diff --git a/.gitignore b/.gitignore
index c6769a2..d588b7d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,3 +10,7 @@ tools/kernel-capture/libkernelcapture.so
 /.wrangler/
 /attestation.json
 /kernels-qwen38-dflash2
+kernels-c9/
+kernels-gdnT/
+kernels-varA/
+kernels-varB/
diff --git a/crates/kern-manifest/src/types.rs b/crates/kern-manifest/src/types.rs
index e8d5040..e6f9247 100644
--- a/crates/kern-manifest/src/types.rs
+++ b/crates/kern-manifest/src/types.rs
@@ -586,6 +586,14 @@ pub struct Step {
     /// Dynamic shared memory in bytes, if the step needs any.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub shared_mem: Option<Expr>,
+    /// Programmatic dependent launch: the runtime launches this step with
+    /// `CU_LAUNCH_ATTRIBUTE_PROGRAMMATIC_STREAM_SERIALIZATION`, so it may
+    /// start while the preceding launch is still draining. Only for kernels
+    /// that put every read of an upstream product and every global write
+    /// behind `griddepcontrol.wait` (their own inputs — weights, state —
+    /// may stream ahead of it).
+    #[serde(default, skip_serializing_if = "is_false")]
+    pub pdl: bool,
     /// Wiring: where each step param comes from — a forwarded interface
     /// arg, a scratch buffer, or an implementation-private literal.
     pub args: Vec<StepArg>,
diff --git a/crates/kern-runtime/src/compile.rs b/crates/kern-runtime/src/compile.rs
index eae524d..32d2826 100644
--- a/crates/kern-runtime/src/compile.rs
+++ b/crates/kern-runtime/src/compile.rs
@@ -68,6 +68,8 @@ pub(crate) enum LaunchKind {
         block: [u32; 3],
         grid: [CExpr; 3],
         shared_mem: Option<CExpr>,
+        /// Launch with programmatic stream serialization (see `Step::pdl`).
+        pdl: bool,
     },
     /// `extern:cublaslt_bf16_tn` / `..._acc` (beta 0.0 / 1.0).
     Gemm { beta: f32 },
@@ -350,6 +352,7 @@ fn compile_dispatch(
                     compile_expr(&st.grid[2], syms)?,
                 ],
                 shared_mem: st.shared_mem.as_ref().map(|e| compile_expr(e, syms)).transpose()?,
+                pdl: st.pdl,
             },
         };
         launches.push(Launch {
diff --git a/crates/kern-runtime/src/lib.rs b/crates/kern-runtime/src/lib.rs
index c81f144..da95d4b 100644
--- a/crates/kern-runtime/src/lib.rs
+++ b/crates/kern-runtime/src/lib.rs
@@ -626,7 +626,7 @@ impl Runtime {
         }
         match &l.kind {
             LaunchKind::Gemm { beta } => gemm_bf16_tn(&self.blt, &self.stream, &vals, *beta),
-            LaunchKind::Cubin { func, block, grid, shared_mem } => {
+            LaunchKind::Cubin { func, block, grid, shared_mem, pdl } => {
                 let grid =
                     (grid[0].eval(env)? as u32, grid[1].eval(env)? as u32, grid[2].eval(env)? as u32);
                 let smem = match shared_mem {
@@ -638,6 +638,34 @@ impl Runtime {
                 let raw: Vec<u64> = vals.iter().map(|r| r.val).collect();
                 let mut params: Vec<*mut c_void> =
                     raw.iter().map(|s| s as *const u64 as *mut c_void).collect();
+                if *pdl {
+                    // Programmatic dependent launch: inside stream capture
+                    // this becomes a programmatic graph edge, so the kernel
+                    // may begin (and stream its own inputs) while the
+                    // previous launch drains; its griddepcontrol.wait keeps
+                    // the data dependency.
+                    let mut attr = sys::CUlaunchAttribute {
+                        id: sys::CUlaunchAttributeID::CU_LAUNCH_ATTRIBUTE_PROGRAMMATIC_STREAM_SERIALIZATION,
+                        pad: [0; 4],
+                        value: sys::CUlaunchAttributeValue { programmaticStreamSerializationAllowed: 1 },
+                    };
+                    let cfg = sys::CUlaunchConfig {
+                        gridDimX: grid.0,
+                        gridDimY: grid.1,
+                        gridDimZ: grid.2,
+                        blockDimX: block[0],
+                        blockDimY: block[1],
+                        blockDimZ: block[2],
+                        sharedMemBytes: smem,
+                        hStream: self.stream.cu_stream(),
+                        attrs: &mut attr,
+                        numAttrs: 1,
+                    };
+                    let r = unsafe {
+                        sys::cuLaunchKernelEx(&cfg, *func, params.as_mut_ptr(), std::ptr::null_mut())
+                    };
+                    return cuda_check(r, "cuLaunchKernelEx");
+                }
                 unsafe {
                     cu::launch_kernel(
                         *func,
diff --git a/docs/qwen38-compare-c10-plain-tray14.json b/docs/qwen38-compare-c10-plain-tray14.json
new file mode 100644
index 0000000..de1e92b
--- /dev/null
+++ b/docs/qwen38-compare-c10-plain-tray14.json
@@ -0,0 +1,2078 @@
+{
+ "config": {
+  "gpu": 3,
+  "chunk": 512,
+  "eager": false,
+  "steps": 400,
+  "capacity": 4704,
+  "only": null,
+  "out": "/mnt/shared/home/susun/agent_code/kern-out/compare-c10-final-plain-tray14.json",
+  "ref": "/mnt/shared/home/susun/agent_code/kern/docs/qwen38-ref.json",
+  "manifest": "examples/qwen3.8-27b.json",
+  "kernels": "kernels-qwen38",
+  "draft": false,
+  "spec": false
+ },
+ "results": [
+  {
+   "index": 0,
+   "match": 124,
+   "ref_len": 400,
+   "byte_identical": false,
+   "prefill": "prefill: 43 tokens in 1 chunk(s) of <= 512 (86.248078ms, 499 tok/s, eager, emits next_token)",
+   "decode": "400 tokens generated, 9.5 ms/step (105.8 tok/s)",
+   "wall_s": 14.0,
+   "generated": [
+    264,
+    3074,
+    4202,
+    421,
+    1467,
+    524,
+    2353,
+    866,
+    314,
+    279,
+    3672,
+    25,
+    264,
+    25347,
+    2512,
+    279,
+    348,
+    32,
+    322,
+    6020,
+    9,
+    995,
+    978,
+    12076,
+    430,
+    1001,
+    52513,
+    506,
+    9117,
+    828,
+    383,
+    279,
+    3603,
+    314,
+    279,
+    3128,
+    70595,
+    11,
+    3482,
+    279,
+    1788,
+    3603,
+    264,
+    7397,
+    4600,
+    725,
+    1433,
+    7246,
+    264,
+    9613,
+    11,
+    14527,
+    54087,
+    71433,
+    7397,
+    501,
+    13810,
+    494,
+    279,
+    2919,
+    321,
+    264,
+    14457,
+    11,
+    83345,
+    3000,
+    3294,
+    279,
+    18137,
+    13,
+    561,
+    7110,
+    725,
+    5572,
+    87338,
+    888,
+    279,
+    2081,
+    11,
+    279,
+    25570,
+    1990,
+    13988,
+    11,
+    430,
+    413,
+    279,
+    1330,
+    7189,
+    998,
+    4313,
+    310,
+    6266,
+    310,
+    1754,
+    975,
+    13,
+    271,
+    4639,
+    3603,
+    11,
+    279,
+    7110,
+    1362,
+    524,
+    5894,
+    13,
+    1216,
+    33461,
+    279,
+    348,
+    32,
+    322,
+    6020,
+    9,
+    430,
+    264,
+    3255,
+    421,
+    995,
+    41065,
+    680,
+    314,
+    1141,
+    5976,
+    12096,
+    321,
+    25286,
+    303,
+    279,
+    4592,
+    314,
+    2361,
+    13,
+    561,
+    7397,
+    4600,
+    725,
+    22557,
+    11,
+    440,
+    1141,
+    40630,
+    11,
+    46368,
+    3447,
+    11,
+    6273,
+    1040,
+    264,
+    11937,
+    314,
+    264,
+    7875,
+    18641,
+    421,
+    995,
+    978,
+    20496,
+    3023,
+    321,
+    3162,
+    290,
+    8366,
+    279,
+    45366,
+    13,
+    561,
+    7110,
+    725,
+    1765,
+    8129,
+    11,
+    8785,
+    321,
+    57879,
+    11,
+    9047,
+    310,
+    381,
+    279,
+    7167,
+    33528,
+    421,
+    1362,
+    3222,
+    279,
+    11937,
+    303,
+    1925,
+    13,
+    271,
+    1465,
+    5820,
+    310,
+    11319,
+    279,
+    4368,
+    13,
+    5339,
+    11,
+    551,
+    9711,
+    279,
+    13769,
+    12688,
+    25,
+    279,
+    2321,
+    11,
+    279,
+    13534,
+    11,
+    279,
+    24834,
+    314,
+    11679,
+    290,
+    19712,
+    11,
+    279,
+    13234,
+    314,
+    28279,
+    11,
+    279,
+    8831,
+    2218,
+    5248,
+    76,
+    11,
+    17257,
+    1657,
+    11,
+    264,
+    14979,
+    29262,
+    13,
+    4844,
+    551,
+    6316,
+    310,
+    279,
+    7397,
+    4600,
+    725,
+    3874,
+    321,
+    4439,
+    1092,
+    424,
+    1362,
+    2977,
+    303,
+    1782,
+    3665,
+    13,
+    357,
+    1001,
+    79088,
+    54087,
+    71433,
+    7397,
+    828,
+    2493,
+    381,
+    264,
+    21172,
+    314,
+    279,
+    17257,
+    383,
+    264,
+    13988,
+    9249,
+    26,
+    264,
+    1001,
+    26418,
+    5143,
+    3000,
+    828,
+    1362,
+    381,
+    279,
+    46219,
+    314,
+    264,
+    13238,
+    49992,
+    4560,
+    13,
+    3113,
+    91469,
+    279,
+    20514,
+    291,
+    2099,
+    303,
+    63001,
+    3447,
+    11,
+    279,
+    7110,
+    6316,
+    264,
+    37753,
+    47978,
+    1227,
+    1083,
+    264,
+    1228,
+    470,
+    29094,
+    13,
+    271,
+    760,
+    1727,
+    2923,
+    557,
+    310,
+    9150,
+    279,
+    1330,
+    9331,
+    3008,
+    539,
+    3008,
+    11,
+    3221,
+    364,
+    77650,
+    321,
+    364,
+    3387,
+    314,
+    3522,
+    13,
+    561,
+    45366,
+    1018,
+    279,
+    348,
+    32,
+    322,
+    6020,
+    9,
+    557,
+    1001,
+    52513,
+    2340,
+    694,
+    279,
+    7397,
+    4600,
+    725,
+    1433,
+    8826,
+    264,
+    8026,
+    303,
+    279,
+    2919,
+    506,
+    279,
+    1788,
+    6258,
+    13,
+    561,
+    7110,
+    10036,
+    279,
+    75332,
+    11,
+    1179,
+    4439,
+    1092,
+    1000,
+    599,
+    310,
+    381,
+    804,
+    364,
+    2107,
+    11889,
+    310,
+    3222,
+    25,
+    8106,
+    279,
+    348,
+    32,
+    322,
+    6020,
+    9,
+    995,
+    524,
+    56507,
+    694,
+    995
+   ],
+   "text": "The harbor master kept a ledger of every ship that wintered in the bay, noting cargo, crew, and the state of each hull in a cramped, looping hand. In the spring of the third year he found a single entry that did not match any of the others: a vessel called the *Aurora* had been recorded as \u201clost at sea\u201d on the night of the equinox, yet the same night a fisherman\u2019s log described a bright, silver\u2011scaled fish leaping from the water and a strange, humming light above the mast. The master\u2019s pen hovered over the page, the ink still wet, as if the two records were trying to speak to each other.\n\nThat night, the master could not sleep. He imagined the *Aurora* as a story that had slipped out of its proper chapter and landed in the margin of another. The fisherman\u2019s tale, with its vivid, sensory details, felt like a fragment of a larger narrative that had been torn away and pasted onto the ledger. The master\u2019s own notes, dry and factual, seemed to be the missing spine that could hold the fragment in place.\n\nHe began to trace the thread. First, he listed the concrete facts: the date, the coordinates, the cargo of salted cod, the crew of twelve, the weather\u2014calm, moonless, a thin fog. Then he turned to the fisherman\u2019s description and asked what it could mean in those terms. A \u201csilver\u2011scaled fish\u201d might be a reflection of the moon on a wet deck; a \u201chumming light\u201d could be the vibration of a newly repaired engine. By grounding the mythic image in measurable details, the master turned a vague anecdote into a testable hypothesis.\n\nThe next step was to compare the two accounts side by side, looking for contradictions and for points of contact. The ledger said the *Aurora* was \u201clost,\u201d but the fisherman\u2019s log placed a ship in the water at the same hour. The master noted the discrepancy, then asked what would have to be true for both statements to hold: perhaps the *Aurora* had not vanished but had\n"
+  },
+  {
+   "index": 1,
+   "match": 139,
+   "ref_len": 400,
+   "byte_identical": false,
+   "prefill": "prefill: 41 tokens in 1 chunk(s) of <= 512 (82.77022ms, 495 tok/s, eager, emits next_token)",
+   "decode": "400 tokens generated, 9.7 ms/step (102.8 tok/s)",
+   "wall_s": 12.4,
+   "generated": [
+    264,
+    36334,
+    21388,
+    314,
+    5484,
+    5415,
+    321,
+    264,
+    716,
+    314,
+    34844,
+    542,
+    5564,
+    58091,
+    1172,
+    279,
+    7991,
+    846,
+    13,
+    561,
+    1118,
+    3603,
+    1172,
+    279,
+    491,
+    12515,
+    11,
+    279,
+    6561,
+    2002,
+    6749,
+    92169,
+    1056,
+    5316,
+    25624,
+    11,
+    3482,
+    279,
+    36334,
+    467,
+    78078,
+    5432,
+    68,
+    1990,
+    9047,
+    310,
+    288,
+    92536,
+    13,
+    561,
+    2015,
+    14607,
+    421,
+    279,
+    53952,
+    725,
+    27374,
+    5257,
+    557,
+    874,
+    4860,
+    264,
+    4145,
+    1001,
+    75,
+    704,
+    54087,
+    429,
+    54087,
+    70735,
+    828,
+    3255,
+    26,
+    424,
+    557,
+    264,
+    61010,
+    23850,
+    314,
+    25586,
+    11,
+    1754,
+    440,
+    1141,
+    1765,
+    79299,
+    11,
+    25113,
+    11,
+    321,
+    3558,
+    13,
+    271,
+    1206,
+    12689,
+    3908,
+    421,
+    23850,
+    11,
+    567,
+    1902,
+    1118,
+    2466,
+    25,
+    2972,
+    4993,
+    635,
+    567,
+    7276,
+    264,
+    1785,
+    314,
+    1599,
+    18766,
+    33765,
+    25586,
+    303,
+    264,
+    1560,
+    421,
+    14908,
+    586,
+    6805,
+    1332,
+    449,
+    2099,
+    668,
+    1304,
+    11,
+    1204,
+    3349,
+    424,
+    668,
+    381,
+    11,
+    321,
+    1204,
+    279,
+    3000,
+    668,
+    35600,
+    170286,
+    561,
+    4087,
+    14995,
+    303,
+    264,
+    16338,
+    716,
+    314,
+    36655,
+    421,
+    4089,
+    279,
+    4220,
+    14193,
+    430,
+    264,
+    3074,
+    1001,
+    24507,
+    11432,
+    828,
+    17796,
+    11,
+    1345,
+    1990,
+    9976,
+    3627,
+    314,
+    279,
+    3714,
+    5255,
+    421,
+    2873,
+    279,
+    1785,
+    1141,
+    3542,
+    13,
+    271,
+    13962,
+    220,
+    16,
+    13,
+    5369,
+    264,
+    3074,
+    7099,
+    310,
+    264,
+    4220,
+    1785,
+    271,
+    2381,
+    264,
+    3074,
+    18766,
+    33765,
+    7099,
+    279,
+    1298,
+    686,
+    518,
+    12323,
+    369,
+    271,
+    75370,
+    198,
+    59,
+    35790,
+    88216,
+    8017,
+    90,
+    82,
+    8017,
+    29533,
+    35790,
+    88216,
+    14728,
+    82,
+    92,
+    33289,
+    35790,
+    88216,
+    23531,
+    77,
+    14728,
+    81,
+    2069,
+    198,
+    59,
+    60,
+    271,
+    2777,
+    17237,
+    82,
+    55821,
+    321,
+    17237,
+    82,
+    15313,
+    8,
+    513,
+    279,
+    1576,
+    321,
+    2099,
+    25704,
+    16384,
+    494,
+    279,
+    7099,
+    11,
+    17237,
+    77,
+    55821,
+    321,
+    17237,
+    77,
+    15313,
+    8,
+    513,
+    279,
+    18766,
+    2932,
+    14505,
+    383,
+    2892,
+    3008,
+    11,
+    321,
+    17237,
+    81,
+    55821,
+    369,
+    279,
+    7099,
+    10264,
+    13,
+    18050,
+    4350,
+    3679,
+    25586,
+    513,
+    8826,
+    303,
+    8240,
+    11,
+    279,
+    1788,
+    1304,
+    79476,
+    413,
+    567,
+    8032,
+    279,
+    3714,
+    31591,
+    539,
+    2972,
+    8678,
+    54087,
+    8877,
+    332,
+    5981,
+    25,
+    271,
+    75370,
+    198,
+    59,
+    35790,
+    88216,
+    8017,
+    90,
+    82,
+    8017,
+    29533,
+    35790,
+    88216,
+    14728,
+    82,
+    92,
+    33289,
+    35790,
+    88216,
+    23531,
+    77,
+    14728,
+    81,
+    14717,
+    59,
+    1272,
+    90,
+    10684,
+    3307,
+    7563,
+    198,
+    59,
+    60,
+    271,
+    8160,
+    17237,
+    81,
+    14717,
+    59,
+    1272,
+    90,
+    10684,
+    3307,
+    55821,
+    369,
+    279,
+    10264,
+    314,
+    79299,
+    314,
+    449,
+    348,
+    24507,
+    11432,
+    9,
+    3074,
+    7099,
+    421,
+    1000,
+    7936,
+    6681,
+    279,
+    1788,
+    1298,
+    686,
+    518,
+    30619,
+    430,
+    279,
+    4220,
+    5436,
+    13,
+    561,
+    2279,
+    314,
+    279,
+    1785,
+    369,
+    271,
+    75370,
+    198,
+    59,
+    43660,
+    283,
+    307,
+    6,
+    471,
+    307,
+    283,
+    1088,
+    35790,
+    88216,
+    23531
+   ],
+   "text": "When the observatory finally reopened after the renovation, the docents discovered that the old refractor had been quietly recollimated by a retired machinist who lived nearby. He left no note, only a faint smell of machine oil and a set of brass shims tucked under the focuser. The first night under the new sky, the star field looked sharper than anyone remembered, yet the faintest nebulae still seemed to wobble. The team realized that the telescope\u2019s optical train was no longer a simple \u201clens\u2011and\u2011mirror\u201d story; it was a layered puzzle of surfaces, each with its own curvature, spacing, and material.\n\nTo untangle that puzzle, we must first ask: **how do we describe a system of many refracting surfaces in a way that lets us predict where an image will form, how large it will be, and how the light will bend?** The answer lies in a compact set of equations that treat the whole assembly as a single \u201cequivalent\u201d lens, while still keeping track of the individual elements that give the system its character.\n\n### 1. From a single surface to a whole system\n\nFor a single refracting surface the paraxial relation is\n\n\\[\n\\frac{n'}{s'}-\\frac{n}{s}=\\frac{n'-n}{r},\n\\]\n\nwhere \\(s\\) and \\(s'\\) are the object and image distances measured from the surface, \\(n\\) and \\(n'\\) are the refractive indices on either side, and \\(r\\) is the surface radius.  \n\nWhen several surfaces are placed in sequence, the same form survives if we replace the individual quantities by **system\u2011wide** ones:\n\n\\[\n\\frac{n'}{s'}-\\frac{n}{s}=\\frac{n'-n}{r_{\\text{eq}}}.\n\\]\n\nHere \\(r_{\\text{eq}}\\) is the radius of curvature of an *equivalent* single surface that would produce exactly the same paraxial imaging as the whole stack. The power of the system is\n\n\\[\n\\Phi = n' - n = \\frac{n'-\n"
+  },
+  {
+   "index": 2,
+   "match": 341,
+   "ref_len": 400,
+   "byte_identical": false,
+   "prefill": "prefill: 48 tokens in 1 chunk(s) of <= 512 (87.176143ms, 551 tok/s, eager, emits next_token)",
+   "decode": "400 tokens generated, 9.5 ms/step (105.5 tok/s)",
+   "wall_s": 11.7,
+   "generated": [
+    7094,
+    781,
+    14428,
+    279,
+    53627,
+    287,
+    440,
+    22945,
+    9399,
+    49954,
+    11,
+    8132,
+    1396,
+    2418,
+    314,
+    272,
+    46016,
+    11,
+    1396,
+    10582,
+    314,
+    274,
+    6960,
+    11,
+    1396,
+    22263,
+    23178,
+    17629,
+    22149,
+    13,
+    733,
+    279,
+    4688,
+    7094,
+    781,
+    5743,
+    310,
+    279,
+    1788,
+    22945,
+    52604,
+    321,
+    1669,
+    279,
+    14367,
+    995,
+    7481,
+    13,
+    271,
+    760,
+    42159,
+    557,
+    2496,
+    7807,
+    13,
+    561,
+    14367,
+    8341,
+    7594,
+    795,
+    13,
+    271,
+    4277,
+    271,
+    760,
+    1118,
+    3065,
+    488,
+    5125,
+    883,
+    279,
+    17198,
+    20139,
+    369,
+    421,
+    424,
+    369,
+    524,
+    264,
+    1925,
+    13,
+    1049,
+    369,
+    264,
+    1817,
+    11884,
+    279,
+    30385,
+    314,
+    264,
+    1925,
+    13,
+    561,
+    4128,
+    488,
+    2418,
+    383,
+    557,
+    14367,
+    2634,
+    264,
+    12828,
+    3998,
+    321,
+    668,
+    381,
+    14367,
+    2634,
+    1495,
+    303,
+    264,
+    12828,
+    13,
+    561,
+    12052,
+    513,
+    524,
+    39527,
+    303,
+    16665,
+    748,
+    1691,
+    430,
+    303,
+    264,
+    46446,
+    440,
+    2919,
+    13,
+    561,
+    272,
+    46016,
+    29528,
+    513,
+    524,
+    19101,
+    13,
+    2302,
+    513,
+    279,
+    4757,
+    579,
+    1560,
+    314,
+    25108,
+    948,
+    279,
+    2919,
+    1898,
+    35937,
+    3294,
+    1141,
+    7303,
+    13,
+    271,
+    760,
+    2018,
+    3065,
+    488,
+    5125,
+    369,
+    279,
+    20513,
+    13,
+    2717,
+    279,
+    18682,
+    314,
+    4950,
+    13,
+    561,
+    9084,
+    314,
+    264,
+    2086,
+    2993,
+    314,
+    4950,
+    13,
+    561,
+    3594,
+    369,
+    11702,
+    440,
+    279,
+    11639,
+    314,
+    2919,
+    6992,
+    1472,
+    51930,
+    11,
+    314,
+    39465,
+    3133,
+    279,
+    60847,
+    11,
+    314,
+    19053,
+    7842,
+    303,
+    32655,
+    421,
+    2714,
+    5995,
+    310,
+    5640,
+    1472,
+    13988,
+    7339,
+    13,
+    561,
+    20513,
+    369,
+    279,
+    4950,
+    314,
+    264,
+    1785,
+    303,
+    53454,
+    13,
+    1049,
+    369,
+    279,
+    4950,
+    314,
+    264,
+    1925,
+    421,
+    682,
+    978,
+    3604,
+    279,
+    1788,
+    3065,
+    364,
+    5600,
+    15706,
+    1578,
+    321,
+    682,
+    874,
+    14181,
+    314,
+    21835,
+    13,
+    271,
+    760,
+    4688,
+    3065,
+    488,
+    5125,
+    369,
+    279,
+    21388,
+    13,
+    561,
+    21388,
+    314,
+    27572,
+    8056,
+    4766,
+    11,
+    314,
+    69064,
+    11,
+    314,
+    279,
+    3820,
+    61306,
+    314,
+    2919,
+    421,
+    682,
+    978,
+    11362,
+    303,
+    264,
+    55294,
+    364,
+    264,
+    1936,
+    13,
+    561,
+    21388,
+    369,
+    524,
+    45322,
+    13,
+    1049,
+    369,
+    279,
+    21388,
+    314,
+    264,
+    1925,
+    421,
+    369,
+    13282,
+    303,
+    279,
+    1560,
+    421,
+    264,
+    54826,
+    16853,
+    369,
+    13282,
+    13,
+    1049,
+    369,
+    279,
+    21388,
+    314,
+    17454,
+    13,
+    271,
+    4277,
+    271,
+    760,
+    296,
+    7679,
+    358,
+    41713,
+    513,
+    279,
+    17198,
+    20139,
+    579,
+    4779,
+    13,
+    8618,
+    799,
+    369,
+    264,
+    21599,
+    1332,
+    279,
+    14367,
+    2957,
+    10298,
+    321,
+    1179,
+    5324,
+    1141,
+    3838,
+    13,
+    561,
+    2919,
+    303,
+    449,
+    296,
+    7679,
+    358,
+    369,
+    1990,
+    13,
+    1049,
+    369,
+    279,
+    1829,
+    314,
+    14799,
+    13,
+    1049,
+    369,
+    2400,
+    314,
+    259,
+    985,
+    1283,
+    501,
+    3726,
+    494,
+    279,
+    272,
+    46016,
+    321,
+    279,
+    55463,
+    19559,
+    11,
+    321,
+    279,
+    259,
+    985,
+    1283,
+    1236,
+    279,
+    2919,
+    80913,
+    11,
+    321,
+    279,
+    93582,
+    13205,
+    279,
+    65562
+   ],
+   "text": "The floodplain census took three summers to complete. In the first summer the crews mapped oxbow lakes and counted heron rookeries from canoes, losing two clipboards and one outboard motor to the river. In the second summer they walked the levees with transect tapes, logging every stand of cypress, every patch of sedge, every abandoned crawfish trap. In the third summer they returned to the same transects and found the river had moved.\n\nThe census was never finished. The river kept adding data.\n\n---\n\nThe first thing you notice about the floodplain is that it is not a place. It is a process wearing the costume of a place. The land you stand on was riverbed a decade ago and will be riverbed again in a decade. The trees are not rooted in soil so much as in a negotiation with water. The cypress knees are not roots. They are the tree's way of breathing when the water table rises above its feet.\n\nThe second thing you notice is the silence. Not the absence of sound. The presence of a different kind of sound. The air is thick with the noise of water moving through vegetation, of insects working the canopy, of birds calling in frequencies that seem designed to travel through wet wood. The silence is the sound of a system in equilibrium. It is the sound of a place that has been doing the same thing for ten thousand years and has no intention of stopping.\n\nThe third thing you notice is the smell. The smell of decomposing matter, of sulfur, of the particular sweetness of water that has been sitting in a basin for a week. The smell is not unpleasant. It is the smell of a place that is alive in the way that a compost heap is alive. It is the smell of transformation.\n\n---\n\nThe oxbow lakes are the floodplain's memory. Each one is a scar where the river once ran and then changed its mind. The water in an oxbow is still. It is the color of tea. It is full of tannins leached from the cypress and the tupelo, and the tannins make the water acidic, and the acidity keeps the algae\n"
+  },
+  {
+   "index": 3,
+   "match": 185,
+   "ref_len": 400,
+   "byte_identical": false,
+   "prefill": "prefill: 42 tokens in 1 chunk(s) of <= 512 (87.373231ms, 481 tok/s, eager, emits next_token)",
+   "decode": "400 tokens generated, 9.5 ms/step (105.8 tok/s)",
+   "wall_s": 11.5,
+   "generated": [
+    271,
+    8160,
+    369,
+    449,
+    15673,
+    314,
+    1439,
+    2250,
+    41724,
+    303,
+    13983,
+    3992,
+    11,
+    3055,
+    440,
+    264,
+    13769,
+    3010,
+    364,
+    1754,
+    13,
+    271,
+    13962,
+    220,
+    16,
+    13,
+    8169,
+    264,
+    19269,
+    7541,
+    2751,
+    303,
+    18266,
+    271,
+    332,
+    760,
+    68868,
+    64700,
+    198,
+    33807,
+    303,
+    279,
+    1814,
+    369,
+    1801,
+    314,
+    13280,
+    18167,
+    318,
+    63339,
+    321,
+    33470,
+    8,
+    421,
+    513,
+    14537,
+    6992,
+    13,
+    3095,
+    279,
+    9039,
+    35937,
+    11,
+    1439,
+    18167,
+    8457,
+    4649,
+    321,
+    3166,
+    10281,
+    11,
+    13969,
+    1070,
+    310,
+    4336,
+    9781,
+    4473,
+    10431,
+    494,
+    799,
+    2361,
+    13,
+    1061,
+    369,
+    2512,
+    2972,
+    65819,
+    14046,
+    159034,
+    271,
+    7732,
+    279,
+    14046,
+    314,
+    264,
+    3074,
+    18546,
+    369,
+    28486,
+    11,
+    948,
+    488,
+    29283,
+    421,
+    13280,
+    6977,
+    539,
+    279,
+    30995,
+    314,
+    31137,
+    303,
+    264,
+    10632,
+    5759,
+    1040,
+    264,
+    9229,
+    13759,
+    11,
+    279,
+    2702,
+    2222,
+    303,
+    2984,
+    8772,
+    4927,
+    13,
+    1368,
+    264,
+    13759,
+    998,
+    5617,
+    430,
+    799,
+    6235,
+    11,
+    632,
+    47272,
+    6365,
+    11,
+    279,
+    8364,
+    303,
+    7094,
+    1000,
+    5073,
+    424,
+    310,
+    14084,
+    13,
+    16643,
+    2956,
+    310,
+    14084,
+    11,
+    279,
+    5138,
+    7035,
+    1000,
+    1870,
+    685,
+    11,
+    13188,
+    13969,
+    279,
+    13759,
+    310,
+    77238,
+    11,
+    16966,
+    11,
+    466,
+    1442,
+    17638,
+    13,
+    271,
+    332,
+    80711,
+    12996,
+    64700,
+    198,
+    49338,
+    264,
+    2972,
+    16,
+    11,
+    15,
+    15,
+    15,
+    19858,
+    23046,
+    9229,
+    13759,
+    159034,
+    12142,
+    50110,
+    506,
+    264,
+    4238,
+    314,
+    883,
+    220,
+    15,
+    13,
+    15,
+    15,
+    15,
+    15,
+    15,
+    21,
+    14493,
+    791,
+    16618,
+    364,
+    1396,
+    8122,
+    66401,
+    5096,
+    303,
+    9039,
+    13,
+    198,
+    9,
+    256,
+    1368,
+    279,
+    9039,
+    35937,
+    539,
+    2972,
+    20,
+    15,
+    56498,
+    332,
+    494,
+    12050,
+    310,
+    7094,
+    11,
+    421,
+    220,
+    16,
+    11,
+    15,
+    15,
+    15,
+    19858,
+    13759,
+    668,
+    8952,
+    539,
+    16757,
+    2972,
+    18,
+    13,
+    21,
+    14493,
+    159034,
+    198,
+    9,
+    256,
+    1946,
+    27358,
+    411,
+    11,
+    23438,
+    4433,
+    2972,
+    4431,
+    10188,
+    33124,
+    332,
+    318,
+    1719,
+    10174,
+    9789,
+    488,
+    1436,
+    303,
+    279,
+    5459,
+    7099,
+    553,
+    4081,
+    33124,
+    1121,
+    1040,
+    10439,
+    15110,
+    1837,
+    11,
+    10377,
+    279,
+    1330,
+    10029,
+    314,
+    279,
+    13759,
+    310,
+    14549,
+    9781,
+    3162,
+    1754,
+    975,
+    430,
+    279,
+    9041,
+    47929,
+    303,
+    279,
+    8364,
+    321,
+    16577,
+    303,
+    279,
+    8981,
+    13,
+    271,
+    11862,
+    271,
+    13962,
+    220,
+    17,
+    13,
+    8169,
+    264,
+    710,
+    46038,
+    326,
+    793,
+    12851,
+    1049,
+    2436,
+    8405,
+    271,
+    332,
+    760,
+    68868,
+    64700,
+    198,
+    760,
+    328,
+    82,
+    23279,
+    1,
+    466,
+    412,
+    375,
+    2631,
+    4950,
+    369,
+    3404,
+    264,
+    1067,
+    314,
+    2972,
+    85,
+    17804,
+    8613,
+    539,
+    19695,
+    1602,
+    29592,
+    321,
+    1179,
+    5698,
+    159034,
+    271,
+    4350,
+    488,
+    1118,
+    8364,
+    2919,
+    11,
+    279,
+    5445,
+    314,
+    279,
+    69862,
+    5055,
+    3882,
+    1518,
+    279,
+    2919,
+    506,
+    279,
+    1844,
+    13,
+    1061,
+    11121,
+    264,
+    6000,
+    314,
+    3882,
+    2919,
+    506,
+    279,
+    5445,
+    421
+   ],
+   "text": "Explain, in plain language and with one concrete example each, why a bridge expands in summer, why a kettle sings before it boils, and why the far side of the moon is never seen from Earth.\n\nHere is an explanation of these three phenomena in plain language, along with a concrete example for each.\n\n### 1. Why a Bridge Expands in Summer\n\n**The Explanation:**\nEverything in the world is made of tiny particles (atoms and molecules) that are constantly moving. When the temperature rises, these particles gain energy and move faster, causing them to push slightly further apart from one another. This is called **thermal expansion**.\n\nWhile the expansion of a single atom is invisible, when you multiply that tiny movement by the billions of atoms in a massive structure like a steel bridge, the total change in length becomes significant. If a bridge were built as one solid, unbroken piece, the heat in summer would cause it to stretch. Without room to stretch, the internal pressure would build up, potentially causing the bridge to buckle, crack, or even collapse.\n\n**Concrete Example:**\nImagine a **1,000-foot-long steel bridge**. Steel expands at a rate of about 0.000006 inches per inch for every degree Fahrenheit increase in temperature.\n*   If the temperature rises by **50\u00b0F** from winter to summer, that 1,000-foot bridge will expand by roughly **3.6 inches**.\n*   To accommodate this, engineers install **expansion joints** (the grooves you see in the road surface). These joints act like shock absorbers, allowing the two ends of the bridge to slide slightly past each other as the metal stretches in the heat and contracts in the cold.\n\n***\n\n### 2. Why a Kettle Sings Before It Boils\n\n**The Explanation:**\nThe \"singing\" or whistling sound is actually a result of **vibration caused by steam being trapped and then released**.\n\nWhen you first heat water, the bottom of the kettle gets hot before the water at the top. This creates a layer of hot water at the bottom that\n"
+  },
+  {
+   "index": 4,
+   "match": 178,
+   "ref_len": 400,
+   "byte_identical": false,
+   "prefill": "prefill: 47 tokens in 1 chunk(s) of <= 512 (83.434189ms, 563 tok/s, eager, emits next_token)",
+   "decode": "400 tokens generated, 9.5 ms/step (105.8 tok/s)",
+   "wall_s": 11.7,
+   "generated": [
+    9896,
+    2250,
+    75842,
+    440,
+    11140,
+    421,
+    998,
+    40361,
+    42749,
+    13,
+    3113,
+    35442,
+    1292,
+    995,
+    5158,
+    264,
+    11183,
+    883,
+    264,
+    5054,
+    852,
+    1362,
+    6512,
+    279,
+    9117,
+    579,
+    295,
+    8792,
+    1472,
+    279,
+    6220,
+    18687,
+    11,
+    321,
+    1292,
+    4035,
+    2574,
+    424,
+    310,
+    1024,
+    6764,
+    11,
+    852,
+    1301,
+    424,
+    303,
+    799,
+    11362,
+    321,
+    1179,
+    1018,
+    11,
+    328,
+    2523,
+    2908,
+    1801,
+    279,
+    9117,
+    4950,
+    1040,
+    264,
+    27809,
+    36252,
+    1149,
+    2838,
+    3738,
+    411,
+    430,
+    279,
+    8166,
+    27463,
+    13,
+    271,
+    760,
+    11183,
+    557,
+    2496,
+    4501,
+    13,
+    1049,
+    7338,
+    303,
+    264,
+    25635,
+    364,
+    42744,
+    1578,
+    11,
+    22175,
+    279,
+    3820,
+    20513,
+    314,
+    2433,
+    421,
+    513,
+    9939,
+    694,
+    524,
+    3482,
+    5354,
+    13,
+    3095,
+    1292,
+    5326,
+    3106,
+    424,
+    310,
+    264,
+    20609,
+    11,
+    781,
+    2512,
+    424,
+    328,
+    1700,
+    5143,
+    1,
+    321,
+    328,
+    64,
+    2545,
+    2166,
+    1248,
+    1149,
+    2838,
+    75931,
+    5356,
+    424,
+    10598,
+    13,
+    561,
+    4688,
+    2243,
+    557,
+    2577,
+    11,
+    694,
+    424,
+    557,
+    874,
+    4860,
+    279,
+    11183,
+    1292,
+    4035,
+    5158,
+    506,
+    35442,
+    11,
+    321,
+    1292,
+    8341,
+    279,
+    3889,
+    303,
+    264,
+    3618,
+    58205,
+    9043,
+    4045,
+    31193,
+    11,
+    864,
+    557,
+    264,
+    9940,
+    11,
+    1521,
+    1292,
+    18559,
+    424,
+    14537,
+    11,
+    303,
+    1024,
+    1901,
+    11,
+    1345,
+    11107,
+    279,
+    41586,
+    1752,
+    11,
+    1345,
+    295,
+    2379,
+    50692,
+    11,
+    1345,
+    13879,
+    310,
+    279,
+    338,
+    68042,
+    17061,
+    888,
+    264,
+    5418,
+    295,
+    9395,
+    300,
+    13,
+    271,
+    7691,
+    4501,
+    1024,
+    1118,
+    2236,
+    506,
+    25292,
+    39663,
+    13,
+    1049,
+    557,
+    264,
+    4281,
+    314,
+    2716,
+    7112,
+    883,
+    279,
+    1208,
+    852,
+    11815,
+    303,
+    279,
+    13721,
+    3559,
+    279,
+    324,
+    55965,
+    25,
+    279,
+    7397,
+    4600,
+    852,
+    8341,
+    264,
+    44603,
+    314,
+    1396,
+    13071,
+    11,
+    279,
+    54897,
+    852,
+    5898,
+    1024,
+    3200,
+    9677,
+    579,
+    14882,
+    321,
+    1429,
+    279,
+    3117,
+    310,
+    3569,
+    264,
+    25439,
+    11,
+    279,
+    7914,
+    852,
+    14129,
+    16002,
+    18009,
+    321,
+    11252,
+    1754,
+    799,
+    12613,
+    264,
+    2086,
+    17415,
+    13,
+    561,
+    2236,
+    557,
+    1575,
+    3816,
+    13,
+    357,
+    9451,
+    5947,
+    421,
+    424,
+    995,
+    328,
+    1719,
+    28342,
+    314,
+    40554,
+    12269,
+    1,
+    321,
+    328,
+    1719,
+    10647,
+    8897,
+    314,
+    264,
+    11717,
+    14293,
+    383,
+    264,
+    9766,
+    29199,
+    1149,
+    2838,
+    44457,
+    279,
+    3286,
+    321,
+    17750,
+    424,
+    3294,
+    1024,
+    17471,
+    11,
+    1332,
+    424,
+    19013,
+    364,
+    4590,
+    1578,
+    2980,
+    264,
+    13071,
+    39957,
+    279,
+    3136,
+    1724,
+    321,
+    279,
+    8720,
+    53507,
+    383,
+    279,
+    6220,
+    13,
+    2838,
+    38891,
+    685,
+    279,
+    9378,
+    321,
+    8341,
+    279,
+    3286,
+    11,
+    1381,
+    440,
+    264,
+    25888,
+    3438,
+    6607,
+    11,
+    303,
+    264,
+    25635,
+    440,
+    279,
+    3889,
+    11183,
+    13,
+    271,
+    760,
+    2018,
+    2236,
+    3738,
+    2943,
+    1578,
+    13,
+    1049,
+    557,
+    264,
+    11183,
+    883,
+    264,
+    5054,
+    852,
+    26124,
+    264,
+    324,
+    55965,
+    321,
+    10782,
+    421,
+    279,
+    3000,
+    995,
+    978,
+    6316,
+    974,
+    364,
+    25292,
+    1578,
+    11
+   ],
+   "text": "The lighthouse keeper's daughter learned to read from shipping manifests and weather logs, so her first stories were inventories: forty barrels of salt cod, a crate of oranges gone soft, one bishop travelling incognito. By twelve she had filled three notebooks with lists that were secretly poems. By fifteen she had written a novel about a woman who could hear the sea's moods through the floorboards, and she'd given it to her father, who read it in one sitting and then said, \"You've made the sea sound like a drunk uncle.\" She took this as the highest praise.\n\nThe novel was never published. It sat in a drawer for eleven years, gathering the particular silence of things that are loved but not yet ready. When she finally sent it to a publisher, they called it \"charming\" and \"a little too long.\" She rewrote it twice. The third version was better, but it was no longer the novel she'd written at fifteen, and she kept the original in a box labelled DO NOT EDIT, which was a lie, because she edited it constantly, in her head, while walking the cliff path, while mending nets, while listening to the gulls argue over a dead mackerel.\n\nShe published her first book at thirty-four. It was a collection of short stories about the people who lived in the village below the lighthouse: the fisherman who kept a diary of every storm, the widow who sold her late husband's boat and used the money to buy a piano, the boy who collected bottle caps and believed each one contained a different ocean. The book was well received. A critic wrote that it had \"the patience of tide tables\" and \"the sudden violence of a wave breaking on a flat shore.\" She framed the review and hung it above her desk, where it stayed for six years until a storm blew the window open and the glass shattered on the floor. She swept up the pieces and kept the review, now with a jagged edge, in a drawer with the original novel.\n\nThe second book took four years. It was a novel about a woman who inherited a lighthouse and discovered that the light had been turned off for thirty years,\n"
+  }
+ ]
+}
\ No newline at end of file
diff --git a/docs/qwen38-compare-c10-spec-tray14.json b/docs/qwen38-compare-c10-spec-tray14.json
new file mode 100644
index 0000000..9257c98
--- /dev/null
+++ b/docs/qwen38-compare-c10-spec-tray14.json
@@ -0,0 +1,2078 @@
+{
+ "config": {
+  "gpu": 0,
+  "chunk": 512,
+  "eager": false,
+  "steps": 400,
+  "capacity": 4704,
+  "only": null,
+  "out": "/mnt/shared/home/susun/agent_code/kern-out/compare-c10-final-spec-tray14.json",
+  "ref": "/mnt/shared/home/susun/agent_code/kern/docs/qwen38-ref.json",
+  "manifest": "examples/qwen3.8-27b-dflash2.json",
+  "kernels": "kernels-qwen38-dflash2",
+  "draft": true,
+  "spec": true
+ },
+ "results": [
+  {
+   "index": 0,
+   "match": 66,
+   "ref_len": 400,
+   "byte_identical": false,
+   "prefill": "prefill: 43 tokens in 1 chunk(s) of <= 512 (86.192974ms, 499 tok/s, eager, emits next_token)",
+   "decode": "spec: 399 tokens in 168 rounds (2.38 tokens/round, 19.6% drafts accepted, per position [99, 62, 29, 17, 11, 7, 6]), 14.11 ms/round (168.3 tok/s)",
+   "wall_s": 13.1,
+   "generated": [
+    264,
+    3074,
+    4202,
+    421,
+    1467,
+    524,
+    2353,
+    866,
+    314,
+    279,
+    3672,
+    25,
+    264,
+    25347,
+    2512,
+    279,
+    348,
+    32,
+    322,
+    6020,
+    9,
+    995,
+    978,
+    12076,
+    430,
+    1001,
+    52513,
+    506,
+    9117,
+    828,
+    383,
+    279,
+    3603,
+    314,
+    279,
+    3128,
+    70595,
+    11,
+    3482,
+    279,
+    1788,
+    3603,
+    264,
+    7397,
+    4600,
+    725,
+    1433,
+    7246,
+    264,
+    9613,
+    11,
+    14527,
+    54087,
+    71433,
+    7397,
+    501,
+    13810,
+    494,
+    279,
+    2919,
+    321,
+    264,
+    14457,
+    11,
+    83345,
+    3000,
+    888,
+    279,
+    54517,
+    13,
+    561,
+    7110,
+    725,
+    5572,
+    17621,
+    36412,
+    26,
+    279,
+    1330,
+    7189,
+    1362,
+    524,
+    2107,
+    381,
+    804,
+    11,
+    321,
+    279,
+    48086,
+    9290,
+    1384,
+    310,
+    2466,
+    264,
+    3296,
+    421,
+    995,
+    54779,
+    66509,
+    364,
+    22892,
+    25,
+    2972,
+    12195,
+    11,
+    413,
+    3977,
+    11,
+    14995,
+    23221,
+    279,
+    7099,
+    314,
+    279,
+    1814,
+    567,
+    1683,
+    567,
+    1366,
+    170286,
+    271,
+    4639,
+    3296,
+    369,
+    279,
+    4560,
+    314,
+    1396,
+    21831,
+    421,
+    10695,
+    13,
+    1049,
+    11690,
+    440,
+    264,
+    4145,
+    21254,
+    27090,
+    7397,
+    421,
+    2682,
+    27155,
+    1040,
+    264,
+    17312,
+    11,
+    264,
+    3000,
+    421,
+    2696,
+    82,
+    1040,
+    264,
+    27786,
+    68220,
+    16312,
+    424,
+    50110,
+    1083,
+    264,
+    8516,
+    314,
+    5721,
+    11,
+    1754,
+    2569,
+    264,
+    2029,
+    421,
+    628,
+    381,
+    9770,
+    11,
+    264,
+    3255,
+    421,
+    628,
+    381,
+    2044,
+    787,
+    11,
+    264,
+    2253,
+    421,
+    628,
+    381,
+    12152,
+    13,
+    3113,
+    43900,
+    421,
+    8516,
+    567,
+    3166,
+    494,
+    279,
+    13769,
+    318,
+    1719,
+    45366,
+    11,
+    279,
+    7397,
+    4600,
+    725,
+    1433,
+    8,
+    310,
+    279,
+    7859,
+    318,
+    1719,
+    6775,
+    314,
+    6337,
+    4924,
+    681,
+    321,
+    567,
+    6779,
+    1204,
+    279,
+    1788,
+    1654,
+    421,
+    65199,
+    264,
+    54517,
+    21831,
+    1048,
+    1172,
+    72020,
+    279,
+    2172,
+    11991,
+    89282,
+    314,
+    279,
+    1483,
+    1330,
+    22892,
+    13,
+    271,
+    13962,
+    220,
+    16,
+    13,
+    5369,
+    264,
+    11000,
+    1473,
+    80222,
+    310,
+    264,
+    3146,
+    6519,
+    271,
+    760,
+    54517,
+    7110,
+    725,
+    52926,
+    43866,
+    264,
+    19565,
+    3377,
+    25,
+    2972,
+    4993,
+    635,
+    567,
+    9973,
+    864,
+    314,
+    1330,
+    49314,
+    9331,
+    369,
+    804,
+    170286,
+    561,
+    4087,
+    369,
+    524,
+    264,
+    3074,
+    5728,
+    694,
+    264,
+    1817,
+    25,
+    271,
+    16,
+    13,
+    2972,
+    27382,
+    1386,
+    279,
+    60826,
+    159029,
+    357,
+    1001,
+    52513,
+    828,
+    8026,
+    321,
+    264,
+    1001,
+    27392,
+    828,
+    3000,
+    4021,
+    1027,
+    27642,
+    1973,
+    15673,
+    13,
+    198,
+    17,
+    13,
+    2972,
+    38,
+    1792,
+    9209,
+    5721,
+    159029,
+    561,
+    45366,
+    11,
+    279,
+    7397,
+    4600,
+    725,
+    1433,
+    11,
+    40554,
+    12269,
+    11,
+    321,
+    279,
+    2876,
+    314,
+    279,
+    38866,
+    660,
+    3512,
+    795,
+    3387,
+    13,
+    198,
+    18,
+    13,
+    2972,
+    1774,
+    264,
+    29094,
+    159029,
+    18200,
+    279,
+    348,
+    32,
+    322,
+    6020,
+    9,
+    557,
+    524,
+    5384,
+    694,
+    7920,
+    303,
+    264,
+    29262,
+    5883,
+    11,
+    321,
+    279,
+    1001,
+    4009,
+    828,
+    557,
+    264,
+    5843,
+    1096,
+    1489,
+    1129,
+    49722,
+    13,
+    198,
+    19,
+    13,
+    2972,
+    2198,
+    279,
+    29094,
+    159029,
+    4109,
+    3315,
+    29262,
+    557
+   ],
+   "text": "The harbor master kept a ledger of every ship that wintered in the bay, noting cargo, crew, and the state of each hull in a cramped, looping hand. In the spring of the third year he found a single entry that did not match any of the others: a vessel called the *Aurora* had been recorded as \u201clost at sea\u201d on the night of the equinox, yet the same night a fisherman\u2019s log described a bright, silver\u2011scaled fish leaping from the water and a strange, humming light over the harbor. The master\u2019s pen trembled; the two records could not both be true, and the contradiction forced him to ask a question that had haunted sailors for centuries: **what, if anything, lies beneath the surface of the world we think we know?**\n\nThat question is the engine of every mystery that follows. It begins with a simple observation\u2014a fish that glitters like a mirror, a light that hums like a distant choir\u2014and it expands into a chain of evidence, each link a fact that can be checked, a story that can be retold, a law that can be tested. By tracing that chain we move from the concrete (the ledger, the fisherman\u2019s log) to the abstract (the nature of knowledge itself), and we discover how the same method that solves a harbor mystery also underpins the great scientific revolutions of the last two centuries.\n\n### 1. From a Single Anomaly to a General Method\n\nThe harbor master\u2019s dilemma illustrates a universal problem: **how do we decide which of two conflicting accounts is true?** The answer is not a single rule but a process:\n\n1. **Identify the anomaly.** A \u201clost\u201d ship and a \u201cseen\u201d light cannot coexist without explanation.\n2. **Gather independent evidence.** The ledger, the fisherman\u2019s log, tide tables, and the condition of the hull all become data points.\n3. **Form a hypothesis.** Perhaps the *Aurora* was not lost but hidden in a fog bank, and the \u201clight\u201d was a bioluminescent bloom.\n4. **Test the hypothesis.** Check whether fog was\n"
+  },
+  {
+   "index": 1,
+   "match": 139,
+   "ref_len": 400,
+   "byte_identical": false,
+   "prefill": "prefill: 41 tokens in 1 chunk(s) of <= 512 (87.633552ms, 468 tok/s, eager, emits next_token)",
+   "decode": "spec: 399 tokens in 143 rounds (2.79 tokens/round, 25.6% drafts accepted, per position [101, 62, 41, 21, 15, 9, 7]), 14.11 ms/round (197.7 tok/s)",
+   "wall_s": 10.6,
+   "generated": [
+    264,
+    36334,
+    21388,
+    314,
+    5484,
+    5415,
+    321,
+    264,
+    716,
+    314,
+    34844,
+    542,
+    5564,
+    58091,
+    1172,
+    279,
+    7991,
+    846,
+    13,
+    561,
+    1118,
+    3603,
+    1172,
+    279,
+    491,
+    12515,
+    11,
+    279,
+    6561,
+    2002,
+    6749,
+    92169,
+    1056,
+    5316,
+    25624,
+    11,
+    3482,
+    279,
+    36334,
+    467,
+    78078,
+    5432,
+    68,
+    1990,
+    9047,
+    310,
+    288,
+    92536,
+    13,
+    561,
+    2015,
+    14607,
+    421,
+    279,
+    53952,
+    725,
+    27374,
+    5257,
+    557,
+    874,
+    4860,
+    264,
+    4145,
+    1001,
+    75,
+    704,
+    54087,
+    429,
+    54087,
+    70735,
+    828,
+    3255,
+    26,
+    424,
+    557,
+    264,
+    61010,
+    23850,
+    314,
+    25586,
+    11,
+    1754,
+    440,
+    1141,
+    1765,
+    79299,
+    11,
+    25113,
+    11,
+    321,
+    3558,
+    13,
+    271,
+    1206,
+    12689,
+    3908,
+    421,
+    23850,
+    11,
+    567,
+    1902,
+    1118,
+    2466,
+    25,
+    2972,
+    4993,
+    635,
+    567,
+    7276,
+    264,
+    1785,
+    314,
+    1599,
+    18766,
+    33765,
+    25586,
+    303,
+    264,
+    1560,
+    421,
+    14908,
+    586,
+    6805,
+    1332,
+    449,
+    2099,
+    668,
+    1304,
+    11,
+    1204,
+    3349,
+    424,
+    668,
+    381,
+    11,
+    321,
+    1204,
+    279,
+    3000,
+    668,
+    35600,
+    170286,
+    561,
+    4087,
+    14995,
+    303,
+    264,
+    16338,
+    716,
+    314,
+    36655,
+    421,
+    4089,
+    279,
+    4220,
+    14193,
+    430,
+    264,
+    3074,
+    1001,
+    24507,
+    11432,
+    828,
+    17796,
+    11,
+    1345,
+    1990,
+    9976,
+    3627,
+    314,
+    279,
+    3714,
+    5255,
+    421,
+    2873,
+    279,
+    1785,
+    1141,
+    3542,
+    13,
+    271,
+    13962,
+    220,
+    16,
+    13,
+    5369,
+    264,
+    3074,
+    7099,
+    310,
+    264,
+    4220,
+    1785,
+    271,
+    2381,
+    264,
+    3074,
+    18766,
+    33765,
+    7099,
+    279,
+    1298,
+    686,
+    518,
+    12323,
+    369,
+    271,
+    75370,
+    198,
+    59,
+    35790,
+    88216,
+    8017,
+    90,
+    82,
+    8017,
+    29533,
+    35790,
+    88216,
+    14728,
+    82,
+    92,
+    33289,
+    35790,
+    88216,
+    23531,
+    77,
+    14728,
+    81,
+    2069,
+    198,
+    59,
+    60,
+    271,
+    2777,
+    17237,
+    82,
+    55821,
+    321,
+    17237,
+    82,
+    15313,
+    8,
+    513,
+    279,
+    1576,
+    321,
+    2099,
+    25704,
+    16384,
+    494,
+    279,
+    7099,
+    11,
+    17237,
+    77,
+    55821,
+    321,
+    17237,
+    77,
+    15313,
+    8,
+    513,
+    279,
+    18766,
+    2932,
+    14505,
+    383,
+    2892,
+    3008,
+    11,
+    321,
+    17237,
+    81,
+    55821,
+    369,
+    279,
+    7099,
+    10264,
+    13,
+    18050,
+    4350,
+    3679,
+    25586,
+    513,
+    8826,
+    303,
+    8240,
+    11,
+    279,
+    1788,
+    1304,
+    79476,
+    413,
+    567,
+    8032,
+    279,
+    3714,
+    31591,
+    539,
+    2972,
+    8678,
+    54087,
+    8877,
+    332,
+    5981,
+    25,
+    271,
+    75370,
+    198,
+    59,
+    35790,
+    88216,
+    8017,
+    90,
+    82,
+    8017,
+    29533,
+    35790,
+    88216,
+    14728,
+    82,
+    92,
+    33289,
+    35790,
+    88216,
+    23531,
+    77,
+    14728,
+    81,
+    14717,
+    59,
+    1272,
+    90,
+    10684,
+    3307,
+    7563,
+    198,
+    59,
+    60,
+    271,
+    8160,
+    17237,
+    81,
+    14717,
+    59,
+    1272,
+    90,
+    10684,
+    3307,
+    55821,
+    369,
+    279,
+    10264,
+    314,
+    79299,
+    314,
+    449,
+    348,
+    24507,
+    11432,
+    9,
+    3074,
+    7099,
+    421,
+    1000,
+    7936,
+    6681,
+    279,
+    1788,
+    1298,
+    686,
+    518,
+    30619,
+    430,
+    279,
+    4220,
+    5436,
+    13,
+    561,
+    2279,
+    314,
+    279,
+    1785,
+    369,
+    271,
+    75370,
+    198,
+    59,
+    43660,
+    283,
+    307,
+    6,
+    471,
+    307,
+    283,
+    1088,
+    35790,
+    88216,
+    23531
+   ],
+   "text": "When the observatory finally reopened after the renovation, the docents discovered that the old refractor had been quietly recollimated by a retired machinist who lived nearby. He left no note, only a faint smell of machine oil and a set of brass shims tucked under the focuser. The first night under the new sky, the star field looked sharper than anyone remembered, yet the faintest nebulae still seemed to wobble. The team realized that the telescope\u2019s optical train was no longer a simple \u201clens\u2011and\u2011mirror\u201d story; it was a layered puzzle of surfaces, each with its own curvature, spacing, and material.\n\nTo untangle that puzzle, we must first ask: **how do we describe a system of many refracting surfaces in a way that lets us predict where an image will form, how large it will be, and how the light will bend?** The answer lies in a compact set of equations that treat the whole assembly as a single \u201cequivalent\u201d lens, while still keeping track of the individual elements that give the system its character.\n\n### 1. From a single surface to a whole system\n\nFor a single refracting surface the paraxial relation is\n\n\\[\n\\frac{n'}{s'}-\\frac{n}{s}=\\frac{n'-n}{r},\n\\]\n\nwhere \\(s\\) and \\(s'\\) are the object and image distances measured from the surface, \\(n\\) and \\(n'\\) are the refractive indices on either side, and \\(r\\) is the surface radius.  \n\nWhen several surfaces are placed in sequence, the same form survives if we replace the individual quantities by **system\u2011wide** ones:\n\n\\[\n\\frac{n'}{s'}-\\frac{n}{s}=\\frac{n'-n}{r_{\\text{eq}}}.\n\\]\n\nHere \\(r_{\\text{eq}}\\) is the radius of curvature of an *equivalent* single surface that would produce exactly the same paraxial imaging as the whole stack. The power of the system is\n\n\\[\n\\Phi = n' - n = \\frac{n'-\n"
+  },
+  {
+   "index": 2,
+   "match": 147,
+   "ref_len": 400,
+   "byte_identical": false,
+   "prefill": "prefill: 48 tokens in 1 chunk(s) of <= 512 (92.620306ms, 518 tok/s, eager, emits next_token)",
+   "decode": "spec: 399 tokens in 161 rounds (2.48 tokens/round, 21.1% drafts accepted, per position [115, 58, 33, 17, 6, 5, 4]), 14.25 ms/round (173.8 tok/s)",
+   "wall_s": 10.6,
+   "generated": [
+    7094,
+    781,
+    14428,
+    279,
+    53627,
+    287,
+    440,
+    22945,
+    9399,
+    49954,
+    11,
+    8132,
+    1396,
+    2418,
+    314,
+    272,
+    46016,
+    11,
+    1396,
+    10582,
+    314,
+    274,
+    6960,
+    11,
+    1396,
+    22263,
+    23178,
+    17629,
+    22149,
+    13,
+    733,
+    279,
+    4688,
+    7094,
+    781,
+    5743,
+    310,
+    279,
+    1788,
+    22945,
+    52604,
+    321,
+    1669,
+    279,
+    14367,
+    995,
+    7481,
+    13,
+    271,
+    760,
+    42159,
+    557,
+    2496,
+    7807,
+    13,
+    561,
+    14367,
+    8341,
+    7594,
+    795,
+    13,
+    271,
+    4277,
+    271,
+    760,
+    1118,
+    3065,
+    488,
+    5125,
+    883,
+    279,
+    17198,
+    20139,
+    369,
+    421,
+    424,
+    369,
+    524,
+    264,
+    1925,
+    13,
+    1049,
+    369,
+    264,
+    1817,
+    11884,
+    279,
+    30385,
+    314,
+    264,
+    1925,
+    13,
+    561,
+    4128,
+    488,
+    2418,
+    383,
+    557,
+    14367,
+    2634,
+    264,
+    12828,
+    3998,
+    321,
+    668,
+    381,
+    14367,
+    2634,
+    1495,
+    303,
+    264,
+    12828,
+    13,
+    561,
+    12052,
+    513,
+    524,
+    39527,
+    303,
+    16665,
+    748,
+    1691,
+    430,
+    303,
+    264,
+    46446,
+    440,
+    2919,
+    13,
+    561,
+    272,
+    46016,
+    29528,
+    513,
+    524,
+    19101,
+    13,
+    2302,
+    513,
+    279,
+    4757,
+    579,
+    1560,
+    314,
+    25108,
+    948,
+    279,
+    4751,
+    369,
+    264,
+    21128,
+    13,
+    271,
+    760,
+    2018,
+    3065,
+    488,
+    5125,
+    369,
+    279,
+    20513,
+    13,
+    2717,
+    279,
+    18682,
+    314,
+    4950,
+    13,
+    561,
+    9084,
+    314,
+    264,
+    2086,
+    2993,
+    314,
+    4950,
+    13,
+    561,
+    3594,
+    369,
+    11702,
+    440,
+    279,
+    11639,
+    314,
+    2919,
+    6992,
+    1472,
+    51930,
+    11,
+    314,
+    39465,
+    3133,
+    279,
+    7099,
+    22770,
+    11,
+    314,
+    19053,
+    7842,
+    303,
+    32655,
+    421,
+    2714,
+    5995,
+    310,
+    5640,
+    1472,
+    13988,
+    3594,
+    13,
+    561,
+    20513,
+    369,
+    279,
+    4950,
+    314,
+    264,
+    1785,
+    303,
+    53454,
+    13,
+    2441,
+    279,
+    4950,
+    314,
+    264,
+    1785,
+    52962,
+    310,
+    381,
+    303,
+    53454,
+    13,
+    271,
+    760,
+    4688,
+    3065,
+    488,
+    5125,
+    369,
+    279,
+    21388,
+    13,
+    326,
+    13898,
+    322,
+    13,
+    93048,
+    3374,
+    13,
+    561,
+    3820,
+    61306,
+    314,
+    2919,
+    421,
+    682,
+    978,
+    1472,
+    264,
+    15706,
+    1578,
+    314,
+    15463,
+    37302,
+    321,
+    3578,
+    495,
+    643,
+    344,
+    13,
+    561,
+    21388,
+    314,
+    264,
+    1925,
+    421,
+    682,
+    978,
+    20245,
+    286,
+    4924,
+    364,
+    264,
+    1546,
+    1248,
+    854,
+    13,
+    271,
+    4277,
+    271,
+    760,
+    53627,
+    287,
+    998,
+    5617,
+    303,
+    279,
+    220,
+    16,
+    24,
+    18,
+    15,
+    82,
+    13,
+    561,
+    12638,
+    29158,
+    314,
+    47070,
+    3571,
+    440,
+    836,
+    10258,
+    286,
+    6875,
+    321,
+    836,
+    13769,
+    321,
+    836,
+    42197,
+    421,
+    264,
+    14367,
+    1362,
+    381,
+    1801,
+    310,
+    34512,
+    13,
+    2302,
+    5617,
+    279,
+    53627,
+    287,
+    310,
+    5827,
+    279,
+    22787,
+    4881,
+    321,
+    279,
+    13176,
+    41276,
+    321,
+    279,
+    2526,
+    24039,
+    421,
+    995,
+    14276,
+    685,
+    3055,
+    279,
+    13558,
+    13,
+    2302,
+    1467,
+    524,
+    1870,
+    1070,
+    310,
+    5827,
+    279,
+    17198,
+    20139,
+    13,
+    2302,
+    5617,
+    1070,
+    310,
+    1236,
+    279,
+    17198,
+    20139,
+    708,
+    3023,
+    13,
+    271,
+    760,
+    53627,
+    287,
+    6237,
+    13,
+    1690,
+    264,
+    1345,
+    13,
+    561,
+    14367,
+    19013,
+    303,
+    1141,
+    5323,
+    13,
+    561,
+    22787,
+    13538
+   ],
+   "text": "The floodplain census took three summers to complete. In the first summer the crews mapped oxbow lakes and counted heron rookeries from canoes, losing two clipboards and one outboard motor to the river. In the second summer they walked the levees with transect tapes, logging every stand of cypress, every patch of sedge, every abandoned crawfish trap. In the third summer they returned to the same transects and found the river had moved.\n\nThe census was never finished. The river kept adding data.\n\n---\n\nThe first thing you notice about the floodplain is that it is not a place. It is a process wearing the costume of a place. The land you stand on was riverbed a decade ago and will be riverbed again in a decade. The trees are not rooted in soil so much as in a negotiation with water. The cypress knees are not roots. They are the tree's way of breathing when the ground is a lake.\n\nThe second thing you notice is the silence. Not the absence of sound. The presence of a different kind of sound. The air is thick with the noise of water moving through vegetation, of insects working the surface tension, of birds calling in frequencies that seem designed to travel through wet air. The silence is the sound of a system in equilibrium. Or the sound of a system pretending to be in equilibrium.\n\nThe third thing you notice is the smell. Sulfur. Decomposition. The particular sweetness of water that has been through a thousand years of leaf litter and root exudate. The smell of a place that has been digesting itself for a very long time.\n\n---\n\nThe levees were built in the 1930s. The Army Corps of Engineers came with their surveying equipment and their concrete and their certainty that a river could be made to behave. They built the levees to protect the cotton fields and the railroads and the small towns that had grown up along the banks. They did not build them to protect the floodplain. They built them to make the floodplain go away.\n\nThe levees worked. For a while. The river stayed in its channel. The cotton grew\n"
+  },
+  {
+   "index": 3,
+   "match": 91,
+   "ref_len": 400,
+   "byte_identical": false,
+   "prefill": "prefill: 42 tokens in 1 chunk(s) of <= 512 (90.546385ms, 464 tok/s, eager, emits next_token)",
+   "decode": "spec: 399 tokens in 134 rounds (2.98 tokens/round, 28.4% drafts accepted, per position [102, 68, 43, 26, 14, 9, 4]), 14.15 ms/round (210.4 tok/s)",
+   "wall_s": 9.9,
+   "generated": [
+    271,
+    8160,
+    369,
+    449,
+    15673,
+    314,
+    1439,
+    2250,
+    41724,
+    303,
+    13983,
+    3992,
+    11,
+    3055,
+    440,
+    264,
+    13769,
+    3010,
+    364,
+    1754,
+    13,
+    271,
+    13962,
+    220,
+    16,
+    13,
+    8169,
+    264,
+    19269,
+    7541,
+    2751,
+    303,
+    18266,
+    271,
+    332,
+    760,
+    68868,
+    64700,
+    198,
+    33807,
+    303,
+    279,
+    1814,
+    369,
+    1801,
+    314,
+    13280,
+    18167,
+    318,
+    63339,
+    321,
+    33470,
+    8,
+    421,
+    513,
+    14537,
+    6992,
+    13,
+    3095,
+    279,
+    9039,
+    35937,
+    11,
+    1439,
+    18167,
+    8457,
+    4649,
+    321,
+    3166,
+    10281,
+    11,
+    13969,
+    1070,
+    310,
+    4336,
+    9781,
+    4473,
+    10431,
+    494,
+    799,
+    2361,
+    13,
+    1061,
+    369,
+    2512,
+    2972,
+    65819,
+    14046,
+    159034,
+    271,
+    7732,
+    264,
+    13759,
+    2493,
+    1353,
+    1040,
+    264,
+    6235,
+    11,
+    632,
+    49971,
+    2424,
+    314,
+    9229,
+    466,
+    13769,
+    11,
+    424,
+    369,
+    3404,
+    264,
+    4281,
+    314,
+    30995,
+    314,
+    1439,
+    18167,
+    13,
+    733,
+    279,
+    7094,
+    8364,
+    11,
+    279,
+    3558,
+    86914,
+    4649,
+    11,
+    279,
+    18167,
+    343,
+    41670,
+    777,
+    67274,
+    11,
+    321,
+    279,
+    4307,
+    5759,
+    21219,
+    5055,
+    4860,
+    13,
+    1368,
+    23438,
+    3105,
+    914,
+    2605,
+    364,
+    411,
+    11,
+    279,
+    13759,
+    1362,
+    77238,
+    466,
+    16966,
+    1172,
+    279,
+    7035,
+    314,
+    1141,
+    1765,
+    14046,
+    13,
+    271,
+    332,
+    80711,
+    12996,
+    64700,
+    198,
+    49338,
+    264,
+    2972,
+    4670,
+    9041,
+    46379,
+    332,
+    2047,
+    383,
+    264,
+    3882,
+    1740,
+    26103,
+    303,
+    5589,
+    13,
+    1368,
+    488,
+    6420,
+    424,
+    440,
+    264,
+    1564,
+    12489,
+    11,
+    424,
+    668,
+    381,
+    264,
+    13280,
+    18830,
+    314,
+    264,
+    2463,
+    24529,
+    4860,
+    1056,
+    424,
+    557,
+    303,
+    5869,
+    13,
+    357,
+    13759,
+    369,
+    1066,
+    264,
+    14120,
+    2243,
+    314,
+    411,
+    46379,
+    26,
+    1521,
+    424,
+    369,
+    11168,
+    314,
+    19434,
+    1248,
+    11,
+    421,
+    13280,
+    14046,
+    11039,
+    685,
+    310,
+    3679,
+    14493,
+    314,
+    6977,
+    11,
+    864,
+    369,
+    3069,
+    488,
+    1436,
+    31092,
+    318,
+    4431,
+    10188,
+    33124,
+    8,
+    1881,
+    13753,
+    314,
+    279,
+    13759,
+    13,
+    271,
+    11862,
+    271,
+    13962,
+    220,
+    17,
+    13,
+    8169,
+    264,
+    710,
+    46038,
+    326,
+    793,
+    12851,
+    1049,
+    2436,
+    8405,
+    271,
+    332,
+    760,
+    68868,
+    64700,
+    198,
+    760,
+    328,
+    82,
+    23279,
+    1,
+    466,
+    412,
+    375,
+    2631,
+    4950,
+    369,
+    3404,
+    264,
+    1067,
+    314,
+    2972,
+    85,
+    17804,
+    8613,
+    539,
+    19695,
+    1602,
+    29592,
+    321,
+    1179,
+    5698,
+    159034,
+    271,
+    4350,
+    488,
+    1118,
+    8364,
+    2919,
+    11,
+    42237,
+    1304,
+    506,
+    279,
+    5445,
+    314,
+    279,
+    69862,
+    1332,
+    424,
+    369,
+    36056,
+    13,
+    4081,
+    42237,
+    513,
+    1801,
+    314,
+    19695,
+    13,
+    4213,
+    11,
+    279,
+    2919,
+    4918,
+    685,
+    303,
+    279,
+    69862,
+    369,
+    1990,
+    6792,
+    13,
+    1577,
+    279,
+    19695,
+    42237,
+    9704,
+    11,
+    781,
+    4062,
+    279,
+    34635,
+    2919,
+    321,
+    21146,
+    17638,
+    318,
+    1252,
+    1081,
+    1142,
+    1083,
+    14058,
+    553,
+    1061,
+    10725,
+    69089,
+    11121,
+    264,
+    50240,
+    11,
+    54651,
+    2631,
+    11639,
+    13,
+    271,
+    2053,
+    279,
+    2919,
+    5055,
+    11784,
+    310,
+    48161,
+    11,
+    279,
+    4307,
+    7867,
+    314,
+    2919,
+    8772,
+    3882,
+    13,
+    561
+   ],
+   "text": "Explain, in plain language and with one concrete example each, why a bridge expands in summer, why a kettle sings before it boils, and why the far side of the moon is never seen from Earth.\n\nHere is an explanation of these three phenomena in plain language, along with a concrete example for each.\n\n### 1. Why a Bridge Expands in Summer\n\n**The Explanation:**\nEverything in the world is made of tiny particles (atoms and molecules) that are constantly moving. When the temperature rises, these particles gain energy and move faster, causing them to push slightly further apart from one another. This is called **thermal expansion**.\n\nWhile a bridge might look like a solid, unchanging block of steel or concrete, it is actually a collection of billions of these particles. In the summer heat, the material absorbs energy, the particles vibrate more intensely, and the entire structure physically gets longer. If engineers didn't account for this, the bridge could buckle or crack under the pressure of its own expansion.\n\n**Concrete Example:**\nImagine a **long metal ruler** left on a hot car dashboard in July. If you measure it with a caliper, it will be a tiny fraction of a millimeter longer than it was in January. A bridge is just a giant version of this ruler; because it is hundreds of meters long, that tiny expansion adds up to several inches of movement, which is why you see gaps (expansion joints) between sections of the bridge.\n\n***\n\n### 2. Why a Kettle Sings Before It Boils\n\n**The Explanation:**\nThe \"singing\" or whistling sound is actually a result of **vibration caused by steam being trapped and then released**.\n\nWhen you first heat water, bubbles form at the bottom of the kettle where it is hottest. These bubbles are made of steam. However, the water higher up in the kettle is still cool. As the steam bubbles rise, they hit the cooler water and instantly collapse (condense back into liquid). This rapid collapsing creates a chaotic, rattling noise.\n\nAs the water gets closer to boiling, the entire volume of water becomes hot. The\n"
+  },
+  {
+   "index": 4,
+   "match": 91,
+   "ref_len": 400,
+   "byte_identical": false,
+   "prefill": "prefill: 47 tokens in 1 chunk(s) of <= 512 (88.850289ms, 529 tok/s, eager, emits next_token)",
+   "decode": "spec: 399 tokens in 180 rounds (2.22 tokens/round, 17.5% drafts accepted, per position [113, 62, 30, 11, 2, 2, 1]), 14.13 ms/round (156.9 tok/s)",
+   "wall_s": 10.4,
+   "generated": [
+    9896,
+    2250,
+    75842,
+    440,
+    11140,
+    421,
+    998,
+    40361,
+    42749,
+    13,
+    3113,
+    35442,
+    1292,
+    995,
+    5158,
+    264,
+    11183,
+    883,
+    264,
+    5054,
+    852,
+    1362,
+    6512,
+    279,
+    9117,
+    579,
+    295,
+    8792,
+    1472,
+    279,
+    6220,
+    18687,
+    11,
+    321,
+    1292,
+    4035,
+    2574,
+    424,
+    310,
+    1024,
+    6764,
+    11,
+    852,
+    1301,
+    424,
+    303,
+    799,
+    11362,
+    321,
+    1179,
+    1018,
+    11,
+    328,
+    2523,
+    2908,
+    1801,
+    279,
+    9117,
+    4950,
+    1040,
+    264,
+    27809,
+    36252,
+    1149,
+    2838,
+    3738,
+    411,
+    430,
+    279,
+    8166,
+    27463,
+    13,
+    271,
+    760,
+    11183,
+    557,
+    2496,
+    4501,
+    13,
+    1049,
+    7338,
+    303,
+    264,
+    25635,
+    364,
+    42744,
+    1578,
+    11,
+    22175,
+    279,
+    3820,
+    15333,
+    314,
+    264,
+    324,
+    55965,
+    2218,
+    40825,
+    321,
+    27056,
+    5415,
+    321,
+    279,
+    36334,
+    43651,
+    12293,
+    314,
+    279,
+    29262,
+    49172,
+    13,
+    3095,
+    1292,
+    5326,
+    3738,
+    424,
+    680,
+    11,
+    279,
+    6602,
+    995,
+    83237,
+    1083,
+    21030,
+    16382,
+    11,
+    321,
+    1292,
+    3272,
+    25,
+    411,
+    369,
+    1092,
+    424,
+    3254,
+    310,
+    381,
+    32471,
+    13,
+    271,
+    7691,
+    557,
+    16570,
+    52616,
+    948,
+    1292,
+    2047,
+    279,
+    12552,
+    13,
+    561,
+    48331,
+    557,
+    264,
+    10439,
+    314,
+    11639,
+    321,
+    12106,
+    11,
+    264,
+    1925,
+    1332,
+    1208,
+    14466,
+    888,
+    1754,
+    975,
+    321,
+    279,
+    3000,
+    3571,
+    494,
+    8799,
+    51432,
+    4370,
+    314,
+    264,
+    33433,
+    1292,
+    4035,
+    47158,
+    1396,
+    3603,
+    13,
+    2838,
+    47224,
+    264,
+    2956,
+    3294,
+    264,
+    509,
+    5085,
+    579,
+    7808,
+    303,
+    264,
+    2548,
+    6092,
+    11,
+    321,
+    279,
+    21388,
+    314,
+    32313,
+    321,
+    11835,
+    500,
+    724,
+    290,
+    1472,
+    279,
+    6220,
+    18687,
+    1040,
+    264,
+    4779,
+    1292,
+    7448,
+    914,
+    4847,
+    1925,
+    13,
+    271,
+    20079,
+    1118,
+    4501,
+    3255,
+    9434,
+    303,
+    264,
+    30333,
+    14005,
+    440,
+    264,
+    33606,
+    314,
+    2943,
+    7493,
+    10672,
+    13,
+    1049,
+    557,
+    883,
+    264,
+    5054,
+    852,
+    8341,
+    264,
+    324,
+    55965,
+    421,
+    5532,
+    914,
+    1017,
+    11,
+    852,
+    47158,
+    264,
+    33433,
+    421,
+    25774,
+    303,
+    279,
+    3594,
+    11,
+    852,
+    11938,
+    310,
+    17199,
+    421,
+    2496,
+    3571,
+    13,
+    561,
+    6238,
+    5947,
+    310,
+    1024,
+    25,
+    328,
+    2523,
+    599,
+    264,
+    7932,
+    364,
+    279,
+    11656,
+    1801,
+    18541,
+    13,
+    1165,
+    4035,
+    1040,
+    310,
+    3305,
+    777,
+    1149,
+    2838,
+    1301,
+    279,
+    6321,
+    2250,
+    2942,
+    11,
+    1179,
+    46120,
+    424,
+    310,
+    279,
+    6784,
+    3294,
+    1024,
+    17471,
+    11,
+    1332,
+    424,
+    19013,
+    364,
+    279,
+    2645,
+    314,
+    1024,
+    2199,
+    13,
+    271,
+    760,
+    7112,
+    3571,
+    303,
+    26104,
+    11,
+    1040,
+    8831,
+    5757,
+    13,
+    357,
+    35528,
+    314,
+    2250,
+    1578,
+    11,
+    1179,
+    264,
+    17198,
+    314,
+    1070,
+    13,
+    2838,
+    5947,
+    883,
+    279,
+    3096,
+    852,
+    295,
+    2789,
+    50692,
+    303,
+    279,
+    72103,
+    11,
+    836,
+    18645,
+    748,
+    1562,
+    35283,
+    781,
+    1362,
+    2579,
+    279,
+    58186,
+    303,
+    279,
+    32313,
+    494,
+    264,
+    5821,
+    13,
+    2838,
+    5947,
+    883,
+    279,
+    2235,
+    855,
+    852,
+    5898,
+    1675,
+    24231,
+    314,
+    279,
+    324,
+    55965,
+    11,
+    1754,
+    799,
+    9781,
+    2086,
+    11,
+    430,
+    413
+   ],
+   "text": "The lighthouse keeper's daughter learned to read from shipping manifests and weather logs, so her first stories were inventories: forty barrels of salt cod, a crate of oranges gone soft, one bishop travelling incognito. By twelve she had filled three notebooks with lists that were secretly poems. By fifteen she had written a novel about a woman who could hear the sea's moods through the floorboards, and she'd given it to her father, who read it in one sitting and then said, \"You've made the sea sound like a drunk uncle.\" She took this as the highest praise.\n\nThe novel was never published. It sat in a drawer for eleven years, gathering the particular dust of a lighthouse\u2014salt and lamp oil and the faint metallic taste of the foghorn. When she finally took it out, the pages had warped into gentle waves, and she thought: this is what it means to be preserved.\n\nShe was twenty-six when she left the island. The mainland was a shock of noise and colour, a place where people talked over each other and the light came from electric bulbs instead of a flame she'd tended every night. She rented a room above a chandler's shop in a port town, and the smell of rope and tar seeped through the floorboards like a memory she couldn't quite place.\n\nHer first published story appeared in a literary magazine with a circulation of four hundred copies. It was about a woman who kept a lighthouse that wasn't there, who tended a flame that burned in the air, who spoke to ships that never came. The editor wrote to her: \"You have a gift for the impossible made ordinary. We'd like to publish more.\" She read the letter three times, then pinned it to the wall above her desk, where it stayed for the rest of her life.\n\nThe stories came in clusters, like weather systems. A drought of three years, then a flood of them. She wrote about the women who mended nets in the harbour, their fingers so calloused they could feel the knots in the rope from a distance. She wrote about the old man who sold postcards of the lighthouse, each one slightly different, as if\n"
+  }
+ ]
+}
\ No newline at end of file
diff --git a/docs/qwen38-vllm-perf-plain-pinned-tray14.json b/docs/qwen38-vllm-perf-plain-pinned-tray14.json
new file mode 100644
index 0000000..13d3daf
--- /dev/null
+++ b/docs/qwen38-vllm-perf-plain-pinned-tray14.json
@@ -0,0 +1,2048 @@
+{
+ "mode": "plain",
+ "pinned": true,
+ "results": [
+  {
+   "prompt_tokens": 43,
+   "out_tokens": 400,
+   "seconds": 4.177,
+   "tok_s": 95.8,
+   "output_token_ids": [
+    264,
+    3074,
+    4202,
+    421,
+    995,
+    978,
+    26164,
+    680,
+    321,
+    56546,
+    25,
+    1001,
+    53,
+    34373,
+    348,
+    32,
+    322,
+    6020,
+    9,
+    1317,
+    220,
+    16,
+    17,
+    15,
+    19008,
+    1317,
+    220,
+    16,
+    19,
+    13234,
+    1317,
+    38866,
+    4950,
+    1920,
+    561,
+    3889,
+    1500,
+    11,
+    18869,
+    2392,
+    1196,
+    23221,
+    279,
+    25570,
+    11,
+    1301,
+    1001,
+    53,
+    34373,
+    348,
+    32,
+    322,
+    6020,
+    9,
+    1317,
+    220,
+    16,
+    17,
+    15,
+    19008,
+    1317,
+    220,
+    16,
+    19,
+    13234,
+    1317,
+    38866,
+    41423,
+    1920,
+    561,
+    7110,
+    725,
+    44579,
+    995,
+    41065,
+    11,
+    321,
+    279,
+    25423,
+    557,
+    1801,
+    303,
+    264,
+    2086,
+    26609,
+    314,
+    25570,
+    11,
+    264,
+    36334,
+    6105,
+    421,
+    1467,
+    524,
+    2353,
+    279,
+    14170,
+    3565,
+    13,
+    271,
+    4350,
+    279,
+    45366,
+    557,
+    2843,
+    23354,
+    539,
+    264,
+    55394,
+    41549,
+    11,
+    279,
+    75332,
+    5924,
+    279,
+    73013,
+    314,
+    264,
+    7875,
+    5515,
+    13,
+    561,
+    41549,
+    17736,
+    421,
+    279,
+    348,
+    32,
+    322,
+    6020,
+    9,
+    995,
+    978,
+    49992,
+    1518,
+    279,
+    43062,
+    11,
+    264,
+    3591,
+    421,
+    1000,
+    42359,
+    279,
+    24024,
+    314,
+    279,
+    6355,
+    5832,
+    13,
+    561,
+    29236,
+    17605,
+    11,
+    4611,
+    11,
+    13874,
+    310,
+    279,
+    6105,
+    25570,
+    430,
+    5721,
+    421,
+    279,
+    4202,
+    995,
+    978,
+    27811,
+    1238,
+    279,
+    2029,
+    11,
+    21861,
+    279,
+    12368,
+    2496,
+    9721,
+    13,
+    561,
+    10682,
+    304,
+    54664,
+    524,
+    383,
+    279,
+    8026,
+    725,
+    23611,
+    694,
+    383,
+    279,
+    13280,
+    11,
+    38226,
+    2029,
+    314,
+    279,
+    25570,
+    725,
+    1829,
+    13,
+    271,
+    1919,
+    8961,
+    43866,
+    264,
+    25971,
+    16988,
+    25,
+    264,
+    3074,
+    11,
+    13769,
+    7472,
+    628,
+    64336,
+    449,
+    4307,
+    18641,
+    13,
+    733,
+    279,
+    1788,
+    1560,
+    11,
+    264,
+    20336,
+    1068,
+    2957,
+    13293,
+    421,
+    264,
+    8240,
+    314,
+    4947,
+    2218,
+    16,
+    11,
+    220,
+    16,
+    11,
+    220,
+    17,
+    11,
+    220,
+    18,
+    11,
+    220,
+    20,
+    11,
+    220,
+    23,
+    2218,
+    265,
+    40568,
+    303,
+    264,
+    5215,
+    421,
+    9047,
+    310,
+    7276,
+    279,
+    6310,
+    314,
+    264,
+    37442,
+    6823,
+    13,
+    3113,
+    32127,
+    5330,
+    279,
+    7859,
+    72632,
+    12323,
+    310,
+    279,
+    47643,
+    2099,
+    314,
+    37910,
+    66803,
+    11,
+    279,
+    14377,
+    5924,
+    264,
+    3255,
+    421,
+    5316,
+    1362,
+    1732,
+    13,
+    561,
+    4947,
+    998,
+    874,
+    4860,
+    17210,
+    26,
+    781,
+    998,
+    279,
+    23469,
+    11,
+    279,
+    61368,
+    11,
+    279,
+    491,
+    30459,
+    24572,
+    1754,
+    2182,
+    13,
+    271,
+    35944,
+    2361,
+    1105,
+    494,
+    279,
+    1814,
+    314,
+    14246,
+    13,
+    357,
+    8369,
+    725,
+    4131,
+    54087,
+    2772,
+    1433,
+    12076,
+    264,
+    6937,
+    314,
+    9229,
+    47205,
+    440,
+    264,
+    21330,
+    448,
+    7913,
+    314,
+    220,
+    20,
+    23,
+    15,
+    373,
+    107,
+    5602,
+    64,
+    13,
+    357,
+    1936,
+    2843,
+    11,
+    264,
+    30687,
+    1433,
+    9711,
+    279,
+    1788,
+    6937,
+    506,
+    220,
+    21,
+    17,
+    15,
+    373,
+    107,
+    5602,
+    64,
+    11,
+    279,
+    2222,
+    1801,
+    303,
+    264,
+    2086,
+    5572,
+    13,
+    561,
+    5819,
+    6436,
+    11,
+    65937,
+    279,
+    3889,
+    4202,
+    11,
+    995,
+    2582,
+    26380,
+    279,
+    47205,
+    310,
+    264
+   ]
+  },
+  {
+   "prompt_tokens": 41,
+   "out_tokens": 400,
+   "seconds": 4.171,
+   "tok_s": 95.9,
+   "output_token_ids": [
+    264,
+    36334,
+    21388,
+    314,
+    5484,
+    5415,
+    321,
+    264,
+    716,
+    314,
+    34844,
+    542,
+    5564,
+    58091,
+    1172,
+    279,
+    7991,
+    846,
+    13,
+    561,
+    1118,
+    3603,
+    1172,
+    279,
+    491,
+    12515,
+    11,
+    279,
+    6561,
+    2002,
+    6749,
+    92169,
+    1056,
+    5316,
+    25624,
+    11,
+    3482,
+    279,
+    36334,
+    467,
+    78078,
+    5432,
+    68,
+    1990,
+    9047,
+    310,
+    288,
+    92536,
+    13,
+    561,
+    2015,
+    14607,
+    421,
+    279,
+    53952,
+    725,
+    27374,
+    5257,
+    557,
+    874,
+    4860,
+    264,
+    4145,
+    1001,
+    75,
+    704,
+    54087,
+    429,
+    54087,
+    70735,
+    828,
+    3255,
+    26,
+    424,
+    557,
+    264,
+    61010,
+    23850,
+    314,
+    25586,
+    11,
+    1754,
+    440,
+    1141,
+    1765,
+    79299,
+    11,
+    25113,
+    11,
+    321,
+    3558,
+    13,
+    271,
+    4639,
+    3603,
+    5924,
+    264,
+    1105,
+    3788,
+    303,
+    1204,
+    264,
+    7072,
+    54087,
+    5911,
+    53952,
+    628,
+    381,
+    7246,
+    11,
+    16384,
+    11,
+    321,
+    13260,
+    34773,
+    13,
+    20543,
+    369,
+    279,
+    2923,
+    54087,
+    1634,
+    54087,
+    9239,
+    41223,
+    314,
+    279,
+    3377,
+    11,
+    279,
+    795,
+    279,
+    13234,
+    19577,
+    11,
+    321,
+    279,
+    4602,
+    781,
+    4221,
+    310,
+    4087,
+    13,
+    271,
+    4277,
+    271,
+    13962,
+    220,
+    16,
+    13,
+    561,
+    27374,
+    6576,
+    271,
+    760,
+    13736,
+    369,
+    264,
+    2972,
+    32897,
+    54087,
+    38429,
+    18766,
+    5444,
+    332,
+    318,
+    19186,
+    33800,
+    11,
+    1754,
+    440,
+    264,
+    3929,
+    321,
+    264,
+    1142,
+    7099,
+    8,
+    7854,
+    539,
+    264,
+    2972,
+    18452,
+    17312,
+    332,
+    421,
+    58469,
+    279,
+    22690,
+    1142,
+    1472,
+    279,
+    1788,
+    33800,
+    13,
+    561,
+    9468,
+    4868,
+    513,
+    25,
+    271,
+    91,
+    26099,
+    735,
+    39115,
+    314,
+    79299,
+    318,
+    3693,
+    8,
+    735,
+    9926,
+    318,
+    77,
+    8,
+    735,
+    198,
+    91,
+    22031,
+    91,
+    746,
+    14894,
+    91,
+    40698,
+    91,
+    198,
+    91,
+    220,
+    16,
+    318,
+    6735,
+    314,
+    1118,
+    17796,
+    8,
+    735,
+    478,
+    16,
+    20,
+    15,
+    735,
+    220,
+    16,
+    13,
+    20,
+    16,
+    21,
+    23,
+    735,
+    198,
+    91,
+    220,
+    17,
+    318,
+    1371,
+    314,
+    1118,
+    17796,
+    8,
+    220,
+    735,
+    1317,
+    16,
+    17,
+    15,
+    735,
+    220,
+    16,
+    13,
+    20,
+    16,
+    21,
+    23,
+    735,
+    198,
+    91,
+    220,
+    18,
+    318,
+    6735,
+    314,
+    2018,
+    17796,
+    8,
+    735,
+    478,
+    17,
+    15,
+    15,
+    735,
+    220,
+    16,
+    13,
+    21,
+    17,
+    15,
+    15,
+    735,
+    198,
+    91,
+    220,
+    19,
+    318,
+    1371,
+    314,
+    2018,
+    17796,
+    8,
+    220,
+    735,
+    1317,
+    16,
+    23,
+    15,
+    735,
+    220,
+    16,
+    13,
+    21,
+    17,
+    15,
+    15,
+    735,
+    198,
+    91,
+    220,
+    20,
+    318,
+    18452,
+    17312,
+    8,
+    260,
+    735,
+    11654,
+    252,
+    318,
+    25393,
+    8,
+    735,
+    1317,
+    735,
+    271,
+    760,
+    2972,
+    686,
+    518,
+    4681,
+    778,
+    332,
+    318,
+    2959,
+    54087,
+    951,
+    54087,
+    2959,
+    25704,
+    8,
+    513,
+    25,
+    271,
+    12,
+    26260,
+    25586,
+    220,
+    16,
+    321,
+    220,
+    17,
+    25,
+    2972,
+    16,
+    15,
+    9186,
+    332,
+    198,
+    12,
+    26260,
+    25586,
+    220,
+    17,
+    321,
+    220,
+    18,
+    25,
+    2972,
+    17,
+    20,
+    9186,
+    332,
+    198,
+    12,
+    26260,
+    25586,
+    220,
+    18,
+    321,
+    220,
+    19,
+    25,
+    2972
+   ]
+  },
+  {
+   "prompt_tokens": 48,
+   "out_tokens": 400,
+   "seconds": 4.174,
+   "tok_s": 95.8,
+   "output_token_ids": [
+    7094,
+    781,
+    14428,
+    279,
+    53627,
+    287,
+    440,
+    22945,
+    9399,
+    49954,
+    11,
+    8132,
+    1396,
+    2418,
+    314,
+    272,
+    46016,
+    11,
+    1396,
+    10582,
+    314,
+    274,
+    6960,
+    11,
+    1396,
+    22263,
+    23178,
+    17629,
+    22149,
+    13,
+    733,
+    279,
+    4688,
+    7094,
+    781,
+    5743,
+    310,
+    279,
+    1788,
+    22945,
+    52604,
+    321,
+    1669,
+    279,
+    14367,
+    995,
+    7481,
+    13,
+    271,
+    760,
+    42159,
+    557,
+    2496,
+    7807,
+    13,
+    561,
+    14367,
+    8341,
+    7594,
+    795,
+    13,
+    271,
+    4277,
+    271,
+    760,
+    1118,
+    3065,
+    488,
+    5125,
+    883,
+    279,
+    17198,
+    20139,
+    369,
+    421,
+    424,
+    369,
+    524,
+    264,
+    1925,
+    13,
+    1049,
+    369,
+    264,
+    1817,
+    11884,
+    279,
+    30385,
+    314,
+    264,
+    1925,
+    13,
+    561,
+    4128,
+    488,
+    2418,
+    383,
+    557,
+    264,
+    14367,
+    16534,
+    34227,
+    1578,
+    3998,
+    11,
+    264,
+    21128,
+    2634,
+    1330,
+    7493,
+    1578,
+    3998,
+    11,
+    264,
+    13245,
+    6220,
+    264,
+    15706,
+    1578,
+    3998,
+    13,
+    561,
+    16665,
+    369,
+    264,
+    10537,
+    38121,
+    267,
+    314,
+    71384,
+    321,
+    58371,
+    11,
+    321,
+    279,
+    12052,
+    513,
+    1066,
+    279,
+    5362,
+    38592,
+    303,
+    264,
+    1248,
+    24270,
+    13,
+    271,
+    760,
+    2018,
+    3065,
+    488,
+    5125,
+    369,
+    279,
+    2919,
+    13,
+    2717,
+    279,
+    14367,
+    4924,
+    11,
+    864,
+    369,
+    264,
+    43089,
+    314,
+    13477,
+    15285,
+    13964,
+    1472,
+    279,
+    17865,
+    11,
+    694,
+    279,
+    2919,
+    421,
+    369,
+    16360,
+    745,
+    13,
+    733,
+    279,
+    16665,
+    13,
+    733,
+    279,
+    19101,
+    13,
+    733,
+    279,
+    3594,
+    11,
+    864,
+    33999,
+    314,
+    259,
+    985,
+    258,
+    321,
+    5571,
+    13,
+    733,
+    279,
+    20513,
+    11,
+    864,
+    369,
+    524,
+    20513,
+    694,
+    279,
+    3238,
+    2696,
+    314,
+    264,
+    3410,
+    2526,
+    28051,
+    1881,
+    2919,
+    321,
+    4128,
+    13,
+    271,
+    760,
+    4688,
+    3065,
+    488,
+    5125,
+    369,
+    279,
+    1208,
+    13,
+    2302,
+    513,
+    524,
+    1532,
+    13,
+    2302,
+    513,
+    16360,
+    13,
+    733,
+    279,
+    53627,
+    287,
+    11,
+    864,
+    513,
+    836,
+    5515,
+    440,
+    279,
+    14367,
+    13,
+    733,
+    279,
+    22002,
+    757,
+    11,
+    864,
+    513,
+    836,
+    28425,
+    13,
+    733,
+    279,
+    22263,
+    4837,
+    69880,
+    82,
+    11,
+    864,
+    513,
+    836,
+    30717,
+    13,
+    733,
+    279,
+    23178,
+    17629,
+    43082,
+    11,
+    864,
+    513,
+    836,
+    39604,
+    13,
+    271,
+    4277,
+    271,
+    760,
+    53627,
+    68,
+    369,
+    279,
+    1379,
+    2894,
+    5759,
+    303,
+    279,
+    17198,
+    20139,
+    11,
+    321,
+    424,
+    369,
+    524,
+    264,
+    5759,
+    506,
+    660,
+    13,
+    1049,
+    369,
+    264,
+    10896,
+    13,
+    357,
+    10896,
+    1801,
+    539,
+    279,
+    29158,
+    314,
+    47070,
+    303,
+    220,
+    16,
+    24,
+    17,
+    23,
+    11,
+    1238,
+    279,
+    8252,
+    54242,
+    11,
+    421,
+    279,
+    14367,
+    1000,
+    381,
+    8341,
+    303,
+    1141,
+    5323,
+    13,
+    561,
+    10896,
+    557,
+    5158,
+    303,
+    13769,
+    321,
+    9114,
+    11,
+    321,
+    424,
+    682,
+    978,
+    8341,
+    11,
+    9712,
+    11,
+    364,
+    75211,
+    75351,
+    1578,
+    13,
+    271,
+    760,
+    53627,
+    68,
+    369,
+    1048,
+    264,
+    9940,
+    13,
+    8938,
+    279,
+    14367,
+    369,
+    524,
+    264,
+    3065,
+    421,
+    628,
+    381,
+    8341,
+    13,
+    1049,
+    369,
+    264,
+    5176,
+    11,
+    264,
+    13727,
+    11
+   ]
+  },
+  {
+   "prompt_tokens": 42,
+   "out_tokens": 400,
+   "seconds": 4.171,
+   "tok_s": 95.9,
+   "output_token_ids": [
+    271,
+    8160,
+    369,
+    449,
+    15673,
+    314,
+    1439,
+    2250,
+    41724,
+    303,
+    13983,
+    3992,
+    11,
+    3055,
+    440,
+    264,
+    13769,
+    3010,
+    364,
+    1754,
+    13,
+    271,
+    13962,
+    220,
+    16,
+    13,
+    8169,
+    264,
+    19269,
+    7541,
+    2751,
+    303,
+    18266,
+    271,
+    332,
+    760,
+    68868,
+    64700,
+    198,
+    33807,
+    303,
+    279,
+    1814,
+    369,
+    1801,
+    314,
+    13280,
+    18167,
+    318,
+    63339,
+    321,
+    33470,
+    8,
+    421,
+    513,
+    14537,
+    6992,
+    13,
+    3095,
+    279,
+    9039,
+    35937,
+    11,
+    1439,
+    18167,
+    8457,
+    4649,
+    321,
+    3166,
+    10281,
+    11,
+    13969,
+    1070,
+    310,
+    4336,
+    9781,
+    4473,
+    10431,
+    494,
+    799,
+    2361,
+    13,
+    5788,
+    411,
+    2222,
+    369,
+    85928,
+    303,
+    264,
+    3074,
+    18546,
+    11,
+    948,
+    488,
+    29283,
+    424,
+    539,
+    279,
+    30995,
+    314,
+    31137,
+    303,
+    264,
+    10632,
+    5759,
+    1040,
+    264,
+    9229,
+    13759,
+    11,
+    279,
+    2702,
+    14046,
+    8772,
+    4927,
+    13,
+    1368,
+    264,
+    13759,
+    998,
+    5617,
+    430,
+    799,
+    6235,
+    11,
+    632,
+    47272,
+    6365,
+    11,
+    411,
+    14046,
+    303,
+    279,
+    8364,
+    1000,
+    5073,
+    279,
+    9041,
+    310,
+    77238,
+    466,
+    16966,
+    1521,
+    424,
+    682,
+    26241,
+    310,
+    708,
+    13,
+    271,
+    332,
+    80711,
+    12996,
+    64700,
+    198,
+    37405,
+    314,
+    264,
+    2972,
+    52195,
+    46379,
+    2047,
+    303,
+    264,
+    3882,
+    1740,
+    159034,
+    1368,
+    488,
+    6420,
+    279,
+    46379,
+    440,
+    264,
+    1564,
+    12489,
+    303,
+    279,
+    12050,
+    321,
+    1179,
+    1495,
+    1238,
+    424,
+    682,
+    978,
+    11362,
+    303,
+    279,
+    6797,
+    364,
+    449,
+    6258,
+    11,
+    279,
+    46379,
+    668,
+    381,
+    21219,
+    4860,
+    303,
+    279,
+    7094,
+    13,
+    357,
+    13759,
+    22839,
+    9331,
+    364,
+    411,
+    539,
+    25780,
+    328,
+    4431,
+    10188,
+    33124,
+    62804,
+    70,
+    2538,
+    9896,
+    440,
+    21970,
+    466,
+    9041,
+    23702,
+    2218,
+    40486,
+    13753,
+    314,
+    279,
+    13759,
+    13,
+    733,
+    7094,
+    11,
+    279,
+    13759,
+    50110,
+    1083,
+    1439,
+    31092,
+    11,
+    10377,
+    279,
+    5759,
+    310,
+    14084,
+    1973,
+    14293,
+    13,
+    271,
+    13962,
+    220,
+    17,
+    13,
+    8169,
+    264,
+    710,
+    46038,
+    326,
+    793,
+    12851,
+    1049,
+    2436,
+    8405,
+    271,
+    332,
+    760,
+    68868,
+    64700,
+    198,
+    760,
+    328,
+    82,
+    23279,
+    1,
+    466,
+    412,
+    375,
+    2631,
+    4950,
+    369,
+    3404,
+    264,
+    1067,
+    314,
+    2972,
+    408,
+    263,
+    663,
+    332,
+    321,
+    2972,
+    85,
+    17804,
+    332,
+    8613,
+    539,
+    19695,
+    13,
+    1577,
+    2919,
+    73944,
+    685,
+    11,
+    424,
+    3070,
+    914,
+    42244,
+    21146,
+    16360,
+    506,
+    2957,
+    13,
+    21153,
+    42237,
+    1304,
+    506,
+    279,
+    5445,
+    314,
+    279,
+    69862,
+    1332,
+    424,
+    369,
+    36056,
+    13,
+    1577,
+    1439,
+    42237,
+    9704,
+    1472,
+    279,
+    34635,
+    2919,
+    3294,
+    1070,
+    11,
+    781,
+    3428,
+    17638,
+    318,
+    53710,
+    8,
+    1518,
+    18176,
+    279,
+    7099,
+    13,
+    1061,
+    10725,
+    17638,
+    11121,
+    13280,
+    10439,
+    55348,
+    321,
+    74610,
+    13,
+    271,
+    623,
+    264,
+    412,
+    375,
+    2631,
+    69862,
+    11,
+    279,
+    2790,
+    11407,
+    411,
+    19695,
+    1472,
+    264,
+    2526,
+    11,
+    3050,
+    8306,
+    318,
+    1719,
+    38240,
+    553,
+    561,
+    19695,
+    8177,
+    3594,
+    310,
+    343,
+    41670,
+    506,
+    264,
+    3050,
+    11304,
+    11,
+    1691,
+    1040,
+    39005,
+    3808,
+    279
+   ]
+  },
+  {
+   "prompt_tokens": 47,
+   "out_tokens": 400,
+   "seconds": 4.163,
+   "tok_s": 96.1,
+   "output_token_ids": [
+    9896,
+    2250,
+    75842,
+    440,
+    11140,
+    421,
+    998,
+    40361,
+    42749,
+    13,
+    3113,
+    35442,
+    1292,
+    995,
+    5158,
+    264,
+    11183,
+    883,
+    264,
+    5054,
+    852,
+    1362,
+    6512,
+    279,
+    9117,
+    579,
+    295,
+    8792,
+    1472,
+    279,
+    6220,
+    18687,
+    11,
+    321,
+    1292,
+    4035,
+    2574,
+    424,
+    310,
+    1024,
+    6764,
+    11,
+    852,
+    1301,
+    424,
+    303,
+    799,
+    11362,
+    321,
+    1179,
+    1018,
+    11,
+    328,
+    2523,
+    2908,
+    1801,
+    279,
+    9117,
+    4950,
+    1040,
+    264,
+    27809,
+    36252,
+    1149,
+    2838,
+    3738,
+    411,
+    430,
+    279,
+    8166,
+    27463,
+    13,
+    271,
+    760,
+    11183,
+    557,
+    2496,
+    4501,
+    13,
+    1049,
+    7338,
+    303,
+    264,
+    25635,
+    364,
+    42744,
+    1578,
+    11,
+    22175,
+    279,
+    3820,
+    20513,
+    314,
+    2433,
+    421,
+    513,
+    9939,
+    694,
+    524,
+    3482,
+    5354,
+    13,
+    3095,
+    1292,
+    5326,
+    3106,
+    424,
+    310,
+    264,
+    20609,
+    11,
+    781,
+    5743,
+    424,
+    440,
+    264,
+    5020,
+    421,
+    557,
+    4409,
+    2993,
+    25,
+    328,
+    32,
+    6037,
+    2351,
+    364,
+    16078,
+    13,
+    1165,
+    2224,
+    524,
+    2617,
+    1092,
+    279,
+    3255,
+    369,
+    1149,
+    2838,
+    15513,
+    13,
+    561,
+    9117,
+    579,
+    295,
+    8792,
+    998,
+    279,
+    3255,
+    13,
+    1921,
+    279,
+    9117,
+    579,
+    295,
+    8792,
+    11,
+    424,
+    6316,
+    680,
+    11,
+    998,
+    524,
+    264,
+    3255,
+    303,
+    279,
+    1560,
+    421,
+    34440,
+    15513,
+    7112,
+    13,
+    271,
+    7691,
+    557,
+    16570,
+    52616,
+    948,
+    1292,
+    6426,
+    310,
+    3165,
+    264,
+    2086,
+    2993,
+    314,
+    2236,
+    13,
+    2717,
+    264,
+    11183,
+    13,
+    2717,
+    264,
+    1103,
+    13,
+    23879,
+    303,
+    1881,
+    13,
+    2838,
+    2512,
+    424,
+    264,
+    328,
+    23674,
+    278,
+    3150,
+    62804,
+    64,
+    2236,
+    421,
+    1000,
+    1732,
+    279,
+    35091,
+    314,
+    279,
+    259,
+    3232,
+    888,
+    264,
+    3074,
+    1007,
+    11,
+    1754,
+    12096,
+    11811,
+    310,
+    264,
+    3050,
+    82507,
+    5215,
+    11,
+    1754,
+    799,
+    8222,
+    264,
+    2086,
+    1304,
+    25,
+    264,
+    6321,
+    11,
+    264,
+    10793,
+    11,
+    264,
+    378,
+    355,
+    15202,
+    11,
+    264,
+    716,
+    314,
+    10895,
+    364,
+    295,
+    2379,
+    264,
+    4041,
+    11,
+    264,
+    10125,
+    888,
+    52673,
+    303,
+    264,
+    72103,
+    50203,
+    13,
+    561,
+    7346,
+    1000,
+    381,
+    8173,
+    539,
+    264,
+    3074,
+    4368,
+    25,
+    279,
+    6103,
+    11,
+    8455,
+    944,
+    314,
+    264,
+    5054,
+    6618,
+    310,
+    41957,
+    1024,
+    6764,
+    364,
+    9101,
+    13,
+    271,
+    760,
+    2236,
+    3738,
+    2943,
+    1578,
+    310,
+    3165,
+    13,
+    2838,
+    11815,
+    303,
+    264,
+    2526,
+    3626,
+    383,
+    279,
+    13255,
+    11,
+    321,
+    1292,
+    8341,
+    264,
+    1433,
+    314,
+    279,
+    259,
+    3232,
+    11,
+    279,
+    8831,
+    11,
+    279,
+    1528,
+    314,
+    279,
+    9117,
+    11,
+    279,
+    1528,
+    314,
+    1024,
+    1765,
+    4593,
+    13,
+    2838,
+    5947,
+    303,
+    279,
+    55287,
+    11,
+    948,
+    279,
+    3000,
+    557,
+    4101,
+    321,
+    279,
+    9117,
+    557,
+    1990,
+    28177,
+    1092,
+    424,
+    4674,
+    310,
+    381,
+    13,
+    2838,
+    1301,
+    303,
+    279,
+    1238,
+    2083,
+    2305,
+    2218,
+    787,
+    48345,
+    53340,
+    11,
+    11592,
+    494,
+    279,
+    15917,
+    4843,
+    11,
+    264,
+    2236,
+    314,
+    95716,
+    28741,
+    299,
+    421,
+    1292,
+    1301,
+    364,
+    279,
+    1560,
+    781,
+    17083,
+    36049,
+    11
+   ]
+  }
+ ],
+ "metrics": {},
+ "mean_tok_s": 95.9
+}
\ No newline at end of file
diff --git a/docs/qwen38-vllm-perf-plain-tray14.json b/docs/qwen38-vllm-perf-plain-tray14.json
new file mode 100644
index 0000000..d44eec1
--- /dev/null
+++ b/docs/qwen38-vllm-perf-plain-tray14.json
@@ -0,0 +1,2048 @@
+{
+ "mode": "plain",
+ "pinned": false,
+ "results": [
+  {
+   "prompt_tokens": 43,
+   "out_tokens": 400,
+   "seconds": 4.218,
+   "tok_s": 94.8,
+   "output_token_ids": [
+    264,
+    3074,
+    4202,
+    421,
+    1467,
+    524,
+    2353,
+    866,
+    314,
+    279,
+    3672,
+    25,
+    264,
+    25347,
+    2512,
+    279,
+    348,
+    32,
+    322,
+    6020,
+    9,
+    995,
+    978,
+    12076,
+    430,
+    1001,
+    52513,
+    506,
+    9117,
+    828,
+    383,
+    279,
+    3603,
+    314,
+    279,
+    3128,
+    70595,
+    11,
+    3482,
+    279,
+    1788,
+    3603,
+    264,
+    7397,
+    4600,
+    725,
+    1433,
+    7246,
+    264,
+    9613,
+    11,
+    14527,
+    54087,
+    71433,
+    7397,
+    501,
+    13810,
+    494,
+    279,
+    2919,
+    321,
+    264,
+    14457,
+    11,
+    83345,
+    3000,
+    3294,
+    279,
+    18137,
+    13,
+    561,
+    7110,
+    725,
+    5572,
+    87338,
+    888,
+    279,
+    2081,
+    11,
+    279,
+    25570,
+    1990,
+    13988,
+    11,
+    430,
+    413,
+    279,
+    1330,
+    7189,
+    998,
+    4313,
+    310,
+    6266,
+    310,
+    1754,
+    975,
+    13,
+    271,
+    4639,
+    3603,
+    11,
+    279,
+    7110,
+    1362,
+    524,
+    5894,
+    13,
+    1216,
+    33461,
+    279,
+    348,
+    32,
+    322,
+    6020,
+    9,
+    430,
+    264,
+    3255,
+    421,
+    995,
+    41065,
+    680,
+    314,
+    1141,
+    1765,
+    18641,
+    11,
+    264,
+    22557,
+    421,
+    995,
+    978,
+    3125,
+    303,
+    799,
+    3992,
+    321,
+    1179,
+    2044,
+    787,
+    303,
+    2361,
+    11,
+    1754,
+    2243,
+    45328,
+    279,
+    5893,
+    314,
+    279,
+    3889,
+    1345,
+    9722,
+    279,
+    4105,
+    13,
+    561,
+    7397,
+    4600,
+    725,
+    2605,
+    557,
+    279,
+    1001,
+    2955,
+    828,
+    314,
+    279,
+    8026,
+    725,
+    23611,
+    25,
+    279,
+    4557,
+    314,
+    279,
+    25347,
+    5924,
+    279,
+    4557,
+    314,
+    264,
+    7397,
+    11,
+    279,
+    25457,
+    314,
+    279,
+    9117,
+    5924,
+    279,
+    25457,
+    314,
+    279,
+    3603,
+    12515,
+    11,
+    321,
+    279,
+    13234,
+    725,
+    20513,
+    5924,
+    279,
+    20513,
+    314,
+    279,
+    2919,
+    13,
+    14205,
+    279,
+    1330,
+    9331,
+    998,
+    524,
+    18922,
+    26,
+    279,
+    7397,
+    4600,
+    3568,
+    264,
+    7472,
+    27090,
+    83345,
+    3000,
+    38697,
+    279,
+    54517,
+    7110,
+    725,
+    45366,
+    2496,
+    9444,
+    13,
+    561,
+    7110,
+    14607,
+    421,
+    264,
+    14053,
+    628,
+    2496,
+    381,
+    264,
+    4574,
+    2880,
+    26,
+    424,
+    369,
+    2590,
+    264,
+    491,
+    3255,
+    421,
+    22651,
+    279,
+    4779,
+    314,
+    279,
+    1118,
+    13,
+    271,
+    1465,
+    3272,
+    314,
+    279,
+    1560,
+    264,
+    20336,
+    1068,
+    13514,
+    264,
+    10741,
+    13,
+    561,
+    10741,
+    369,
+    264,
+    8240,
+    314,
+    17210,
+    11,
+    1754,
+    2923,
+    264,
+    13280,
+    3255,
+    883,
+    4947,
+    13,
+    3095,
+    279,
+    10741,
+    369,
+    1301,
+    68030,
+    11,
+    279,
+    17210,
+    3512,
+    10054,
+    11,
+    321,
+    279,
+    10054,
+    3512,
+    5167,
+    303,
+    279,
+    11117,
+    725,
+    3838,
+    13,
+    561,
+    3889,
+    10741,
+    11,
+    279,
+    20699,
+    2243,
+    11,
+    321,
+    279,
+    10189,
+    6588,
+    513,
+    2250,
+    2086,
+    1001,
+    41409,
+    828,
+    21988,
+    279,
+    1788,
+    7793,
+    13,
+    1368,
+    799,
+    314,
+    1070,
+    369,
+    27811,
+    27090,
+    1777,
+    44543,
+    11,
+    264,
+    3299,
+    5605,
+    68723,
+    18863,
+    11,
+    264,
+    6588,
+    33461,
+    41915,
+    21010,
+    7793,
+    628,
+    381,
+    5384,
+    11,
+    1442,
+    3384,
+    279,
+    975,
+    1330,
+    6922,
+    33295,
+    13,
+    561,
+    7110,
+    5312,
+    421,
+    279,
+    348,
+    32,
+    322,
+    6020,
+    9,
+    515,
+    82,
+    45366,
+    4202,
+    321,
+    279,
+    7397,
+    4600,
+    725,
+    1433,
+    998,
+    1040,
+    1330,
+    74519,
+    314,
+    279,
+    1788,
+    55877,
+    25
+   ]
+  },
+  {
+   "prompt_tokens": 41,
+   "out_tokens": 400,
+   "seconds": 4.198,
+   "tok_s": 95.3,
+   "output_token_ids": [
+    264,
+    36334,
+    21388,
+    314,
+    5484,
+    5415,
+    321,
+    264,
+    716,
+    314,
+    34844,
+    542,
+    5564,
+    58091,
+    1172,
+    279,
+    7991,
+    846,
+    13,
+    561,
+    1118,
+    3603,
+    1172,
+    279,
+    491,
+    12515,
+    11,
+    279,
+    6561,
+    2002,
+    6749,
+    92169,
+    1056,
+    5316,
+    25624,
+    11,
+    3482,
+    279,
+    36334,
+    467,
+    78078,
+    5432,
+    68,
+    1990,
+    9047,
+    310,
+    288,
+    92536,
+    13,
+    561,
+    2015,
+    14607,
+    421,
+    279,
+    53952,
+    725,
+    27374,
+    5257,
+    557,
+    874,
+    4860,
+    264,
+    4145,
+    1001,
+    75,
+    704,
+    54087,
+    429,
+    54087,
+    70735,
+    828,
+    3255,
+    26,
+    424,
+    557,
+    264,
+    61010,
+    23850,
+    314,
+    25586,
+    11,
+    1754,
+    440,
+    1141,
+    1765,
+    79299,
+    11,
+    25113,
+    11,
+    321,
+    3558,
+    13,
+    271,
+    4639,
+    3603,
+    5924,
+    264,
+    1105,
+    3788,
+    303,
+    1204,
+    264,
+    7072,
+    54087,
+    5911,
+    53952,
+    628,
+    381,
+    7246,
+    11,
+    16384,
+    11,
+    321,
+    13260,
+    34773,
+    13,
+    20543,
+    369,
+    279,
+    2923,
+    54087,
+    1634,
+    54087,
+    9239,
+    41223,
+    314,
+    279,
+    3377,
+    11,
+    279,
+    795,
+    279,
+    13234,
+    19577,
+    11,
+    321,
+    279,
+    4602,
+    781,
+    4221,
+    310,
+    4087,
+    13,
+    271,
+    4277,
+    271,
+    13962,
+    220,
+    16,
+    13,
+    561,
+    27374,
+    6576,
+    271,
+    760,
+    13736,
+    369,
+    264,
+    2972,
+    32897,
+    54087,
+    38429,
+    18766,
+    5444,
+    332,
+    318,
+    19186,
+    33800,
+    11,
+    1754,
+    440,
+    264,
+    3929,
+    321,
+    264,
+    1142,
+    7099,
+    8,
+    7854,
+    539,
+    264,
+    2972,
+    18452,
+    17312,
+    332,
+    421,
+    58469,
+    279,
+    22690,
+    1142,
+    1472,
+    279,
+    1788,
+    33800,
+    13,
+    561,
+    9468,
+    4868,
+    513,
+    25,
+    271,
+    91,
+    26099,
+    735,
+    39115,
+    314,
+    79299,
+    318,
+    3693,
+    8,
+    735,
+    9926,
+    318,
+    77,
+    8,
+    735,
+    198,
+    91,
+    22031,
+    91,
+    746,
+    14894,
+    91,
+    40698,
+    91,
+    198,
+    91,
+    220,
+    16,
+    318,
+    6735,
+    314,
+    1118,
+    17796,
+    8,
+    735,
+    478,
+    16,
+    20,
+    15,
+    735,
+    220,
+    16,
+    13,
+    20,
+    16,
+    21,
+    23,
+    735,
+    198,
+    91,
+    220,
+    17,
+    318,
+    1371,
+    314,
+    1118,
+    17796,
+    8,
+    220,
+    735,
+    1317,
+    17,
+    15,
+    15,
+    735,
+    220,
+    16,
+    13,
+    20,
+    16,
+    21,
+    23,
+    735,
+    198,
+    91,
+    220,
+    18,
+    318,
+    6735,
+    314,
+    2018,
+    17796,
+    8,
+    735,
+    478,
+    16,
+    17,
+    15,
+    735,
+    220,
+    16,
+    13,
+    21,
+    17,
+    15,
+    15,
+    735,
+    198,
+    91,
+    220,
+    19,
+    318,
+    1371,
+    314,
+    2018,
+    17796,
+    8,
+    220,
+    735,
+    1317,
+    16,
+    23,
+    15,
+    735,
+    220,
+    16,
+    13,
+    21,
+    17,
+    15,
+    15,
+    735,
+    198,
+    91,
+    220,
+    20,
+    318,
+    18452,
+    17312,
+    8,
+    260,
+    735,
+    11654,
+    252,
+    318,
+    25393,
+    8,
+    735,
+    1317,
+    735,
+    271,
+    760,
+    2972,
+    686,
+    518,
+    4681,
+    778,
+    332,
+    318,
+    2959,
+    54087,
+    951,
+    54087,
+    2959,
+    25704,
+    8,
+    513,
+    25,
+    271,
+    12,
+    26260,
+    25586,
+    220,
+    16,
+    321,
+    220,
+    17,
+    25,
+    2972,
+    16,
+    15,
+    9186,
+    332,
+    198,
+    12,
+    26260,
+    25586,
+    220,
+    17,
+    321,
+    220,
+    18,
+    25,
+    2972,
+    17,
+    20,
+    9186,
+    332,
+    198,
+    12,
+    26260,
+    25586,
+    220,
+    18,
+    321,
+    220,
+    19,
+    25,
+    2972
+   ]
+  },
+  {
+   "prompt_tokens": 48,
+   "out_tokens": 400,
+   "seconds": 4.205,
+   "tok_s": 95.1,
+   "output_token_ids": [
+    7094,
+    781,
+    14428,
+    279,
+    53627,
+    287,
+    440,
+    22945,
+    9399,
+    49954,
+    11,
+    8132,
+    1396,
+    2418,
+    314,
+    272,
+    46016,
+    11,
+    1396,
+    10582,
+    314,
+    274,
+    6960,
+    11,
+    1396,
+    22263,
+    23178,
+    17629,
+    22149,
+    13,
+    733,
+    279,
+    4688,
+    7094,
+    781,
+    5743,
+    310,
+    279,
+    1788,
+    22945,
+    52604,
+    321,
+    1669,
+    279,
+    14367,
+    995,
+    7481,
+    13,
+    271,
+    760,
+    42159,
+    557,
+    2496,
+    7807,
+    13,
+    561,
+    14367,
+    8341,
+    7594,
+    795,
+    13,
+    271,
+    4277,
+    271,
+    760,
+    1118,
+    3065,
+    488,
+    5125,
+    883,
+    279,
+    17198,
+    20139,
+    369,
+    421,
+    424,
+    369,
+    524,
+    264,
+    1925,
+    13,
+    1049,
+    369,
+    264,
+    1817,
+    11884,
+    279,
+    30385,
+    314,
+    264,
+    1925,
+    13,
+    561,
+    4128,
+    488,
+    2418,
+    383,
+    557,
+    14367,
+    2634,
+    264,
+    12828,
+    3998,
+    321,
+    668,
+    381,
+    14367,
+    2634,
+    1495,
+    303,
+    264,
+    12828,
+    13,
+    561,
+    12052,
+    513,
+    524,
+    39527,
+    303,
+    16665,
+    748,
+    1691,
+    430,
+    303,
+    264,
+    46446,
+    440,
+    2919,
+    13,
+    561,
+    272,
+    46016,
+    29528,
+    513,
+    524,
+    19101,
+    13,
+    2302,
+    513,
+    279,
+    4757,
+    579,
+    1560,
+    314,
+    25108,
+    948,
+    279,
+    2919,
+    1898,
+    35937,
+    3294,
+    1141,
+    7303,
+    13,
+    271,
+    760,
+    2018,
+    3065,
+    488,
+    5125,
+    369,
+    279,
+    20513,
+    13,
+    2717,
+    279,
+    18682,
+    314,
+    4950,
+    13,
+    561,
+    9084,
+    314,
+    264,
+    2086,
+    2993,
+    314,
+    4950,
+    13,
+    561,
+    3594,
+    369,
+    11702,
+    440,
+    279,
+    11639,
+    314,
+    2919,
+    6992,
+    1472,
+    51930,
+    11,
+    314,
+    39465,
+    3133,
+    279,
+    25348,
+    11,
+    314,
+    19053,
+    7842,
+    303,
+    32655,
+    421,
+    2714,
+    5995,
+    310,
+    5640,
+    1472,
+    13988,
+    3594,
+    13,
+    561,
+    20513,
+    369,
+    279,
+    4950,
+    314,
+    264,
+    1785,
+    303,
+    53454,
+    13,
+    271,
+    760,
+    4688,
+    3065,
+    488,
+    5125,
+    369,
+    279,
+    21388,
+    13,
+    326,
+    13898,
+    322,
+    13,
+    93048,
+    3374,
+    13,
+    561,
+    3820,
+    61306,
+    314,
+    2919,
+    421,
+    682,
+    978,
+    1472,
+    279,
+    16665,
+    321,
+    2445,
+    680,
+    279,
+    975,
+    3008,
+    14885,
+    4156,
+    279,
+    16665,
+    1042,
+    708,
+    13,
+    561,
+    21388,
+    369,
+    524,
+    45322,
+    13,
+    1049,
+    369,
+    279,
+    21388,
+    314,
+    264,
+    1925,
+    421,
+    369,
+    3604,
+    1141,
+    944,
+    13,
+    271,
+    4277,
+    271,
+    760,
+    42159,
+    40670,
+    6237,
+    303,
+    7036,
+    314,
+    2943,
+    13,
+    8771,
+    383,
+    279,
+    2919,
+    11,
+    1330,
+    383,
+    279,
+    4128,
+    13,
+    561,
+    2919,
+    2015,
+    11354,
+    264,
+    80611,
+    11,
+    264,
+    22619,
+    4821,
+    11,
+    264,
+    44497,
+    11,
+    321,
+    264,
+    28868,
+    428,
+    314,
+    10482,
+    421,
+    3804,
+    8981,
+    539,
+    279,
+    2018,
+    6258,
+    13,
+    561,
+    4128,
+    2015,
+    11354,
+    264,
+    22945,
+    9399,
+    16473,
+    11,
+    264,
+    88821,
+    11,
+    264,
+    16665,
+    21059,
+    11,
+    321,
+    264,
+    35935,
+    421,
+    9896,
+    685,
+    539,
+    279,
+    4688,
+    1834,
+    13,
+    271,
+    760,
+    2919,
+    2015,
+    579,
+    2531,
+    557,
+    310,
+    2336,
+    279,
+    11407,
+    13,
+    2717,
+    279,
+    1822,
+    5323,
+    13,
+    561,
+    1822,
+    5323,
+    557,
+    2582,
+    23098,
+    13,
+    561,
+    1142,
+    11407,
+    11,
+    279,
+    296,
+    7679,
+    4102,
+    11,
+    279,
+    1678,
+    1336,
+    82,
+    11,
+    279,
+    7247
+   ]
+  },
+  {
+   "prompt_tokens": 42,
+   "out_tokens": 400,
+   "seconds": 4.2,
+   "tok_s": 95.2,
+   "output_token_ids": [
+    271,
+    8160,
+    369,
+    449,
+    15673,
+    314,
+    1439,
+    2250,
+    41724,
+    303,
+    13983,
+    3992,
+    11,
+    3055,
+    440,
+    264,
+    13769,
+    3010,
+    364,
+    1754,
+    13,
+    271,
+    13962,
+    220,
+    16,
+    13,
+    8169,
+    264,
+    19269,
+    7541,
+    2751,
+    303,
+    18266,
+    271,
+    332,
+    760,
+    68868,
+    64700,
+    198,
+    33807,
+    303,
+    279,
+    1814,
+    369,
+    1801,
+    314,
+    13280,
+    18167,
+    318,
+    63339,
+    321,
+    33470,
+    8,
+    421,
+    513,
+    14537,
+    6992,
+    13,
+    3095,
+    279,
+    9039,
+    35937,
+    11,
+    1439,
+    18167,
+    8457,
+    4649,
+    321,
+    3166,
+    10281,
+    11,
+    13969,
+    1070,
+    310,
+    4336,
+    9781,
+    4473,
+    10431,
+    494,
+    799,
+    2361,
+    13,
+    5788,
+    411,
+    2222,
+    369,
+    85928,
+    303,
+    264,
+    3074,
+    18546,
+    11,
+    948,
+    488,
+    29283,
+    424,
+    539,
+    279,
+    30995,
+    314,
+    31137,
+    303,
+    264,
+    10632,
+    5759,
+    1040,
+    264,
+    9229,
+    13759,
+    11,
+    279,
+    2702,
+    14046,
+    8772,
+    4927,
+    13,
+    1368,
+    264,
+    13759,
+    998,
+    5617,
+    430,
+    799,
+    6235,
+    11,
+    632,
+    47272,
+    6365,
+    11,
+    411,
+    14046,
+    303,
+    279,
+    8364,
+    1000,
+    5073,
+    279,
+    9041,
+    310,
+    77238,
+    466,
+    16966,
+    1521,
+    424,
+    682,
+    26241,
+    310,
+    708,
+    13,
+    271,
+    332,
+    80711,
+    12996,
+    64700,
+    198,
+    37405,
+    314,
+    264,
+    2972,
+    52195,
+    46379,
+    2047,
+    303,
+    264,
+    3882,
+    1740,
+    159034,
+    1368,
+    488,
+    6420,
+    279,
+    46379,
+    440,
+    264,
+    1564,
+    12489,
+    303,
+    279,
+    12050,
+    321,
+    1179,
+    1495,
+    1238,
+    424,
+    682,
+    978,
+    11362,
+    303,
+    279,
+    6797,
+    364,
+    449,
+    6258,
+    11,
+    279,
+    46379,
+    668,
+    381,
+    21219,
+    4860,
+    303,
+    279,
+    7094,
+    13,
+    357,
+    13759,
+    22839,
+    9331,
+    364,
+    411,
+    539,
+    25780,
+    328,
+    4431,
+    10188,
+    33124,
+    62804,
+    70,
+    2538,
+    9896,
+    440,
+    21970,
+    466,
+    9041,
+    23702,
+    2218,
+    40486,
+    13753,
+    314,
+    279,
+    13759,
+    13,
+    733,
+    7094,
+    11,
+    279,
+    13759,
+    50110,
+    1083,
+    1439,
+    31092,
+    11,
+    10377,
+    279,
+    5759,
+    310,
+    14084,
+    1973,
+    14293,
+    13,
+    271,
+    13962,
+    220,
+    17,
+    13,
+    8169,
+    264,
+    710,
+    46038,
+    326,
+    793,
+    12851,
+    1049,
+    2436,
+    8405,
+    271,
+    332,
+    760,
+    68868,
+    64700,
+    198,
+    760,
+    328,
+    82,
+    23279,
+    1,
+    466,
+    412,
+    375,
+    2631,
+    4950,
+    369,
+    3404,
+    264,
+    1067,
+    314,
+    2972,
+    408,
+    263,
+    663,
+    332,
+    321,
+    2972,
+    85,
+    17804,
+    332,
+    8613,
+    539,
+    19695,
+    13,
+    1577,
+    2919,
+    73944,
+    685,
+    11,
+    424,
+    3070,
+    914,
+    42244,
+    21146,
+    16360,
+    506,
+    2957,
+    13,
+    21153,
+    42237,
+    1304,
+    506,
+    279,
+    5445,
+    314,
+    279,
+    69862,
+    1332,
+    424,
+    369,
+    36056,
+    13,
+    1577,
+    1439,
+    42237,
+    9704,
+    1472,
+    279,
+    34635,
+    2919,
+    3294,
+    1070,
+    11,
+    781,
+    3428,
+    17638,
+    318,
+    53710,
+    8,
+    1518,
+    18176,
+    279,
+    7099,
+    13,
+    1061,
+    10725,
+    17638,
+    11121,
+    13280,
+    10439,
+    55348,
+    321,
+    74610,
+    13,
+    271,
+    623,
+    264,
+    412,
+    375,
+    2631,
+    69862,
+    11,
+    279,
+    2790,
+    11407,
+    411,
+    19695,
+    1472,
+    264,
+    2526,
+    11,
+    3050,
+    8306,
+    318,
+    1719,
+    38240,
+    553,
+    561,
+    19695,
+    8177,
+    3594,
+    310,
+    343,
+    41670,
+    506,
+    264,
+    3050,
+    11304,
+    11,
+    1691,
+    1040,
+    39005,
+    3808,
+    279
+   ]
+  },
+  {
+   "prompt_tokens": 47,
+   "out_tokens": 400,
+   "seconds": 4.198,
+   "tok_s": 95.3,
+   "output_token_ids": [
+    9896,
+    2250,
+    75842,
+    440,
+    11140,
+    421,
+    998,
+    40361,
+    42749,
+    13,
+    3113,
+    35442,
+    1292,
+    995,
+    5158,
+    264,
+    11183,
+    883,
+    264,
+    5054,
+    852,
+    1362,
+    6512,
+    279,
+    9117,
+    579,
+    295,
+    8792,
+    1472,
+    279,
+    6220,
+    18687,
+    11,
+    321,
+    1292,
+    4035,
+    2574,
+    424,
+    310,
+    1024,
+    6764,
+    11,
+    852,
+    1301,
+    424,
+    303,
+    799,
+    11362,
+    321,
+    1179,
+    1018,
+    11,
+    328,
+    2523,
+    2908,
+    1801,
+    279,
+    9117,
+    4950,
+    1040,
+    264,
+    27809,
+    36252,
+    1149,
+    2838,
+    3738,
+    411,
+    430,
+    279,
+    8166,
+    27463,
+    13,
+    271,
+    760,
+    11183,
+    557,
+    2496,
+    4501,
+    13,
+    1049,
+    7338,
+    303,
+    264,
+    25635,
+    364,
+    42744,
+    1578,
+    11,
+    22175,
+    279,
+    3820,
+    20513,
+    314,
+    2433,
+    421,
+    513,
+    9939,
+    694,
+    524,
+    3482,
+    5354,
+    13,
+    3095,
+    1292,
+    5326,
+    3106,
+    424,
+    310,
+    264,
+    20609,
+    11,
+    781,
+    5743,
+    424,
+    440,
+    264,
+    5020,
+    421,
+    557,
+    4409,
+    2993,
+    25,
+    328,
+    32,
+    6037,
+    2351,
+    364,
+    16078,
+    13,
+    1165,
+    2224,
+    524,
+    2617,
+    1092,
+    279,
+    3255,
+    369,
+    1149,
+    2838,
+    15513,
+    13,
+    561,
+    9117,
+    579,
+    295,
+    8792,
+    998,
+    279,
+    3255,
+    13,
+    1921,
+    279,
+    9117,
+    579,
+    295,
+    8792,
+    11,
+    424,
+    6316,
+    680,
+    11,
+    998,
+    524,
+    264,
+    3255,
+    303,
+    279,
+    1560,
+    421,
+    34440,
+    15513,
+    7112,
+    13,
+    271,
+    7691,
+    557,
+    16570,
+    52616,
+    948,
+    1292,
+    6426,
+    310,
+    3165,
+    264,
+    2086,
+    2993,
+    314,
+    2236,
+    13,
+    2717,
+    264,
+    11183,
+    13,
+    2717,
+    264,
+    1103,
+    13,
+    23879,
+    303,
+    1881,
+    13,
+    357,
+    2236,
+    421,
+    1000,
+    381,
+    264,
+    3626,
+    440,
+    1599,
+    11683,
+    11,
+    1332,
+    1754,
+    2956,
+    5467,
+    264,
+    2086,
+    2993,
+    314,
+    7793,
+    11,
+    321,
+    279,
+    6396,
+    1362,
+    37922,
+    1472,
+    1070,
+    303,
+    866,
+    1906,
+    13,
+    2838,
+    2512,
+    424,
+    348,
+    760,
+    78812,
+    579,
+    211358,
+    565,
+    11961,
+    321,
+    424,
+    3738,
+    1024,
+    2943,
+    1578,
+    310,
+    3165,
+    13,
+    271,
+    760,
+    444,
+    1461,
+    565,
+    557,
+    17249,
+    1083,
+    28279,
+    13753,
+    11,
+    799,
+    364,
+    1754,
+    2182,
+    11,
+    694,
+    279,
+    3818,
+    998,
+    524,
+    279,
+    3818,
+    314,
+    279,
+    12786,
+    13,
+    5869,
+    557,
+    279,
+    2182,
+    314,
+    11679,
+    13,
+    7165,
+    557,
+    279,
+    2182,
+    314,
+    279,
+    1118,
+    3000,
+    13,
+    5297,
+    557,
+    279,
+    2182,
+    314,
+    279,
+    295,
+    2379,
+    13,
+    5632,
+    557,
+    279,
+    2182,
+    314,
+    279,
+    1248,
+    20513,
+    13,
+    1541,
+    748,
+    383,
+    11,
+    1472,
+    279,
+    1007,
+    314,
+    1024,
+    1765,
+    3154,
+    11,
+    264,
+    1007,
+    421,
+    7481,
+    524,
+    539,
+    279,
+    12680,
+    314,
+    279,
+    9114,
+    694,
+    539,
+    279,
+    12680,
+    314,
+    6326,
+    13,
+    271,
+    4699,
+    3644,
+    12613,
+    264,
+    2086,
+    1304,
+    13,
+    733,
+    279,
+    2182,
+    314,
+    11679,
+    11,
+    1292,
+    5947,
+    264,
+    3878,
+    314,
+    2716,
+    11,
+    16724,
+    9378,
+    883,
+    279,
+    3096,
+    852,
+    995,
+    8341,
+    279,
+    324,
+    55965,
+    1518,
+    1024,
+    6351,
+    11,
+    2019,
+    1142,
+    7840,
+    21116,
+    13,
+    733,
+    279,
+    2182,
+    314,
+    279,
+    1118,
+    3000,
+    11,
+    1292,
+    5947,
+    264,
+    1248,
+    11,
+    6103,
+    8627,
+    883
+   ]
+  }
+ ],
+ "metrics": {},
+ "mean_tok_s": 95.1
+}
\ No newline at end of file
diff --git a/docs/qwen38-vllm-perf-spec-tray14.json b/docs/qwen38-vllm-perf-spec-tray14.json
new file mode 100644
index 0000000..b446736
--- /dev/null
+++ b/docs/qwen38-vllm-perf-spec-tray14.json
@@ -0,0 +1,2061 @@
+{
+ "mode": "spec",
+ "pinned": false,
+ "results": [
+  {
+   "prompt_tokens": 43,
+   "out_tokens": 400,
+   "seconds": 2.558,
+   "tok_s": 156.4,
+   "output_token_ids": [
+    264,
+    3074,
+    4202,
+    421,
+    1467,
+    524,
+    2353,
+    866,
+    314,
+    279,
+    3672,
+    25,
+    264,
+    25347,
+    2512,
+    279,
+    348,
+    32,
+    322,
+    6020,
+    9,
+    995,
+    978,
+    12076,
+    430,
+    1001,
+    52513,
+    506,
+    9117,
+    828,
+    383,
+    279,
+    3603,
+    314,
+    279,
+    3128,
+    70595,
+    11,
+    3482,
+    279,
+    1788,
+    3603,
+    264,
+    7397,
+    4600,
+    725,
+    1433,
+    7246,
+    264,
+    9613,
+    11,
+    14527,
+    54087,
+    71433,
+    7397,
+    501,
+    13810,
+    494,
+    279,
+    2919,
+    321,
+    264,
+    14457,
+    11,
+    83345,
+    3000,
+    888,
+    279,
+    54517,
+    13,
+    561,
+    7110,
+    725,
+    5572,
+    17621,
+    36412,
+    26,
+    279,
+    1330,
+    7189,
+    1362,
+    524,
+    2107,
+    381,
+    804,
+    11,
+    321,
+    279,
+    48086,
+    9290,
+    1384,
+    310,
+    2466,
+    264,
+    3296,
+    421,
+    995,
+    54779,
+    66509,
+    364,
+    22892,
+    25,
+    2972,
+    12195,
+    11,
+    413,
+    3977,
+    11,
+    14995,
+    23221,
+    279,
+    7099,
+    314,
+    279,
+    1814,
+    567,
+    1683,
+    567,
+    1366,
+    170286,
+    271,
+    4639,
+    3296,
+    369,
+    279,
+    4560,
+    314,
+    1396,
+    21831,
+    421,
+    10695,
+    13,
+    1049,
+    31950,
+    586,
+    494,
+    279,
+    13769,
+    2218,
+    741,
+    383,
+    264,
+    45366,
+    11,
+    264,
+    7397,
+    4600,
+    725,
+    11155,
+    383,
+    264,
+    8981,
+    3603,
+    2218,
+    17583,
+    279,
+    7859,
+    21215,
+    314,
+    279,
+    9496,
+    11,
+    1332,
+    279,
+    1001,
+    76,
+    91968,
+    828,
+    369,
+    524,
+    264,
+    3074,
+    6038,
+    694,
+    264,
+    4220,
+    17865,
+    314,
+    72872,
+    5154,
+    13,
+    271,
+    13962,
+    220,
+    16,
+    13,
+    561,
+    46403,
+    314,
+    279,
+    4196,
+    725,
+    16613,
+    2228,
+    760,
+    1118,
+    2172,
+    21831,
+    369,
+    279,
+    799,
+    421,
+    6348,
+    9704,
+    310,
+    279,
+    1546,
+    3299,
+    348,
+    76,
+    91968,
+    77340,
+    279,
+    6042,
+    314,
+    279,
+    79699,
+    13,
+    35725,
+    30404,
+    33461,
+    264,
+    14120,
+    34895,
+    17504,
+    279,
+    9114,
+    11,
+    264,
+    52862,
+    18075,
+    11,
+    466,
+    264,
+    28407,
+    11155,
+    13,
+    17722,
+    7785,
+    39356,
+    279,
+    34895,
+    440,
+    264,
+    3882,
+    11,
+    27044,
+    1528,
+    421,
+    16676,
+    220,
+    16,
+    18,
+    13,
+    23,
+    373,
+    107,
+    65,
+    14300,
+    1578,
+    3998,
+    21010,
+    5971,
+    16419,
+    13,
+    14205,
+    279,
+    3255,
+    1503,
+    524,
+    809,
+    1017,
+    13,
+    1165,
+    628,
+    11319,
+    279,
+    15039,
+    1142,
+    310,
+    264,
+    18830,
+    314,
+    264,
+    2018,
+    1238,
+    421,
+    23439,
+    11,
+    694,
+    279,
+    9402,
+    348,
+    14372,
+    9,
+    8198,
+    264,
+    9813,
+    2081,
+    13,
+    561,
+    2354,
+    1680,
+    430,
+    29144,
+    22525,
+    11,
+    886,
+    9826,
+    11,
+    321,
+    6136,
+    29144,
+    53785,
+    2369,
+    1754,
+    28647,
+    264,
+    2086,
+    1001,
+    3765,
+    4299,
+    2340,
+    3482,
+    6643,
+    682,
+    978,
+    9064,
+    722,
+    10457,
+    13,
+    561,
+    21831,
+    11,
+    8646,
+    11,
+    369,
+    524,
+    264,
+    3074,
+    2029,
+    310,
+    381,
+    41012,
+    694,
+    264,
+    46398,
+    1332,
+    36352,
+    11,
+    21254,
+    11,
+    321,
+    26896,
+    3258,
+    13,
+    271,
+    13962,
+    220,
+    17,
+    13,
+    561,
+    46403,
+    314,
+    88492,
+    2022,
+    2228,
+    2592,
+    279,
+    15039,
+    5820,
+    440,
+    264,
+    7960,
+    314,
+    4649,
+    11,
+    279,
+    1727,
+    21831,
+    369,
+    1204,
+    421,
+    4649,
+    5924,
+    7605,
+    314,
+    4924,
+    13,
+    561,
+    1001,
+    18140,
+    3377,
+    828,
+    314,
+    24086,
+    16561,
+    3069,
+    6745,
+    11036
+   ]
+  },
+  {
+   "prompt_tokens": 41,
+   "out_tokens": 400,
+   "seconds": 1.938,
+   "tok_s": 206.4,
+   "output_token_ids": [
+    264,
+    36334,
+    21388,
+    314,
+    5484,
+    5415,
+    321,
+    264,
+    716,
+    314,
+    34844,
+    542,
+    5564,
+    58091,
+    1172,
+    279,
+    7991,
+    846,
+    13,
+    561,
+    1118,
+    3603,
+    1172,
+    279,
+    491,
+    12515,
+    11,
+    279,
+    6561,
+    2002,
+    6749,
+    92169,
+    1056,
+    5316,
+    25624,
+    11,
+    3482,
+    279,
+    36334,
+    467,
+    78078,
+    5432,
+    68,
+    1990,
+    9047,
+    310,
+    288,
+    92536,
+    13,
+    561,
+    2015,
+    14607,
+    421,
+    279,
+    53952,
+    725,
+    27374,
+    5257,
+    557,
+    874,
+    4860,
+    264,
+    4145,
+    1001,
+    75,
+    704,
+    54087,
+    429,
+    54087,
+    70735,
+    828,
+    3255,
+    26,
+    424,
+    557,
+    264,
+    61010,
+    23850,
+    314,
+    25586,
+    11,
+    1754,
+    440,
+    1141,
+    1765,
+    79299,
+    11,
+    25113,
+    11,
+    321,
+    3558,
+    13,
+    271,
+    4639,
+    3603,
+    5924,
+    264,
+    1105,
+    3788,
+    303,
+    1204,
+    264,
+    7072,
+    54087,
+    5911,
+    53952,
+    628,
+    381,
+    7246,
+    11,
+    16384,
+    11,
+    321,
+    13260,
+    34773,
+    13,
+    20543,
+    369,
+    279,
+    2923,
+    54087,
+    1634,
+    54087,
+    9239,
+    41223,
+    314,
+    279,
+    3377,
+    11,
+    279,
+    795,
+    279,
+    13234,
+    19577,
+    11,
+    321,
+    279,
+    4602,
+    781,
+    4221,
+    310,
+    4087,
+    13,
+    271,
+    4277,
+    271,
+    13962,
+    220,
+    16,
+    13,
+    561,
+    27374,
+    6576,
+    271,
+    760,
+    13736,
+    369,
+    264,
+    2972,
+    32897,
+    54087,
+    38429,
+    18766,
+    5444,
+    332,
+    318,
+    19186,
+    33800,
+    11,
+    1754,
+    440,
+    264,
+    3929,
+    321,
+    264,
+    1142,
+    7099,
+    8,
+    7854,
+    539,
+    264,
+    2972,
+    18452,
+    17312,
+    332,
+    421,
+    58469,
+    279,
+    22690,
+    1142,
+    1472,
+    279,
+    1788,
+    33800,
+    13,
+    561,
+    9468,
+    4868,
+    513,
+    25,
+    271,
+    91,
+    26099,
+    735,
+    39115,
+    314,
+    79299,
+    318,
+    3693,
+    8,
+    735,
+    9926,
+    318,
+    77,
+    8,
+    735,
+    198,
+    91,
+    22031,
+    91,
+    746,
+    14894,
+    91,
+    40698,
+    91,
+    198,
+    91,
+    220,
+    16,
+    318,
+    6735,
+    314,
+    1118,
+    17796,
+    8,
+    735,
+    478,
+    16,
+    20,
+    15,
+    735,
+    220,
+    16,
+    13,
+    20,
+    16,
+    21,
+    23,
+    735,
+    198,
+    91,
+    220,
+    17,
+    318,
+    1371,
+    314,
+    1118,
+    17796,
+    8,
+    220,
+    735,
+    1317,
+    16,
+    17,
+    15,
+    735,
+    220,
+    16,
+    13,
+    20,
+    16,
+    21,
+    23,
+    735,
+    198,
+    91,
+    220,
+    18,
+    318,
+    6735,
+    314,
+    2018,
+    17796,
+    8,
+    735,
+    478,
+    17,
+    15,
+    15,
+    735,
+    220,
+    16,
+    13,
+    21,
+    17,
+    15,
+    15,
+    735,
+    198,
+    91,
+    220,
+    19,
+    318,
+    1371,
+    314,
+    2018,
+    17796,
+    8,
+    220,
+    735,
+    1317,
+    16,
+    23,
+    15,
+    735,
+    220,
+    16,
+    13,
+    21,
+    17,
+    15,
+    15,
+    735,
+    198,
+    91,
+    220,
+    20,
+    318,
+    18452,
+    17312,
+    8,
+    260,
+    735,
+    11654,
+    252,
+    318,
+    25393,
+    8,
+    735,
+    1317,
+    735,
+    271,
+    760,
+    2972,
+    686,
+    518,
+    4681,
+    778,
+    332,
+    318,
+    2959,
+    54087,
+    951,
+    54087,
+    2959,
+    25704,
+    8,
+    513,
+    25,
+    271,
+    12,
+    26260,
+    7099,
+    220,
+    16,
+    321,
+    7099,
+    220,
+    17,
+    25,
+    2972,
+    16,
+    15,
+    9186,
+    332,
+    198,
+    12,
+    26260,
+    7099,
+    220,
+    17,
+    321,
+    7099,
+    220,
+    18,
+    25,
+    2972,
+    17,
+    20,
+    9186,
+    332,
+    198,
+    12,
+    26260,
+    7099,
+    220,
+    18,
+    321,
+    7099,
+    220
+   ]
+  },
+  {
+   "prompt_tokens": 48,
+   "out_tokens": 400,
+   "seconds": 2.389,
+   "tok_s": 167.4,
+   "output_token_ids": [
+    7094,
+    781,
+    14428,
+    279,
+    53627,
+    287,
+    440,
+    22945,
+    9399,
+    49954,
+    11,
+    8132,
+    1396,
+    2418,
+    314,
+    272,
+    46016,
+    11,
+    1396,
+    10582,
+    314,
+    274,
+    6960,
+    11,
+    1396,
+    22263,
+    23178,
+    17629,
+    22149,
+    13,
+    733,
+    279,
+    4688,
+    7094,
+    781,
+    5743,
+    310,
+    279,
+    1788,
+    22945,
+    52604,
+    321,
+    1669,
+    279,
+    14367,
+    995,
+    7481,
+    13,
+    271,
+    760,
+    42159,
+    557,
+    2496,
+    7807,
+    13,
+    561,
+    14367,
+    8341,
+    7594,
+    795,
+    13,
+    271,
+    4277,
+    271,
+    760,
+    1118,
+    3065,
+    488,
+    5125,
+    883,
+    279,
+    17198,
+    20139,
+    369,
+    421,
+    424,
+    369,
+    524,
+    264,
+    1925,
+    13,
+    1049,
+    369,
+    264,
+    1817,
+    11884,
+    279,
+    30385,
+    314,
+    264,
+    1925,
+    13,
+    561,
+    4128,
+    488,
+    2418,
+    383,
+    557,
+    14367,
+    2634,
+    264,
+    12828,
+    3998,
+    321,
+    668,
+    381,
+    14367,
+    2634,
+    1495,
+    303,
+    264,
+    12828,
+    13,
+    561,
+    12052,
+    513,
+    524,
+    39527,
+    303,
+    16665,
+    748,
+    1691,
+    430,
+    303,
+    264,
+    46446,
+    440,
+    2919,
+    13,
+    561,
+    272,
+    46016,
+    29528,
+    513,
+    524,
+    19101,
+    13,
+    2302,
+    513,
+    279,
+    4757,
+    579,
+    1560,
+    314,
+    25108,
+    948,
+    279,
+    2919,
+    1898,
+    35937,
+    3294,
+    1141,
+    7303,
+    13,
+    271,
+    760,
+    2018,
+    3065,
+    488,
+    5125,
+    369,
+    279,
+    20513,
+    13,
+    2717,
+    279,
+    18682,
+    314,
+    4950,
+    13,
+    561,
+    9084,
+    314,
+    264,
+    2086,
+    2993,
+    314,
+    4950,
+    13,
+    561,
+    3594,
+    369,
+    11702,
+    440,
+    279,
+    11639,
+    314,
+    2919,
+    6992,
+    1472,
+    51930,
+    11,
+    314,
+    39465,
+    3133,
+    279,
+    25348,
+    11,
+    314,
+    19053,
+    7842,
+    303,
+    32655,
+    421,
+    2714,
+    5995,
+    310,
+    5640,
+    1472,
+    13988,
+    3594,
+    13,
+    561,
+    20513,
+    369,
+    279,
+    4950,
+    314,
+    264,
+    1785,
+    303,
+    53454,
+    11,
+    321,
+    424,
+    369,
+    279,
+    27705,
+    4818,
+    3065,
+    303,
+    279,
+    1814,
+    13,
+    271,
+    760,
+    4688,
+    3065,
+    488,
+    5125,
+    369,
+    279,
+    21388,
+    13,
+    326,
+    13898,
+    322,
+    13,
+    93048,
+    3374,
+    13,
+    561,
+    5983,
+    11,
+    9922,
+    5571,
+    314,
+    16845,
+    4766,
+    14293,
+    1441,
+    303,
+    38957,
+    261,
+    29799,
+    25348,
+    13,
+    1049,
+    369,
+    279,
+    21388,
+    314,
+    264,
+    1925,
+    1332,
+    4397,
+    369,
+    524,
+    449,
+    13004,
+    694,
+    264,
+    10106,
+    13,
+    271,
+    4277,
+    271,
+    760,
+    296,
+    7679,
+    358,
+    41713,
+    513,
+    279,
+    17198,
+    20139,
+    579,
+    4779,
+    13,
+    3095,
+    279,
+    14367,
+    4203,
+    3199,
+    11,
+    424,
+    10583,
+    4661,
+    264,
+    44234,
+    1129,
+    314,
+    2919,
+    421,
+    13569,
+    39248,
+    440,
+    56084,
+    11,
+    13632,
+    15049,
+    11,
+    321,
+    8460,
+    13,
+    8618,
+    296,
+    7679,
+    358,
+    369,
+    264,
+    3150,
+    314,
+    264,
+    3820,
+    35600,
+    303,
+    279,
+    14367,
+    579,
+    1752,
+    11,
+    264,
+    3820,
+    3098,
+    314,
+    36900,
+    11,
+    264,
+    3820,
+    6343,
+    314,
+    279,
+    17865,
+    421,
+    668,
+    2496,
+    2905,
+    1495,
+    13,
+    271,
+    760,
+    1024,
+    263,
+    897,
+    547,
+    4547,
+    513,
+    279,
+    17198,
+    20139,
+    579,
+    2945,
+    41272,
+    13,
+    561,
+    19053,
+    16821,
+    303,
+    5297,
+    321,
+    5106,
+    303,
+    5963,
+    13,
+    2302,
+    1870,
+    836,
+    91838,
+    303,
+    279,
+    272,
+    46016,
+    12052,
+    11,
+    321,
+    279,
+    91838,
+    513,
+    748
+   ]
+  },
+  {
+   "prompt_tokens": 42,
+   "out_tokens": 400,
+   "seconds": 2.244,
+   "tok_s": 178.3,
+   "output_token_ids": [
+    271,
+    8160,
+    369,
+    449,
+    15673,
+    314,
+    1439,
+    2250,
+    41724,
+    303,
+    13983,
+    3992,
+    11,
+    3055,
+    440,
+    264,
+    13769,
+    3010,
+    364,
+    1754,
+    13,
+    271,
+    13962,
+    220,
+    16,
+    13,
+    8169,
+    264,
+    19269,
+    7541,
+    2751,
+    303,
+    18266,
+    271,
+    332,
+    760,
+    68868,
+    64700,
+    198,
+    33807,
+    303,
+    279,
+    1814,
+    369,
+    1801,
+    314,
+    13280,
+    18167,
+    318,
+    63339,
+    321,
+    33470,
+    8,
+    421,
+    513,
+    14537,
+    6992,
+    13,
+    3095,
+    279,
+    9039,
+    35937,
+    11,
+    1439,
+    18167,
+    8457,
+    4649,
+    321,
+    3166,
+    10281,
+    11,
+    13969,
+    1070,
+    310,
+    4336,
+    9781,
+    4473,
+    10431,
+    494,
+    799,
+    2361,
+    13,
+    5788,
+    411,
+    2222,
+    369,
+    85928,
+    303,
+    264,
+    3074,
+    18546,
+    11,
+    948,
+    488,
+    29283,
+    424,
+    539,
+    279,
+    30995,
+    314,
+    31137,
+    303,
+    264,
+    10632,
+    5759,
+    1040,
+    264,
+    9229,
+    13759,
+    11,
+    279,
+    2702,
+    14046,
+    8772,
+    4927,
+    13,
+    1368,
+    264,
+    13759,
+    998,
+    5617,
+    430,
+    799,
+    6235,
+    11,
+    632,
+    47272,
+    6365,
+    11,
+    411,
+    14046,
+    303,
+    279,
+    8364,
+    1000,
+    5073,
+    279,
+    9041,
+    310,
+    77238,
+    466,
+    16966,
+    1521,
+    424,
+    682,
+    26241,
+    310,
+    708,
+    13,
+    271,
+    332,
+    80711,
+    12996,
+    64700,
+    198,
+    37405,
+    314,
+    264,
+    2972,
+    52195,
+    46379,
+    2047,
+    303,
+    264,
+    3882,
+    1740,
+    159034,
+    1368,
+    488,
+    6420,
+    279,
+    46379,
+    440,
+    264,
+    1564,
+    12489,
+    303,
+    279,
+    12050,
+    321,
+    1179,
+    1495,
+    1238,
+    424,
+    682,
+    978,
+    11362,
+    303,
+    279,
+    6797,
+    364,
+    449,
+    6258,
+    11,
+    279,
+    46379,
+    668,
+    381,
+    21219,
+    4860,
+    303,
+    279,
+    7094,
+    13,
+    357,
+    13759,
+    22839,
+    9331,
+    364,
+    411,
+    539,
+    25780,
+    328,
+    4431,
+    10188,
+    33124,
+    62804,
+    70,
+    2538,
+    9896,
+    440,
+    21970,
+    466,
+    9041,
+    23702,
+    2218,
+    40486,
+    13753,
+    314,
+    279,
+    13759,
+    13,
+    733,
+    7094,
+    11,
+    279,
+    13759,
+    50110,
+    1083,
+    1439,
+    31092,
+    11,
+    10377,
+    279,
+    5759,
+    310,
+    14084,
+    1973,
+    14293,
+    13,
+    271,
+    13962,
+    220,
+    17,
+    13,
+    8169,
+    264,
+    710,
+    46038,
+    326,
+    793,
+    12851,
+    1049,
+    2436,
+    8405,
+    271,
+    332,
+    760,
+    68868,
+    64700,
+    198,
+    760,
+    328,
+    82,
+    23279,
+    1,
+    466,
+    412,
+    375,
+    2631,
+    4950,
+    369,
+    3404,
+    264,
+    1067,
+    314,
+    2972,
+    408,
+    263,
+    663,
+    332,
+    321,
+    2972,
+    85,
+    17804,
+    332,
+    8613,
+    539,
+    19695,
+    13,
+    1577,
+    2919,
+    73944,
+    685,
+    11,
+    424,
+    3070,
+    914,
+    42244,
+    21146,
+    16360,
+    506,
+    2957,
+    13,
+    21153,
+    42237,
+    1304,
+    506,
+    279,
+    5445,
+    314,
+    279,
+    69862,
+    1332,
+    424,
+    369,
+    36056,
+    13,
+    1577,
+    1439,
+    42237,
+    9704,
+    1472,
+    279,
+    34635,
+    2919,
+    3294,
+    1070,
+    11,
+    781,
+    3428,
+    17638,
+    318,
+    53710,
+    8,
+    1518,
+    18176,
+    279,
+    7099,
+    13,
+    1061,
+    10725,
+    17638,
+    11121,
+    13280,
+    10439,
+    55348,
+    321,
+    74610,
+    13,
+    271,
+    623,
+    264,
+    412,
+    375,
+    2631,
+    69862,
+    11,
+    279,
+    2790,
+    11407,
+    411,
+    19695,
+    1472,
+    264,
+    2526,
+    11,
+    3050,
+    8306,
+    318,
+    1719,
+    38240,
+    553,
+    561,
+    19695,
+    8177,
+    3594,
+    310,
+    343,
+    41670,
+    506,
+    264,
+    3050,
+    11304,
+    11,
+    1691,
+    1040,
+    39005,
+    3808,
+    279
+   ]
+  },
+  {
+   "prompt_tokens": 47,
+   "out_tokens": 400,
+   "seconds": 2.693,
+   "tok_s": 148.5,
+   "output_token_ids": [
+    9896,
+    2250,
+    75842,
+    440,
+    11140,
+    421,
+    998,
+    40361,
+    42749,
+    13,
+    3113,
+    35442,
+    1292,
+    995,
+    5158,
+    264,
+    11183,
+    883,
+    264,
+    5054,
+    852,
+    1362,
+    6512,
+    279,
+    9117,
+    579,
+    295,
+    8792,
+    1472,
+    279,
+    6220,
+    18687,
+    11,
+    321,
+    1292,
+    4035,
+    2574,
+    424,
+    310,
+    1024,
+    6764,
+    11,
+    852,
+    1301,
+    424,
+    303,
+    799,
+    11362,
+    321,
+    1179,
+    1018,
+    11,
+    328,
+    2523,
+    2908,
+    1801,
+    279,
+    9117,
+    4950,
+    1040,
+    264,
+    27809,
+    36252,
+    1149,
+    2838,
+    3738,
+    411,
+    430,
+    279,
+    8166,
+    27463,
+    13,
+    271,
+    760,
+    11183,
+    557,
+    2496,
+    4501,
+    13,
+    1049,
+    7338,
+    303,
+    264,
+    25635,
+    364,
+    42744,
+    1578,
+    11,
+    22175,
+    279,
+    3820,
+    20513,
+    314,
+    2433,
+    421,
+    513,
+    9939,
+    694,
+    524,
+    3482,
+    5354,
+    13,
+    3095,
+    1292,
+    5326,
+    3106,
+    424,
+    310,
+    264,
+    20609,
+    11,
+    781,
+    5743,
+    424,
+    440,
+    264,
+    5020,
+    421,
+    557,
+    4409,
+    2993,
+    25,
+    328,
+    32,
+    6037,
+    2351,
+    364,
+    16078,
+    13,
+    1165,
+    2224,
+    524,
+    2617,
+    1092,
+    279,
+    3255,
+    369,
+    1149,
+    2838,
+    15513,
+    13,
+    561,
+    9117,
+    579,
+    295,
+    8792,
+    998,
+    279,
+    3255,
+    13,
+    1921,
+    279,
+    9117,
+    579,
+    295,
+    8792,
+    11,
+    424,
+    6316,
+    680,
+    11,
+    998,
+    524,
+    264,
+    3255,
+    421,
+    1000,
+    4410,
+    13,
+    271,
+    4272,
+    1292,
+    5947,
+    975,
+    2433,
+    13,
+    10382,
+    7112,
+    364,
+    30801,
+    421,
+    6945,
+    303,
+    11988,
+    321,
+    26957,
+    69204,
+    13,
+    357,
+    48555,
+    883,
+    279,
+    324,
+    55965,
+    11,
+    864,
+    5898,
+    25989,
+    391,
+    321,
+    1801,
+    1024,
+    264,
+    2526,
+    803,
+    303,
+    264,
+    2526,
+    1814,
+    13,
+    2838,
+    11876,
+    264,
+    855,
+    852,
+    14129,
+    47558,
+    321,
+    11252,
+    421,
+    1396,
+    3046,
+    995,
+    264,
+    5016,
+    421,
+    557,
+    9781,
+    2166,
+    3349,
+    364,
+    1141,
+    17435,
+    13,
+    2302,
+    995,
+    1330,
+    2748,
+    852,
+    13538,
+    685,
+    5122,
+    1024,
+    75898,
+    888,
+    1024,
+    16388,
+    11,
+    852,
+    9218,
+    421,
+    836,
+    6351,
+    579,
+    22157,
+    995,
+    264,
+    3820,
+    35091,
+    11,
+    1040,
+    16382,
+    2272,
+    264,
+    3820,
+    29199,
+    13,
+    271,
+    760,
+    324,
+    55965,
+    557,
+    27572,
+    2641,
+    290,
+    303,
+    220,
+    16,
+    24,
+    23,
+    22,
+    13,
+    2838,
+    3804,
+    1142,
+    2957,
+    11,
+    303,
+    279,
+    40297,
+    11,
+    321,
+    14217,
+    303,
+    279,
+    51624,
+    579,
+    2956,
+    1332,
+    279,
+    6220,
+    18687,
+    1990,
+    272,
+    1072,
+    290,
+    303,
+    279,
+    2235,
+    5215,
+    13,
+    2838,
+    16810,
+    1024,
+    2351,
+    310,
+    279,
+    7339,
+    321,
+    6408,
+    4161,
+    13,
+    2441,
+    4598,
+    11,
+    1292,
+    6408,
+    4156,
+    11,
+    864,
+    557,
+    279,
+    1788,
+    3065,
+    13,
+    561,
+    9117,
+    557,
+    1990,
+    1017,
+    11,
+    1990,
+    4396,
+    1041,
+    11,
+    1990,
+    27809,
+    13,
+    1049,
+    995,
+    4777,
+    10184,
+    15899,
+    364,
+    1024,
+    13,
+    271,
+    7691,
+    557,
+    31347,
+    46780,
+    948,
+    279,
+    2018,
+    11183,
+    3571,
+    13,
+    2717,
+    279,
+    324,
+    55965,
+    11183,
+    13,
+    357,
+    2086,
+    799,
+    13,
+    9680,
+    264,
+    5054,
+    303,
+    264,
+    33101,
+    6092,
+    852,
+    13205,
+    264,
+    1103,
+    314,
+    1396,
+    14882,
+    421,
+    15734,
+    11,
+    321,
+    852,
+    11690,
+    310,
+    5125,
+    421
+   ]
+  }
+ ],
+ "metrics": {
+  "vllm:spec_decode_num_drafts": 818,
+  "vllm:spec_decode_num_draft_tokens": 5726,
+  "vllm:spec_decode_num_accepted_tokens": 1190,
+  "vllm:spec_decode_num_accepted_tokens_per_pos": [
+   521,
+   307,
+   164,
+   93,
+   45,
+   35,
+   25
+  ]
+ },
+ "mean_tok_s": 171.4
+}
\ No newline at end of file
diff --git a/examples/qwen3.8-27b-dflash2.json b/examples/qwen3.8-27b-dflash2.json
index 97471fb..75eebd3 100644
--- a/examples/qwen3.8-27b-dflash2.json
+++ b/examples/qwen3.8-27b-dflash2.json
@@ -6335,34 +6335,46 @@
     ]
    }
   },
-  "gemma_norm_qhead": {
+  "attn_prep": {
    "params": [
+    "out buffer<bf16>",
     "out buffer<bf16>",
     "in buffer<bf16>",
     "in buffer<f32>",
+    "in buffer<f32>",
+    "in buffer<bf16>",
+    "in buffer<bf16>",
+    "inout ptr",
+    "inout ptr",
+    "in buffer<i64>",
     "i32",
     "i32",
-    "i32",
-    "i32",
-    "i32",
-    "i32",
+    "i64",
+    "i64",
+    "i64",
     "i32",
     "f32"
    ],
    "impl": {
     "steps": [
      {
-      "symbol": "kern_gemma_rms_norm_bf16",
+      "symbol": "kern_attn_prep_bf16",
       "params": [
+       "out buffer<bf16>",
        "out buffer<bf16>",
        "in buffer<bf16>",
        "in buffer<f32>",
+       "in buffer<f32>",
+       "in buffer<bf16>",
+       "in buffer<bf16>",
+       "inout ptr",
+       "inout ptr",
+       "in buffer<i64>",
        "i32",
        "i32",
-       "i32",
-       "i32",
-       "i32",
-       "i32",
+       "i64",
+       "i64",
+       "i64",
        "i32",
        "f32"
       ],
@@ -6377,7 +6389,7 @@
          {
           "sym": "tokens"
          },
-         24
+         28
         ]
        },
        1,
@@ -6416,43 +6428,56 @@
        },
        {
         "arg": 10
+       },
+       {
+        "arg": 11
+       },
+       {
+        "arg": 12
+       },
+       {
+        "arg": 13
+       },
+       {
+        "arg": 14
+       },
+       {
+        "arg": 15
+       },
+       {
+        "arg": 16
        }
       ],
-      "shared_mem": 4096,
-      "cubin": "gemma_rms_norm.cubin"
+      "cubin": "attn_prep.cubin"
      }
     ]
    }
   },
-  "gemma_norm_khead": {
+  "gemma_fused_norm": {
    "params": [
     "out buffer<bf16>",
     "in buffer<bf16>",
+    "inout buffer<bf16>",
     "in buffer<f32>",
     "i32",
     "i32",
     "i32",
     "i32",
-    "i32",
-    "i32",
-    "i32",
     "f32"
    ],
    "impl": {
     "steps": [
      {
-      "symbol": "kern_gemma_rms_norm_bf16",
+      "symbol": "kern_gemma_fused_add_rms_norm_bf16",
       "params": [
        "out buffer<bf16>",
        "in buffer<bf16>",
+       "inout buffer<bf16>",
        "in buffer<f32>",
        "i32",
        "i32",
        "i32",
        "i32",
-       "i32",
-       "i32",
-       "i32",
        "f32"
       ],
       "block": [
@@ -6462,12 +6487,7 @@
       ],
       "grid": [
        {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
+        "sym": "tokens"
        },
        1,
        1
@@ -6499,41 +6519,257 @@
        },
        {
         "arg": 8
+       }
+      ],
+      "shared_mem": 43008,
+      "cubin": "gemma_rms_norm.cubin"
+     }
+    ]
+   }
+  },
+  "silu_mul": {
+   "params": [
+    "out buffer<bf16>",
+    "in buffer<bf16>",
+    "i32",
+    "i32",
+    "f32",
+    "i32"
+   ],
+   "impl": {
+    "steps": [
+     {
+      "symbol": "_ZN4vllm18act_and_mul_kernelIN3c108BFloat16E14__nv_bfloat162XadL_ZNS_11silu_kernelIS2_EET_RKS5_fEEXadL_ZNS_18packed_silu_kernelIS3_EES5_S7_fEELb1ELb1ELb0ELb1EEEvPS5_PS6_ifff",
+      "params": [
+       "out buffer<bf16>",
+       "in buffer<bf16>",
+       "i32",
+       "i32",
+       "f32",
+       "i32"
+      ],
+      "block": [
+       1024,
+       1,
+       1
+      ],
+      "grid": [
+       {
+        "sym": "tokens"
        },
+       1,
+       1
+      ],
+      "args": [
        {
-        "arg": 9
+        "arg": 0
        },
        {
-        "arg": 10
+        "arg": 1
+       },
+       {
+        "arg": 2
+       },
+       {
+        "arg": 3
+       },
+       {
+        "arg": 4
+       },
+       {
+        "arg": 5
+       }
+      ]
+     }
+    ]
+   }
+  },
+  "gemm8_gateup_silu": {
+   "params": [
+    "out buffer<bf16>",
+    "in buffer<bf16>",
+    "in buffer<bf16>",
+    "i32",
+    "i32",
+    "i32"
+   ],
+   "impl": {
+    "steps": [
+     {
+      "symbol": "kern_gemm8_gateup_silu_bf16",
+      "params": [
+       "out buffer<bf16>",
+       "in buffer<bf16>",
+       "in buffer<bf16>",
+       "i32",
+       "i32",
+       "i32"
+      ],
+      "block": [
+       288,
+       1,
+       1
+      ],
+      "grid": [
+       152,
+       1,
+       1
+      ],
+      "args": [
+       {
+        "arg": 0
+       },
+       {
+        "arg": 1
+       },
+       {
+        "arg": 2
+       },
+       {
+        "arg": 3
+       },
+       {
+        "arg": 4
+       },
+       {
+        "arg": 5
        }
       ],
-      "shared_mem": 4096,
-      "cubin": "gemma_rms_norm.cubin"
+      "shared_mem": 224128,
+      "cubin": "gemm8.cubin",
+      "pdl": true
      }
     ]
    }
   },
-  "gemma_fused_norm": {
+  "gemm8_dual": {
+   "params": [
+    "out buffer<bf16>",
+    "out buffer<bf16>",
+    "in buffer<bf16>",
+    "in buffer<bf16>",
+    "in buffer<bf16>",
+    "i32",
+    "i32",
+    "i32",
+    "i32"
+   ],
+   "impl": {
+    "steps": [
+     {
+      "symbol": "kern_gemm8_dual_bf16",
+      "params": [
+       "out buffer<bf16>",
+       "out buffer<bf16>",
+       "in buffer<bf16>",
+       "in buffer<bf16>",
+       "in buffer<bf16>",
+       "i32",
+       "i32",
+       "i32",
+       "i32"
+      ],
+      "block": [
+       288,
+       1,
+       1
+      ],
+      "grid": [
+       152,
+       1,
+       1
+      ],
+      "args": [
+       {
+        "arg": 0
+       },
+       {
+        "arg": 1
+       },
+       {
+        "arg": 2
+       },
+       {
+        "arg": 3
+       },
+       {
+        "arg": 4
+       },
+       {
+        "arg": 5
+       },
+       {
+        "arg": 6
+       },
+       {
+        "arg": 7
+       },
+       {
+        "arg": 8
+       }
+      ],
+      "shared_mem": 224128,
+      "cubin": "gemm8.cubin",
+      "pdl": true
+     }
+    ]
+   }
+  },
+  "gemm8_sgate_add_norm": {
    "params": [
     "out buffer<bf16>",
     "in buffer<bf16>",
+    "in buffer<bf16>",
+    "in buffer<bf16>",
     "inout buffer<bf16>",
     "in buffer<f32>",
     "i32",
     "i32",
     "i32",
     "i32",
+    "i32",
+    "i32",
     "f32"
    ],
    "impl": {
+    "scratch": {
+     "y": {
+      "dtype": "bf16",
+      "shape": [
+       8,
+       5120
+      ]
+     },
+     "partial": {
+      "dtype": "f32",
+      "shape": [
+       8,
+       8,
+       5120
+      ]
+     },
+     "gcnt": {
+      "dtype": "i32",
+      "shape": [
+       2
+      ]
+     }
+    },
     "steps": [
      {
-      "symbol": "kern_gemma_fused_add_rms_norm_bf16",
+      "symbol": "kern_gemm8_sgate_add_norm_bf16",
       "params": [
        "out buffer<bf16>",
        "in buffer<bf16>",
+       "in buffer<bf16>",
+       "in buffer<bf16>",
        "inout buffer<bf16>",
        "in buffer<f32>",
+       "out buffer<bf16>",
+       "out buffer<f32>",
+       "out buffer<i32>",
+       "i32",
+       "i32",
        "i32",
        "i32",
        "i32",
@@ -6541,14 +6777,12 @@
        "f32"
       ],
       "block": [
-       512,
+       288,
        1,
        1
       ],
       "grid": [
-       {
-        "sym": "tokens"
-       },
+       152,
        1,
        1
       ],
@@ -6571,6 +6805,15 @@
        {
         "arg": 5
        },
+       {
+        "scratch": "y"
+       },
+       {
+        "scratch": "partial"
+       },
+       {
+        "scratch": "gcnt"
+       },
        {
         "arg": 6
        },
@@ -6579,44 +6822,87 @@
        },
        {
         "arg": 8
+       },
+       {
+        "arg": 9
+       },
+       {
+        "arg": 10
+       },
+       {
+        "arg": 11
+       },
+       {
+        "arg": 12
        }
       ],
-      "shared_mem": 43008,
-      "cubin": "gemma_rms_norm.cubin"
+      "shared_mem": 224128,
+      "cubin": "gemm8.cubin",
+      "pdl": true
      }
     ]
    }
   },
-  "silu_mul": {
+  "gemm8_add_norm": {
    "params": [
     "out buffer<bf16>",
     "in buffer<bf16>",
+    "in buffer<bf16>",
+    "inout buffer<bf16>",
+    "in buffer<f32>",
     "i32",
     "i32",
-    "f32",
-    "i32"
+    "i32",
+    "f32"
    ],
    "impl": {
+    "scratch": {
+     "y": {
+      "dtype": "bf16",
+      "shape": [
+       8,
+       5120
+      ]
+     },
+     "partial": {
+      "dtype": "f32",
+      "shape": [
+       8,
+       8,
+       5120
+      ]
+     },
+     "gcnt": {
+      "dtype": "i32",
+      "shape": [
+       2
+      ]
+     }
+    },
     "steps": [
      {
-      "symbol": "_ZN4vllm18act_and_mul_kernelIN3c108BFloat16E14__nv_bfloat162XadL_ZNS_11silu_kernelIS2_EET_RKS5_fEEXadL_ZNS_18packed_silu_kernelIS3_EES5_S7_fEELb1ELb1ELb0ELb1EEEvPS5_PS6_ifff",
+      "symbol": "kern_gemm8_add_norm_bf16",
       "params": [
        "out buffer<bf16>",
        "in buffer<bf16>",
+       "in buffer<bf16>",
+       "inout buffer<bf16>",
+       "in buffer<f32>",
+       "out buffer<bf16>",
+       "out buffer<f32>",
+       "out buffer<i32>",
        "i32",
        "i32",
-       "f32",
-       "i32"
+       "i32",
+       "f32"
       ],
       "block": [
-       1024,
+       288,
        1,
        1
       ],
       "grid": [
-       {
-        "sym": "tokens"
-       },
+       152,
        1,
        1
       ],
@@ -6636,10 +6922,31 @@
        {
         "arg": 4
        },
+       {
+        "scratch": "y"
+       },
+       {
+        "scratch": "partial"
+       },
+       {
+        "scratch": "gcnt"
+       },
        {
         "arg": 5
+       },
+       {
+        "arg": 6
+       },
+       {
+        "arg": 7
+       },
+       {
+        "arg": 8
        }
-      ]
+      ],
+      "shared_mem": 224128,
+      "cubin": "gemm8.cubin",
+      "pdl": true
      }
     ]
    }
@@ -7743,179 +8050,6 @@
     ]
    }
   },
-  "mrope": {
-   "params": [
-    "inout buffer<bf16>",
-    "inout buffer<bf16>",
-    "in buffer<bf16>",
-    "in buffer<bf16>",
-    "i32",
-    "i64",
-    "i64"
-   ],
-   "impl": {
-    "steps": [
-     {
-      "symbol": "_triton_mrope_forward",
-      "params": [
-       "inout buffer<bf16>",
-       "inout buffer<bf16>",
-       "in buffer<bf16>",
-       "in buffer<bf16>",
-       "i32",
-       "i64",
-       "i64"
-      ],
-      "block": [
-       128,
-       1,
-       1
-      ],
-      "grid": [
-       {
-        "sym": "tokens"
-       },
-       1,
-       1
-      ],
-      "args": [
-       {
-        "arg": 0
-       },
-       {
-        "arg": 1
-       },
-       {
-        "arg": 2
-       },
-       {
-        "arg": 3
-       },
-       {
-        "arg": 4
-       },
-       {
-        "arg": 5
-       },
-       {
-        "arg": 6
-       }
-      ],
-      "cubin": "cc5d7bff4bfd.cubin",
-      "sha256": "cc5d7bff4bfd36a94e8e43bfb6651908905734c8314019db41d3e469b3517483"
-     }
-    ]
-   }
-  },
-  "reshape_and_cache": {
-   "params": [
-    "in buffer<bf16>",
-    "in buffer<bf16>",
-    "inout ptr",
-    "inout ptr",
-    "in buffer<i64>",
-    "in buffer<f32>",
-    "in buffer<f32>",
-    "i64",
-    "i64",
-    "i64",
-    "i64",
-    "i64",
-    "i64",
-    "i64",
-    "i64",
-    "i64"
-   ],
-   "impl": {
-    "steps": [
-     {
-      "symbol": "reshape_and_cache_kernel_flash",
-      "params": [
-       "in buffer<bf16>",
-       "in buffer<bf16>",
-       "inout ptr",
-       "inout ptr",
-       "in buffer<i64>",
-       "in buffer<f32>",
-       "in buffer<f32>",
-       "i64",
-       "i64",
-       "i64",
-       "i64",
-       "i64",
-       "i64",
-       "i64",
-       "i64",
-       "i64"
-      ],
-      "block": [
-       512,
-       1,
-       1
-      ],
-      "grid": [
-       {
-        "sym": "tokens"
-       },
-       1,
-       1
-      ],
-      "args": [
-       {
-        "arg": 0
-       },
-       {
-        "arg": 1
-       },
-       {
-        "arg": 2
-       },
-       {
-        "arg": 3
-       },
-       {
-        "arg": 4
-       },
-       {
-        "arg": 5
-       },
-       {
-        "arg": 6
-       },
-       {
-        "arg": 7
-       },
-       {
-        "arg": 8
-       },
-       {
-        "arg": 9
-       },
-       {
-        "arg": 10
-       },
-       {
-        "arg": 11
-       },
-       {
-        "arg": 12
-       },
-       {
-        "arg": 13
-       },
-       {
-        "arg": 14
-       },
-       {
-        "arg": 15
-       }
-      ],
-      "cubin": "93d7768f15ce.cubin",
-      "sha256": "93d7768f15ce02e252ad208dca2f6803d8ad65296d57f3444dccb9d1e6b88ac5"
-     }
-    ]
-   }
-  },
   "attn_prefill": {
    "params": [
     "out buffer<bf16>",
@@ -8421,49 +8555,111 @@
     ]
    }
   },
-  "conv_update_spec": {
+  "gdn_spec": {
    "params": [
+    "inout buffer<bf16>",
     "in buffer<bf16>",
-    "in buffer<bf16>",
+    "inout ptr",
     "inout ptr",
     "in buffer<i32>",
     "in buffer<i32>",
     "in buffer<i32>",
+    "in buffer<i32>",
+    "in buffer<bf16>",
+    "in buffer<f32>",
+    "in buffer<bf16>",
     "out buffer<bf16>",
+    "in buffer<bf16>",
+    "out buffer<bf16>",
+    "out buffer<bf16>",
+    "out buffer<bf16>",
+    "out buffer<f32>",
+    "out buffer<f32>",
+    "out buffer<bf16>",
+    "out buffer<bf16>",
+    "out buffer<bf16>",
+    "f32",
+    "f32",
     "i32",
     "i32",
-    "i64",
-    "i64",
-    "i64",
-    "i64"
+    "i32",
+    "i32",
+    "i32"
    ],
    "impl": {
+    "scratch": {
+     "xcnt": {
+      "dtype": "i32",
+      "shape": [
+       16
+      ]
+     },
+     "hpart": {
+      "dtype": "f32",
+      "shape": [
+       8,
+       48,
+       4
+      ]
+     },
+     "hcnt": {
+      "dtype": "i32",
+      "shape": [
+       48
+      ]
+     },
+     "gcnt": {
+      "dtype": "i32",
+      "shape": [
+       1
+      ]
+     }
+    },
     "steps": [
      {
-      "symbol": "_causal_conv1d_update_kernel",
+      "symbol": "kern_gdn_spec_bf16",
       "params": [
+       "inout buffer<bf16>",
        "in buffer<bf16>",
-       "in buffer<bf16>",
+       "inout ptr",
        "inout ptr",
        "in buffer<i32>",
        "in buffer<i32>",
        "in buffer<i32>",
+       "in buffer<i32>",
+       "in buffer<bf16>",
+       "in buffer<f32>",
+       "in buffer<bf16>",
+       "out buffer<bf16>",
+       "in buffer<bf16>",
+       "out buffer<bf16>",
+       "out buffer<bf16>",
+       "out buffer<bf16>",
+       "out buffer<f32>",
+       "out buffer<f32>",
        "out buffer<bf16>",
+       "out buffer<bf16>",
+       "out buffer<bf16>",
+       "out buffer<i32>",
+       "out buffer<f32>",
+       "out buffer<i32>",
+       "out buffer<i32>",
+       "f32",
+       "f32",
        "i32",
        "i32",
-       "i64",
-       "i64",
-       "i64",
-       "i64"
+       "i32",
+       "i32",
+       "i32"
       ],
       "block": [
-       128,
+       256,
        1,
        1
       ],
       "grid": [
+       192,
        1,
-       40,
        1
       ],
       "args": [
@@ -8505,138 +8701,66 @@
        },
        {
         "arg": 12
-       }
-      ],
-      "cubin": "159465ea16be.cubin",
-      "sha256": "159465ea16be78a5440e457e32159ace2a074114488122dc35be1b30962006b1"
-     }
-    ]
-   }
-  },
-  "recurrent_spec": {
-   "params": [
-    "in buffer<f32>",
-    "in buffer<bf16>",
-    "in buffer<bf16>",
-    "in buffer<bf16>",
-    "f32",
-    "f32",
-    "in buffer<bf16>",
-    "in buffer<bf16>",
-    "in buffer<bf16>",
-    "out buffer<bf16>",
-    "inout ptr",
-    "inout ptr",
-    "in buffer<i32>",
-    "in buffer<i32>",
-    "in buffer<i32>",
-    "f32",
-    "i64",
-    "i64",
-    "i64",
-    "i64"
-   ],
-   "impl": {
-    "steps": [
-     {
-      "symbol": "fused_sigmoid_gating_delta_rule_update_kernel",
-      "params": [
-       "in buffer<f32>",
-       "in buffer<bf16>",
-       "in buffer<bf16>",
-       "in buffer<bf16>",
-       "f32",
-       "f32",
-       "in buffer<bf16>",
-       "in buffer<bf16>",
-       "in buffer<bf16>",
-       "out buffer<bf16>",
-       "inout ptr",
-       "inout ptr",
-       "in buffer<i32>",
-       "in buffer<i32>",
-       "in buffer<i32>",
-       "f32",
-       "i64",
-       "i64",
-       "i64",
-       "i64"
-      ],
-      "block": [
-       128,
-       1,
-       1
-      ],
-      "grid": [
-       1,
-       4,
-       48
-      ],
-      "args": [
-       {
-        "arg": 0
        },
        {
-        "arg": 1
+        "arg": 13
        },
        {
-        "arg": 2
+        "arg": 14
        },
        {
-        "arg": 3
+        "arg": 15
        },
        {
-        "arg": 4
+        "arg": 16
        },
        {
-        "arg": 5
+        "arg": 17
        },
        {
-        "arg": 6
+        "arg": 18
        },
        {
-        "arg": 7
+        "arg": 19
        },
        {
-        "arg": 8
+        "arg": 20
        },
        {
-        "arg": 9
+        "scratch": "xcnt"
        },
        {
-        "arg": 10
+        "scratch": "hpart"
        },
        {
-        "arg": 11
+        "scratch": "hcnt"
        },
        {
-        "arg": 12
+        "scratch": "gcnt"
        },
        {
-        "arg": 13
+        "arg": 21
        },
        {
-        "arg": 14
+        "arg": 22
        },
        {
-        "arg": 15
+        "arg": 23
        },
        {
-        "arg": 16
+        "arg": 24
        },
        {
-        "arg": 17
+        "arg": 25
        },
        {
-        "arg": 18
+        "arg": 26
        },
        {
-        "arg": 19
+        "arg": 27
        }
       ],
-      "shared_mem": 512,
-      "cubin": "786e2578f9bd.cubin",
-      "sha256": "786e2578f9bd5555dbb6babb53c89016aea1a1c2752f1bb0ad76b9904ad3330a"
+      "cubin": "gdn_spec.cubin"
      }
     ]
    }
@@ -11718,67 +11842,49 @@
      ]
     },
     {
-     "label": "l3.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l3.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
       {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.3.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "buf": "k_n"
       },
       {
-       "i32": 24
+       "buf": "qkv"
       },
       {
-       "i32": 14336
+       "buf": "model.layers.3.self_attn.q_norm.weight_p1"
       },
       {
-       "i32": 512
+       "buf": "model.layers.3.self_attn.k_norm.weight_p1"
       },
       {
-       "i32": 6144
+       "buf": "cos_g"
       },
       {
-       "i32": 256
+       "buf": "sin_g"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l3.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
+       "state": "kv"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
+       "state": "kv",
+       "offset": 512
       },
       {
-       "buf": "model.layers.3.self_attn.k_norm.weight_p1"
+       "buf": "slot_mapping"
       },
       {
-       "i32": 256
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
        "expr": {
@@ -11790,107 +11896,20 @@
         ]
        }
       },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l3.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l3.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv"
-      },
-      {
-       "state": "kv",
-       "offset": 512
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -14052,107 +14071,23 @@
      ]
     },
     {
-     "label": "l7.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l7.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.7.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l7.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
       {
        "buf": "k_n"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.7.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
+       "buf": "qkv"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l7.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "model.layers.7.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "k_n"
+       "buf": "model.layers.7.self_attn.k_norm.weight_p1"
       },
       {
        "buf": "cos_g"
@@ -14160,28 +14095,6 @@
       {
        "buf": "sin_g"
       },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l7.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
       {
        "state": "kv",
        "offset": 3211264
@@ -14194,38 +14107,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -16364,107 +16278,23 @@
      ]
     },
     {
-     "label": "l11.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l11.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.11.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l11.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
       {
        "buf": "k_n"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.11.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
+       "buf": "qkv"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l11.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "model.layers.11.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "k_n"
+       "buf": "model.layers.11.self_attn.k_norm.weight_p1"
       },
       {
        "buf": "cos_g"
@@ -16472,28 +16302,6 @@
       {
        "buf": "sin_g"
       },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l11.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
       {
        "state": "kv",
        "offset": 6422528
@@ -16506,38 +16314,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -18676,107 +18485,23 @@
      ]
     },
     {
-     "label": "l15.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l15.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.15.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l15.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
       {
        "buf": "k_n"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.15.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
+       "buf": "qkv"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l15.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "model.layers.15.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "k_n"
+       "buf": "model.layers.15.self_attn.k_norm.weight_p1"
       },
       {
        "buf": "cos_g"
@@ -18784,28 +18509,6 @@
       {
        "buf": "sin_g"
       },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l15.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
       {
        "state": "kv",
        "offset": 9633792
@@ -18818,38 +18521,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -20988,107 +20692,23 @@
      ]
     },
     {
-     "label": "l19.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l19.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.19.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l19.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
       {
        "buf": "k_n"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.19.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
+       "buf": "qkv"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l19.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "model.layers.19.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "k_n"
+       "buf": "model.layers.19.self_attn.k_norm.weight_p1"
       },
       {
        "buf": "cos_g"
@@ -21096,28 +20716,6 @@
       {
        "buf": "sin_g"
       },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l19.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
       {
        "state": "kv",
        "offset": 12845056
@@ -21130,38 +20728,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -23324,107 +22923,23 @@
      ]
     },
     {
-     "label": "l23.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l23.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.23.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l23.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
       {
        "buf": "k_n"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.23.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
+       "buf": "qkv"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l23.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "model.layers.23.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "k_n"
+       "buf": "model.layers.23.self_attn.k_norm.weight_p1"
       },
       {
        "buf": "cos_g"
@@ -23432,28 +22947,6 @@
       {
        "buf": "sin_g"
       },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l23.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
       {
        "state": "kv",
        "offset": 16056320
@@ -23466,38 +22959,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -25636,107 +25130,23 @@
      ]
     },
     {
-     "label": "l27.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l27.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.27.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l27.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
       {
        "buf": "k_n"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.27.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
+       "buf": "qkv"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l27.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "model.layers.27.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "k_n"
+       "buf": "model.layers.27.self_attn.k_norm.weight_p1"
       },
       {
        "buf": "cos_g"
@@ -25744,28 +25154,6 @@
       {
        "buf": "sin_g"
       },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l27.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
       {
        "state": "kv",
        "offset": 19267584
@@ -25778,38 +25166,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -27948,107 +27337,23 @@
      ]
     },
     {
-     "label": "l31.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l31.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.31.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l31.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
       {
        "buf": "k_n"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.31.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
+       "buf": "qkv"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l31.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "model.layers.31.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "k_n"
+       "buf": "model.layers.31.self_attn.k_norm.weight_p1"
       },
       {
        "buf": "cos_g"
@@ -28056,28 +27361,6 @@
       {
        "buf": "sin_g"
       },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l31.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
       {
        "state": "kv",
        "offset": 22478848
@@ -28090,38 +27373,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -30284,107 +29568,23 @@
      ]
     },
     {
-     "label": "l35.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l35.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.35.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l35.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
       {
        "buf": "k_n"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.35.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
+       "buf": "qkv"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l35.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "model.layers.35.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "k_n"
+       "buf": "model.layers.35.self_attn.k_norm.weight_p1"
       },
       {
        "buf": "cos_g"
@@ -30392,28 +29592,6 @@
       {
        "buf": "sin_g"
       },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l35.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
       {
        "state": "kv",
        "offset": 25690112
@@ -30426,38 +29604,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -32596,107 +31775,23 @@
      ]
     },
     {
-     "label": "l39.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l39.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.39.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l39.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
       {
        "buf": "k_n"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.39.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
+       "buf": "qkv"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l39.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "model.layers.39.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "k_n"
+       "buf": "model.layers.39.self_attn.k_norm.weight_p1"
       },
       {
        "buf": "cos_g"
@@ -32704,28 +31799,6 @@
       {
        "buf": "sin_g"
       },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l39.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
       {
        "state": "kv",
        "offset": 28901376
@@ -32738,38 +31811,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -34908,107 +33982,23 @@
      ]
     },
     {
-     "label": "l43.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l43.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.43.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l43.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
       {
        "buf": "k_n"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.43.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
+       "buf": "qkv"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l43.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "model.layers.43.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "k_n"
+       "buf": "model.layers.43.self_attn.k_norm.weight_p1"
       },
       {
        "buf": "cos_g"
@@ -35016,28 +34006,6 @@
       {
        "buf": "sin_g"
       },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l43.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
       {
        "state": "kv",
        "offset": 32112640
@@ -35050,38 +34018,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -37220,107 +36189,23 @@
      ]
     },
     {
-     "label": "l47.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l47.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.47.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l47.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
       {
        "buf": "k_n"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.47.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
+       "buf": "qkv"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l47.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "model.layers.47.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "k_n"
+       "buf": "model.layers.47.self_attn.k_norm.weight_p1"
       },
       {
        "buf": "cos_g"
@@ -37328,28 +36213,6 @@
       {
        "buf": "sin_g"
       },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l47.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
       {
        "state": "kv",
        "offset": 35323904
@@ -37362,38 +36225,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -39556,107 +38420,23 @@
      ]
     },
     {
-     "label": "l51.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l51.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.51.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l51.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
       {
        "buf": "k_n"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.51.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
+       "buf": "qkv"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l51.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "model.layers.51.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "k_n"
+       "buf": "model.layers.51.self_attn.k_norm.weight_p1"
       },
       {
        "buf": "cos_g"
@@ -39664,28 +38444,6 @@
       {
        "buf": "sin_g"
       },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l51.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
       {
        "state": "kv",
        "offset": 38535168
@@ -39698,38 +38456,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -41868,107 +40627,23 @@
      ]
     },
     {
-     "label": "l55.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l55.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.55.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l55.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
       {
        "buf": "k_n"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.55.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
+       "buf": "qkv"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l55.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "model.layers.55.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "k_n"
+       "buf": "model.layers.55.self_attn.k_norm.weight_p1"
       },
       {
        "buf": "cos_g"
@@ -41976,28 +40651,6 @@
       {
        "buf": "sin_g"
       },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l55.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
       {
        "state": "kv",
        "offset": 41746432
@@ -42010,38 +40663,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -44180,107 +42834,23 @@
      ]
     },
     {
-     "label": "l59.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l59.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.59.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l59.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
       {
        "buf": "k_n"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.59.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
+       "buf": "qkv"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l59.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "model.layers.59.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "k_n"
+       "buf": "model.layers.59.self_attn.k_norm.weight_p1"
       },
       {
        "buf": "cos_g"
@@ -44288,28 +42858,6 @@
       {
        "buf": "sin_g"
       },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l59.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
       {
        "state": "kv",
        "offset": 44957696
@@ -44322,38 +42870,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -46516,107 +45065,23 @@
      ]
     },
     {
-     "label": "l63.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l63.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.63.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l63.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
       {
        "buf": "k_n"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.63.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
+       "buf": "qkv"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l63.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "model.layers.63.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "k_n"
+       "buf": "model.layers.63.self_attn.k_norm.weight_p1"
       },
       {
        "buf": "cos_g"
@@ -46624,28 +45089,6 @@
       {
        "buf": "sin_g"
       },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l63.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
       {
        "state": "kv",
        "offset": 48168960
@@ -46658,38 +45101,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -47143,44 +45587,29 @@
      ]
     },
     {
-     "label": "l0.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l0.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.0.linear_attn.in_proj_qkvz.weight"
-      },
       {
        "buf": "qkvz"
       },
       {
-       "sym": "tokens"
+       "buf": "ba"
       },
       {
-       "i32": 16384
+       "buf": "x"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l0.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
+       "buf": "model.layers.0.linear_attn.in_proj_qkvz.weight"
       },
       {
        "buf": "model.layers.0.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "ba"
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 16384
       },
       {
        "i32": 96
@@ -47191,8 +45620,8 @@
      ]
     },
     {
-     "label": "l0.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l0.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
        "buf": "qkvz"
@@ -47203,6 +45632,10 @@
       {
        "state": "gdn"
       },
+      {
+       "state": "gdn",
+       "offset": 204800
+      },
       {
        "buf": "gdn.spec_line_index",
        "offset": 4
@@ -47214,47 +45647,22 @@
        "buf": "cu_seqlens_q"
       },
       {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
+       "buf": "gdn.spec_slots"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l0.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
+       "buf": "ba"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.0.linear_attn.A_log"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.0.linear_attn.dt_bias"
       },
       {
-       "buf": "model.layers.0.linear_attn.A_log"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.0.linear_attn.dt_bias"
+       "buf": "model.layers.0.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -47272,360 +45680,328 @@
        "buf": "beta"
       },
       {
-       "i32": 16384
+       "buf": "a_c"
       },
       {
-       "i32": 96
+       "buf": "b_c"
       },
       {
-       "i32": 96
+       "buf": "z_c"
       },
       {
-       "i32": 2048
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 2048
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 6144
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 385
       },
       {
-       "i64": 0
+       "i32": 1703936
       },
       {
-       "i64": 0
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l0.a_copy",
-     "kernel": "copy_rows",
+     "label": "l0.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "core_attn_out"
       },
       {
-       "i32": 48
+       "buf": "model.layers.0.linear_attn.out_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l0.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.0.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "ba"
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 5120
       },
       {
-       "i32": 96
+       "i32": 6144
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l0.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l0.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "model.layers.0.linear_attn.A_log"
+       "buf": "act"
       },
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "b_c"
+       "buf": "model.layers.0.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "model.layers.0.linear_attn.dt_bias"
+       "sym": "tokens"
       },
       {
-       "f32": 1.0
+       "i32": 17408
       },
       {
-       "f32": 20.0
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l0.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "buf": "gdn_q"
+       "buf": "x"
       },
       {
-       "buf": "gdn_k"
+       "buf": "act"
       },
       {
-       "buf": "gdn_v"
+       "buf": "model.layers.0.mlp.down_proj.weight"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "residual"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "model.layers.1.input_layernorm.weight_p1"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "sym": "tokens"
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 5120
       },
       {
-       "buf": "gdn.spec_slots"
+       "i32": 17408
       },
       {
-       "buf": "gdn.one"
-      },
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l1.in_proj",
+     "kernel": "gemm8_dual",
+     "args": [
       {
-       "f32": 0.0883883461356163
+       "buf": "qkvz"
       },
       {
-       "i64": 1
+       "buf": "ba"
       },
       {
-       "sym": "tokens"
+       "buf": "x"
       },
       {
-       "i64": 0
+       "buf": "model.layers.1.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l0.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
+       "buf": "model.layers.1.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 16384
       },
       {
-       "i32": 16384
+       "i32": 96
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l0.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l1.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "qkvz"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.1.linear_attn.conv1d.weight"
       },
       {
-       "buf": "model.layers.0.linear_attn.norm.weight"
+       "state": "gdn"
       },
       {
-       "buf": "z_c"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 128
+       "buf": "gdn.spec_line_index",
+       "offset": 8
       },
       {
-       "i32": 128
+       "buf": "gdn.one"
       },
       {
-       "i32": 128
+       "buf": "cu_seqlens_q"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "buf": "gdn.spec_slots",
+       "offset": 32
       },
       {
-       "f32": 9.999999974752427e-07
+       "buf": "ba"
       },
       {
-       "i64": 0
+       "buf": "model.layers.1.linear_attn.A_log"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l0.out_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "model.layers.1.linear_attn.dt_bias"
+      },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.0.linear_attn.out_proj.weight"
+       "buf": "model.layers.1.linear_attn.norm.weight"
       },
       {
-       "buf": "y"
+       "buf": "gdn_q"
       },
       {
-       "sym": "tokens"
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
+       "buf": "gdn_v"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l0.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "g"
+      },
       {
-       "buf": "x"
+       "buf": "beta"
       },
       {
-       "buf": "y"
+       "buf": "a_c"
       },
       {
-       "buf": "residual"
+       "buf": "b_c"
       },
       {
-       "buf": "model.layers.0.post_attention_layernorm.weight_p1"
+       "buf": "z_c"
       },
       {
-       "i32": 5120
+       "f32": 0.0883883461356163
+      },
+      {
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
       },
       {
-       "f32": 9.999999974752427e-07
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l0.gate_up",
-     "kernel": "gemm",
+     "label": "l1.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.0.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 34816
+       "buf": "model.layers.1.linear_attn.out_proj.weight"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l0.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "residual"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.1.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 17408
+       "sym": "tokens"
       },
       {
-       "i32": 0
+       "i32": 5120
       },
       {
-       "f32": 1.0
+       "i32": 6144
       },
       {
-       "i32": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l0.down_proj",
-     "kernel": "gemm",
+     "label": "l1.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
        "buf": "act"
       },
       {
-       "buf": "model.layers.0.mlp.down_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.1.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 17408
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l0.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l1.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.1.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.1.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.2.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -47634,7 +46010,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -47642,45 +46018,30 @@
      ]
     },
     {
-     "label": "l1.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l2.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.1.linear_attn.in_proj_qkvz.weight"
-      },
       {
        "buf": "qkvz"
       },
       {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
+       "buf": "ba"
       },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l1.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.1.linear_attn.in_proj_ba.weight"
+       "buf": "model.layers.2.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.2.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
+      {
+       "i32": 16384
+      },
       {
        "i32": 96
       },
@@ -47690,21 +46051,25 @@
      ]
     },
     {
-     "label": "l1.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l2.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.1.linear_attn.conv1d.weight"
+       "buf": "model.layers.2.linear_attn.conv1d.weight"
       },
       {
        "state": "gdn"
       },
+      {
+       "state": "gdn",
+       "offset": 204800
+      },
       {
        "buf": "gdn.spec_line_index",
-       "offset": 8
+       "offset": 12
       },
       {
        "buf": "gdn.one"
@@ -47713,47 +46078,23 @@
        "buf": "cu_seqlens_q"
       },
       {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
+       "buf": "gdn.spec_slots",
+       "offset": 64
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l1.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
+       "buf": "ba"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.2.linear_attn.A_log"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.2.linear_attn.dt_bias"
       },
       {
-       "buf": "model.layers.1.linear_attn.A_log"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.1.linear_attn.dt_bias"
+       "buf": "model.layers.2.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -47771,192 +46112,195 @@
        "buf": "beta"
       },
       {
-       "i32": 16384
+       "buf": "a_c"
       },
       {
-       "i32": 96
+       "buf": "b_c"
       },
       {
-       "i32": 96
+       "buf": "z_c"
       },
       {
-       "i32": 2048
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 2048
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 6144
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 385
       },
       {
-       "i64": 0
+       "i32": 1703936
       },
       {
-       "i64": 0
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l1.a_copy",
-     "kernel": "copy_rows",
+     "label": "l2.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "core_attn_out"
       },
       {
-       "i32": 48
+       "buf": "model.layers.2.linear_attn.out_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l1.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.2.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "ba"
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 5120
       },
       {
-       "i32": 96
+       "i32": 6144
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l1.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l2.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "model.layers.1.linear_attn.A_log"
+       "buf": "act"
       },
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "b_c"
+       "buf": "model.layers.2.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "model.layers.1.linear_attn.dt_bias"
+       "sym": "tokens"
       },
       {
-       "f32": 1.0
+       "i32": 17408
       },
       {
-       "f32": 20.0
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l2.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "buf": "gdn_q"
+       "buf": "x"
       },
       {
-       "buf": "gdn_k"
+       "buf": "act"
       },
       {
-       "buf": "gdn_v"
+       "buf": "model.layers.2.mlp.down_proj.weight"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "residual"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "model.layers.3.input_layernorm.weight_p1"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "sym": "tokens"
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 5120
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 32
+       "i32": 17408
       },
       {
-       "buf": "gdn.one"
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l3.qkv_proj",
+     "kernel": "gemm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "model.layers.3.self_attn.qkv_proj.weight"
       },
       {
-       "i64": 1
+       "buf": "qkv"
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l1.z_copy",
-     "kernel": "copy_rows",
+     "label": "l3.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "q_n"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "k_n"
       },
       {
-       "i32": 6144
+       "buf": "qkv"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.3.self_attn.q_norm.weight_p1"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l1.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
+       "buf": "model.layers.3.self_attn.k_norm.weight_p1"
+      },
       {
-       "buf": "core_attn_out"
+       "buf": "cos_g"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "sin_g"
       },
       {
-       "buf": "model.layers.1.linear_attn.norm.weight"
+       "state": "kv"
       },
       {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
+       "state": "kv",
+       "offset": 512
       },
       {
-       "i32": 128
+       "buf": "slot_mapping"
       },
       {
-       "i32": 128
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
        "expr": {
@@ -47964,168 +46308,141 @@
          {
           "sym": "tokens"
          },
-         48
+         4
         ]
        }
       },
       {
-       "f32": 9.999999974752427e-07
+       "i64": 25690112
       },
       {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 0
+       "i64": 512
+      },
+      {
+       "i32": 784
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l1.out_proj",
-     "kernel": "gemm",
+     "label": "l3.attn",
+     "kernel": "attn",
      "args": [
       {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.1.linear_attn.out_proj.weight"
+       "buf": "attn_out"
       },
       {
-       "buf": "y"
+       "buf": "q_n"
       },
       {
-       "sym": "tokens"
+       "state": "kv"
       },
       {
-       "i32": 5120
+       "state": "kv",
+       "offset": 512
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l1.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
+       "buf": "block_table"
       },
       {
-       "buf": "y"
+       "buf": "seq_lens"
       },
       {
-       "buf": "residual"
+       "f32": 0.0625
       },
       {
-       "buf": "model.layers.1.post_attention_layernorm.weight_p1"
+       "buf": "kv_scales"
       },
       {
-       "i32": 5120
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "sym": "tokens"
+       "f32": 1.0
       },
       {
-       "i32": 5120
+       "f32": 0.0
       },
       {
-       "i32": 5120
+       "i64": 8
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l1.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
+       "i64": 6144
       },
       {
-       "buf": "model.layers.1.mlp.gate_up_proj.weight"
+       "i64": 256
       },
       {
-       "buf": "gate_up"
+       "i64": 6144
       },
       {
-       "sym": "tokens"
+       "i64": 256
       },
       {
-       "i32": 34816
+       "i64": 0
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l1.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "seq_lens"
       },
       {
-       "buf": "gate_up"
+       "i64": 25690112
       },
       {
-       "i32": 17408
+       "i64": 2048
       },
       {
-       "i32": 0
+       "i64": 512
       },
       {
-       "f32": 1.0
+       "i64": 25690112
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l1.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "i64": 2048
       },
       {
-       "buf": "model.layers.1.mlp.down_proj.weight"
+       "i64": 512
       },
       {
-       "buf": "y"
+       "buf": "cu_seqlens_q"
       },
       {
-       "sym": "tokens"
+       "i32": 1
       },
       {
-       "i32": 5120
+       "i64": 0
       },
       {
-       "i32": 17408
+       "i64": 0
       }
      ]
     },
     {
-     "label": "l1.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l3.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "buf": "model.layers.2.input_layernorm.weight_p1"
+       "buf": "model.layers.3.self_attn.o_proj.weight"
       },
       {
-       "i32": 5120
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.3.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -48134,7 +46451,16 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
+      },
+      {
+       "i32": 256
+      },
+      {
+       "i32": 14336
+      },
+      {
+       "i32": 512
       },
       {
        "f32": 9.999999974752427e-07
@@ -48142,23 +46468,23 @@
      ]
     },
     {
-     "label": "l2.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l3.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.2.linear_attn.in_proj_qkvz.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.3.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -48166,94 +46492,116 @@
      ]
     },
     {
-     "label": "l2.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l3.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.2.linear_attn.in_proj_ba.weight"
+       "buf": "act"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.3.mlp.down_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 96
+       "buf": "model.layers.4.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l2.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l4.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.2.linear_attn.conv1d.weight"
+       "buf": "ba"
       },
       {
-       "state": "gdn"
+       "buf": "x"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 12
+       "buf": "model.layers.4.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gdn.one"
+       "buf": "model.layers.4.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "cu_seqlens_q"
+       "sym": "tokens"
       },
       {
-       "buf": "qkvz"
+       "i32": 16384
       },
       {
-       "i32": 1
+       "i32": 96
       },
       {
-       "i32": 385
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l4.gdn",
+     "kernel": "gdn_spec",
+     "args": [
+      {
+       "buf": "qkvz"
       },
       {
-       "i64": 16384
+       "buf": "model.layers.4.linear_attn.conv1d.weight"
       },
       {
-       "i64": 16384
+       "state": "gdn"
       },
       {
-       "i64": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l2.post_conv",
-     "kernel": "post_conv",
-     "args": [
+       "buf": "gdn.spec_line_index",
+       "offset": 16
+      },
       {
-       "buf": "qkvz"
+       "buf": "gdn.one"
       },
       {
-       "buf": "ba",
+       "buf": "cu_seqlens_q"
+      },
+      {
+       "buf": "gdn.spec_slots",
        "offset": 96
       },
       {
        "buf": "ba"
       },
       {
-       "buf": "model.layers.2.linear_attn.A_log"
+       "buf": "model.layers.4.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.2.linear_attn.dt_bias"
+       "buf": "model.layers.4.linear_attn.dt_bias"
+      },
+      {
+       "buf": "core_attn_out"
+      },
+      {
+       "buf": "model.layers.4.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -48271,361 +46619,328 @@
        "buf": "beta"
       },
       {
-       "i32": 16384
+       "buf": "a_c"
       },
       {
-       "i32": 96
+       "buf": "b_c"
       },
       {
-       "i32": 96
+       "buf": "z_c"
       },
       {
-       "i32": 2048
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 2048
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 6144
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 385
       },
       {
-       "i64": 0
+       "i32": 1703936
       },
       {
-       "i64": 0
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l2.a_copy",
-     "kernel": "copy_rows",
+     "label": "l4.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "core_attn_out"
       },
       {
-       "i32": 48
+       "buf": "model.layers.4.linear_attn.out_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l2.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.4.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "ba"
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 5120
       },
       {
-       "i32": 96
+       "i32": 6144
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l2.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l4.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "model.layers.2.linear_attn.A_log"
+       "buf": "act"
       },
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "b_c"
+       "buf": "model.layers.4.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "model.layers.2.linear_attn.dt_bias"
+       "sym": "tokens"
       },
       {
-       "f32": 1.0
+       "i32": 17408
       },
       {
-       "f32": 20.0
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l4.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "buf": "gdn_q"
+       "buf": "x"
       },
       {
-       "buf": "gdn_k"
+       "buf": "act"
       },
       {
-       "buf": "gdn_v"
+       "buf": "model.layers.4.mlp.down_proj.weight"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "residual"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "model.layers.5.input_layernorm.weight_p1"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "sym": "tokens"
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 5120
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 64
+       "i32": 17408
       },
       {
-       "buf": "gdn.one"
-      },
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l5.in_proj",
+     "kernel": "gemm8_dual",
+     "args": [
       {
-       "f32": 0.0883883461356163
+       "buf": "qkvz"
       },
       {
-       "i64": 1
+       "buf": "ba"
       },
       {
-       "sym": "tokens"
+       "buf": "x"
       },
       {
-       "i64": 0
+       "buf": "model.layers.5.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l2.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
+       "buf": "model.layers.5.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 16384
       },
       {
-       "i32": 16384
+       "i32": 96
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l2.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l5.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "qkvz"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.5.linear_attn.conv1d.weight"
       },
       {
-       "buf": "model.layers.2.linear_attn.norm.weight"
+       "state": "gdn"
       },
       {
-       "buf": "z_c"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 128
+       "buf": "gdn.spec_line_index",
+       "offset": 20
       },
       {
-       "i32": 128
+       "buf": "gdn.one"
       },
       {
-       "i32": 128
+       "buf": "cu_seqlens_q"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "buf": "gdn.spec_slots",
+       "offset": 128
       },
       {
-       "f32": 9.999999974752427e-07
+       "buf": "ba"
       },
       {
-       "i64": 0
+       "buf": "model.layers.5.linear_attn.A_log"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l2.out_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "model.layers.5.linear_attn.dt_bias"
+      },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.2.linear_attn.out_proj.weight"
+       "buf": "model.layers.5.linear_attn.norm.weight"
       },
       {
-       "buf": "y"
+       "buf": "gdn_q"
       },
       {
-       "sym": "tokens"
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
+       "buf": "gdn_v"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l2.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "g"
+      },
       {
-       "buf": "x"
+       "buf": "beta"
       },
       {
-       "buf": "y"
+       "buf": "a_c"
       },
       {
-       "buf": "residual"
+       "buf": "b_c"
       },
       {
-       "buf": "model.layers.2.post_attention_layernorm.weight_p1"
+       "buf": "z_c"
       },
       {
-       "i32": 5120
+       "f32": 0.0883883461356163
+      },
+      {
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
       },
       {
-       "f32": 9.999999974752427e-07
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l2.gate_up",
-     "kernel": "gemm",
+     "label": "l5.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.2.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 34816
+       "buf": "model.layers.5.linear_attn.out_proj.weight"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l2.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "residual"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.5.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 17408
+       "sym": "tokens"
       },
       {
-       "i32": 0
+       "i32": 5120
       },
       {
-       "f32": 1.0
+       "i32": 6144
       },
       {
-       "i32": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l2.down_proj",
-     "kernel": "gemm",
+     "label": "l5.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
        "buf": "act"
       },
       {
-       "buf": "model.layers.2.mlp.down_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.5.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 17408
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l2.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l5.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.5.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.3.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.6.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -48634,7 +46949,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -48642,23 +46957,32 @@
      ]
     },
     {
-     "label": "l3.qkv_proj",
-     "kernel": "gemm",
+     "label": "l6.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.3.self_attn.qkv_proj.weight"
+       "buf": "model.layers.6.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "qkv"
+       "buf": "model.layers.6.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 14336
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -48666,184 +46990,287 @@
      ]
     },
     {
-     "label": "l3.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l6.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "q_n"
+       "buf": "qkvz"
       },
       {
-       "buf": "qkv"
+       "buf": "model.layers.6.linear_attn.conv1d.weight"
       },
       {
-       "buf": "model.layers.3.self_attn.q_norm.weight_p1"
+       "state": "gdn"
       },
       {
-       "i32": 256
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "buf": "gdn.spec_line_index",
+       "offset": 24
       },
       {
-       "i32": 24
+       "buf": "gdn.one"
       },
       {
-       "i32": 14336
+       "buf": "cu_seqlens_q"
       },
       {
-       "i32": 512
+       "buf": "gdn.spec_slots",
+       "offset": 160
       },
       {
-       "i32": 6144
+       "buf": "ba"
       },
       {
-       "i32": 256
+       "buf": "model.layers.6.linear_attn.A_log"
+      },
+      {
+       "buf": "model.layers.6.linear_attn.dt_bias"
+      },
+      {
+       "buf": "core_attn_out"
+      },
+      {
+       "buf": "model.layers.6.linear_attn.norm.weight"
+      },
+      {
+       "buf": "gdn_q"
+      },
+      {
+       "buf": "gdn_k"
+      },
+      {
+       "buf": "gdn_v"
+      },
+      {
+       "buf": "g"
+      },
+      {
+       "buf": "beta"
+      },
+      {
+       "buf": "a_c"
+      },
+      {
+       "buf": "b_c"
+      },
+      {
+       "buf": "z_c"
+      },
+      {
+       "f32": 0.0883883461356163
       },
       {
        "f32": 9.999999974752427e-07
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 385
+      },
+      {
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l3.k_norm",
-     "kernel": "gemma_norm_khead",
+     "label": "l6.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "k_n"
+       "buf": "x"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
+       "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.3.self_attn.k_norm.weight_p1"
+       "buf": "model.layers.6.linear_attn.out_proj.weight"
       },
       {
-       "i32": 256
+       "buf": "residual"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
+       "buf": "model.layers.6.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 4
+       "sym": "tokens"
       },
       {
-       "i32": 14336
+       "i32": 5120
       },
       {
-       "i32": 256
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l6.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
+     "args": [
+      {
+       "buf": "act"
       },
       {
-       "i32": 1024
+       "buf": "x"
       },
       {
-       "i32": 256
+       "buf": "model.layers.6.mlp.gate_up_proj.weight"
       },
       {
-       "f32": 9.999999974752427e-07
+       "sym": "tokens"
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l3.rope",
-     "kernel": "mrope",
+     "label": "l6.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "q_n"
+       "buf": "x"
       },
       {
-       "buf": "k_n"
+       "buf": "act"
       },
       {
-       "buf": "cos_g"
+       "buf": "model.layers.6.mlp.down_proj.weight"
       },
       {
-       "buf": "sin_g"
+       "buf": "residual"
       },
       {
-       "i32": 0
+       "buf": "model.layers.7.input_layernorm.weight_p1"
       },
       {
-       "i64": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l3.kv_write",
-     "kernel": "reshape_and_cache",
+     "label": "l7.qkv_proj",
+     "kernel": "gemm",
      "args": [
       {
-       "buf": "k_n"
+       "buf": "x"
       },
       {
-       "buf": "qkv",
-       "offset": 26624
+       "buf": "model.layers.7.self_attn.qkv_proj.weight"
       },
       {
-       "state": "kv"
+       "buf": "qkv"
       },
       {
-       "state": "kv",
-       "offset": 512
+       "sym": "tokens"
       },
       {
-       "buf": "slot_mapping"
+       "i32": 14336
       },
       {
-       "buf": "kv_scales"
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l7.prep",
+     "kernel": "attn_prep",
+     "args": [
+      {
+       "buf": "q_n"
       },
       {
-       "buf": "kv_scales",
-       "offset": 4
+       "buf": "k_n"
       },
       {
-       "i64": 1024
+       "buf": "qkv"
       },
       {
-       "i64": 14336
+       "buf": "model.layers.7.self_attn.q_norm.weight_p1"
       },
       {
-       "i64": 25690112
+       "buf": "model.layers.7.self_attn.k_norm.weight_p1"
       },
       {
-       "i64": 512
+       "buf": "cos_g"
       },
       {
-       "i64": 0
+       "buf": "sin_g"
       },
       {
-       "i64": 0
+       "state": "kv",
+       "offset": 3211264
+      },
+      {
+       "state": "kv",
+       "offset": 3211776
+      },
+      {
+       "buf": "slot_mapping"
+      },
+      {
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
+      },
+      {
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
+      },
+      {
+       "i64": 25690112
       },
       {
        "i64": 2048
       },
       {
-       "i64": 0
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l3.attn",
+     "label": "l7.attn",
      "kernel": "attn",
      "args": [
       {
@@ -48853,11 +47280,12 @@
        "buf": "q_n"
       },
       {
-       "state": "kv"
+       "state": "kv",
+       "offset": 3211264
       },
       {
        "state": "kv",
-       "offset": 512
+       "offset": 3211776
       },
       {
        "buf": "block_table"
@@ -48935,11 +47363,11 @@
      ]
     },
     {
-     "label": "l3.gate",
-     "kernel": "sigmoid_mul",
+     "label": "l7.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "gated"
+       "buf": "x"
       },
       {
        "buf": "attn_out"
@@ -48949,7 +47377,22 @@
        "offset": 512
       },
       {
-       "i32": 24
+       "buf": "model.layers.7.self_attn.o_proj.weight"
+      },
+      {
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.7.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 6144
       },
       {
        "i32": 256
@@ -48959,51 +47402,54 @@
       },
       {
        "i32": 512
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l3.o_proj",
-     "kernel": "gemm",
+     "label": "l7.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "gated"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.3.self_attn.o_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.7.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l3.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l7.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.7.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.3.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.8.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -49012,7 +47458,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -49020,23 +47466,32 @@
      ]
     },
     {
-     "label": "l3.gate_up",
-     "kernel": "gemm",
+     "label": "l8.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.3.mlp.gate_up_proj.weight"
+       "buf": "model.layers.8.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.8.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -49044,71 +47499,116 @@
      ]
     },
     {
-     "label": "l3.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l8.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.8.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 28
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l3.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "gdn.one"
+      },
       {
-       "buf": "act"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "model.layers.3.mlp.down_proj.weight"
+       "buf": "gdn.spec_slots",
+       "offset": 192
       },
       {
-       "buf": "y"
+       "buf": "ba"
+      },
+      {
+       "buf": "model.layers.8.linear_attn.A_log"
+      },
+      {
+       "buf": "model.layers.8.linear_attn.dt_bias"
+      },
+      {
+       "buf": "core_attn_out"
+      },
+      {
+       "buf": "model.layers.8.linear_attn.norm.weight"
+      },
+      {
+       "buf": "gdn_q"
+      },
+      {
+       "buf": "gdn_k"
+      },
+      {
+       "buf": "gdn_v"
+      },
+      {
+       "buf": "g"
+      },
+      {
+       "buf": "beta"
+      },
+      {
+       "buf": "a_c"
+      },
+      {
+       "buf": "b_c"
+      },
+      {
+       "buf": "z_c"
+      },
+      {
+       "f32": 0.0883883461356163
+      },
+      {
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 385
       },
       {
-       "i32": 17408
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l3.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l8.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.8.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.4.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.8.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -49117,7 +47617,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -49125,23 +47625,23 @@
      ]
     },
     {
-     "label": "l4.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l8.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.4.linear_attn.in_proj_qkvz.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.8.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -49149,94 +47649,116 @@
      ]
     },
     {
-     "label": "l4.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l8.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.4.linear_attn.in_proj_ba.weight"
+       "buf": "act"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.8.mlp.down_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 96
+       "buf": "model.layers.9.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l4.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l9.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.4.linear_attn.conv1d.weight"
+       "buf": "ba"
       },
       {
-       "state": "gdn"
+       "buf": "x"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 16
+       "buf": "model.layers.9.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gdn.one"
+       "buf": "model.layers.9.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "cu_seqlens_q"
+       "sym": "tokens"
       },
       {
-       "buf": "qkvz"
+       "i32": 16384
       },
       {
-       "i32": 1
+       "i32": 96
       },
       {
-       "i32": 385
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l9.gdn",
+     "kernel": "gdn_spec",
+     "args": [
+      {
+       "buf": "qkvz"
       },
       {
-       "i64": 16384
+       "buf": "model.layers.9.linear_attn.conv1d.weight"
       },
       {
-       "i64": 16384
+       "state": "gdn"
       },
       {
-       "i64": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l4.post_conv",
-     "kernel": "post_conv",
-     "args": [
+       "buf": "gdn.spec_line_index",
+       "offset": 32
+      },
       {
-       "buf": "qkvz"
+       "buf": "gdn.one"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "cu_seqlens_q"
+      },
+      {
+       "buf": "gdn.spec_slots",
+       "offset": 224
       },
       {
        "buf": "ba"
       },
       {
-       "buf": "model.layers.4.linear_attn.A_log"
+       "buf": "model.layers.9.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.4.linear_attn.dt_bias"
+       "buf": "model.layers.9.linear_attn.dt_bias"
+      },
+      {
+       "buf": "core_attn_out"
+      },
+      {
+       "buf": "model.layers.9.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -49254,361 +47776,112 @@
        "buf": "beta"
       },
       {
-       "i32": 16384
+       "buf": "a_c"
       },
       {
-       "i32": 96
+       "buf": "b_c"
       },
       {
-       "i32": 96
+       "buf": "z_c"
       },
       {
-       "i32": 2048
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 2048
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 6144
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 385
       },
       {
-       "i64": 0
+       "i32": 1703936
       },
       {
-       "i64": 0
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l4.a_copy",
-     "kernel": "copy_rows",
+     "label": "l9.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "core_attn_out"
       },
       {
-       "i32": 48
+       "buf": "model.layers.9.linear_attn.out_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l4.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.9.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "ba"
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 5120
       },
       {
-       "i32": 96
+       "i32": 6144
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l4.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l9.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "model.layers.4.linear_attn.A_log"
+       "buf": "act"
       },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.4.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 96
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l4.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l4.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.4.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l4.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.4.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l4.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.4.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
+       "buf": "model.layers.9.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l4.gate_up",
-     "kernel": "gemm",
+     "label": "l9.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
-      {
-       "buf": "model.layers.4.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l4.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l4.down_proj",
-     "kernel": "gemm",
-     "args": [
       {
        "buf": "act"
       },
       {
-       "buf": "model.layers.4.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l4.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
+       "buf": "model.layers.9.mlp.down_proj.weight"
       },
       {
        "buf": "residual"
       },
       {
-       "buf": "model.layers.5.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
+       "buf": "model.layers.10.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -49617,7 +47890,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -49625,45 +47898,30 @@
      ]
     },
     {
-     "label": "l5.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l10.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.5.linear_attn.in_proj_qkvz.weight"
-      },
       {
        "buf": "qkvz"
       },
       {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
+       "buf": "ba"
       },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l5.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.5.linear_attn.in_proj_ba.weight"
+       "buf": "model.layers.10.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.10.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
+      {
+       "i32": 16384
+      },
       {
        "i32": 96
       },
@@ -49673,21 +47931,25 @@
      ]
     },
     {
-     "label": "l5.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l10.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.5.linear_attn.conv1d.weight"
+       "buf": "model.layers.10.linear_attn.conv1d.weight"
       },
       {
        "state": "gdn"
       },
+      {
+       "state": "gdn",
+       "offset": 204800
+      },
       {
        "buf": "gdn.spec_line_index",
-       "offset": 20
+       "offset": 36
       },
       {
        "buf": "gdn.one"
@@ -49696,47 +47958,23 @@
        "buf": "cu_seqlens_q"
       },
       {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
+       "buf": "gdn.spec_slots",
+       "offset": 256
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l5.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
+       "buf": "ba"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.10.linear_attn.A_log"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.10.linear_attn.dt_bias"
       },
       {
-       "buf": "model.layers.5.linear_attn.A_log"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.5.linear_attn.dt_bias"
+       "buf": "model.layers.10.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -49753,85 +47991,6 @@
       {
        "buf": "beta"
       },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l5.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l5.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l5.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.5.linear_attn.A_log"
-      },
       {
        "buf": "a_c"
       },
@@ -49839,48025 +47998,106 @@
        "buf": "b_c"
       },
       {
-       "buf": "model.layers.5.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 128
-      },
-      {
-       "buf": "gdn.one"
+       "buf": "z_c"
       },
       {
        "f32": 0.0883883461356163
       },
       {
-       "i64": 1
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l5.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
+       "i32": 385
       },
       {
-       "i32": 6144
+       "i32": 1703936
       },
       {
-       "i32": 16384
+       "i32": 10
       },
       {
-       "i32": 6144
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l5.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l10.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.5.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l5.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
+       "buf": "model.layers.10.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.5.linear_attn.out_proj.weight"
+       "buf": "residual"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.10.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l5.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.5.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l5.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.5.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l5.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l5.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.5.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l5.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.6.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l6.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.6.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l6.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.6.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l6.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.6.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 24
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l6.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.6.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.6.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l6.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l6.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l6.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.6.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.6.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 160
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l6.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l6.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.6.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l6.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.6.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l6.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.6.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l6.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.6.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l6.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l6.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.6.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l6.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.7.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l7.qkv_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.7.self_attn.qkv_proj.weight"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l7.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.7.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l7.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.7.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l7.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l7.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv",
-       "offset": 3211264
-      },
-      {
-       "state": "kv",
-       "offset": 3211776
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l7.attn",
-     "kernel": "attn",
-     "args": [
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv",
-       "offset": 3211264
-      },
-      {
-       "state": "kv",
-       "offset": 3211776
-      },
-      {
-       "buf": "block_table"
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "f32": 0.0625
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 0.0
-      },
-      {
-       "i64": 8
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 0
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l7.gate",
-     "kernel": "sigmoid_mul",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "qkv",
-       "offset": 512
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l7.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "model.layers.7.self_attn.o_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l7.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.7.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l7.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.7.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l7.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l7.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.7.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l7.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.8.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l8.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.8.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l8.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.8.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l8.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.8.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 28
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l8.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.8.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.8.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l8.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l8.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l8.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.8.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.8.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 192
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l8.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l8.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.8.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l8.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.8.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l8.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.8.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l8.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.8.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l8.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l8.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.8.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l8.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.9.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l9.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.9.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l9.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.9.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l9.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.9.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 32
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l9.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.9.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.9.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l9.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l9.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l9.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.9.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.9.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 224
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l9.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l9.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.9.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l9.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.9.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l9.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.9.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l9.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.9.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l9.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l9.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.9.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l9.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.10.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l10.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.10.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l10.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.10.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l10.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.10.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 36
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l10.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.10.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.10.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l10.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l10.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l10.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.10.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.10.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 256
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l10.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l10.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.10.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l10.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.10.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l10.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.10.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l10.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.10.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l10.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l10.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.10.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l10.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.11.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l11.qkv_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.11.self_attn.qkv_proj.weight"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l11.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.11.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l11.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.11.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l11.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l11.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv",
-       "offset": 6422528
-      },
-      {
-       "state": "kv",
-       "offset": 6423040
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l11.attn",
-     "kernel": "attn",
-     "args": [
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv",
-       "offset": 6422528
-      },
-      {
-       "state": "kv",
-       "offset": 6423040
-      },
-      {
-       "buf": "block_table"
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "f32": 0.0625
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 0.0
-      },
-      {
-       "i64": 8
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 0
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l11.gate",
-     "kernel": "sigmoid_mul",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "qkv",
-       "offset": 512
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l11.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "model.layers.11.self_attn.o_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l11.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.11.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l11.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.11.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l11.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l11.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.11.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l11.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.12.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l12.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.12.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l12.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.12.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l12.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.12.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 40
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l12.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.12.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.12.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l12.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l12.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l12.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.12.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.12.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 288
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l12.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l12.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.12.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l12.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.12.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l12.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.12.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l12.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.12.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l12.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l12.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.12.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l12.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.13.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l13.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.13.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l13.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.13.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l13.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.13.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 44
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l13.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.13.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.13.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l13.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l13.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l13.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.13.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.13.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 320
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l13.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l13.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.13.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l13.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.13.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l13.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.13.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l13.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.13.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l13.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l13.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.13.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l13.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.14.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l14.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.14.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l14.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.14.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l14.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.14.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 48
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l14.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.14.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.14.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l14.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l14.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l14.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.14.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.14.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 352
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l14.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l14.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.14.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l14.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.14.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l14.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.14.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l14.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.14.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l14.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l14.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.14.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l14.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.15.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l15.qkv_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.15.self_attn.qkv_proj.weight"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l15.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.15.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l15.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.15.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l15.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l15.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv",
-       "offset": 9633792
-      },
-      {
-       "state": "kv",
-       "offset": 9634304
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l15.attn",
-     "kernel": "attn",
-     "args": [
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv",
-       "offset": 9633792
-      },
-      {
-       "state": "kv",
-       "offset": 9634304
-      },
-      {
-       "buf": "block_table"
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "f32": 0.0625
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 0.0
-      },
-      {
-       "i64": 8
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 0
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l15.gate",
-     "kernel": "sigmoid_mul",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "qkv",
-       "offset": 512
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l15.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "model.layers.15.self_attn.o_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l15.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.15.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l15.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.15.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l15.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l15.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.15.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l15.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.16.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l16.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.16.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l16.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.16.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l16.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.16.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 52
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l16.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.16.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.16.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l16.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l16.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l16.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.16.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.16.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 384
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l16.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l16.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.16.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l16.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.16.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l16.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.16.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l16.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.16.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l16.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l16.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.16.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l16.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.17.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l17.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.17.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l17.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.17.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l17.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.17.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 56
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l17.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.17.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.17.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l17.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l17.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l17.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.17.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.17.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 416
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l17.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l17.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.17.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l17.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.17.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l17.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.17.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l17.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.17.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l17.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l17.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.17.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l17.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.18.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l18.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.18.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l18.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.18.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l18.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.18.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 60
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l18.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.18.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.18.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l18.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l18.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l18.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.18.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.18.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 448
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l18.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l18.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.18.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l18.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.18.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l18.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.18.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l18.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.18.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l18.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l18.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.18.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l18.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.19.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l19.qkv_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.19.self_attn.qkv_proj.weight"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l19.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.19.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l19.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.19.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l19.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l19.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv",
-       "offset": 12845056
-      },
-      {
-       "state": "kv",
-       "offset": 12845568
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l19.attn",
-     "kernel": "attn",
-     "args": [
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv",
-       "offset": 12845056
-      },
-      {
-       "state": "kv",
-       "offset": 12845568
-      },
-      {
-       "buf": "block_table"
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "f32": 0.0625
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 0.0
-      },
-      {
-       "i64": 8
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 0
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l19.gate",
-     "kernel": "sigmoid_mul",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "qkv",
-       "offset": 512
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l19.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "model.layers.19.self_attn.o_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l19.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.19.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l19.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.19.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l19.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l19.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.19.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l19.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.20.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l20.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.20.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l20.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.20.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l20.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.20.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 64
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l20.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.20.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.20.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l20.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l20.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l20.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.20.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.20.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 480
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l20.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l20.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.20.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l20.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.20.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l20.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.20.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l20.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.20.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l20.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l20.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.20.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l20.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.21.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l21.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.21.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l21.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.21.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l21.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.21.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 68
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l21.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.21.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.21.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l21.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l21.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l21.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.21.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.21.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 512
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l21.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l21.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.21.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l21.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.21.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l21.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.21.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l21.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.21.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l21.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l21.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.21.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l21.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.22.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l22.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.22.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l22.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.22.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l22.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.22.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 72
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l22.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.22.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.22.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l22.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l22.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l22.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.22.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.22.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 544
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l22.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l22.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.22.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l22.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.22.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l22.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.22.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l22.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.22.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l22.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l22.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.22.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l22.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.23.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l23.qkv_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.23.self_attn.qkv_proj.weight"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l23.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.23.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l23.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.23.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l23.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l23.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv",
-       "offset": 16056320
-      },
-      {
-       "state": "kv",
-       "offset": 16056832
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l23.attn",
-     "kernel": "attn",
-     "args": [
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv",
-       "offset": 16056320
-      },
-      {
-       "state": "kv",
-       "offset": 16056832
-      },
-      {
-       "buf": "block_table"
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "f32": 0.0625
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 0.0
-      },
-      {
-       "i64": 8
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 0
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l23.gate",
-     "kernel": "sigmoid_mul",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "qkv",
-       "offset": 512
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l23.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "model.layers.23.self_attn.o_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l23.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.23.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l23.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.23.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l23.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l23.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.23.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l23.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.24.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l24.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.24.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l24.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.24.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l24.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.24.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 76
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l24.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.24.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.24.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l24.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l24.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l24.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.24.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.24.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 576
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l24.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l24.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.24.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l24.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.24.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l24.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.24.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l24.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.24.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l24.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l24.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.24.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l24.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.25.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l25.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.25.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l25.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.25.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l25.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.25.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 80
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l25.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.25.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.25.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l25.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l25.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l25.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.25.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.25.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 608
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l25.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l25.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.25.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l25.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.25.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l25.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.25.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l25.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.25.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l25.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l25.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.25.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l25.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.26.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l26.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.26.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l26.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.26.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l26.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.26.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 84
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l26.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.26.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.26.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l26.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l26.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l26.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.26.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.26.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 640
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l26.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l26.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.26.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l26.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.26.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l26.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.26.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l26.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.26.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l26.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l26.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.26.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l26.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.27.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l27.qkv_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.27.self_attn.qkv_proj.weight"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l27.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.27.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l27.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.27.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l27.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l27.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv",
-       "offset": 19267584
-      },
-      {
-       "state": "kv",
-       "offset": 19268096
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l27.attn",
-     "kernel": "attn",
-     "args": [
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv",
-       "offset": 19267584
-      },
-      {
-       "state": "kv",
-       "offset": 19268096
-      },
-      {
-       "buf": "block_table"
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "f32": 0.0625
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 0.0
-      },
-      {
-       "i64": 8
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 0
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l27.gate",
-     "kernel": "sigmoid_mul",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "qkv",
-       "offset": 512
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l27.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "model.layers.27.self_attn.o_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l27.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.27.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l27.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.27.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l27.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l27.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.27.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l27.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.28.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l28.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.28.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l28.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.28.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l28.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.28.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 88
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l28.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.28.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.28.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l28.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l28.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l28.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.28.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.28.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 672
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l28.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l28.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.28.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l28.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.28.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l28.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.28.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l28.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.28.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l28.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l28.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.28.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l28.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.29.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l29.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.29.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l29.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.29.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l29.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.29.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 92
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l29.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.29.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.29.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l29.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l29.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l29.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.29.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.29.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 704
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l29.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l29.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.29.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l29.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.29.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l29.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.29.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l29.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.29.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l29.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l29.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.29.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l29.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.30.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l30.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.30.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l30.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.30.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l30.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.30.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 96
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l30.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.30.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.30.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l30.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l30.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l30.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.30.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.30.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 736
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l30.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l30.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.30.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l30.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.30.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l30.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.30.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l30.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.30.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l30.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l30.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.30.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l30.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.31.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l31.qkv_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.31.self_attn.qkv_proj.weight"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l31.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.31.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l31.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.31.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l31.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l31.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv",
-       "offset": 22478848
-      },
-      {
-       "state": "kv",
-       "offset": 22479360
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l31.attn",
-     "kernel": "attn",
-     "args": [
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv",
-       "offset": 22478848
-      },
-      {
-       "state": "kv",
-       "offset": 22479360
-      },
-      {
-       "buf": "block_table"
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "f32": 0.0625
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 0.0
-      },
-      {
-       "i64": 8
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 0
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l31.gate",
-     "kernel": "sigmoid_mul",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "qkv",
-       "offset": 512
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l31.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "model.layers.31.self_attn.o_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l31.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.31.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l31.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.31.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l31.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l31.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.31.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l31.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.32.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l32.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.32.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l32.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.32.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l32.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.32.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 100
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l32.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.32.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.32.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l32.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l32.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l32.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.32.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.32.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 768
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l32.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l32.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.32.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l32.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.32.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l32.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.32.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l32.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.32.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l32.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l32.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.32.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l32.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.33.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l33.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.33.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l33.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.33.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l33.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.33.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 104
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l33.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.33.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.33.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l33.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l33.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l33.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.33.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.33.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 800
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l33.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l33.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.33.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l33.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.33.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l33.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.33.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l33.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.33.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l33.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l33.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.33.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l33.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.34.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l34.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.34.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l34.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.34.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l34.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.34.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 108
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l34.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.34.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.34.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l34.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l34.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l34.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.34.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.34.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 832
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l34.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l34.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.34.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l34.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.34.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l34.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.34.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l34.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.34.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l34.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l34.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.34.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l34.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.35.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l35.qkv_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.35.self_attn.qkv_proj.weight"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l35.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.35.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l35.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.35.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l35.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l35.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv",
-       "offset": 25690112
-      },
-      {
-       "state": "kv",
-       "offset": 25690624
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l35.attn",
-     "kernel": "attn",
-     "args": [
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv",
-       "offset": 25690112
-      },
-      {
-       "state": "kv",
-       "offset": 25690624
-      },
-      {
-       "buf": "block_table"
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "f32": 0.0625
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 0.0
-      },
-      {
-       "i64": 8
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 0
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l35.gate",
-     "kernel": "sigmoid_mul",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "qkv",
-       "offset": 512
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l35.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "model.layers.35.self_attn.o_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l35.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.35.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l35.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.35.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l35.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l35.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.35.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l35.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.36.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l36.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.36.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l36.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.36.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l36.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.36.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 112
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l36.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.36.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.36.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l36.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l36.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l36.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.36.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.36.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 864
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l36.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l36.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.36.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l36.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.36.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l36.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.36.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l36.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.36.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l36.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l36.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.36.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l36.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.37.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l37.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.37.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l37.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.37.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l37.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.37.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 116
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l37.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.37.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.37.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l37.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l37.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l37.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.37.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.37.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 896
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l37.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l37.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.37.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l37.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.37.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l37.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.37.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l37.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.37.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l37.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l37.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.37.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l37.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.38.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l38.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.38.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l38.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.38.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l38.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.38.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 120
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l38.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.38.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.38.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l38.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l38.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l38.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.38.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.38.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 928
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l38.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l38.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.38.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l38.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.38.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l38.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.38.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l38.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.38.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l38.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l38.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.38.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l38.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.39.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l39.qkv_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.39.self_attn.qkv_proj.weight"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l39.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.39.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l39.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.39.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l39.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l39.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv",
-       "offset": 28901376
-      },
-      {
-       "state": "kv",
-       "offset": 28901888
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l39.attn",
-     "kernel": "attn",
-     "args": [
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv",
-       "offset": 28901376
-      },
-      {
-       "state": "kv",
-       "offset": 28901888
-      },
-      {
-       "buf": "block_table"
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "f32": 0.0625
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 0.0
-      },
-      {
-       "i64": 8
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 0
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l39.gate",
-     "kernel": "sigmoid_mul",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "qkv",
-       "offset": 512
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l39.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "model.layers.39.self_attn.o_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l39.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.39.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l39.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.39.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l39.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l39.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.39.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l39.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.40.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l40.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.40.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l40.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.40.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l40.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.40.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 124
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l40.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.40.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.40.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l40.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l40.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l40.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.40.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.40.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 960
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l40.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l40.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.40.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l40.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.40.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l40.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.40.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l40.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.40.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l40.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l40.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.40.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l40.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.41.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l41.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.41.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l41.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.41.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l41.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.41.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 128
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l41.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.41.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.41.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l41.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l41.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l41.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.41.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.41.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 992
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l41.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l41.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.41.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l41.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.41.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l41.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.41.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l41.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.41.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l41.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l41.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.41.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l41.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.42.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l42.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.42.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l42.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.42.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l42.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.42.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 132
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l42.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.42.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.42.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l42.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l42.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l42.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.42.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.42.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 1024
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l42.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l42.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.42.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l42.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.42.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l42.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.42.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l42.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.42.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l42.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l42.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.42.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l42.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.43.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l43.qkv_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.43.self_attn.qkv_proj.weight"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l43.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.43.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l43.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.43.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l43.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l43.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv",
-       "offset": 32112640
-      },
-      {
-       "state": "kv",
-       "offset": 32113152
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l43.attn",
-     "kernel": "attn",
-     "args": [
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv",
-       "offset": 32112640
-      },
-      {
-       "state": "kv",
-       "offset": 32113152
-      },
-      {
-       "buf": "block_table"
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "f32": 0.0625
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 0.0
-      },
-      {
-       "i64": 8
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 0
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l43.gate",
-     "kernel": "sigmoid_mul",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "qkv",
-       "offset": 512
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l43.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "model.layers.43.self_attn.o_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l43.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.43.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l43.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.43.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l43.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l43.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.43.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l43.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.44.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l44.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.44.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l44.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.44.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l44.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.44.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 136
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l44.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.44.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.44.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l44.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l44.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l44.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.44.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.44.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 1056
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l44.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l44.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.44.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l44.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.44.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l44.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.44.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l44.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.44.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l44.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l44.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.44.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l44.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.45.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l45.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.45.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l45.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.45.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l45.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.45.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 140
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l45.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.45.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.45.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l45.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l45.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l45.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.45.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.45.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 1088
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l45.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l45.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.45.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l45.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.45.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l45.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.45.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l45.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.45.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l45.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l45.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.45.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l45.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.46.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l46.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.46.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l46.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.46.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l46.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.46.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 144
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l46.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.46.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.46.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l46.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l46.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l46.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.46.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.46.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 1120
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l46.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l46.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.46.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l46.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.46.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l46.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.46.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l46.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.46.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l46.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l46.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.46.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l46.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.47.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l47.qkv_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.47.self_attn.qkv_proj.weight"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l47.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.47.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l47.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.47.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l47.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l47.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv",
-       "offset": 35323904
-      },
-      {
-       "state": "kv",
-       "offset": 35324416
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l47.attn",
-     "kernel": "attn",
-     "args": [
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv",
-       "offset": 35323904
-      },
-      {
-       "state": "kv",
-       "offset": 35324416
-      },
-      {
-       "buf": "block_table"
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "f32": 0.0625
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 0.0
-      },
-      {
-       "i64": 8
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 0
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l47.gate",
-     "kernel": "sigmoid_mul",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "qkv",
-       "offset": 512
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l47.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "model.layers.47.self_attn.o_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l47.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.47.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l47.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.47.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l47.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l47.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.47.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l47.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.48.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l48.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.48.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l48.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.48.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l48.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.48.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 148
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l48.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.48.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.48.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l48.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l48.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l48.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.48.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.48.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 1152
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l48.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l48.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.48.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l48.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.48.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l48.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.48.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l48.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.48.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l48.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l48.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.48.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l48.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.49.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l49.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.49.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l49.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.49.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l49.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.49.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 152
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l49.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.49.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.49.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l49.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l49.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l49.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.49.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.49.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 1184
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l49.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l49.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.49.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l49.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.49.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l49.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.49.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l49.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.49.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l49.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l49.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.49.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l49.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.50.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l50.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.50.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l50.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.50.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l50.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.50.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 156
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l50.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.50.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.50.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l50.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l50.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l50.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.50.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.50.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 1216
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l50.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l50.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.50.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l50.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.50.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l50.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.50.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l50.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.50.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l50.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l50.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.50.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l50.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.51.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l51.qkv_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.51.self_attn.qkv_proj.weight"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l51.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.51.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l51.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.51.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l51.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l51.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv",
-       "offset": 38535168
-      },
-      {
-       "state": "kv",
-       "offset": 38535680
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l51.attn",
-     "kernel": "attn",
-     "args": [
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv",
-       "offset": 38535168
-      },
-      {
-       "state": "kv",
-       "offset": 38535680
-      },
-      {
-       "buf": "block_table"
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "f32": 0.0625
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 0.0
-      },
-      {
-       "i64": 8
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 0
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l51.gate",
-     "kernel": "sigmoid_mul",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "qkv",
-       "offset": 512
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l51.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "model.layers.51.self_attn.o_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l51.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.51.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l51.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.51.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l51.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l51.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.51.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l51.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.52.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l52.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.52.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l52.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.52.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l52.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.52.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 160
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l52.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.52.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.52.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l52.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l52.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l52.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.52.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.52.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 1248
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l52.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l52.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.52.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l52.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.52.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l52.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.52.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l52.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.52.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l52.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l52.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.52.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l52.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.53.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l53.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.53.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l53.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.53.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l53.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.53.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 164
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l53.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.53.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.53.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l53.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l53.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l53.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.53.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.53.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 1280
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l53.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l53.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.53.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l53.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.53.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l53.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.53.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l53.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.53.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l53.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l53.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.53.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l53.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.54.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l54.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.54.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l54.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.54.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l54.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.54.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 168
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l54.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.54.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.54.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l54.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l54.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l54.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.54.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.54.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 1312
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l54.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l54.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.54.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l54.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.54.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l54.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.54.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l54.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.54.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l54.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l54.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.54.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l54.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.55.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l55.qkv_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.55.self_attn.qkv_proj.weight"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l55.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.55.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l55.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.55.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l55.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l55.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv",
-       "offset": 41746432
-      },
-      {
-       "state": "kv",
-       "offset": 41746944
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l55.attn",
-     "kernel": "attn",
-     "args": [
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv",
-       "offset": 41746432
-      },
-      {
-       "state": "kv",
-       "offset": 41746944
-      },
-      {
-       "buf": "block_table"
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "f32": 0.0625
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 0.0
-      },
-      {
-       "i64": 8
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 0
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l55.gate",
-     "kernel": "sigmoid_mul",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "qkv",
-       "offset": 512
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l55.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "model.layers.55.self_attn.o_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l55.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.55.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l55.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.55.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l55.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l55.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.55.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l55.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.56.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l56.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.56.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l56.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.56.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l56.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.56.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 172
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l56.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.56.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.56.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l56.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l56.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l56.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.56.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.56.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 1344
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l56.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l56.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.56.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l56.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.56.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l56.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.56.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l56.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.56.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l56.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l56.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.56.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l56.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.57.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l57.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.57.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l57.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.57.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l57.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.57.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 176
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l57.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.57.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.57.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l57.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l57.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l57.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.57.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.57.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 1376
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l57.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l57.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.57.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l57.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.57.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l57.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.57.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l57.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.57.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l57.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l57.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.57.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l57.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.58.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l58.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.58.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l58.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.58.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l58.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.58.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 180
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l58.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.58.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.58.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l58.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l58.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l58.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.58.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.58.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 1408
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l58.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l58.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.58.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l58.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.58.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l58.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.58.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l58.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.58.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l58.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l58.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.58.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l58.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.59.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l59.qkv_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.59.self_attn.qkv_proj.weight"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l59.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.59.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l59.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.59.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l59.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l59.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv",
-       "offset": 44957696
-      },
-      {
-       "state": "kv",
-       "offset": 44958208
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l59.attn",
-     "kernel": "attn",
-     "args": [
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv",
-       "offset": 44957696
-      },
-      {
-       "state": "kv",
-       "offset": 44958208
-      },
-      {
-       "buf": "block_table"
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "f32": 0.0625
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 0.0
-      },
-      {
-       "i64": 8
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 0
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l59.gate",
-     "kernel": "sigmoid_mul",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "qkv",
-       "offset": 512
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l59.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "model.layers.59.self_attn.o_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l59.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.59.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l59.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.59.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l59.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l59.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.59.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l59.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.60.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l60.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.60.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l60.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.60.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l60.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.60.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 184
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l60.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.60.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.60.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l60.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l60.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l60.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.60.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.60.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 1440
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l60.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l60.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.60.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l60.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.60.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l60.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.60.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l60.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.60.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l60.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l60.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.60.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l60.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.61.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l61.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.61.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l61.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.61.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l61.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.61.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 188
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l61.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.61.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.61.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l61.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l61.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l61.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.61.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.61.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 1472
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l61.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l61.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.61.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l61.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.61.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l61.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.61.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l61.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.61.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l61.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l61.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.61.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l61.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.62.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l62.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.62.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l62.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.62.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l62.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.62.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 192
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l62.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.62.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.62.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l62.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l62.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l62.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.62.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.62.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 1504
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l62.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l62.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.62.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l62.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.62.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l62.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.62.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l62.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.62.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l62.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l62.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.62.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l62.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.63.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l63.qkv_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.63.self_attn.qkv_proj.weight"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l63.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.63.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l63.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.63.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l63.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l63.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv",
-       "offset": 48168960
-      },
-      {
-       "state": "kv",
-       "offset": 48169472
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l63.attn",
-     "kernel": "attn",
-     "args": [
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv",
-       "offset": 48168960
-      },
-      {
-       "state": "kv",
-       "offset": 48169472
-      },
-      {
-       "buf": "block_table"
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "f32": 0.0625
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 0.0
-      },
-      {
-       "i64": 8
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 0
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l63.gate",
-     "kernel": "sigmoid_mul",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "qkv",
-       "offset": 512
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l63.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "model.layers.63.self_attn.o_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l63.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.63.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l63.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.63.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l63.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l63.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.63.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l63.final_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.norm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "lm_head",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "lm_head.weight"
-      },
-      {
-       "buf": "logits"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 248320
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "sample",
-     "kernel": "argmax_row",
-     "args": [
-      {
-       "buf": "logits"
-      },
-      {
-       "buf": "next_token"
-      },
-      {
-       "i32": 248320
-      }
-     ]
-    }
-   ]
-  },
-  "decode_spec": {
-   "dispatches": [
-    {
-     "label": "embed",
-     "kernel": "embedding",
-     "args": [
-      {
-       "buf": "token_ids"
-      },
-      {
-       "buf": "model.embed_tokens.weight"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "rope_cos",
-     "kernel": "embedding",
-     "args": [
-      {
-       "buf": "positions"
-      },
-      {
-       "buf": "rope.cos"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 32
-      }
-     ]
-    },
-    {
-     "label": "rope_sin",
-     "kernel": "embedding",
-     "args": [
-      {
-       "buf": "positions"
-      },
-      {
-       "buf": "rope.sin"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 32
-      }
-     ]
-    },
-    {
-     "label": "l0.input_norm",
-     "kernel": "gemma_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.0.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l0.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.0.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l0.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.0.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l0.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.0.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 4
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l0.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.0.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.0.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l0.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l0.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l0.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.0.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.0.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots"
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l0.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l0.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.0.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l0.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.0.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l0.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.0.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l0.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.0.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l0.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l0.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.0.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l0.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.1.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l1.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.1.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l1.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.1.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l1.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.1.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 8
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l1.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.1.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.1.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l1.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l1.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l1.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.1.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.1.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 32
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l1.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l1.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.1.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l1.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.1.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l1.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.1.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l1.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.1.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l1.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l1.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.1.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l1.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.2.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l2.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.2.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l2.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.2.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l2.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.2.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 12
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l2.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.2.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.2.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l2.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l2.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l2.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.2.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.2.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 64
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l2.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l2.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.2.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l2.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.2.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l2.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.2.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l2.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.2.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l2.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l2.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.2.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l2.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.3.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l3.qkv_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.3.self_attn.qkv_proj.weight"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l3.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.3.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l3.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.3.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l3.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l3.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv"
-      },
-      {
-       "state": "kv",
-       "offset": 512
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l3.attn",
-     "kernel": "attn",
-     "args": [
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv"
-      },
-      {
-       "state": "kv",
-       "offset": 512
-      },
-      {
-       "buf": "block_table"
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "f32": 0.0625
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 0.0
-      },
-      {
-       "i64": 8
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 0
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l3.gate",
-     "kernel": "sigmoid_mul",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "qkv",
-       "offset": 512
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l3.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "model.layers.3.self_attn.o_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l3.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.3.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l3.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.3.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l3.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l3.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.3.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l3.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.4.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l4.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.4.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l4.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.4.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l4.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.4.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 16
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l4.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.4.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.4.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l4.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l4.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l4.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.4.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.4.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 96
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l4.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l4.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.4.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l4.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.4.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l4.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.4.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l4.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.4.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l4.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l4.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.4.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l4.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.5.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l5.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.5.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l5.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.5.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l5.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.5.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 20
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l5.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.5.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.5.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l5.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l5.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l5.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.5.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.5.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 128
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l5.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l5.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.5.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l5.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.5.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l5.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.5.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l5.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.5.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l5.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l5.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.5.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l5.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.6.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l5.fc_tap",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "draft.fc.0.weight"
-      },
-      {
-       "buf": "fc_out"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l6.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.6.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l6.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.6.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l6.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.6.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 24
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l6.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.6.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.6.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l6.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l6.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l6.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.6.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.6.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 160
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l6.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l6.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.6.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l6.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.6.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l6.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.6.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l6.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.6.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l6.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l6.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.6.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l6.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.7.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l7.qkv_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.7.self_attn.qkv_proj.weight"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l7.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.7.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l7.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.7.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l7.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l7.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv",
-       "offset": 3211264
-      },
-      {
-       "state": "kv",
-       "offset": 3211776
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l7.attn",
-     "kernel": "attn",
-     "args": [
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv",
-       "offset": 3211264
-      },
-      {
-       "state": "kv",
-       "offset": 3211776
-      },
-      {
-       "buf": "block_table"
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "f32": 0.0625
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 0.0
-      },
-      {
-       "i64": 8
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 0
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l7.gate",
-     "kernel": "sigmoid_mul",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "qkv",
-       "offset": 512
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l7.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "model.layers.7.self_attn.o_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l7.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.7.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l7.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.7.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l7.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l7.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.7.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l7.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.8.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l8.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.8.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l8.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.8.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l8.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.8.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 28
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l8.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.8.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.8.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l8.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l8.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l8.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.8.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.8.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 192
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l8.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l8.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.8.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l8.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.8.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l8.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.8.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l8.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.8.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l8.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l8.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.8.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l8.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.9.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l9.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.9.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l9.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.9.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l9.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.9.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 32
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l9.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.9.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.9.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l9.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l9.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l9.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.9.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.9.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 224
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l9.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l9.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.9.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l9.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.9.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l9.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.9.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l9.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.9.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l9.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l9.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.9.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l9.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.10.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l10.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.10.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l10.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.10.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l10.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.10.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 36
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l10.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.10.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.10.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l10.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l10.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l10.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.10.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.10.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 256
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l10.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l10.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.10.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l10.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.10.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l10.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.10.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l10.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.10.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l10.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l10.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.10.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l10.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.11.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l11.qkv_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.11.self_attn.qkv_proj.weight"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l11.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.11.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l11.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.11.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l11.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l11.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv",
-       "offset": 6422528
-      },
-      {
-       "state": "kv",
-       "offset": 6423040
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l11.attn",
-     "kernel": "attn",
-     "args": [
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv",
-       "offset": 6422528
-      },
-      {
-       "state": "kv",
-       "offset": 6423040
-      },
-      {
-       "buf": "block_table"
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "f32": 0.0625
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 0.0
-      },
-      {
-       "i64": 8
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 0
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l11.gate",
-     "kernel": "sigmoid_mul",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "qkv",
-       "offset": 512
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l11.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "model.layers.11.self_attn.o_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l11.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.11.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l11.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.11.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l11.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l11.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.11.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l11.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.12.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l12.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.12.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l12.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.12.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l12.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.12.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 40
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l12.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.12.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.12.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l12.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l12.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l12.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.12.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.12.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 288
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l12.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l12.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.12.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l12.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.12.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l12.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.12.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l12.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.12.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l12.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l12.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.12.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l12.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.13.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l13.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.13.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l13.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.13.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l13.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.13.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 44
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l13.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.13.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.13.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l13.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l13.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l13.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.13.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.13.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 320
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l13.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l13.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.13.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l13.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.13.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l13.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.13.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l13.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.13.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l13.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l13.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.13.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l13.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.14.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l14.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.14.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l14.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.14.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l14.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.14.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 48
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l14.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.14.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.14.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l14.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l14.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l14.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.14.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.14.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 352
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l14.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l14.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.14.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l14.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.14.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l14.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.14.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l14.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.14.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l14.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l14.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.14.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l14.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.15.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l15.qkv_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.15.self_attn.qkv_proj.weight"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l15.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.15.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l15.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.15.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l15.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l15.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv",
-       "offset": 9633792
-      },
-      {
-       "state": "kv",
-       "offset": 9634304
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l15.attn",
-     "kernel": "attn",
-     "args": [
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv",
-       "offset": 9633792
-      },
-      {
-       "state": "kv",
-       "offset": 9634304
-      },
-      {
-       "buf": "block_table"
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "f32": 0.0625
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 0.0
-      },
-      {
-       "i64": 8
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 0
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l15.gate",
-     "kernel": "sigmoid_mul",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "qkv",
-       "offset": 512
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l15.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "model.layers.15.self_attn.o_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l15.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.15.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l15.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.15.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l15.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l15.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.15.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l15.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.16.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l16.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.16.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l16.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.16.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l16.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.16.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 52
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l16.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.16.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.16.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l16.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l16.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l16.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.16.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.16.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 384
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l16.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l16.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.16.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l16.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.16.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l16.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.16.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l16.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.16.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l16.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l16.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.16.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l16.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.17.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l17.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.17.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l17.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.17.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l17.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.17.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 56
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l17.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.17.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.17.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l17.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l17.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l17.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.17.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.17.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 416
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l17.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l17.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.17.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l17.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.17.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l17.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.17.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l17.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.17.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l17.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l17.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.17.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l17.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.18.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l18.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.18.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l18.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.18.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l18.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.18.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 60
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l18.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.18.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.18.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l18.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l18.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l18.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.18.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.18.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 448
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l18.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l18.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.18.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l18.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.18.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l18.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.18.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l18.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.18.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l18.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l18.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.18.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l18.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.19.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l19.qkv_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.19.self_attn.qkv_proj.weight"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l19.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.19.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l19.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.19.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l19.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l19.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv",
-       "offset": 12845056
-      },
-      {
-       "state": "kv",
-       "offset": 12845568
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l19.attn",
-     "kernel": "attn",
-     "args": [
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv",
-       "offset": 12845056
-      },
-      {
-       "state": "kv",
-       "offset": 12845568
-      },
-      {
-       "buf": "block_table"
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "f32": 0.0625
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 0.0
-      },
-      {
-       "i64": 8
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 0
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l19.gate",
-     "kernel": "sigmoid_mul",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "qkv",
-       "offset": 512
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l19.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "model.layers.19.self_attn.o_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l19.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.19.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l19.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.19.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l19.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l19.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.19.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l19.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.20.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l19.fc_tap",
-     "kernel": "gemm_acc",
-     "args": [
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "draft.fc.1.weight"
-      },
-      {
-       "buf": "fc_out"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l20.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.20.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l20.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.20.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l20.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.20.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 64
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l20.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.20.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.20.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l20.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l20.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l20.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.20.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.20.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 480
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l20.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l20.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.20.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l20.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.20.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l20.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.20.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l20.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.20.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l20.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l20.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.20.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l20.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.21.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l21.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.21.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l21.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.21.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l21.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.21.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 68
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l21.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.21.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.21.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l21.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l21.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l21.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.21.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.21.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 512
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l21.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l21.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.21.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l21.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.21.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l21.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.21.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l21.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.21.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l21.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l21.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.21.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l21.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.22.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l22.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.22.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l22.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.22.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l22.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.22.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 72
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l22.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.22.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.22.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l22.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l22.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l22.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.22.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.22.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 544
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l22.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l22.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.22.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l22.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.22.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l22.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.22.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l22.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.22.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l22.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l22.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.22.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l22.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.23.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l23.qkv_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.23.self_attn.qkv_proj.weight"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l23.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.23.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l23.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.23.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l23.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l23.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv",
-       "offset": 16056320
-      },
-      {
-       "state": "kv",
-       "offset": 16056832
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l23.attn",
-     "kernel": "attn",
-     "args": [
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv",
-       "offset": 16056320
-      },
-      {
-       "state": "kv",
-       "offset": 16056832
-      },
-      {
-       "buf": "block_table"
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "f32": 0.0625
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 0.0
-      },
-      {
-       "i64": 8
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 0
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l23.gate",
-     "kernel": "sigmoid_mul",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "qkv",
-       "offset": 512
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l23.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "model.layers.23.self_attn.o_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l23.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.23.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l23.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.23.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l23.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l23.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.23.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l23.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.24.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l24.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.24.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l24.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.24.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l24.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.24.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 76
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l24.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.24.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.24.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l24.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l24.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l24.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.24.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.24.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 576
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l24.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l24.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.24.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l24.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.24.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l24.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.24.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l24.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.24.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l24.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l24.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.24.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l24.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.25.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l25.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.25.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l25.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.25.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l25.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.25.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 80
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l25.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.25.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.25.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l25.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l25.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l25.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.25.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.25.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 608
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l25.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l25.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.25.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l25.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.25.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l25.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.25.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l25.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.25.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l25.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l25.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.25.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l25.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.26.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l26.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.26.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l26.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.26.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l26.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.26.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 84
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l26.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.26.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.26.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l26.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l26.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l26.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.26.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.26.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 640
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l26.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l26.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.26.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l26.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.26.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l26.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.26.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l26.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.26.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l26.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l26.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.26.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l26.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.27.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l27.qkv_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.27.self_attn.qkv_proj.weight"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l27.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.27.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l27.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.27.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l27.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l27.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv",
-       "offset": 19267584
-      },
-      {
-       "state": "kv",
-       "offset": 19268096
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l27.attn",
-     "kernel": "attn",
-     "args": [
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv",
-       "offset": 19267584
-      },
-      {
-       "state": "kv",
-       "offset": 19268096
-      },
-      {
-       "buf": "block_table"
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "f32": 0.0625
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 0.0
-      },
-      {
-       "i64": 8
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 0
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l27.gate",
-     "kernel": "sigmoid_mul",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "qkv",
-       "offset": 512
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l27.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "model.layers.27.self_attn.o_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l27.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.27.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l27.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.27.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l27.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l27.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.27.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l27.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.28.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l28.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.28.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l28.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.28.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l28.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.28.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 88
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l28.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.28.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.28.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l28.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l28.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l28.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.28.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.28.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 672
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l28.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l28.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.28.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l28.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.28.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l28.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.28.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l28.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.28.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l28.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l28.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.28.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l28.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.29.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l29.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.29.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l29.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.29.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l29.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.29.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 92
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l29.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.29.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.29.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l29.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l29.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l29.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.29.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.29.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 704
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l29.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l29.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.29.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l29.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.29.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l29.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.29.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l29.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.29.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l29.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l29.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.29.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l29.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.30.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l30.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.30.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l30.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.30.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l30.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.30.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 96
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l30.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.30.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.30.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l30.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l30.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l30.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.30.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.30.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 736
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l30.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l30.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.30.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l30.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.30.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l30.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.30.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l30.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.30.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l30.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l30.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.30.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l30.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.31.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l31.qkv_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.31.self_attn.qkv_proj.weight"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l31.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.31.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l31.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.31.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l31.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l31.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv",
-       "offset": 22478848
-      },
-      {
-       "state": "kv",
-       "offset": 22479360
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l31.attn",
-     "kernel": "attn",
-     "args": [
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv",
-       "offset": 22478848
-      },
-      {
-       "state": "kv",
-       "offset": 22479360
-      },
-      {
-       "buf": "block_table"
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "f32": 0.0625
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 0.0
-      },
-      {
-       "i64": 8
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 0
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l31.gate",
-     "kernel": "sigmoid_mul",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "qkv",
-       "offset": 512
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l31.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "model.layers.31.self_attn.o_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l31.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.31.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l31.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.31.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l31.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l31.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.31.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l31.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.32.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l32.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.32.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l32.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.32.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l32.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.32.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 100
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l32.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.32.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.32.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l32.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l32.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l32.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.32.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.32.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 768
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l32.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l32.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.32.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l32.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.32.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l32.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.32.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l32.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.32.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l32.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l32.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.32.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l32.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.33.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l33.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.33.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l33.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.33.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l33.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.33.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 104
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l33.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.33.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.33.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l33.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l33.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l33.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.33.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.33.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 800
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l33.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l33.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.33.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l33.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.33.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l33.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.33.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l33.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.33.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l33.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l33.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.33.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l33.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.34.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l33.fc_tap",
-     "kernel": "gemm_acc",
-     "args": [
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "draft.fc.2.weight"
-      },
-      {
-       "buf": "fc_out"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l34.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.34.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l34.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.34.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l34.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.34.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 108
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l34.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.34.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.34.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l34.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l34.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l34.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.34.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.34.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 832
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l34.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l34.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.34.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l34.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.34.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l34.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.34.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l34.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.34.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l34.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l34.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.34.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l34.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.35.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l35.qkv_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.35.self_attn.qkv_proj.weight"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l35.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.35.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l35.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.35.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l35.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l35.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv",
-       "offset": 25690112
-      },
-      {
-       "state": "kv",
-       "offset": 25690624
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l35.attn",
-     "kernel": "attn",
-     "args": [
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv",
-       "offset": 25690112
-      },
-      {
-       "state": "kv",
-       "offset": 25690624
-      },
-      {
-       "buf": "block_table"
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "f32": 0.0625
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 0.0
-      },
-      {
-       "i64": 8
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 0
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l35.gate",
-     "kernel": "sigmoid_mul",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "qkv",
-       "offset": 512
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l35.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "model.layers.35.self_attn.o_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l35.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.35.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l35.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.35.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l35.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l35.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.35.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l35.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.36.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l36.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.36.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l36.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.36.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l36.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.36.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 112
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l36.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.36.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.36.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l36.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l36.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l36.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.36.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.36.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 864
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l36.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l36.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.36.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l36.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.36.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l36.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.36.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l36.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.36.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l36.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l36.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.36.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l36.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.37.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l37.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.37.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l37.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.37.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l37.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.37.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 116
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l37.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.37.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.37.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l37.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l37.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l37.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.37.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.37.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 896
-      },
-      {
-       "buf": "gdn.one"
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 1
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l37.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l37.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.37.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
-      },
-      {
-       "f32": 9.999999974752427e-07
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 6144
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l37.out_proj",
-     "kernel": "gemm",
+     "label": "l10.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.37.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.10.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l37.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l10.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.10.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.37.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.11.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -97866,7 +48106,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -97874,23 +48114,23 @@
      ]
     },
     {
-     "label": "l37.gate_up",
+     "label": "l11.qkv_proj",
      "kernel": "gemm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.37.mlp.gate_up_proj.weight"
+       "buf": "model.layers.11.self_attn.qkv_proj.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "qkv"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 14336
       },
       {
        "i32": 5120
@@ -97898,80 +48138,72 @@
      ]
     },
     {
-     "label": "l37.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l11.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
+       "buf": "q_n"
       },
       {
-       "i32": 0
+       "buf": "k_n"
       },
       {
-       "f32": 1.0
+       "buf": "qkv"
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l37.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "model.layers.11.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "model.layers.37.mlp.down_proj.weight"
+       "buf": "model.layers.11.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "y"
+       "buf": "cos_g"
       },
       {
-       "sym": "tokens"
+       "buf": "sin_g"
       },
       {
-       "i32": 5120
+       "state": "kv",
+       "offset": 6422528
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l37.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
+       "state": "kv",
+       "offset": 6423040
       },
       {
-       "buf": "y"
+       "buf": "slot_mapping"
       },
       {
-       "buf": "residual"
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "buf": "model.layers.38.input_layernorm.weight_p1"
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i32": 5120
+       "i64": 25690112
       },
       {
-       "sym": "tokens"
+       "i64": 2048
       },
       {
-       "i32": 5120
+       "i64": 512
       },
       {
-       "i32": 5120
+       "i32": 784
       },
       {
        "f32": 9.999999974752427e-07
@@ -97979,90 +48211,89 @@
      ]
     },
     {
-     "label": "l38.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l11.attn",
+     "kernel": "attn",
      "args": [
       {
-       "buf": "x"
+       "buf": "attn_out"
       },
       {
-       "buf": "model.layers.38.linear_attn.in_proj_qkvz.weight"
+       "buf": "q_n"
       },
       {
-       "buf": "qkvz"
+       "state": "kv",
+       "offset": 6422528
       },
       {
-       "sym": "tokens"
+       "state": "kv",
+       "offset": 6423040
       },
       {
-       "i32": 16384
+       "buf": "block_table"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l38.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
+       "buf": "seq_lens"
+      },
       {
-       "buf": "x"
+       "f32": 0.0625
       },
       {
-       "buf": "model.layers.38.linear_attn.in_proj_ba.weight"
+       "buf": "kv_scales"
       },
       {
-       "buf": "ba"
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "sym": "tokens"
+       "f32": 1.0
       },
       {
-       "i32": 96
+       "f32": 0.0
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l38.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
+       "i64": 8
+      },
       {
-       "buf": "qkvz"
+       "i64": 6144
       },
       {
-       "buf": "model.layers.38.linear_attn.conv1d.weight"
+       "i64": 256
       },
       {
-       "state": "gdn"
+       "i64": 6144
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 120
+       "i64": 256
       },
       {
-       "buf": "gdn.one"
+       "i64": 0
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "seq_lens"
       },
       {
-       "buf": "qkvz"
+       "i64": 25690112
       },
       {
-       "i32": 1
+       "i64": 2048
       },
       {
-       "i32": 385
+       "i64": 512
       },
       {
-       "i64": 16384
+       "i64": 25690112
       },
       {
-       "i64": 16384
+       "i64": 2048
+      },
+      {
+       "i64": 512
+      },
+      {
+       "buf": "cu_seqlens_q"
+      },
+      {
+       "i32": 1
       },
       {
        "i64": 0
@@ -98073,291 +48304,252 @@
      ]
     },
     {
-     "label": "l38.post_conv",
-     "kernel": "post_conv",
+     "label": "l11.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "attn_out"
       },
       {
-       "buf": "ba"
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "buf": "model.layers.38.linear_attn.A_log"
+       "buf": "model.layers.11.self_attn.o_proj.weight"
       },
       {
-       "buf": "model.layers.38.linear_attn.dt_bias"
+       "buf": "residual"
       },
       {
-       "buf": "gdn_q"
+       "buf": "model.layers.11.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "gdn_k"
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_v"
+       "i32": 5120
       },
       {
-       "buf": "g"
+       "i32": 6144
       },
       {
-       "buf": "beta"
+       "i32": 256
       },
       {
-       "i32": 16384
+       "i32": 14336
       },
       {
-       "i32": 96
+       "i32": 512
       },
       {
-       "i32": 96
-      },
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l11.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
+     "args": [
       {
-       "i32": 2048
+       "buf": "act"
       },
       {
-       "i32": 2048
+       "buf": "x"
       },
       {
-       "i32": 6144
+       "buf": "model.layers.11.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 17408
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l38.a_copy",
-     "kernel": "copy_rows",
+     "label": "l11.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "act"
       },
       {
-       "i32": 48
+       "buf": "model.layers.11.mlp.down_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l38.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.12.input_layernorm.weight_p1"
       },
       {
-       "buf": "ba"
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 5120
       },
       {
-       "i32": 96
+       "i32": 17408
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l38.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l12.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "model.layers.38.linear_attn.A_log"
+       "buf": "qkvz"
       },
       {
-       "buf": "a_c"
+       "buf": "ba"
       },
       {
-       "buf": "b_c"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.38.linear_attn.dt_bias"
+       "buf": "model.layers.12.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "f32": 1.0
+       "buf": "model.layers.12.linear_attn.in_proj_ba.weight"
       },
       {
-       "f32": 20.0
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_q"
+       "i32": 16384
       },
       {
-       "buf": "gdn_k"
+       "i32": 96
       },
       {
-       "buf": "gdn_v"
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l12.gdn",
+     "kernel": "gdn_spec",
+     "args": [
+      {
+       "buf": "qkvz"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.12.linear_attn.conv1d.weight"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "state": "gdn"
       },
       {
        "state": "gdn",
        "offset": 204800
       },
       {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 928
+       "buf": "gdn.spec_line_index",
+       "offset": 40
       },
       {
        "buf": "gdn.one"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "cu_seqlens_q"
       },
       {
-       "i64": 1
+       "buf": "gdn.spec_slots",
+       "offset": 288
       },
       {
-       "sym": "tokens"
+       "buf": "ba"
       },
       {
-       "i64": 0
+       "buf": "model.layers.12.linear_attn.A_log"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l38.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
+       "buf": "model.layers.12.linear_attn.dt_bias"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "core_attn_out"
       },
       {
-       "i32": 6144
+       "buf": "model.layers.12.linear_attn.norm.weight"
       },
       {
-       "i32": 16384
+       "buf": "gdn_q"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l38.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
+       "buf": "gdn_k"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "gdn_v"
       },
       {
-       "buf": "model.layers.38.linear_attn.norm.weight"
+       "buf": "g"
       },
       {
-       "buf": "z_c"
+       "buf": "beta"
       },
       {
-       "i32": 128
+       "buf": "a_c"
       },
       {
-       "i32": 128
+       "buf": "b_c"
       },
       {
-       "i32": 128
+       "buf": "z_c"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "f32": 0.0883883461356163
       },
       {
        "f32": 9.999999974752427e-07
       },
       {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l38.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.38.linear_attn.out_proj.weight"
+       "sym": "tokens"
       },
       {
-       "buf": "y"
+       "i32": 385
       },
       {
-       "sym": "tokens"
+       "i32": 1703936
       },
       {
-       "i32": 5120
+       "i32": 10
       },
       {
-       "i32": 6144
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l38.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l12.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.12.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.38.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.12.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -98366,7 +48558,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -98374,23 +48566,23 @@
      ]
     },
     {
-     "label": "l38.gate_up",
-     "kernel": "gemm",
+     "label": "l12.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.38.mlp.gate_up_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.12.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -98398,197 +48590,191 @@
      ]
     },
     {
-     "label": "l38.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l12.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
+      {
+       "buf": "x"
+      },
       {
        "buf": "act"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.12.mlp.down_proj.weight"
       },
       {
-       "i32": 17408
+       "buf": "residual"
       },
       {
-       "i32": 0
+       "buf": "model.layers.13.input_layernorm.weight_p1"
       },
       {
-       "f32": 1.0
+       "sym": "tokens"
       },
       {
-       "i32": 0
+       "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l38.down_proj",
-     "kernel": "gemm",
+     "label": "l13.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "model.layers.38.mlp.down_proj.weight"
+       "buf": "ba"
       },
       {
-       "buf": "y"
+       "buf": "x"
+      },
+      {
+       "buf": "model.layers.13.linear_attn.in_proj_qkvz.weight"
+      },
+      {
+       "buf": "model.layers.13.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 16384
       },
       {
-       "i32": 17408
+       "i32": 96
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l38.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l13.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "x"
+       "buf": "qkvz"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.13.linear_attn.conv1d.weight"
       },
       {
-       "buf": "residual"
+       "state": "gdn"
       },
       {
-       "buf": "model.layers.39.input_layernorm.weight_p1"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 5120
+       "buf": "gdn.spec_line_index",
+       "offset": 44
       },
       {
-       "sym": "tokens"
+       "buf": "gdn.one"
       },
       {
-       "i32": 5120
+       "buf": "cu_seqlens_q"
       },
       {
-       "i32": 5120
+       "buf": "gdn.spec_slots",
+       "offset": 320
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l39.qkv_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "ba"
+      },
       {
-       "buf": "x"
+       "buf": "model.layers.13.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.39.self_attn.qkv_proj.weight"
+       "buf": "model.layers.13.linear_attn.dt_bias"
       },
       {
-       "buf": "qkv"
+       "buf": "core_attn_out"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.13.linear_attn.norm.weight"
       },
       {
-       "i32": 14336
+       "buf": "gdn_q"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l39.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
+       "buf": "gdn_k"
+      },
       {
-       "buf": "q_n"
+       "buf": "gdn_v"
       },
       {
-       "buf": "qkv"
+       "buf": "g"
       },
       {
-       "buf": "model.layers.39.self_attn.q_norm.weight_p1"
+       "buf": "beta"
       },
       {
-       "i32": 256
+       "buf": "a_c"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "buf": "b_c"
       },
       {
-       "i32": 24
+       "buf": "z_c"
       },
       {
-       "i32": 14336
+       "f32": 0.0883883461356163
+      },
+      {
+       "f32": 9.999999974752427e-07
+      },
+      {
+       "sym": "tokens"
       },
       {
-       "i32": 512
+       "i32": 385
       },
       {
-       "i32": 6144
+       "i32": 1703936
       },
       {
-       "i32": 256
+       "i32": 10
       },
       {
-       "f32": 9.999999974752427e-07
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l39.k_norm",
-     "kernel": "gemma_norm_khead",
+     "label": "l13.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.39.self_attn.k_norm.weight_p1"
+       "buf": "x"
       },
       {
-       "i32": 256
+       "buf": "core_attn_out"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
+       "buf": "model.layers.13.linear_attn.out_proj.weight"
       },
       {
-       "i32": 4
+       "buf": "residual"
       },
       {
-       "i32": 14336
+       "buf": "model.layers.13.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 256
+       "sym": "tokens"
       },
       {
-       "i32": 1024
+       "i32": 5120
       },
       {
-       "i32": 256
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -98596,253 +48782,263 @@
      ]
     },
     {
-     "label": "l39.rope",
-     "kernel": "mrope",
+     "label": "l13.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
+       "buf": "act"
       },
       {
-       "buf": "cos_g"
+       "buf": "x"
       },
       {
-       "buf": "sin_g"
+       "buf": "model.layers.13.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 17408
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l39.kv_write",
-     "kernel": "reshape_and_cache",
+     "label": "l13.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "k_n"
+       "buf": "x"
       },
       {
-       "buf": "qkv",
-       "offset": 26624
+       "buf": "act"
       },
       {
-       "state": "kv",
-       "offset": 28901376
+       "buf": "model.layers.13.mlp.down_proj.weight"
       },
       {
-       "state": "kv",
-       "offset": 28901888
+       "buf": "residual"
       },
       {
-       "buf": "slot_mapping"
+       "buf": "model.layers.14.input_layernorm.weight_p1"
       },
       {
-       "buf": "kv_scales"
+       "sym": "tokens"
       },
       {
-       "buf": "kv_scales",
-       "offset": 4
+       "i32": 5120
       },
       {
-       "i64": 1024
+       "i32": 17408
       },
       {
-       "i64": 14336
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l14.in_proj",
+     "kernel": "gemm8_dual",
+     "args": [
+      {
+       "buf": "qkvz"
       },
       {
-       "i64": 25690112
+       "buf": "ba"
       },
       {
-       "i64": 512
+       "buf": "x"
       },
       {
-       "i64": 0
+       "buf": "model.layers.14.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i64": 0
+       "buf": "model.layers.14.linear_attn.in_proj_ba.weight"
       },
       {
-       "i64": 2048
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 16384
       },
       {
-       "i64": 0
+       "i32": 96
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l39.attn",
-     "kernel": "attn",
+     "label": "l14.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "attn_out"
+       "buf": "qkvz"
       },
       {
-       "buf": "q_n"
+       "buf": "model.layers.14.linear_attn.conv1d.weight"
       },
       {
-       "state": "kv",
-       "offset": 28901376
+       "state": "gdn"
       },
       {
-       "state": "kv",
-       "offset": 28901888
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "buf": "block_table"
+       "buf": "gdn.spec_line_index",
+       "offset": 48
       },
       {
-       "buf": "seq_lens"
+       "buf": "gdn.one"
       },
       {
-       "f32": 0.0625
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "kv_scales"
+       "buf": "gdn.spec_slots",
+       "offset": 352
       },
       {
-       "buf": "kv_scales",
-       "offset": 4
+       "buf": "ba"
       },
       {
-       "f32": 1.0
+       "buf": "model.layers.14.linear_attn.A_log"
       },
       {
-       "f32": 0.0
+       "buf": "model.layers.14.linear_attn.dt_bias"
       },
       {
-       "i64": 8
+       "buf": "core_attn_out"
       },
       {
-       "i64": 6144
+       "buf": "model.layers.14.linear_attn.norm.weight"
       },
       {
-       "i64": 256
+       "buf": "gdn_q"
       },
       {
-       "i64": 6144
+       "buf": "gdn_k"
       },
       {
-       "i64": 256
+       "buf": "gdn_v"
       },
       {
-       "i64": 0
+       "buf": "g"
       },
       {
-       "buf": "seq_lens"
+       "buf": "beta"
       },
       {
-       "i64": 25690112
+       "buf": "a_c"
       },
       {
-       "i64": 2048
+       "buf": "b_c"
       },
       {
-       "i64": 512
+       "buf": "z_c"
       },
       {
-       "i64": 25690112
+       "f32": 0.0883883461356163
       },
       {
-       "i64": 2048
+       "f32": 9.999999974752427e-07
       },
       {
-       "i64": 512
+       "sym": "tokens"
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 385
       },
       {
-       "i32": 1
+       "i32": 1703936
       },
       {
-       "i64": 0
+       "i32": 10
       },
       {
-       "i64": 0
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l39.gate",
-     "kernel": "sigmoid_mul",
+     "label": "l14.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "gated"
+       "buf": "x"
       },
       {
-       "buf": "attn_out"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "qkv",
-       "offset": 512
+       "buf": "model.layers.14.linear_attn.out_proj.weight"
       },
       {
-       "i32": 24
+       "buf": "residual"
       },
       {
-       "i32": 256
+       "buf": "model.layers.14.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 14336
+       "sym": "tokens"
       },
       {
-       "i32": 512
+       "i32": 5120
+      },
+      {
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l39.o_proj",
-     "kernel": "gemm",
+     "label": "l14.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "gated"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.39.self_attn.o_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.14.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l39.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l14.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.14.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.39.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.15.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -98851,7 +49047,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -98859,23 +49055,23 @@
      ]
     },
     {
-     "label": "l39.gate_up",
+     "label": "l15.qkv_proj",
      "kernel": "gemm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.39.mlp.gate_up_proj.weight"
+       "buf": "model.layers.15.self_attn.qkv_proj.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "qkv"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 14336
       },
       {
        "i32": 5120
@@ -98883,80 +49079,72 @@
      ]
     },
     {
-     "label": "l39.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l15.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
+       "buf": "q_n"
       },
       {
-       "i32": 0
+       "buf": "k_n"
       },
       {
-       "f32": 1.0
+       "buf": "qkv"
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l39.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "model.layers.15.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "model.layers.39.mlp.down_proj.weight"
+       "buf": "model.layers.15.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "y"
+       "buf": "cos_g"
       },
       {
-       "sym": "tokens"
+       "buf": "sin_g"
       },
       {
-       "i32": 5120
+       "state": "kv",
+       "offset": 9633792
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l39.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
+       "state": "kv",
+       "offset": 9634304
       },
       {
-       "buf": "y"
+       "buf": "slot_mapping"
       },
       {
-       "buf": "residual"
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "buf": "model.layers.40.input_layernorm.weight_p1"
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i32": 5120
+       "i64": 25690112
       },
       {
-       "sym": "tokens"
+       "i64": 2048
       },
       {
-       "i32": 5120
+       "i64": 512
       },
       {
-       "i32": 5120
+       "i32": 784
       },
       {
        "f32": 9.999999974752427e-07
@@ -98964,90 +49152,89 @@
      ]
     },
     {
-     "label": "l40.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l15.attn",
+     "kernel": "attn",
      "args": [
       {
-       "buf": "x"
+       "buf": "attn_out"
       },
       {
-       "buf": "model.layers.40.linear_attn.in_proj_qkvz.weight"
+       "buf": "q_n"
       },
       {
-       "buf": "qkvz"
+       "state": "kv",
+       "offset": 9633792
       },
       {
-       "sym": "tokens"
+       "state": "kv",
+       "offset": 9634304
       },
       {
-       "i32": 16384
+       "buf": "block_table"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l40.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
+       "buf": "seq_lens"
+      },
       {
-       "buf": "x"
+       "f32": 0.0625
       },
       {
-       "buf": "model.layers.40.linear_attn.in_proj_ba.weight"
+       "buf": "kv_scales"
       },
       {
-       "buf": "ba"
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "sym": "tokens"
+       "f32": 1.0
       },
       {
-       "i32": 96
+       "f32": 0.0
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l40.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
+       "i64": 8
+      },
       {
-       "buf": "qkvz"
+       "i64": 6144
       },
       {
-       "buf": "model.layers.40.linear_attn.conv1d.weight"
+       "i64": 256
       },
       {
-       "state": "gdn"
+       "i64": 6144
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 124
+       "i64": 256
       },
       {
-       "buf": "gdn.one"
+       "i64": 0
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "seq_lens"
       },
       {
-       "buf": "qkvz"
+       "i64": 25690112
       },
       {
-       "i32": 1
+       "i64": 2048
       },
       {
-       "i32": 385
+       "i64": 512
       },
       {
-       "i64": 16384
+       "i64": 25690112
       },
       {
-       "i64": 16384
+       "i64": 2048
+      },
+      {
+       "i64": 512
+      },
+      {
+       "buf": "cu_seqlens_q"
+      },
+      {
+       "i32": 1
       },
       {
        "i64": 0
@@ -99058,291 +49245,309 @@
      ]
     },
     {
-     "label": "l40.post_conv",
-     "kernel": "post_conv",
+     "label": "l15.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "attn_out"
       },
       {
-       "buf": "ba"
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "buf": "model.layers.40.linear_attn.A_log"
+       "buf": "model.layers.15.self_attn.o_proj.weight"
       },
       {
-       "buf": "model.layers.40.linear_attn.dt_bias"
+       "buf": "residual"
       },
       {
-       "buf": "gdn_q"
+       "buf": "model.layers.15.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "gdn_k"
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_v"
+       "i32": 5120
       },
       {
-       "buf": "g"
+       "i32": 6144
       },
       {
-       "buf": "beta"
+       "i32": 256
       },
       {
-       "i32": 16384
+       "i32": 14336
       },
       {
-       "i32": 96
+       "i32": 512
       },
       {
-       "i32": 96
-      },
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l15.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
+     "args": [
       {
-       "i32": 2048
+       "buf": "act"
       },
       {
-       "i32": 2048
+       "buf": "x"
       },
       {
-       "i32": 6144
+       "buf": "model.layers.15.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 17408
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l40.a_copy",
-     "kernel": "copy_rows",
+     "label": "l15.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "act"
       },
       {
-       "i32": 48
+       "buf": "model.layers.15.mlp.down_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.16.input_layernorm.weight_p1"
       },
       {
-       "i32": 48
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l40.b_copy",
-     "kernel": "copy_rows",
+     "label": "l16.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "b_c"
+       "buf": "qkvz"
       },
       {
        "buf": "ba"
       },
       {
-       "i32": 48
+       "buf": "x"
+      },
+      {
+       "buf": "model.layers.16.linear_attn.in_proj_qkvz.weight"
+      },
+      {
+       "buf": "model.layers.16.linear_attn.in_proj_ba.weight"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 16384
       },
       {
        "i32": 96
       },
       {
-       "i32": 48
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l40.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l16.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "model.layers.40.linear_attn.A_log"
+       "buf": "qkvz"
       },
       {
-       "buf": "a_c"
+       "buf": "model.layers.16.linear_attn.conv1d.weight"
       },
       {
-       "buf": "b_c"
+       "state": "gdn"
       },
       {
-       "buf": "model.layers.40.linear_attn.dt_bias"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 52
       },
       {
-       "f32": 20.0
+       "buf": "gdn.one"
       },
       {
-       "buf": "gdn_q"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "gdn_k"
+       "buf": "gdn.spec_slots",
+       "offset": 384
       },
       {
-       "buf": "gdn_v"
+       "buf": "ba"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.16.linear_attn.A_log"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "model.layers.16.linear_attn.dt_bias"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "core_attn_out"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "model.layers.16.linear_attn.norm.weight"
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 960
+       "buf": "gdn_q"
       },
       {
-       "buf": "gdn.one"
+       "buf": "gdn_k"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "gdn_v"
       },
       {
-       "i64": 1
+       "buf": "g"
       },
       {
-       "sym": "tokens"
+       "buf": "beta"
       },
       {
-       "i64": 0
+       "buf": "a_c"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l40.z_copy",
-     "kernel": "copy_rows",
-     "args": [
+       "buf": "b_c"
+      },
       {
        "buf": "z_c"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 6144
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 16384
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 385
+      },
+      {
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l40.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l16.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.40.linear_attn.norm.weight"
+       "buf": "model.layers.16.linear_attn.out_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.16.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l40.out_proj",
-     "kernel": "gemm",
+     "label": "l16.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.40.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.16.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l40.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l16.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.16.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.40.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.17.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -99351,7 +49556,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -99359,23 +49564,32 @@
      ]
     },
     {
-     "label": "l40.gate_up",
-     "kernel": "gemm",
+     "label": "l17.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.40.mlp.gate_up_proj.weight"
+       "buf": "model.layers.17.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.17.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -99383,71 +49597,116 @@
      ]
     },
     {
-     "label": "l40.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l17.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.17.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 56
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l40.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "gdn.one"
+      },
       {
-       "buf": "act"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "model.layers.40.mlp.down_proj.weight"
+       "buf": "gdn.spec_slots",
+       "offset": 416
       },
       {
-       "buf": "y"
+       "buf": "ba"
+      },
+      {
+       "buf": "model.layers.17.linear_attn.A_log"
+      },
+      {
+       "buf": "model.layers.17.linear_attn.dt_bias"
+      },
+      {
+       "buf": "core_attn_out"
+      },
+      {
+       "buf": "model.layers.17.linear_attn.norm.weight"
+      },
+      {
+       "buf": "gdn_q"
+      },
+      {
+       "buf": "gdn_k"
+      },
+      {
+       "buf": "gdn_v"
+      },
+      {
+       "buf": "g"
+      },
+      {
+       "buf": "beta"
+      },
+      {
+       "buf": "a_c"
+      },
+      {
+       "buf": "b_c"
+      },
+      {
+       "buf": "z_c"
+      },
+      {
+       "f32": 0.0883883461356163
+      },
+      {
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 385
       },
       {
-       "i32": 17408
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l40.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l17.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.17.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.41.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.17.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -99456,7 +49715,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -99464,23 +49723,23 @@
      ]
     },
     {
-     "label": "l41.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l17.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.41.linear_attn.in_proj_qkvz.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.17.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -99488,94 +49747,116 @@
      ]
     },
     {
-     "label": "l41.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l17.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.41.linear_attn.in_proj_ba.weight"
+       "buf": "act"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.17.mlp.down_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 96
+       "buf": "model.layers.18.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l41.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l18.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.41.linear_attn.conv1d.weight"
+       "buf": "ba"
       },
       {
-       "state": "gdn"
+       "buf": "x"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 128
+       "buf": "model.layers.18.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gdn.one"
+       "buf": "model.layers.18.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "cu_seqlens_q"
+       "sym": "tokens"
       },
       {
-       "buf": "qkvz"
+       "i32": 16384
       },
       {
-       "i32": 1
+       "i32": 96
       },
       {
-       "i32": 385
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l18.gdn",
+     "kernel": "gdn_spec",
+     "args": [
+      {
+       "buf": "qkvz"
       },
       {
-       "i64": 16384
+       "buf": "model.layers.18.linear_attn.conv1d.weight"
       },
       {
-       "i64": 16384
+       "state": "gdn"
       },
       {
-       "i64": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l41.post_conv",
-     "kernel": "post_conv",
-     "args": [
+       "buf": "gdn.spec_line_index",
+       "offset": 60
+      },
       {
-       "buf": "qkvz"
+       "buf": "gdn.one"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "cu_seqlens_q"
+      },
+      {
+       "buf": "gdn.spec_slots",
+       "offset": 448
       },
       {
        "buf": "ba"
       },
       {
-       "buf": "model.layers.41.linear_attn.A_log"
+       "buf": "model.layers.18.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.41.linear_attn.dt_bias"
+       "buf": "model.layers.18.linear_attn.dt_bias"
+      },
+      {
+       "buf": "core_attn_out"
+      },
+      {
+       "buf": "model.layers.18.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -99593,192 +49874,196 @@
        "buf": "beta"
       },
       {
-       "i32": 16384
+       "buf": "a_c"
       },
       {
-       "i32": 96
+       "buf": "b_c"
       },
       {
-       "i32": 96
+       "buf": "z_c"
       },
       {
-       "i32": 2048
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 2048
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 6144
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 385
       },
       {
-       "i64": 0
+       "i32": 1703936
       },
       {
-       "i64": 0
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l41.a_copy",
-     "kernel": "copy_rows",
+     "label": "l18.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "core_attn_out"
       },
       {
-       "i32": 48
+       "buf": "model.layers.18.linear_attn.out_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l41.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.18.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "ba"
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 5120
       },
       {
-       "i32": 96
+       "i32": 6144
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l41.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l18.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "model.layers.41.linear_attn.A_log"
+       "buf": "act"
       },
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "b_c"
+       "buf": "model.layers.18.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "model.layers.41.linear_attn.dt_bias"
+       "sym": "tokens"
       },
       {
-       "f32": 1.0
+       "i32": 17408
       },
       {
-       "f32": 20.0
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l18.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "buf": "gdn_q"
+       "buf": "x"
       },
       {
-       "buf": "gdn_k"
+       "buf": "act"
       },
       {
-       "buf": "gdn_v"
+       "buf": "model.layers.18.mlp.down_proj.weight"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "residual"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "model.layers.19.input_layernorm.weight_p1"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "sym": "tokens"
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 5120
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 992
+       "i32": 17408
       },
       {
-       "buf": "gdn.one"
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l19.qkv_proj",
+     "kernel": "gemm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "model.layers.19.self_attn.qkv_proj.weight"
       },
       {
-       "i64": 1
+       "buf": "qkv"
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l41.z_copy",
-     "kernel": "copy_rows",
+     "label": "l19.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "q_n"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "k_n"
       },
       {
-       "i32": 6144
+       "buf": "qkv"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.19.self_attn.q_norm.weight_p1"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l41.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
+       "buf": "model.layers.19.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "cos_g"
       },
       {
-       "buf": "model.layers.41.linear_attn.norm.weight"
+       "buf": "sin_g"
       },
       {
-       "buf": "z_c"
+       "state": "kv",
+       "offset": 12845056
       },
       {
-       "i32": 128
+       "state": "kv",
+       "offset": 12845568
       },
       {
-       "i32": 128
+       "buf": "slot_mapping"
       },
       {
-       "i32": 128
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
        "expr": {
@@ -99786,168 +50071,208 @@
          {
           "sym": "tokens"
          },
-         48
+         4
         ]
        }
       },
       {
-       "f32": 9.999999974752427e-07
+       "i64": 25690112
       },
       {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 0
+       "i64": 512
+      },
+      {
+       "i32": 784
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l41.out_proj",
-     "kernel": "gemm",
+     "label": "l19.attn",
+     "kernel": "attn",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "attn_out"
       },
       {
-       "buf": "model.layers.41.linear_attn.out_proj.weight"
+       "buf": "q_n"
       },
       {
-       "buf": "y"
+       "state": "kv",
+       "offset": 12845056
       },
       {
-       "sym": "tokens"
+       "state": "kv",
+       "offset": 12845568
       },
       {
-       "i32": 5120
+       "buf": "block_table"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l41.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "seq_lens"
+      },
       {
-       "buf": "x"
+       "f32": 0.0625
       },
       {
-       "buf": "y"
+       "buf": "kv_scales"
       },
       {
-       "buf": "residual"
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "buf": "model.layers.41.post_attention_layernorm.weight_p1"
+       "f32": 1.0
       },
       {
-       "i32": 5120
+       "f32": 0.0
       },
       {
-       "sym": "tokens"
+       "i64": 8
       },
       {
-       "i32": 5120
+       "i64": 6144
       },
       {
-       "i32": 5120
+       "i64": 256
       },
       {
-       "f32": 9.999999974752427e-07
+       "i64": 6144
+      },
+      {
+       "i64": 256
+      },
+      {
+       "i64": 0
+      },
+      {
+       "buf": "seq_lens"
+      },
+      {
+       "i64": 25690112
+      },
+      {
+       "i64": 2048
+      },
+      {
+       "i64": 512
+      },
+      {
+       "i64": 25690112
+      },
+      {
+       "i64": 2048
+      },
+      {
+       "i64": 512
+      },
+      {
+       "buf": "cu_seqlens_q"
+      },
+      {
+       "i32": 1
+      },
+      {
+       "i64": 0
+      },
+      {
+       "i64": 0
       }
      ]
     },
     {
-     "label": "l41.gate_up",
-     "kernel": "gemm",
+     "label": "l19.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.41.mlp.gate_up_proj.weight"
+       "buf": "attn_out"
       },
       {
-       "buf": "gate_up"
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.19.self_attn.o_proj.weight"
       },
       {
-       "i32": 34816
+       "buf": "residual"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l41.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
+       "buf": "model.layers.19.post_attention_layernorm.weight_p1"
+      },
       {
-       "buf": "act"
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
       },
       {
-       "buf": "gate_up"
+       "i32": 6144
       },
       {
-       "i32": 17408
+       "i32": 256
       },
       {
-       "i32": 0
+       "i32": 14336
       },
       {
-       "f32": 1.0
+       "i32": 512
       },
       {
-       "i32": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l41.down_proj",
-     "kernel": "gemm",
+     "label": "l19.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
        "buf": "act"
       },
       {
-       "buf": "model.layers.41.mlp.down_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.19.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 17408
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l41.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l19.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.19.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.42.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.20.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -99956,7 +50281,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -99964,45 +50289,30 @@
      ]
     },
     {
-     "label": "l42.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l20.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.42.linear_attn.in_proj_qkvz.weight"
-      },
       {
        "buf": "qkvz"
       },
       {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
+       "buf": "ba"
       },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l42.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.42.linear_attn.in_proj_ba.weight"
+       "buf": "model.layers.20.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.20.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
+      {
+       "i32": 16384
+      },
       {
        "i32": 96
       },
@@ -100012,21 +50322,25 @@
      ]
     },
     {
-     "label": "l42.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l20.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.42.linear_attn.conv1d.weight"
+       "buf": "model.layers.20.linear_attn.conv1d.weight"
       },
       {
        "state": "gdn"
       },
+      {
+       "state": "gdn",
+       "offset": 204800
+      },
       {
        "buf": "gdn.spec_line_index",
-       "offset": 132
+       "offset": 64
       },
       {
        "buf": "gdn.one"
@@ -100035,314 +50349,362 @@
        "buf": "cu_seqlens_q"
       },
       {
-       "buf": "qkvz"
+       "buf": "gdn.spec_slots",
+       "offset": 480
       },
       {
-       "i32": 1
+       "buf": "ba"
       },
       {
-       "i32": 385
+       "buf": "model.layers.20.linear_attn.A_log"
       },
       {
-       "i64": 16384
+       "buf": "model.layers.20.linear_attn.dt_bias"
       },
       {
-       "i64": 16384
+       "buf": "core_attn_out"
       },
       {
-       "i64": 0
+       "buf": "model.layers.20.linear_attn.norm.weight"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l42.post_conv",
-     "kernel": "post_conv",
-     "args": [
+       "buf": "gdn_q"
+      },
       {
-       "buf": "qkvz"
+       "buf": "gdn_k"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "gdn_v"
       },
       {
-       "buf": "ba"
+       "buf": "g"
       },
       {
-       "buf": "model.layers.42.linear_attn.A_log"
+       "buf": "beta"
       },
       {
-       "buf": "model.layers.42.linear_attn.dt_bias"
+       "buf": "a_c"
       },
       {
-       "buf": "gdn_q"
+       "buf": "b_c"
       },
       {
-       "buf": "gdn_k"
+       "buf": "z_c"
       },
       {
-       "buf": "gdn_v"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "g"
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "beta"
+       "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 385
       },
       {
-       "i32": 96
+       "i32": 1703936
       },
       {
-       "i32": 96
+       "i32": 10
       },
       {
-       "i32": 2048
+       "i32": 851968
+      }
+     ]
+    },
+    {
+     "label": "l20.out_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "i32": 2048
+       "buf": "core_attn_out"
       },
       {
-       "i32": 6144
+       "buf": "model.layers.20.linear_attn.out_proj.weight"
+      },
+      {
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.20.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l42.a_copy",
-     "kernel": "copy_rows",
+     "label": "l20.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "act"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "x"
       },
       {
-       "i32": 48
+       "buf": "model.layers.20.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 96
+       "sym": "tokens"
+      },
+      {
+       "i32": 17408
       },
       {
-       "i32": 48
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l42.b_copy",
-     "kernel": "copy_rows",
+     "label": "l20.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "b_c"
+       "buf": "x"
       },
       {
-       "buf": "ba"
+       "buf": "act"
       },
       {
-       "i32": 48
+       "buf": "model.layers.20.mlp.down_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.21.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
       },
       {
-       "i32": 48
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l42.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l21.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "model.layers.42.linear_attn.A_log"
+       "buf": "qkvz"
       },
       {
-       "buf": "a_c"
+       "buf": "ba"
       },
       {
-       "buf": "b_c"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.42.linear_attn.dt_bias"
+       "buf": "model.layers.21.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "f32": 1.0
+       "buf": "model.layers.21.linear_attn.in_proj_ba.weight"
       },
       {
-       "f32": 20.0
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_q"
+       "i32": 16384
       },
       {
-       "buf": "gdn_k"
+       "i32": 96
       },
       {
-       "buf": "gdn_v"
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l21.gdn",
+     "kernel": "gdn_spec",
+     "args": [
+      {
+       "buf": "qkvz"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.21.linear_attn.conv1d.weight"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "state": "gdn"
       },
       {
        "state": "gdn",
        "offset": 204800
       },
+      {
+       "buf": "gdn.spec_line_index",
+       "offset": 68
+      },
+      {
+       "buf": "gdn.one"
+      },
       {
        "buf": "cu_seqlens_q"
       },
       {
        "buf": "gdn.spec_slots",
-       "offset": 1024
+       "offset": 512
       },
       {
-       "buf": "gdn.one"
+       "buf": "ba"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "model.layers.21.linear_attn.A_log"
       },
       {
-       "i64": 1
+       "buf": "model.layers.21.linear_attn.dt_bias"
       },
       {
-       "sym": "tokens"
+       "buf": "core_attn_out"
       },
       {
-       "i64": 0
+       "buf": "model.layers.21.linear_attn.norm.weight"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l42.z_copy",
-     "kernel": "copy_rows",
-     "args": [
+       "buf": "gdn_q"
+      },
+      {
+       "buf": "gdn_k"
+      },
+      {
+       "buf": "gdn_v"
+      },
+      {
+       "buf": "g"
+      },
+      {
+       "buf": "beta"
+      },
+      {
+       "buf": "a_c"
+      },
+      {
+       "buf": "b_c"
+      },
       {
        "buf": "z_c"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 6144
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 16384
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 385
+      },
+      {
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l42.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l21.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.42.linear_attn.norm.weight"
+       "buf": "model.layers.21.linear_attn.out_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.21.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l42.out_proj",
-     "kernel": "gemm",
+     "label": "l21.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.42.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.21.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l42.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l21.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.21.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.42.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.22.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -100351,7 +50713,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -100359,23 +50721,32 @@
      ]
     },
     {
-     "label": "l42.gate_up",
-     "kernel": "gemm",
+     "label": "l22.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.42.mlp.gate_up_proj.weight"
+       "buf": "model.layers.22.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.22.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -100383,197 +50754,182 @@
      ]
     },
     {
-     "label": "l42.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l22.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.22.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 72
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l42.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "gdn.one"
       },
       {
-       "buf": "model.layers.42.mlp.down_proj.weight"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "y"
+       "buf": "gdn.spec_slots",
+       "offset": 544
       },
       {
-       "sym": "tokens"
+       "buf": "ba"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.22.linear_attn.A_log"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l42.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "model.layers.22.linear_attn.dt_bias"
+      },
       {
-       "buf": "x"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.22.linear_attn.norm.weight"
       },
       {
-       "buf": "residual"
+       "buf": "gdn_q"
       },
       {
-       "buf": "model.layers.43.input_layernorm.weight_p1"
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
+       "buf": "gdn_v"
       },
       {
-       "sym": "tokens"
+       "buf": "g"
       },
       {
-       "i32": 5120
+       "buf": "beta"
       },
       {
-       "i32": 5120
+       "buf": "a_c"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l43.qkv_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "b_c"
+      },
       {
-       "buf": "x"
+       "buf": "z_c"
       },
       {
-       "buf": "model.layers.43.self_attn.qkv_proj.weight"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "qkv"
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 14336
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l43.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l22.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.43.self_attn.q_norm.weight_p1"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 256
+       "buf": "model.layers.22.linear_attn.out_proj.weight"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "buf": "residual"
       },
       {
-       "i32": 24
+       "buf": "model.layers.22.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 14336
+       "sym": "tokens"
       },
       {
-       "i32": 512
+       "i32": 5120
       },
       {
        "i32": 6144
       },
-      {
-       "i32": 256
-      },
       {
        "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l43.k_norm",
-     "kernel": "gemma_norm_khead",
+     "label": "l22.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "k_n"
+       "buf": "act"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
+       "buf": "x"
       },
       {
-       "buf": "model.layers.43.self_attn.k_norm.weight_p1"
+       "buf": "model.layers.22.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 256
+       "sym": "tokens"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
+       "i32": 17408
+      },
+      {
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l22.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "i32": 4
+       "buf": "act"
       },
       {
-       "i32": 14336
+       "buf": "model.layers.22.mlp.down_proj.weight"
       },
       {
-       "i32": 256
+       "buf": "residual"
       },
       {
-       "i32": 1024
+       "buf": "model.layers.23.input_layernorm.weight_p1"
       },
       {
-       "i32": 256
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -100581,92 +50937,104 @@
      ]
     },
     {
-     "label": "l43.rope",
-     "kernel": "mrope",
+     "label": "l23.qkv_proj",
+     "kernel": "gemm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
+       "buf": "x"
       },
       {
-       "buf": "cos_g"
+       "buf": "model.layers.23.self_attn.qkv_proj.weight"
       },
       {
-       "buf": "sin_g"
+       "buf": "qkv"
       },
       {
-       "i32": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l43.kv_write",
-     "kernel": "reshape_and_cache",
+     "label": "l23.prep",
+     "kernel": "attn_prep",
      "args": [
+      {
+       "buf": "q_n"
+      },
       {
        "buf": "k_n"
       },
       {
-       "buf": "qkv",
-       "offset": 26624
+       "buf": "qkv"
       },
       {
-       "state": "kv",
-       "offset": 32112640
+       "buf": "model.layers.23.self_attn.q_norm.weight_p1"
       },
       {
-       "state": "kv",
-       "offset": 32113152
+       "buf": "model.layers.23.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "slot_mapping"
+       "buf": "cos_g"
       },
       {
-       "buf": "kv_scales"
+       "buf": "sin_g"
       },
       {
-       "buf": "kv_scales",
-       "offset": 4
+       "state": "kv",
+       "offset": 16056320
       },
       {
-       "i64": 1024
+       "state": "kv",
+       "offset": 16056832
       },
       {
-       "i64": 14336
+       "buf": "slot_mapping"
       },
       {
-       "i64": 25690112
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 512
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i64": 0
+       "i64": 25690112
       },
       {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l43.attn",
+     "label": "l23.attn",
      "kernel": "attn",
      "args": [
       {
@@ -100677,11 +51045,11 @@
       },
       {
        "state": "kv",
-       "offset": 32112640
+       "offset": 16056320
       },
       {
        "state": "kv",
-       "offset": 32113152
+       "offset": 16056832
       },
       {
        "buf": "block_table"
@@ -100759,11 +51127,11 @@
      ]
     },
     {
-     "label": "l43.gate",
-     "kernel": "sigmoid_mul",
+     "label": "l23.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "gated"
+       "buf": "x"
       },
       {
        "buf": "attn_out"
@@ -100773,7 +51141,22 @@
        "offset": 512
       },
       {
-       "i32": 24
+       "buf": "model.layers.23.self_attn.o_proj.weight"
+      },
+      {
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.23.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 6144
       },
       {
        "i32": 256
@@ -100783,51 +51166,54 @@
       },
       {
        "i32": 512
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l43.o_proj",
-     "kernel": "gemm",
+     "label": "l23.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "gated"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.43.self_attn.o_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.23.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l43.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l23.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.23.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.43.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.24.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -100836,7 +51222,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -100844,23 +51230,32 @@
      ]
     },
     {
-     "label": "l43.gate_up",
-     "kernel": "gemm",
+     "label": "l24.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.43.mlp.gate_up_proj.weight"
+       "buf": "model.layers.24.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.24.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -100868,293 +51263,281 @@
      ]
     },
     {
-     "label": "l43.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l24.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.24.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 76
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l43.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "gdn.one"
       },
       {
-       "buf": "model.layers.43.mlp.down_proj.weight"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "y"
+       "buf": "gdn.spec_slots",
+       "offset": 576
       },
       {
-       "sym": "tokens"
+       "buf": "ba"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.24.linear_attn.A_log"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l43.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "model.layers.24.linear_attn.dt_bias"
+      },
       {
-       "buf": "x"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.24.linear_attn.norm.weight"
       },
       {
-       "buf": "residual"
+       "buf": "gdn_q"
       },
       {
-       "buf": "model.layers.44.input_layernorm.weight_p1"
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
+       "buf": "gdn_v"
       },
       {
-       "sym": "tokens"
+       "buf": "g"
       },
       {
-       "i32": 5120
+       "buf": "beta"
       },
       {
-       "i32": 5120
+       "buf": "a_c"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l44.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
+       "buf": "b_c"
+      },
       {
-       "buf": "x"
+       "buf": "z_c"
       },
       {
-       "buf": "model.layers.44.linear_attn.in_proj_qkvz.weight"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "qkvz"
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l44.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l24.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.44.linear_attn.in_proj_ba.weight"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.24.linear_attn.out_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 96
+       "buf": "model.layers.24.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l44.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l24.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.44.linear_attn.conv1d.weight"
+       "buf": "x"
       },
       {
-       "state": "gdn"
+       "buf": "model.layers.24.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 136
+       "sym": "tokens"
       },
       {
-       "buf": "gdn.one"
+       "i32": 17408
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l24.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "buf": "qkvz"
+       "buf": "act"
       },
       {
-       "i32": 1
+       "buf": "model.layers.24.mlp.down_proj.weight"
       },
       {
-       "i32": 385
+       "buf": "residual"
       },
       {
-       "i64": 16384
+       "buf": "model.layers.25.input_layernorm.weight_p1"
       },
       {
-       "i64": 16384
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l44.post_conv",
-     "kernel": "post_conv",
+     "label": "l25.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
        "buf": "qkvz"
       },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
       {
        "buf": "ba"
       },
       {
-       "buf": "model.layers.44.linear_attn.A_log"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.44.linear_attn.dt_bias"
+       "buf": "model.layers.25.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gdn_q"
+       "buf": "model.layers.25.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "gdn_k"
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_v"
+       "i32": 16384
       },
       {
-       "buf": "g"
+       "i32": 96
       },
       {
-       "buf": "beta"
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l25.gdn",
+     "kernel": "gdn_spec",
+     "args": [
       {
-       "i32": 16384
+       "buf": "qkvz"
       },
       {
-       "i32": 96
+       "buf": "model.layers.25.linear_attn.conv1d.weight"
       },
       {
-       "i32": 96
+       "state": "gdn"
       },
       {
-       "i32": 2048
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 2048
+       "buf": "gdn.spec_line_index",
+       "offset": 80
       },
       {
-       "i32": 6144
+       "buf": "gdn.one"
       },
       {
-       "sym": "tokens"
+       "buf": "cu_seqlens_q"
       },
       {
-       "i64": 0
+       "buf": "gdn.spec_slots",
+       "offset": 608
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l44.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
+       "buf": "ba"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.25.linear_attn.A_log"
       },
       {
-       "i32": 48
+       "buf": "model.layers.25.linear_attn.dt_bias"
       },
       {
-       "i32": 96
+       "buf": "core_attn_out"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l44.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.25.linear_attn.norm.weight"
       },
       {
-       "buf": "ba"
+       "buf": "gdn_q"
       },
       {
-       "i32": 48
+       "buf": "gdn_k"
       },
       {
-       "i32": 96
+       "buf": "gdn_v"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l44.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
+       "buf": "g"
+      },
       {
-       "buf": "model.layers.44.linear_attn.A_log"
+       "buf": "beta"
       },
       {
        "buf": "a_c"
@@ -101163,276 +51546,265 @@
        "buf": "b_c"
       },
       {
-       "buf": "model.layers.44.linear_attn.dt_bias"
+       "buf": "z_c"
       },
       {
-       "f32": 1.0
+       "f32": 0.0883883461356163
       },
       {
-       "f32": 20.0
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "gdn_q"
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_k"
+       "i32": 385
       },
       {
-       "buf": "gdn_v"
+       "i32": 1703936
       },
       {
-       "buf": "core_attn_out"
+       "i32": 10
       },
       {
-       "state": "gdn",
-       "offset": 204800
-      },
+       "i32": 851968
+      }
+     ]
+    },
+    {
+     "label": "l25.out_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "x"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 1056
+       "buf": "model.layers.25.linear_attn.out_proj.weight"
       },
       {
-       "buf": "gdn.one"
+       "buf": "residual"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "model.layers.25.post_attention_layernorm.weight_p1"
       },
       {
-       "i64": 1
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 6144
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l44.z_copy",
-     "kernel": "copy_rows",
+     "label": "l25.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "act"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "x"
       },
       {
-       "i32": 6144
+       "buf": "model.layers.25.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 16384
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 17408
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l44.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l25.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.44.linear_attn.norm.weight"
+       "buf": "model.layers.25.mlp.down_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.26.input_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l44.out_proj",
-     "kernel": "gemm",
+     "label": "l26.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "qkvz"
       },
       {
-       "buf": "model.layers.44.linear_attn.out_proj.weight"
+       "buf": "ba"
       },
       {
-       "buf": "y"
+       "buf": "x"
+      },
+      {
+       "buf": "model.layers.26.linear_attn.in_proj_qkvz.weight"
+      },
+      {
+       "buf": "model.layers.26.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 16384
       },
       {
-       "i32": 6144
+       "i32": 96
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l44.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l26.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "x"
+       "buf": "qkvz"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.26.linear_attn.conv1d.weight"
       },
       {
-       "buf": "residual"
+       "state": "gdn"
       },
       {
-       "buf": "model.layers.44.post_attention_layernorm.weight_p1"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 5120
+       "buf": "gdn.spec_line_index",
+       "offset": 84
       },
       {
-       "sym": "tokens"
+       "buf": "gdn.one"
       },
       {
-       "i32": 5120
+       "buf": "cu_seqlens_q"
       },
       {
-       "i32": 5120
+       "buf": "gdn.spec_slots",
+       "offset": 640
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l44.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
+       "buf": "ba"
       },
       {
-       "buf": "model.layers.44.mlp.gate_up_proj.weight"
+       "buf": "model.layers.26.linear_attn.A_log"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.26.linear_attn.dt_bias"
       },
       {
-       "sym": "tokens"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 34816
+       "buf": "model.layers.26.linear_attn.norm.weight"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l44.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
+       "buf": "gdn_q"
+      },
       {
-       "buf": "act"
+       "buf": "gdn_k"
       },
       {
-       "buf": "gate_up"
+       "buf": "gdn_v"
       },
       {
-       "i32": 17408
+       "buf": "g"
       },
       {
-       "i32": 0
+       "buf": "beta"
       },
       {
-       "f32": 1.0
+       "buf": "a_c"
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l44.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "b_c"
+      },
       {
-       "buf": "act"
+       "buf": "z_c"
       },
       {
-       "buf": "model.layers.44.mlp.down_proj.weight"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "y"
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 385
       },
       {
-       "i32": 17408
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l44.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l26.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.26.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.45.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.26.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -101441,7 +51813,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -101449,23 +51821,23 @@
      ]
     },
     {
-     "label": "l45.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l26.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.45.linear_attn.in_proj_qkvz.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.26.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -101473,242 +51845,219 @@
      ]
     },
     {
-     "label": "l45.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l26.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.45.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
+       "buf": "act"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l45.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
+       "buf": "model.layers.26.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.45.linear_attn.conv1d.weight"
+       "buf": "residual"
       },
       {
-       "state": "gdn"
+       "buf": "model.layers.27.input_layernorm.weight_p1"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 140
+       "sym": "tokens"
       },
       {
-       "buf": "gdn.one"
+       "i32": 5120
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 17408
       },
       {
-       "buf": "qkvz"
-      },
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l27.qkv_proj",
+     "kernel": "gemm",
+     "args": [
       {
-       "i32": 1
+       "buf": "x"
       },
       {
-       "i32": 385
+       "buf": "model.layers.27.self_attn.qkv_proj.weight"
       },
       {
-       "i64": 16384
+       "buf": "qkv"
       },
       {
-       "i64": 16384
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l45.post_conv",
-     "kernel": "post_conv",
+     "label": "l27.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
+       "buf": "q_n"
       },
       {
-       "buf": "model.layers.45.linear_attn.A_log"
+       "buf": "k_n"
       },
       {
-       "buf": "model.layers.45.linear_attn.dt_bias"
+       "buf": "qkv"
       },
       {
-       "buf": "gdn_q"
+       "buf": "model.layers.27.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "gdn_k"
+       "buf": "model.layers.27.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "gdn_v"
+       "buf": "cos_g"
       },
       {
-       "buf": "g"
+       "buf": "sin_g"
       },
       {
-       "buf": "beta"
+       "state": "kv",
+       "offset": 19267584
       },
       {
-       "i32": 16384
+       "state": "kv",
+       "offset": 19268096
       },
       {
-       "i32": 96
+       "buf": "slot_mapping"
       },
       {
-       "i32": 96
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i32": 2048
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i32": 2048
+       "i64": 25690112
       },
       {
-       "i32": 6144
+       "i64": 2048
       },
       {
-       "sym": "tokens"
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l45.a_copy",
-     "kernel": "copy_rows",
+     "label": "l27.attn",
+     "kernel": "attn",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "attn_out"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "q_n"
       },
       {
-       "i32": 48
+       "state": "kv",
+       "offset": 19267584
       },
       {
-       "i32": 96
+       "state": "kv",
+       "offset": 19268096
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l45.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "block_table"
       },
       {
-       "buf": "ba"
+       "buf": "seq_lens"
       },
       {
-       "i32": 48
+       "f32": 0.0625
       },
       {
-       "i32": 96
+       "buf": "kv_scales"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l45.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.45.linear_attn.A_log"
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "buf": "a_c"
+       "f32": 1.0
       },
       {
-       "buf": "b_c"
+       "f32": 0.0
       },
       {
-       "buf": "model.layers.45.linear_attn.dt_bias"
+       "i64": 8
       },
       {
-       "f32": 1.0
+       "i64": 6144
       },
       {
-       "f32": 20.0
+       "i64": 256
       },
       {
-       "buf": "gdn_q"
+       "i64": 6144
       },
       {
-       "buf": "gdn_k"
+       "i64": 256
       },
       {
-       "buf": "gdn_v"
+       "i64": 0
       },
       {
-       "buf": "core_attn_out"
+       "buf": "seq_lens"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "i64": 25690112
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "i64": 2048
       },
       {
-       "buf": "cu_seqlens_q"
+       "i64": 512
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 1088
+       "i64": 25690112
       },
       {
-       "buf": "gdn.one"
+       "i64": 2048
       },
       {
-       "f32": 0.0883883461356163
+       "i64": 512
       },
       {
-       "i64": 1
+       "buf": "cu_seqlens_q"
       },
       {
-       "sym": "tokens"
+       "i32": 1
       },
       {
        "i64": 0
@@ -101719,115 +52068,93 @@
      ]
     },
     {
-     "label": "l45.z_copy",
-     "kernel": "copy_rows",
+     "label": "l27.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "x"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "attn_out"
       },
       {
-       "i32": 6144
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "i32": 16384
+       "buf": "model.layers.27.self_attn.o_proj.weight"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l45.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
+       "buf": "residual"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.27.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "model.layers.45.linear_attn.norm.weight"
+       "sym": "tokens"
       },
       {
-       "buf": "z_c"
+       "i32": 5120
       },
       {
-       "i32": 128
+       "i32": 6144
       },
       {
-       "i32": 128
+       "i32": 256
       },
       {
-       "i32": 128
+       "i32": 14336
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 512
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l45.out_proj",
-     "kernel": "gemm",
+     "label": "l27.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.45.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.27.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l45.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l27.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.27.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.45.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.28.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -101836,7 +52163,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -101844,23 +52171,32 @@
      ]
     },
     {
-     "label": "l45.gate_up",
-     "kernel": "gemm",
+     "label": "l28.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.45.mlp.gate_up_proj.weight"
+       "buf": "model.layers.28.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.28.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -101868,308 +52204,266 @@
      ]
     },
     {
-     "label": "l45.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l28.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
+       "buf": "qkvz"
       },
       {
-       "i32": 17408
+       "buf": "model.layers.28.linear_attn.conv1d.weight"
       },
       {
-       "i32": 0
+       "state": "gdn"
       },
       {
-       "f32": 1.0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l45.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "gdn.spec_line_index",
+       "offset": 88
       },
       {
-       "buf": "model.layers.45.mlp.down_proj.weight"
+       "buf": "gdn.one"
       },
       {
-       "buf": "y"
+       "buf": "cu_seqlens_q"
       },
       {
-       "sym": "tokens"
+       "buf": "gdn.spec_slots",
+       "offset": 672
       },
       {
-       "i32": 5120
+       "buf": "ba"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l45.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
+       "buf": "model.layers.28.linear_attn.A_log"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.28.linear_attn.dt_bias"
       },
       {
-       "buf": "residual"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.46.input_layernorm.weight_p1"
+       "buf": "model.layers.28.linear_attn.norm.weight"
       },
       {
-       "i32": 5120
+       "buf": "gdn_q"
       },
       {
-       "sym": "tokens"
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
+       "buf": "gdn_v"
       },
       {
-       "i32": 5120
+       "buf": "g"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l46.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
+       "buf": "beta"
       },
       {
-       "buf": "model.layers.46.linear_attn.in_proj_qkvz.weight"
+       "buf": "a_c"
       },
       {
-       "buf": "qkvz"
+       "buf": "b_c"
       },
       {
-       "sym": "tokens"
+       "buf": "z_c"
       },
       {
-       "i32": 16384
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l46.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "model.layers.46.linear_attn.in_proj_ba.weight"
+       "sym": "tokens"
       },
       {
-       "buf": "ba"
+       "i32": 385
       },
       {
-       "sym": "tokens"
+       "i32": 1703936
       },
       {
-       "i32": 96
+       "i32": 10
       },
       {
-       "i32": 5120
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l46.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l28.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.46.linear_attn.conv1d.weight"
+       "buf": "core_attn_out"
       },
       {
-       "state": "gdn"
+       "buf": "model.layers.28.linear_attn.out_proj.weight"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 144
+       "buf": "residual"
       },
       {
-       "buf": "gdn.one"
+       "buf": "model.layers.28.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "cu_seqlens_q"
+       "sym": "tokens"
       },
       {
-       "buf": "qkvz"
+       "i32": 5120
       },
       {
-       "i32": 1
+       "i32": 6144
       },
       {
-       "i32": 385
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l28.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
+     "args": [
+      {
+       "buf": "act"
       },
       {
-       "i64": 16384
+       "buf": "x"
       },
       {
-       "i64": 16384
+       "buf": "model.layers.28.mlp.gate_up_proj.weight"
       },
       {
-       "i64": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 17408
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l46.post_conv",
-     "kernel": "post_conv",
+     "label": "l28.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
+       "buf": "x"
       },
       {
-       "buf": "ba"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.46.linear_attn.A_log"
+       "buf": "model.layers.28.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.46.linear_attn.dt_bias"
+       "buf": "residual"
       },
       {
-       "buf": "gdn_q"
+       "buf": "model.layers.29.input_layernorm.weight_p1"
       },
       {
-       "buf": "gdn_k"
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_v"
+       "i32": 5120
       },
       {
-       "buf": "g"
+       "i32": 17408
       },
       {
-       "buf": "beta"
-      },
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l29.in_proj",
+     "kernel": "gemm8_dual",
+     "args": [
       {
-       "i32": 16384
+       "buf": "qkvz"
       },
       {
-       "i32": 96
+       "buf": "ba"
       },
       {
-       "i32": 96
+       "buf": "x"
       },
       {
-       "i32": 2048
+       "buf": "model.layers.29.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i32": 2048
+       "buf": "model.layers.29.linear_attn.in_proj_ba.weight"
       },
       {
-       "i32": 6144
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 16384
       },
       {
-       "i64": 0
+       "i32": 96
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l46.a_copy",
-     "kernel": "copy_rows",
+     "label": "l29.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
+       "buf": "qkvz"
       },
       {
-       "i32": 96
+       "buf": "model.layers.29.linear_attn.conv1d.weight"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l46.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "state": "gdn"
       },
       {
-       "buf": "ba"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 48
+       "buf": "gdn.spec_line_index",
+       "offset": 92
       },
       {
-       "i32": 96
+       "buf": "gdn.one"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l46.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
+       "buf": "cu_seqlens_q"
+      },
       {
-       "buf": "model.layers.46.linear_attn.A_log"
+       "buf": "gdn.spec_slots",
+       "offset": 704
       },
       {
-       "buf": "a_c"
+       "buf": "ba"
       },
       {
-       "buf": "b_c"
+       "buf": "model.layers.29.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.46.linear_attn.dt_bias"
+       "buf": "model.layers.29.linear_attn.dt_bias"
       },
       {
-       "f32": 1.0
+       "buf": "core_attn_out"
       },
       {
-       "f32": 20.0
+       "buf": "model.layers.29.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -102181,153 +52475,118 @@
        "buf": "gdn_v"
       },
       {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "g"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "beta"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "a_c"
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 1120
+       "buf": "b_c"
       },
       {
-       "buf": "gdn.one"
+       "buf": "z_c"
       },
       {
        "f32": 0.0883883461356163
       },
       {
-       "i64": 1
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l46.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
+       "i32": 385
       },
       {
-       "i32": 6144
+       "i32": 1703936
       },
       {
-       "i32": 16384
+       "i32": 10
       },
       {
-       "i32": 6144
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l46.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l29.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.46.linear_attn.norm.weight"
+       "buf": "model.layers.29.linear_attn.out_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.29.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l46.out_proj",
-     "kernel": "gemm",
+     "label": "l29.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.46.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.29.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l46.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l29.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.29.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.46.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.30.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -102336,7 +52595,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -102344,23 +52603,32 @@
      ]
     },
     {
-     "label": "l46.gate_up",
-     "kernel": "gemm",
+     "label": "l30.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.46.mlp.gate_up_proj.weight"
+       "buf": "model.layers.30.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.30.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -102368,197 +52636,182 @@
      ]
     },
     {
-     "label": "l46.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l30.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.30.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 96
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l46.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "gdn.one"
       },
       {
-       "buf": "model.layers.46.mlp.down_proj.weight"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "y"
+       "buf": "gdn.spec_slots",
+       "offset": 736
       },
       {
-       "sym": "tokens"
+       "buf": "ba"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.30.linear_attn.A_log"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l46.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "model.layers.30.linear_attn.dt_bias"
+      },
       {
-       "buf": "x"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.30.linear_attn.norm.weight"
       },
       {
-       "buf": "residual"
+       "buf": "gdn_q"
       },
       {
-       "buf": "model.layers.47.input_layernorm.weight_p1"
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
+       "buf": "gdn_v"
       },
       {
-       "sym": "tokens"
+       "buf": "g"
       },
       {
-       "i32": 5120
+       "buf": "beta"
       },
       {
-       "i32": 5120
+       "buf": "a_c"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l47.qkv_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "b_c"
+      },
       {
-       "buf": "x"
+       "buf": "z_c"
       },
       {
-       "buf": "model.layers.47.self_attn.qkv_proj.weight"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "qkv"
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 14336
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l47.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l30.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.47.self_attn.q_norm.weight_p1"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 256
+       "buf": "model.layers.30.linear_attn.out_proj.weight"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "buf": "residual"
       },
       {
-       "i32": 24
+       "buf": "model.layers.30.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 14336
+       "sym": "tokens"
       },
       {
-       "i32": 512
+       "i32": 5120
       },
       {
        "i32": 6144
       },
-      {
-       "i32": 256
-      },
       {
        "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l47.k_norm",
-     "kernel": "gemma_norm_khead",
+     "label": "l30.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "k_n"
+       "buf": "act"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
+       "buf": "x"
       },
       {
-       "buf": "model.layers.47.self_attn.k_norm.weight_p1"
+       "buf": "model.layers.30.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 256
+       "sym": "tokens"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
+       "i32": 17408
       },
       {
-       "i32": 4
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l30.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "i32": 14336
+       "buf": "act"
       },
       {
-       "i32": 256
+       "buf": "model.layers.30.mlp.down_proj.weight"
       },
       {
-       "i32": 1024
+       "buf": "residual"
       },
       {
-       "i32": 256
+       "buf": "model.layers.31.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -102566,92 +52819,104 @@
      ]
     },
     {
-     "label": "l47.rope",
-     "kernel": "mrope",
+     "label": "l31.qkv_proj",
+     "kernel": "gemm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
+       "buf": "x"
       },
       {
-       "buf": "cos_g"
+       "buf": "model.layers.31.self_attn.qkv_proj.weight"
       },
       {
-       "buf": "sin_g"
+       "buf": "qkv"
       },
       {
-       "i32": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l47.kv_write",
-     "kernel": "reshape_and_cache",
+     "label": "l31.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "k_n"
+       "buf": "q_n"
       },
       {
-       "buf": "qkv",
-       "offset": 26624
+       "buf": "k_n"
       },
       {
-       "state": "kv",
-       "offset": 35323904
+       "buf": "qkv"
       },
       {
-       "state": "kv",
-       "offset": 35324416
+       "buf": "model.layers.31.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "slot_mapping"
+       "buf": "model.layers.31.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "kv_scales"
+       "buf": "cos_g"
       },
       {
-       "buf": "kv_scales",
-       "offset": 4
+       "buf": "sin_g"
       },
       {
-       "i64": 1024
+       "state": "kv",
+       "offset": 22478848
       },
       {
-       "i64": 14336
+       "state": "kv",
+       "offset": 22479360
       },
       {
-       "i64": 25690112
+       "buf": "slot_mapping"
       },
       {
-       "i64": 512
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 0
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i64": 0
+       "i64": 25690112
       },
       {
        "i64": 2048
       },
       {
-       "i64": 0
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l47.attn",
+     "label": "l31.attn",
      "kernel": "attn",
      "args": [
       {
@@ -102662,11 +52927,11 @@
       },
       {
        "state": "kv",
-       "offset": 35323904
+       "offset": 22478848
       },
       {
        "state": "kv",
-       "offset": 35324416
+       "offset": 22479360
       },
       {
        "buf": "block_table"
@@ -102744,11 +53009,11 @@
      ]
     },
     {
-     "label": "l47.gate",
-     "kernel": "sigmoid_mul",
+     "label": "l31.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "gated"
+       "buf": "x"
       },
       {
        "buf": "attn_out"
@@ -102758,136 +53023,13 @@
        "offset": 512
       },
       {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l47.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "model.layers.47.self_attn.o_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l47.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
+       "buf": "model.layers.31.self_attn.o_proj.weight"
       },
       {
        "buf": "residual"
       },
       {
-       "buf": "model.layers.47.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l47.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.47.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l47.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l47.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.47.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
+       "buf": "model.layers.31.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -102896,37 +53038,16 @@
        "i32": 5120
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l47.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.48.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
+       "i32": 6144
       },
       {
-       "sym": "tokens"
+       "i32": 256
       },
       {
-       "i32": 5120
+       "i32": 14336
       },
       {
-       "i32": 5120
+       "i32": 512
       },
       {
        "f32": 9.999999974752427e-07
@@ -102934,23 +53055,23 @@
      ]
     },
     {
-     "label": "l47.fc_tap",
-     "kernel": "gemm_acc",
+     "label": "l31.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "residual"
+       "buf": "act"
       },
       {
-       "buf": "draft.fc.3.weight"
+       "buf": "x"
       },
       {
-       "buf": "fc_out"
+       "buf": "model.layers.31.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -102958,45 +53079,63 @@
      ]
     },
     {
-     "label": "l48.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l31.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.48.linear_attn.in_proj_qkvz.weight"
+       "buf": "act"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.31.mlp.down_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.32.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l48.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l32.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.48.linear_attn.in_proj_ba.weight"
+       "buf": "model.layers.32.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.32.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
+      {
+       "i32": 16384
+      },
       {
        "i32": 96
       },
@@ -103006,21 +53145,25 @@
      ]
     },
     {
-     "label": "l48.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l32.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.48.linear_attn.conv1d.weight"
+       "buf": "model.layers.32.linear_attn.conv1d.weight"
       },
       {
        "state": "gdn"
       },
+      {
+       "state": "gdn",
+       "offset": 204800
+      },
       {
        "buf": "gdn.spec_line_index",
-       "offset": 148
+       "offset": 100
       },
       {
        "buf": "gdn.one"
@@ -103029,314 +53172,362 @@
        "buf": "cu_seqlens_q"
       },
       {
-       "buf": "qkvz"
+       "buf": "gdn.spec_slots",
+       "offset": 768
       },
       {
-       "i32": 1
+       "buf": "ba"
       },
       {
-       "i32": 385
+       "buf": "model.layers.32.linear_attn.A_log"
       },
       {
-       "i64": 16384
+       "buf": "model.layers.32.linear_attn.dt_bias"
       },
       {
-       "i64": 16384
+       "buf": "core_attn_out"
       },
       {
-       "i64": 0
+       "buf": "model.layers.32.linear_attn.norm.weight"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l48.post_conv",
-     "kernel": "post_conv",
-     "args": [
+       "buf": "gdn_q"
+      },
       {
-       "buf": "qkvz"
+       "buf": "gdn_k"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "gdn_v"
       },
       {
-       "buf": "ba"
+       "buf": "g"
       },
       {
-       "buf": "model.layers.48.linear_attn.A_log"
+       "buf": "beta"
       },
       {
-       "buf": "model.layers.48.linear_attn.dt_bias"
+       "buf": "a_c"
       },
       {
-       "buf": "gdn_q"
+       "buf": "b_c"
       },
       {
-       "buf": "gdn_k"
+       "buf": "z_c"
       },
       {
-       "buf": "gdn_v"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "g"
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "beta"
+       "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 385
       },
       {
-       "i32": 96
+       "i32": 1703936
       },
       {
-       "i32": 96
+       "i32": 10
       },
       {
-       "i32": 2048
+       "i32": 851968
+      }
+     ]
+    },
+    {
+     "label": "l32.out_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "i32": 2048
+       "buf": "core_attn_out"
       },
       {
-       "i32": 6144
+       "buf": "model.layers.32.linear_attn.out_proj.weight"
+      },
+      {
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.32.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l48.a_copy",
-     "kernel": "copy_rows",
+     "label": "l32.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "act"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "x"
       },
       {
-       "i32": 48
+       "buf": "model.layers.32.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 96
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 17408
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l48.b_copy",
-     "kernel": "copy_rows",
+     "label": "l32.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "b_c"
+       "buf": "x"
       },
       {
-       "buf": "ba"
+       "buf": "act"
       },
       {
-       "i32": 48
+       "buf": "model.layers.32.mlp.down_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.33.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 17408
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l48.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l33.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "model.layers.48.linear_attn.A_log"
+       "buf": "qkvz"
       },
       {
-       "buf": "a_c"
+       "buf": "ba"
       },
       {
-       "buf": "b_c"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.48.linear_attn.dt_bias"
+       "buf": "model.layers.33.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "f32": 1.0
+       "buf": "model.layers.33.linear_attn.in_proj_ba.weight"
       },
       {
-       "f32": 20.0
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_q"
+       "i32": 16384
       },
       {
-       "buf": "gdn_k"
+       "i32": 96
       },
       {
-       "buf": "gdn_v"
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l33.gdn",
+     "kernel": "gdn_spec",
+     "args": [
+      {
+       "buf": "qkvz"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.33.linear_attn.conv1d.weight"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "state": "gdn"
       },
       {
        "state": "gdn",
        "offset": 204800
       },
+      {
+       "buf": "gdn.spec_line_index",
+       "offset": 104
+      },
+      {
+       "buf": "gdn.one"
+      },
       {
        "buf": "cu_seqlens_q"
       },
       {
        "buf": "gdn.spec_slots",
-       "offset": 1152
+       "offset": 800
       },
       {
-       "buf": "gdn.one"
+       "buf": "ba"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "model.layers.33.linear_attn.A_log"
+      },
+      {
+       "buf": "model.layers.33.linear_attn.dt_bias"
       },
       {
-       "i64": 1
+       "buf": "core_attn_out"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.33.linear_attn.norm.weight"
       },
       {
-       "i64": 0
+       "buf": "gdn_q"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l48.z_copy",
-     "kernel": "copy_rows",
-     "args": [
+       "buf": "gdn_k"
+      },
+      {
+       "buf": "gdn_v"
+      },
+      {
+       "buf": "g"
+      },
+      {
+       "buf": "beta"
+      },
+      {
+       "buf": "a_c"
+      },
+      {
+       "buf": "b_c"
+      },
       {
        "buf": "z_c"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 6144
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 16384
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 385
+      },
+      {
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l48.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l33.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.48.linear_attn.norm.weight"
+       "buf": "model.layers.33.linear_attn.out_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.33.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l48.out_proj",
-     "kernel": "gemm",
+     "label": "l33.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.48.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.33.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l48.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l33.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.33.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.48.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.34.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -103345,7 +53536,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -103353,23 +53544,32 @@
      ]
     },
     {
-     "label": "l48.gate_up",
-     "kernel": "gemm",
+     "label": "l34.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.48.mlp.gate_up_proj.weight"
+       "buf": "model.layers.34.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.34.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -103377,128 +53577,149 @@
      ]
     },
     {
-     "label": "l48.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l34.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.34.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 108
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l48.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "gdn.one"
+      },
       {
-       "buf": "act"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "model.layers.48.mlp.down_proj.weight"
+       "buf": "gdn.spec_slots",
+       "offset": 832
       },
       {
-       "buf": "y"
+       "buf": "ba"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.34.linear_attn.A_log"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.34.linear_attn.dt_bias"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l48.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "core_attn_out"
+      },
       {
-       "buf": "x"
+       "buf": "model.layers.34.linear_attn.norm.weight"
       },
       {
-       "buf": "y"
+       "buf": "gdn_q"
       },
       {
-       "buf": "residual"
+       "buf": "gdn_k"
       },
       {
-       "buf": "model.layers.49.input_layernorm.weight_p1"
+       "buf": "gdn_v"
       },
       {
-       "i32": 5120
+       "buf": "g"
       },
       {
-       "sym": "tokens"
+       "buf": "beta"
       },
       {
-       "i32": 5120
+       "buf": "a_c"
       },
       {
-       "i32": 5120
+       "buf": "b_c"
+      },
+      {
+       "buf": "z_c"
+      },
+      {
+       "f32": 0.0883883461356163
       },
       {
        "f32": 9.999999974752427e-07
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 385
+      },
+      {
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l49.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l34.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.49.linear_attn.in_proj_qkvz.weight"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.34.linear_attn.out_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.34.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l49.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l34.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.49.linear_attn.in_proj_ba.weight"
+       "buf": "x"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.34.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 96
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -103506,218 +53727,219 @@
      ]
     },
     {
-     "label": "l49.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l34.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.49.linear_attn.conv1d.weight"
+       "buf": "act"
       },
       {
-       "state": "gdn"
+       "buf": "model.layers.34.mlp.down_proj.weight"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 152
+       "buf": "residual"
       },
       {
-       "buf": "gdn.one"
+       "buf": "model.layers.35.input_layernorm.weight_p1"
       },
       {
-       "buf": "cu_seqlens_q"
+       "sym": "tokens"
       },
       {
-       "buf": "qkvz"
+       "i32": 5120
       },
       {
-       "i32": 1
+       "i32": 17408
       },
       {
-       "i32": 385
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l35.qkv_proj",
+     "kernel": "gemm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "i64": 16384
+       "buf": "model.layers.35.self_attn.qkv_proj.weight"
       },
       {
-       "i64": 16384
+       "buf": "qkv"
       },
       {
-       "i64": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l49.post_conv",
-     "kernel": "post_conv",
+     "label": "l35.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
+       "buf": "q_n"
       },
       {
-       "buf": "model.layers.49.linear_attn.A_log"
+       "buf": "k_n"
       },
       {
-       "buf": "model.layers.49.linear_attn.dt_bias"
+       "buf": "qkv"
       },
       {
-       "buf": "gdn_q"
+       "buf": "model.layers.35.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "gdn_k"
+       "buf": "model.layers.35.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "gdn_v"
+       "buf": "cos_g"
       },
       {
-       "buf": "g"
+       "buf": "sin_g"
       },
       {
-       "buf": "beta"
+       "state": "kv",
+       "offset": 25690112
       },
       {
-       "i32": 16384
+       "state": "kv",
+       "offset": 25690624
       },
       {
-       "i32": 96
+       "buf": "slot_mapping"
       },
       {
-       "i32": 96
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i32": 2048
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i32": 2048
+       "i64": 25690112
       },
       {
-       "i32": 6144
+       "i64": 2048
       },
       {
-       "sym": "tokens"
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l49.a_copy",
-     "kernel": "copy_rows",
+     "label": "l35.attn",
+     "kernel": "attn",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "attn_out"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "q_n"
       },
       {
-       "i32": 48
+       "state": "kv",
+       "offset": 25690112
       },
       {
-       "i32": 96
+       "state": "kv",
+       "offset": 25690624
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l49.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "block_table"
       },
       {
-       "buf": "ba"
+       "buf": "seq_lens"
       },
       {
-       "i32": 48
+       "f32": 0.0625
       },
       {
-       "i32": 96
+       "buf": "kv_scales"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l49.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.49.linear_attn.A_log"
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "buf": "a_c"
+       "f32": 1.0
       },
       {
-       "buf": "b_c"
+       "f32": 0.0
       },
       {
-       "buf": "model.layers.49.linear_attn.dt_bias"
+       "i64": 8
       },
       {
-       "f32": 1.0
+       "i64": 6144
       },
       {
-       "f32": 20.0
+       "i64": 256
       },
       {
-       "buf": "gdn_q"
+       "i64": 6144
       },
       {
-       "buf": "gdn_k"
+       "i64": 256
       },
       {
-       "buf": "gdn_v"
+       "i64": 0
       },
       {
-       "buf": "core_attn_out"
+       "buf": "seq_lens"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "i64": 25690112
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "i64": 2048
       },
       {
-       "buf": "cu_seqlens_q"
+       "i64": 512
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 1184
+       "i64": 25690112
       },
       {
-       "buf": "gdn.one"
+       "i64": 2048
       },
       {
-       "f32": 0.0883883461356163
+       "i64": 512
       },
       {
-       "i64": 1
+       "buf": "cu_seqlens_q"
       },
       {
-       "sym": "tokens"
+       "i32": 1
       },
       {
        "i64": 0
@@ -103728,115 +53950,93 @@
      ]
     },
     {
-     "label": "l49.z_copy",
-     "kernel": "copy_rows",
+     "label": "l35.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "x"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "attn_out"
       },
       {
-       "i32": 6144
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "i32": 16384
+       "buf": "model.layers.35.self_attn.o_proj.weight"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l49.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
+       "buf": "residual"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.35.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "model.layers.49.linear_attn.norm.weight"
+       "sym": "tokens"
       },
       {
-       "buf": "z_c"
+       "i32": 5120
       },
       {
-       "i32": 128
+       "i32": 6144
       },
       {
-       "i32": 128
+       "i32": 256
       },
       {
-       "i32": 128
+       "i32": 14336
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 512
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l49.out_proj",
-     "kernel": "gemm",
+     "label": "l35.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.49.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.35.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l49.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l35.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.35.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.49.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.36.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -103845,7 +54045,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -103853,23 +54053,32 @@
      ]
     },
     {
-     "label": "l49.gate_up",
-     "kernel": "gemm",
+     "label": "l36.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.49.mlp.gate_up_proj.weight"
+       "buf": "model.layers.36.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.36.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -103877,293 +54086,281 @@
      ]
     },
     {
-     "label": "l49.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l36.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.36.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 112
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l49.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "gdn.one"
       },
       {
-       "buf": "model.layers.49.mlp.down_proj.weight"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "y"
+       "buf": "gdn.spec_slots",
+       "offset": 864
       },
       {
-       "sym": "tokens"
+       "buf": "ba"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.36.linear_attn.A_log"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l49.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "model.layers.36.linear_attn.dt_bias"
+      },
       {
-       "buf": "x"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.36.linear_attn.norm.weight"
       },
       {
-       "buf": "residual"
+       "buf": "gdn_q"
       },
       {
-       "buf": "model.layers.50.input_layernorm.weight_p1"
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
+       "buf": "gdn_v"
       },
       {
-       "sym": "tokens"
+       "buf": "g"
       },
       {
-       "i32": 5120
+       "buf": "beta"
       },
       {
-       "i32": 5120
+       "buf": "a_c"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l50.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
+       "buf": "b_c"
+      },
       {
-       "buf": "x"
+       "buf": "z_c"
       },
       {
-       "buf": "model.layers.50.linear_attn.in_proj_qkvz.weight"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "qkvz"
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l50.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l36.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.50.linear_attn.in_proj_ba.weight"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.36.linear_attn.out_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 96
+       "buf": "model.layers.36.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l50.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l36.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.50.linear_attn.conv1d.weight"
+       "buf": "x"
       },
       {
-       "state": "gdn"
+       "buf": "model.layers.36.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 156
+       "sym": "tokens"
       },
       {
-       "buf": "gdn.one"
+       "i32": 17408
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l36.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "buf": "qkvz"
+       "buf": "act"
       },
       {
-       "i32": 1
+       "buf": "model.layers.36.mlp.down_proj.weight"
       },
       {
-       "i32": 385
+       "buf": "residual"
       },
       {
-       "i64": 16384
+       "buf": "model.layers.37.input_layernorm.weight_p1"
       },
       {
-       "i64": 16384
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l50.post_conv",
-     "kernel": "post_conv",
+     "label": "l37.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
        "buf": "qkvz"
       },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
       {
        "buf": "ba"
       },
       {
-       "buf": "model.layers.50.linear_attn.A_log"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.50.linear_attn.dt_bias"
+       "buf": "model.layers.37.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gdn_q"
+       "buf": "model.layers.37.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "gdn_k"
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_v"
+       "i32": 16384
       },
       {
-       "buf": "g"
+       "i32": 96
       },
       {
-       "buf": "beta"
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l37.gdn",
+     "kernel": "gdn_spec",
+     "args": [
       {
-       "i32": 16384
+       "buf": "qkvz"
       },
       {
-       "i32": 96
+       "buf": "model.layers.37.linear_attn.conv1d.weight"
       },
       {
-       "i32": 96
+       "state": "gdn"
       },
       {
-       "i32": 2048
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 2048
+       "buf": "gdn.spec_line_index",
+       "offset": 116
       },
       {
-       "i32": 6144
+       "buf": "gdn.one"
       },
       {
-       "sym": "tokens"
+       "buf": "cu_seqlens_q"
       },
       {
-       "i64": 0
+       "buf": "gdn.spec_slots",
+       "offset": 896
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l50.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
+       "buf": "ba"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.37.linear_attn.A_log"
       },
       {
-       "i32": 48
+       "buf": "model.layers.37.linear_attn.dt_bias"
       },
       {
-       "i32": 96
+       "buf": "core_attn_out"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l50.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.37.linear_attn.norm.weight"
       },
       {
-       "buf": "ba"
+       "buf": "gdn_q"
       },
       {
-       "i32": 48
+       "buf": "gdn_k"
       },
       {
-       "i32": 96
+       "buf": "gdn_v"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l50.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
+       "buf": "g"
+      },
       {
-       "buf": "model.layers.50.linear_attn.A_log"
+       "buf": "beta"
       },
       {
        "buf": "a_c"
@@ -104172,276 +54369,265 @@
        "buf": "b_c"
       },
       {
-       "buf": "model.layers.50.linear_attn.dt_bias"
+       "buf": "z_c"
       },
       {
-       "f32": 1.0
+       "f32": 0.0883883461356163
       },
       {
-       "f32": 20.0
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "gdn_q"
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_k"
+       "i32": 385
       },
       {
-       "buf": "gdn_v"
+       "i32": 1703936
       },
       {
-       "buf": "core_attn_out"
+       "i32": 10
       },
       {
-       "state": "gdn",
-       "offset": 204800
-      },
+       "i32": 851968
+      }
+     ]
+    },
+    {
+     "label": "l37.out_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "x"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 1216
+       "buf": "model.layers.37.linear_attn.out_proj.weight"
       },
       {
-       "buf": "gdn.one"
+       "buf": "residual"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "model.layers.37.post_attention_layernorm.weight_p1"
       },
       {
-       "i64": 1
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 6144
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l50.z_copy",
-     "kernel": "copy_rows",
+     "label": "l37.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "act"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "x"
       },
       {
-       "i32": 6144
+       "buf": "model.layers.37.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 16384
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 17408
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l50.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l37.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.50.linear_attn.norm.weight"
+       "buf": "model.layers.37.mlp.down_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.38.input_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l50.out_proj",
-     "kernel": "gemm",
+     "label": "l38.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "qkvz"
       },
       {
-       "buf": "model.layers.50.linear_attn.out_proj.weight"
+       "buf": "ba"
       },
       {
-       "buf": "y"
+       "buf": "x"
+      },
+      {
+       "buf": "model.layers.38.linear_attn.in_proj_qkvz.weight"
+      },
+      {
+       "buf": "model.layers.38.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 16384
       },
       {
-       "i32": 6144
+       "i32": 96
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l50.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l38.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "x"
+       "buf": "qkvz"
+      },
+      {
+       "buf": "model.layers.38.linear_attn.conv1d.weight"
       },
       {
-       "buf": "y"
+       "state": "gdn"
       },
       {
-       "buf": "residual"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "buf": "model.layers.50.post_attention_layernorm.weight_p1"
+       "buf": "gdn.spec_line_index",
+       "offset": 120
       },
       {
-       "i32": 5120
+       "buf": "gdn.one"
       },
       {
-       "sym": "tokens"
+       "buf": "cu_seqlens_q"
       },
       {
-       "i32": 5120
+       "buf": "gdn.spec_slots",
+       "offset": 928
       },
       {
-       "i32": 5120
+       "buf": "ba"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l50.gate_up",
-     "kernel": "gemm",
-     "args": [
+       "buf": "model.layers.38.linear_attn.A_log"
+      },
       {
-       "buf": "x"
+       "buf": "model.layers.38.linear_attn.dt_bias"
       },
       {
-       "buf": "model.layers.50.mlp.gate_up_proj.weight"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.38.linear_attn.norm.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "gdn_q"
       },
       {
-       "i32": 34816
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l50.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
+       "buf": "gdn_v"
+      },
       {
-       "buf": "act"
+       "buf": "g"
       },
       {
-       "buf": "gate_up"
+       "buf": "beta"
       },
       {
-       "i32": 17408
+       "buf": "a_c"
       },
       {
-       "i32": 0
+       "buf": "b_c"
       },
       {
-       "f32": 1.0
+       "buf": "z_c"
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l50.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "f32": 0.0883883461356163
+      },
       {
-       "buf": "act"
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "model.layers.50.mlp.down_proj.weight"
+       "sym": "tokens"
       },
       {
-       "buf": "y"
+       "i32": 385
       },
       {
-       "sym": "tokens"
+       "i32": 1703936
       },
       {
-       "i32": 5120
+       "i32": 10
       },
       {
-       "i32": 17408
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l50.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l38.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.38.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.51.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.38.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -104450,7 +54636,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -104458,23 +54644,23 @@
      ]
     },
     {
-     "label": "l51.qkv_proj",
-     "kernel": "gemm",
+     "label": "l38.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.51.self_attn.qkv_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkv"
+       "buf": "model.layers.38.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 14336
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -104482,45 +54668,32 @@
      ]
     },
     {
-     "label": "l51.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l38.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.51.self_attn.q_norm.weight_p1"
+       "buf": "x"
       },
       {
-       "i32": 256
+       "buf": "act"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "buf": "model.layers.38.mlp.down_proj.weight"
       },
       {
-       "i32": 24
+       "buf": "residual"
       },
       {
-       "i32": 14336
+       "buf": "model.layers.39.input_layernorm.weight_p1"
       },
       {
-       "i32": 512
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 5120
       },
       {
-       "i32": 256
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -104528,55 +54701,32 @@
      ]
     },
     {
-     "label": "l51.k_norm",
-     "kernel": "gemma_norm_khead",
+     "label": "l39.qkv_proj",
+     "kernel": "gemm",
      "args": [
       {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.51.self_attn.k_norm.weight_p1"
+       "buf": "x"
       },
       {
-       "i32": 256
+       "buf": "model.layers.39.self_attn.qkv_proj.weight"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
+       "buf": "qkv"
       },
       {
-       "i32": 4
+       "sym": "tokens"
       },
       {
        "i32": 14336
       },
       {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l51.rope",
-     "kernel": "mrope",
+     "label": "l39.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
@@ -104585,82 +54735,70 @@
        "buf": "k_n"
       },
       {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
+       "buf": "qkv"
       },
       {
-       "i32": 0
+       "buf": "model.layers.39.self_attn.q_norm.weight_p1"
       },
       {
-       "i64": 0
+       "buf": "model.layers.39.self_attn.k_norm.weight_p1"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l51.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
+       "buf": "cos_g"
       },
       {
-       "buf": "qkv",
-       "offset": 26624
+       "buf": "sin_g"
       },
       {
        "state": "kv",
-       "offset": 38535168
+       "offset": 28901376
       },
       {
        "state": "kv",
-       "offset": 38535680
+       "offset": 28901888
       },
       {
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l51.attn",
+     "label": "l39.attn",
      "kernel": "attn",
      "args": [
       {
@@ -104671,11 +54809,11 @@
       },
       {
        "state": "kv",
-       "offset": 38535168
+       "offset": 28901376
       },
       {
        "state": "kv",
-       "offset": 38535680
+       "offset": 28901888
       },
       {
        "buf": "block_table"
@@ -104753,11 +54891,11 @@
      ]
     },
     {
-     "label": "l51.gate",
-     "kernel": "sigmoid_mul",
+     "label": "l39.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "gated"
+       "buf": "x"
       },
       {
        "buf": "attn_out"
@@ -104767,31 +54905,13 @@
        "offset": 512
       },
       {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l51.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
+       "buf": "model.layers.39.self_attn.o_proj.weight"
       },
       {
-       "buf": "model.layers.51.self_attn.o_proj.weight"
+       "buf": "residual"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.39.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -104801,36 +54921,15 @@
       },
       {
        "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l51.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.51.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
+       "i32": 256
       },
       {
-       "i32": 5120
+       "i32": 14336
       },
       {
-       "i32": 5120
+       "i32": 512
       },
       {
        "f32": 9.999999974752427e-07
@@ -104838,95 +54937,47 @@
      ]
     },
     {
-     "label": "l51.gate_up",
-     "kernel": "gemm",
+     "label": "l39.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.51.mlp.gate_up_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.39.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l51.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
       {
        "i32": 17408
       },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l51.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.51.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
       {
        "i32": 5120
-      },
-      {
-       "i32": 17408
       }
      ]
     },
     {
-     "label": "l51.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l39.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.39.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.52.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.40.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -104935,7 +54986,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -104943,45 +54994,30 @@
      ]
     },
     {
-     "label": "l52.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l40.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.52.linear_attn.in_proj_qkvz.weight"
-      },
       {
        "buf": "qkvz"
       },
       {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
+       "buf": "ba"
       },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l52.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.52.linear_attn.in_proj_ba.weight"
+       "buf": "model.layers.40.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.40.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
+      {
+       "i32": 16384
+      },
       {
        "i32": 96
       },
@@ -104991,21 +55027,25 @@
      ]
     },
     {
-     "label": "l52.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l40.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.52.linear_attn.conv1d.weight"
+       "buf": "model.layers.40.linear_attn.conv1d.weight"
       },
       {
        "state": "gdn"
       },
+      {
+       "state": "gdn",
+       "offset": 204800
+      },
       {
        "buf": "gdn.spec_line_index",
-       "offset": 160
+       "offset": 124
       },
       {
        "buf": "gdn.one"
@@ -105014,47 +55054,23 @@
        "buf": "cu_seqlens_q"
       },
       {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
+       "buf": "gdn.spec_slots",
+       "offset": 960
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l52.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
+       "buf": "ba"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.40.linear_attn.A_log"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.40.linear_attn.dt_bias"
       },
       {
-       "buf": "model.layers.52.linear_attn.A_log"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.52.linear_attn.dt_bias"
+       "buf": "model.layers.40.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -105072,361 +55088,328 @@
        "buf": "beta"
       },
       {
-       "i32": 16384
+       "buf": "a_c"
       },
       {
-       "i32": 96
+       "buf": "b_c"
       },
       {
-       "i32": 96
+       "buf": "z_c"
       },
       {
-       "i32": 2048
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 2048
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 6144
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 385
       },
       {
-       "i64": 0
+       "i32": 1703936
       },
       {
-       "i64": 0
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l52.a_copy",
-     "kernel": "copy_rows",
+     "label": "l40.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "core_attn_out"
       },
       {
-       "i32": 48
+       "buf": "model.layers.40.linear_attn.out_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l52.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.40.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "ba"
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 5120
       },
       {
-       "i32": 96
+       "i32": 6144
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l52.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l40.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "model.layers.52.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.52.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
+       "buf": "act"
       },
       {
-       "buf": "gdn_k"
+       "buf": "x"
       },
       {
-       "buf": "gdn_v"
+       "buf": "model.layers.40.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "core_attn_out"
+       "sym": "tokens"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "i32": 17408
       },
       {
-       "state": "gdn",
-       "offset": 204800
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l40.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "buf": "cu_seqlens_q"
+       "buf": "x"
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 1248
+       "buf": "act"
       },
       {
-       "buf": "gdn.one"
+       "buf": "model.layers.40.mlp.down_proj.weight"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "residual"
       },
       {
-       "i64": 1
+       "buf": "model.layers.41.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l52.z_copy",
-     "kernel": "copy_rows",
+     "label": "l41.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "qkvz"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "ba"
       },
       {
-       "i32": 6144
+       "buf": "x"
+      },
+      {
+       "buf": "model.layers.41.linear_attn.in_proj_qkvz.weight"
+      },
+      {
+       "buf": "model.layers.41.linear_attn.in_proj_ba.weight"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 16384
       },
       {
-       "i32": 6144
+       "i32": 96
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l52.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l41.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "qkvz"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.41.linear_attn.conv1d.weight"
       },
       {
-       "buf": "model.layers.52.linear_attn.norm.weight"
+       "state": "gdn"
       },
       {
-       "buf": "z_c"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 128
+       "buf": "gdn.spec_line_index",
+       "offset": 128
       },
       {
-       "i32": 128
+       "buf": "gdn.one"
       },
       {
-       "i32": 128
+       "buf": "cu_seqlens_q"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "buf": "gdn.spec_slots",
+       "offset": 992
       },
       {
-       "f32": 9.999999974752427e-07
+       "buf": "ba"
       },
       {
-       "i64": 0
+       "buf": "model.layers.41.linear_attn.A_log"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l52.out_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "model.layers.41.linear_attn.dt_bias"
+      },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.52.linear_attn.out_proj.weight"
+       "buf": "model.layers.41.linear_attn.norm.weight"
       },
       {
-       "buf": "y"
+       "buf": "gdn_q"
       },
       {
-       "sym": "tokens"
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
+       "buf": "gdn_v"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l52.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "g"
+      },
       {
-       "buf": "x"
+       "buf": "beta"
       },
       {
-       "buf": "y"
+       "buf": "a_c"
       },
       {
-       "buf": "residual"
+       "buf": "b_c"
       },
       {
-       "buf": "model.layers.52.post_attention_layernorm.weight_p1"
+       "buf": "z_c"
       },
       {
-       "i32": 5120
+       "f32": 0.0883883461356163
+      },
+      {
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
       },
       {
-       "f32": 9.999999974752427e-07
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l52.gate_up",
-     "kernel": "gemm",
+     "label": "l41.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.52.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 34816
+       "buf": "model.layers.41.linear_attn.out_proj.weight"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l52.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "residual"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.41.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 17408
+       "sym": "tokens"
       },
       {
-       "i32": 0
+       "i32": 5120
       },
       {
-       "f32": 1.0
+       "i32": 6144
       },
       {
-       "i32": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l52.down_proj",
-     "kernel": "gemm",
+     "label": "l41.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
        "buf": "act"
       },
       {
-       "buf": "model.layers.52.mlp.down_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.41.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 17408
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l52.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l41.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.41.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.53.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.42.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -105435,7 +55418,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -105443,45 +55426,30 @@
      ]
     },
     {
-     "label": "l53.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l42.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.53.linear_attn.in_proj_qkvz.weight"
-      },
       {
        "buf": "qkvz"
       },
       {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
+       "buf": "ba"
       },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l53.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.53.linear_attn.in_proj_ba.weight"
+       "buf": "model.layers.42.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.42.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
+      {
+       "i32": 16384
+      },
       {
        "i32": 96
       },
@@ -105491,21 +55459,25 @@
      ]
     },
     {
-     "label": "l53.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l42.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.53.linear_attn.conv1d.weight"
+       "buf": "model.layers.42.linear_attn.conv1d.weight"
       },
       {
        "state": "gdn"
       },
+      {
+       "state": "gdn",
+       "offset": 204800
+      },
       {
        "buf": "gdn.spec_line_index",
-       "offset": 164
+       "offset": 132
       },
       {
        "buf": "gdn.one"
@@ -105514,47 +55486,23 @@
        "buf": "cu_seqlens_q"
       },
       {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
+       "buf": "gdn.spec_slots",
+       "offset": 1024
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l53.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
+       "buf": "ba"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.42.linear_attn.A_log"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.42.linear_attn.dt_bias"
       },
       {
-       "buf": "model.layers.53.linear_attn.A_log"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.53.linear_attn.dt_bias"
+       "buf": "model.layers.42.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -105572,192 +55520,196 @@
        "buf": "beta"
       },
       {
-       "i32": 16384
+       "buf": "a_c"
       },
       {
-       "i32": 96
+       "buf": "b_c"
       },
       {
-       "i32": 96
+       "buf": "z_c"
       },
       {
-       "i32": 2048
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 2048
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 6144
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 385
       },
       {
-       "i64": 0
+       "i32": 1703936
       },
       {
-       "i64": 0
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l53.a_copy",
-     "kernel": "copy_rows",
+     "label": "l42.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "core_attn_out"
       },
       {
-       "i32": 48
+       "buf": "model.layers.42.linear_attn.out_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l53.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.42.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "ba"
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 5120
       },
       {
-       "i32": 96
+       "i32": 6144
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l53.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l42.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "model.layers.53.linear_attn.A_log"
+       "buf": "act"
       },
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "b_c"
+       "buf": "model.layers.42.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "model.layers.53.linear_attn.dt_bias"
+       "sym": "tokens"
       },
       {
-       "f32": 1.0
+       "i32": 17408
       },
       {
-       "f32": 20.0
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l42.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "buf": "gdn_q"
+       "buf": "x"
       },
       {
-       "buf": "gdn_k"
+       "buf": "act"
       },
       {
-       "buf": "gdn_v"
+       "buf": "model.layers.42.mlp.down_proj.weight"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "residual"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "model.layers.43.input_layernorm.weight_p1"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "sym": "tokens"
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 5120
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 1280
+       "i32": 17408
       },
       {
-       "buf": "gdn.one"
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l43.qkv_proj",
+     "kernel": "gemm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "model.layers.43.self_attn.qkv_proj.weight"
       },
       {
-       "i64": 1
+       "buf": "qkv"
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l53.z_copy",
-     "kernel": "copy_rows",
+     "label": "l43.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "q_n"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "k_n"
       },
       {
-       "i32": 6144
+       "buf": "qkv"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.43.self_attn.q_norm.weight_p1"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l53.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
+       "buf": "model.layers.43.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "cos_g"
       },
       {
-       "buf": "model.layers.53.linear_attn.norm.weight"
+       "buf": "sin_g"
       },
       {
-       "buf": "z_c"
+       "state": "kv",
+       "offset": 32112640
       },
       {
-       "i32": 128
+       "state": "kv",
+       "offset": 32113152
       },
       {
-       "i32": 128
+       "buf": "slot_mapping"
       },
       {
-       "i32": 128
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
        "expr": {
@@ -105765,168 +55717,208 @@
          {
           "sym": "tokens"
          },
-         48
+         4
         ]
        }
       },
       {
-       "f32": 9.999999974752427e-07
+       "i64": 25690112
       },
       {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 0
+       "i64": 512
+      },
+      {
+       "i32": 784
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l53.out_proj",
-     "kernel": "gemm",
+     "label": "l43.attn",
+     "kernel": "attn",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "attn_out"
       },
       {
-       "buf": "model.layers.53.linear_attn.out_proj.weight"
+       "buf": "q_n"
       },
       {
-       "buf": "y"
+       "state": "kv",
+       "offset": 32112640
       },
       {
-       "sym": "tokens"
+       "state": "kv",
+       "offset": 32113152
       },
       {
-       "i32": 5120
+       "buf": "block_table"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l53.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "seq_lens"
+      },
       {
-       "buf": "x"
+       "f32": 0.0625
       },
       {
-       "buf": "y"
+       "buf": "kv_scales"
       },
       {
-       "buf": "residual"
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "buf": "model.layers.53.post_attention_layernorm.weight_p1"
+       "f32": 1.0
       },
       {
-       "i32": 5120
+       "f32": 0.0
       },
       {
-       "sym": "tokens"
+       "i64": 8
       },
       {
-       "i32": 5120
+       "i64": 6144
       },
       {
-       "i32": 5120
+       "i64": 256
       },
       {
-       "f32": 9.999999974752427e-07
+       "i64": 6144
+      },
+      {
+       "i64": 256
+      },
+      {
+       "i64": 0
+      },
+      {
+       "buf": "seq_lens"
+      },
+      {
+       "i64": 25690112
+      },
+      {
+       "i64": 2048
+      },
+      {
+       "i64": 512
+      },
+      {
+       "i64": 25690112
+      },
+      {
+       "i64": 2048
+      },
+      {
+       "i64": 512
+      },
+      {
+       "buf": "cu_seqlens_q"
+      },
+      {
+       "i32": 1
+      },
+      {
+       "i64": 0
+      },
+      {
+       "i64": 0
       }
      ]
     },
     {
-     "label": "l53.gate_up",
-     "kernel": "gemm",
+     "label": "l43.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.53.mlp.gate_up_proj.weight"
+       "buf": "attn_out"
       },
       {
-       "buf": "gate_up"
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.43.self_attn.o_proj.weight"
       },
       {
-       "i32": 34816
+       "buf": "residual"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l53.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
+       "buf": "model.layers.43.post_attention_layernorm.weight_p1"
+      },
       {
-       "buf": "act"
+       "sym": "tokens"
       },
       {
-       "buf": "gate_up"
+       "i32": 5120
       },
       {
-       "i32": 17408
+       "i32": 6144
       },
       {
-       "i32": 0
+       "i32": 256
+      },
+      {
+       "i32": 14336
       },
       {
-       "f32": 1.0
+       "i32": 512
       },
       {
-       "i32": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l53.down_proj",
-     "kernel": "gemm",
+     "label": "l43.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
        "buf": "act"
       },
       {
-       "buf": "model.layers.53.mlp.down_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.43.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 17408
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l53.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l43.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.43.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.54.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.44.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -105935,7 +55927,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -105943,45 +55935,30 @@
      ]
     },
     {
-     "label": "l54.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l44.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.54.linear_attn.in_proj_qkvz.weight"
-      },
       {
        "buf": "qkvz"
       },
       {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
+       "buf": "ba"
       },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l54.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.54.linear_attn.in_proj_ba.weight"
+       "buf": "model.layers.44.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.44.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
+      {
+       "i32": 16384
+      },
       {
        "i32": 96
       },
@@ -105991,21 +55968,25 @@
      ]
     },
     {
-     "label": "l54.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l44.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.54.linear_attn.conv1d.weight"
+       "buf": "model.layers.44.linear_attn.conv1d.weight"
       },
       {
        "state": "gdn"
       },
+      {
+       "state": "gdn",
+       "offset": 204800
+      },
       {
        "buf": "gdn.spec_line_index",
-       "offset": 168
+       "offset": 136
       },
       {
        "buf": "gdn.one"
@@ -106014,314 +55995,362 @@
        "buf": "cu_seqlens_q"
       },
       {
-       "buf": "qkvz"
+       "buf": "gdn.spec_slots",
+       "offset": 1056
       },
       {
-       "i32": 1
+       "buf": "ba"
       },
       {
-       "i32": 385
+       "buf": "model.layers.44.linear_attn.A_log"
       },
       {
-       "i64": 16384
+       "buf": "model.layers.44.linear_attn.dt_bias"
       },
       {
-       "i64": 16384
+       "buf": "core_attn_out"
       },
       {
-       "i64": 0
+       "buf": "model.layers.44.linear_attn.norm.weight"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l54.post_conv",
-     "kernel": "post_conv",
-     "args": [
+       "buf": "gdn_q"
+      },
       {
-       "buf": "qkvz"
+       "buf": "gdn_k"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "gdn_v"
       },
       {
-       "buf": "ba"
+       "buf": "g"
       },
       {
-       "buf": "model.layers.54.linear_attn.A_log"
+       "buf": "beta"
       },
       {
-       "buf": "model.layers.54.linear_attn.dt_bias"
+       "buf": "a_c"
       },
       {
-       "buf": "gdn_q"
+       "buf": "b_c"
       },
       {
-       "buf": "gdn_k"
+       "buf": "z_c"
       },
       {
-       "buf": "gdn_v"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "g"
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "beta"
+       "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 385
       },
       {
-       "i32": 96
+       "i32": 1703936
       },
       {
-       "i32": 96
+       "i32": 10
       },
       {
-       "i32": 2048
+       "i32": 851968
+      }
+     ]
+    },
+    {
+     "label": "l44.out_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "i32": 2048
+       "buf": "core_attn_out"
       },
       {
-       "i32": 6144
+       "buf": "model.layers.44.linear_attn.out_proj.weight"
+      },
+      {
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.44.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l54.a_copy",
-     "kernel": "copy_rows",
+     "label": "l44.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "act"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "x"
       },
       {
-       "i32": 48
+       "buf": "model.layers.44.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 96
+       "sym": "tokens"
+      },
+      {
+       "i32": 17408
       },
       {
-       "i32": 48
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l54.b_copy",
-     "kernel": "copy_rows",
+     "label": "l44.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "b_c"
+       "buf": "x"
       },
       {
-       "buf": "ba"
+       "buf": "act"
       },
       {
-       "i32": 48
+       "buf": "model.layers.44.mlp.down_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.45.input_layernorm.weight_p1"
       },
       {
-       "i32": 48
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l54.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l45.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "model.layers.54.linear_attn.A_log"
+       "buf": "qkvz"
       },
       {
-       "buf": "a_c"
+       "buf": "ba"
       },
       {
-       "buf": "b_c"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.54.linear_attn.dt_bias"
+       "buf": "model.layers.45.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "f32": 1.0
+       "buf": "model.layers.45.linear_attn.in_proj_ba.weight"
       },
       {
-       "f32": 20.0
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_q"
+       "i32": 16384
       },
       {
-       "buf": "gdn_k"
+       "i32": 96
       },
       {
-       "buf": "gdn_v"
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l45.gdn",
+     "kernel": "gdn_spec",
+     "args": [
+      {
+       "buf": "qkvz"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.45.linear_attn.conv1d.weight"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "state": "gdn"
       },
       {
        "state": "gdn",
        "offset": 204800
       },
+      {
+       "buf": "gdn.spec_line_index",
+       "offset": 140
+      },
+      {
+       "buf": "gdn.one"
+      },
       {
        "buf": "cu_seqlens_q"
       },
       {
        "buf": "gdn.spec_slots",
-       "offset": 1312
+       "offset": 1088
       },
       {
-       "buf": "gdn.one"
+       "buf": "ba"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "model.layers.45.linear_attn.A_log"
       },
       {
-       "i64": 1
+       "buf": "model.layers.45.linear_attn.dt_bias"
       },
       {
-       "sym": "tokens"
+       "buf": "core_attn_out"
       },
       {
-       "i64": 0
+       "buf": "model.layers.45.linear_attn.norm.weight"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l54.z_copy",
-     "kernel": "copy_rows",
-     "args": [
+       "buf": "gdn_q"
+      },
+      {
+       "buf": "gdn_k"
+      },
+      {
+       "buf": "gdn_v"
+      },
+      {
+       "buf": "g"
+      },
+      {
+       "buf": "beta"
+      },
+      {
+       "buf": "a_c"
+      },
+      {
+       "buf": "b_c"
+      },
       {
        "buf": "z_c"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 6144
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 16384
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 385
+      },
+      {
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l54.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l45.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.54.linear_attn.norm.weight"
+       "buf": "model.layers.45.linear_attn.out_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.45.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l54.out_proj",
-     "kernel": "gemm",
+     "label": "l45.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.54.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.45.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l54.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l45.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.45.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.54.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.46.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -106330,7 +56359,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -106338,23 +56367,32 @@
      ]
     },
     {
-     "label": "l54.gate_up",
-     "kernel": "gemm",
+     "label": "l46.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.54.mlp.gate_up_proj.weight"
+       "buf": "model.layers.46.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.46.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -106362,197 +56400,182 @@
      ]
     },
     {
-     "label": "l54.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l46.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.46.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 144
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l54.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "gdn.one"
       },
       {
-       "buf": "model.layers.54.mlp.down_proj.weight"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "y"
+       "buf": "gdn.spec_slots",
+       "offset": 1120
       },
       {
-       "sym": "tokens"
+       "buf": "ba"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.46.linear_attn.A_log"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l54.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "model.layers.46.linear_attn.dt_bias"
+      },
       {
-       "buf": "x"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.46.linear_attn.norm.weight"
       },
       {
-       "buf": "residual"
+       "buf": "gdn_q"
       },
       {
-       "buf": "model.layers.55.input_layernorm.weight_p1"
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
+       "buf": "gdn_v"
       },
       {
-       "sym": "tokens"
+       "buf": "g"
       },
       {
-       "i32": 5120
+       "buf": "beta"
       },
       {
-       "i32": 5120
+       "buf": "a_c"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l55.qkv_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "b_c"
+      },
       {
-       "buf": "x"
+       "buf": "z_c"
       },
       {
-       "buf": "model.layers.55.self_attn.qkv_proj.weight"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "qkv"
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 14336
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l55.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l46.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.55.self_attn.q_norm.weight_p1"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 256
+       "buf": "model.layers.46.linear_attn.out_proj.weight"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "buf": "residual"
       },
       {
-       "i32": 24
+       "buf": "model.layers.46.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 14336
+       "sym": "tokens"
       },
       {
-       "i32": 512
+       "i32": 5120
       },
       {
        "i32": 6144
       },
-      {
-       "i32": 256
-      },
       {
        "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l55.k_norm",
-     "kernel": "gemma_norm_khead",
+     "label": "l46.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "k_n"
+       "buf": "act"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
+       "buf": "x"
       },
       {
-       "buf": "model.layers.55.self_attn.k_norm.weight_p1"
+       "buf": "model.layers.46.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 256
+       "sym": "tokens"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
+       "i32": 17408
+      },
+      {
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l46.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "i32": 4
+       "buf": "act"
       },
       {
-       "i32": 14336
+       "buf": "model.layers.46.mlp.down_proj.weight"
       },
       {
-       "i32": 256
+       "buf": "residual"
       },
       {
-       "i32": 1024
+       "buf": "model.layers.47.input_layernorm.weight_p1"
       },
       {
-       "i32": 256
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -106560,92 +56583,104 @@
      ]
     },
     {
-     "label": "l55.rope",
-     "kernel": "mrope",
+     "label": "l47.qkv_proj",
+     "kernel": "gemm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
+       "buf": "x"
       },
       {
-       "buf": "cos_g"
+       "buf": "model.layers.47.self_attn.qkv_proj.weight"
       },
       {
-       "buf": "sin_g"
+       "buf": "qkv"
       },
       {
-       "i32": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l55.kv_write",
-     "kernel": "reshape_and_cache",
+     "label": "l47.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "k_n"
+       "buf": "q_n"
       },
       {
-       "buf": "qkv",
-       "offset": 26624
+       "buf": "k_n"
       },
       {
-       "state": "kv",
-       "offset": 41746432
+       "buf": "qkv"
       },
       {
-       "state": "kv",
-       "offset": 41746944
+       "buf": "model.layers.47.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "slot_mapping"
+       "buf": "model.layers.47.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "kv_scales"
+       "buf": "cos_g"
       },
       {
-       "buf": "kv_scales",
-       "offset": 4
+       "buf": "sin_g"
       },
       {
-       "i64": 1024
+       "state": "kv",
+       "offset": 35323904
       },
       {
-       "i64": 14336
+       "state": "kv",
+       "offset": 35324416
       },
       {
-       "i64": 25690112
+       "buf": "slot_mapping"
       },
       {
-       "i64": 512
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 0
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i64": 0
+       "i64": 25690112
       },
       {
        "i64": 2048
       },
       {
-       "i64": 0
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l55.attn",
+     "label": "l47.attn",
      "kernel": "attn",
      "args": [
       {
@@ -106656,11 +56691,11 @@
       },
       {
        "state": "kv",
-       "offset": 41746432
+       "offset": 35323904
       },
       {
        "state": "kv",
-       "offset": 41746944
+       "offset": 35324416
       },
       {
        "buf": "block_table"
@@ -106738,11 +56773,11 @@
      ]
     },
     {
-     "label": "l55.gate",
-     "kernel": "sigmoid_mul",
+     "label": "l47.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "gated"
+       "buf": "x"
       },
       {
        "buf": "attn_out"
@@ -106752,31 +56787,13 @@
        "offset": 512
       },
       {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l55.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
+       "buf": "model.layers.47.self_attn.o_proj.weight"
       },
       {
-       "buf": "model.layers.55.self_attn.o_proj.weight"
+       "buf": "residual"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.47.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -106786,36 +56803,15 @@
       },
       {
        "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l55.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.55.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
+       "i32": 256
       },
       {
-       "i32": 5120
+       "i32": 14336
       },
       {
-       "i32": 5120
+       "i32": 512
       },
       {
        "f32": 9.999999974752427e-07
@@ -106823,95 +56819,47 @@
      ]
     },
     {
-     "label": "l55.gate_up",
-     "kernel": "gemm",
+     "label": "l47.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.55.mlp.gate_up_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.47.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l55.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
       {
        "i32": 17408
       },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l55.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.55.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
       {
        "i32": 5120
-      },
-      {
-       "i32": 17408
       }
      ]
     },
     {
-     "label": "l55.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l47.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.47.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.56.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.48.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -106920,7 +56868,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -106928,45 +56876,30 @@
      ]
     },
     {
-     "label": "l56.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l48.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.56.linear_attn.in_proj_qkvz.weight"
-      },
       {
        "buf": "qkvz"
       },
       {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
+       "buf": "ba"
       },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l56.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.56.linear_attn.in_proj_ba.weight"
+       "buf": "model.layers.48.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.48.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
+      {
+       "i32": 16384
+      },
       {
        "i32": 96
       },
@@ -106976,21 +56909,25 @@
      ]
     },
     {
-     "label": "l56.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l48.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.56.linear_attn.conv1d.weight"
+       "buf": "model.layers.48.linear_attn.conv1d.weight"
       },
       {
        "state": "gdn"
       },
+      {
+       "state": "gdn",
+       "offset": 204800
+      },
       {
        "buf": "gdn.spec_line_index",
-       "offset": 172
+       "offset": 148
       },
       {
        "buf": "gdn.one"
@@ -106999,47 +56936,23 @@
        "buf": "cu_seqlens_q"
       },
       {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
+       "buf": "gdn.spec_slots",
+       "offset": 1152
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l56.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
+       "buf": "ba"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.48.linear_attn.A_log"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.48.linear_attn.dt_bias"
       },
       {
-       "buf": "model.layers.56.linear_attn.A_log"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.56.linear_attn.dt_bias"
+       "buf": "model.layers.48.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -107057,331 +56970,271 @@
        "buf": "beta"
       },
       {
-       "i32": 16384
+       "buf": "a_c"
       },
       {
-       "i32": 96
+       "buf": "b_c"
       },
       {
-       "i32": 96
+       "buf": "z_c"
       },
       {
-       "i32": 2048
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 2048
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 6144
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 385
       },
       {
-       "i64": 0
+       "i32": 1703936
       },
       {
-       "i64": 0
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l56.a_copy",
-     "kernel": "copy_rows",
+     "label": "l48.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "core_attn_out"
       },
       {
-       "i32": 48
+       "buf": "model.layers.48.linear_attn.out_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l56.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.48.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "ba"
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 5120
       },
       {
-       "i32": 96
+       "i32": 6144
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l56.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l48.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "model.layers.56.linear_attn.A_log"
+       "buf": "act"
       },
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "b_c"
+       "buf": "model.layers.48.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "model.layers.56.linear_attn.dt_bias"
+       "sym": "tokens"
       },
       {
-       "f32": 1.0
+       "i32": 17408
       },
       {
-       "f32": 20.0
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l48.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "buf": "gdn_q"
+       "buf": "x"
       },
       {
-       "buf": "gdn_k"
+       "buf": "act"
       },
       {
-       "buf": "gdn_v"
+       "buf": "model.layers.48.mlp.down_proj.weight"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "residual"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "model.layers.49.input_layernorm.weight_p1"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "sym": "tokens"
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 5120
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 1344
+       "i32": 17408
       },
       {
-       "buf": "gdn.one"
-      },
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l49.in_proj",
+     "kernel": "gemm8_dual",
+     "args": [
       {
-       "f32": 0.0883883461356163
+       "buf": "qkvz"
       },
       {
-       "i64": 1
+       "buf": "ba"
       },
       {
-       "sym": "tokens"
+       "buf": "x"
       },
       {
-       "i64": 0
+       "buf": "model.layers.49.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l56.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
+       "buf": "model.layers.49.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 16384
       },
       {
-       "i32": 16384
+       "i32": 96
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l56.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l49.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
+       "buf": "qkvz"
       },
       {
-       "buf": "model.layers.56.linear_attn.norm.weight"
+       "buf": "model.layers.49.linear_attn.conv1d.weight"
       },
       {
-       "buf": "z_c"
+       "state": "gdn"
       },
       {
-       "i32": 128
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 128
+       "buf": "gdn.spec_line_index",
+       "offset": 152
       },
       {
-       "i32": 128
+       "buf": "gdn.one"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "buf": "cu_seqlens_q"
       },
       {
-       "f32": 9.999999974752427e-07
+       "buf": "gdn.spec_slots",
+       "offset": 1184
       },
       {
-       "i64": 0
+       "buf": "ba"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l56.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
+       "buf": "model.layers.49.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.56.linear_attn.out_proj.weight"
+       "buf": "model.layers.49.linear_attn.dt_bias"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.49.linear_attn.norm.weight"
       },
       {
-       "i32": 5120
+       "buf": "gdn_q"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l56.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
+       "buf": "gdn_k"
       },
       {
-       "buf": "y"
+       "buf": "gdn_v"
       },
       {
-       "buf": "residual"
+       "buf": "g"
       },
       {
-       "buf": "model.layers.56.post_attention_layernorm.weight_p1"
+       "buf": "beta"
       },
       {
-       "i32": 5120
+       "buf": "a_c"
       },
       {
-       "sym": "tokens"
+       "buf": "b_c"
       },
       {
-       "i32": 5120
+       "buf": "z_c"
       },
       {
-       "i32": 5120
+       "f32": 0.0883883461356163
       },
       {
        "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l56.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
       },
       {
-       "buf": "model.layers.56.mlp.gate_up_proj.weight"
+       "sym": "tokens"
       },
       {
-       "buf": "gate_up"
+       "i32": 385
       },
       {
-       "sym": "tokens"
+       "i32": 1703936
       },
       {
-       "i32": 34816
+       "i32": 10
       },
       {
-       "i32": 5120
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l56.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l49.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
+       "buf": "x"
       },
       {
-       "f32": 1.0
+       "buf": "core_attn_out"
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l56.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "model.layers.49.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.56.mlp.down_proj.weight"
+       "buf": "residual"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.49.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -107390,83 +57243,95 @@
        "i32": 5120
       },
       {
-       "i32": 17408
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l56.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l49.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.57.input_layernorm.weight_p1"
+       "buf": "x"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.49.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l57.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l49.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.57.linear_attn.in_proj_qkvz.weight"
+       "buf": "act"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.49.mlp.down_proj.weight"
+      },
+      {
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.50.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l57.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l50.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.57.linear_attn.in_proj_ba.weight"
+       "buf": "model.layers.50.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.50.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
+      {
+       "i32": 16384
+      },
       {
        "i32": 96
       },
@@ -107476,21 +57341,25 @@
      ]
     },
     {
-     "label": "l57.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l50.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.57.linear_attn.conv1d.weight"
+       "buf": "model.layers.50.linear_attn.conv1d.weight"
       },
       {
        "state": "gdn"
       },
+      {
+       "state": "gdn",
+       "offset": 204800
+      },
       {
        "buf": "gdn.spec_line_index",
-       "offset": 176
+       "offset": 156
       },
       {
        "buf": "gdn.one"
@@ -107499,47 +57368,23 @@
        "buf": "cu_seqlens_q"
       },
       {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
+       "buf": "gdn.spec_slots",
+       "offset": 1216
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l57.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
+       "buf": "ba"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.50.linear_attn.A_log"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.50.linear_attn.dt_bias"
       },
       {
-       "buf": "model.layers.57.linear_attn.A_log"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.57.linear_attn.dt_bias"
+       "buf": "model.layers.50.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -107557,192 +57402,196 @@
        "buf": "beta"
       },
       {
-       "i32": 16384
+       "buf": "a_c"
       },
       {
-       "i32": 96
+       "buf": "b_c"
       },
       {
-       "i32": 96
+       "buf": "z_c"
       },
       {
-       "i32": 2048
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 2048
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 6144
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 385
       },
       {
-       "i64": 0
+       "i32": 1703936
       },
       {
-       "i64": 0
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l57.a_copy",
-     "kernel": "copy_rows",
+     "label": "l50.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "core_attn_out"
       },
       {
-       "i32": 48
+       "buf": "model.layers.50.linear_attn.out_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l57.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.50.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "ba"
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 5120
       },
       {
-       "i32": 96
+       "i32": 6144
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l57.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l50.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "model.layers.57.linear_attn.A_log"
+       "buf": "act"
       },
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "b_c"
+       "buf": "model.layers.50.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "model.layers.57.linear_attn.dt_bias"
+       "sym": "tokens"
       },
       {
-       "f32": 1.0
+       "i32": 17408
       },
       {
-       "f32": 20.0
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l50.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "buf": "gdn_q"
+       "buf": "x"
       },
       {
-       "buf": "gdn_k"
+       "buf": "act"
       },
       {
-       "buf": "gdn_v"
+       "buf": "model.layers.50.mlp.down_proj.weight"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "residual"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "model.layers.51.input_layernorm.weight_p1"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "sym": "tokens"
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 5120
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 1376
+       "i32": 17408
       },
       {
-       "buf": "gdn.one"
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l51.qkv_proj",
+     "kernel": "gemm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "model.layers.51.self_attn.qkv_proj.weight"
       },
       {
-       "i64": 1
+       "buf": "qkv"
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l57.z_copy",
-     "kernel": "copy_rows",
+     "label": "l51.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "q_n"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "k_n"
       },
       {
-       "i32": 6144
+       "buf": "qkv"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.51.self_attn.q_norm.weight_p1"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l57.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
+       "buf": "model.layers.51.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "cos_g"
       },
       {
-       "buf": "model.layers.57.linear_attn.norm.weight"
+       "buf": "sin_g"
       },
       {
-       "buf": "z_c"
+       "state": "kv",
+       "offset": 38535168
       },
       {
-       "i32": 128
+       "state": "kv",
+       "offset": 38535680
       },
       {
-       "i32": 128
+       "buf": "slot_mapping"
       },
       {
-       "i32": 128
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
        "expr": {
@@ -107750,168 +57599,208 @@
          {
           "sym": "tokens"
          },
-         48
+         4
         ]
        }
       },
       {
-       "f32": 9.999999974752427e-07
+       "i64": 25690112
       },
       {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 0
+       "i64": 512
+      },
+      {
+       "i32": 784
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l57.out_proj",
-     "kernel": "gemm",
+     "label": "l51.attn",
+     "kernel": "attn",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "attn_out"
       },
       {
-       "buf": "model.layers.57.linear_attn.out_proj.weight"
+       "buf": "q_n"
       },
       {
-       "buf": "y"
+       "state": "kv",
+       "offset": 38535168
       },
       {
-       "sym": "tokens"
+       "state": "kv",
+       "offset": 38535680
       },
       {
-       "i32": 5120
+       "buf": "block_table"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l57.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "seq_lens"
+      },
       {
-       "buf": "x"
+       "f32": 0.0625
       },
       {
-       "buf": "y"
+       "buf": "kv_scales"
       },
       {
-       "buf": "residual"
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "buf": "model.layers.57.post_attention_layernorm.weight_p1"
+       "f32": 1.0
       },
       {
-       "i32": 5120
+       "f32": 0.0
       },
       {
-       "sym": "tokens"
+       "i64": 8
       },
       {
-       "i32": 5120
+       "i64": 6144
       },
       {
-       "i32": 5120
+       "i64": 256
       },
       {
-       "f32": 9.999999974752427e-07
+       "i64": 6144
+      },
+      {
+       "i64": 256
+      },
+      {
+       "i64": 0
+      },
+      {
+       "buf": "seq_lens"
+      },
+      {
+       "i64": 25690112
+      },
+      {
+       "i64": 2048
+      },
+      {
+       "i64": 512
+      },
+      {
+       "i64": 25690112
+      },
+      {
+       "i64": 2048
+      },
+      {
+       "i64": 512
+      },
+      {
+       "buf": "cu_seqlens_q"
+      },
+      {
+       "i32": 1
+      },
+      {
+       "i64": 0
+      },
+      {
+       "i64": 0
       }
      ]
     },
     {
-     "label": "l57.gate_up",
-     "kernel": "gemm",
+     "label": "l51.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.57.mlp.gate_up_proj.weight"
+       "buf": "attn_out"
       },
       {
-       "buf": "gate_up"
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.51.self_attn.o_proj.weight"
       },
       {
-       "i32": 34816
+       "buf": "residual"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l57.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
+       "buf": "model.layers.51.post_attention_layernorm.weight_p1"
+      },
       {
-       "buf": "act"
+       "sym": "tokens"
       },
       {
-       "buf": "gate_up"
+       "i32": 5120
       },
       {
-       "i32": 17408
+       "i32": 6144
       },
       {
-       "i32": 0
+       "i32": 256
       },
       {
-       "f32": 1.0
+       "i32": 14336
       },
       {
-       "i32": 0
+       "i32": 512
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l57.down_proj",
-     "kernel": "gemm",
+     "label": "l51.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
        "buf": "act"
       },
       {
-       "buf": "model.layers.57.mlp.down_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.51.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 17408
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l57.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l51.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.51.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.58.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.52.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -107920,7 +57809,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -107928,17 +57817,23 @@
      ]
     },
     {
-     "label": "l58.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l52.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.58.linear_attn.in_proj_qkvz.weight"
+       "buf": "model.layers.52.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.52.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
@@ -107946,367 +57841,398 @@
       {
        "i32": 16384
       },
+      {
+       "i32": 96
+      },
       {
        "i32": 5120
       }
      ]
     },
     {
-     "label": "l58.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l52.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "x"
+       "buf": "qkvz"
       },
       {
-       "buf": "model.layers.58.linear_attn.in_proj_ba.weight"
+       "buf": "model.layers.52.linear_attn.conv1d.weight"
       },
       {
-       "buf": "ba"
+       "state": "gdn"
       },
       {
-       "sym": "tokens"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 96
+       "buf": "gdn.spec_line_index",
+       "offset": 160
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l58.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
+       "buf": "gdn.one"
+      },
       {
-       "buf": "qkvz"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "model.layers.58.linear_attn.conv1d.weight"
+       "buf": "gdn.spec_slots",
+       "offset": 1248
       },
       {
-       "state": "gdn"
+       "buf": "ba"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 180
+       "buf": "model.layers.52.linear_attn.A_log"
       },
       {
-       "buf": "gdn.one"
+       "buf": "model.layers.52.linear_attn.dt_bias"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.52.linear_attn.norm.weight"
       },
       {
-       "i32": 1
+       "buf": "gdn_q"
       },
       {
-       "i32": 385
+       "buf": "gdn_k"
       },
       {
-       "i64": 16384
+       "buf": "gdn_v"
       },
       {
-       "i64": 16384
+       "buf": "g"
       },
       {
-       "i64": 0
+       "buf": "beta"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l58.post_conv",
-     "kernel": "post_conv",
-     "args": [
+       "buf": "a_c"
+      },
       {
-       "buf": "qkvz"
+       "buf": "b_c"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "z_c"
       },
       {
-       "buf": "ba"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "model.layers.58.linear_attn.A_log"
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "model.layers.58.linear_attn.dt_bias"
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_q"
+       "i32": 385
       },
       {
-       "buf": "gdn_k"
+       "i32": 1703936
       },
       {
-       "buf": "gdn_v"
+       "i32": 10
       },
       {
-       "buf": "g"
+       "i32": 851968
+      }
+     ]
+    },
+    {
+     "label": "l52.out_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "buf": "beta"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.52.linear_attn.out_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
       },
       {
-       "i32": 96
+       "buf": "model.layers.52.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 2048
+       "sym": "tokens"
       },
       {
-       "i32": 2048
+       "i32": 5120
       },
       {
        "i32": 6144
       },
+      {
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l52.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
+     "args": [
+      {
+       "buf": "act"
+      },
+      {
+       "buf": "x"
+      },
+      {
+       "buf": "model.layers.52.mlp.gate_up_proj.weight"
+      },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 17408
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l58.a_copy",
-     "kernel": "copy_rows",
+     "label": "l52.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "act"
       },
       {
-       "i32": 48
+       "buf": "model.layers.52.mlp.down_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.53.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
       },
       {
-       "i32": 48
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l58.b_copy",
-     "kernel": "copy_rows",
+     "label": "l53.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "b_c"
+       "buf": "qkvz"
       },
       {
        "buf": "ba"
       },
       {
-       "i32": 48
+       "buf": "x"
+      },
+      {
+       "buf": "model.layers.53.linear_attn.in_proj_qkvz.weight"
+      },
+      {
+       "buf": "model.layers.53.linear_attn.in_proj_ba.weight"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 16384
       },
       {
        "i32": 96
       },
       {
-       "i32": 48
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l58.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l53.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "model.layers.58.linear_attn.A_log"
+       "buf": "qkvz"
       },
       {
-       "buf": "a_c"
+       "buf": "model.layers.53.linear_attn.conv1d.weight"
       },
       {
-       "buf": "b_c"
+       "state": "gdn"
       },
       {
-       "buf": "model.layers.58.linear_attn.dt_bias"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 164
       },
       {
-       "f32": 20.0
+       "buf": "gdn.one"
       },
       {
-       "buf": "gdn_q"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "gdn_k"
+       "buf": "gdn.spec_slots",
+       "offset": 1280
       },
       {
-       "buf": "gdn_v"
+       "buf": "ba"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.53.linear_attn.A_log"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "model.layers.53.linear_attn.dt_bias"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "core_attn_out"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "model.layers.53.linear_attn.norm.weight"
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 1408
+       "buf": "gdn_q"
       },
       {
-       "buf": "gdn.one"
+       "buf": "gdn_k"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "gdn_v"
       },
       {
-       "i64": 1
+       "buf": "g"
       },
       {
-       "sym": "tokens"
+       "buf": "beta"
       },
       {
-       "i64": 0
+       "buf": "a_c"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l58.z_copy",
-     "kernel": "copy_rows",
-     "args": [
+       "buf": "b_c"
+      },
       {
        "buf": "z_c"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 6144
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 16384
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 385
+      },
+      {
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l58.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l53.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.58.linear_attn.norm.weight"
+       "buf": "model.layers.53.linear_attn.out_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.53.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l58.out_proj",
-     "kernel": "gemm",
+     "label": "l53.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.58.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.53.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l58.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l53.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.53.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.58.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.54.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -108315,7 +58241,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -108323,23 +58249,32 @@
      ]
     },
     {
-     "label": "l58.gate_up",
-     "kernel": "gemm",
+     "label": "l54.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.58.mlp.gate_up_proj.weight"
+       "buf": "model.layers.54.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.54.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -108347,197 +58282,182 @@
      ]
     },
     {
-     "label": "l58.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l54.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.54.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 168
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l58.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "gdn.one"
       },
       {
-       "buf": "model.layers.58.mlp.down_proj.weight"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "y"
+       "buf": "gdn.spec_slots",
+       "offset": 1312
       },
       {
-       "sym": "tokens"
+       "buf": "ba"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.54.linear_attn.A_log"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l58.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "model.layers.54.linear_attn.dt_bias"
+      },
       {
-       "buf": "x"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.54.linear_attn.norm.weight"
       },
       {
-       "buf": "residual"
+       "buf": "gdn_q"
       },
       {
-       "buf": "model.layers.59.input_layernorm.weight_p1"
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
+       "buf": "gdn_v"
       },
       {
-       "sym": "tokens"
+       "buf": "g"
       },
       {
-       "i32": 5120
+       "buf": "beta"
       },
       {
-       "i32": 5120
+       "buf": "a_c"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l59.qkv_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "b_c"
+      },
       {
-       "buf": "x"
+       "buf": "z_c"
       },
       {
-       "buf": "model.layers.59.self_attn.qkv_proj.weight"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "qkv"
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 14336
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l59.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l54.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.59.self_attn.q_norm.weight_p1"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 256
+       "buf": "model.layers.54.linear_attn.out_proj.weight"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "buf": "residual"
       },
       {
-       "i32": 24
+       "buf": "model.layers.54.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 14336
+       "sym": "tokens"
       },
       {
-       "i32": 512
+       "i32": 5120
       },
       {
        "i32": 6144
       },
-      {
-       "i32": 256
-      },
       {
        "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l59.k_norm",
-     "kernel": "gemma_norm_khead",
+     "label": "l54.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "k_n"
+       "buf": "act"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
+       "buf": "x"
       },
       {
-       "buf": "model.layers.59.self_attn.k_norm.weight_p1"
+       "buf": "model.layers.54.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 256
+       "sym": "tokens"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
+       "i32": 17408
+      },
+      {
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l54.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "i32": 4
+       "buf": "act"
       },
       {
-       "i32": 14336
+       "buf": "model.layers.54.mlp.down_proj.weight"
       },
       {
-       "i32": 256
+       "buf": "residual"
       },
       {
-       "i32": 1024
+       "buf": "model.layers.55.input_layernorm.weight_p1"
       },
       {
-       "i32": 256
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -108545,92 +58465,104 @@
      ]
     },
     {
-     "label": "l59.rope",
-     "kernel": "mrope",
+     "label": "l55.qkv_proj",
+     "kernel": "gemm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
+       "buf": "x"
       },
       {
-       "buf": "cos_g"
+       "buf": "model.layers.55.self_attn.qkv_proj.weight"
       },
       {
-       "buf": "sin_g"
+       "buf": "qkv"
       },
       {
-       "i32": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l59.kv_write",
-     "kernel": "reshape_and_cache",
+     "label": "l55.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "k_n"
+       "buf": "q_n"
       },
       {
-       "buf": "qkv",
-       "offset": 26624
+       "buf": "k_n"
       },
       {
-       "state": "kv",
-       "offset": 44957696
+       "buf": "qkv"
       },
       {
-       "state": "kv",
-       "offset": 44958208
+       "buf": "model.layers.55.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "slot_mapping"
+       "buf": "model.layers.55.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "kv_scales"
+       "buf": "cos_g"
       },
       {
-       "buf": "kv_scales",
-       "offset": 4
+       "buf": "sin_g"
       },
       {
-       "i64": 1024
+       "state": "kv",
+       "offset": 41746432
       },
       {
-       "i64": 14336
+       "state": "kv",
+       "offset": 41746944
       },
       {
-       "i64": 25690112
+       "buf": "slot_mapping"
       },
       {
-       "i64": 512
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 0
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i64": 0
+       "i64": 25690112
       },
       {
        "i64": 2048
       },
       {
-       "i64": 0
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l59.attn",
+     "label": "l55.attn",
      "kernel": "attn",
      "args": [
       {
@@ -108641,11 +58573,11 @@
       },
       {
        "state": "kv",
-       "offset": 44957696
+       "offset": 41746432
       },
       {
        "state": "kv",
-       "offset": 44958208
+       "offset": 41746944
       },
       {
        "buf": "block_table"
@@ -108723,11 +58655,11 @@
      ]
     },
     {
-     "label": "l59.gate",
-     "kernel": "sigmoid_mul",
+     "label": "l55.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "gated"
+       "buf": "x"
       },
       {
        "buf": "attn_out"
@@ -108737,7 +58669,22 @@
        "offset": 512
       },
       {
-       "i32": 24
+       "buf": "model.layers.55.self_attn.o_proj.weight"
+      },
+      {
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.55.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 6144
       },
       {
        "i32": 256
@@ -108747,51 +58694,54 @@
       },
       {
        "i32": 512
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l59.o_proj",
-     "kernel": "gemm",
+     "label": "l55.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "gated"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.59.self_attn.o_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.55.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l59.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l55.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.55.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.59.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.56.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -108800,7 +58750,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -108808,23 +58758,32 @@
      ]
     },
     {
-     "label": "l59.gate_up",
-     "kernel": "gemm",
+     "label": "l56.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.59.mlp.gate_up_proj.weight"
+       "buf": "model.layers.56.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.56.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -108832,128 +58791,149 @@
      ]
     },
     {
-     "label": "l59.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l56.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.56.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 172
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l59.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "gdn.one"
+      },
       {
-       "buf": "act"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "model.layers.59.mlp.down_proj.weight"
+       "buf": "gdn.spec_slots",
+       "offset": 1344
       },
       {
-       "buf": "y"
+       "buf": "ba"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.56.linear_attn.A_log"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.56.linear_attn.dt_bias"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l59.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "core_attn_out"
+      },
       {
-       "buf": "x"
+       "buf": "model.layers.56.linear_attn.norm.weight"
       },
       {
-       "buf": "y"
+       "buf": "gdn_q"
       },
       {
-       "buf": "residual"
+       "buf": "gdn_k"
+      },
+      {
+       "buf": "gdn_v"
+      },
+      {
+       "buf": "g"
+      },
+      {
+       "buf": "beta"
+      },
+      {
+       "buf": "a_c"
+      },
+      {
+       "buf": "b_c"
+      },
+      {
+       "buf": "z_c"
       },
       {
-       "buf": "model.layers.60.input_layernorm.weight_p1"
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 5120
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
       },
       {
-       "f32": 9.999999974752427e-07
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l60.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l56.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.60.linear_attn.in_proj_qkvz.weight"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.56.linear_attn.out_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.56.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l60.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l56.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.60.linear_attn.in_proj_ba.weight"
+       "buf": "x"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.56.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 96
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -108961,85 +58941,59 @@
      ]
     },
     {
-     "label": "l60.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l56.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.60.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 184
-      },
-      {
-       "buf": "gdn.one"
+       "buf": "x"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "act"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.56.mlp.down_proj.weight"
       },
       {
-       "i32": 1
+       "buf": "residual"
       },
       {
-       "i32": 385
+       "buf": "model.layers.57.input_layernorm.weight_p1"
       },
       {
-       "i64": 16384
+       "sym": "tokens"
       },
       {
-       "i64": 16384
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 17408
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l60.post_conv",
-     "kernel": "post_conv",
+     "label": "l57.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
        "buf": "qkvz"
       },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
       {
        "buf": "ba"
       },
       {
-       "buf": "model.layers.60.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.60.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
+       "buf": "x"
       },
       {
-       "buf": "gdn_v"
+       "buf": "model.layers.57.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "g"
+       "buf": "model.layers.57.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "beta"
+       "sym": "tokens"
       },
       {
        "i32": 16384
@@ -109048,92 +59002,55 @@
        "i32": 96
       },
       {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l60.a_copy",
-     "kernel": "copy_rows",
+     "label": "l57.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
+       "buf": "qkvz"
       },
       {
-       "i32": 48
+       "buf": "model.layers.57.linear_attn.conv1d.weight"
       },
       {
-       "i32": 96
+       "state": "gdn"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l60.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "buf": "ba"
+       "buf": "gdn.spec_line_index",
+       "offset": 176
       },
       {
-       "i32": 48
+       "buf": "gdn.one"
       },
       {
-       "i32": 96
+       "buf": "cu_seqlens_q"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l60.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.60.linear_attn.A_log"
+       "buf": "gdn.spec_slots",
+       "offset": 1376
       },
       {
-       "buf": "a_c"
+       "buf": "ba"
       },
       {
-       "buf": "b_c"
+       "buf": "model.layers.57.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.60.linear_attn.dt_bias"
+       "buf": "model.layers.57.linear_attn.dt_bias"
       },
       {
-       "f32": 1.0
+       "buf": "core_attn_out"
       },
       {
-       "f32": 20.0
+       "buf": "model.layers.57.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -109145,153 +59062,118 @@
        "buf": "gdn_v"
       },
       {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "g"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "beta"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "a_c"
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 1440
+       "buf": "b_c"
       },
       {
-       "buf": "gdn.one"
+       "buf": "z_c"
       },
       {
        "f32": 0.0883883461356163
       },
       {
-       "i64": 1
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l60.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
+       "i32": 385
       },
       {
-       "i32": 6144
+       "i32": 1703936
       },
       {
-       "i32": 16384
+       "i32": 10
       },
       {
-       "i32": 6144
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l60.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l57.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.60.linear_attn.norm.weight"
+       "buf": "model.layers.57.linear_attn.out_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.57.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l60.out_proj",
-     "kernel": "gemm",
+     "label": "l57.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.60.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.57.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l60.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l57.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.57.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.60.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.58.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -109300,7 +59182,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -109308,23 +59190,32 @@
      ]
     },
     {
-     "label": "l60.gate_up",
-     "kernel": "gemm",
+     "label": "l58.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.60.mlp.gate_up_proj.weight"
+       "buf": "model.layers.58.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.58.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -109332,128 +59223,149 @@
      ]
     },
     {
-     "label": "l60.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l58.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.58.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 180
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l60.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "gdn.one"
+      },
       {
-       "buf": "act"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "model.layers.60.mlp.down_proj.weight"
+       "buf": "gdn.spec_slots",
+       "offset": 1408
       },
       {
-       "buf": "y"
+       "buf": "ba"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.58.linear_attn.A_log"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.58.linear_attn.dt_bias"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l60.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "core_attn_out"
+      },
       {
-       "buf": "x"
+       "buf": "model.layers.58.linear_attn.norm.weight"
       },
       {
-       "buf": "y"
+       "buf": "gdn_q"
       },
       {
-       "buf": "residual"
+       "buf": "gdn_k"
       },
       {
-       "buf": "model.layers.61.input_layernorm.weight_p1"
+       "buf": "gdn_v"
       },
       {
-       "i32": 5120
+       "buf": "g"
       },
       {
-       "sym": "tokens"
+       "buf": "beta"
       },
       {
-       "i32": 5120
+       "buf": "a_c"
       },
       {
-       "i32": 5120
+       "buf": "b_c"
+      },
+      {
+       "buf": "z_c"
+      },
+      {
+       "f32": 0.0883883461356163
       },
       {
        "f32": 9.999999974752427e-07
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 385
+      },
+      {
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l61.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l58.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.61.linear_attn.in_proj_qkvz.weight"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.58.linear_attn.out_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.58.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l61.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l58.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.61.linear_attn.in_proj_ba.weight"
+       "buf": "x"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.58.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 96
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -109461,218 +59373,219 @@
      ]
     },
     {
-     "label": "l61.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l58.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.61.linear_attn.conv1d.weight"
+       "buf": "act"
       },
       {
-       "state": "gdn"
+       "buf": "model.layers.58.mlp.down_proj.weight"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 188
+       "buf": "residual"
       },
       {
-       "buf": "gdn.one"
+       "buf": "model.layers.59.input_layernorm.weight_p1"
       },
       {
-       "buf": "cu_seqlens_q"
+       "sym": "tokens"
       },
       {
-       "buf": "qkvz"
+       "i32": 5120
       },
       {
-       "i32": 1
+       "i32": 17408
       },
       {
-       "i32": 385
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l59.qkv_proj",
+     "kernel": "gemm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "i64": 16384
+       "buf": "model.layers.59.self_attn.qkv_proj.weight"
       },
       {
-       "i64": 16384
+       "buf": "qkv"
       },
       {
-       "i64": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l61.post_conv",
-     "kernel": "post_conv",
+     "label": "l59.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
+       "buf": "q_n"
       },
       {
-       "buf": "model.layers.61.linear_attn.A_log"
+       "buf": "k_n"
       },
       {
-       "buf": "model.layers.61.linear_attn.dt_bias"
+       "buf": "qkv"
       },
       {
-       "buf": "gdn_q"
+       "buf": "model.layers.59.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "gdn_k"
+       "buf": "model.layers.59.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "gdn_v"
+       "buf": "cos_g"
       },
       {
-       "buf": "g"
+       "buf": "sin_g"
       },
       {
-       "buf": "beta"
+       "state": "kv",
+       "offset": 44957696
       },
       {
-       "i32": 16384
+       "state": "kv",
+       "offset": 44958208
       },
       {
-       "i32": 96
+       "buf": "slot_mapping"
       },
       {
-       "i32": 96
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i32": 2048
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i32": 2048
+       "i64": 25690112
       },
       {
-       "i32": 6144
+       "i64": 2048
       },
       {
-       "sym": "tokens"
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l61.a_copy",
-     "kernel": "copy_rows",
+     "label": "l59.attn",
+     "kernel": "attn",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "attn_out"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "q_n"
       },
       {
-       "i32": 48
+       "state": "kv",
+       "offset": 44957696
       },
       {
-       "i32": 96
+       "state": "kv",
+       "offset": 44958208
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l61.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "block_table"
       },
       {
-       "buf": "ba"
+       "buf": "seq_lens"
       },
       {
-       "i32": 48
+       "f32": 0.0625
       },
       {
-       "i32": 96
+       "buf": "kv_scales"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l61.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.61.linear_attn.A_log"
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "buf": "a_c"
+       "f32": 1.0
       },
       {
-       "buf": "b_c"
+       "f32": 0.0
       },
       {
-       "buf": "model.layers.61.linear_attn.dt_bias"
+       "i64": 8
       },
       {
-       "f32": 1.0
+       "i64": 6144
       },
       {
-       "f32": 20.0
+       "i64": 256
       },
       {
-       "buf": "gdn_q"
+       "i64": 6144
       },
       {
-       "buf": "gdn_k"
+       "i64": 256
       },
       {
-       "buf": "gdn_v"
+       "i64": 0
       },
       {
-       "buf": "core_attn_out"
+       "buf": "seq_lens"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "i64": 25690112
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "i64": 2048
       },
       {
-       "buf": "cu_seqlens_q"
+       "i64": 512
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 1472
+       "i64": 25690112
       },
       {
-       "buf": "gdn.one"
+       "i64": 2048
       },
       {
-       "f32": 0.0883883461356163
+       "i64": 512
       },
       {
-       "i64": 1
+       "buf": "cu_seqlens_q"
       },
       {
-       "sym": "tokens"
+       "i32": 1
       },
       {
        "i64": 0
@@ -109683,115 +59596,93 @@
      ]
     },
     {
-     "label": "l61.z_copy",
-     "kernel": "copy_rows",
+     "label": "l59.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "x"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "attn_out"
       },
       {
-       "i32": 6144
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "i32": 16384
+       "buf": "model.layers.59.self_attn.o_proj.weight"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l61.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
+       "buf": "residual"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.59.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "model.layers.61.linear_attn.norm.weight"
+       "sym": "tokens"
       },
       {
-       "buf": "z_c"
+       "i32": 5120
       },
       {
-       "i32": 128
+       "i32": 6144
       },
       {
-       "i32": 128
+       "i32": 256
       },
       {
-       "i32": 128
+       "i32": 14336
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 512
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l61.out_proj",
-     "kernel": "gemm",
+     "label": "l59.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.61.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.59.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l61.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l59.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.59.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.61.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.60.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -109800,7 +59691,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -109808,23 +59699,32 @@
      ]
     },
     {
-     "label": "l61.gate_up",
-     "kernel": "gemm",
+     "label": "l60.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.61.mlp.gate_up_proj.weight"
+       "buf": "model.layers.60.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.60.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -109832,152 +59732,149 @@
      ]
     },
     {
-     "label": "l61.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l60.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.60.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 184
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l61.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "gdn.one"
       },
       {
-       "buf": "model.layers.61.mlp.down_proj.weight"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "y"
+       "buf": "gdn.spec_slots",
+       "offset": 1440
       },
       {
-       "sym": "tokens"
+       "buf": "ba"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.60.linear_attn.A_log"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l61.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "model.layers.60.linear_attn.dt_bias"
+      },
       {
-       "buf": "x"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.60.linear_attn.norm.weight"
       },
       {
-       "buf": "residual"
+       "buf": "gdn_q"
       },
       {
-       "buf": "model.layers.62.input_layernorm.weight_p1"
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
+       "buf": "gdn_v"
       },
       {
-       "sym": "tokens"
+       "buf": "g"
       },
       {
-       "i32": 5120
+       "buf": "beta"
       },
       {
-       "i32": 5120
+       "buf": "a_c"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l61.fc_tap",
-     "kernel": "gemm_acc",
-     "args": [
+       "buf": "b_c"
+      },
       {
-       "buf": "residual"
+       "buf": "z_c"
       },
       {
-       "buf": "draft.fc.4.weight"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "fc_out"
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l62.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l60.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.62.linear_attn.in_proj_qkvz.weight"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.60.linear_attn.out_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.60.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l62.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l60.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.62.linear_attn.in_proj_ba.weight"
+       "buf": "x"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.60.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 96
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -109985,164 +59882,131 @@
      ]
     },
     {
-     "label": "l62.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l60.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.62.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 192
-      },
-      {
-       "buf": "gdn.one"
+       "buf": "x"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "act"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.60.mlp.down_proj.weight"
       },
       {
-       "i32": 1
+       "buf": "residual"
       },
       {
-       "i32": 385
+       "buf": "model.layers.61.input_layernorm.weight_p1"
       },
       {
-       "i64": 16384
+       "sym": "tokens"
       },
       {
-       "i64": 16384
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 17408
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l62.post_conv",
-     "kernel": "post_conv",
+     "label": "l61.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
        "buf": "qkvz"
       },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
       {
        "buf": "ba"
       },
       {
-       "buf": "model.layers.62.linear_attn.A_log"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.62.linear_attn.dt_bias"
+       "buf": "model.layers.61.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gdn_q"
+       "buf": "model.layers.61.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "gdn_k"
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_v"
+       "i32": 16384
       },
       {
-       "buf": "g"
+       "i32": 96
       },
       {
-       "buf": "beta"
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l61.gdn",
+     "kernel": "gdn_spec",
+     "args": [
       {
-       "i32": 16384
+       "buf": "qkvz"
       },
       {
-       "i32": 96
+       "buf": "model.layers.61.linear_attn.conv1d.weight"
       },
       {
-       "i32": 96
+       "state": "gdn"
       },
       {
-       "i32": 2048
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 2048
+       "buf": "gdn.spec_line_index",
+       "offset": 188
       },
       {
-       "i32": 6144
+       "buf": "gdn.one"
       },
       {
-       "sym": "tokens"
+       "buf": "cu_seqlens_q"
       },
       {
-       "i64": 0
+       "buf": "gdn.spec_slots",
+       "offset": 1472
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l62.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
+       "buf": "ba"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.61.linear_attn.A_log"
       },
       {
-       "i32": 48
+       "buf": "model.layers.61.linear_attn.dt_bias"
       },
       {
-       "i32": 96
+       "buf": "core_attn_out"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l62.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.61.linear_attn.norm.weight"
       },
       {
-       "buf": "ba"
+       "buf": "gdn_q"
       },
       {
-       "i32": 48
+       "buf": "gdn_k"
       },
       {
-       "i32": 96
+       "buf": "gdn_v"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l62.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
+       "buf": "g"
+      },
       {
-       "buf": "model.layers.62.linear_attn.A_log"
+       "buf": "beta"
       },
       {
        "buf": "a_c"
@@ -110151,276 +60015,265 @@
        "buf": "b_c"
       },
       {
-       "buf": "model.layers.62.linear_attn.dt_bias"
+       "buf": "z_c"
       },
       {
-       "f32": 1.0
+       "f32": 0.0883883461356163
       },
       {
-       "f32": 20.0
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "gdn_q"
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_k"
+       "i32": 385
       },
       {
-       "buf": "gdn_v"
+       "i32": 1703936
       },
       {
-       "buf": "core_attn_out"
+       "i32": 10
       },
       {
-       "state": "gdn",
-       "offset": 204800
-      },
+       "i32": 851968
+      }
+     ]
+    },
+    {
+     "label": "l61.out_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "x"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 1504
+       "buf": "model.layers.61.linear_attn.out_proj.weight"
       },
       {
-       "buf": "gdn.one"
+       "buf": "residual"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "model.layers.61.post_attention_layernorm.weight_p1"
       },
       {
-       "i64": 1
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 6144
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l62.z_copy",
-     "kernel": "copy_rows",
+     "label": "l61.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "act"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "x"
       },
       {
-       "i32": 6144
+       "buf": "model.layers.61.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 16384
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 17408
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l62.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l61.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.62.linear_attn.norm.weight"
+       "buf": "model.layers.61.mlp.down_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.62.input_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l62.out_proj",
-     "kernel": "gemm",
+     "label": "l62.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "qkvz"
       },
       {
-       "buf": "model.layers.62.linear_attn.out_proj.weight"
+       "buf": "ba"
       },
       {
-       "buf": "y"
+       "buf": "x"
+      },
+      {
+       "buf": "model.layers.62.linear_attn.in_proj_qkvz.weight"
+      },
+      {
+       "buf": "model.layers.62.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 16384
       },
       {
-       "i32": 6144
+       "i32": 96
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l62.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l62.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "x"
+       "buf": "qkvz"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.62.linear_attn.conv1d.weight"
       },
       {
-       "buf": "residual"
+       "state": "gdn"
       },
       {
-       "buf": "model.layers.62.post_attention_layernorm.weight_p1"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 5120
+       "buf": "gdn.spec_line_index",
+       "offset": 192
       },
       {
-       "sym": "tokens"
+       "buf": "gdn.one"
       },
       {
-       "i32": 5120
+       "buf": "cu_seqlens_q"
       },
       {
-       "i32": 5120
+       "buf": "gdn.spec_slots",
+       "offset": 1504
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l62.gate_up",
-     "kernel": "gemm",
-     "args": [
+       "buf": "ba"
+      },
       {
-       "buf": "x"
+       "buf": "model.layers.62.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.62.mlp.gate_up_proj.weight"
+       "buf": "model.layers.62.linear_attn.dt_bias"
       },
       {
-       "buf": "gate_up"
+       "buf": "core_attn_out"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.62.linear_attn.norm.weight"
       },
       {
-       "i32": 34816
+       "buf": "gdn_q"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l62.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
+       "buf": "gdn_k"
+      },
       {
-       "buf": "act"
+       "buf": "gdn_v"
+      },
+      {
+       "buf": "g"
+      },
+      {
+       "buf": "beta"
       },
       {
-       "buf": "gate_up"
+       "buf": "a_c"
       },
       {
-       "i32": 17408
+       "buf": "b_c"
       },
       {
-       "i32": 0
+       "buf": "z_c"
       },
       {
-       "f32": 1.0
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l62.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "model.layers.62.mlp.down_proj.weight"
+       "sym": "tokens"
       },
       {
-       "buf": "y"
+       "i32": 385
       },
       {
-       "sym": "tokens"
+       "i32": 1703936
       },
       {
-       "i32": 5120
+       "i32": 10
       },
       {
-       "i32": 17408
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l62.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l62.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.62.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.63.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.62.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -110429,7 +60282,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -110437,23 +60290,23 @@
      ]
     },
     {
-     "label": "l63.qkv_proj",
-     "kernel": "gemm",
+     "label": "l62.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.63.self_attn.qkv_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkv"
+       "buf": "model.layers.62.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 14336
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -110461,45 +60314,32 @@
      ]
     },
     {
-     "label": "l63.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l62.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.63.self_attn.q_norm.weight_p1"
+       "buf": "x"
       },
       {
-       "i32": 256
+       "buf": "act"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "buf": "model.layers.62.mlp.down_proj.weight"
       },
       {
-       "i32": 24
+       "buf": "residual"
       },
       {
-       "i32": 14336
+       "buf": "model.layers.63.input_layernorm.weight_p1"
       },
       {
-       "i32": 512
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 5120
       },
       {
-       "i32": 256
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -110507,55 +60347,32 @@
      ]
     },
     {
-     "label": "l63.k_norm",
-     "kernel": "gemma_norm_khead",
+     "label": "l63.qkv_proj",
+     "kernel": "gemm",
      "args": [
       {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.63.self_attn.k_norm.weight_p1"
+       "buf": "x"
       },
       {
-       "i32": 256
+       "buf": "model.layers.63.self_attn.qkv_proj.weight"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
+       "buf": "qkv"
       },
       {
-       "i32": 4
+       "sym": "tokens"
       },
       {
        "i32": 14336
       },
       {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l63.rope",
-     "kernel": "mrope",
+     "label": "l63.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
@@ -110564,32 +60381,19 @@
        "buf": "k_n"
       },
       {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
+       "buf": "qkv"
       },
       {
-       "i32": 0
+       "buf": "model.layers.63.self_attn.q_norm.weight_p1"
       },
       {
-       "i64": 0
+       "buf": "model.layers.63.self_attn.k_norm.weight_p1"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l63.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
+       "buf": "cos_g"
       },
       {
-       "buf": "qkv",
-       "offset": 26624
+       "buf": "sin_g"
       },
       {
        "state": "kv",
@@ -110603,38 +60407,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -110732,11 +60537,11 @@
      ]
     },
     {
-     "label": "l63.gate",
-     "kernel": "sigmoid_mul",
+     "label": "l63.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "gated"
+       "buf": "x"
       },
       {
        "buf": "attn_out"
@@ -110746,31 +60551,13 @@
        "offset": 512
       },
       {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l63.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
+       "buf": "model.layers.63.self_attn.o_proj.weight"
       },
       {
-       "buf": "model.layers.63.self_attn.o_proj.weight"
+       "buf": "residual"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.63.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -110780,36 +60567,15 @@
       },
       {
        "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l63.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.63.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
+       "i32": 256
       },
       {
-       "i32": 5120
+       "i32": 14336
       },
       {
-       "i32": 5120
+       "i32": 512
       },
       {
        "f32": 9.999999974752427e-07
@@ -110817,23 +60583,23 @@
      ]
     },
     {
-     "label": "l63.gate_up",
-     "kernel": "gemm",
+     "label": "l63.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.63.mlp.gate_up_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.63.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -110841,72 +60607,24 @@
      ]
     },
     {
-     "label": "l63.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l63.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
+       "buf": "x"
       },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l63.down_proj",
-     "kernel": "gemm",
-     "args": [
       {
        "buf": "act"
       },
       {
        "buf": "model.layers.63.mlp.down_proj.weight"
       },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l63.final_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
       {
        "buf": "residual"
       },
       {
        "buf": "model.norm.weight_p1"
       },
-      {
-       "i32": 5120
-      },
       {
        "sym": "tokens"
       },
@@ -110914,7 +60632,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -110962,7 +60680,7 @@
     }
    ]
   },
-  "verify": {
+  "decode_spec": {
    "dispatches": [
     {
      "label": "embed",
@@ -111067,44 +60785,29 @@
      ]
     },
     {
-     "label": "l0.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l0.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.0.linear_attn.in_proj_qkvz.weight"
-      },
       {
        "buf": "qkvz"
       },
       {
-       "sym": "tokens"
+       "buf": "ba"
       },
       {
-       "i32": 16384
+       "buf": "x"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l0.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
+       "buf": "model.layers.0.linear_attn.in_proj_qkvz.weight"
       },
       {
        "buf": "model.layers.0.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "ba"
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 16384
       },
       {
        "i32": 96
@@ -111115,8 +60818,8 @@
      ]
     },
     {
-     "label": "l0.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l0.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
        "buf": "qkvz"
@@ -111127,58 +60830,37 @@
       {
        "state": "gdn"
       },
+      {
+       "state": "gdn",
+       "offset": 204800
+      },
       {
        "buf": "gdn.spec_line_index",
        "offset": 4
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "gdn.one"
       },
       {
        "buf": "cu_seqlens_q"
       },
       {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
+       "buf": "gdn.spec_slots"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l0.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
+       "buf": "ba"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.0.linear_attn.A_log"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.0.linear_attn.dt_bias"
       },
       {
-       "buf": "model.layers.0.linear_attn.A_log"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.0.linear_attn.dt_bias"
+       "buf": "model.layers.0.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -111195,85 +60877,6 @@
       {
        "buf": "beta"
       },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l0.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l0.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "i32": 48
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l0.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.0.linear_attn.A_log"
-      },
       {
        "buf": "a_c"
       },
@@ -111281,170 +60884,106 @@
        "buf": "b_c"
       },
       {
-       "buf": "model.layers.0.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots"
-      },
-      {
-       "buf": "num_accepted_tokens"
+       "buf": "z_c"
       },
       {
        "f32": 0.0883883461356163
       },
       {
-       "i64": 1
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l0.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
+       "i32": 385
       },
       {
-       "i32": 6144
+       "i32": 1703936
       },
       {
-       "i32": 16384
+       "i32": 10
       },
       {
-       "i32": 6144
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l0.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l0.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.0.linear_attn.norm.weight"
-      },
-      {
-       "buf": "z_c"
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
+       "buf": "model.layers.0.linear_attn.out_proj.weight"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "buf": "residual"
       },
       {
-       "f32": 9.999999974752427e-07
+       "buf": "model.layers.0.post_attention_layernorm.weight_p1"
       },
       {
-       "i64": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 5120
+      },
+      {
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l0.out_proj",
-     "kernel": "gemm",
+     "label": "l0.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.0.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.0.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l0.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l0.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.0.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.0.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.1.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -111453,7 +60992,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -111461,23 +61000,32 @@
      ]
     },
     {
-     "label": "l0.gate_up",
-     "kernel": "gemm",
+     "label": "l1.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.0.mlp.gate_up_proj.weight"
+       "buf": "model.layers.1.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.1.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -111485,71 +61033,116 @@
      ]
     },
     {
-     "label": "l0.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l1.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.1.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 8
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l0.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "gdn.one"
+      },
       {
-       "buf": "act"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "model.layers.0.mlp.down_proj.weight"
+       "buf": "gdn.spec_slots",
+       "offset": 32
       },
       {
-       "buf": "y"
+       "buf": "ba"
+      },
+      {
+       "buf": "model.layers.1.linear_attn.A_log"
+      },
+      {
+       "buf": "model.layers.1.linear_attn.dt_bias"
+      },
+      {
+       "buf": "core_attn_out"
+      },
+      {
+       "buf": "model.layers.1.linear_attn.norm.weight"
+      },
+      {
+       "buf": "gdn_q"
+      },
+      {
+       "buf": "gdn_k"
+      },
+      {
+       "buf": "gdn_v"
+      },
+      {
+       "buf": "g"
+      },
+      {
+       "buf": "beta"
+      },
+      {
+       "buf": "a_c"
+      },
+      {
+       "buf": "b_c"
+      },
+      {
+       "buf": "z_c"
+      },
+      {
+       "f32": 0.0883883461356163
+      },
+      {
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 385
       },
       {
-       "i32": 17408
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l0.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l1.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.1.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.1.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.1.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -111558,7 +61151,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -111566,23 +61159,23 @@
      ]
     },
     {
-     "label": "l1.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l1.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.1.linear_attn.in_proj_qkvz.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.1.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -111590,94 +61183,116 @@
      ]
     },
     {
-     "label": "l1.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l1.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.1.linear_attn.in_proj_ba.weight"
+       "buf": "act"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.1.mlp.down_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 96
+       "buf": "model.layers.2.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l1.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l2.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.1.linear_attn.conv1d.weight"
+       "buf": "ba"
       },
       {
-       "state": "gdn"
+       "buf": "x"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 8
+       "buf": "model.layers.2.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "model.layers.2.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "cu_seqlens_q"
+       "sym": "tokens"
       },
       {
-       "buf": "qkvz"
+       "i32": 16384
       },
       {
-       "i32": 1
+       "i32": 96
       },
       {
-       "i32": 385
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l2.gdn",
+     "kernel": "gdn_spec",
+     "args": [
+      {
+       "buf": "qkvz"
       },
       {
-       "i64": 16384
+       "buf": "model.layers.2.linear_attn.conv1d.weight"
       },
       {
-       "i64": 16384
+       "state": "gdn"
       },
       {
-       "i64": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l1.post_conv",
-     "kernel": "post_conv",
-     "args": [
+       "buf": "gdn.spec_line_index",
+       "offset": 12
+      },
       {
-       "buf": "qkvz"
+       "buf": "gdn.one"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "cu_seqlens_q"
+      },
+      {
+       "buf": "gdn.spec_slots",
+       "offset": 64
       },
       {
        "buf": "ba"
       },
       {
-       "buf": "model.layers.1.linear_attn.A_log"
+       "buf": "model.layers.2.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.1.linear_attn.dt_bias"
+       "buf": "model.layers.2.linear_attn.dt_bias"
+      },
+      {
+       "buf": "core_attn_out"
+      },
+      {
+       "buf": "model.layers.2.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -111695,192 +61310,195 @@
        "buf": "beta"
       },
       {
-       "i32": 16384
+       "buf": "a_c"
       },
       {
-       "i32": 96
+       "buf": "b_c"
       },
       {
-       "i32": 96
+       "buf": "z_c"
       },
       {
-       "i32": 2048
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 2048
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 6144
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 385
       },
       {
-       "i64": 0
+       "i32": 1703936
       },
       {
-       "i64": 0
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l1.a_copy",
-     "kernel": "copy_rows",
+     "label": "l2.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "core_attn_out"
       },
       {
-       "i32": 48
+       "buf": "model.layers.2.linear_attn.out_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l1.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.2.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "ba"
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 5120
       },
       {
-       "i32": 96
+       "i32": 6144
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l1.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l2.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "model.layers.1.linear_attn.A_log"
+       "buf": "act"
       },
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "b_c"
+       "buf": "model.layers.2.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "model.layers.1.linear_attn.dt_bias"
+       "sym": "tokens"
       },
       {
-       "f32": 1.0
+       "i32": 17408
       },
       {
-       "f32": 20.0
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l2.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "buf": "gdn_q"
+       "buf": "x"
       },
       {
-       "buf": "gdn_k"
+       "buf": "act"
       },
       {
-       "buf": "gdn_v"
+       "buf": "model.layers.2.mlp.down_proj.weight"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "residual"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "model.layers.3.input_layernorm.weight_p1"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "sym": "tokens"
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 5120
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 32
+       "i32": 17408
       },
       {
-       "buf": "num_accepted_tokens"
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l3.qkv_proj",
+     "kernel": "gemm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "model.layers.3.self_attn.qkv_proj.weight"
       },
       {
-       "i64": 1
+       "buf": "qkv"
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l1.z_copy",
-     "kernel": "copy_rows",
+     "label": "l3.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "q_n"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "k_n"
       },
       {
-       "i32": 6144
+       "buf": "qkv"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.3.self_attn.q_norm.weight_p1"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l1.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
+       "buf": "model.layers.3.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "cos_g"
       },
       {
-       "buf": "model.layers.1.linear_attn.norm.weight"
+       "buf": "sin_g"
       },
       {
-       "buf": "z_c"
+       "state": "kv"
       },
       {
-       "i32": 128
+       "state": "kv",
+       "offset": 512
       },
       {
-       "i32": 128
+       "buf": "slot_mapping"
       },
       {
-       "i32": 128
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
        "expr": {
@@ -111888,168 +61506,207 @@
          {
           "sym": "tokens"
          },
-         48
+         4
         ]
        }
       },
       {
-       "f32": 9.999999974752427e-07
+       "i64": 25690112
       },
       {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 0
+       "i64": 512
+      },
+      {
+       "i32": 784
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l1.out_proj",
-     "kernel": "gemm",
+     "label": "l3.attn",
+     "kernel": "attn",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "attn_out"
       },
       {
-       "buf": "model.layers.1.linear_attn.out_proj.weight"
+       "buf": "q_n"
       },
       {
-       "buf": "y"
+       "state": "kv"
       },
       {
-       "sym": "tokens"
+       "state": "kv",
+       "offset": 512
       },
       {
-       "i32": 5120
+       "buf": "block_table"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l1.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "seq_lens"
+      },
       {
-       "buf": "x"
+       "f32": 0.0625
       },
       {
-       "buf": "y"
+       "buf": "kv_scales"
       },
       {
-       "buf": "residual"
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "buf": "model.layers.1.post_attention_layernorm.weight_p1"
+       "f32": 1.0
       },
       {
-       "i32": 5120
+       "f32": 0.0
       },
       {
-       "sym": "tokens"
+       "i64": 8
       },
       {
-       "i32": 5120
+       "i64": 6144
       },
       {
-       "i32": 5120
+       "i64": 256
       },
       {
-       "f32": 9.999999974752427e-07
+       "i64": 6144
+      },
+      {
+       "i64": 256
+      },
+      {
+       "i64": 0
+      },
+      {
+       "buf": "seq_lens"
+      },
+      {
+       "i64": 25690112
+      },
+      {
+       "i64": 2048
+      },
+      {
+       "i64": 512
+      },
+      {
+       "i64": 25690112
+      },
+      {
+       "i64": 2048
+      },
+      {
+       "i64": 512
+      },
+      {
+       "buf": "cu_seqlens_q"
+      },
+      {
+       "i32": 1
+      },
+      {
+       "i64": 0
+      },
+      {
+       "i64": 0
       }
      ]
     },
     {
-     "label": "l1.gate_up",
-     "kernel": "gemm",
+     "label": "l3.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.1.mlp.gate_up_proj.weight"
+       "buf": "attn_out"
       },
       {
-       "buf": "gate_up"
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.3.self_attn.o_proj.weight"
       },
       {
-       "i32": 34816
+       "buf": "residual"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l1.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
+       "buf": "model.layers.3.post_attention_layernorm.weight_p1"
+      },
       {
-       "buf": "act"
+       "sym": "tokens"
       },
       {
-       "buf": "gate_up"
+       "i32": 5120
       },
       {
-       "i32": 17408
+       "i32": 6144
       },
       {
-       "i32": 0
+       "i32": 256
       },
       {
-       "f32": 1.0
+       "i32": 14336
       },
       {
-       "i32": 0
+       "i32": 512
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l1.down_proj",
-     "kernel": "gemm",
+     "label": "l3.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
        "buf": "act"
       },
       {
-       "buf": "model.layers.1.mlp.down_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.3.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 17408
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l1.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l3.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.3.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.2.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.4.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -112058,7 +61715,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -112066,17 +61723,23 @@
      ]
     },
     {
-     "label": "l2.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l4.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.2.linear_attn.in_proj_qkvz.weight"
+       "buf": "model.layers.4.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.4.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
@@ -112084,367 +61747,398 @@
       {
        "i32": 16384
       },
+      {
+       "i32": 96
+      },
       {
        "i32": 5120
       }
      ]
     },
     {
-     "label": "l2.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l4.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "x"
+       "buf": "qkvz"
       },
       {
-       "buf": "model.layers.2.linear_attn.in_proj_ba.weight"
+       "buf": "model.layers.4.linear_attn.conv1d.weight"
       },
       {
-       "buf": "ba"
+       "state": "gdn"
       },
       {
-       "sym": "tokens"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 96
+       "buf": "gdn.spec_line_index",
+       "offset": 16
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l2.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
+       "buf": "gdn.one"
+      },
       {
-       "buf": "qkvz"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "model.layers.2.linear_attn.conv1d.weight"
+       "buf": "gdn.spec_slots",
+       "offset": 96
       },
       {
-       "state": "gdn"
+       "buf": "ba"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 12
+       "buf": "model.layers.4.linear_attn.A_log"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "model.layers.4.linear_attn.dt_bias"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.4.linear_attn.norm.weight"
       },
       {
-       "i32": 1
+       "buf": "gdn_q"
       },
       {
-       "i32": 385
+       "buf": "gdn_k"
       },
       {
-       "i64": 16384
+       "buf": "gdn_v"
       },
       {
-       "i64": 16384
+       "buf": "g"
       },
       {
-       "i64": 0
+       "buf": "beta"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l2.post_conv",
-     "kernel": "post_conv",
-     "args": [
+       "buf": "a_c"
+      },
       {
-       "buf": "qkvz"
+       "buf": "b_c"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "z_c"
       },
       {
-       "buf": "ba"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "model.layers.2.linear_attn.A_log"
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "model.layers.2.linear_attn.dt_bias"
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_q"
+       "i32": 385
       },
       {
-       "buf": "gdn_k"
+       "i32": 1703936
       },
       {
-       "buf": "gdn_v"
+       "i32": 10
       },
       {
-       "buf": "g"
+       "i32": 851968
+      }
+     ]
+    },
+    {
+     "label": "l4.out_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "buf": "beta"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.4.linear_attn.out_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
       },
       {
-       "i32": 96
+       "buf": "model.layers.4.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 2048
+       "sym": "tokens"
       },
       {
-       "i32": 2048
+       "i32": 5120
       },
       {
        "i32": 6144
       },
+      {
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l4.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
+     "args": [
+      {
+       "buf": "act"
+      },
+      {
+       "buf": "x"
+      },
+      {
+       "buf": "model.layers.4.mlp.gate_up_proj.weight"
+      },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 17408
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l2.a_copy",
-     "kernel": "copy_rows",
+     "label": "l4.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "act"
       },
       {
-       "i32": 48
+       "buf": "model.layers.4.mlp.down_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.5.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 17408
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l2.b_copy",
-     "kernel": "copy_rows",
+     "label": "l5.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "b_c"
+       "buf": "qkvz"
       },
       {
        "buf": "ba"
       },
       {
-       "i32": 48
+       "buf": "x"
+      },
+      {
+       "buf": "model.layers.5.linear_attn.in_proj_qkvz.weight"
+      },
+      {
+       "buf": "model.layers.5.linear_attn.in_proj_ba.weight"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 16384
       },
       {
        "i32": 96
       },
       {
-       "i32": 48
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l2.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l5.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "model.layers.2.linear_attn.A_log"
+       "buf": "qkvz"
       },
       {
-       "buf": "a_c"
+       "buf": "model.layers.5.linear_attn.conv1d.weight"
       },
       {
-       "buf": "b_c"
+       "state": "gdn"
       },
       {
-       "buf": "model.layers.2.linear_attn.dt_bias"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 20
       },
       {
-       "f32": 20.0
+       "buf": "gdn.one"
       },
       {
-       "buf": "gdn_q"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "gdn_k"
+       "buf": "gdn.spec_slots",
+       "offset": 128
       },
       {
-       "buf": "gdn_v"
+       "buf": "ba"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.5.linear_attn.A_log"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "model.layers.5.linear_attn.dt_bias"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "core_attn_out"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "model.layers.5.linear_attn.norm.weight"
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 64
+       "buf": "gdn_q"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "gdn_k"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "gdn_v"
       },
       {
-       "i64": 1
+       "buf": "g"
       },
       {
-       "sym": "tokens"
+       "buf": "beta"
       },
       {
-       "i64": 0
+       "buf": "a_c"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l2.z_copy",
-     "kernel": "copy_rows",
-     "args": [
+       "buf": "b_c"
+      },
       {
        "buf": "z_c"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 6144
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 16384
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 385
+      },
+      {
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l2.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l5.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.2.linear_attn.norm.weight"
+       "buf": "model.layers.5.linear_attn.out_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.5.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l2.out_proj",
-     "kernel": "gemm",
+     "label": "l5.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.2.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.5.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l2.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l5.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.5.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.2.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.6.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -112453,7 +62147,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -112461,23 +62155,23 @@
      ]
     },
     {
-     "label": "l2.gate_up",
+     "label": "l5.fc_tap",
      "kernel": "gemm",
      "args": [
       {
-       "buf": "x"
+       "buf": "residual"
       },
       {
-       "buf": "model.layers.2.mlp.gate_up_proj.weight"
+       "buf": "draft.fc.0.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "fc_out"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 5120
       },
       {
        "i32": 5120
@@ -112485,197 +62179,215 @@
      ]
     },
     {
-     "label": "l2.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l6.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "ba"
       },
       {
-       "i32": 17408
+       "buf": "x"
       },
       {
-       "i32": 0
+       "buf": "model.layers.6.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "f32": 1.0
+       "buf": "model.layers.6.linear_attn.in_proj_ba.weight"
       },
       {
-       "i32": 0
+       "sym": "tokens"
+      },
+      {
+       "i32": 16384
+      },
+      {
+       "i32": 96
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l2.down_proj",
-     "kernel": "gemm",
+     "label": "l6.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "model.layers.2.mlp.down_proj.weight"
+       "buf": "model.layers.6.linear_attn.conv1d.weight"
       },
       {
-       "buf": "y"
+       "state": "gdn"
       },
       {
-       "sym": "tokens"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 5120
+       "buf": "gdn.spec_line_index",
+       "offset": 24
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l2.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "gdn.one"
+      },
       {
-       "buf": "x"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "y"
+       "buf": "gdn.spec_slots",
+       "offset": 160
       },
       {
-       "buf": "residual"
+       "buf": "ba"
       },
       {
-       "buf": "model.layers.3.input_layernorm.weight_p1"
+       "buf": "model.layers.6.linear_attn.A_log"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.6.linear_attn.dt_bias"
       },
       {
-       "sym": "tokens"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.6.linear_attn.norm.weight"
       },
       {
-       "i32": 5120
+       "buf": "gdn_q"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l3.qkv_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "gdn_k"
+      },
       {
-       "buf": "x"
+       "buf": "gdn_v"
       },
       {
-       "buf": "model.layers.3.self_attn.qkv_proj.weight"
+       "buf": "g"
       },
       {
-       "buf": "qkv"
+       "buf": "beta"
+      },
+      {
+       "buf": "a_c"
+      },
+      {
+       "buf": "b_c"
+      },
+      {
+       "buf": "z_c"
+      },
+      {
+       "f32": 0.0883883461356163
+      },
+      {
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 14336
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l3.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l6.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.3.self_attn.q_norm.weight_p1"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 256
+       "buf": "model.layers.6.linear_attn.out_proj.weight"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "buf": "residual"
       },
       {
-       "i32": 24
+       "buf": "model.layers.6.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 14336
+       "sym": "tokens"
       },
       {
-       "i32": 512
+       "i32": 5120
       },
       {
        "i32": 6144
       },
-      {
-       "i32": 256
-      },
       {
        "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l3.k_norm",
-     "kernel": "gemma_norm_khead",
+     "label": "l6.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "k_n"
+       "buf": "act"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
+       "buf": "x"
       },
       {
-       "buf": "model.layers.3.self_attn.k_norm.weight_p1"
+       "buf": "model.layers.6.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 256
+       "sym": "tokens"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
+       "i32": 17408
+      },
+      {
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l6.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "i32": 4
+       "buf": "act"
       },
       {
-       "i32": 14336
+       "buf": "model.layers.6.mlp.down_proj.weight"
       },
       {
-       "i32": 256
+       "buf": "residual"
       },
       {
-       "i32": 1024
+       "buf": "model.layers.7.input_layernorm.weight_p1"
       },
       {
-       "i32": 256
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -112683,92 +62395,105 @@
      ]
     },
     {
-     "label": "l3.rope",
-     "kernel": "mrope",
+     "label": "l7.qkv_proj",
+     "kernel": "gemm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
+       "buf": "x"
       },
       {
-       "buf": "cos_g"
+       "buf": "model.layers.7.self_attn.qkv_proj.weight"
       },
       {
-       "buf": "sin_g"
+       "buf": "qkv"
       },
       {
-       "i32": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l3.kv_write",
-     "kernel": "reshape_and_cache",
+     "label": "l7.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "k_n"
+       "buf": "q_n"
       },
       {
-       "buf": "qkv",
-       "offset": 26624
+       "buf": "k_n"
       },
       {
-       "state": "kv"
+       "buf": "qkv"
       },
       {
-       "state": "kv",
-       "offset": 512
+       "buf": "model.layers.7.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "slot_mapping"
+       "buf": "model.layers.7.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "kv_scales"
+       "buf": "cos_g"
       },
       {
-       "buf": "kv_scales",
-       "offset": 4
+       "buf": "sin_g"
       },
       {
-       "i64": 1024
+       "state": "kv",
+       "offset": 3211264
       },
       {
-       "i64": 14336
+       "state": "kv",
+       "offset": 3211776
       },
       {
-       "i64": 25690112
+       "buf": "slot_mapping"
       },
       {
-       "i64": 512
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 0
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i64": 0
+       "i64": 25690112
       },
       {
        "i64": 2048
       },
       {
-       "i64": 0
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l3.attn",
-     "kernel": "attn_prefill",
+     "label": "l7.attn",
+     "kernel": "attn",
      "args": [
       {
        "buf": "attn_out"
@@ -112777,11 +62502,12 @@
        "buf": "q_n"
       },
       {
-       "state": "kv"
+       "state": "kv",
+       "offset": 3211264
       },
       {
        "state": "kv",
-       "offset": 512
+       "offset": 3211776
       },
       {
        "buf": "block_table"
@@ -112859,11 +62585,11 @@
      ]
     },
     {
-     "label": "l3.gate",
-     "kernel": "sigmoid_mul",
+     "label": "l7.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "gated"
+       "buf": "x"
       },
       {
        "buf": "attn_out"
@@ -112873,7 +62599,22 @@
        "offset": 512
       },
       {
-       "i32": 24
+       "buf": "model.layers.7.self_attn.o_proj.weight"
+      },
+      {
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.7.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 6144
       },
       {
        "i32": 256
@@ -112883,51 +62624,54 @@
       },
       {
        "i32": 512
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l3.o_proj",
-     "kernel": "gemm",
+     "label": "l7.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "gated"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.3.self_attn.o_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.7.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l3.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l7.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.7.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.3.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.8.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -112936,7 +62680,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -112944,23 +62688,32 @@
      ]
     },
     {
-     "label": "l3.gate_up",
-     "kernel": "gemm",
+     "label": "l8.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.3.mlp.gate_up_proj.weight"
+       "buf": "model.layers.8.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.8.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -112968,293 +62721,281 @@
      ]
     },
     {
-     "label": "l3.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l8.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.8.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 28
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l3.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "gdn.one"
       },
       {
-       "buf": "model.layers.3.mlp.down_proj.weight"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "y"
+       "buf": "gdn.spec_slots",
+       "offset": 192
       },
       {
-       "sym": "tokens"
+       "buf": "ba"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.8.linear_attn.A_log"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l3.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "model.layers.8.linear_attn.dt_bias"
+      },
       {
-       "buf": "x"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.8.linear_attn.norm.weight"
       },
       {
-       "buf": "residual"
+       "buf": "gdn_q"
       },
       {
-       "buf": "model.layers.4.input_layernorm.weight_p1"
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
+       "buf": "gdn_v"
       },
       {
-       "sym": "tokens"
+       "buf": "g"
       },
       {
-       "i32": 5120
+       "buf": "beta"
       },
       {
-       "i32": 5120
+       "buf": "a_c"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l4.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
+       "buf": "b_c"
+      },
       {
-       "buf": "x"
+       "buf": "z_c"
       },
       {
-       "buf": "model.layers.4.linear_attn.in_proj_qkvz.weight"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "qkvz"
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l4.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l8.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.4.linear_attn.in_proj_ba.weight"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.8.linear_attn.out_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 96
+       "buf": "model.layers.8.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l4.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l8.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.4.linear_attn.conv1d.weight"
+       "buf": "x"
       },
       {
-       "state": "gdn"
+       "buf": "model.layers.8.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 16
+       "sym": "tokens"
       },
       {
-       "buf": "num_accepted_tokens"
+       "i32": 17408
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l8.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "buf": "qkvz"
+       "buf": "act"
       },
       {
-       "i32": 1
+       "buf": "model.layers.8.mlp.down_proj.weight"
       },
       {
-       "i32": 385
+       "buf": "residual"
       },
       {
-       "i64": 16384
+       "buf": "model.layers.9.input_layernorm.weight_p1"
       },
       {
-       "i64": 16384
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l4.post_conv",
-     "kernel": "post_conv",
+     "label": "l9.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
        "buf": "qkvz"
       },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
       {
        "buf": "ba"
       },
       {
-       "buf": "model.layers.4.linear_attn.A_log"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.4.linear_attn.dt_bias"
+       "buf": "model.layers.9.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gdn_q"
+       "buf": "model.layers.9.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "gdn_k"
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_v"
+       "i32": 16384
       },
       {
-       "buf": "g"
+       "i32": 96
       },
       {
-       "buf": "beta"
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l9.gdn",
+     "kernel": "gdn_spec",
+     "args": [
       {
-       "i32": 16384
+       "buf": "qkvz"
       },
       {
-       "i32": 96
+       "buf": "model.layers.9.linear_attn.conv1d.weight"
       },
       {
-       "i32": 96
+       "state": "gdn"
       },
       {
-       "i32": 2048
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 2048
+       "buf": "gdn.spec_line_index",
+       "offset": 32
       },
       {
-       "i32": 6144
+       "buf": "gdn.one"
       },
       {
-       "sym": "tokens"
+       "buf": "cu_seqlens_q"
       },
       {
-       "i64": 0
+       "buf": "gdn.spec_slots",
+       "offset": 224
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l4.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
+       "buf": "ba"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.9.linear_attn.A_log"
       },
       {
-       "i32": 48
+       "buf": "model.layers.9.linear_attn.dt_bias"
       },
       {
-       "i32": 96
+       "buf": "core_attn_out"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l4.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.9.linear_attn.norm.weight"
       },
       {
-       "buf": "ba"
+       "buf": "gdn_q"
       },
       {
-       "i32": 48
+       "buf": "gdn_k"
       },
       {
-       "i32": 96
+       "buf": "gdn_v"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l4.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
+       "buf": "g"
+      },
       {
-       "buf": "model.layers.4.linear_attn.A_log"
+       "buf": "beta"
       },
       {
        "buf": "a_c"
@@ -113263,276 +63004,265 @@
        "buf": "b_c"
       },
       {
-       "buf": "model.layers.4.linear_attn.dt_bias"
+       "buf": "z_c"
       },
       {
-       "f32": 1.0
+       "f32": 0.0883883461356163
       },
       {
-       "f32": 20.0
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "gdn_q"
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_k"
+       "i32": 385
       },
       {
-       "buf": "gdn_v"
+       "i32": 1703936
       },
       {
-       "buf": "core_attn_out"
+       "i32": 10
       },
       {
-       "state": "gdn",
-       "offset": 204800
-      },
+       "i32": 851968
+      }
+     ]
+    },
+    {
+     "label": "l9.out_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "x"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 96
+       "buf": "model.layers.9.linear_attn.out_proj.weight"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "residual"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "model.layers.9.post_attention_layernorm.weight_p1"
       },
       {
-       "i64": 1
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 6144
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l4.z_copy",
-     "kernel": "copy_rows",
+     "label": "l9.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "act"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "x"
       },
       {
-       "i32": 6144
+       "buf": "model.layers.9.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 16384
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 17408
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l4.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l9.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.4.linear_attn.norm.weight"
+       "buf": "model.layers.9.mlp.down_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.10.input_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l4.out_proj",
-     "kernel": "gemm",
+     "label": "l10.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "qkvz"
       },
       {
-       "buf": "model.layers.4.linear_attn.out_proj.weight"
+       "buf": "ba"
       },
       {
-       "buf": "y"
+       "buf": "x"
+      },
+      {
+       "buf": "model.layers.10.linear_attn.in_proj_qkvz.weight"
+      },
+      {
+       "buf": "model.layers.10.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 16384
       },
       {
-       "i32": 6144
+       "i32": 96
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l4.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l10.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "x"
+       "buf": "qkvz"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.10.linear_attn.conv1d.weight"
       },
       {
-       "buf": "residual"
+       "state": "gdn"
       },
       {
-       "buf": "model.layers.4.post_attention_layernorm.weight_p1"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 5120
+       "buf": "gdn.spec_line_index",
+       "offset": 36
       },
       {
-       "sym": "tokens"
+       "buf": "gdn.one"
       },
       {
-       "i32": 5120
+       "buf": "cu_seqlens_q"
       },
       {
-       "i32": 5120
+       "buf": "gdn.spec_slots",
+       "offset": 256
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l4.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
+       "buf": "ba"
       },
       {
-       "buf": "model.layers.4.mlp.gate_up_proj.weight"
+       "buf": "model.layers.10.linear_attn.A_log"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.10.linear_attn.dt_bias"
       },
       {
-       "sym": "tokens"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 34816
+       "buf": "model.layers.10.linear_attn.norm.weight"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l4.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
+       "buf": "gdn_q"
+      },
       {
-       "buf": "act"
+       "buf": "gdn_k"
       },
       {
-       "buf": "gate_up"
+       "buf": "gdn_v"
       },
       {
-       "i32": 17408
+       "buf": "g"
       },
       {
-       "i32": 0
+       "buf": "beta"
       },
       {
-       "f32": 1.0
+       "buf": "a_c"
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l4.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "b_c"
+      },
       {
-       "buf": "act"
+       "buf": "z_c"
       },
       {
-       "buf": "model.layers.4.mlp.down_proj.weight"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "y"
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 385
       },
       {
-       "i32": 17408
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l4.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l10.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.10.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.5.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.10.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -113541,7 +63271,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -113549,23 +63279,23 @@
      ]
     },
     {
-     "label": "l5.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l10.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.5.linear_attn.in_proj_qkvz.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.10.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -113573,242 +63303,219 @@
      ]
     },
     {
-     "label": "l5.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l10.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.5.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
+       "buf": "act"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l5.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
+       "buf": "model.layers.10.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.5.linear_attn.conv1d.weight"
+       "buf": "residual"
       },
       {
-       "state": "gdn"
+       "buf": "model.layers.11.input_layernorm.weight_p1"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 20
+       "sym": "tokens"
       },
       {
-       "buf": "num_accepted_tokens"
+       "i32": 5120
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 17408
       },
       {
-       "buf": "qkvz"
-      },
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l11.qkv_proj",
+     "kernel": "gemm",
+     "args": [
       {
-       "i32": 1
+       "buf": "x"
       },
       {
-       "i32": 385
+       "buf": "model.layers.11.self_attn.qkv_proj.weight"
       },
       {
-       "i64": 16384
+       "buf": "qkv"
       },
       {
-       "i64": 16384
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l5.post_conv",
-     "kernel": "post_conv",
+     "label": "l11.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
+       "buf": "q_n"
       },
       {
-       "buf": "model.layers.5.linear_attn.A_log"
+       "buf": "k_n"
       },
       {
-       "buf": "model.layers.5.linear_attn.dt_bias"
+       "buf": "qkv"
       },
       {
-       "buf": "gdn_q"
+       "buf": "model.layers.11.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "gdn_k"
+       "buf": "model.layers.11.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "gdn_v"
+       "buf": "cos_g"
       },
       {
-       "buf": "g"
+       "buf": "sin_g"
       },
       {
-       "buf": "beta"
+       "state": "kv",
+       "offset": 6422528
       },
       {
-       "i32": 16384
+       "state": "kv",
+       "offset": 6423040
       },
       {
-       "i32": 96
+       "buf": "slot_mapping"
       },
       {
-       "i32": 96
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i32": 2048
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i32": 2048
+       "i64": 25690112
       },
       {
-       "i32": 6144
+       "i64": 2048
       },
       {
-       "sym": "tokens"
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l5.a_copy",
-     "kernel": "copy_rows",
+     "label": "l11.attn",
+     "kernel": "attn",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "attn_out"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "q_n"
       },
       {
-       "i32": 48
+       "state": "kv",
+       "offset": 6422528
       },
       {
-       "i32": 96
+       "state": "kv",
+       "offset": 6423040
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l5.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "block_table"
       },
       {
-       "buf": "ba"
+       "buf": "seq_lens"
       },
       {
-       "i32": 48
+       "f32": 0.0625
       },
       {
-       "i32": 96
+       "buf": "kv_scales"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l5.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.5.linear_attn.A_log"
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "buf": "a_c"
+       "f32": 1.0
       },
       {
-       "buf": "b_c"
+       "f32": 0.0
       },
       {
-       "buf": "model.layers.5.linear_attn.dt_bias"
+       "i64": 8
       },
       {
-       "f32": 1.0
+       "i64": 6144
       },
       {
-       "f32": 20.0
+       "i64": 256
       },
       {
-       "buf": "gdn_q"
+       "i64": 6144
       },
       {
-       "buf": "gdn_k"
+       "i64": 256
       },
       {
-       "buf": "gdn_v"
+       "i64": 0
       },
       {
-       "buf": "core_attn_out"
+       "buf": "seq_lens"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "i64": 25690112
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "i64": 2048
       },
       {
-       "buf": "cu_seqlens_q"
+       "i64": 512
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 128
+       "i64": 25690112
       },
       {
-       "buf": "num_accepted_tokens"
+       "i64": 2048
       },
       {
-       "f32": 0.0883883461356163
+       "i64": 512
       },
       {
-       "i64": 1
+       "buf": "cu_seqlens_q"
       },
       {
-       "sym": "tokens"
+       "i32": 1
       },
       {
        "i64": 0
@@ -113819,115 +63526,93 @@
      ]
     },
     {
-     "label": "l5.z_copy",
-     "kernel": "copy_rows",
+     "label": "l11.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "x"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "attn_out"
       },
       {
-       "i32": 6144
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "i32": 16384
+       "buf": "model.layers.11.self_attn.o_proj.weight"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l5.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
+       "buf": "residual"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.11.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "model.layers.5.linear_attn.norm.weight"
+       "sym": "tokens"
       },
       {
-       "buf": "z_c"
+       "i32": 5120
       },
       {
-       "i32": 128
+       "i32": 6144
       },
       {
-       "i32": 128
+       "i32": 256
       },
       {
-       "i32": 128
+       "i32": 14336
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 512
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l5.out_proj",
-     "kernel": "gemm",
+     "label": "l11.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.5.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.11.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l5.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l11.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.11.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.5.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.12.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -113936,7 +63621,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -113944,23 +63629,32 @@
      ]
     },
     {
-     "label": "l5.gate_up",
-     "kernel": "gemm",
+     "label": "l12.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.5.mlp.gate_up_proj.weight"
+       "buf": "model.layers.12.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.12.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -113968,152 +63662,149 @@
      ]
     },
     {
-     "label": "l5.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l12.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.12.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 40
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l5.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "gdn.one"
       },
       {
-       "buf": "model.layers.5.mlp.down_proj.weight"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "y"
+       "buf": "gdn.spec_slots",
+       "offset": 288
       },
       {
-       "sym": "tokens"
+       "buf": "ba"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.12.linear_attn.A_log"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l5.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "model.layers.12.linear_attn.dt_bias"
+      },
       {
-       "buf": "x"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.12.linear_attn.norm.weight"
       },
       {
-       "buf": "residual"
+       "buf": "gdn_q"
       },
       {
-       "buf": "model.layers.6.input_layernorm.weight_p1"
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
+       "buf": "gdn_v"
       },
       {
-       "sym": "tokens"
+       "buf": "g"
       },
       {
-       "i32": 5120
+       "buf": "beta"
       },
       {
-       "i32": 5120
+       "buf": "a_c"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l5.fc_tap",
-     "kernel": "gemm",
-     "args": [
+       "buf": "b_c"
+      },
       {
-       "buf": "residual"
+       "buf": "z_c"
       },
       {
-       "buf": "draft.fc.0.weight"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "fc_out"
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l6.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l12.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.6.linear_attn.in_proj_qkvz.weight"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.12.linear_attn.out_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.12.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l6.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l12.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.6.linear_attn.in_proj_ba.weight"
+       "buf": "x"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.12.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 96
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -114121,337 +63812,239 @@
      ]
     },
     {
-     "label": "l6.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l12.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.6.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 24
-      },
-      {
-       "buf": "num_accepted_tokens"
+       "buf": "x"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "act"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.12.mlp.down_proj.weight"
       },
       {
-       "i32": 1
+       "buf": "residual"
       },
       {
-       "i32": 385
+       "buf": "model.layers.13.input_layernorm.weight_p1"
       },
       {
-       "i64": 16384
+       "sym": "tokens"
       },
       {
-       "i64": 16384
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 17408
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l6.post_conv",
-     "kernel": "post_conv",
+     "label": "l13.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
        "buf": "qkvz"
       },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
       {
        "buf": "ba"
       },
       {
-       "buf": "model.layers.6.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.6.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
-      },
-      {
-       "buf": "g"
-      },
-      {
-       "buf": "beta"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
+       "buf": "x"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l6.a_copy",
-     "kernel": "copy_rows",
-     "args": [
+       "buf": "model.layers.13.linear_attn.in_proj_qkvz.weight"
+      },
       {
-       "buf": "a_c"
+       "buf": "model.layers.13.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 16384
       },
       {
        "i32": 96
       },
       {
-       "i32": 48
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l6.b_copy",
-     "kernel": "copy_rows",
+     "label": "l13.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "b_c"
+       "buf": "qkvz"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.13.linear_attn.conv1d.weight"
       },
       {
-       "i32": 48
+       "state": "gdn"
       },
       {
-       "i32": 96
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l6.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.6.linear_attn.A_log"
+       "buf": "gdn.spec_line_index",
+       "offset": 44
       },
       {
-       "buf": "a_c"
+       "buf": "gdn.one"
       },
       {
-       "buf": "b_c"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "model.layers.6.linear_attn.dt_bias"
+       "buf": "gdn.spec_slots",
+       "offset": 320
       },
       {
-       "f32": 1.0
+       "buf": "ba"
       },
       {
-       "f32": 20.0
+       "buf": "model.layers.13.linear_attn.A_log"
       },
       {
-       "buf": "gdn_q"
+       "buf": "model.layers.13.linear_attn.dt_bias"
       },
       {
-       "buf": "gdn_k"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "gdn_v"
+       "buf": "model.layers.13.linear_attn.norm.weight"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "gdn_q"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "gdn_k"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "gdn_v"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "g"
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 160
+       "buf": "beta"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "a_c"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "b_c"
       },
       {
-       "i64": 1
+       "buf": "z_c"
       },
       {
-       "sym": "tokens"
+       "f32": 0.0883883461356163
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l6.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
+       "sym": "tokens"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "i32": 385
       },
       {
-       "i32": 6144
+       "i32": 1703936
       },
       {
-       "i32": 16384
+       "i32": 10
       },
       {
-       "i32": 6144
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l6.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l13.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.6.linear_attn.norm.weight"
+       "buf": "model.layers.13.linear_attn.out_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.13.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l6.out_proj",
-     "kernel": "gemm",
+     "label": "l13.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.6.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.13.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l6.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l13.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.13.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.6.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.14.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -114460,7 +64053,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -114468,23 +64061,32 @@
      ]
     },
     {
-     "label": "l6.gate_up",
-     "kernel": "gemm",
+     "label": "l14.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.6.mlp.gate_up_proj.weight"
+       "buf": "model.layers.14.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.14.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -114492,197 +64094,182 @@
      ]
     },
     {
-     "label": "l6.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l14.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.14.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 48
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l6.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "gdn.one"
       },
       {
-       "buf": "model.layers.6.mlp.down_proj.weight"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "y"
+       "buf": "gdn.spec_slots",
+       "offset": 352
       },
       {
-       "sym": "tokens"
+       "buf": "ba"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.14.linear_attn.A_log"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l6.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "model.layers.14.linear_attn.dt_bias"
+      },
       {
-       "buf": "x"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.14.linear_attn.norm.weight"
       },
       {
-       "buf": "residual"
+       "buf": "gdn_q"
       },
       {
-       "buf": "model.layers.7.input_layernorm.weight_p1"
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
+       "buf": "gdn_v"
       },
       {
-       "sym": "tokens"
+       "buf": "g"
       },
       {
-       "i32": 5120
+       "buf": "beta"
       },
       {
-       "i32": 5120
+       "buf": "a_c"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l7.qkv_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "b_c"
+      },
       {
-       "buf": "x"
+       "buf": "z_c"
       },
       {
-       "buf": "model.layers.7.self_attn.qkv_proj.weight"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "qkv"
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 14336
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l7.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l14.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.7.self_attn.q_norm.weight_p1"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 256
+       "buf": "model.layers.14.linear_attn.out_proj.weight"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "buf": "residual"
       },
       {
-       "i32": 24
+       "buf": "model.layers.14.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 14336
+       "sym": "tokens"
       },
       {
-       "i32": 512
+       "i32": 5120
       },
       {
        "i32": 6144
       },
-      {
-       "i32": 256
-      },
       {
        "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l7.k_norm",
-     "kernel": "gemma_norm_khead",
+     "label": "l14.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "k_n"
+       "buf": "act"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
+       "buf": "x"
       },
       {
-       "buf": "model.layers.7.self_attn.k_norm.weight_p1"
+       "buf": "model.layers.14.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 256
+       "sym": "tokens"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
+       "i32": 17408
       },
       {
-       "i32": 4
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l14.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "i32": 14336
+       "buf": "act"
       },
       {
-       "i32": 256
+       "buf": "model.layers.14.mlp.down_proj.weight"
+      },
+      {
+       "buf": "residual"
       },
       {
-       "i32": 1024
+       "buf": "model.layers.15.input_layernorm.weight_p1"
       },
       {
-       "i32": 256
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -114690,93 +64277,105 @@
      ]
     },
     {
-     "label": "l7.rope",
-     "kernel": "mrope",
+     "label": "l15.qkv_proj",
+     "kernel": "gemm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
+       "buf": "x"
       },
       {
-       "buf": "cos_g"
+       "buf": "model.layers.15.self_attn.qkv_proj.weight"
       },
       {
-       "buf": "sin_g"
+       "buf": "qkv"
       },
       {
-       "i32": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l7.kv_write",
-     "kernel": "reshape_and_cache",
+     "label": "l15.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "k_n"
+       "buf": "q_n"
       },
       {
-       "buf": "qkv",
-       "offset": 26624
+       "buf": "k_n"
       },
       {
-       "state": "kv",
-       "offset": 3211264
+       "buf": "qkv"
       },
       {
-       "state": "kv",
-       "offset": 3211776
+       "buf": "model.layers.15.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "slot_mapping"
+       "buf": "model.layers.15.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "kv_scales"
+       "buf": "cos_g"
       },
       {
-       "buf": "kv_scales",
-       "offset": 4
+       "buf": "sin_g"
       },
       {
-       "i64": 1024
+       "state": "kv",
+       "offset": 9633792
       },
       {
-       "i64": 14336
+       "state": "kv",
+       "offset": 9634304
       },
       {
-       "i64": 25690112
+       "buf": "slot_mapping"
       },
       {
-       "i64": 512
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 0
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i64": 0
+       "i64": 25690112
       },
       {
        "i64": 2048
       },
       {
-       "i64": 0
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l7.attn",
-     "kernel": "attn_prefill",
+     "label": "l15.attn",
+     "kernel": "attn",
      "args": [
       {
        "buf": "attn_out"
@@ -114786,11 +64385,11 @@
       },
       {
        "state": "kv",
-       "offset": 3211264
+       "offset": 9633792
       },
       {
        "state": "kv",
-       "offset": 3211776
+       "offset": 9634304
       },
       {
        "buf": "block_table"
@@ -114868,11 +64467,11 @@
      ]
     },
     {
-     "label": "l7.gate",
-     "kernel": "sigmoid_mul",
+     "label": "l15.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "gated"
+       "buf": "x"
       },
       {
        "buf": "attn_out"
@@ -114882,7 +64481,22 @@
        "offset": 512
       },
       {
-       "i32": 24
+       "buf": "model.layers.15.self_attn.o_proj.weight"
+      },
+      {
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.15.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 6144
       },
       {
        "i32": 256
@@ -114892,51 +64506,54 @@
       },
       {
        "i32": 512
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l7.o_proj",
-     "kernel": "gemm",
+     "label": "l15.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "gated"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.7.self_attn.o_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.15.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l7.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l15.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.15.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.7.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.16.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -114945,7 +64562,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -114953,23 +64570,32 @@
      ]
     },
     {
-     "label": "l7.gate_up",
-     "kernel": "gemm",
+     "label": "l16.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.7.mlp.gate_up_proj.weight"
+       "buf": "model.layers.16.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.16.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -114977,128 +64603,149 @@
      ]
     },
     {
-     "label": "l7.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l16.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.16.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 52
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l7.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "gdn.one"
+      },
       {
-       "buf": "act"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "model.layers.7.mlp.down_proj.weight"
+       "buf": "gdn.spec_slots",
+       "offset": 384
       },
       {
-       "buf": "y"
+       "buf": "ba"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.16.linear_attn.A_log"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.16.linear_attn.dt_bias"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l7.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "core_attn_out"
+      },
       {
-       "buf": "x"
+       "buf": "model.layers.16.linear_attn.norm.weight"
       },
       {
-       "buf": "y"
+       "buf": "gdn_q"
       },
       {
-       "buf": "residual"
+       "buf": "gdn_k"
+      },
+      {
+       "buf": "gdn_v"
+      },
+      {
+       "buf": "g"
+      },
+      {
+       "buf": "beta"
+      },
+      {
+       "buf": "a_c"
+      },
+      {
+       "buf": "b_c"
+      },
+      {
+       "buf": "z_c"
       },
       {
-       "buf": "model.layers.8.input_layernorm.weight_p1"
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 5120
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
       },
       {
-       "f32": 9.999999974752427e-07
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l8.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l16.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.8.linear_attn.in_proj_qkvz.weight"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.16.linear_attn.out_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.16.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l8.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l16.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.8.linear_attn.in_proj_ba.weight"
+       "buf": "x"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.16.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 96
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -115106,85 +64753,59 @@
      ]
     },
     {
-     "label": "l8.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l16.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.8.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 28
-      },
-      {
-       "buf": "num_accepted_tokens"
+       "buf": "x"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "act"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.16.mlp.down_proj.weight"
       },
       {
-       "i32": 1
+       "buf": "residual"
       },
       {
-       "i32": 385
+       "buf": "model.layers.17.input_layernorm.weight_p1"
       },
       {
-       "i64": 16384
+       "sym": "tokens"
       },
       {
-       "i64": 16384
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 17408
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l8.post_conv",
-     "kernel": "post_conv",
+     "label": "l17.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
        "buf": "qkvz"
       },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
       {
        "buf": "ba"
       },
       {
-       "buf": "model.layers.8.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.8.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
+       "buf": "x"
       },
       {
-       "buf": "gdn_v"
+       "buf": "model.layers.17.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "g"
+       "buf": "model.layers.17.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "beta"
+       "sym": "tokens"
       },
       {
        "i32": 16384
@@ -115193,92 +64814,55 @@
        "i32": 96
       },
       {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l8.a_copy",
-     "kernel": "copy_rows",
+     "label": "l17.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
+       "buf": "qkvz"
       },
       {
-       "i32": 48
+       "buf": "model.layers.17.linear_attn.conv1d.weight"
       },
       {
-       "i32": 96
+       "state": "gdn"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l8.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "buf": "ba"
+       "buf": "gdn.spec_line_index",
+       "offset": 56
       },
       {
-       "i32": 48
+       "buf": "gdn.one"
       },
       {
-       "i32": 96
+       "buf": "cu_seqlens_q"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l8.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.8.linear_attn.A_log"
+       "buf": "gdn.spec_slots",
+       "offset": 416
       },
       {
-       "buf": "a_c"
+       "buf": "ba"
       },
       {
-       "buf": "b_c"
+       "buf": "model.layers.17.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.8.linear_attn.dt_bias"
+       "buf": "model.layers.17.linear_attn.dt_bias"
       },
       {
-       "f32": 1.0
+       "buf": "core_attn_out"
       },
       {
-       "f32": 20.0
+       "buf": "model.layers.17.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -115290,153 +64874,118 @@
        "buf": "gdn_v"
       },
       {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "g"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "beta"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "a_c"
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 192
+       "buf": "b_c"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "z_c"
       },
       {
        "f32": 0.0883883461356163
       },
       {
-       "i64": 1
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l8.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
+       "i32": 385
       },
       {
-       "i32": 6144
+       "i32": 1703936
       },
       {
-       "i32": 16384
+       "i32": 10
       },
       {
-       "i32": 6144
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l8.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l17.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.8.linear_attn.norm.weight"
+       "buf": "model.layers.17.linear_attn.out_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.17.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l8.out_proj",
-     "kernel": "gemm",
+     "label": "l17.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.8.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.17.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l8.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l17.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.17.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.8.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.18.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -115445,7 +64994,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -115453,23 +65002,32 @@
      ]
     },
     {
-     "label": "l8.gate_up",
-     "kernel": "gemm",
+     "label": "l18.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.8.mlp.gate_up_proj.weight"
+       "buf": "model.layers.18.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.18.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -115477,128 +65035,149 @@
      ]
     },
     {
-     "label": "l8.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l18.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.18.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 60
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l8.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "gdn.one"
+      },
       {
-       "buf": "act"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "model.layers.8.mlp.down_proj.weight"
+       "buf": "gdn.spec_slots",
+       "offset": 448
       },
       {
-       "buf": "y"
+       "buf": "ba"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.18.linear_attn.A_log"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.18.linear_attn.dt_bias"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l8.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "core_attn_out"
+      },
       {
-       "buf": "x"
+       "buf": "model.layers.18.linear_attn.norm.weight"
       },
       {
-       "buf": "y"
+       "buf": "gdn_q"
       },
       {
-       "buf": "residual"
+       "buf": "gdn_k"
       },
       {
-       "buf": "model.layers.9.input_layernorm.weight_p1"
+       "buf": "gdn_v"
       },
       {
-       "i32": 5120
+       "buf": "g"
       },
       {
-       "sym": "tokens"
+       "buf": "beta"
       },
       {
-       "i32": 5120
+       "buf": "a_c"
       },
       {
-       "i32": 5120
+       "buf": "b_c"
+      },
+      {
+       "buf": "z_c"
+      },
+      {
+       "f32": 0.0883883461356163
       },
       {
        "f32": 9.999999974752427e-07
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 385
+      },
+      {
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l9.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l18.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.9.linear_attn.in_proj_qkvz.weight"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.18.linear_attn.out_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.18.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l9.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l18.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.9.linear_attn.in_proj_ba.weight"
+       "buf": "x"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.18.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 96
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -115606,218 +65185,219 @@
      ]
     },
     {
-     "label": "l9.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l18.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.9.linear_attn.conv1d.weight"
+       "buf": "act"
       },
       {
-       "state": "gdn"
+       "buf": "model.layers.18.mlp.down_proj.weight"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 32
+       "buf": "residual"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "model.layers.19.input_layernorm.weight_p1"
       },
       {
-       "buf": "cu_seqlens_q"
+       "sym": "tokens"
       },
       {
-       "buf": "qkvz"
+       "i32": 5120
       },
       {
-       "i32": 1
+       "i32": 17408
       },
       {
-       "i32": 385
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l19.qkv_proj",
+     "kernel": "gemm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "i64": 16384
+       "buf": "model.layers.19.self_attn.qkv_proj.weight"
       },
       {
-       "i64": 16384
+       "buf": "qkv"
       },
       {
-       "i64": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l9.post_conv",
-     "kernel": "post_conv",
+     "label": "l19.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
+       "buf": "q_n"
       },
       {
-       "buf": "model.layers.9.linear_attn.A_log"
+       "buf": "k_n"
       },
       {
-       "buf": "model.layers.9.linear_attn.dt_bias"
+       "buf": "qkv"
       },
       {
-       "buf": "gdn_q"
+       "buf": "model.layers.19.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "gdn_k"
+       "buf": "model.layers.19.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "gdn_v"
+       "buf": "cos_g"
       },
       {
-       "buf": "g"
+       "buf": "sin_g"
       },
       {
-       "buf": "beta"
+       "state": "kv",
+       "offset": 12845056
       },
       {
-       "i32": 16384
+       "state": "kv",
+       "offset": 12845568
       },
       {
-       "i32": 96
+       "buf": "slot_mapping"
       },
       {
-       "i32": 96
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i32": 2048
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i32": 2048
+       "i64": 25690112
       },
       {
-       "i32": 6144
+       "i64": 2048
       },
       {
-       "sym": "tokens"
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l9.a_copy",
-     "kernel": "copy_rows",
+     "label": "l19.attn",
+     "kernel": "attn",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "attn_out"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "q_n"
       },
       {
-       "i32": 48
+       "state": "kv",
+       "offset": 12845056
       },
       {
-       "i32": 96
+       "state": "kv",
+       "offset": 12845568
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l9.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "block_table"
       },
       {
-       "buf": "ba"
+       "buf": "seq_lens"
       },
       {
-       "i32": 48
+       "f32": 0.0625
       },
       {
-       "i32": 96
+       "buf": "kv_scales"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l9.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.9.linear_attn.A_log"
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "buf": "a_c"
+       "f32": 1.0
       },
       {
-       "buf": "b_c"
+       "f32": 0.0
       },
       {
-       "buf": "model.layers.9.linear_attn.dt_bias"
+       "i64": 8
       },
       {
-       "f32": 1.0
+       "i64": 6144
       },
       {
-       "f32": 20.0
+       "i64": 256
       },
       {
-       "buf": "gdn_q"
+       "i64": 6144
       },
       {
-       "buf": "gdn_k"
+       "i64": 256
       },
       {
-       "buf": "gdn_v"
+       "i64": 0
       },
       {
-       "buf": "core_attn_out"
+       "buf": "seq_lens"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "i64": 25690112
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "i64": 2048
       },
       {
-       "buf": "cu_seqlens_q"
+       "i64": 512
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 224
+       "i64": 25690112
       },
       {
-       "buf": "num_accepted_tokens"
+       "i64": 2048
       },
       {
-       "f32": 0.0883883461356163
+       "i64": 512
       },
       {
-       "i64": 1
+       "buf": "cu_seqlens_q"
       },
       {
-       "sym": "tokens"
+       "i32": 1
       },
       {
        "i64": 0
@@ -115828,115 +65408,93 @@
      ]
     },
     {
-     "label": "l9.z_copy",
-     "kernel": "copy_rows",
+     "label": "l19.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "x"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "attn_out"
       },
       {
-       "i32": 6144
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "i32": 16384
+       "buf": "model.layers.19.self_attn.o_proj.weight"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l9.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
+       "buf": "residual"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.19.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "model.layers.9.linear_attn.norm.weight"
+       "sym": "tokens"
       },
       {
-       "buf": "z_c"
+       "i32": 5120
       },
       {
-       "i32": 128
+       "i32": 6144
       },
       {
-       "i32": 128
+       "i32": 256
       },
       {
-       "i32": 128
+       "i32": 14336
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 512
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l9.out_proj",
-     "kernel": "gemm",
+     "label": "l19.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.9.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.19.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l9.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l19.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.19.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.9.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.20.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -115945,7 +65503,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -115953,23 +65511,23 @@
      ]
     },
     {
-     "label": "l9.gate_up",
-     "kernel": "gemm",
+     "label": "l19.fc_tap",
+     "kernel": "gemm_acc",
      "args": [
       {
-       "buf": "x"
+       "buf": "residual"
       },
       {
-       "buf": "model.layers.9.mlp.gate_up_proj.weight"
+       "buf": "draft.fc.1.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "fc_out"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 5120
       },
       {
        "i32": 5120
@@ -115977,308 +65535,299 @@
      ]
     },
     {
-     "label": "l9.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l20.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "ba"
       },
       {
-       "i32": 17408
+       "buf": "x"
       },
       {
-       "i32": 0
+       "buf": "model.layers.20.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "f32": 1.0
+       "buf": "model.layers.20.linear_attn.in_proj_ba.weight"
       },
       {
-       "i32": 0
+       "sym": "tokens"
+      },
+      {
+       "i32": 16384
+      },
+      {
+       "i32": 96
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l9.down_proj",
-     "kernel": "gemm",
+     "label": "l20.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "model.layers.9.mlp.down_proj.weight"
+       "buf": "model.layers.20.linear_attn.conv1d.weight"
       },
       {
-       "buf": "y"
+       "state": "gdn"
       },
       {
-       "sym": "tokens"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 5120
+       "buf": "gdn.spec_line_index",
+       "offset": 64
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l9.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
+       "buf": "gdn.one"
       },
       {
-       "buf": "y"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "residual"
+       "buf": "gdn.spec_slots",
+       "offset": 480
       },
       {
-       "buf": "model.layers.10.input_layernorm.weight_p1"
+       "buf": "ba"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.20.linear_attn.A_log"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.20.linear_attn.dt_bias"
       },
       {
-       "i32": 5120
+       "buf": "core_attn_out"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.20.linear_attn.norm.weight"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l10.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
+       "buf": "gdn_q"
+      },
       {
-       "buf": "x"
+       "buf": "gdn_k"
       },
       {
-       "buf": "model.layers.10.linear_attn.in_proj_qkvz.weight"
+       "buf": "gdn_v"
       },
       {
-       "buf": "qkvz"
+       "buf": "g"
       },
       {
-       "sym": "tokens"
+       "buf": "beta"
       },
       {
-       "i32": 16384
+       "buf": "a_c"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l10.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
+       "buf": "b_c"
+      },
       {
-       "buf": "x"
+       "buf": "z_c"
       },
       {
-       "buf": "model.layers.10.linear_attn.in_proj_ba.weight"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "ba"
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 96
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l10.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l20.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.10.linear_attn.conv1d.weight"
+       "buf": "core_attn_out"
       },
       {
-       "state": "gdn"
+       "buf": "model.layers.20.linear_attn.out_proj.weight"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 36
+       "buf": "residual"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "model.layers.20.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "cu_seqlens_q"
+       "sym": "tokens"
       },
       {
-       "buf": "qkvz"
+       "i32": 5120
       },
       {
-       "i32": 1
+       "i32": 6144
       },
       {
-       "i32": 385
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l20.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
+     "args": [
+      {
+       "buf": "act"
       },
       {
-       "i64": 16384
+       "buf": "x"
       },
       {
-       "i64": 16384
+       "buf": "model.layers.20.mlp.gate_up_proj.weight"
       },
       {
-       "i64": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 17408
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l10.post_conv",
-     "kernel": "post_conv",
+     "label": "l20.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
+       "buf": "x"
       },
       {
-       "buf": "ba"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.10.linear_attn.A_log"
+       "buf": "model.layers.20.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.10.linear_attn.dt_bias"
+       "buf": "residual"
       },
       {
-       "buf": "gdn_q"
+       "buf": "model.layers.21.input_layernorm.weight_p1"
       },
       {
-       "buf": "gdn_k"
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_v"
+       "i32": 5120
       },
       {
-       "buf": "g"
+       "i32": 17408
       },
       {
-       "buf": "beta"
-      },
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l21.in_proj",
+     "kernel": "gemm8_dual",
+     "args": [
       {
-       "i32": 16384
+       "buf": "qkvz"
       },
       {
-       "i32": 96
+       "buf": "ba"
       },
       {
-       "i32": 96
+       "buf": "x"
       },
       {
-       "i32": 2048
+       "buf": "model.layers.21.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i32": 2048
+       "buf": "model.layers.21.linear_attn.in_proj_ba.weight"
       },
       {
-       "i32": 6144
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 16384
       },
       {
-       "i64": 0
+       "i32": 96
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l10.a_copy",
-     "kernel": "copy_rows",
+     "label": "l21.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
+       "buf": "qkvz"
       },
       {
-       "i32": 48
+       "buf": "model.layers.21.linear_attn.conv1d.weight"
       },
       {
-       "i32": 96
+       "state": "gdn"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l10.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "buf": "ba"
+       "buf": "gdn.spec_line_index",
+       "offset": 68
       },
       {
-       "i32": 48
+       "buf": "gdn.one"
       },
       {
-       "i32": 96
+       "buf": "cu_seqlens_q"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l10.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.10.linear_attn.A_log"
+       "buf": "gdn.spec_slots",
+       "offset": 512
       },
       {
-       "buf": "a_c"
+       "buf": "ba"
       },
       {
-       "buf": "b_c"
+       "buf": "model.layers.21.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.10.linear_attn.dt_bias"
+       "buf": "model.layers.21.linear_attn.dt_bias"
       },
       {
-       "f32": 1.0
+       "buf": "core_attn_out"
       },
       {
-       "f32": 20.0
+       "buf": "model.layers.21.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -116290,153 +65839,118 @@
        "buf": "gdn_v"
       },
       {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "g"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "beta"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "a_c"
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 256
+       "buf": "b_c"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "z_c"
       },
       {
        "f32": 0.0883883461356163
       },
       {
-       "i64": 1
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l10.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
+       "i32": 385
       },
       {
-       "i32": 6144
+       "i32": 1703936
       },
       {
-       "i32": 16384
+       "i32": 10
       },
       {
-       "i32": 6144
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l10.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l21.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.10.linear_attn.norm.weight"
+       "buf": "model.layers.21.linear_attn.out_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.21.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l10.out_proj",
-     "kernel": "gemm",
+     "label": "l21.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.10.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.21.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l10.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l21.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.21.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.10.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.22.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -116445,7 +65959,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -116453,23 +65967,32 @@
      ]
     },
     {
-     "label": "l10.gate_up",
-     "kernel": "gemm",
+     "label": "l22.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.10.mlp.gate_up_proj.weight"
+       "buf": "model.layers.22.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.22.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -116477,71 +66000,116 @@
      ]
     },
     {
-     "label": "l10.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l22.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
+      },
+      {
+       "buf": "model.layers.22.linear_attn.conv1d.weight"
+      },
+      {
+       "state": "gdn"
+      },
+      {
+       "state": "gdn",
+       "offset": 204800
+      },
+      {
+       "buf": "gdn.spec_line_index",
+       "offset": 72
+      },
+      {
+       "buf": "gdn.one"
+      },
+      {
+       "buf": "cu_seqlens_q"
+      },
+      {
+       "buf": "gdn.spec_slots",
+       "offset": 544
+      },
+      {
+       "buf": "ba"
+      },
+      {
+       "buf": "model.layers.22.linear_attn.A_log"
+      },
+      {
+       "buf": "model.layers.22.linear_attn.dt_bias"
+      },
+      {
+       "buf": "core_attn_out"
+      },
+      {
+       "buf": "model.layers.22.linear_attn.norm.weight"
+      },
+      {
+       "buf": "gdn_q"
+      },
+      {
+       "buf": "gdn_k"
+      },
+      {
+       "buf": "gdn_v"
+      },
+      {
+       "buf": "g"
       },
       {
-       "buf": "gate_up"
+       "buf": "beta"
       },
       {
-       "i32": 17408
+       "buf": "a_c"
       },
       {
-       "i32": 0
+       "buf": "b_c"
       },
       {
-       "f32": 1.0
+       "buf": "z_c"
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l10.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "f32": 0.0883883461356163
+      },
       {
-       "buf": "act"
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "model.layers.10.mlp.down_proj.weight"
+       "sym": "tokens"
       },
       {
-       "buf": "y"
+       "i32": 385
       },
       {
-       "sym": "tokens"
+       "i32": 1703936
       },
       {
-       "i32": 5120
+       "i32": 10
       },
       {
-       "i32": 17408
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l10.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l22.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.22.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.11.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.22.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -116550,7 +66118,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -116558,23 +66126,23 @@
      ]
     },
     {
-     "label": "l11.qkv_proj",
-     "kernel": "gemm",
+     "label": "l22.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.11.self_attn.qkv_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkv"
+       "buf": "model.layers.22.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 14336
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -116582,45 +66150,32 @@
      ]
     },
     {
-     "label": "l11.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l22.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.11.self_attn.q_norm.weight_p1"
+       "buf": "x"
       },
       {
-       "i32": 256
+       "buf": "act"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "buf": "model.layers.22.mlp.down_proj.weight"
       },
       {
-       "i32": 24
+       "buf": "residual"
       },
       {
-       "i32": 14336
+       "buf": "model.layers.23.input_layernorm.weight_p1"
       },
       {
-       "i32": 512
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 5120
       },
       {
-       "i32": 256
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -116628,55 +66183,32 @@
      ]
     },
     {
-     "label": "l11.k_norm",
-     "kernel": "gemma_norm_khead",
+     "label": "l23.qkv_proj",
+     "kernel": "gemm",
      "args": [
       {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.11.self_attn.k_norm.weight_p1"
+       "buf": "x"
       },
       {
-       "i32": 256
+       "buf": "model.layers.23.self_attn.qkv_proj.weight"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
+       "buf": "qkv"
       },
       {
-       "i32": 4
+       "sym": "tokens"
       },
       {
        "i32": 14336
       },
       {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l11.rope",
-     "kernel": "mrope",
+     "label": "l23.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
@@ -116685,83 +66217,71 @@
        "buf": "k_n"
       },
       {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
+       "buf": "qkv"
       },
       {
-       "i32": 0
+       "buf": "model.layers.23.self_attn.q_norm.weight_p1"
       },
       {
-       "i64": 0
+       "buf": "model.layers.23.self_attn.k_norm.weight_p1"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l11.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
+       "buf": "cos_g"
       },
       {
-       "buf": "qkv",
-       "offset": 26624
+       "buf": "sin_g"
       },
       {
        "state": "kv",
-       "offset": 6422528
+       "offset": 16056320
       },
       {
        "state": "kv",
-       "offset": 6423040
+       "offset": 16056832
       },
       {
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l11.attn",
-     "kernel": "attn_prefill",
+     "label": "l23.attn",
+     "kernel": "attn",
      "args": [
       {
        "buf": "attn_out"
@@ -116771,11 +66291,11 @@
       },
       {
        "state": "kv",
-       "offset": 6422528
+       "offset": 16056320
       },
       {
        "state": "kv",
-       "offset": 6423040
+       "offset": 16056832
       },
       {
        "buf": "block_table"
@@ -116853,11 +66373,11 @@
      ]
     },
     {
-     "label": "l11.gate",
-     "kernel": "sigmoid_mul",
+     "label": "l23.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "gated"
+       "buf": "x"
       },
       {
        "buf": "attn_out"
@@ -116867,7 +66387,22 @@
        "offset": 512
       },
       {
-       "i32": 24
+       "buf": "model.layers.23.self_attn.o_proj.weight"
+      },
+      {
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.23.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 6144
       },
       {
        "i32": 256
@@ -116877,51 +66412,54 @@
       },
       {
        "i32": 512
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l11.o_proj",
-     "kernel": "gemm",
+     "label": "l23.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "gated"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.11.self_attn.o_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.23.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l11.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l23.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.23.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.11.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.24.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -116930,7 +66468,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -116938,23 +66476,32 @@
      ]
     },
     {
-     "label": "l11.gate_up",
-     "kernel": "gemm",
+     "label": "l24.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.11.mlp.gate_up_proj.weight"
+       "buf": "model.layers.24.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.24.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -116962,308 +66509,266 @@
      ]
     },
     {
-     "label": "l11.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l24.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
+       "buf": "qkvz"
       },
       {
-       "i32": 17408
+       "buf": "model.layers.24.linear_attn.conv1d.weight"
       },
       {
-       "i32": 0
+       "state": "gdn"
       },
       {
-       "f32": 1.0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l11.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "gdn.spec_line_index",
+       "offset": 76
       },
       {
-       "buf": "model.layers.11.mlp.down_proj.weight"
+       "buf": "gdn.one"
       },
       {
-       "buf": "y"
+       "buf": "cu_seqlens_q"
       },
       {
-       "sym": "tokens"
+       "buf": "gdn.spec_slots",
+       "offset": 576
       },
       {
-       "i32": 5120
+       "buf": "ba"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l11.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
+       "buf": "model.layers.24.linear_attn.A_log"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.24.linear_attn.dt_bias"
       },
       {
-       "buf": "residual"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.12.input_layernorm.weight_p1"
+       "buf": "model.layers.24.linear_attn.norm.weight"
       },
       {
-       "i32": 5120
+       "buf": "gdn_q"
       },
       {
-       "sym": "tokens"
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
+       "buf": "gdn_v"
       },
       {
-       "i32": 5120
+       "buf": "g"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l12.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
+       "buf": "beta"
       },
       {
-       "buf": "model.layers.12.linear_attn.in_proj_qkvz.weight"
+       "buf": "a_c"
       },
       {
-       "buf": "qkvz"
+       "buf": "b_c"
       },
       {
-       "sym": "tokens"
+       "buf": "z_c"
       },
       {
-       "i32": 16384
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l12.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "model.layers.12.linear_attn.in_proj_ba.weight"
+       "sym": "tokens"
       },
       {
-       "buf": "ba"
+       "i32": 385
       },
       {
-       "sym": "tokens"
+       "i32": 1703936
       },
       {
-       "i32": 96
+       "i32": 10
       },
       {
-       "i32": 5120
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l12.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l24.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.12.linear_attn.conv1d.weight"
+       "buf": "core_attn_out"
       },
       {
-       "state": "gdn"
+       "buf": "model.layers.24.linear_attn.out_proj.weight"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 40
+       "buf": "residual"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "model.layers.24.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "cu_seqlens_q"
+       "sym": "tokens"
       },
       {
-       "buf": "qkvz"
+       "i32": 5120
       },
       {
-       "i32": 1
+       "i32": 6144
       },
       {
-       "i32": 385
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l24.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
+     "args": [
+      {
+       "buf": "act"
       },
       {
-       "i64": 16384
+       "buf": "x"
       },
       {
-       "i64": 16384
+       "buf": "model.layers.24.mlp.gate_up_proj.weight"
       },
       {
-       "i64": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 17408
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l12.post_conv",
-     "kernel": "post_conv",
+     "label": "l24.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
+       "buf": "x"
       },
       {
-       "buf": "ba"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.12.linear_attn.A_log"
+       "buf": "model.layers.24.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.12.linear_attn.dt_bias"
+       "buf": "residual"
       },
       {
-       "buf": "gdn_q"
+       "buf": "model.layers.25.input_layernorm.weight_p1"
       },
       {
-       "buf": "gdn_k"
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_v"
+       "i32": 5120
       },
       {
-       "buf": "g"
+       "i32": 17408
       },
       {
-       "buf": "beta"
-      },
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l25.in_proj",
+     "kernel": "gemm8_dual",
+     "args": [
       {
-       "i32": 16384
+       "buf": "qkvz"
       },
       {
-       "i32": 96
+       "buf": "ba"
       },
       {
-       "i32": 96
+       "buf": "x"
       },
       {
-       "i32": 2048
+       "buf": "model.layers.25.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i32": 2048
+       "buf": "model.layers.25.linear_attn.in_proj_ba.weight"
       },
       {
-       "i32": 6144
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 16384
       },
       {
-       "i64": 0
+       "i32": 96
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l12.a_copy",
-     "kernel": "copy_rows",
+     "label": "l25.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
+       "buf": "qkvz"
       },
       {
-       "i32": 48
+       "buf": "model.layers.25.linear_attn.conv1d.weight"
       },
       {
-       "i32": 96
+       "state": "gdn"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l12.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "buf": "ba"
+       "buf": "gdn.spec_line_index",
+       "offset": 80
       },
       {
-       "i32": 48
+       "buf": "gdn.one"
       },
       {
-       "i32": 96
+       "buf": "cu_seqlens_q"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l12.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.12.linear_attn.A_log"
+       "buf": "gdn.spec_slots",
+       "offset": 608
       },
       {
-       "buf": "a_c"
+       "buf": "ba"
       },
       {
-       "buf": "b_c"
+       "buf": "model.layers.25.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.12.linear_attn.dt_bias"
+       "buf": "model.layers.25.linear_attn.dt_bias"
       },
       {
-       "f32": 1.0
+       "buf": "core_attn_out"
       },
       {
-       "f32": 20.0
+       "buf": "model.layers.25.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -117275,153 +66780,118 @@
        "buf": "gdn_v"
       },
       {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "g"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "beta"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "a_c"
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 288
+       "buf": "b_c"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "z_c"
       },
       {
        "f32": 0.0883883461356163
       },
       {
-       "i64": 1
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l12.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
+       "i32": 385
       },
       {
-       "i32": 6144
+       "i32": 1703936
       },
       {
-       "i32": 16384
+       "i32": 10
       },
       {
-       "i32": 6144
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l12.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l25.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.12.linear_attn.norm.weight"
+       "buf": "model.layers.25.linear_attn.out_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.25.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l12.out_proj",
-     "kernel": "gemm",
+     "label": "l25.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.12.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.25.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l12.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l25.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.25.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.12.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.26.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -117430,7 +66900,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -117438,23 +66908,32 @@
      ]
     },
     {
-     "label": "l12.gate_up",
-     "kernel": "gemm",
+     "label": "l26.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.12.mlp.gate_up_proj.weight"
+       "buf": "model.layers.26.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.26.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -117462,128 +66941,149 @@
      ]
     },
     {
-     "label": "l12.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l26.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.26.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 84
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l12.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "gdn.one"
+      },
       {
-       "buf": "act"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "model.layers.12.mlp.down_proj.weight"
+       "buf": "gdn.spec_slots",
+       "offset": 640
       },
       {
-       "buf": "y"
+       "buf": "ba"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.26.linear_attn.A_log"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.26.linear_attn.dt_bias"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l12.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "core_attn_out"
+      },
       {
-       "buf": "x"
+       "buf": "model.layers.26.linear_attn.norm.weight"
       },
       {
-       "buf": "y"
+       "buf": "gdn_q"
       },
       {
-       "buf": "residual"
+       "buf": "gdn_k"
       },
       {
-       "buf": "model.layers.13.input_layernorm.weight_p1"
+       "buf": "gdn_v"
       },
       {
-       "i32": 5120
+       "buf": "g"
       },
       {
-       "sym": "tokens"
+       "buf": "beta"
       },
       {
-       "i32": 5120
+       "buf": "a_c"
       },
       {
-       "i32": 5120
+       "buf": "b_c"
+      },
+      {
+       "buf": "z_c"
+      },
+      {
+       "f32": 0.0883883461356163
       },
       {
        "f32": 9.999999974752427e-07
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 385
+      },
+      {
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l13.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l26.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.13.linear_attn.in_proj_qkvz.weight"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.26.linear_attn.out_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.26.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l13.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l26.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.13.linear_attn.in_proj_ba.weight"
+       "buf": "x"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.26.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 96
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -117591,218 +67091,219 @@
      ]
     },
     {
-     "label": "l13.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l26.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.13.linear_attn.conv1d.weight"
+       "buf": "act"
       },
       {
-       "state": "gdn"
+       "buf": "model.layers.26.mlp.down_proj.weight"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 44
+       "buf": "residual"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "model.layers.27.input_layernorm.weight_p1"
       },
       {
-       "buf": "cu_seqlens_q"
+       "sym": "tokens"
       },
       {
-       "buf": "qkvz"
+       "i32": 5120
       },
       {
-       "i32": 1
+       "i32": 17408
       },
       {
-       "i32": 385
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l27.qkv_proj",
+     "kernel": "gemm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "i64": 16384
+       "buf": "model.layers.27.self_attn.qkv_proj.weight"
       },
       {
-       "i64": 16384
+       "buf": "qkv"
       },
       {
-       "i64": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l13.post_conv",
-     "kernel": "post_conv",
+     "label": "l27.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
+       "buf": "q_n"
       },
       {
-       "buf": "model.layers.13.linear_attn.A_log"
+       "buf": "k_n"
       },
       {
-       "buf": "model.layers.13.linear_attn.dt_bias"
+       "buf": "qkv"
       },
       {
-       "buf": "gdn_q"
+       "buf": "model.layers.27.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "gdn_k"
+       "buf": "model.layers.27.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "gdn_v"
+       "buf": "cos_g"
       },
       {
-       "buf": "g"
+       "buf": "sin_g"
       },
       {
-       "buf": "beta"
+       "state": "kv",
+       "offset": 19267584
       },
       {
-       "i32": 16384
+       "state": "kv",
+       "offset": 19268096
       },
       {
-       "i32": 96
+       "buf": "slot_mapping"
       },
       {
-       "i32": 96
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i32": 2048
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i32": 2048
+       "i64": 25690112
       },
       {
-       "i32": 6144
+       "i64": 2048
       },
       {
-       "sym": "tokens"
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l13.a_copy",
-     "kernel": "copy_rows",
+     "label": "l27.attn",
+     "kernel": "attn",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "attn_out"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "q_n"
       },
       {
-       "i32": 48
+       "state": "kv",
+       "offset": 19267584
       },
       {
-       "i32": 96
+       "state": "kv",
+       "offset": 19268096
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l13.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "block_table"
       },
       {
-       "buf": "ba"
+       "buf": "seq_lens"
       },
       {
-       "i32": 48
+       "f32": 0.0625
       },
       {
-       "i32": 96
+       "buf": "kv_scales"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l13.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.13.linear_attn.A_log"
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "buf": "a_c"
+       "f32": 1.0
       },
       {
-       "buf": "b_c"
+       "f32": 0.0
       },
       {
-       "buf": "model.layers.13.linear_attn.dt_bias"
+       "i64": 8
       },
       {
-       "f32": 1.0
+       "i64": 6144
       },
       {
-       "f32": 20.0
+       "i64": 256
       },
       {
-       "buf": "gdn_q"
+       "i64": 6144
       },
       {
-       "buf": "gdn_k"
+       "i64": 256
       },
       {
-       "buf": "gdn_v"
+       "i64": 0
       },
       {
-       "buf": "core_attn_out"
+       "buf": "seq_lens"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "i64": 25690112
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "i64": 2048
       },
       {
-       "buf": "cu_seqlens_q"
+       "i64": 512
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 320
+       "i64": 25690112
       },
       {
-       "buf": "num_accepted_tokens"
+       "i64": 2048
       },
       {
-       "f32": 0.0883883461356163
+       "i64": 512
       },
       {
-       "i64": 1
+       "buf": "cu_seqlens_q"
       },
       {
-       "sym": "tokens"
+       "i32": 1
       },
       {
        "i64": 0
@@ -117813,115 +67314,93 @@
      ]
     },
     {
-     "label": "l13.z_copy",
-     "kernel": "copy_rows",
+     "label": "l27.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "x"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "attn_out"
       },
       {
-       "i32": 6144
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "i32": 16384
+       "buf": "model.layers.27.self_attn.o_proj.weight"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l13.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
+       "buf": "residual"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.27.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "model.layers.13.linear_attn.norm.weight"
+       "sym": "tokens"
       },
       {
-       "buf": "z_c"
+       "i32": 5120
       },
       {
-       "i32": 128
+       "i32": 6144
       },
       {
-       "i32": 128
+       "i32": 256
       },
       {
-       "i32": 128
+       "i32": 14336
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 512
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l13.out_proj",
-     "kernel": "gemm",
+     "label": "l27.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.13.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.27.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l13.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l27.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.27.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.13.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.28.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -117930,7 +67409,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -117938,23 +67417,32 @@
      ]
     },
     {
-     "label": "l13.gate_up",
-     "kernel": "gemm",
+     "label": "l28.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.13.mlp.gate_up_proj.weight"
+       "buf": "model.layers.28.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.28.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -117962,71 +67450,116 @@
      ]
     },
     {
-     "label": "l13.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l28.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.28.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 88
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l13.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "gdn.one"
+      },
       {
-       "buf": "act"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "model.layers.13.mlp.down_proj.weight"
+       "buf": "gdn.spec_slots",
+       "offset": 672
       },
       {
-       "buf": "y"
+       "buf": "ba"
+      },
+      {
+       "buf": "model.layers.28.linear_attn.A_log"
+      },
+      {
+       "buf": "model.layers.28.linear_attn.dt_bias"
+      },
+      {
+       "buf": "core_attn_out"
+      },
+      {
+       "buf": "model.layers.28.linear_attn.norm.weight"
+      },
+      {
+       "buf": "gdn_q"
+      },
+      {
+       "buf": "gdn_k"
+      },
+      {
+       "buf": "gdn_v"
+      },
+      {
+       "buf": "g"
+      },
+      {
+       "buf": "beta"
+      },
+      {
+       "buf": "a_c"
+      },
+      {
+       "buf": "b_c"
+      },
+      {
+       "buf": "z_c"
+      },
+      {
+       "f32": 0.0883883461356163
+      },
+      {
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 385
+      },
+      {
+       "i32": 1703936
+      },
+      {
+       "i32": 10
       },
       {
-       "i32": 17408
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l13.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l28.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.28.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.14.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.28.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -118035,7 +67568,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -118043,23 +67576,23 @@
      ]
     },
     {
-     "label": "l14.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l28.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.14.linear_attn.in_proj_qkvz.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.28.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -118067,466 +67600,398 @@
      ]
     },
     {
-     "label": "l14.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l28.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.14.linear_attn.in_proj_ba.weight"
+       "buf": "act"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.28.mlp.down_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 96
+       "buf": "model.layers.29.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l14.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l29.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.14.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 48
-      },
-      {
-       "buf": "num_accepted_tokens"
-      },
-      {
-       "buf": "cu_seqlens_q"
+       "buf": "ba"
       },
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "i32": 1
+       "buf": "model.layers.29.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i32": 385
+       "buf": "model.layers.29.linear_attn.in_proj_ba.weight"
       },
       {
-       "i64": 16384
+       "sym": "tokens"
       },
       {
-       "i64": 16384
+       "i32": 16384
       },
       {
-       "i64": 0
+       "i32": 96
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l14.post_conv",
-     "kernel": "post_conv",
+     "label": "l29.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.29.linear_attn.conv1d.weight"
       },
       {
-       "buf": "ba"
+       "state": "gdn"
       },
       {
-       "buf": "model.layers.14.linear_attn.A_log"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "buf": "model.layers.14.linear_attn.dt_bias"
+       "buf": "gdn.spec_line_index",
+       "offset": 92
       },
       {
-       "buf": "gdn_q"
+       "buf": "gdn.one"
       },
       {
-       "buf": "gdn_k"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "gdn_v"
+       "buf": "gdn.spec_slots",
+       "offset": 704
       },
       {
-       "buf": "g"
+       "buf": "ba"
       },
       {
-       "buf": "beta"
+       "buf": "model.layers.29.linear_attn.A_log"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.29.linear_attn.dt_bias"
       },
       {
-       "i32": 96
+       "buf": "core_attn_out"
       },
       {
-       "i32": 96
+       "buf": "model.layers.29.linear_attn.norm.weight"
       },
       {
-       "i32": 2048
+       "buf": "gdn_q"
       },
       {
-       "i32": 2048
+       "buf": "gdn_k"
       },
       {
-       "i32": 6144
+       "buf": "gdn_v"
       },
       {
-       "sym": "tokens"
+       "buf": "g"
       },
       {
-       "i64": 0
+       "buf": "beta"
       },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l14.a_copy",
-     "kernel": "copy_rows",
-     "args": [
       {
        "buf": "a_c"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "b_c"
       },
       {
-       "i32": 48
+       "buf": "z_c"
       },
       {
-       "i32": 96
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l14.b_copy",
-     "kernel": "copy_rows",
-     "args": [
+       "f32": 9.999999974752427e-07
+      },
       {
-       "buf": "b_c"
+       "sym": "tokens"
       },
       {
-       "buf": "ba"
+       "i32": 385
       },
       {
-       "i32": 48
+       "i32": 1703936
       },
       {
-       "i32": 96
+       "i32": 10
       },
       {
-       "i32": 48
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l14.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l29.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "model.layers.14.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.14.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
-      },
-      {
-       "buf": "gdn_v"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "gdn.spec_slots",
-       "offset": 352
+       "buf": "model.layers.29.linear_attn.out_proj.weight"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "residual"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "model.layers.29.post_attention_layernorm.weight_p1"
       },
       {
-       "i64": 1
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 6144
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l14.z_copy",
-     "kernel": "copy_rows",
+     "label": "l29.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "act"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "x"
       },
       {
-       "i32": 6144
+       "buf": "model.layers.29.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 16384
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 17408
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l14.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l29.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.14.linear_attn.norm.weight"
+       "buf": "model.layers.29.mlp.down_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.30.input_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l14.out_proj",
-     "kernel": "gemm",
+     "label": "l30.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "qkvz"
       },
       {
-       "buf": "model.layers.14.linear_attn.out_proj.weight"
+       "buf": "ba"
       },
       {
-       "buf": "y"
+       "buf": "x"
+      },
+      {
+       "buf": "model.layers.30.linear_attn.in_proj_qkvz.weight"
+      },
+      {
+       "buf": "model.layers.30.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 16384
       },
       {
-       "i32": 6144
+       "i32": 96
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l14.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l30.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "x"
+       "buf": "qkvz"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.30.linear_attn.conv1d.weight"
       },
       {
-       "buf": "residual"
+       "state": "gdn"
       },
       {
-       "buf": "model.layers.14.post_attention_layernorm.weight_p1"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 5120
+       "buf": "gdn.spec_line_index",
+       "offset": 96
       },
       {
-       "sym": "tokens"
+       "buf": "gdn.one"
       },
       {
-       "i32": 5120
+       "buf": "cu_seqlens_q"
       },
       {
-       "i32": 5120
+       "buf": "gdn.spec_slots",
+       "offset": 736
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l14.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
+       "buf": "ba"
       },
       {
-       "buf": "model.layers.14.mlp.gate_up_proj.weight"
+       "buf": "model.layers.30.linear_attn.A_log"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.30.linear_attn.dt_bias"
       },
       {
-       "sym": "tokens"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 34816
+       "buf": "model.layers.30.linear_attn.norm.weight"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l14.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
+       "buf": "gdn_q"
+      },
       {
-       "buf": "act"
+       "buf": "gdn_k"
       },
       {
-       "buf": "gate_up"
+       "buf": "gdn_v"
       },
       {
-       "i32": 17408
+       "buf": "g"
       },
       {
-       "i32": 0
+       "buf": "beta"
       },
       {
-       "f32": 1.0
+       "buf": "a_c"
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l14.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "b_c"
+      },
       {
-       "buf": "act"
+       "buf": "z_c"
       },
       {
-       "buf": "model.layers.14.mlp.down_proj.weight"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "y"
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 385
       },
       {
-       "i32": 17408
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l14.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l30.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.30.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.15.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.30.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -118535,7 +68000,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -118543,23 +68008,23 @@
      ]
     },
     {
-     "label": "l15.qkv_proj",
-     "kernel": "gemm",
+     "label": "l30.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.15.self_attn.qkv_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkv"
+       "buf": "model.layers.30.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 14336
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -118567,45 +68032,32 @@
      ]
     },
     {
-     "label": "l15.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l30.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.15.self_attn.q_norm.weight_p1"
+       "buf": "x"
       },
       {
-       "i32": 256
+       "buf": "act"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "buf": "model.layers.30.mlp.down_proj.weight"
       },
       {
-       "i32": 24
+       "buf": "residual"
       },
       {
-       "i32": 14336
+       "buf": "model.layers.31.input_layernorm.weight_p1"
       },
       {
-       "i32": 512
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 5120
       },
       {
-       "i32": 256
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -118613,55 +68065,32 @@
      ]
     },
     {
-     "label": "l15.k_norm",
-     "kernel": "gemma_norm_khead",
+     "label": "l31.qkv_proj",
+     "kernel": "gemm",
      "args": [
       {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.15.self_attn.k_norm.weight_p1"
+       "buf": "x"
       },
       {
-       "i32": 256
+       "buf": "model.layers.31.self_attn.qkv_proj.weight"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
+       "buf": "qkv"
       },
       {
-       "i32": 4
+       "sym": "tokens"
       },
       {
        "i32": 14336
       },
       {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l15.rope",
-     "kernel": "mrope",
+     "label": "l31.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
@@ -118670,83 +68099,71 @@
        "buf": "k_n"
       },
       {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
+       "buf": "qkv"
       },
       {
-       "i32": 0
+       "buf": "model.layers.31.self_attn.q_norm.weight_p1"
       },
       {
-       "i64": 0
+       "buf": "model.layers.31.self_attn.k_norm.weight_p1"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l15.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
+       "buf": "cos_g"
       },
       {
-       "buf": "qkv",
-       "offset": 26624
+       "buf": "sin_g"
       },
       {
        "state": "kv",
-       "offset": 9633792
+       "offset": 22478848
       },
       {
        "state": "kv",
-       "offset": 9634304
+       "offset": 22479360
       },
       {
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l15.attn",
-     "kernel": "attn_prefill",
+     "label": "l31.attn",
+     "kernel": "attn",
      "args": [
       {
        "buf": "attn_out"
@@ -118756,11 +68173,11 @@
       },
       {
        "state": "kv",
-       "offset": 9633792
+       "offset": 22478848
       },
       {
        "state": "kv",
-       "offset": 9634304
+       "offset": 22479360
       },
       {
        "buf": "block_table"
@@ -118838,11 +68255,11 @@
      ]
     },
     {
-     "label": "l15.gate",
-     "kernel": "sigmoid_mul",
+     "label": "l31.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "gated"
+       "buf": "x"
       },
       {
        "buf": "attn_out"
@@ -118852,7 +68269,22 @@
        "offset": 512
       },
       {
-       "i32": 24
+       "buf": "model.layers.31.self_attn.o_proj.weight"
+      },
+      {
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.31.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 6144
       },
       {
        "i32": 256
@@ -118862,51 +68294,54 @@
       },
       {
        "i32": 512
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l15.o_proj",
-     "kernel": "gemm",
+     "label": "l31.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "gated"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.15.self_attn.o_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.31.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l15.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l31.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.31.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.15.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.32.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -118915,7 +68350,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -118923,23 +68358,32 @@
      ]
     },
     {
-     "label": "l15.gate_up",
-     "kernel": "gemm",
+     "label": "l32.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.15.mlp.gate_up_proj.weight"
+       "buf": "model.layers.32.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.32.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -118947,293 +68391,281 @@
      ]
     },
     {
-     "label": "l15.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l32.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.32.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 100
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l15.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "gdn.one"
       },
       {
-       "buf": "model.layers.15.mlp.down_proj.weight"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "y"
+       "buf": "gdn.spec_slots",
+       "offset": 768
       },
       {
-       "sym": "tokens"
+       "buf": "ba"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.32.linear_attn.A_log"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l15.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "model.layers.32.linear_attn.dt_bias"
+      },
       {
-       "buf": "x"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.32.linear_attn.norm.weight"
       },
       {
-       "buf": "residual"
+       "buf": "gdn_q"
       },
       {
-       "buf": "model.layers.16.input_layernorm.weight_p1"
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
+       "buf": "gdn_v"
       },
       {
-       "sym": "tokens"
+       "buf": "g"
       },
       {
-       "i32": 5120
+       "buf": "beta"
       },
       {
-       "i32": 5120
+       "buf": "a_c"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l16.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
+       "buf": "b_c"
+      },
       {
-       "buf": "x"
+       "buf": "z_c"
       },
       {
-       "buf": "model.layers.16.linear_attn.in_proj_qkvz.weight"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "qkvz"
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l16.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l32.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.16.linear_attn.in_proj_ba.weight"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.32.linear_attn.out_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 96
+       "buf": "model.layers.32.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l16.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l32.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.16.linear_attn.conv1d.weight"
+       "buf": "x"
       },
       {
-       "state": "gdn"
+       "buf": "model.layers.32.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 52
+       "sym": "tokens"
       },
       {
-       "buf": "num_accepted_tokens"
+       "i32": 17408
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l32.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "buf": "qkvz"
+       "buf": "act"
       },
       {
-       "i32": 1
+       "buf": "model.layers.32.mlp.down_proj.weight"
       },
       {
-       "i32": 385
+       "buf": "residual"
       },
       {
-       "i64": 16384
+       "buf": "model.layers.33.input_layernorm.weight_p1"
       },
       {
-       "i64": 16384
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l16.post_conv",
-     "kernel": "post_conv",
+     "label": "l33.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
        "buf": "qkvz"
       },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
       {
        "buf": "ba"
       },
       {
-       "buf": "model.layers.16.linear_attn.A_log"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.16.linear_attn.dt_bias"
+       "buf": "model.layers.33.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gdn_q"
+       "buf": "model.layers.33.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "gdn_k"
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_v"
+       "i32": 16384
       },
       {
-       "buf": "g"
+       "i32": 96
       },
       {
-       "buf": "beta"
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l33.gdn",
+     "kernel": "gdn_spec",
+     "args": [
       {
-       "i32": 16384
+       "buf": "qkvz"
       },
       {
-       "i32": 96
+       "buf": "model.layers.33.linear_attn.conv1d.weight"
       },
       {
-       "i32": 96
+       "state": "gdn"
       },
       {
-       "i32": 2048
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 2048
+       "buf": "gdn.spec_line_index",
+       "offset": 104
       },
       {
-       "i32": 6144
+       "buf": "gdn.one"
       },
       {
-       "sym": "tokens"
+       "buf": "cu_seqlens_q"
       },
       {
-       "i64": 0
+       "buf": "gdn.spec_slots",
+       "offset": 800
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l16.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
+       "buf": "ba"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.33.linear_attn.A_log"
       },
       {
-       "i32": 48
+       "buf": "model.layers.33.linear_attn.dt_bias"
       },
       {
-       "i32": 96
+       "buf": "core_attn_out"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l16.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.33.linear_attn.norm.weight"
       },
       {
-       "buf": "ba"
+       "buf": "gdn_q"
       },
       {
-       "i32": 48
+       "buf": "gdn_k"
       },
       {
-       "i32": 96
+       "buf": "gdn_v"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l16.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
+       "buf": "g"
+      },
       {
-       "buf": "model.layers.16.linear_attn.A_log"
+       "buf": "beta"
       },
       {
        "buf": "a_c"
@@ -119242,141 +68674,133 @@
        "buf": "b_c"
       },
       {
-       "buf": "model.layers.16.linear_attn.dt_bias"
+       "buf": "z_c"
       },
       {
-       "f32": 1.0
+       "f32": 0.0883883461356163
       },
       {
-       "f32": 20.0
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "gdn_q"
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_k"
+       "i32": 385
       },
       {
-       "buf": "gdn_v"
+       "i32": 1703936
       },
       {
-       "buf": "core_attn_out"
+       "i32": 10
       },
       {
-       "state": "gdn",
-       "offset": 204800
-      },
+       "i32": 851968
+      }
+     ]
+    },
+    {
+     "label": "l33.out_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "x"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 384
+       "buf": "model.layers.33.linear_attn.out_proj.weight"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "residual"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "model.layers.33.post_attention_layernorm.weight_p1"
       },
       {
-       "i64": 1
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 6144
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l16.z_copy",
-     "kernel": "copy_rows",
+     "label": "l33.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "act"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "x"
       },
       {
-       "i32": 6144
+       "buf": "model.layers.33.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 16384
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 17408
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l16.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l33.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.16.linear_attn.norm.weight"
+       "buf": "model.layers.33.mlp.down_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.34.input_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l16.out_proj",
-     "kernel": "gemm",
+     "label": "l33.fc_tap",
+     "kernel": "gemm_acc",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "residual"
       },
       {
-       "buf": "model.layers.16.linear_attn.out_proj.weight"
+       "buf": "draft.fc.2.weight"
       },
       {
-       "buf": "y"
+       "buf": "fc_out"
       },
       {
        "sym": "tokens"
@@ -119385,190 +68809,187 @@
        "i32": 5120
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l16.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l34.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "x"
+       "buf": "qkvz"
       },
       {
-       "buf": "y"
+       "buf": "ba"
       },
       {
-       "buf": "residual"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.16.post_attention_layernorm.weight_p1"
+       "buf": "model.layers.34.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.34.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 16384
       },
       {
-       "i32": 5120
+       "i32": 96
       },
       {
-       "f32": 9.999999974752427e-07
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l16.gate_up",
-     "kernel": "gemm",
+     "label": "l34.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "x"
+       "buf": "qkvz"
       },
       {
-       "buf": "model.layers.16.mlp.gate_up_proj.weight"
+       "buf": "model.layers.34.linear_attn.conv1d.weight"
       },
       {
-       "buf": "gate_up"
+       "state": "gdn"
       },
       {
-       "sym": "tokens"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 34816
+       "buf": "gdn.spec_line_index",
+       "offset": 108
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l16.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
+       "buf": "gdn.one"
+      },
       {
-       "buf": "act"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "gate_up"
+       "buf": "gdn.spec_slots",
+       "offset": 832
       },
       {
-       "i32": 17408
+       "buf": "ba"
       },
       {
-       "i32": 0
+       "buf": "model.layers.34.linear_attn.A_log"
       },
       {
-       "f32": 1.0
+       "buf": "model.layers.34.linear_attn.dt_bias"
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l16.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "core_attn_out"
+      },
       {
-       "buf": "act"
+       "buf": "model.layers.34.linear_attn.norm.weight"
       },
       {
-       "buf": "model.layers.16.mlp.down_proj.weight"
+       "buf": "gdn_q"
       },
       {
-       "buf": "y"
+       "buf": "gdn_k"
       },
       {
-       "sym": "tokens"
+       "buf": "gdn_v"
       },
       {
-       "i32": 5120
+       "buf": "g"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l16.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "beta"
+      },
       {
-       "buf": "x"
+       "buf": "a_c"
       },
       {
-       "buf": "y"
+       "buf": "b_c"
       },
       {
-       "buf": "residual"
+       "buf": "z_c"
       },
       {
-       "buf": "model.layers.17.input_layernorm.weight_p1"
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 5120
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
       },
       {
-       "f32": 9.999999974752427e-07
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l17.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l34.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.17.linear_attn.in_proj_qkvz.weight"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.34.linear_attn.out_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.34.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l17.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l34.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.17.linear_attn.in_proj_ba.weight"
+       "buf": "x"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.34.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 96
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -119576,218 +68997,219 @@
      ]
     },
     {
-     "label": "l17.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l34.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.17.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 56
-      },
-      {
-       "buf": "num_accepted_tokens"
+       "buf": "x"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "act"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.34.mlp.down_proj.weight"
       },
       {
-       "i32": 1
+       "buf": "residual"
       },
       {
-       "i32": 385
+       "buf": "model.layers.35.input_layernorm.weight_p1"
       },
       {
-       "i64": 16384
+       "sym": "tokens"
       },
       {
-       "i64": 16384
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 17408
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l17.post_conv",
-     "kernel": "post_conv",
+     "label": "l35.qkv_proj",
+     "kernel": "gemm",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.35.self_attn.qkv_proj.weight"
       },
       {
-       "buf": "ba"
+       "buf": "qkv"
       },
       {
-       "buf": "model.layers.17.linear_attn.A_log"
+       "sym": "tokens"
       },
       {
-       "buf": "model.layers.17.linear_attn.dt_bias"
+       "i32": 14336
       },
       {
-       "buf": "gdn_q"
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l35.prep",
+     "kernel": "attn_prep",
+     "args": [
       {
-       "buf": "gdn_k"
+       "buf": "q_n"
       },
       {
-       "buf": "gdn_v"
+       "buf": "k_n"
       },
       {
-       "buf": "g"
+       "buf": "qkv"
       },
       {
-       "buf": "beta"
+       "buf": "model.layers.35.self_attn.q_norm.weight_p1"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.35.self_attn.k_norm.weight_p1"
       },
       {
-       "i32": 96
+       "buf": "cos_g"
       },
       {
-       "i32": 96
+       "buf": "sin_g"
       },
       {
-       "i32": 2048
+       "state": "kv",
+       "offset": 25690112
       },
       {
-       "i32": 2048
+       "state": "kv",
+       "offset": 25690624
       },
       {
-       "i32": 6144
+       "buf": "slot_mapping"
       },
       {
-       "sym": "tokens"
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 0
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l17.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
+       "i64": 25690112
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "i64": 2048
       },
       {
-       "i32": 48
+       "i64": 512
       },
       {
-       "i32": 96
+       "i32": 784
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l17.b_copy",
-     "kernel": "copy_rows",
+     "label": "l35.attn",
+     "kernel": "attn",
      "args": [
       {
-       "buf": "b_c"
+       "buf": "attn_out"
       },
       {
-       "buf": "ba"
+       "buf": "q_n"
       },
       {
-       "i32": 48
+       "state": "kv",
+       "offset": 25690112
       },
       {
-       "i32": 96
+       "state": "kv",
+       "offset": 25690624
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l17.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
+       "buf": "block_table"
+      },
       {
-       "buf": "model.layers.17.linear_attn.A_log"
+       "buf": "seq_lens"
       },
       {
-       "buf": "a_c"
+       "f32": 0.0625
       },
       {
-       "buf": "b_c"
+       "buf": "kv_scales"
       },
       {
-       "buf": "model.layers.17.linear_attn.dt_bias"
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
        "f32": 1.0
       },
       {
-       "f32": 20.0
+       "f32": 0.0
       },
       {
-       "buf": "gdn_q"
+       "i64": 8
       },
       {
-       "buf": "gdn_k"
+       "i64": 6144
       },
       {
-       "buf": "gdn_v"
+       "i64": 256
       },
       {
-       "buf": "core_attn_out"
+       "i64": 6144
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "i64": 256
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "i64": 0
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "seq_lens"
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 416
+       "i64": 25690112
       },
       {
-       "buf": "num_accepted_tokens"
+       "i64": 2048
       },
       {
-       "f32": 0.0883883461356163
+       "i64": 512
+      },
+      {
+       "i64": 25690112
       },
       {
-       "i64": 1
+       "i64": 2048
       },
       {
-       "sym": "tokens"
+       "i64": 512
+      },
+      {
+       "buf": "cu_seqlens_q"
+      },
+      {
+       "i32": 1
       },
       {
        "i64": 0
@@ -119798,115 +69220,93 @@
      ]
     },
     {
-     "label": "l17.z_copy",
-     "kernel": "copy_rows",
+     "label": "l35.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "x"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "attn_out"
       },
       {
-       "i32": 6144
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "i32": 16384
+       "buf": "model.layers.35.self_attn.o_proj.weight"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l17.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
+       "buf": "residual"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.35.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "model.layers.17.linear_attn.norm.weight"
+       "sym": "tokens"
       },
       {
-       "buf": "z_c"
+       "i32": 5120
       },
       {
-       "i32": 128
+       "i32": 6144
       },
       {
-       "i32": 128
+       "i32": 256
       },
       {
-       "i32": 128
+       "i32": 14336
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 512
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l17.out_proj",
-     "kernel": "gemm",
+     "label": "l35.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.17.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.35.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l17.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l35.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.35.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.17.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.36.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -119915,7 +69315,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -119923,23 +69323,32 @@
      ]
     },
     {
-     "label": "l17.gate_up",
-     "kernel": "gemm",
+     "label": "l36.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.17.mlp.gate_up_proj.weight"
+       "buf": "model.layers.36.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.36.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -119947,308 +69356,266 @@
      ]
     },
     {
-     "label": "l17.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l36.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
+       "buf": "qkvz"
       },
       {
-       "i32": 17408
+       "buf": "model.layers.36.linear_attn.conv1d.weight"
       },
       {
-       "i32": 0
+       "state": "gdn"
       },
       {
-       "f32": 1.0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l17.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "gdn.spec_line_index",
+       "offset": 112
       },
       {
-       "buf": "model.layers.17.mlp.down_proj.weight"
+       "buf": "gdn.one"
       },
       {
-       "buf": "y"
+       "buf": "cu_seqlens_q"
       },
       {
-       "sym": "tokens"
+       "buf": "gdn.spec_slots",
+       "offset": 864
       },
       {
-       "i32": 5120
+       "buf": "ba"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l17.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
+       "buf": "model.layers.36.linear_attn.A_log"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.36.linear_attn.dt_bias"
       },
       {
-       "buf": "residual"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.18.input_layernorm.weight_p1"
+       "buf": "model.layers.36.linear_attn.norm.weight"
       },
       {
-       "i32": 5120
+       "buf": "gdn_q"
       },
       {
-       "sym": "tokens"
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
+       "buf": "gdn_v"
       },
       {
-       "i32": 5120
+       "buf": "g"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l18.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
+       "buf": "beta"
       },
       {
-       "buf": "model.layers.18.linear_attn.in_proj_qkvz.weight"
+       "buf": "a_c"
       },
       {
-       "buf": "qkvz"
+       "buf": "b_c"
       },
       {
-       "sym": "tokens"
+       "buf": "z_c"
       },
       {
-       "i32": 16384
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l18.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "model.layers.18.linear_attn.in_proj_ba.weight"
+       "sym": "tokens"
       },
       {
-       "buf": "ba"
+       "i32": 385
       },
       {
-       "sym": "tokens"
+       "i32": 1703936
       },
       {
-       "i32": 96
+       "i32": 10
       },
       {
-       "i32": 5120
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l18.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l36.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.18.linear_attn.conv1d.weight"
+       "buf": "core_attn_out"
       },
       {
-       "state": "gdn"
+       "buf": "model.layers.36.linear_attn.out_proj.weight"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 60
+       "buf": "residual"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "model.layers.36.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "cu_seqlens_q"
+       "sym": "tokens"
       },
       {
-       "buf": "qkvz"
+       "i32": 5120
       },
       {
-       "i32": 1
+       "i32": 6144
       },
       {
-       "i32": 385
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l36.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
+     "args": [
+      {
+       "buf": "act"
       },
       {
-       "i64": 16384
+       "buf": "x"
       },
       {
-       "i64": 16384
+       "buf": "model.layers.36.mlp.gate_up_proj.weight"
       },
       {
-       "i64": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 17408
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l18.post_conv",
-     "kernel": "post_conv",
+     "label": "l36.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
+       "buf": "x"
       },
       {
-       "buf": "ba"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.18.linear_attn.A_log"
+       "buf": "model.layers.36.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.18.linear_attn.dt_bias"
+       "buf": "residual"
       },
       {
-       "buf": "gdn_q"
+       "buf": "model.layers.37.input_layernorm.weight_p1"
       },
       {
-       "buf": "gdn_k"
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_v"
+       "i32": 5120
       },
       {
-       "buf": "g"
+       "i32": 17408
       },
       {
-       "buf": "beta"
-      },
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l37.in_proj",
+     "kernel": "gemm8_dual",
+     "args": [
       {
-       "i32": 16384
+       "buf": "qkvz"
       },
       {
-       "i32": 96
+       "buf": "ba"
       },
       {
-       "i32": 96
+       "buf": "x"
       },
       {
-       "i32": 2048
+       "buf": "model.layers.37.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i32": 2048
+       "buf": "model.layers.37.linear_attn.in_proj_ba.weight"
       },
       {
-       "i32": 6144
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 16384
       },
       {
-       "i64": 0
+       "i32": 96
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l18.a_copy",
-     "kernel": "copy_rows",
+     "label": "l37.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
+       "buf": "qkvz"
       },
       {
-       "i32": 48
+       "buf": "model.layers.37.linear_attn.conv1d.weight"
       },
       {
-       "i32": 96
+       "state": "gdn"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l18.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "buf": "ba"
+       "buf": "gdn.spec_line_index",
+       "offset": 116
       },
       {
-       "i32": 48
+       "buf": "gdn.one"
       },
       {
-       "i32": 96
+       "buf": "cu_seqlens_q"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l18.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.18.linear_attn.A_log"
+       "buf": "gdn.spec_slots",
+       "offset": 896
       },
       {
-       "buf": "a_c"
+       "buf": "ba"
       },
       {
-       "buf": "b_c"
+       "buf": "model.layers.37.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.18.linear_attn.dt_bias"
+       "buf": "model.layers.37.linear_attn.dt_bias"
       },
       {
-       "f32": 1.0
+       "buf": "core_attn_out"
       },
       {
-       "f32": 20.0
+       "buf": "model.layers.37.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -120260,153 +69627,118 @@
        "buf": "gdn_v"
       },
       {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "g"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "beta"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "a_c"
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 448
+       "buf": "b_c"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "z_c"
       },
       {
        "f32": 0.0883883461356163
       },
       {
-       "i64": 1
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l18.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
+       "i32": 385
       },
       {
-       "i32": 6144
+       "i32": 1703936
       },
       {
-       "i32": 16384
+       "i32": 10
       },
       {
-       "i32": 6144
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l18.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l37.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.18.linear_attn.norm.weight"
+       "buf": "model.layers.37.linear_attn.out_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.37.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l18.out_proj",
-     "kernel": "gemm",
+     "label": "l37.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.18.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.37.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l18.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l37.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.37.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.18.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.38.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -120415,7 +69747,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -120423,23 +69755,32 @@
      ]
     },
     {
-     "label": "l18.gate_up",
-     "kernel": "gemm",
+     "label": "l38.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.18.mlp.gate_up_proj.weight"
+       "buf": "model.layers.38.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.38.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -120447,197 +69788,182 @@
      ]
     },
     {
-     "label": "l18.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l38.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.38.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 120
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l18.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "gdn.one"
       },
       {
-       "buf": "model.layers.18.mlp.down_proj.weight"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "y"
+       "buf": "gdn.spec_slots",
+       "offset": 928
       },
       {
-       "sym": "tokens"
+       "buf": "ba"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.38.linear_attn.A_log"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l18.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "model.layers.38.linear_attn.dt_bias"
+      },
       {
-       "buf": "x"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.38.linear_attn.norm.weight"
       },
       {
-       "buf": "residual"
+       "buf": "gdn_q"
       },
       {
-       "buf": "model.layers.19.input_layernorm.weight_p1"
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
+       "buf": "gdn_v"
       },
       {
-       "sym": "tokens"
+       "buf": "g"
       },
       {
-       "i32": 5120
+       "buf": "beta"
       },
       {
-       "i32": 5120
+       "buf": "a_c"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l19.qkv_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "b_c"
+      },
       {
-       "buf": "x"
+       "buf": "z_c"
       },
       {
-       "buf": "model.layers.19.self_attn.qkv_proj.weight"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "qkv"
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 14336
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l19.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l38.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.19.self_attn.q_norm.weight_p1"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 256
+       "buf": "model.layers.38.linear_attn.out_proj.weight"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "buf": "residual"
       },
       {
-       "i32": 24
+       "buf": "model.layers.38.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 14336
+       "sym": "tokens"
       },
       {
-       "i32": 512
+       "i32": 5120
       },
       {
        "i32": 6144
       },
-      {
-       "i32": 256
-      },
       {
        "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l19.k_norm",
-     "kernel": "gemma_norm_khead",
+     "label": "l38.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "k_n"
+       "buf": "act"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
+       "buf": "x"
       },
       {
-       "buf": "model.layers.19.self_attn.k_norm.weight_p1"
+       "buf": "model.layers.38.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 256
+       "sym": "tokens"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
+       "i32": 17408
       },
       {
-       "i32": 4
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l38.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "i32": 14336
+       "buf": "act"
       },
       {
-       "i32": 256
+       "buf": "model.layers.38.mlp.down_proj.weight"
+      },
+      {
+       "buf": "residual"
       },
       {
-       "i32": 1024
+       "buf": "model.layers.39.input_layernorm.weight_p1"
       },
       {
-       "i32": 256
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -120645,93 +69971,105 @@
      ]
     },
     {
-     "label": "l19.rope",
-     "kernel": "mrope",
+     "label": "l39.qkv_proj",
+     "kernel": "gemm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
+       "buf": "x"
       },
       {
-       "buf": "cos_g"
+       "buf": "model.layers.39.self_attn.qkv_proj.weight"
       },
       {
-       "buf": "sin_g"
+       "buf": "qkv"
       },
       {
-       "i32": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l19.kv_write",
-     "kernel": "reshape_and_cache",
+     "label": "l39.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "k_n"
+       "buf": "q_n"
       },
       {
-       "buf": "qkv",
-       "offset": 26624
+       "buf": "k_n"
       },
       {
-       "state": "kv",
-       "offset": 12845056
+       "buf": "qkv"
       },
       {
-       "state": "kv",
-       "offset": 12845568
+       "buf": "model.layers.39.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "slot_mapping"
+       "buf": "model.layers.39.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "kv_scales"
+       "buf": "cos_g"
       },
       {
-       "buf": "kv_scales",
-       "offset": 4
+       "buf": "sin_g"
       },
       {
-       "i64": 1024
+       "state": "kv",
+       "offset": 28901376
       },
       {
-       "i64": 14336
+       "state": "kv",
+       "offset": 28901888
       },
       {
-       "i64": 25690112
+       "buf": "slot_mapping"
       },
       {
-       "i64": 512
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 0
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i64": 0
+       "i64": 25690112
       },
       {
        "i64": 2048
       },
       {
-       "i64": 0
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l19.attn",
-     "kernel": "attn_prefill",
+     "label": "l39.attn",
+     "kernel": "attn",
      "args": [
       {
        "buf": "attn_out"
@@ -120741,11 +70079,11 @@
       },
       {
        "state": "kv",
-       "offset": 12845056
+       "offset": 28901376
       },
       {
        "state": "kv",
-       "offset": 12845568
+       "offset": 28901888
       },
       {
        "buf": "block_table"
@@ -120823,11 +70161,11 @@
      ]
     },
     {
-     "label": "l19.gate",
-     "kernel": "sigmoid_mul",
+     "label": "l39.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "gated"
+       "buf": "x"
       },
       {
        "buf": "attn_out"
@@ -120837,31 +70175,13 @@
        "offset": 512
       },
       {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l19.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
+       "buf": "model.layers.39.self_attn.o_proj.weight"
       },
       {
-       "buf": "model.layers.19.self_attn.o_proj.weight"
+       "buf": "residual"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.39.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -120871,36 +70191,15 @@
       },
       {
        "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l19.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.19.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
+       "i32": 256
       },
       {
-       "i32": 5120
+       "i32": 14336
       },
       {
-       "i32": 5120
+       "i32": 512
       },
       {
        "f32": 9.999999974752427e-07
@@ -120908,95 +70207,47 @@
      ]
     },
     {
-     "label": "l19.gate_up",
-     "kernel": "gemm",
+     "label": "l39.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.19.mlp.gate_up_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.39.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l19.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
       {
        "i32": 17408
       },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l19.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.19.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
       {
        "i32": 5120
-      },
-      {
-       "i32": 17408
       }
      ]
     },
     {
-     "label": "l19.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l39.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.39.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.20.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.40.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -121005,7 +70256,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -121013,41 +70264,23 @@
      ]
     },
     {
-     "label": "l19.fc_tap",
-     "kernel": "gemm_acc",
+     "label": "l40.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "residual"
-      },
-      {
-       "buf": "draft.fc.1.weight"
-      },
-      {
-       "buf": "fc_out"
-      },
-      {
-       "sym": "tokens"
+       "buf": "qkvz"
       },
       {
-       "i32": 5120
+       "buf": "ba"
       },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l20.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.20.linear_attn.in_proj_qkvz.weight"
+       "buf": "model.layers.40.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.40.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
@@ -121055,27 +70288,6 @@
       {
        "i32": 16384
       },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l20.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.20.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
       {
        "i32": 96
       },
@@ -121085,70 +70297,50 @@
      ]
     },
     {
-     "label": "l20.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l40.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.20.linear_attn.conv1d.weight"
+       "buf": "model.layers.40.linear_attn.conv1d.weight"
       },
       {
        "state": "gdn"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 64
-      },
-      {
-       "buf": "num_accepted_tokens"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 385
+       "buf": "gdn.spec_line_index",
+       "offset": 124
       },
       {
-       "i64": 16384
+       "buf": "gdn.one"
       },
       {
-       "i64": 16384
+       "buf": "cu_seqlens_q"
       },
       {
-       "i64": 0
+       "buf": "gdn.spec_slots",
+       "offset": 960
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l20.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
+       "buf": "ba"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.40.linear_attn.A_log"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.40.linear_attn.dt_bias"
       },
       {
-       "buf": "model.layers.20.linear_attn.A_log"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.20.linear_attn.dt_bias"
+       "buf": "model.layers.40.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -121166,361 +70358,328 @@
        "buf": "beta"
       },
       {
-       "i32": 16384
+       "buf": "a_c"
       },
       {
-       "i32": 96
+       "buf": "b_c"
       },
       {
-       "i32": 96
+       "buf": "z_c"
       },
       {
-       "i32": 2048
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 2048
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 6144
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 385
       },
       {
-       "i64": 0
+       "i32": 1703936
       },
       {
-       "i64": 0
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l20.a_copy",
-     "kernel": "copy_rows",
+     "label": "l40.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "core_attn_out"
       },
       {
-       "i32": 48
+       "buf": "model.layers.40.linear_attn.out_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l20.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.40.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "ba"
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 5120
       },
       {
-       "i32": 96
+       "i32": 6144
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l20.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l40.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "model.layers.20.linear_attn.A_log"
+       "buf": "act"
       },
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "b_c"
+       "buf": "model.layers.40.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "model.layers.20.linear_attn.dt_bias"
+       "sym": "tokens"
       },
       {
-       "f32": 1.0
+       "i32": 17408
       },
       {
-       "f32": 20.0
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l40.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "buf": "gdn_q"
+       "buf": "x"
       },
       {
-       "buf": "gdn_k"
+       "buf": "act"
       },
       {
-       "buf": "gdn_v"
+       "buf": "model.layers.40.mlp.down_proj.weight"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "residual"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "model.layers.41.input_layernorm.weight_p1"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "sym": "tokens"
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 5120
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 480
+       "i32": 17408
       },
       {
-       "buf": "num_accepted_tokens"
-      },
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l41.in_proj",
+     "kernel": "gemm8_dual",
+     "args": [
       {
-       "f32": 0.0883883461356163
+       "buf": "qkvz"
       },
       {
-       "i64": 1
+       "buf": "ba"
       },
       {
-       "sym": "tokens"
+       "buf": "x"
       },
       {
-       "i64": 0
+       "buf": "model.layers.41.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l20.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
+       "buf": "model.layers.41.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 16384
       },
       {
-       "i32": 16384
+       "i32": 96
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l20.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l41.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "qkvz"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.41.linear_attn.conv1d.weight"
       },
       {
-       "buf": "model.layers.20.linear_attn.norm.weight"
+       "state": "gdn"
       },
       {
-       "buf": "z_c"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 128
+       "buf": "gdn.spec_line_index",
+       "offset": 128
       },
       {
-       "i32": 128
+       "buf": "gdn.one"
       },
       {
-       "i32": 128
+       "buf": "cu_seqlens_q"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "buf": "gdn.spec_slots",
+       "offset": 992
       },
       {
-       "f32": 9.999999974752427e-07
+       "buf": "ba"
       },
       {
-       "i64": 0
+       "buf": "model.layers.41.linear_attn.A_log"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l20.out_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "model.layers.41.linear_attn.dt_bias"
+      },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.20.linear_attn.out_proj.weight"
+       "buf": "model.layers.41.linear_attn.norm.weight"
       },
       {
-       "buf": "y"
+       "buf": "gdn_q"
       },
       {
-       "sym": "tokens"
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
+       "buf": "gdn_v"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l20.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "g"
+      },
       {
-       "buf": "x"
+       "buf": "beta"
       },
       {
-       "buf": "y"
+       "buf": "a_c"
       },
       {
-       "buf": "residual"
+       "buf": "b_c"
       },
       {
-       "buf": "model.layers.20.post_attention_layernorm.weight_p1"
+       "buf": "z_c"
       },
       {
-       "i32": 5120
+       "f32": 0.0883883461356163
+      },
+      {
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
       },
       {
-       "f32": 9.999999974752427e-07
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l20.gate_up",
-     "kernel": "gemm",
+     "label": "l41.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.20.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 34816
+       "buf": "model.layers.41.linear_attn.out_proj.weight"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l20.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "residual"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.41.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 17408
+       "sym": "tokens"
       },
       {
-       "i32": 0
+       "i32": 5120
       },
       {
-       "f32": 1.0
+       "i32": 6144
       },
       {
-       "i32": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l20.down_proj",
-     "kernel": "gemm",
+     "label": "l41.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
        "buf": "act"
       },
       {
-       "buf": "model.layers.20.mlp.down_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.41.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 17408
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l20.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l41.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.41.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.21.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.42.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -121529,7 +70688,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -121537,45 +70696,30 @@
      ]
     },
     {
-     "label": "l21.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l42.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.21.linear_attn.in_proj_qkvz.weight"
-      },
       {
        "buf": "qkvz"
       },
       {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
+       "buf": "ba"
       },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l21.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.21.linear_attn.in_proj_ba.weight"
+       "buf": "model.layers.42.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.42.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
+      {
+       "i32": 16384
+      },
       {
        "i32": 96
       },
@@ -121585,412 +70729,466 @@
      ]
     },
     {
-     "label": "l21.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l42.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.21.linear_attn.conv1d.weight"
+       "buf": "model.layers.42.linear_attn.conv1d.weight"
       },
       {
        "state": "gdn"
       },
+      {
+       "state": "gdn",
+       "offset": 204800
+      },
       {
        "buf": "gdn.spec_line_index",
-       "offset": 68
+       "offset": 132
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "gdn.one"
       },
       {
        "buf": "cu_seqlens_q"
       },
       {
-       "buf": "qkvz"
+       "buf": "gdn.spec_slots",
+       "offset": 1024
       },
       {
-       "i32": 1
+       "buf": "ba"
       },
       {
-       "i32": 385
+       "buf": "model.layers.42.linear_attn.A_log"
       },
       {
-       "i64": 16384
+       "buf": "model.layers.42.linear_attn.dt_bias"
       },
       {
-       "i64": 16384
+       "buf": "core_attn_out"
       },
       {
-       "i64": 0
+       "buf": "model.layers.42.linear_attn.norm.weight"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l21.post_conv",
-     "kernel": "post_conv",
-     "args": [
+       "buf": "gdn_q"
+      },
       {
-       "buf": "qkvz"
+       "buf": "gdn_k"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "gdn_v"
       },
       {
-       "buf": "ba"
+       "buf": "g"
       },
       {
-       "buf": "model.layers.21.linear_attn.A_log"
+       "buf": "beta"
       },
       {
-       "buf": "model.layers.21.linear_attn.dt_bias"
+       "buf": "a_c"
       },
       {
-       "buf": "gdn_q"
+       "buf": "b_c"
       },
       {
-       "buf": "gdn_k"
+       "buf": "z_c"
       },
       {
-       "buf": "gdn_v"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "g"
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "beta"
+       "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 385
       },
       {
-       "i32": 96
+       "i32": 1703936
       },
       {
-       "i32": 96
+       "i32": 10
       },
       {
-       "i32": 2048
+       "i32": 851968
+      }
+     ]
+    },
+    {
+     "label": "l42.out_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "i32": 2048
+       "buf": "core_attn_out"
       },
       {
-       "i32": 6144
+       "buf": "model.layers.42.linear_attn.out_proj.weight"
+      },
+      {
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.42.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l21.a_copy",
-     "kernel": "copy_rows",
+     "label": "l42.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "act"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "x"
+      },
+      {
+       "buf": "model.layers.42.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 48
+       "sym": "tokens"
       },
       {
-       "i32": 96
+       "i32": 17408
       },
       {
-       "i32": 48
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l21.b_copy",
-     "kernel": "copy_rows",
+     "label": "l42.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "b_c"
+       "buf": "x"
       },
       {
-       "buf": "ba"
+       "buf": "act"
+      },
+      {
+       "buf": "model.layers.42.mlp.down_proj.weight"
       },
       {
-       "i32": 48
+       "buf": "residual"
       },
       {
-       "i32": 96
+       "buf": "model.layers.43.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 17408
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l21.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l43.qkv_proj",
+     "kernel": "gemm",
      "args": [
       {
-       "buf": "model.layers.21.linear_attn.A_log"
+       "buf": "x"
       },
       {
-       "buf": "a_c"
+       "buf": "model.layers.43.self_attn.qkv_proj.weight"
       },
       {
-       "buf": "b_c"
+       "buf": "qkv"
       },
       {
-       "buf": "model.layers.21.linear_attn.dt_bias"
+       "sym": "tokens"
       },
       {
-       "f32": 1.0
+       "i32": 14336
       },
       {
-       "f32": 20.0
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l43.prep",
+     "kernel": "attn_prep",
+     "args": [
+      {
+       "buf": "q_n"
       },
       {
-       "buf": "gdn_q"
+       "buf": "k_n"
       },
       {
-       "buf": "gdn_k"
+       "buf": "qkv"
       },
       {
-       "buf": "gdn_v"
+       "buf": "model.layers.43.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.43.self_attn.k_norm.weight_p1"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "cos_g"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "sin_g"
       },
       {
-       "buf": "cu_seqlens_q"
+       "state": "kv",
+       "offset": 32112640
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 512
+       "state": "kv",
+       "offset": 32113152
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "slot_mapping"
       },
       {
-       "f32": 0.0883883461356163
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
+      },
+      {
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i64": 1
+       "i64": 25690112
       },
       {
-       "sym": "tokens"
+       "i64": 2048
       },
       {
-       "i64": 0
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l21.z_copy",
-     "kernel": "copy_rows",
+     "label": "l43.attn",
+     "kernel": "attn",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "attn_out"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "q_n"
       },
       {
-       "i32": 6144
+       "state": "kv",
+       "offset": 32112640
       },
       {
-       "i32": 16384
+       "state": "kv",
+       "offset": 32113152
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l21.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
+       "buf": "block_table"
+      },
       {
-       "buf": "core_attn_out"
+       "buf": "seq_lens"
+      },
+      {
+       "f32": 0.0625
+      },
+      {
+       "buf": "kv_scales"
+      },
+      {
+       "buf": "kv_scales",
+       "offset": 4
+      },
+      {
+       "f32": 1.0
+      },
+      {
+       "f32": 0.0
+      },
+      {
+       "i64": 8
       },
       {
-       "buf": "core_attn_out"
+       "i64": 6144
       },
       {
-       "buf": "model.layers.21.linear_attn.norm.weight"
+       "i64": 256
       },
       {
-       "buf": "z_c"
+       "i64": 6144
       },
       {
-       "i32": 128
+       "i64": 256
       },
       {
-       "i32": 128
+       "i64": 0
       },
       {
-       "i32": 128
+       "buf": "seq_lens"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i64": 25690112
       },
       {
-       "f32": 9.999999974752427e-07
+       "i64": 2048
       },
       {
-       "i64": 0
+       "i64": 512
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l21.out_proj",
-     "kernel": "gemm",
-     "args": [
+       "i64": 25690112
+      },
       {
-       "buf": "core_attn_out"
+       "i64": 2048
       },
       {
-       "buf": "model.layers.21.linear_attn.out_proj.weight"
+       "i64": 512
       },
       {
-       "buf": "y"
+       "buf": "cu_seqlens_q"
       },
       {
-       "sym": "tokens"
+       "i32": 1
       },
       {
-       "i32": 5120
+       "i64": 0
       },
       {
-       "i32": 6144
+       "i64": 0
       }
      ]
     },
     {
-     "label": "l21.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l43.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "buf": "model.layers.21.post_attention_layernorm.weight_p1"
+       "buf": "model.layers.43.self_attn.o_proj.weight"
       },
       {
-       "i32": 5120
+       "buf": "residual"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.43.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 5120
+       "sym": "tokens"
       },
       {
        "i32": 5120
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l21.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.21.mlp.gate_up_proj.weight"
+       "i32": 6144
       },
       {
-       "buf": "gate_up"
+       "i32": 256
       },
       {
-       "sym": "tokens"
+       "i32": 14336
       },
       {
-       "i32": 34816
+       "i32": 512
       },
       {
-       "i32": 5120
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l21.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l43.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
        "buf": "act"
       },
       {
-       "buf": "gate_up"
+       "buf": "x"
       },
       {
-       "i32": 17408
+       "buf": "model.layers.43.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 0
+       "sym": "tokens"
       },
       {
-       "f32": 1.0
+       "i32": 17408
       },
       {
-       "i32": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l21.down_proj",
-     "kernel": "gemm",
+     "label": "l43.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
+      {
+       "buf": "x"
+      },
       {
        "buf": "act"
       },
       {
-       "buf": "model.layers.21.mlp.down_proj.weight"
+       "buf": "model.layers.43.mlp.down_proj.weight"
       },
       {
-       "buf": "y"
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.44.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -122000,422 +71198,429 @@
       },
       {
        "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l21.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l44.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "x"
+       "buf": "qkvz"
       },
       {
-       "buf": "y"
+       "buf": "ba"
       },
       {
-       "buf": "residual"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.22.input_layernorm.weight_p1"
+       "buf": "model.layers.44.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.44.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 16384
       },
       {
-       "i32": 5120
+       "i32": 96
       },
       {
-       "f32": 9.999999974752427e-07
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l22.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l44.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "x"
+       "buf": "qkvz"
       },
       {
-       "buf": "model.layers.22.linear_attn.in_proj_qkvz.weight"
+       "buf": "model.layers.44.linear_attn.conv1d.weight"
       },
       {
-       "buf": "qkvz"
+       "state": "gdn"
       },
       {
-       "sym": "tokens"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 16384
+       "buf": "gdn.spec_line_index",
+       "offset": 136
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l22.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
+       "buf": "gdn.one"
+      },
       {
-       "buf": "x"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "model.layers.22.linear_attn.in_proj_ba.weight"
+       "buf": "gdn.spec_slots",
+       "offset": 1056
       },
       {
        "buf": "ba"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.44.linear_attn.A_log"
       },
       {
-       "i32": 96
+       "buf": "model.layers.44.linear_attn.dt_bias"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l22.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
+       "buf": "core_attn_out"
+      },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.44.linear_attn.norm.weight"
       },
       {
-       "buf": "model.layers.22.linear_attn.conv1d.weight"
+       "buf": "gdn_q"
       },
       {
-       "state": "gdn"
+       "buf": "gdn_k"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 72
+       "buf": "gdn_v"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "g"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "beta"
       },
       {
-       "buf": "qkvz"
+       "buf": "a_c"
       },
       {
-       "i32": 1
+       "buf": "b_c"
       },
       {
-       "i32": 385
+       "buf": "z_c"
       },
       {
-       "i64": 16384
+       "f32": 0.0883883461356163
       },
       {
-       "i64": 16384
+       "f32": 9.999999974752427e-07
       },
       {
-       "i64": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 385
+      },
+      {
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l22.post_conv",
-     "kernel": "post_conv",
+     "label": "l44.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "core_attn_out"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.44.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.22.linear_attn.A_log"
+       "buf": "residual"
       },
       {
-       "buf": "model.layers.22.linear_attn.dt_bias"
+       "buf": "model.layers.44.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "gdn_q"
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_k"
+       "i32": 5120
       },
       {
-       "buf": "gdn_v"
+       "i32": 6144
       },
       {
-       "buf": "g"
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l44.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
+     "args": [
+      {
+       "buf": "act"
       },
       {
-       "buf": "beta"
+       "buf": "x"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.44.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 96
+       "sym": "tokens"
       },
       {
-       "i32": 96
+       "i32": 17408
       },
       {
-       "i32": 2048
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l44.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "i32": 2048
+       "buf": "act"
       },
       {
-       "i32": 6144
+       "buf": "model.layers.44.mlp.down_proj.weight"
+      },
+      {
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.45.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l22.a_copy",
-     "kernel": "copy_rows",
+     "label": "l45.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "qkvz"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "ba"
       },
       {
-       "i32": 48
+       "buf": "x"
       },
       {
-       "i32": 96
+       "buf": "model.layers.45.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l22.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.45.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "ba"
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 16384
       },
       {
        "i32": 96
       },
       {
-       "i32": 48
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l22.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l45.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "model.layers.22.linear_attn.A_log"
+       "buf": "qkvz"
       },
       {
-       "buf": "a_c"
+       "buf": "model.layers.45.linear_attn.conv1d.weight"
       },
       {
-       "buf": "b_c"
+       "state": "gdn"
       },
       {
-       "buf": "model.layers.22.linear_attn.dt_bias"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 140
       },
       {
-       "f32": 20.0
+       "buf": "gdn.one"
       },
       {
-       "buf": "gdn_q"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "gdn_k"
+       "buf": "gdn.spec_slots",
+       "offset": 1088
       },
       {
-       "buf": "gdn_v"
+       "buf": "ba"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.45.linear_attn.A_log"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "model.layers.45.linear_attn.dt_bias"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "core_attn_out"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "model.layers.45.linear_attn.norm.weight"
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 544
+       "buf": "gdn_q"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "gdn_k"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "gdn_v"
       },
       {
-       "i64": 1
+       "buf": "g"
       },
       {
-       "sym": "tokens"
+       "buf": "beta"
       },
       {
-       "i64": 0
+       "buf": "a_c"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l22.z_copy",
-     "kernel": "copy_rows",
-     "args": [
+       "buf": "b_c"
+      },
       {
        "buf": "z_c"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 6144
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 16384
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 385
+      },
+      {
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l22.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l45.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.22.linear_attn.norm.weight"
+       "buf": "model.layers.45.linear_attn.out_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.45.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l22.out_proj",
-     "kernel": "gemm",
+     "label": "l45.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.22.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.45.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l22.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l45.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.45.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.22.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.46.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -122424,7 +71629,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -122432,23 +71637,32 @@
      ]
     },
     {
-     "label": "l22.gate_up",
-     "kernel": "gemm",
+     "label": "l46.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.22.mlp.gate_up_proj.weight"
+       "buf": "model.layers.46.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.46.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -122456,71 +71670,116 @@
      ]
     },
     {
-     "label": "l22.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l46.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.46.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 144
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l22.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "gdn.one"
+      },
       {
-       "buf": "act"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "model.layers.22.mlp.down_proj.weight"
+       "buf": "gdn.spec_slots",
+       "offset": 1120
       },
       {
-       "buf": "y"
+       "buf": "ba"
+      },
+      {
+       "buf": "model.layers.46.linear_attn.A_log"
+      },
+      {
+       "buf": "model.layers.46.linear_attn.dt_bias"
+      },
+      {
+       "buf": "core_attn_out"
+      },
+      {
+       "buf": "model.layers.46.linear_attn.norm.weight"
+      },
+      {
+       "buf": "gdn_q"
+      },
+      {
+       "buf": "gdn_k"
+      },
+      {
+       "buf": "gdn_v"
+      },
+      {
+       "buf": "g"
+      },
+      {
+       "buf": "beta"
+      },
+      {
+       "buf": "a_c"
+      },
+      {
+       "buf": "b_c"
+      },
+      {
+       "buf": "z_c"
+      },
+      {
+       "f32": 0.0883883461356163
+      },
+      {
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 385
       },
       {
-       "i32": 17408
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l22.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l46.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.46.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.23.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.46.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -122529,7 +71788,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -122537,23 +71796,23 @@
      ]
     },
     {
-     "label": "l23.qkv_proj",
-     "kernel": "gemm",
+     "label": "l46.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.23.self_attn.qkv_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkv"
+       "buf": "model.layers.46.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 14336
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -122561,45 +71820,32 @@
      ]
     },
     {
-     "label": "l23.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l46.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.23.self_attn.q_norm.weight_p1"
+       "buf": "x"
       },
       {
-       "i32": 256
+       "buf": "act"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "buf": "model.layers.46.mlp.down_proj.weight"
       },
       {
-       "i32": 24
+       "buf": "residual"
       },
       {
-       "i32": 14336
+       "buf": "model.layers.47.input_layernorm.weight_p1"
       },
       {
-       "i32": 512
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 5120
       },
       {
-       "i32": 256
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -122607,55 +71853,32 @@
      ]
     },
     {
-     "label": "l23.k_norm",
-     "kernel": "gemma_norm_khead",
+     "label": "l47.qkv_proj",
+     "kernel": "gemm",
      "args": [
       {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.23.self_attn.k_norm.weight_p1"
+       "buf": "x"
       },
       {
-       "i32": 256
+       "buf": "model.layers.47.self_attn.qkv_proj.weight"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
+       "buf": "qkv"
       },
       {
-       "i32": 4
+       "sym": "tokens"
       },
       {
        "i32": 14336
       },
       {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l23.rope",
-     "kernel": "mrope",
+     "label": "l47.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
@@ -122664,83 +71887,71 @@
        "buf": "k_n"
       },
       {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
+       "buf": "qkv"
       },
       {
-       "i32": 0
+       "buf": "model.layers.47.self_attn.q_norm.weight_p1"
       },
       {
-       "i64": 0
+       "buf": "model.layers.47.self_attn.k_norm.weight_p1"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l23.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
+       "buf": "cos_g"
       },
       {
-       "buf": "qkv",
-       "offset": 26624
+       "buf": "sin_g"
       },
       {
        "state": "kv",
-       "offset": 16056320
+       "offset": 35323904
       },
       {
        "state": "kv",
-       "offset": 16056832
+       "offset": 35324416
       },
       {
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l23.attn",
-     "kernel": "attn_prefill",
+     "label": "l47.attn",
+     "kernel": "attn",
      "args": [
       {
        "buf": "attn_out"
@@ -122750,11 +71961,11 @@
       },
       {
        "state": "kv",
-       "offset": 16056320
+       "offset": 35323904
       },
       {
        "state": "kv",
-       "offset": 16056832
+       "offset": 35324416
       },
       {
        "buf": "block_table"
@@ -122832,11 +72043,11 @@
      ]
     },
     {
-     "label": "l23.gate",
-     "kernel": "sigmoid_mul",
+     "label": "l47.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "gated"
+       "buf": "x"
       },
       {
        "buf": "attn_out"
@@ -122846,31 +72057,13 @@
        "offset": 512
       },
       {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l23.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
+       "buf": "model.layers.47.self_attn.o_proj.weight"
       },
       {
-       "buf": "model.layers.23.self_attn.o_proj.weight"
+       "buf": "residual"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.47.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -122880,36 +72073,15 @@
       },
       {
        "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l23.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.23.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
       },
       {
-       "sym": "tokens"
+       "i32": 256
       },
       {
-       "i32": 5120
+       "i32": 14336
       },
       {
-       "i32": 5120
+       "i32": 512
       },
       {
        "f32": 9.999999974752427e-07
@@ -122917,23 +72089,23 @@
      ]
     },
     {
-     "label": "l23.gate_up",
-     "kernel": "gemm",
+     "label": "l47.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.23.mlp.gate_up_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.47.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -122941,41 +72113,23 @@
      ]
     },
     {
-     "label": "l23.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l47.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
+       "buf": "x"
       },
       {
-       "f32": 1.0
+       "buf": "act"
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l23.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "model.layers.47.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.23.mlp.down_proj.weight"
+       "buf": "residual"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.48.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -122985,27 +72139,24 @@
       },
       {
        "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l23.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l47.fc_tap",
+     "kernel": "gemm_acc",
      "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
       {
        "buf": "residual"
       },
       {
-       "buf": "model.layers.24.input_layernorm.weight_p1"
+       "buf": "draft.fc.3.weight"
       },
       {
-       "i32": 5120
+       "buf": "fc_out"
       },
       {
        "sym": "tokens"
@@ -123015,52 +72166,34 @@
       },
       {
        "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l24.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l48.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.24.linear_attn.in_proj_qkvz.weight"
-      },
       {
        "buf": "qkvz"
       },
       {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
+       "buf": "ba"
       },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l24.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.24.linear_attn.in_proj_ba.weight"
+       "buf": "model.layers.48.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.48.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
+      {
+       "i32": 16384
+      },
       {
        "i32": 96
       },
@@ -123070,70 +72203,50 @@
      ]
     },
     {
-     "label": "l24.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l48.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.24.linear_attn.conv1d.weight"
+       "buf": "model.layers.48.linear_attn.conv1d.weight"
       },
       {
        "state": "gdn"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 76
-      },
-      {
-       "buf": "num_accepted_tokens"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 385
+       "buf": "gdn.spec_line_index",
+       "offset": 148
       },
       {
-       "i64": 16384
+       "buf": "gdn.one"
       },
       {
-       "i64": 16384
+       "buf": "cu_seqlens_q"
       },
       {
-       "i64": 0
+       "buf": "gdn.spec_slots",
+       "offset": 1152
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l24.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
+       "buf": "ba"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.48.linear_attn.A_log"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.48.linear_attn.dt_bias"
       },
       {
-       "buf": "model.layers.24.linear_attn.A_log"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.24.linear_attn.dt_bias"
+       "buf": "model.layers.48.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -123151,361 +72264,328 @@
        "buf": "beta"
       },
       {
-       "i32": 16384
+       "buf": "a_c"
       },
       {
-       "i32": 96
+       "buf": "b_c"
       },
       {
-       "i32": 96
+       "buf": "z_c"
       },
       {
-       "i32": 2048
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 2048
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 6144
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 385
       },
       {
-       "i64": 0
+       "i32": 1703936
       },
       {
-       "i64": 0
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l24.a_copy",
-     "kernel": "copy_rows",
+     "label": "l48.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "core_attn_out"
       },
       {
-       "i32": 48
+       "buf": "model.layers.48.linear_attn.out_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l24.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.48.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "ba"
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 5120
       },
       {
-       "i32": 96
+       "i32": 6144
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l24.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l48.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "model.layers.24.linear_attn.A_log"
+       "buf": "act"
       },
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "b_c"
+       "buf": "model.layers.48.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "model.layers.24.linear_attn.dt_bias"
+       "sym": "tokens"
       },
       {
-       "f32": 1.0
+       "i32": 17408
       },
       {
-       "f32": 20.0
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l48.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "buf": "gdn_q"
+       "buf": "x"
       },
       {
-       "buf": "gdn_k"
+       "buf": "act"
       },
       {
-       "buf": "gdn_v"
+       "buf": "model.layers.48.mlp.down_proj.weight"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "residual"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "model.layers.49.input_layernorm.weight_p1"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "sym": "tokens"
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 5120
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 576
+       "i32": 17408
       },
       {
-       "buf": "num_accepted_tokens"
-      },
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l49.in_proj",
+     "kernel": "gemm8_dual",
+     "args": [
       {
-       "f32": 0.0883883461356163
+       "buf": "qkvz"
       },
       {
-       "i64": 1
+       "buf": "ba"
       },
       {
-       "sym": "tokens"
+       "buf": "x"
       },
       {
-       "i64": 0
+       "buf": "model.layers.49.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l24.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
+       "buf": "model.layers.49.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 16384
       },
       {
-       "i32": 16384
+       "i32": 96
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l24.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l49.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
+       "buf": "qkvz"
       },
       {
-       "buf": "model.layers.24.linear_attn.norm.weight"
+       "buf": "model.layers.49.linear_attn.conv1d.weight"
       },
       {
-       "buf": "z_c"
+       "state": "gdn"
       },
       {
-       "i32": 128
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 128
+       "buf": "gdn.spec_line_index",
+       "offset": 152
       },
       {
-       "i32": 128
+       "buf": "gdn.one"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "buf": "cu_seqlens_q"
       },
       {
-       "f32": 9.999999974752427e-07
+       "buf": "gdn.spec_slots",
+       "offset": 1184
       },
       {
-       "i64": 0
+       "buf": "ba"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l24.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
+       "buf": "model.layers.49.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.24.linear_attn.out_proj.weight"
+       "buf": "model.layers.49.linear_attn.dt_bias"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.49.linear_attn.norm.weight"
       },
       {
-       "i32": 5120
+       "buf": "gdn_q"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l24.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
+       "buf": "gdn_k"
       },
       {
-       "buf": "y"
+       "buf": "gdn_v"
       },
       {
-       "buf": "residual"
+       "buf": "g"
       },
       {
-       "buf": "model.layers.24.post_attention_layernorm.weight_p1"
+       "buf": "beta"
       },
       {
-       "i32": 5120
+       "buf": "a_c"
       },
       {
-       "sym": "tokens"
+       "buf": "b_c"
       },
       {
-       "i32": 5120
+       "buf": "z_c"
       },
       {
-       "i32": 5120
+       "f32": 0.0883883461356163
       },
       {
        "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l24.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
       },
       {
-       "buf": "model.layers.24.mlp.gate_up_proj.weight"
+       "sym": "tokens"
       },
       {
-       "buf": "gate_up"
+       "i32": 385
       },
       {
-       "sym": "tokens"
+       "i32": 1703936
       },
       {
-       "i32": 34816
+       "i32": 10
       },
       {
-       "i32": 5120
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l24.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l49.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "act"
+       "buf": "x"
       },
       {
-       "buf": "gate_up"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 17408
+       "buf": "model.layers.49.linear_attn.out_proj.weight"
       },
       {
-       "i32": 0
+       "buf": "residual"
       },
       {
-       "f32": 1.0
+       "buf": "model.layers.49.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 0
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l24.down_proj",
-     "kernel": "gemm",
+     "label": "l49.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
        "buf": "act"
       },
       {
-       "buf": "model.layers.24.mlp.down_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.49.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 17408
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l24.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l49.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.49.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.25.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.50.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -123514,7 +72594,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -123522,45 +72602,30 @@
      ]
     },
     {
-     "label": "l25.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l50.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.25.linear_attn.in_proj_qkvz.weight"
-      },
       {
        "buf": "qkvz"
       },
       {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
+       "buf": "ba"
       },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l25.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.25.linear_attn.in_proj_ba.weight"
+       "buf": "model.layers.50.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.50.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
+      {
+       "i32": 16384
+      },
       {
        "i32": 96
       },
@@ -123570,70 +72635,50 @@
      ]
     },
     {
-     "label": "l25.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l50.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.25.linear_attn.conv1d.weight"
+       "buf": "model.layers.50.linear_attn.conv1d.weight"
       },
       {
        "state": "gdn"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 80
-      },
-      {
-       "buf": "num_accepted_tokens"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 385
+       "buf": "gdn.spec_line_index",
+       "offset": 156
       },
       {
-       "i64": 16384
+       "buf": "gdn.one"
       },
       {
-       "i64": 16384
+       "buf": "cu_seqlens_q"
       },
       {
-       "i64": 0
+       "buf": "gdn.spec_slots",
+       "offset": 1216
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l25.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
+       "buf": "ba"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.50.linear_attn.A_log"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.50.linear_attn.dt_bias"
       },
       {
-       "buf": "model.layers.25.linear_attn.A_log"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.25.linear_attn.dt_bias"
+       "buf": "model.layers.50.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -123651,192 +72696,196 @@
        "buf": "beta"
       },
       {
-       "i32": 16384
+       "buf": "a_c"
       },
       {
-       "i32": 96
+       "buf": "b_c"
       },
       {
-       "i32": 96
+       "buf": "z_c"
       },
       {
-       "i32": 2048
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 2048
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 6144
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 385
       },
       {
-       "i64": 0
+       "i32": 1703936
       },
       {
-       "i64": 0
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l25.a_copy",
-     "kernel": "copy_rows",
+     "label": "l50.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "core_attn_out"
       },
       {
-       "i32": 48
+       "buf": "model.layers.50.linear_attn.out_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l25.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.50.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "ba"
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 5120
       },
       {
-       "i32": 96
+       "i32": 6144
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l25.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l50.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "model.layers.25.linear_attn.A_log"
+       "buf": "act"
       },
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "b_c"
+       "buf": "model.layers.50.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "model.layers.25.linear_attn.dt_bias"
+       "sym": "tokens"
       },
       {
-       "f32": 1.0
+       "i32": 17408
       },
       {
-       "f32": 20.0
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l50.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "buf": "gdn_q"
+       "buf": "x"
       },
       {
-       "buf": "gdn_k"
+       "buf": "act"
       },
       {
-       "buf": "gdn_v"
+       "buf": "model.layers.50.mlp.down_proj.weight"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "residual"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "model.layers.51.input_layernorm.weight_p1"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "sym": "tokens"
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 5120
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 608
+       "i32": 17408
       },
       {
-       "buf": "num_accepted_tokens"
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l51.qkv_proj",
+     "kernel": "gemm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "model.layers.51.self_attn.qkv_proj.weight"
       },
       {
-       "i64": 1
+       "buf": "qkv"
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l25.z_copy",
-     "kernel": "copy_rows",
+     "label": "l51.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "q_n"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "k_n"
       },
       {
-       "i32": 6144
+       "buf": "qkv"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.51.self_attn.q_norm.weight_p1"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l25.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
+       "buf": "model.layers.51.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "cos_g"
       },
       {
-       "buf": "model.layers.25.linear_attn.norm.weight"
+       "buf": "sin_g"
       },
       {
-       "buf": "z_c"
+       "state": "kv",
+       "offset": 38535168
       },
       {
-       "i32": 128
+       "state": "kv",
+       "offset": 38535680
       },
       {
-       "i32": 128
+       "buf": "slot_mapping"
       },
       {
-       "i32": 128
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
        "expr": {
@@ -123844,168 +72893,142 @@
          {
           "sym": "tokens"
          },
-         48
+         4
         ]
        }
       },
       {
-       "f32": 9.999999974752427e-07
+       "i64": 25690112
       },
       {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 0
+       "i64": 512
+      },
+      {
+       "i32": 784
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l25.out_proj",
-     "kernel": "gemm",
+     "label": "l51.attn",
+     "kernel": "attn",
      "args": [
       {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.25.linear_attn.out_proj.weight"
+       "buf": "attn_out"
       },
       {
-       "buf": "y"
+       "buf": "q_n"
       },
       {
-       "sym": "tokens"
+       "state": "kv",
+       "offset": 38535168
       },
       {
-       "i32": 5120
+       "state": "kv",
+       "offset": 38535680
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l25.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
+       "buf": "block_table"
       },
       {
-       "buf": "y"
+       "buf": "seq_lens"
       },
       {
-       "buf": "residual"
+       "f32": 0.0625
       },
       {
-       "buf": "model.layers.25.post_attention_layernorm.weight_p1"
+       "buf": "kv_scales"
       },
       {
-       "i32": 5120
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "sym": "tokens"
+       "f32": 1.0
       },
       {
-       "i32": 5120
+       "f32": 0.0
       },
       {
-       "i32": 5120
+       "i64": 8
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l25.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
+       "i64": 6144
       },
       {
-       "buf": "model.layers.25.mlp.gate_up_proj.weight"
+       "i64": 256
       },
       {
-       "buf": "gate_up"
+       "i64": 6144
       },
       {
-       "sym": "tokens"
+       "i64": 256
       },
       {
-       "i32": 34816
+       "i64": 0
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l25.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "seq_lens"
       },
       {
-       "buf": "gate_up"
+       "i64": 25690112
       },
       {
-       "i32": 17408
+       "i64": 2048
       },
       {
-       "i32": 0
+       "i64": 512
       },
       {
-       "f32": 1.0
+       "i64": 25690112
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l25.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "i64": 2048
       },
       {
-       "buf": "model.layers.25.mlp.down_proj.weight"
+       "i64": 512
       },
       {
-       "buf": "y"
+       "buf": "cu_seqlens_q"
       },
       {
-       "sym": "tokens"
+       "i32": 1
       },
       {
-       "i32": 5120
+       "i64": 0
       },
       {
-       "i32": 17408
+       "i64": 0
       }
      ]
     },
     {
-     "label": "l25.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l51.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "buf": "model.layers.26.input_layernorm.weight_p1"
+       "buf": "model.layers.51.self_attn.o_proj.weight"
       },
       {
-       "i32": 5120
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.51.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -124014,7 +73037,16 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
+      },
+      {
+       "i32": 256
+      },
+      {
+       "i32": 14336
+      },
+      {
+       "i32": 512
       },
       {
        "f32": 9.999999974752427e-07
@@ -124022,23 +73054,23 @@
      ]
     },
     {
-     "label": "l26.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l51.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.26.linear_attn.in_proj_qkvz.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.51.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -124046,94 +73078,116 @@
      ]
     },
     {
-     "label": "l26.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l51.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.26.linear_attn.in_proj_ba.weight"
+       "buf": "act"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.51.mlp.down_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 96
+       "buf": "model.layers.52.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l26.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l52.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.26.linear_attn.conv1d.weight"
+       "buf": "ba"
       },
       {
-       "state": "gdn"
+       "buf": "x"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 84
+       "buf": "model.layers.52.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "model.layers.52.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "cu_seqlens_q"
+       "sym": "tokens"
+      },
+      {
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
+      {
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l52.gdn",
+     "kernel": "gdn_spec",
+     "args": [
       {
        "buf": "qkvz"
       },
       {
-       "i32": 1
+       "buf": "model.layers.52.linear_attn.conv1d.weight"
       },
       {
-       "i32": 385
+       "state": "gdn"
       },
       {
-       "i64": 16384
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i64": 16384
+       "buf": "gdn.spec_line_index",
+       "offset": 160
       },
       {
-       "i64": 0
+       "buf": "gdn.one"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l26.post_conv",
-     "kernel": "post_conv",
-     "args": [
+       "buf": "cu_seqlens_q"
+      },
       {
-       "buf": "qkvz"
+       "buf": "gdn.spec_slots",
+       "offset": 1248
+      },
+      {
+       "buf": "ba"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.52.linear_attn.A_log"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.52.linear_attn.dt_bias"
       },
       {
-       "buf": "model.layers.26.linear_attn.A_log"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.26.linear_attn.dt_bias"
+       "buf": "model.layers.52.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -124151,331 +73205,328 @@
        "buf": "beta"
       },
       {
-       "i32": 16384
+       "buf": "a_c"
       },
       {
-       "i32": 96
+       "buf": "b_c"
       },
       {
-       "i32": 96
+       "buf": "z_c"
       },
       {
-       "i32": 2048
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 2048
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 6144
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 385
       },
       {
-       "i64": 0
+       "i32": 1703936
       },
       {
-       "i64": 0
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l26.a_copy",
-     "kernel": "copy_rows",
+     "label": "l52.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "core_attn_out"
       },
       {
-       "i32": 48
+       "buf": "model.layers.52.linear_attn.out_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l26.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.52.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "ba"
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 5120
       },
       {
-       "i32": 96
+       "i32": 6144
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l26.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l52.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "model.layers.26.linear_attn.A_log"
+       "buf": "act"
       },
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "b_c"
+       "buf": "model.layers.52.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "model.layers.26.linear_attn.dt_bias"
+       "sym": "tokens"
       },
       {
-       "f32": 1.0
+       "i32": 17408
       },
       {
-       "f32": 20.0
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l52.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "buf": "gdn_q"
+       "buf": "x"
       },
       {
-       "buf": "gdn_k"
+       "buf": "act"
       },
       {
-       "buf": "gdn_v"
+       "buf": "model.layers.52.mlp.down_proj.weight"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "residual"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "model.layers.53.input_layernorm.weight_p1"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "sym": "tokens"
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 5120
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 640
+       "i32": 17408
       },
       {
-       "buf": "num_accepted_tokens"
-      },
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l53.in_proj",
+     "kernel": "gemm8_dual",
+     "args": [
       {
-       "f32": 0.0883883461356163
+       "buf": "qkvz"
       },
       {
-       "i64": 1
+       "buf": "ba"
       },
       {
-       "sym": "tokens"
+       "buf": "x"
       },
       {
-       "i64": 0
+       "buf": "model.layers.53.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l26.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
+       "buf": "model.layers.53.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 16384
       },
       {
-       "i32": 16384
+       "i32": 96
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l26.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l53.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "qkvz"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.53.linear_attn.conv1d.weight"
       },
       {
-       "buf": "model.layers.26.linear_attn.norm.weight"
+       "state": "gdn"
       },
       {
-       "buf": "z_c"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 128
+       "buf": "gdn.spec_line_index",
+       "offset": 164
       },
       {
-       "i32": 128
+       "buf": "gdn.one"
       },
       {
-       "i32": 128
+       "buf": "cu_seqlens_q"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "buf": "gdn.spec_slots",
+       "offset": 1280
       },
       {
-       "f32": 9.999999974752427e-07
+       "buf": "ba"
       },
       {
-       "i64": 0
+       "buf": "model.layers.53.linear_attn.A_log"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l26.out_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "model.layers.53.linear_attn.dt_bias"
+      },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.26.linear_attn.out_proj.weight"
+       "buf": "model.layers.53.linear_attn.norm.weight"
       },
       {
-       "buf": "y"
+       "buf": "gdn_q"
       },
       {
-       "sym": "tokens"
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
+       "buf": "gdn_v"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l26.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "g"
+      },
       {
-       "buf": "x"
+       "buf": "beta"
       },
       {
-       "buf": "y"
+       "buf": "a_c"
       },
       {
-       "buf": "residual"
+       "buf": "b_c"
       },
       {
-       "buf": "model.layers.26.post_attention_layernorm.weight_p1"
+       "buf": "z_c"
       },
       {
-       "i32": 5120
+       "f32": 0.0883883461356163
+      },
+      {
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
       },
       {
-       "f32": 9.999999974752427e-07
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l26.gate_up",
-     "kernel": "gemm",
+     "label": "l53.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.26.mlp.gate_up_proj.weight"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.53.linear_attn.out_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 34816
+       "buf": "model.layers.53.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l26.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l53.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
        "buf": "act"
       },
       {
-       "buf": "gate_up"
+       "buf": "x"
       },
       {
-       "i32": 17408
+       "buf": "model.layers.53.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 0
+       "sym": "tokens"
       },
       {
-       "f32": 1.0
+       "i32": 17408
       },
       {
-       "i32": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l26.down_proj",
-     "kernel": "gemm",
+     "label": "l53.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
+      {
+       "buf": "x"
+      },
       {
        "buf": "act"
       },
       {
-       "buf": "model.layers.26.mlp.down_proj.weight"
+       "buf": "model.layers.53.mlp.down_proj.weight"
       },
       {
-       "buf": "y"
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.54.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -124485,304 +73536,307 @@
       },
       {
        "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l26.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l54.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "x"
+       "buf": "qkvz"
       },
       {
-       "buf": "y"
+       "buf": "ba"
       },
       {
-       "buf": "residual"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.27.input_layernorm.weight_p1"
+       "buf": "model.layers.54.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.54.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 16384
       },
       {
-       "i32": 5120
+       "i32": 96
       },
       {
-       "f32": 9.999999974752427e-07
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l27.qkv_proj",
-     "kernel": "gemm",
+     "label": "l54.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "x"
+       "buf": "qkvz"
       },
       {
-       "buf": "model.layers.27.self_attn.qkv_proj.weight"
+       "buf": "model.layers.54.linear_attn.conv1d.weight"
       },
       {
-       "buf": "qkv"
+       "state": "gdn"
       },
       {
-       "sym": "tokens"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 14336
+       "buf": "gdn.spec_line_index",
+       "offset": 168
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l27.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
+       "buf": "gdn.one"
+      },
       {
-       "buf": "q_n"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "qkv"
+       "buf": "gdn.spec_slots",
+       "offset": 1312
       },
       {
-       "buf": "model.layers.27.self_attn.q_norm.weight_p1"
+       "buf": "ba"
       },
       {
-       "i32": 256
+       "buf": "model.layers.54.linear_attn.A_log"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "buf": "model.layers.54.linear_attn.dt_bias"
       },
       {
-       "i32": 24
+       "buf": "core_attn_out"
       },
       {
-       "i32": 14336
+       "buf": "model.layers.54.linear_attn.norm.weight"
       },
       {
-       "i32": 512
+       "buf": "gdn_q"
       },
       {
-       "i32": 6144
+       "buf": "gdn_k"
       },
       {
-       "i32": 256
+       "buf": "gdn_v"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l27.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
+       "buf": "g"
+      },
       {
-       "buf": "k_n"
+       "buf": "beta"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
+       "buf": "a_c"
       },
       {
-       "buf": "model.layers.27.self_attn.k_norm.weight_p1"
+       "buf": "b_c"
       },
       {
-       "i32": 256
+       "buf": "z_c"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 4
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 14336
+       "sym": "tokens"
       },
       {
-       "i32": 256
+       "i32": 385
       },
       {
-       "i32": 1024
+       "i32": 1703936
       },
       {
-       "i32": 256
+       "i32": 10
       },
       {
-       "f32": 9.999999974752427e-07
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l27.rope",
-     "kernel": "mrope",
+     "label": "l54.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "q_n"
+       "buf": "x"
       },
       {
-       "buf": "k_n"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "cos_g"
+       "buf": "model.layers.54.linear_attn.out_proj.weight"
       },
       {
-       "buf": "sin_g"
+       "buf": "residual"
       },
       {
-       "i32": 0
+       "buf": "model.layers.54.post_attention_layernorm.weight_p1"
       },
       {
-       "i64": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 5120
+      },
+      {
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l27.kv_write",
-     "kernel": "reshape_and_cache",
+     "label": "l54.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
+       "buf": "act"
       },
       {
-       "state": "kv",
-       "offset": 19267584
+       "buf": "x"
       },
       {
-       "state": "kv",
-       "offset": 19268096
+       "buf": "model.layers.54.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "slot_mapping"
+       "sym": "tokens"
       },
       {
-       "buf": "kv_scales"
+       "i32": 17408
       },
       {
-       "buf": "kv_scales",
-       "offset": 4
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l54.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "i64": 1024
+       "buf": "x"
       },
       {
-       "i64": 14336
+       "buf": "act"
       },
       {
-       "i64": 25690112
+       "buf": "model.layers.54.mlp.down_proj.weight"
       },
       {
-       "i64": 512
+       "buf": "residual"
       },
       {
-       "i64": 0
+       "buf": "model.layers.55.input_layernorm.weight_p1"
       },
       {
-       "i64": 0
+       "sym": "tokens"
       },
       {
-       "i64": 2048
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 17408
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l27.attn",
-     "kernel": "attn_prefill",
+     "label": "l55.qkv_proj",
+     "kernel": "gemm",
      "args": [
       {
-       "buf": "attn_out"
+       "buf": "x"
       },
       {
-       "buf": "q_n"
+       "buf": "model.layers.55.self_attn.qkv_proj.weight"
       },
       {
-       "state": "kv",
-       "offset": 19267584
+       "buf": "qkv"
       },
       {
-       "state": "kv",
-       "offset": 19268096
+       "sym": "tokens"
       },
       {
-       "buf": "block_table"
+       "i32": 14336
       },
       {
-       "buf": "seq_lens"
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l55.prep",
+     "kernel": "attn_prep",
+     "args": [
       {
-       "f32": 0.0625
+       "buf": "q_n"
       },
       {
-       "buf": "kv_scales"
+       "buf": "k_n"
       },
       {
-       "buf": "kv_scales",
-       "offset": 4
+       "buf": "qkv"
       },
       {
-       "f32": 1.0
+       "buf": "model.layers.55.self_attn.q_norm.weight_p1"
       },
       {
-       "f32": 0.0
+       "buf": "model.layers.55.self_attn.k_norm.weight_p1"
       },
       {
-       "i64": 8
+       "buf": "cos_g"
       },
       {
-       "i64": 6144
+       "buf": "sin_g"
       },
       {
-       "i64": 256
+       "state": "kv",
+       "offset": 41746432
       },
       {
-       "i64": 6144
+       "state": "kv",
+       "offset": 41746944
       },
       {
-       "i64": 256
+       "buf": "slot_mapping"
       },
       {
-       "i64": 0
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "buf": "seq_lens"
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
@@ -124794,203 +73848,194 @@
        "i64": 512
       },
       {
-       "i64": 25690112
+       "i32": 784
       },
       {
-       "i64": 2048
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l55.attn",
+     "kernel": "attn",
+     "args": [
+      {
+       "buf": "attn_out"
       },
       {
-       "i64": 512
+       "buf": "q_n"
       },
       {
-       "buf": "cu_seqlens_q"
+       "state": "kv",
+       "offset": 41746432
       },
       {
-       "i32": 1
+       "state": "kv",
+       "offset": 41746944
       },
       {
-       "i64": 0
+       "buf": "block_table"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l27.gate",
-     "kernel": "sigmoid_mul",
-     "args": [
+       "buf": "seq_lens"
+      },
       {
-       "buf": "gated"
+       "f32": 0.0625
       },
       {
-       "buf": "attn_out"
+       "buf": "kv_scales"
       },
       {
-       "buf": "qkv",
-       "offset": 512
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "i32": 24
+       "f32": 1.0
       },
       {
-       "i32": 256
+       "f32": 0.0
       },
       {
-       "i32": 14336
+       "i64": 8
       },
       {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l27.o_proj",
-     "kernel": "gemm",
-     "args": [
+       "i64": 6144
+      },
       {
-       "buf": "gated"
+       "i64": 256
       },
       {
-       "buf": "model.layers.27.self_attn.o_proj.weight"
+       "i64": 6144
       },
       {
-       "buf": "y"
+       "i64": 256
       },
       {
-       "sym": "tokens"
+       "i64": 0
       },
       {
-       "i32": 5120
+       "buf": "seq_lens"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l27.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "i64": 25690112
+      },
       {
-       "buf": "x"
+       "i64": 2048
       },
       {
-       "buf": "y"
+       "i64": 512
       },
       {
-       "buf": "residual"
+       "i64": 25690112
       },
       {
-       "buf": "model.layers.27.post_attention_layernorm.weight_p1"
+       "i64": 2048
       },
       {
-       "i32": 5120
+       "i64": 512
       },
       {
-       "sym": "tokens"
+       "buf": "cu_seqlens_q"
       },
       {
-       "i32": 5120
+       "i32": 1
       },
       {
-       "i32": 5120
+       "i64": 0
       },
       {
-       "f32": 9.999999974752427e-07
+       "i64": 0
       }
      ]
     },
     {
-     "label": "l27.gate_up",
-     "kernel": "gemm",
+     "label": "l55.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.27.mlp.gate_up_proj.weight"
+       "buf": "attn_out"
       },
       {
-       "buf": "gate_up"
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.55.self_attn.o_proj.weight"
       },
       {
-       "i32": 34816
+       "buf": "residual"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l27.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
+       "buf": "model.layers.55.post_attention_layernorm.weight_p1"
+      },
       {
-       "buf": "act"
+       "sym": "tokens"
       },
       {
-       "buf": "gate_up"
+       "i32": 5120
       },
       {
-       "i32": 17408
+       "i32": 6144
       },
       {
-       "i32": 0
+       "i32": 256
       },
       {
-       "f32": 1.0
+       "i32": 14336
       },
       {
-       "i32": 0
+       "i32": 512
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l27.down_proj",
-     "kernel": "gemm",
+     "label": "l55.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
        "buf": "act"
       },
       {
-       "buf": "model.layers.27.mlp.down_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.55.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 17408
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l27.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l55.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.55.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.28.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.56.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -124999,7 +74044,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -125007,45 +74052,30 @@
      ]
     },
     {
-     "label": "l28.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l56.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.28.linear_attn.in_proj_qkvz.weight"
-      },
       {
        "buf": "qkvz"
       },
       {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
+       "buf": "ba"
       },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l28.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.28.linear_attn.in_proj_ba.weight"
+       "buf": "model.layers.56.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.56.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
+      {
+       "i32": 16384
+      },
       {
        "i32": 96
       },
@@ -125055,70 +74085,50 @@
      ]
     },
     {
-     "label": "l28.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l56.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.28.linear_attn.conv1d.weight"
+       "buf": "model.layers.56.linear_attn.conv1d.weight"
       },
       {
        "state": "gdn"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 88
-      },
-      {
-       "buf": "num_accepted_tokens"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 385
+       "buf": "gdn.spec_line_index",
+       "offset": 172
       },
       {
-       "i64": 16384
+       "buf": "gdn.one"
       },
       {
-       "i64": 16384
+       "buf": "cu_seqlens_q"
       },
       {
-       "i64": 0
+       "buf": "gdn.spec_slots",
+       "offset": 1344
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l28.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
+       "buf": "ba"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.56.linear_attn.A_log"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.56.linear_attn.dt_bias"
       },
       {
-       "buf": "model.layers.28.linear_attn.A_log"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.28.linear_attn.dt_bias"
+       "buf": "model.layers.56.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -125136,361 +74146,328 @@
        "buf": "beta"
       },
       {
-       "i32": 16384
+       "buf": "a_c"
       },
       {
-       "i32": 96
+       "buf": "b_c"
       },
       {
-       "i32": 96
+       "buf": "z_c"
       },
       {
-       "i32": 2048
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 2048
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 6144
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 385
       },
       {
-       "i64": 0
+       "i32": 1703936
       },
       {
-       "i64": 0
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l28.a_copy",
-     "kernel": "copy_rows",
+     "label": "l56.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "core_attn_out"
       },
       {
-       "i32": 48
+       "buf": "model.layers.56.linear_attn.out_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l28.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.56.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "ba"
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 5120
       },
       {
-       "i32": 96
+       "i32": 6144
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l28.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l56.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "model.layers.28.linear_attn.A_log"
+       "buf": "act"
       },
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "b_c"
+       "buf": "model.layers.56.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "model.layers.28.linear_attn.dt_bias"
+       "sym": "tokens"
       },
       {
-       "f32": 1.0
+       "i32": 17408
       },
       {
-       "f32": 20.0
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l56.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "buf": "gdn_q"
+       "buf": "x"
       },
       {
-       "buf": "gdn_k"
+       "buf": "act"
       },
       {
-       "buf": "gdn_v"
+       "buf": "model.layers.56.mlp.down_proj.weight"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "residual"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "model.layers.57.input_layernorm.weight_p1"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "sym": "tokens"
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 5120
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 672
+       "i32": 17408
       },
       {
-       "buf": "num_accepted_tokens"
-      },
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l57.in_proj",
+     "kernel": "gemm8_dual",
+     "args": [
       {
-       "f32": 0.0883883461356163
+       "buf": "qkvz"
       },
       {
-       "i64": 1
+       "buf": "ba"
       },
       {
-       "sym": "tokens"
+       "buf": "x"
       },
       {
-       "i64": 0
+       "buf": "model.layers.57.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l28.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
+       "buf": "model.layers.57.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 16384
       },
       {
-       "i32": 16384
+       "i32": 96
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l28.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l57.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "qkvz"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.57.linear_attn.conv1d.weight"
       },
       {
-       "buf": "model.layers.28.linear_attn.norm.weight"
+       "state": "gdn"
       },
       {
-       "buf": "z_c"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 128
+       "buf": "gdn.spec_line_index",
+       "offset": 176
       },
       {
-       "i32": 128
+       "buf": "gdn.one"
       },
       {
-       "i32": 128
+       "buf": "cu_seqlens_q"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "buf": "gdn.spec_slots",
+       "offset": 1376
       },
       {
-       "f32": 9.999999974752427e-07
+       "buf": "ba"
       },
       {
-       "i64": 0
+       "buf": "model.layers.57.linear_attn.A_log"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l28.out_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "model.layers.57.linear_attn.dt_bias"
+      },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.28.linear_attn.out_proj.weight"
+       "buf": "model.layers.57.linear_attn.norm.weight"
       },
       {
-       "buf": "y"
+       "buf": "gdn_q"
       },
       {
-       "sym": "tokens"
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
+       "buf": "gdn_v"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l28.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "g"
+      },
       {
-       "buf": "x"
+       "buf": "beta"
       },
       {
-       "buf": "y"
+       "buf": "a_c"
       },
       {
-       "buf": "residual"
+       "buf": "b_c"
       },
       {
-       "buf": "model.layers.28.post_attention_layernorm.weight_p1"
+       "buf": "z_c"
       },
       {
-       "i32": 5120
+       "f32": 0.0883883461356163
+      },
+      {
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
       },
       {
-       "f32": 9.999999974752427e-07
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l28.gate_up",
-     "kernel": "gemm",
+     "label": "l57.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.28.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 34816
+       "buf": "model.layers.57.linear_attn.out_proj.weight"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l28.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "residual"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.57.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 17408
+       "sym": "tokens"
       },
       {
-       "i32": 0
+       "i32": 5120
       },
       {
-       "f32": 1.0
+       "i32": 6144
       },
       {
-       "i32": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l28.down_proj",
-     "kernel": "gemm",
+     "label": "l57.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
        "buf": "act"
       },
       {
-       "buf": "model.layers.28.mlp.down_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.57.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 17408
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l28.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l57.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.57.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.29.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.58.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -125499,7 +74476,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -125507,45 +74484,30 @@
      ]
     },
     {
-     "label": "l29.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l58.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.29.linear_attn.in_proj_qkvz.weight"
-      },
       {
        "buf": "qkvz"
       },
       {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
+       "buf": "ba"
       },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l29.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.29.linear_attn.in_proj_ba.weight"
+       "buf": "model.layers.58.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.58.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
+      {
+       "i32": 16384
+      },
       {
        "i32": 96
       },
@@ -125555,70 +74517,50 @@
      ]
     },
     {
-     "label": "l29.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l58.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.29.linear_attn.conv1d.weight"
+       "buf": "model.layers.58.linear_attn.conv1d.weight"
       },
       {
        "state": "gdn"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 92
-      },
-      {
-       "buf": "num_accepted_tokens"
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 385
+       "buf": "gdn.spec_line_index",
+       "offset": 180
       },
       {
-       "i64": 16384
+       "buf": "gdn.one"
       },
       {
-       "i64": 16384
+       "buf": "cu_seqlens_q"
       },
       {
-       "i64": 0
+       "buf": "gdn.spec_slots",
+       "offset": 1408
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l29.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
+       "buf": "ba"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.58.linear_attn.A_log"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.58.linear_attn.dt_bias"
       },
       {
-       "buf": "model.layers.29.linear_attn.A_log"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.29.linear_attn.dt_bias"
+       "buf": "model.layers.58.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -125636,192 +74578,196 @@
        "buf": "beta"
       },
       {
-       "i32": 16384
+       "buf": "a_c"
       },
       {
-       "i32": 96
+       "buf": "b_c"
       },
       {
-       "i32": 96
+       "buf": "z_c"
       },
       {
-       "i32": 2048
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 2048
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 6144
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 385
       },
       {
-       "i64": 0
+       "i32": 1703936
       },
       {
-       "i64": 0
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l29.a_copy",
-     "kernel": "copy_rows",
+     "label": "l58.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "core_attn_out"
       },
       {
-       "i32": 48
+       "buf": "model.layers.58.linear_attn.out_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l29.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.58.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "ba"
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 5120
       },
       {
-       "i32": 96
+       "i32": 6144
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l29.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l58.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "model.layers.29.linear_attn.A_log"
+       "buf": "act"
       },
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "b_c"
+       "buf": "model.layers.58.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "model.layers.29.linear_attn.dt_bias"
+       "sym": "tokens"
       },
       {
-       "f32": 1.0
+       "i32": 17408
       },
       {
-       "f32": 20.0
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l58.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "buf": "gdn_q"
+       "buf": "x"
       },
       {
-       "buf": "gdn_k"
+       "buf": "act"
       },
       {
-       "buf": "gdn_v"
+       "buf": "model.layers.58.mlp.down_proj.weight"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "residual"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "model.layers.59.input_layernorm.weight_p1"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "sym": "tokens"
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 5120
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 704
+       "i32": 17408
       },
       {
-       "buf": "num_accepted_tokens"
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l59.qkv_proj",
+     "kernel": "gemm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "model.layers.59.self_attn.qkv_proj.weight"
       },
       {
-       "i64": 1
+       "buf": "qkv"
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l29.z_copy",
-     "kernel": "copy_rows",
+     "label": "l59.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "q_n"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "k_n"
       },
       {
-       "i32": 6144
+       "buf": "qkv"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.59.self_attn.q_norm.weight_p1"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l29.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
+       "buf": "model.layers.59.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "cos_g"
       },
       {
-       "buf": "model.layers.29.linear_attn.norm.weight"
+       "buf": "sin_g"
       },
       {
-       "buf": "z_c"
+       "state": "kv",
+       "offset": 44957696
       },
       {
-       "i32": 128
+       "state": "kv",
+       "offset": 44958208
       },
       {
-       "i32": 128
+       "buf": "slot_mapping"
       },
       {
-       "i32": 128
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
        "expr": {
@@ -125829,168 +74775,208 @@
          {
           "sym": "tokens"
          },
-         48
+         4
         ]
        }
       },
       {
-       "f32": 9.999999974752427e-07
+       "i64": 25690112
       },
       {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 0
+       "i64": 512
+      },
+      {
+       "i32": 784
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l29.out_proj",
-     "kernel": "gemm",
+     "label": "l59.attn",
+     "kernel": "attn",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "attn_out"
       },
       {
-       "buf": "model.layers.29.linear_attn.out_proj.weight"
+       "buf": "q_n"
       },
       {
-       "buf": "y"
+       "state": "kv",
+       "offset": 44957696
       },
       {
-       "sym": "tokens"
+       "state": "kv",
+       "offset": 44958208
       },
       {
-       "i32": 5120
+       "buf": "block_table"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l29.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "seq_lens"
+      },
       {
-       "buf": "x"
+       "f32": 0.0625
       },
       {
-       "buf": "y"
+       "buf": "kv_scales"
       },
       {
-       "buf": "residual"
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "buf": "model.layers.29.post_attention_layernorm.weight_p1"
+       "f32": 1.0
       },
       {
-       "i32": 5120
+       "f32": 0.0
       },
       {
-       "sym": "tokens"
+       "i64": 8
       },
       {
-       "i32": 5120
+       "i64": 6144
       },
       {
-       "i32": 5120
+       "i64": 256
       },
       {
-       "f32": 9.999999974752427e-07
+       "i64": 6144
+      },
+      {
+       "i64": 256
+      },
+      {
+       "i64": 0
+      },
+      {
+       "buf": "seq_lens"
+      },
+      {
+       "i64": 25690112
+      },
+      {
+       "i64": 2048
+      },
+      {
+       "i64": 512
+      },
+      {
+       "i64": 25690112
+      },
+      {
+       "i64": 2048
+      },
+      {
+       "i64": 512
+      },
+      {
+       "buf": "cu_seqlens_q"
+      },
+      {
+       "i32": 1
+      },
+      {
+       "i64": 0
+      },
+      {
+       "i64": 0
       }
      ]
     },
     {
-     "label": "l29.gate_up",
-     "kernel": "gemm",
+     "label": "l59.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.29.mlp.gate_up_proj.weight"
+       "buf": "attn_out"
       },
       {
-       "buf": "gate_up"
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.59.self_attn.o_proj.weight"
       },
       {
-       "i32": 34816
+       "buf": "residual"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l29.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
+       "buf": "model.layers.59.post_attention_layernorm.weight_p1"
+      },
       {
-       "buf": "act"
+       "sym": "tokens"
       },
       {
-       "buf": "gate_up"
+       "i32": 5120
       },
       {
-       "i32": 17408
+       "i32": 6144
       },
       {
-       "i32": 0
+       "i32": 256
       },
       {
-       "f32": 1.0
+       "i32": 14336
       },
       {
-       "i32": 0
+       "i32": 512
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l29.down_proj",
-     "kernel": "gemm",
+     "label": "l59.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
        "buf": "act"
       },
       {
-       "buf": "model.layers.29.mlp.down_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.59.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 17408
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l29.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l59.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.59.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.30.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.60.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -125999,7 +74985,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -126007,17 +74993,23 @@
      ]
     },
     {
-     "label": "l30.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l60.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.30.linear_attn.in_proj_qkvz.weight"
+       "buf": "model.layers.60.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.60.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
@@ -126025,367 +75017,398 @@
       {
        "i32": 16384
       },
+      {
+       "i32": 96
+      },
       {
        "i32": 5120
       }
      ]
     },
     {
-     "label": "l30.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l60.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "x"
+       "buf": "qkvz"
       },
       {
-       "buf": "model.layers.30.linear_attn.in_proj_ba.weight"
+       "buf": "model.layers.60.linear_attn.conv1d.weight"
       },
       {
-       "buf": "ba"
+       "state": "gdn"
       },
       {
-       "sym": "tokens"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 96
+       "buf": "gdn.spec_line_index",
+       "offset": 184
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l30.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
+       "buf": "gdn.one"
+      },
       {
-       "buf": "qkvz"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "model.layers.30.linear_attn.conv1d.weight"
+       "buf": "gdn.spec_slots",
+       "offset": 1440
       },
       {
-       "state": "gdn"
+       "buf": "ba"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 96
+       "buf": "model.layers.60.linear_attn.A_log"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "model.layers.60.linear_attn.dt_bias"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.60.linear_attn.norm.weight"
       },
       {
-       "i32": 1
+       "buf": "gdn_q"
       },
       {
-       "i32": 385
+       "buf": "gdn_k"
       },
       {
-       "i64": 16384
+       "buf": "gdn_v"
       },
       {
-       "i64": 16384
+       "buf": "g"
       },
       {
-       "i64": 0
+       "buf": "beta"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l30.post_conv",
-     "kernel": "post_conv",
-     "args": [
+       "buf": "a_c"
+      },
       {
-       "buf": "qkvz"
+       "buf": "b_c"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "z_c"
       },
       {
-       "buf": "ba"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "model.layers.30.linear_attn.A_log"
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "model.layers.30.linear_attn.dt_bias"
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_q"
+       "i32": 385
       },
       {
-       "buf": "gdn_k"
+       "i32": 1703936
       },
       {
-       "buf": "gdn_v"
+       "i32": 10
       },
       {
-       "buf": "g"
+       "i32": 851968
+      }
+     ]
+    },
+    {
+     "label": "l60.out_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "buf": "beta"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.60.linear_attn.out_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
       },
       {
-       "i32": 96
+       "buf": "model.layers.60.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 2048
+       "sym": "tokens"
       },
       {
-       "i32": 2048
+       "i32": 5120
       },
       {
        "i32": 6144
       },
+      {
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l60.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
+     "args": [
+      {
+       "buf": "act"
+      },
+      {
+       "buf": "x"
+      },
+      {
+       "buf": "model.layers.60.mlp.gate_up_proj.weight"
+      },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 17408
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l30.a_copy",
-     "kernel": "copy_rows",
+     "label": "l60.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "act"
       },
       {
-       "i32": 48
+       "buf": "model.layers.60.mlp.down_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.61.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 17408
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l30.b_copy",
-     "kernel": "copy_rows",
+     "label": "l61.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "b_c"
+       "buf": "qkvz"
       },
       {
        "buf": "ba"
       },
       {
-       "i32": 48
+       "buf": "x"
+      },
+      {
+       "buf": "model.layers.61.linear_attn.in_proj_qkvz.weight"
+      },
+      {
+       "buf": "model.layers.61.linear_attn.in_proj_ba.weight"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 16384
       },
       {
        "i32": 96
       },
       {
-       "i32": 48
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l30.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l61.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "model.layers.30.linear_attn.A_log"
+       "buf": "qkvz"
       },
       {
-       "buf": "a_c"
+       "buf": "model.layers.61.linear_attn.conv1d.weight"
       },
       {
-       "buf": "b_c"
+       "state": "gdn"
       },
       {
-       "buf": "model.layers.30.linear_attn.dt_bias"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 188
       },
       {
-       "f32": 20.0
+       "buf": "gdn.one"
       },
       {
-       "buf": "gdn_q"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "gdn_k"
+       "buf": "gdn.spec_slots",
+       "offset": 1472
       },
       {
-       "buf": "gdn_v"
+       "buf": "ba"
+      },
+      {
+       "buf": "model.layers.61.linear_attn.A_log"
+      },
+      {
+       "buf": "model.layers.61.linear_attn.dt_bias"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "model.layers.61.linear_attn.norm.weight"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "gdn_q"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "gdn_k"
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 736
+       "buf": "gdn_v"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "g"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "beta"
       },
       {
-       "i64": 1
+       "buf": "a_c"
       },
       {
-       "sym": "tokens"
+       "buf": "b_c"
       },
       {
-       "i64": 0
+       "buf": "z_c"
+      },
+      {
+       "f32": 0.0883883461356163
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l30.z_copy",
-     "kernel": "copy_rows",
-     "args": [
+       "f32": 9.999999974752427e-07
+      },
       {
-       "buf": "z_c"
+       "sym": "tokens"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "i32": 385
       },
       {
-       "i32": 6144
+       "i32": 1703936
       },
       {
-       "i32": 16384
+       "i32": 10
       },
       {
-       "i32": 6144
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l30.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l61.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.30.linear_attn.norm.weight"
+       "buf": "model.layers.61.linear_attn.out_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.61.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l30.out_proj",
-     "kernel": "gemm",
+     "label": "l61.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.30.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.61.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l30.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l61.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.61.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.30.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.62.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -126394,7 +75417,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -126402,23 +75425,23 @@
      ]
     },
     {
-     "label": "l30.gate_up",
-     "kernel": "gemm",
+     "label": "l61.fc_tap",
+     "kernel": "gemm_acc",
      "args": [
       {
-       "buf": "x"
+       "buf": "residual"
       },
       {
-       "buf": "model.layers.30.mlp.gate_up_proj.weight"
+       "buf": "draft.fc.4.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "fc_out"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 5120
       },
       {
        "i32": 5120
@@ -126426,197 +75449,215 @@
      ]
     },
     {
-     "label": "l30.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l62.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "ba"
       },
       {
-       "i32": 17408
+       "buf": "x"
       },
       {
-       "i32": 0
+       "buf": "model.layers.62.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "f32": 1.0
+       "buf": "model.layers.62.linear_attn.in_proj_ba.weight"
       },
       {
-       "i32": 0
+       "sym": "tokens"
+      },
+      {
+       "i32": 16384
+      },
+      {
+       "i32": 96
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l30.down_proj",
-     "kernel": "gemm",
+     "label": "l62.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "model.layers.30.mlp.down_proj.weight"
+       "buf": "model.layers.62.linear_attn.conv1d.weight"
       },
       {
-       "buf": "y"
+       "state": "gdn"
       },
       {
-       "sym": "tokens"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 5120
+       "buf": "gdn.spec_line_index",
+       "offset": 192
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l30.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "gdn.one"
+      },
       {
-       "buf": "x"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "y"
+       "buf": "gdn.spec_slots",
+       "offset": 1504
       },
       {
-       "buf": "residual"
+       "buf": "ba"
       },
       {
-       "buf": "model.layers.31.input_layernorm.weight_p1"
+       "buf": "model.layers.62.linear_attn.A_log"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.62.linear_attn.dt_bias"
       },
       {
-       "sym": "tokens"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.62.linear_attn.norm.weight"
       },
       {
-       "i32": 5120
+       "buf": "gdn_q"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l31.qkv_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "gdn_k"
+      },
       {
-       "buf": "x"
+       "buf": "gdn_v"
       },
       {
-       "buf": "model.layers.31.self_attn.qkv_proj.weight"
+       "buf": "g"
       },
       {
-       "buf": "qkv"
+       "buf": "beta"
+      },
+      {
+       "buf": "a_c"
+      },
+      {
+       "buf": "b_c"
+      },
+      {
+       "buf": "z_c"
+      },
+      {
+       "f32": 0.0883883461356163
+      },
+      {
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 14336
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l31.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l62.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.31.self_attn.q_norm.weight_p1"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 256
+       "buf": "model.layers.62.linear_attn.out_proj.weight"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "buf": "residual"
       },
       {
-       "i32": 24
+       "buf": "model.layers.62.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 14336
+       "sym": "tokens"
       },
       {
-       "i32": 512
+       "i32": 5120
       },
       {
        "i32": 6144
       },
-      {
-       "i32": 256
-      },
       {
        "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l31.k_norm",
-     "kernel": "gemma_norm_khead",
+     "label": "l62.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "k_n"
+       "buf": "act"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
+       "buf": "x"
       },
       {
-       "buf": "model.layers.31.self_attn.k_norm.weight_p1"
+       "buf": "model.layers.62.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 256
+       "sym": "tokens"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
+       "i32": 17408
       },
       {
-       "i32": 4
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l62.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "i32": 14336
+       "buf": "act"
       },
       {
-       "i32": 256
+       "buf": "model.layers.62.mlp.down_proj.weight"
       },
       {
-       "i32": 1024
+       "buf": "residual"
       },
       {
-       "i32": 256
+       "buf": "model.layers.63.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -126624,93 +75665,105 @@
      ]
     },
     {
-     "label": "l31.rope",
-     "kernel": "mrope",
+     "label": "l63.qkv_proj",
+     "kernel": "gemm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
+       "buf": "x"
       },
       {
-       "buf": "cos_g"
+       "buf": "model.layers.63.self_attn.qkv_proj.weight"
       },
       {
-       "buf": "sin_g"
+       "buf": "qkv"
       },
       {
-       "i32": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l31.kv_write",
-     "kernel": "reshape_and_cache",
+     "label": "l63.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "k_n"
+       "buf": "q_n"
       },
       {
-       "buf": "qkv",
-       "offset": 26624
+       "buf": "k_n"
       },
       {
-       "state": "kv",
-       "offset": 22478848
+       "buf": "qkv"
       },
       {
-       "state": "kv",
-       "offset": 22479360
+       "buf": "model.layers.63.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "slot_mapping"
+       "buf": "model.layers.63.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "kv_scales"
+       "buf": "cos_g"
       },
       {
-       "buf": "kv_scales",
-       "offset": 4
+       "buf": "sin_g"
       },
       {
-       "i64": 1024
+       "state": "kv",
+       "offset": 48168960
       },
       {
-       "i64": 14336
+       "state": "kv",
+       "offset": 48169472
       },
       {
-       "i64": 25690112
+       "buf": "slot_mapping"
       },
       {
-       "i64": 512
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 0
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i64": 0
+       "i64": 25690112
       },
       {
        "i64": 2048
       },
       {
-       "i64": 0
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l31.attn",
-     "kernel": "attn_prefill",
+     "label": "l63.attn",
+     "kernel": "attn",
      "args": [
       {
        "buf": "attn_out"
@@ -126720,11 +75773,11 @@
       },
       {
        "state": "kv",
-       "offset": 22478848
+       "offset": 48168960
       },
       {
        "state": "kv",
-       "offset": 22479360
+       "offset": 48169472
       },
       {
        "buf": "block_table"
@@ -126802,11 +75855,11 @@
      ]
     },
     {
-     "label": "l31.gate",
-     "kernel": "sigmoid_mul",
+     "label": "l63.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "gated"
+       "buf": "x"
       },
       {
        "buf": "attn_out"
@@ -126816,31 +75869,13 @@
        "offset": 512
       },
       {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l31.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
+       "buf": "model.layers.63.self_attn.o_proj.weight"
       },
       {
-       "buf": "model.layers.31.self_attn.o_proj.weight"
+       "buf": "residual"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.63.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -126850,36 +75885,15 @@
       },
       {
        "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l31.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
       },
       {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.31.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
+       "i32": 256
       },
       {
-       "i32": 5120
+       "i32": 14336
       },
       {
-       "i32": 5120
+       "i32": 512
       },
       {
        "f32": 9.999999974752427e-07
@@ -126887,23 +75901,23 @@
      ]
     },
     {
-     "label": "l31.gate_up",
-     "kernel": "gemm",
+     "label": "l63.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.31.mlp.gate_up_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.63.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -126911,41 +75925,23 @@
      ]
     },
     {
-     "label": "l31.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l63.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
+       "buf": "x"
       },
       {
-       "f32": 1.0
+       "buf": "act"
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l31.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "model.layers.63.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.31.mlp.down_proj.weight"
+       "buf": "residual"
       },
       {
-       "buf": "y"
+       "buf": "model.norm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -126955,264 +75951,234 @@
       },
       {
        "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l31.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "lm_head",
+     "kernel": "gemm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.32.input_layernorm.weight_p1"
+       "buf": "lm_head.weight"
       },
       {
-       "i32": 5120
+       "buf": "logits"
       },
       {
-       "sym": "tokens"
+       "i32": 1
       },
       {
-       "i32": 5120
+       "i32": 248320
       },
       {
        "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l32.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "sample",
+     "kernel": "argmax_row",
      "args": [
       {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.32.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
+       "buf": "logits"
       },
       {
-       "i32": 16384
+       "buf": "next_token"
       },
       {
-       "i32": 5120
+       "i32": 248320
       }
      ]
-    },
+    }
+   ]
+  },
+  "verify": {
+   "dispatches": [
     {
-     "label": "l32.in_proj_ba",
-     "kernel": "gemm",
+     "label": "embed",
+     "kernel": "embedding",
      "args": [
       {
-       "buf": "x"
+       "buf": "token_ids"
       },
       {
-       "buf": "model.layers.32.linear_attn.in_proj_ba.weight"
+       "buf": "model.embed_tokens.weight"
       },
       {
-       "buf": "ba"
+       "buf": "residual"
       },
       {
        "sym": "tokens"
       },
-      {
-       "i32": 96
-      },
       {
        "i32": 5120
       }
      ]
     },
     {
-     "label": "l32.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "rope_cos",
+     "kernel": "embedding",
      "args": [
       {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.32.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 100
+       "buf": "positions"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "rope.cos"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "cos_g"
       },
       {
-       "buf": "qkvz"
+       "sym": "tokens"
       },
       {
-       "i32": 1
-      },
+       "i32": 32
+      }
+     ]
+    },
+    {
+     "label": "rope_sin",
+     "kernel": "embedding",
+     "args": [
       {
-       "i32": 385
+       "buf": "positions"
       },
       {
-       "i64": 16384
+       "buf": "rope.sin"
       },
       {
-       "i64": 16384
+       "buf": "sin_g"
       },
       {
-       "i64": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 32
       }
      ]
     },
     {
-     "label": "l32.post_conv",
-     "kernel": "post_conv",
+     "label": "l0.input_norm",
+     "kernel": "gemma_norm",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "residual"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.0.input_layernorm.weight_p1"
       },
       {
-       "buf": "model.layers.32.linear_attn.A_log"
+       "i32": 5120
       },
       {
-       "buf": "model.layers.32.linear_attn.dt_bias"
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_q"
+       "i32": 1
       },
       {
-       "buf": "gdn_k"
+       "i32": 5120
       },
       {
-       "buf": "gdn_v"
+       "i32": 0
       },
       {
-       "buf": "g"
+       "i32": 5120
       },
       {
-       "buf": "beta"
+       "i32": 0
       },
       {
-       "i32": 16384
-      },
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l0.in_proj",
+     "kernel": "gemm8_dual",
+     "args": [
       {
-       "i32": 96
+       "buf": "qkvz"
       },
       {
-       "i32": 96
+       "buf": "ba"
       },
       {
-       "i32": 2048
+       "buf": "x"
       },
       {
-       "i32": 2048
+       "buf": "model.layers.0.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i32": 6144
+       "buf": "model.layers.0.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 16384
       },
       {
-       "i64": 0
+       "i32": 96
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l32.a_copy",
-     "kernel": "copy_rows",
+     "label": "l0.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
+       "buf": "qkvz"
       },
       {
-       "i32": 48
+       "buf": "model.layers.0.linear_attn.conv1d.weight"
       },
       {
-       "i32": 96
+       "state": "gdn"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l32.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "buf": "ba"
+       "buf": "gdn.spec_line_index",
+       "offset": 4
       },
       {
-       "i32": 48
+       "buf": "num_accepted_tokens"
       },
       {
-       "i32": 96
+       "buf": "cu_seqlens_q"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l32.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.32.linear_attn.A_log"
+       "buf": "gdn.spec_slots"
       },
       {
-       "buf": "a_c"
+       "buf": "ba"
       },
       {
-       "buf": "b_c"
+       "buf": "model.layers.0.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.32.linear_attn.dt_bias"
+       "buf": "model.layers.0.linear_attn.dt_bias"
       },
       {
-       "f32": 1.0
+       "buf": "core_attn_out"
       },
       {
-       "f32": 20.0
+       "buf": "model.layers.0.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -127224,153 +76190,118 @@
        "buf": "gdn_v"
       },
       {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "g"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "beta"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "a_c"
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 768
+       "buf": "b_c"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "z_c"
       },
       {
        "f32": 0.0883883461356163
       },
       {
-       "i64": 1
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l32.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
+       "i32": 385
       },
       {
-       "i32": 6144
+       "i32": 1703936
       },
       {
-       "i32": 16384
+       "i32": 10
       },
       {
-       "i32": 6144
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l32.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l0.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.32.linear_attn.norm.weight"
+       "buf": "model.layers.0.linear_attn.out_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.0.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l32.out_proj",
-     "kernel": "gemm",
+     "label": "l0.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.32.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.0.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l32.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l0.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.0.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.32.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.1.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -127379,7 +76310,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -127387,23 +76318,32 @@
      ]
     },
     {
-     "label": "l32.gate_up",
-     "kernel": "gemm",
+     "label": "l1.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.32.mlp.gate_up_proj.weight"
+       "buf": "model.layers.1.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.1.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -127411,308 +76351,266 @@
      ]
     },
     {
-     "label": "l32.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l1.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
+       "buf": "qkvz"
       },
       {
-       "i32": 17408
+       "buf": "model.layers.1.linear_attn.conv1d.weight"
       },
       {
-       "i32": 0
+       "state": "gdn"
       },
       {
-       "f32": 1.0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l32.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "gdn.spec_line_index",
+       "offset": 8
       },
       {
-       "buf": "model.layers.32.mlp.down_proj.weight"
+       "buf": "num_accepted_tokens"
       },
       {
-       "buf": "y"
+       "buf": "cu_seqlens_q"
       },
       {
-       "sym": "tokens"
+       "buf": "gdn.spec_slots",
+       "offset": 32
       },
       {
-       "i32": 5120
+       "buf": "ba"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l32.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
+       "buf": "model.layers.1.linear_attn.A_log"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.1.linear_attn.dt_bias"
       },
       {
-       "buf": "residual"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.33.input_layernorm.weight_p1"
+       "buf": "model.layers.1.linear_attn.norm.weight"
       },
       {
-       "i32": 5120
+       "buf": "gdn_q"
       },
       {
-       "sym": "tokens"
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
+       "buf": "gdn_v"
       },
       {
-       "i32": 5120
+       "buf": "g"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l33.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
+       "buf": "beta"
       },
       {
-       "buf": "model.layers.33.linear_attn.in_proj_qkvz.weight"
+       "buf": "a_c"
       },
       {
-       "buf": "qkvz"
+       "buf": "b_c"
       },
       {
-       "sym": "tokens"
+       "buf": "z_c"
       },
       {
-       "i32": 16384
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l33.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "model.layers.33.linear_attn.in_proj_ba.weight"
+       "sym": "tokens"
       },
       {
-       "buf": "ba"
+       "i32": 385
       },
       {
-       "sym": "tokens"
+       "i32": 1703936
       },
       {
-       "i32": 96
+       "i32": 10
       },
       {
-       "i32": 5120
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l33.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l1.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.33.linear_attn.conv1d.weight"
+       "buf": "core_attn_out"
       },
       {
-       "state": "gdn"
+       "buf": "model.layers.1.linear_attn.out_proj.weight"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 104
+       "buf": "residual"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "model.layers.1.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "cu_seqlens_q"
+       "sym": "tokens"
       },
       {
-       "buf": "qkvz"
+       "i32": 5120
       },
       {
-       "i32": 1
+       "i32": 6144
       },
       {
-       "i32": 385
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l1.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
+     "args": [
+      {
+       "buf": "act"
       },
       {
-       "i64": 16384
+       "buf": "x"
       },
       {
-       "i64": 16384
+       "buf": "model.layers.1.mlp.gate_up_proj.weight"
       },
       {
-       "i64": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 17408
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l33.post_conv",
-     "kernel": "post_conv",
+     "label": "l1.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
+       "buf": "x"
       },
       {
-       "buf": "ba"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.33.linear_attn.A_log"
+       "buf": "model.layers.1.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.33.linear_attn.dt_bias"
+       "buf": "residual"
       },
       {
-       "buf": "gdn_q"
+       "buf": "model.layers.2.input_layernorm.weight_p1"
       },
       {
-       "buf": "gdn_k"
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_v"
+       "i32": 5120
       },
       {
-       "buf": "g"
+       "i32": 17408
       },
       {
-       "buf": "beta"
-      },
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l2.in_proj",
+     "kernel": "gemm8_dual",
+     "args": [
       {
-       "i32": 16384
+       "buf": "qkvz"
       },
       {
-       "i32": 96
+       "buf": "ba"
       },
       {
-       "i32": 96
+       "buf": "x"
       },
       {
-       "i32": 2048
+       "buf": "model.layers.2.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i32": 2048
+       "buf": "model.layers.2.linear_attn.in_proj_ba.weight"
       },
       {
-       "i32": 6144
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 16384
       },
       {
-       "i64": 0
+       "i32": 96
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l33.a_copy",
-     "kernel": "copy_rows",
+     "label": "l2.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
+       "buf": "qkvz"
       },
       {
-       "i32": 48
+       "buf": "model.layers.2.linear_attn.conv1d.weight"
       },
       {
-       "i32": 96
+       "state": "gdn"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l33.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "buf": "ba"
+       "buf": "gdn.spec_line_index",
+       "offset": 12
       },
       {
-       "i32": 48
+       "buf": "num_accepted_tokens"
       },
       {
-       "i32": 96
+       "buf": "cu_seqlens_q"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l33.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.33.linear_attn.A_log"
+       "buf": "gdn.spec_slots",
+       "offset": 64
       },
       {
-       "buf": "a_c"
+       "buf": "ba"
       },
       {
-       "buf": "b_c"
+       "buf": "model.layers.2.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.33.linear_attn.dt_bias"
+       "buf": "model.layers.2.linear_attn.dt_bias"
       },
       {
-       "f32": 1.0
+       "buf": "core_attn_out"
       },
       {
-       "f32": 20.0
+       "buf": "model.layers.2.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -127724,153 +76622,118 @@
        "buf": "gdn_v"
       },
       {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "g"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "beta"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "a_c"
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 800
+       "buf": "b_c"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "z_c"
       },
       {
        "f32": 0.0883883461356163
       },
       {
-       "i64": 1
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l33.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
+       "i32": 385
       },
       {
-       "i32": 6144
+       "i32": 1703936
       },
       {
-       "i32": 16384
+       "i32": 10
       },
       {
-       "i32": 6144
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l33.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l2.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.33.linear_attn.norm.weight"
+       "buf": "model.layers.2.linear_attn.out_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.2.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l33.out_proj",
-     "kernel": "gemm",
+     "label": "l2.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.33.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.2.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l33.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l2.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.2.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.33.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.3.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -127879,7 +76742,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -127887,23 +76750,23 @@
      ]
     },
     {
-     "label": "l33.gate_up",
+     "label": "l3.qkv_proj",
      "kernel": "gemm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.33.mlp.gate_up_proj.weight"
+       "buf": "model.layers.3.self_attn.qkv_proj.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "qkv"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 14336
       },
       {
        "i32": 5120
@@ -127911,80 +76774,71 @@
      ]
     },
     {
-     "label": "l33.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l3.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
+       "buf": "q_n"
       },
       {
-       "i32": 0
+       "buf": "k_n"
       },
       {
-       "f32": 1.0
+       "buf": "qkv"
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l33.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "model.layers.3.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "model.layers.33.mlp.down_proj.weight"
+       "buf": "model.layers.3.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "y"
+       "buf": "cos_g"
       },
       {
-       "sym": "tokens"
+       "buf": "sin_g"
       },
       {
-       "i32": 5120
+       "state": "kv"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l33.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
+       "state": "kv",
+       "offset": 512
       },
       {
-       "buf": "y"
+       "buf": "slot_mapping"
       },
       {
-       "buf": "residual"
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "buf": "model.layers.34.input_layernorm.weight_p1"
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i32": 5120
+       "i64": 25690112
       },
       {
-       "sym": "tokens"
+       "i64": 2048
       },
       {
-       "i32": 5120
+       "i64": 512
       },
       {
-       "i32": 5120
+       "i32": 784
       },
       {
        "f32": 9.999999974752427e-07
@@ -127992,114 +76846,88 @@
      ]
     },
     {
-     "label": "l33.fc_tap",
-     "kernel": "gemm_acc",
+     "label": "l3.attn",
+     "kernel": "attn_prefill",
      "args": [
       {
-       "buf": "residual"
+       "buf": "attn_out"
       },
       {
-       "buf": "draft.fc.2.weight"
+       "buf": "q_n"
       },
       {
-       "buf": "fc_out"
+       "state": "kv"
       },
       {
-       "sym": "tokens"
+       "state": "kv",
+       "offset": 512
       },
       {
-       "i32": 5120
+       "buf": "block_table"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l34.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
+       "buf": "seq_lens"
       },
       {
-       "buf": "model.layers.34.linear_attn.in_proj_qkvz.weight"
+       "f32": 0.0625
       },
       {
-       "buf": "qkvz"
+       "buf": "kv_scales"
       },
       {
-       "sym": "tokens"
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "i32": 16384
+       "f32": 1.0
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l34.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
+       "f32": 0.0
       },
       {
-       "buf": "model.layers.34.linear_attn.in_proj_ba.weight"
+       "i64": 8
       },
       {
-       "buf": "ba"
+       "i64": 6144
       },
       {
-       "sym": "tokens"
+       "i64": 256
       },
       {
-       "i32": 96
+       "i64": 6144
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l34.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
-      {
-       "buf": "qkvz"
+       "i64": 256
       },
       {
-       "buf": "model.layers.34.linear_attn.conv1d.weight"
+       "i64": 0
       },
       {
-       "state": "gdn"
+       "buf": "seq_lens"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 108
+       "i64": 25690112
       },
       {
-       "buf": "num_accepted_tokens"
+       "i64": 2048
       },
       {
-       "buf": "cu_seqlens_q"
+       "i64": 512
       },
       {
-       "buf": "qkvz"
+       "i64": 25690112
       },
       {
-       "i32": 1
+       "i64": 2048
       },
       {
-       "i32": 385
+       "i64": 512
       },
       {
-       "i64": 16384
+       "buf": "cu_seqlens_q"
       },
       {
-       "i64": 16384
+       "i32": 1
       },
       {
        "i64": 0
@@ -128110,291 +76938,309 @@
      ]
     },
     {
-     "label": "l34.post_conv",
-     "kernel": "post_conv",
+     "label": "l3.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "attn_out"
       },
       {
-       "buf": "ba"
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "buf": "model.layers.34.linear_attn.A_log"
+       "buf": "model.layers.3.self_attn.o_proj.weight"
       },
       {
-       "buf": "model.layers.34.linear_attn.dt_bias"
+       "buf": "residual"
       },
       {
-       "buf": "gdn_q"
+       "buf": "model.layers.3.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "gdn_k"
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_v"
+       "i32": 5120
       },
       {
-       "buf": "g"
+       "i32": 6144
       },
       {
-       "buf": "beta"
+       "i32": 256
       },
       {
-       "i32": 16384
+       "i32": 14336
       },
       {
-       "i32": 96
+       "i32": 512
       },
       {
-       "i32": 96
-      },
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l3.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
+     "args": [
       {
-       "i32": 2048
+       "buf": "act"
       },
       {
-       "i32": 2048
+       "buf": "x"
       },
       {
-       "i32": 6144
+       "buf": "model.layers.3.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 17408
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l34.a_copy",
-     "kernel": "copy_rows",
+     "label": "l3.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "act"
+      },
+      {
+       "buf": "model.layers.3.mlp.down_proj.weight"
       },
       {
-       "i32": 48
+       "buf": "residual"
       },
       {
-       "i32": 96
+       "buf": "model.layers.4.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 17408
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l34.b_copy",
-     "kernel": "copy_rows",
+     "label": "l4.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "b_c"
+       "buf": "qkvz"
       },
       {
        "buf": "ba"
       },
       {
-       "i32": 48
+       "buf": "x"
+      },
+      {
+       "buf": "model.layers.4.linear_attn.in_proj_qkvz.weight"
+      },
+      {
+       "buf": "model.layers.4.linear_attn.in_proj_ba.weight"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 16384
       },
       {
        "i32": 96
       },
       {
-       "i32": 48
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l34.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l4.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "model.layers.34.linear_attn.A_log"
+       "buf": "qkvz"
       },
       {
-       "buf": "a_c"
+       "buf": "model.layers.4.linear_attn.conv1d.weight"
       },
       {
-       "buf": "b_c"
+       "state": "gdn"
       },
       {
-       "buf": "model.layers.34.linear_attn.dt_bias"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 16
       },
       {
-       "f32": 20.0
+       "buf": "num_accepted_tokens"
       },
       {
-       "buf": "gdn_q"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "gdn_k"
+       "buf": "gdn.spec_slots",
+       "offset": 96
       },
       {
-       "buf": "gdn_v"
+       "buf": "ba"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.4.linear_attn.A_log"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "model.layers.4.linear_attn.dt_bias"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "core_attn_out"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "model.layers.4.linear_attn.norm.weight"
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 832
+       "buf": "gdn_q"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "gdn_k"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "gdn_v"
       },
       {
-       "i64": 1
+       "buf": "g"
       },
       {
-       "sym": "tokens"
+       "buf": "beta"
       },
       {
-       "i64": 0
+       "buf": "a_c"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l34.z_copy",
-     "kernel": "copy_rows",
-     "args": [
+       "buf": "b_c"
+      },
       {
        "buf": "z_c"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 6144
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 16384
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 385
+      },
+      {
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l34.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l4.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.34.linear_attn.norm.weight"
+       "buf": "model.layers.4.linear_attn.out_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.4.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l34.out_proj",
-     "kernel": "gemm",
+     "label": "l4.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.34.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.4.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l34.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l4.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.4.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.34.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.5.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -128403,7 +77249,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -128411,23 +77257,32 @@
      ]
     },
     {
-     "label": "l34.gate_up",
-     "kernel": "gemm",
+     "label": "l5.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.34.mlp.gate_up_proj.weight"
+       "buf": "model.layers.5.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.5.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -128435,451 +77290,413 @@
      ]
     },
     {
-     "label": "l34.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l5.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.5.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 20
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l34.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "num_accepted_tokens"
       },
       {
-       "buf": "model.layers.34.mlp.down_proj.weight"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "y"
+       "buf": "gdn.spec_slots",
+       "offset": 128
       },
       {
-       "sym": "tokens"
+       "buf": "ba"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.5.linear_attn.A_log"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l34.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "model.layers.5.linear_attn.dt_bias"
+      },
       {
-       "buf": "x"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.5.linear_attn.norm.weight"
       },
       {
-       "buf": "residual"
+       "buf": "gdn_q"
       },
       {
-       "buf": "model.layers.35.input_layernorm.weight_p1"
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
+       "buf": "gdn_v"
       },
       {
-       "sym": "tokens"
+       "buf": "g"
       },
       {
-       "i32": 5120
+       "buf": "beta"
       },
       {
-       "i32": 5120
+       "buf": "a_c"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l35.qkv_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "b_c"
+      },
       {
-       "buf": "x"
+       "buf": "z_c"
       },
       {
-       "buf": "model.layers.35.self_attn.qkv_proj.weight"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "qkv"
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 14336
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l35.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l5.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.35.self_attn.q_norm.weight_p1"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 256
+       "buf": "model.layers.5.linear_attn.out_proj.weight"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "buf": "residual"
       },
       {
-       "i32": 24
+       "buf": "model.layers.5.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 14336
+       "sym": "tokens"
       },
       {
-       "i32": 512
+       "i32": 5120
       },
       {
        "i32": 6144
       },
-      {
-       "i32": 256
-      },
       {
        "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l35.k_norm",
-     "kernel": "gemma_norm_khead",
+     "label": "l5.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.35.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
+       "buf": "act"
       },
       {
-       "i32": 14336
+       "buf": "x"
       },
       {
-       "i32": 256
+       "buf": "model.layers.5.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 1024
+       "sym": "tokens"
       },
       {
-       "i32": 256
+       "i32": 17408
       },
       {
-       "f32": 9.999999974752427e-07
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l35.rope",
-     "kernel": "mrope",
+     "label": "l5.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "q_n"
+       "buf": "x"
       },
       {
-       "buf": "k_n"
+       "buf": "act"
       },
       {
-       "buf": "cos_g"
+       "buf": "model.layers.5.mlp.down_proj.weight"
       },
       {
-       "buf": "sin_g"
+       "buf": "residual"
       },
       {
-       "i32": 0
+       "buf": "model.layers.6.input_layernorm.weight_p1"
       },
       {
-       "i64": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l35.kv_write",
-     "kernel": "reshape_and_cache",
+     "label": "l5.fc_tap",
+     "kernel": "gemm",
      "args": [
       {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
+       "buf": "residual"
       },
       {
-       "state": "kv",
-       "offset": 25690112
+       "buf": "draft.fc.0.weight"
       },
       {
-       "state": "kv",
-       "offset": 25690624
+       "buf": "fc_out"
       },
       {
-       "buf": "slot_mapping"
+       "sym": "tokens"
       },
       {
-       "buf": "kv_scales"
+       "i32": 5120
       },
       {
-       "buf": "kv_scales",
-       "offset": 4
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l6.in_proj",
+     "kernel": "gemm8_dual",
+     "args": [
       {
-       "i64": 1024
+       "buf": "qkvz"
       },
       {
-       "i64": 14336
+       "buf": "ba"
       },
       {
-       "i64": 25690112
+       "buf": "x"
       },
       {
-       "i64": 512
+       "buf": "model.layers.6.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i64": 0
+       "buf": "model.layers.6.linear_attn.in_proj_ba.weight"
       },
       {
-       "i64": 0
+       "sym": "tokens"
       },
       {
-       "i64": 2048
+       "i32": 16384
       },
       {
-       "i64": 0
+       "i32": 96
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l35.attn",
-     "kernel": "attn_prefill",
+     "label": "l6.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "attn_out"
+       "buf": "qkvz"
       },
       {
-       "buf": "q_n"
+       "buf": "model.layers.6.linear_attn.conv1d.weight"
       },
       {
-       "state": "kv",
-       "offset": 25690112
+       "state": "gdn"
       },
       {
-       "state": "kv",
-       "offset": 25690624
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "buf": "block_table"
+       "buf": "gdn.spec_line_index",
+       "offset": 24
       },
       {
-       "buf": "seq_lens"
+       "buf": "num_accepted_tokens"
       },
       {
-       "f32": 0.0625
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "kv_scales"
+       "buf": "gdn.spec_slots",
+       "offset": 160
       },
       {
-       "buf": "kv_scales",
-       "offset": 4
+       "buf": "ba"
       },
       {
-       "f32": 1.0
+       "buf": "model.layers.6.linear_attn.A_log"
       },
       {
-       "f32": 0.0
+       "buf": "model.layers.6.linear_attn.dt_bias"
       },
       {
-       "i64": 8
+       "buf": "core_attn_out"
       },
       {
-       "i64": 6144
+       "buf": "model.layers.6.linear_attn.norm.weight"
       },
       {
-       "i64": 256
+       "buf": "gdn_q"
       },
       {
-       "i64": 6144
+       "buf": "gdn_k"
       },
       {
-       "i64": 256
+       "buf": "gdn_v"
       },
       {
-       "i64": 0
+       "buf": "g"
       },
       {
-       "buf": "seq_lens"
+       "buf": "beta"
       },
       {
-       "i64": 25690112
+       "buf": "a_c"
       },
       {
-       "i64": 2048
+       "buf": "b_c"
       },
       {
-       "i64": 512
+       "buf": "z_c"
       },
       {
-       "i64": 25690112
+       "f32": 0.0883883461356163
       },
       {
-       "i64": 2048
+       "f32": 9.999999974752427e-07
       },
       {
-       "i64": 512
+       "sym": "tokens"
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 385
       },
       {
-       "i32": 1
+       "i32": 1703936
       },
       {
-       "i64": 0
+       "i32": 10
       },
       {
-       "i64": 0
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l35.gate",
-     "kernel": "sigmoid_mul",
+     "label": "l6.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "gated"
+       "buf": "x"
       },
       {
-       "buf": "attn_out"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "qkv",
-       "offset": 512
+       "buf": "model.layers.6.linear_attn.out_proj.weight"
       },
       {
-       "i32": 24
+       "buf": "residual"
       },
       {
-       "i32": 256
+       "buf": "model.layers.6.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 14336
+       "sym": "tokens"
       },
       {
-       "i32": 512
+       "i32": 5120
+      },
+      {
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l35.o_proj",
-     "kernel": "gemm",
+     "label": "l6.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "gated"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.35.self_attn.o_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.6.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l35.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l6.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.6.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.35.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.7.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -128888,7 +77705,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -128896,23 +77713,23 @@
      ]
     },
     {
-     "label": "l35.gate_up",
+     "label": "l7.qkv_proj",
      "kernel": "gemm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.35.mlp.gate_up_proj.weight"
+       "buf": "model.layers.7.self_attn.qkv_proj.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "qkv"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 14336
       },
       {
        "i32": 5120
@@ -128920,80 +77737,72 @@
      ]
     },
     {
-     "label": "l35.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l7.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
+       "buf": "q_n"
       },
       {
-       "i32": 0
+       "buf": "k_n"
       },
       {
-       "f32": 1.0
+       "buf": "qkv"
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l35.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "model.layers.7.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "model.layers.35.mlp.down_proj.weight"
+       "buf": "model.layers.7.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "y"
+       "buf": "cos_g"
       },
       {
-       "sym": "tokens"
+       "buf": "sin_g"
       },
       {
-       "i32": 5120
+       "state": "kv",
+       "offset": 3211264
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l35.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
+       "state": "kv",
+       "offset": 3211776
       },
       {
-       "buf": "y"
+       "buf": "slot_mapping"
       },
       {
-       "buf": "residual"
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "buf": "model.layers.36.input_layernorm.weight_p1"
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i32": 5120
+       "i64": 25690112
       },
       {
-       "sym": "tokens"
+       "i64": 2048
       },
       {
-       "i32": 5120
+       "i64": 512
       },
       {
-       "i32": 5120
+       "i32": 784
       },
       {
        "f32": 9.999999974752427e-07
@@ -129001,90 +77810,89 @@
      ]
     },
     {
-     "label": "l36.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l7.attn",
+     "kernel": "attn_prefill",
      "args": [
       {
-       "buf": "x"
+       "buf": "attn_out"
       },
       {
-       "buf": "model.layers.36.linear_attn.in_proj_qkvz.weight"
+       "buf": "q_n"
       },
       {
-       "buf": "qkvz"
+       "state": "kv",
+       "offset": 3211264
       },
       {
-       "sym": "tokens"
+       "state": "kv",
+       "offset": 3211776
       },
       {
-       "i32": 16384
+       "buf": "block_table"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l36.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
+       "buf": "seq_lens"
+      },
       {
-       "buf": "x"
+       "f32": 0.0625
       },
       {
-       "buf": "model.layers.36.linear_attn.in_proj_ba.weight"
+       "buf": "kv_scales"
       },
       {
-       "buf": "ba"
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "sym": "tokens"
+       "f32": 1.0
       },
       {
-       "i32": 96
+       "f32": 0.0
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l36.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
+       "i64": 8
+      },
       {
-       "buf": "qkvz"
+       "i64": 6144
       },
       {
-       "buf": "model.layers.36.linear_attn.conv1d.weight"
+       "i64": 256
       },
       {
-       "state": "gdn"
+       "i64": 6144
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 112
+       "i64": 256
       },
       {
-       "buf": "num_accepted_tokens"
+       "i64": 0
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "seq_lens"
       },
       {
-       "buf": "qkvz"
+       "i64": 25690112
       },
       {
-       "i32": 1
+       "i64": 2048
       },
       {
-       "i32": 385
+       "i64": 512
       },
       {
-       "i64": 16384
+       "i64": 25690112
       },
       {
-       "i64": 16384
+       "i64": 2048
+      },
+      {
+       "i64": 512
+      },
+      {
+       "buf": "cu_seqlens_q"
+      },
+      {
+       "i32": 1
       },
       {
        "i64": 0
@@ -129095,291 +77903,309 @@
      ]
     },
     {
-     "label": "l36.post_conv",
-     "kernel": "post_conv",
+     "label": "l7.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "attn_out"
       },
       {
-       "buf": "ba"
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "buf": "model.layers.36.linear_attn.A_log"
+       "buf": "model.layers.7.self_attn.o_proj.weight"
       },
       {
-       "buf": "model.layers.36.linear_attn.dt_bias"
+       "buf": "residual"
       },
       {
-       "buf": "gdn_q"
+       "buf": "model.layers.7.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "gdn_k"
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_v"
+       "i32": 5120
       },
       {
-       "buf": "g"
+       "i32": 6144
       },
       {
-       "buf": "beta"
+       "i32": 256
       },
       {
-       "i32": 16384
+       "i32": 14336
       },
       {
-       "i32": 96
+       "i32": 512
       },
       {
-       "i32": 96
-      },
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l7.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
+     "args": [
       {
-       "i32": 2048
+       "buf": "act"
       },
       {
-       "i32": 2048
+       "buf": "x"
       },
       {
-       "i32": 6144
+       "buf": "model.layers.7.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 17408
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l36.a_copy",
-     "kernel": "copy_rows",
+     "label": "l7.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "act"
       },
       {
-       "i32": 48
+       "buf": "model.layers.7.mlp.down_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.8.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
       },
       {
-       "i32": 48
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l36.b_copy",
-     "kernel": "copy_rows",
+     "label": "l8.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "b_c"
+       "buf": "qkvz"
       },
       {
        "buf": "ba"
       },
       {
-       "i32": 48
+       "buf": "x"
+      },
+      {
+       "buf": "model.layers.8.linear_attn.in_proj_qkvz.weight"
+      },
+      {
+       "buf": "model.layers.8.linear_attn.in_proj_ba.weight"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 16384
       },
       {
        "i32": 96
       },
       {
-       "i32": 48
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l36.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l8.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "model.layers.36.linear_attn.A_log"
+       "buf": "qkvz"
       },
       {
-       "buf": "a_c"
+       "buf": "model.layers.8.linear_attn.conv1d.weight"
       },
       {
-       "buf": "b_c"
+       "state": "gdn"
       },
       {
-       "buf": "model.layers.36.linear_attn.dt_bias"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 28
       },
       {
-       "f32": 20.0
+       "buf": "num_accepted_tokens"
       },
       {
-       "buf": "gdn_q"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "gdn_k"
+       "buf": "gdn.spec_slots",
+       "offset": 192
       },
       {
-       "buf": "gdn_v"
+       "buf": "ba"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.8.linear_attn.A_log"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "model.layers.8.linear_attn.dt_bias"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "core_attn_out"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "model.layers.8.linear_attn.norm.weight"
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 864
+       "buf": "gdn_q"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "gdn_k"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "gdn_v"
       },
       {
-       "i64": 1
+       "buf": "g"
       },
       {
-       "sym": "tokens"
+       "buf": "beta"
       },
       {
-       "i64": 0
+       "buf": "a_c"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l36.z_copy",
-     "kernel": "copy_rows",
-     "args": [
+       "buf": "b_c"
+      },
       {
        "buf": "z_c"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "f32": 0.0883883461356163
+      },
+      {
+       "f32": 9.999999974752427e-07
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 385
       },
       {
-       "i32": 6144
+       "i32": 1703936
       },
       {
-       "i32": 16384
+       "i32": 10
       },
       {
-       "i32": 6144
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l36.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l8.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.36.linear_attn.norm.weight"
+       "buf": "model.layers.8.linear_attn.out_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.8.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l36.out_proj",
-     "kernel": "gemm",
+     "label": "l8.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.36.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.8.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l36.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l8.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.8.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.36.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.9.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -129388,7 +78214,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -129396,23 +78222,32 @@
      ]
     },
     {
-     "label": "l36.gate_up",
-     "kernel": "gemm",
+     "label": "l9.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.36.mlp.gate_up_proj.weight"
+       "buf": "model.layers.9.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.9.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -129420,71 +78255,116 @@
      ]
     },
     {
-     "label": "l36.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l9.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.9.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 32
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l36.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "num_accepted_tokens"
+      },
       {
-       "buf": "act"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "model.layers.36.mlp.down_proj.weight"
+       "buf": "gdn.spec_slots",
+       "offset": 224
       },
       {
-       "buf": "y"
+       "buf": "ba"
+      },
+      {
+       "buf": "model.layers.9.linear_attn.A_log"
+      },
+      {
+       "buf": "model.layers.9.linear_attn.dt_bias"
+      },
+      {
+       "buf": "core_attn_out"
+      },
+      {
+       "buf": "model.layers.9.linear_attn.norm.weight"
+      },
+      {
+       "buf": "gdn_q"
+      },
+      {
+       "buf": "gdn_k"
+      },
+      {
+       "buf": "gdn_v"
+      },
+      {
+       "buf": "g"
+      },
+      {
+       "buf": "beta"
+      },
+      {
+       "buf": "a_c"
+      },
+      {
+       "buf": "b_c"
+      },
+      {
+       "buf": "z_c"
+      },
+      {
+       "f32": 0.0883883461356163
+      },
+      {
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 385
       },
       {
-       "i32": 17408
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l36.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l9.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.9.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.37.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.9.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -129493,7 +78373,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -129501,23 +78381,23 @@
      ]
     },
     {
-     "label": "l37.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l9.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.37.linear_attn.in_proj_qkvz.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.9.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -129525,94 +78405,116 @@
      ]
     },
     {
-     "label": "l37.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l9.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.37.linear_attn.in_proj_ba.weight"
+       "buf": "act"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.9.mlp.down_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 96
+       "buf": "model.layers.10.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l37.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l10.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.37.linear_attn.conv1d.weight"
+       "buf": "ba"
       },
       {
-       "state": "gdn"
+       "buf": "x"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 116
+       "buf": "model.layers.10.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "model.layers.10.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "cu_seqlens_q"
+       "sym": "tokens"
       },
       {
-       "buf": "qkvz"
+       "i32": 16384
       },
       {
-       "i32": 1
+       "i32": 96
       },
       {
-       "i32": 385
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l10.gdn",
+     "kernel": "gdn_spec",
+     "args": [
+      {
+       "buf": "qkvz"
       },
       {
-       "i64": 16384
+       "buf": "model.layers.10.linear_attn.conv1d.weight"
       },
       {
-       "i64": 16384
+       "state": "gdn"
       },
       {
-       "i64": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l37.post_conv",
-     "kernel": "post_conv",
-     "args": [
+       "buf": "gdn.spec_line_index",
+       "offset": 36
+      },
       {
-       "buf": "qkvz"
+       "buf": "num_accepted_tokens"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "cu_seqlens_q"
+      },
+      {
+       "buf": "gdn.spec_slots",
+       "offset": 256
       },
       {
        "buf": "ba"
       },
       {
-       "buf": "model.layers.37.linear_attn.A_log"
+       "buf": "model.layers.10.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.37.linear_attn.dt_bias"
+       "buf": "model.layers.10.linear_attn.dt_bias"
+      },
+      {
+       "buf": "core_attn_out"
+      },
+      {
+       "buf": "model.layers.10.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -129630,192 +78532,196 @@
        "buf": "beta"
       },
       {
-       "i32": 16384
+       "buf": "a_c"
       },
       {
-       "i32": 96
+       "buf": "b_c"
       },
       {
-       "i32": 96
+       "buf": "z_c"
       },
       {
-       "i32": 2048
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 2048
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 6144
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 385
       },
       {
-       "i64": 0
+       "i32": 1703936
       },
       {
-       "i64": 0
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l37.a_copy",
-     "kernel": "copy_rows",
+     "label": "l10.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "core_attn_out"
       },
       {
-       "i32": 48
+       "buf": "model.layers.10.linear_attn.out_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l37.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.10.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "ba"
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 5120
       },
       {
-       "i32": 96
+       "i32": 6144
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l37.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l10.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "model.layers.37.linear_attn.A_log"
+       "buf": "act"
       },
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "b_c"
+       "buf": "model.layers.10.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "model.layers.37.linear_attn.dt_bias"
+       "sym": "tokens"
       },
       {
-       "f32": 1.0
+       "i32": 17408
       },
       {
-       "f32": 20.0
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l10.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "buf": "gdn_q"
+       "buf": "x"
       },
       {
-       "buf": "gdn_k"
+       "buf": "act"
       },
       {
-       "buf": "gdn_v"
+       "buf": "model.layers.10.mlp.down_proj.weight"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "residual"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "model.layers.11.input_layernorm.weight_p1"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "sym": "tokens"
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 5120
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 896
+       "i32": 17408
       },
       {
-       "buf": "num_accepted_tokens"
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l11.qkv_proj",
+     "kernel": "gemm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "model.layers.11.self_attn.qkv_proj.weight"
       },
       {
-       "i64": 1
+       "buf": "qkv"
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l37.z_copy",
-     "kernel": "copy_rows",
+     "label": "l11.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "q_n"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "k_n"
       },
       {
-       "i32": 6144
+       "buf": "qkv"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.11.self_attn.q_norm.weight_p1"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l37.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
+       "buf": "model.layers.11.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "cos_g"
       },
       {
-       "buf": "model.layers.37.linear_attn.norm.weight"
+       "buf": "sin_g"
       },
       {
-       "buf": "z_c"
+       "state": "kv",
+       "offset": 6422528
       },
       {
-       "i32": 128
+       "state": "kv",
+       "offset": 6423040
       },
       {
-       "i32": 128
+       "buf": "slot_mapping"
       },
       {
-       "i32": 128
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
        "expr": {
@@ -129823,168 +78729,208 @@
          {
           "sym": "tokens"
          },
-         48
+         4
         ]
        }
       },
       {
-       "f32": 9.999999974752427e-07
+       "i64": 25690112
       },
       {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 0
+       "i64": 512
+      },
+      {
+       "i32": 784
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l37.out_proj",
-     "kernel": "gemm",
+     "label": "l11.attn",
+     "kernel": "attn_prefill",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "attn_out"
       },
       {
-       "buf": "model.layers.37.linear_attn.out_proj.weight"
+       "buf": "q_n"
       },
       {
-       "buf": "y"
+       "state": "kv",
+       "offset": 6422528
       },
       {
-       "sym": "tokens"
+       "state": "kv",
+       "offset": 6423040
       },
       {
-       "i32": 5120
+       "buf": "block_table"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l37.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "seq_lens"
+      },
       {
-       "buf": "x"
+       "f32": 0.0625
       },
       {
-       "buf": "y"
+       "buf": "kv_scales"
       },
       {
-       "buf": "residual"
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "buf": "model.layers.37.post_attention_layernorm.weight_p1"
+       "f32": 1.0
       },
       {
-       "i32": 5120
+       "f32": 0.0
       },
       {
-       "sym": "tokens"
+       "i64": 8
       },
       {
-       "i32": 5120
+       "i64": 6144
       },
       {
-       "i32": 5120
+       "i64": 256
       },
       {
-       "f32": 9.999999974752427e-07
+       "i64": 6144
+      },
+      {
+       "i64": 256
+      },
+      {
+       "i64": 0
+      },
+      {
+       "buf": "seq_lens"
+      },
+      {
+       "i64": 25690112
+      },
+      {
+       "i64": 2048
+      },
+      {
+       "i64": 512
+      },
+      {
+       "i64": 25690112
+      },
+      {
+       "i64": 2048
+      },
+      {
+       "i64": 512
+      },
+      {
+       "buf": "cu_seqlens_q"
+      },
+      {
+       "i32": 1
+      },
+      {
+       "i64": 0
+      },
+      {
+       "i64": 0
       }
      ]
     },
     {
-     "label": "l37.gate_up",
-     "kernel": "gemm",
+     "label": "l11.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.37.mlp.gate_up_proj.weight"
+       "buf": "attn_out"
       },
       {
-       "buf": "gate_up"
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.11.self_attn.o_proj.weight"
       },
       {
-       "i32": 34816
+       "buf": "residual"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l37.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
+       "buf": "model.layers.11.post_attention_layernorm.weight_p1"
+      },
       {
-       "buf": "act"
+       "sym": "tokens"
       },
       {
-       "buf": "gate_up"
+       "i32": 5120
       },
       {
-       "i32": 17408
+       "i32": 6144
       },
       {
-       "i32": 0
+       "i32": 256
       },
       {
-       "f32": 1.0
+       "i32": 14336
       },
       {
-       "i32": 0
+       "i32": 512
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l37.down_proj",
-     "kernel": "gemm",
+     "label": "l11.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
        "buf": "act"
       },
       {
-       "buf": "model.layers.37.mlp.down_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.11.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 17408
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l37.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l11.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.11.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.38.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.12.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -129993,7 +78939,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -130001,45 +78947,30 @@
      ]
     },
     {
-     "label": "l38.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l12.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.38.linear_attn.in_proj_qkvz.weight"
-      },
       {
        "buf": "qkvz"
       },
       {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
+       "buf": "ba"
       },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l38.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.38.linear_attn.in_proj_ba.weight"
+       "buf": "model.layers.12.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.12.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
+      {
+       "i32": 16384
+      },
       {
        "i32": 96
       },
@@ -130049,21 +78980,25 @@
      ]
     },
     {
-     "label": "l38.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l12.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.38.linear_attn.conv1d.weight"
+       "buf": "model.layers.12.linear_attn.conv1d.weight"
       },
       {
        "state": "gdn"
       },
+      {
+       "state": "gdn",
+       "offset": 204800
+      },
       {
        "buf": "gdn.spec_line_index",
-       "offset": 120
+       "offset": 40
       },
       {
        "buf": "num_accepted_tokens"
@@ -130072,314 +79007,362 @@
        "buf": "cu_seqlens_q"
       },
       {
-       "buf": "qkvz"
+       "buf": "gdn.spec_slots",
+       "offset": 288
       },
       {
-       "i32": 1
+       "buf": "ba"
       },
       {
-       "i32": 385
+       "buf": "model.layers.12.linear_attn.A_log"
       },
       {
-       "i64": 16384
+       "buf": "model.layers.12.linear_attn.dt_bias"
       },
       {
-       "i64": 16384
+       "buf": "core_attn_out"
       },
       {
-       "i64": 0
+       "buf": "model.layers.12.linear_attn.norm.weight"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l38.post_conv",
-     "kernel": "post_conv",
-     "args": [
+       "buf": "gdn_q"
+      },
       {
-       "buf": "qkvz"
+       "buf": "gdn_k"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "gdn_v"
       },
       {
-       "buf": "ba"
+       "buf": "g"
       },
       {
-       "buf": "model.layers.38.linear_attn.A_log"
+       "buf": "beta"
       },
       {
-       "buf": "model.layers.38.linear_attn.dt_bias"
+       "buf": "a_c"
       },
       {
-       "buf": "gdn_q"
+       "buf": "b_c"
       },
       {
-       "buf": "gdn_k"
+       "buf": "z_c"
       },
       {
-       "buf": "gdn_v"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "g"
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "beta"
+       "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 385
       },
       {
-       "i32": 96
+       "i32": 1703936
       },
       {
-       "i32": 96
+       "i32": 10
       },
       {
-       "i32": 2048
+       "i32": 851968
+      }
+     ]
+    },
+    {
+     "label": "l12.out_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "i32": 2048
+       "buf": "core_attn_out"
       },
       {
-       "i32": 6144
+       "buf": "model.layers.12.linear_attn.out_proj.weight"
+      },
+      {
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.12.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l38.a_copy",
-     "kernel": "copy_rows",
+     "label": "l12.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "act"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "x"
       },
       {
-       "i32": 48
+       "buf": "model.layers.12.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 96
+       "sym": "tokens"
+      },
+      {
+       "i32": 17408
       },
       {
-       "i32": 48
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l38.b_copy",
-     "kernel": "copy_rows",
+     "label": "l12.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "b_c"
+       "buf": "x"
       },
       {
-       "buf": "ba"
+       "buf": "act"
       },
       {
-       "i32": 48
+       "buf": "model.layers.12.mlp.down_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.13.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 17408
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l38.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l13.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "model.layers.38.linear_attn.A_log"
+       "buf": "qkvz"
       },
       {
-       "buf": "a_c"
+       "buf": "ba"
       },
       {
-       "buf": "b_c"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.38.linear_attn.dt_bias"
+       "buf": "model.layers.13.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "f32": 1.0
+       "buf": "model.layers.13.linear_attn.in_proj_ba.weight"
       },
       {
-       "f32": 20.0
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_q"
+       "i32": 16384
       },
       {
-       "buf": "gdn_k"
+       "i32": 96
       },
       {
-       "buf": "gdn_v"
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l13.gdn",
+     "kernel": "gdn_spec",
+     "args": [
+      {
+       "buf": "qkvz"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.13.linear_attn.conv1d.weight"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "state": "gdn"
       },
       {
        "state": "gdn",
        "offset": 204800
       },
+      {
+       "buf": "gdn.spec_line_index",
+       "offset": 44
+      },
+      {
+       "buf": "num_accepted_tokens"
+      },
       {
        "buf": "cu_seqlens_q"
       },
       {
        "buf": "gdn.spec_slots",
-       "offset": 928
+       "offset": 320
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "ba"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "model.layers.13.linear_attn.A_log"
       },
       {
-       "i64": 1
+       "buf": "model.layers.13.linear_attn.dt_bias"
       },
       {
-       "sym": "tokens"
+       "buf": "core_attn_out"
       },
       {
-       "i64": 0
+       "buf": "model.layers.13.linear_attn.norm.weight"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l38.z_copy",
-     "kernel": "copy_rows",
-     "args": [
+       "buf": "gdn_q"
+      },
+      {
+       "buf": "gdn_k"
+      },
+      {
+       "buf": "gdn_v"
+      },
+      {
+       "buf": "g"
+      },
+      {
+       "buf": "beta"
+      },
+      {
+       "buf": "a_c"
+      },
+      {
+       "buf": "b_c"
+      },
       {
        "buf": "z_c"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 6144
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 16384
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 385
+      },
+      {
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l38.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l13.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.38.linear_attn.norm.weight"
+       "buf": "model.layers.13.linear_attn.out_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.13.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l38.out_proj",
-     "kernel": "gemm",
+     "label": "l13.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.38.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.13.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l38.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l13.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.13.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.38.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.14.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -130388,7 +79371,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -130396,23 +79379,32 @@
      ]
     },
     {
-     "label": "l38.gate_up",
-     "kernel": "gemm",
+     "label": "l14.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.38.mlp.gate_up_proj.weight"
+       "buf": "model.layers.14.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.14.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -130420,197 +79412,182 @@
      ]
     },
     {
-     "label": "l38.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l14.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.14.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 48
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l38.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "num_accepted_tokens"
       },
       {
-       "buf": "model.layers.38.mlp.down_proj.weight"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "y"
+       "buf": "gdn.spec_slots",
+       "offset": 352
       },
       {
-       "sym": "tokens"
+       "buf": "ba"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.14.linear_attn.A_log"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l38.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "model.layers.14.linear_attn.dt_bias"
+      },
       {
-       "buf": "x"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.14.linear_attn.norm.weight"
       },
       {
-       "buf": "residual"
+       "buf": "gdn_q"
       },
       {
-       "buf": "model.layers.39.input_layernorm.weight_p1"
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
+       "buf": "gdn_v"
       },
       {
-       "sym": "tokens"
+       "buf": "g"
       },
       {
-       "i32": 5120
+       "buf": "beta"
       },
       {
-       "i32": 5120
+       "buf": "a_c"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l39.qkv_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "b_c"
+      },
       {
-       "buf": "x"
+       "buf": "z_c"
       },
       {
-       "buf": "model.layers.39.self_attn.qkv_proj.weight"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "qkv"
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 14336
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l39.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l14.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.39.self_attn.q_norm.weight_p1"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 256
+       "buf": "model.layers.14.linear_attn.out_proj.weight"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "buf": "residual"
       },
       {
-       "i32": 24
+       "buf": "model.layers.14.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 14336
+       "sym": "tokens"
       },
       {
-       "i32": 512
+       "i32": 5120
       },
       {
        "i32": 6144
       },
-      {
-       "i32": 256
-      },
       {
        "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l39.k_norm",
-     "kernel": "gemma_norm_khead",
+     "label": "l14.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "k_n"
+       "buf": "act"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
+       "buf": "x"
       },
       {
-       "buf": "model.layers.39.self_attn.k_norm.weight_p1"
+       "buf": "model.layers.14.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 256
+       "sym": "tokens"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
+       "i32": 17408
       },
       {
-       "i32": 4
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l14.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "i32": 14336
+       "buf": "act"
       },
       {
-       "i32": 256
+       "buf": "model.layers.14.mlp.down_proj.weight"
       },
       {
-       "i32": 1024
+       "buf": "residual"
       },
       {
-       "i32": 256
+       "buf": "model.layers.15.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -130618,92 +79595,104 @@
      ]
     },
     {
-     "label": "l39.rope",
-     "kernel": "mrope",
+     "label": "l15.qkv_proj",
+     "kernel": "gemm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
+       "buf": "x"
       },
       {
-       "buf": "cos_g"
+       "buf": "model.layers.15.self_attn.qkv_proj.weight"
       },
       {
-       "buf": "sin_g"
+       "buf": "qkv"
       },
       {
-       "i32": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l39.kv_write",
-     "kernel": "reshape_and_cache",
+     "label": "l15.prep",
+     "kernel": "attn_prep",
      "args": [
+      {
+       "buf": "q_n"
+      },
       {
        "buf": "k_n"
       },
       {
-       "buf": "qkv",
-       "offset": 26624
+       "buf": "qkv"
       },
       {
-       "state": "kv",
-       "offset": 28901376
+       "buf": "model.layers.15.self_attn.q_norm.weight_p1"
       },
       {
-       "state": "kv",
-       "offset": 28901888
+       "buf": "model.layers.15.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "slot_mapping"
+       "buf": "cos_g"
       },
       {
-       "buf": "kv_scales"
+       "buf": "sin_g"
       },
       {
-       "buf": "kv_scales",
-       "offset": 4
+       "state": "kv",
+       "offset": 9633792
       },
       {
-       "i64": 1024
+       "state": "kv",
+       "offset": 9634304
       },
       {
-       "i64": 14336
+       "buf": "slot_mapping"
       },
       {
-       "i64": 25690112
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 512
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i64": 0
+       "i64": 25690112
       },
       {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l39.attn",
+     "label": "l15.attn",
      "kernel": "attn_prefill",
      "args": [
       {
@@ -130714,11 +79703,11 @@
       },
       {
        "state": "kv",
-       "offset": 28901376
+       "offset": 9633792
       },
       {
        "state": "kv",
-       "offset": 28901888
+       "offset": 9634304
       },
       {
        "buf": "block_table"
@@ -130796,11 +79785,11 @@
      ]
     },
     {
-     "label": "l39.gate",
-     "kernel": "sigmoid_mul",
+     "label": "l15.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "gated"
+       "buf": "x"
       },
       {
        "buf": "attn_out"
@@ -130810,31 +79799,13 @@
        "offset": 512
       },
       {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l39.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
+       "buf": "model.layers.15.self_attn.o_proj.weight"
       },
       {
-       "buf": "model.layers.39.self_attn.o_proj.weight"
+       "buf": "residual"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.15.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -130844,36 +79815,15 @@
       },
       {
        "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l39.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.39.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
+       "i32": 256
       },
       {
-       "i32": 5120
+       "i32": 14336
       },
       {
-       "i32": 5120
+       "i32": 512
       },
       {
        "f32": 9.999999974752427e-07
@@ -130881,95 +79831,47 @@
      ]
     },
     {
-     "label": "l39.gate_up",
-     "kernel": "gemm",
+     "label": "l15.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.39.mlp.gate_up_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.15.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l39.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
       {
        "i32": 17408
       },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l39.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.39.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
       {
        "i32": 5120
-      },
-      {
-       "i32": 17408
       }
      ]
     },
     {
-     "label": "l39.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l15.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.15.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.40.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.16.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -130978,7 +79880,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -130986,45 +79888,30 @@
      ]
     },
     {
-     "label": "l40.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l16.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.40.linear_attn.in_proj_qkvz.weight"
-      },
       {
        "buf": "qkvz"
       },
       {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
+       "buf": "ba"
       },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l40.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.40.linear_attn.in_proj_ba.weight"
+       "buf": "model.layers.16.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.16.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
+      {
+       "i32": 16384
+      },
       {
        "i32": 96
       },
@@ -131034,21 +79921,25 @@
      ]
     },
     {
-     "label": "l40.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l16.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.40.linear_attn.conv1d.weight"
+       "buf": "model.layers.16.linear_attn.conv1d.weight"
       },
       {
        "state": "gdn"
       },
+      {
+       "state": "gdn",
+       "offset": 204800
+      },
       {
        "buf": "gdn.spec_line_index",
-       "offset": 124
+       "offset": 52
       },
       {
        "buf": "num_accepted_tokens"
@@ -131057,47 +79948,23 @@
        "buf": "cu_seqlens_q"
       },
       {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
+       "buf": "gdn.spec_slots",
+       "offset": 384
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l40.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
+       "buf": "ba"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.16.linear_attn.A_log"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.16.linear_attn.dt_bias"
       },
       {
-       "buf": "model.layers.40.linear_attn.A_log"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.40.linear_attn.dt_bias"
+       "buf": "model.layers.16.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -131115,331 +79982,271 @@
        "buf": "beta"
       },
       {
-       "i32": 16384
+       "buf": "a_c"
       },
       {
-       "i32": 96
+       "buf": "b_c"
       },
       {
-       "i32": 96
+       "buf": "z_c"
       },
       {
-       "i32": 2048
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 2048
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 6144
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 385
       },
       {
-       "i64": 0
+       "i32": 1703936
       },
       {
-       "i64": 0
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l40.a_copy",
-     "kernel": "copy_rows",
+     "label": "l16.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "core_attn_out"
       },
       {
-       "i32": 48
+       "buf": "model.layers.16.linear_attn.out_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l40.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.16.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "ba"
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 5120
       },
       {
-       "i32": 96
+       "i32": 6144
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l40.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l16.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "model.layers.40.linear_attn.A_log"
+       "buf": "act"
       },
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "b_c"
+       "buf": "model.layers.16.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "model.layers.40.linear_attn.dt_bias"
+       "sym": "tokens"
       },
       {
-       "f32": 1.0
+       "i32": 17408
       },
       {
-       "f32": 20.0
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l16.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "buf": "gdn_q"
+       "buf": "x"
       },
       {
-       "buf": "gdn_k"
+       "buf": "act"
       },
       {
-       "buf": "gdn_v"
+       "buf": "model.layers.16.mlp.down_proj.weight"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "residual"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "model.layers.17.input_layernorm.weight_p1"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "sym": "tokens"
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 5120
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 960
+       "i32": 17408
       },
       {
-       "buf": "num_accepted_tokens"
-      },
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l17.in_proj",
+     "kernel": "gemm8_dual",
+     "args": [
       {
-       "f32": 0.0883883461356163
+       "buf": "qkvz"
       },
       {
-       "i64": 1
+       "buf": "ba"
       },
       {
-       "sym": "tokens"
+       "buf": "x"
       },
       {
-       "i64": 0
+       "buf": "model.layers.17.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l40.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
+       "buf": "model.layers.17.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 16384
       },
       {
-       "i32": 16384
+       "i32": 96
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l40.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l17.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
+       "buf": "qkvz"
       },
       {
-       "buf": "model.layers.40.linear_attn.norm.weight"
+       "buf": "model.layers.17.linear_attn.conv1d.weight"
       },
       {
-       "buf": "z_c"
+       "state": "gdn"
       },
       {
-       "i32": 128
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 128
+       "buf": "gdn.spec_line_index",
+       "offset": 56
       },
       {
-       "i32": 128
+       "buf": "num_accepted_tokens"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "buf": "cu_seqlens_q"
       },
       {
-       "f32": 9.999999974752427e-07
+       "buf": "gdn.spec_slots",
+       "offset": 416
       },
       {
-       "i64": 0
+       "buf": "ba"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l40.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
+       "buf": "model.layers.17.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.40.linear_attn.out_proj.weight"
+       "buf": "model.layers.17.linear_attn.dt_bias"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.17.linear_attn.norm.weight"
       },
       {
-       "i32": 5120
+       "buf": "gdn_q"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l40.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
+       "buf": "gdn_k"
       },
       {
-       "buf": "y"
+       "buf": "gdn_v"
       },
       {
-       "buf": "residual"
+       "buf": "g"
       },
       {
-       "buf": "model.layers.40.post_attention_layernorm.weight_p1"
+       "buf": "beta"
       },
       {
-       "i32": 5120
+       "buf": "a_c"
       },
       {
-       "sym": "tokens"
+       "buf": "b_c"
       },
       {
-       "i32": 5120
+       "buf": "z_c"
       },
       {
-       "i32": 5120
+       "f32": 0.0883883461356163
       },
       {
        "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l40.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
       },
       {
-       "buf": "model.layers.40.mlp.gate_up_proj.weight"
+       "sym": "tokens"
       },
       {
-       "buf": "gate_up"
+       "i32": 385
       },
       {
-       "sym": "tokens"
+       "i32": 1703936
       },
       {
-       "i32": 34816
+       "i32": 10
       },
       {
-       "i32": 5120
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l40.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l17.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
+       "buf": "x"
       },
       {
-       "f32": 1.0
+       "buf": "core_attn_out"
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l40.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "model.layers.17.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.40.mlp.down_proj.weight"
+       "buf": "residual"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.17.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -131448,83 +80255,95 @@
        "i32": 5120
       },
       {
-       "i32": 17408
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l40.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l17.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.41.input_layernorm.weight_p1"
+       "buf": "x"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.17.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l41.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l17.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.41.linear_attn.in_proj_qkvz.weight"
+       "buf": "act"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.17.mlp.down_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.18.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l41.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l18.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.41.linear_attn.in_proj_ba.weight"
+       "buf": "model.layers.18.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.18.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
+      {
+       "i32": 16384
+      },
       {
        "i32": 96
       },
@@ -131534,21 +80353,25 @@
      ]
     },
     {
-     "label": "l41.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l18.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.41.linear_attn.conv1d.weight"
+       "buf": "model.layers.18.linear_attn.conv1d.weight"
       },
       {
        "state": "gdn"
       },
+      {
+       "state": "gdn",
+       "offset": 204800
+      },
       {
        "buf": "gdn.spec_line_index",
-       "offset": 128
+       "offset": 60
       },
       {
        "buf": "num_accepted_tokens"
@@ -131557,47 +80380,23 @@
        "buf": "cu_seqlens_q"
       },
       {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
+       "buf": "gdn.spec_slots",
+       "offset": 448
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l41.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
+       "buf": "ba"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.18.linear_attn.A_log"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.18.linear_attn.dt_bias"
       },
       {
-       "buf": "model.layers.41.linear_attn.A_log"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.41.linear_attn.dt_bias"
+       "buf": "model.layers.18.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -131615,192 +80414,196 @@
        "buf": "beta"
       },
       {
-       "i32": 16384
+       "buf": "a_c"
       },
       {
-       "i32": 96
+       "buf": "b_c"
       },
       {
-       "i32": 96
+       "buf": "z_c"
       },
       {
-       "i32": 2048
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 2048
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 6144
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 385
       },
       {
-       "i64": 0
+       "i32": 1703936
       },
       {
-       "i64": 0
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l41.a_copy",
-     "kernel": "copy_rows",
+     "label": "l18.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "core_attn_out"
       },
       {
-       "i32": 48
+       "buf": "model.layers.18.linear_attn.out_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l41.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.18.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "ba"
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 5120
       },
       {
-       "i32": 96
+       "i32": 6144
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l41.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l18.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "model.layers.41.linear_attn.A_log"
+       "buf": "act"
       },
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "b_c"
+       "buf": "model.layers.18.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "model.layers.41.linear_attn.dt_bias"
+       "sym": "tokens"
       },
       {
-       "f32": 1.0
+       "i32": 17408
       },
       {
-       "f32": 20.0
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l18.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "buf": "gdn_q"
+       "buf": "x"
       },
       {
-       "buf": "gdn_k"
+       "buf": "act"
       },
       {
-       "buf": "gdn_v"
+       "buf": "model.layers.18.mlp.down_proj.weight"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "residual"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "model.layers.19.input_layernorm.weight_p1"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "sym": "tokens"
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 5120
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 992
+       "i32": 17408
       },
       {
-       "buf": "num_accepted_tokens"
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l19.qkv_proj",
+     "kernel": "gemm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "model.layers.19.self_attn.qkv_proj.weight"
       },
       {
-       "i64": 1
+       "buf": "qkv"
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l41.z_copy",
-     "kernel": "copy_rows",
+     "label": "l19.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "q_n"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "k_n"
       },
       {
-       "i32": 6144
+       "buf": "qkv"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.19.self_attn.q_norm.weight_p1"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l41.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
+       "buf": "model.layers.19.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "cos_g"
       },
       {
-       "buf": "model.layers.41.linear_attn.norm.weight"
+       "buf": "sin_g"
       },
       {
-       "buf": "z_c"
+       "state": "kv",
+       "offset": 12845056
       },
       {
-       "i32": 128
+       "state": "kv",
+       "offset": 12845568
       },
       {
-       "i32": 128
+       "buf": "slot_mapping"
       },
       {
-       "i32": 128
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
        "expr": {
@@ -131808,168 +80611,208 @@
          {
           "sym": "tokens"
          },
-         48
+         4
         ]
        }
       },
       {
-       "f32": 9.999999974752427e-07
+       "i64": 25690112
       },
       {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 0
+       "i64": 512
+      },
+      {
+       "i32": 784
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l41.out_proj",
-     "kernel": "gemm",
+     "label": "l19.attn",
+     "kernel": "attn_prefill",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "attn_out"
       },
       {
-       "buf": "model.layers.41.linear_attn.out_proj.weight"
+       "buf": "q_n"
       },
       {
-       "buf": "y"
+       "state": "kv",
+       "offset": 12845056
       },
       {
-       "sym": "tokens"
+       "state": "kv",
+       "offset": 12845568
       },
       {
-       "i32": 5120
+       "buf": "block_table"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l41.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "seq_lens"
+      },
       {
-       "buf": "x"
+       "f32": 0.0625
       },
       {
-       "buf": "y"
+       "buf": "kv_scales"
       },
       {
-       "buf": "residual"
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "buf": "model.layers.41.post_attention_layernorm.weight_p1"
+       "f32": 1.0
       },
       {
-       "i32": 5120
+       "f32": 0.0
       },
       {
-       "sym": "tokens"
+       "i64": 8
       },
       {
-       "i32": 5120
+       "i64": 6144
       },
       {
-       "i32": 5120
+       "i64": 256
       },
       {
-       "f32": 9.999999974752427e-07
+       "i64": 6144
+      },
+      {
+       "i64": 256
+      },
+      {
+       "i64": 0
+      },
+      {
+       "buf": "seq_lens"
+      },
+      {
+       "i64": 25690112
+      },
+      {
+       "i64": 2048
+      },
+      {
+       "i64": 512
+      },
+      {
+       "i64": 25690112
+      },
+      {
+       "i64": 2048
+      },
+      {
+       "i64": 512
+      },
+      {
+       "buf": "cu_seqlens_q"
+      },
+      {
+       "i32": 1
+      },
+      {
+       "i64": 0
+      },
+      {
+       "i64": 0
       }
      ]
     },
     {
-     "label": "l41.gate_up",
-     "kernel": "gemm",
+     "label": "l19.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.41.mlp.gate_up_proj.weight"
+       "buf": "attn_out"
       },
       {
-       "buf": "gate_up"
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.19.self_attn.o_proj.weight"
       },
       {
-       "i32": 34816
+       "buf": "residual"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l41.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
+       "buf": "model.layers.19.post_attention_layernorm.weight_p1"
+      },
       {
-       "buf": "act"
+       "sym": "tokens"
       },
       {
-       "buf": "gate_up"
+       "i32": 5120
       },
       {
-       "i32": 17408
+       "i32": 6144
       },
       {
-       "i32": 0
+       "i32": 256
       },
       {
-       "f32": 1.0
+       "i32": 14336
       },
       {
-       "i32": 0
+       "i32": 512
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l41.down_proj",
-     "kernel": "gemm",
+     "label": "l19.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
        "buf": "act"
       },
       {
-       "buf": "model.layers.41.mlp.down_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.19.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 17408
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l41.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l19.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.19.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.42.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.20.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -131978,7 +80821,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -131986,23 +80829,23 @@
      ]
     },
     {
-     "label": "l42.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l19.fc_tap",
+     "kernel": "gemm_acc",
      "args": [
       {
-       "buf": "x"
+       "buf": "residual"
       },
       {
-       "buf": "model.layers.42.linear_attn.in_proj_qkvz.weight"
+       "buf": "draft.fc.1.weight"
       },
       {
-       "buf": "qkvz"
+       "buf": "fc_out"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 5120
       },
       {
        "i32": 5120
@@ -132010,21 +80853,30 @@
      ]
     },
     {
-     "label": "l42.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l20.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.42.linear_attn.in_proj_ba.weight"
+       "buf": "model.layers.20.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.20.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
+      {
+       "i32": 16384
+      },
       {
        "i32": 96
       },
@@ -132034,21 +80886,25 @@
      ]
     },
     {
-     "label": "l42.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l20.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.42.linear_attn.conv1d.weight"
+       "buf": "model.layers.20.linear_attn.conv1d.weight"
       },
       {
        "state": "gdn"
       },
+      {
+       "state": "gdn",
+       "offset": 204800
+      },
       {
        "buf": "gdn.spec_line_index",
-       "offset": 132
+       "offset": 64
       },
       {
        "buf": "num_accepted_tokens"
@@ -132057,314 +80913,362 @@
        "buf": "cu_seqlens_q"
       },
       {
-       "buf": "qkvz"
+       "buf": "gdn.spec_slots",
+       "offset": 480
       },
       {
-       "i32": 1
+       "buf": "ba"
       },
       {
-       "i32": 385
+       "buf": "model.layers.20.linear_attn.A_log"
       },
       {
-       "i64": 16384
+       "buf": "model.layers.20.linear_attn.dt_bias"
       },
       {
-       "i64": 16384
+       "buf": "core_attn_out"
       },
       {
-       "i64": 0
+       "buf": "model.layers.20.linear_attn.norm.weight"
       },
       {
-       "i64": 0
+       "buf": "gdn_q"
+      },
+      {
+       "buf": "gdn_k"
+      },
+      {
+       "buf": "gdn_v"
+      },
+      {
+       "buf": "g"
+      },
+      {
+       "buf": "beta"
+      },
+      {
+       "buf": "a_c"
+      },
+      {
+       "buf": "b_c"
+      },
+      {
+       "buf": "z_c"
+      },
+      {
+       "f32": 0.0883883461356163
+      },
+      {
+       "f32": 9.999999974752427e-07
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 385
+      },
+      {
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l42.post_conv",
-     "kernel": "post_conv",
+     "label": "l20.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "core_attn_out"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.20.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.42.linear_attn.A_log"
+       "buf": "residual"
       },
       {
-       "buf": "model.layers.42.linear_attn.dt_bias"
+       "buf": "model.layers.20.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "gdn_q"
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_k"
+       "i32": 5120
       },
       {
-       "buf": "gdn_v"
+       "i32": 6144
       },
       {
-       "buf": "g"
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l20.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
+     "args": [
+      {
+       "buf": "act"
       },
       {
-       "buf": "beta"
+       "buf": "x"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.20.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 96
+       "sym": "tokens"
       },
       {
-       "i32": 96
+       "i32": 17408
+      },
+      {
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l20.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
+      },
+      {
+       "buf": "act"
       },
       {
-       "i32": 2048
+       "buf": "model.layers.20.mlp.down_proj.weight"
       },
       {
-       "i32": 2048
+       "buf": "residual"
       },
       {
-       "i32": 6144
+       "buf": "model.layers.21.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l42.a_copy",
-     "kernel": "copy_rows",
+     "label": "l21.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "qkvz"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "ba"
       },
       {
-       "i32": 48
+       "buf": "x"
       },
       {
-       "i32": 96
+       "buf": "model.layers.21.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l42.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.21.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "ba"
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 16384
       },
       {
        "i32": 96
       },
       {
-       "i32": 48
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l42.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l21.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "model.layers.42.linear_attn.A_log"
+       "buf": "qkvz"
       },
       {
-       "buf": "a_c"
+       "buf": "model.layers.21.linear_attn.conv1d.weight"
       },
       {
-       "buf": "b_c"
+       "state": "gdn"
       },
       {
-       "buf": "model.layers.42.linear_attn.dt_bias"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 68
       },
       {
-       "f32": 20.0
+       "buf": "num_accepted_tokens"
       },
       {
-       "buf": "gdn_q"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "gdn_k"
+       "buf": "gdn.spec_slots",
+       "offset": 512
       },
       {
-       "buf": "gdn_v"
+       "buf": "ba"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.21.linear_attn.A_log"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "model.layers.21.linear_attn.dt_bias"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "core_attn_out"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "model.layers.21.linear_attn.norm.weight"
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 1024
+       "buf": "gdn_q"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "gdn_k"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "gdn_v"
       },
       {
-       "i64": 1
+       "buf": "g"
       },
       {
-       "sym": "tokens"
+       "buf": "beta"
       },
       {
-       "i64": 0
+       "buf": "a_c"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l42.z_copy",
-     "kernel": "copy_rows",
-     "args": [
+       "buf": "b_c"
+      },
       {
        "buf": "z_c"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 6144
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 16384
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 385
+      },
+      {
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l42.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l21.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.42.linear_attn.norm.weight"
+       "buf": "model.layers.21.linear_attn.out_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.21.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l42.out_proj",
-     "kernel": "gemm",
+     "label": "l21.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.42.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.21.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l42.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l21.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.21.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.42.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.22.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -132373,7 +81277,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -132381,23 +81285,32 @@
      ]
     },
     {
-     "label": "l42.gate_up",
-     "kernel": "gemm",
+     "label": "l22.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.42.mlp.gate_up_proj.weight"
+       "buf": "model.layers.22.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.22.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -132405,197 +81318,182 @@
      ]
     },
     {
-     "label": "l42.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l22.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.22.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 72
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l42.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "num_accepted_tokens"
       },
       {
-       "buf": "model.layers.42.mlp.down_proj.weight"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "y"
+       "buf": "gdn.spec_slots",
+       "offset": 544
       },
       {
-       "sym": "tokens"
+       "buf": "ba"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.22.linear_attn.A_log"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l42.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "model.layers.22.linear_attn.dt_bias"
+      },
       {
-       "buf": "x"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.22.linear_attn.norm.weight"
       },
       {
-       "buf": "residual"
+       "buf": "gdn_q"
       },
       {
-       "buf": "model.layers.43.input_layernorm.weight_p1"
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
+       "buf": "gdn_v"
       },
       {
-       "sym": "tokens"
+       "buf": "g"
       },
       {
-       "i32": 5120
+       "buf": "beta"
       },
       {
-       "i32": 5120
+       "buf": "a_c"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l43.qkv_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "b_c"
+      },
       {
-       "buf": "x"
+       "buf": "z_c"
       },
       {
-       "buf": "model.layers.43.self_attn.qkv_proj.weight"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "qkv"
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 14336
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l43.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l22.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.43.self_attn.q_norm.weight_p1"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 256
+       "buf": "model.layers.22.linear_attn.out_proj.weight"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "buf": "residual"
       },
       {
-       "i32": 24
+       "buf": "model.layers.22.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 14336
+       "sym": "tokens"
       },
       {
-       "i32": 512
+       "i32": 5120
       },
       {
        "i32": 6144
       },
-      {
-       "i32": 256
-      },
       {
        "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l43.k_norm",
-     "kernel": "gemma_norm_khead",
+     "label": "l22.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "k_n"
+       "buf": "act"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
+       "buf": "x"
       },
       {
-       "buf": "model.layers.43.self_attn.k_norm.weight_p1"
+       "buf": "model.layers.22.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 256
+       "sym": "tokens"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
+       "i32": 17408
+      },
+      {
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l22.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "i32": 4
+       "buf": "act"
       },
       {
-       "i32": 14336
+       "buf": "model.layers.22.mlp.down_proj.weight"
       },
       {
-       "i32": 256
+       "buf": "residual"
       },
       {
-       "i32": 1024
+       "buf": "model.layers.23.input_layernorm.weight_p1"
       },
       {
-       "i32": 256
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -132603,92 +81501,104 @@
      ]
     },
     {
-     "label": "l43.rope",
-     "kernel": "mrope",
+     "label": "l23.qkv_proj",
+     "kernel": "gemm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
+       "buf": "x"
       },
       {
-       "buf": "cos_g"
+       "buf": "model.layers.23.self_attn.qkv_proj.weight"
       },
       {
-       "buf": "sin_g"
+       "buf": "qkv"
       },
       {
-       "i32": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l43.kv_write",
-     "kernel": "reshape_and_cache",
+     "label": "l23.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "k_n"
+       "buf": "q_n"
       },
       {
-       "buf": "qkv",
-       "offset": 26624
+       "buf": "k_n"
       },
       {
-       "state": "kv",
-       "offset": 32112640
+       "buf": "qkv"
       },
       {
-       "state": "kv",
-       "offset": 32113152
+       "buf": "model.layers.23.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "slot_mapping"
+       "buf": "model.layers.23.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "kv_scales"
+       "buf": "cos_g"
       },
       {
-       "buf": "kv_scales",
-       "offset": 4
+       "buf": "sin_g"
       },
       {
-       "i64": 1024
+       "state": "kv",
+       "offset": 16056320
       },
       {
-       "i64": 14336
+       "state": "kv",
+       "offset": 16056832
       },
       {
-       "i64": 25690112
+       "buf": "slot_mapping"
       },
       {
-       "i64": 512
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 0
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i64": 0
+       "i64": 25690112
       },
       {
        "i64": 2048
       },
       {
-       "i64": 0
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l43.attn",
+     "label": "l23.attn",
      "kernel": "attn_prefill",
      "args": [
       {
@@ -132699,11 +81609,11 @@
       },
       {
        "state": "kv",
-       "offset": 32112640
+       "offset": 16056320
       },
       {
        "state": "kv",
-       "offset": 32113152
+       "offset": 16056832
       },
       {
        "buf": "block_table"
@@ -132781,11 +81691,11 @@
      ]
     },
     {
-     "label": "l43.gate",
-     "kernel": "sigmoid_mul",
+     "label": "l23.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "gated"
+       "buf": "x"
       },
       {
        "buf": "attn_out"
@@ -132795,31 +81705,13 @@
        "offset": 512
       },
       {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l43.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
+       "buf": "model.layers.23.self_attn.o_proj.weight"
       },
       {
-       "buf": "model.layers.43.self_attn.o_proj.weight"
+       "buf": "residual"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.23.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -132829,36 +81721,15 @@
       },
       {
        "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l43.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.43.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
       },
       {
-       "sym": "tokens"
+       "i32": 256
       },
       {
-       "i32": 5120
+       "i32": 14336
       },
       {
-       "i32": 5120
+       "i32": 512
       },
       {
        "f32": 9.999999974752427e-07
@@ -132866,23 +81737,23 @@
      ]
     },
     {
-     "label": "l43.gate_up",
-     "kernel": "gemm",
+     "label": "l23.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.43.mlp.gate_up_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.23.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -132890,71 +81761,23 @@
      ]
     },
     {
-     "label": "l43.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l23.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
+       "buf": "x"
       },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l43.down_proj",
-     "kernel": "gemm",
-     "args": [
       {
        "buf": "act"
       },
       {
-       "buf": "model.layers.43.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l43.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
+       "buf": "model.layers.23.mlp.down_proj.weight"
       },
       {
        "buf": "residual"
       },
       {
-       "buf": "model.layers.44.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
+       "buf": "model.layers.24.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -132963,7 +81786,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -132971,45 +81794,30 @@
      ]
     },
     {
-     "label": "l44.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l24.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.44.linear_attn.in_proj_qkvz.weight"
-      },
       {
        "buf": "qkvz"
       },
       {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
+       "buf": "ba"
       },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l44.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.44.linear_attn.in_proj_ba.weight"
+       "buf": "model.layers.24.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.24.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
+      {
+       "i32": 16384
+      },
       {
        "i32": 96
       },
@@ -133019,21 +81827,25 @@
      ]
     },
     {
-     "label": "l44.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l24.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.44.linear_attn.conv1d.weight"
+       "buf": "model.layers.24.linear_attn.conv1d.weight"
       },
       {
        "state": "gdn"
       },
+      {
+       "state": "gdn",
+       "offset": 204800
+      },
       {
        "buf": "gdn.spec_line_index",
-       "offset": 136
+       "offset": 76
       },
       {
        "buf": "num_accepted_tokens"
@@ -133042,314 +81854,362 @@
        "buf": "cu_seqlens_q"
       },
       {
-       "buf": "qkvz"
+       "buf": "gdn.spec_slots",
+       "offset": 576
       },
       {
-       "i32": 1
+       "buf": "ba"
       },
       {
-       "i32": 385
+       "buf": "model.layers.24.linear_attn.A_log"
       },
       {
-       "i64": 16384
+       "buf": "model.layers.24.linear_attn.dt_bias"
       },
       {
-       "i64": 16384
+       "buf": "core_attn_out"
       },
       {
-       "i64": 0
+       "buf": "model.layers.24.linear_attn.norm.weight"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l44.post_conv",
-     "kernel": "post_conv",
-     "args": [
+       "buf": "gdn_q"
+      },
       {
-       "buf": "qkvz"
+       "buf": "gdn_k"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "gdn_v"
       },
       {
-       "buf": "ba"
+       "buf": "g"
       },
       {
-       "buf": "model.layers.44.linear_attn.A_log"
+       "buf": "beta"
       },
       {
-       "buf": "model.layers.44.linear_attn.dt_bias"
+       "buf": "a_c"
       },
       {
-       "buf": "gdn_q"
+       "buf": "b_c"
       },
       {
-       "buf": "gdn_k"
+       "buf": "z_c"
       },
       {
-       "buf": "gdn_v"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "g"
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "beta"
+       "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 385
       },
       {
-       "i32": 96
+       "i32": 1703936
       },
       {
-       "i32": 96
+       "i32": 10
       },
       {
-       "i32": 2048
+       "i32": 851968
+      }
+     ]
+    },
+    {
+     "label": "l24.out_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "i32": 2048
+       "buf": "core_attn_out"
       },
       {
-       "i32": 6144
+       "buf": "model.layers.24.linear_attn.out_proj.weight"
+      },
+      {
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.24.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l44.a_copy",
-     "kernel": "copy_rows",
+     "label": "l24.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "act"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "x"
+      },
+      {
+       "buf": "model.layers.24.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 48
+       "sym": "tokens"
       },
       {
-       "i32": 96
+       "i32": 17408
       },
       {
-       "i32": 48
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l44.b_copy",
-     "kernel": "copy_rows",
+     "label": "l24.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "b_c"
+       "buf": "x"
       },
       {
-       "buf": "ba"
+       "buf": "act"
+      },
+      {
+       "buf": "model.layers.24.mlp.down_proj.weight"
       },
       {
-       "i32": 48
+       "buf": "residual"
       },
       {
-       "i32": 96
+       "buf": "model.layers.25.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l44.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l25.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "model.layers.44.linear_attn.A_log"
+       "buf": "qkvz"
       },
       {
-       "buf": "a_c"
+       "buf": "ba"
       },
       {
-       "buf": "b_c"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.44.linear_attn.dt_bias"
+       "buf": "model.layers.25.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "f32": 1.0
+       "buf": "model.layers.25.linear_attn.in_proj_ba.weight"
       },
       {
-       "f32": 20.0
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_q"
+       "i32": 16384
       },
       {
-       "buf": "gdn_k"
+       "i32": 96
       },
       {
-       "buf": "gdn_v"
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l25.gdn",
+     "kernel": "gdn_spec",
+     "args": [
+      {
+       "buf": "qkvz"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.25.linear_attn.conv1d.weight"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "state": "gdn"
       },
       {
        "state": "gdn",
        "offset": 204800
       },
+      {
+       "buf": "gdn.spec_line_index",
+       "offset": 80
+      },
+      {
+       "buf": "num_accepted_tokens"
+      },
       {
        "buf": "cu_seqlens_q"
       },
       {
        "buf": "gdn.spec_slots",
-       "offset": 1056
+       "offset": 608
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "ba"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "model.layers.25.linear_attn.A_log"
       },
       {
-       "i64": 1
+       "buf": "model.layers.25.linear_attn.dt_bias"
       },
       {
-       "sym": "tokens"
+       "buf": "core_attn_out"
       },
       {
-       "i64": 0
+       "buf": "model.layers.25.linear_attn.norm.weight"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l44.z_copy",
-     "kernel": "copy_rows",
-     "args": [
+       "buf": "gdn_q"
+      },
+      {
+       "buf": "gdn_k"
+      },
+      {
+       "buf": "gdn_v"
+      },
+      {
+       "buf": "g"
+      },
+      {
+       "buf": "beta"
+      },
+      {
+       "buf": "a_c"
+      },
+      {
+       "buf": "b_c"
+      },
       {
        "buf": "z_c"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 6144
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 16384
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 385
+      },
+      {
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l44.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l25.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.44.linear_attn.norm.weight"
+       "buf": "model.layers.25.linear_attn.out_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.25.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l44.out_proj",
-     "kernel": "gemm",
+     "label": "l25.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.44.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.25.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l44.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l25.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.25.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.44.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.26.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -133358,7 +82218,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -133366,23 +82226,32 @@
      ]
     },
     {
-     "label": "l44.gate_up",
-     "kernel": "gemm",
+     "label": "l26.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.44.mlp.gate_up_proj.weight"
+       "buf": "model.layers.26.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.26.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -133390,128 +82259,149 @@
      ]
     },
     {
-     "label": "l44.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l26.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.26.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 84
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l44.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "num_accepted_tokens"
+      },
       {
-       "buf": "act"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "model.layers.44.mlp.down_proj.weight"
+       "buf": "gdn.spec_slots",
+       "offset": 640
       },
       {
-       "buf": "y"
+       "buf": "ba"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.26.linear_attn.A_log"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.26.linear_attn.dt_bias"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l44.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "core_attn_out"
+      },
       {
-       "buf": "x"
+       "buf": "model.layers.26.linear_attn.norm.weight"
       },
       {
-       "buf": "y"
+       "buf": "gdn_q"
       },
       {
-       "buf": "residual"
+       "buf": "gdn_k"
       },
       {
-       "buf": "model.layers.45.input_layernorm.weight_p1"
+       "buf": "gdn_v"
       },
       {
-       "i32": 5120
+       "buf": "g"
       },
       {
-       "sym": "tokens"
+       "buf": "beta"
       },
       {
-       "i32": 5120
+       "buf": "a_c"
       },
       {
-       "i32": 5120
+       "buf": "b_c"
+      },
+      {
+       "buf": "z_c"
+      },
+      {
+       "f32": 0.0883883461356163
       },
       {
        "f32": 9.999999974752427e-07
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 385
+      },
+      {
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l45.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l26.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.45.linear_attn.in_proj_qkvz.weight"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.26.linear_attn.out_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.26.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l45.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l26.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.45.linear_attn.in_proj_ba.weight"
+       "buf": "x"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.26.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 96
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -133519,218 +82409,219 @@
      ]
     },
     {
-     "label": "l45.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l26.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.45.linear_attn.conv1d.weight"
+       "buf": "act"
       },
       {
-       "state": "gdn"
+       "buf": "model.layers.26.mlp.down_proj.weight"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 140
+       "buf": "residual"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "model.layers.27.input_layernorm.weight_p1"
       },
       {
-       "buf": "cu_seqlens_q"
+       "sym": "tokens"
       },
       {
-       "buf": "qkvz"
+       "i32": 5120
       },
       {
-       "i32": 1
+       "i32": 17408
       },
       {
-       "i32": 385
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l27.qkv_proj",
+     "kernel": "gemm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "i64": 16384
+       "buf": "model.layers.27.self_attn.qkv_proj.weight"
       },
       {
-       "i64": 16384
+       "buf": "qkv"
       },
       {
-       "i64": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l45.post_conv",
-     "kernel": "post_conv",
+     "label": "l27.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
+       "buf": "q_n"
       },
       {
-       "buf": "model.layers.45.linear_attn.A_log"
+       "buf": "k_n"
       },
       {
-       "buf": "model.layers.45.linear_attn.dt_bias"
+       "buf": "qkv"
       },
       {
-       "buf": "gdn_q"
+       "buf": "model.layers.27.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "gdn_k"
+       "buf": "model.layers.27.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "gdn_v"
+       "buf": "cos_g"
       },
       {
-       "buf": "g"
+       "buf": "sin_g"
       },
       {
-       "buf": "beta"
+       "state": "kv",
+       "offset": 19267584
       },
       {
-       "i32": 16384
+       "state": "kv",
+       "offset": 19268096
       },
       {
-       "i32": 96
+       "buf": "slot_mapping"
       },
       {
-       "i32": 96
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i32": 2048
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i32": 2048
+       "i64": 25690112
       },
       {
-       "i32": 6144
+       "i64": 2048
       },
       {
-       "sym": "tokens"
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l45.a_copy",
-     "kernel": "copy_rows",
+     "label": "l27.attn",
+     "kernel": "attn_prefill",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "attn_out"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "q_n"
       },
       {
-       "i32": 48
+       "state": "kv",
+       "offset": 19267584
       },
       {
-       "i32": 96
+       "state": "kv",
+       "offset": 19268096
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l45.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "block_table"
       },
       {
-       "buf": "ba"
+       "buf": "seq_lens"
       },
       {
-       "i32": 48
+       "f32": 0.0625
       },
       {
-       "i32": 96
+       "buf": "kv_scales"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l45.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.45.linear_attn.A_log"
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "buf": "a_c"
+       "f32": 1.0
       },
       {
-       "buf": "b_c"
+       "f32": 0.0
       },
       {
-       "buf": "model.layers.45.linear_attn.dt_bias"
+       "i64": 8
       },
       {
-       "f32": 1.0
+       "i64": 6144
       },
       {
-       "f32": 20.0
+       "i64": 256
       },
       {
-       "buf": "gdn_q"
+       "i64": 6144
       },
       {
-       "buf": "gdn_k"
+       "i64": 256
       },
       {
-       "buf": "gdn_v"
+       "i64": 0
       },
       {
-       "buf": "core_attn_out"
+       "buf": "seq_lens"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "i64": 25690112
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "i64": 2048
       },
       {
-       "buf": "cu_seqlens_q"
+       "i64": 512
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 1088
+       "i64": 25690112
       },
       {
-       "buf": "num_accepted_tokens"
+       "i64": 2048
       },
       {
-       "f32": 0.0883883461356163
+       "i64": 512
       },
       {
-       "i64": 1
+       "buf": "cu_seqlens_q"
       },
       {
-       "sym": "tokens"
+       "i32": 1
       },
       {
        "i64": 0
@@ -133741,115 +82632,93 @@
      ]
     },
     {
-     "label": "l45.z_copy",
-     "kernel": "copy_rows",
+     "label": "l27.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "x"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "attn_out"
       },
       {
-       "i32": 6144
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "i32": 16384
+       "buf": "model.layers.27.self_attn.o_proj.weight"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l45.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
+       "buf": "residual"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.27.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "model.layers.45.linear_attn.norm.weight"
+       "sym": "tokens"
       },
       {
-       "buf": "z_c"
+       "i32": 5120
       },
       {
-       "i32": 128
+       "i32": 6144
       },
       {
-       "i32": 128
+       "i32": 256
       },
       {
-       "i32": 128
+       "i32": 14336
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 512
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l45.out_proj",
-     "kernel": "gemm",
+     "label": "l27.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.45.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.27.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l45.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l27.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.27.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.45.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.28.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -133858,7 +82727,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -133866,23 +82735,32 @@
      ]
     },
     {
-     "label": "l45.gate_up",
-     "kernel": "gemm",
+     "label": "l28.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.45.mlp.gate_up_proj.weight"
+       "buf": "model.layers.28.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.28.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -133890,293 +82768,281 @@
      ]
     },
     {
-     "label": "l45.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l28.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.28.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 88
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l45.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "num_accepted_tokens"
       },
       {
-       "buf": "model.layers.45.mlp.down_proj.weight"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "y"
+       "buf": "gdn.spec_slots",
+       "offset": 672
       },
       {
-       "sym": "tokens"
+       "buf": "ba"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.28.linear_attn.A_log"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l45.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "model.layers.28.linear_attn.dt_bias"
+      },
       {
-       "buf": "x"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.28.linear_attn.norm.weight"
       },
       {
-       "buf": "residual"
+       "buf": "gdn_q"
       },
       {
-       "buf": "model.layers.46.input_layernorm.weight_p1"
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
+       "buf": "gdn_v"
       },
       {
-       "sym": "tokens"
+       "buf": "g"
       },
       {
-       "i32": 5120
+       "buf": "beta"
       },
       {
-       "i32": 5120
+       "buf": "a_c"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l46.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
+       "buf": "b_c"
+      },
       {
-       "buf": "x"
+       "buf": "z_c"
       },
       {
-       "buf": "model.layers.46.linear_attn.in_proj_qkvz.weight"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "qkvz"
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l46.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l28.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.46.linear_attn.in_proj_ba.weight"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.28.linear_attn.out_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 96
+       "buf": "model.layers.28.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l46.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l28.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.46.linear_attn.conv1d.weight"
+       "buf": "x"
       },
       {
-       "state": "gdn"
+       "buf": "model.layers.28.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 144
+       "sym": "tokens"
       },
       {
-       "buf": "num_accepted_tokens"
+       "i32": 17408
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l28.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "buf": "qkvz"
+       "buf": "act"
       },
       {
-       "i32": 1
+       "buf": "model.layers.28.mlp.down_proj.weight"
       },
       {
-       "i32": 385
+       "buf": "residual"
       },
       {
-       "i64": 16384
+       "buf": "model.layers.29.input_layernorm.weight_p1"
       },
       {
-       "i64": 16384
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l46.post_conv",
-     "kernel": "post_conv",
+     "label": "l29.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
        "buf": "qkvz"
       },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
       {
        "buf": "ba"
       },
       {
-       "buf": "model.layers.46.linear_attn.A_log"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.46.linear_attn.dt_bias"
+       "buf": "model.layers.29.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gdn_q"
+       "buf": "model.layers.29.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "gdn_k"
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_v"
+       "i32": 16384
       },
       {
-       "buf": "g"
+       "i32": 96
       },
       {
-       "buf": "beta"
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l29.gdn",
+     "kernel": "gdn_spec",
+     "args": [
       {
-       "i32": 16384
+       "buf": "qkvz"
       },
       {
-       "i32": 96
+       "buf": "model.layers.29.linear_attn.conv1d.weight"
       },
       {
-       "i32": 96
+       "state": "gdn"
       },
       {
-       "i32": 2048
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 2048
+       "buf": "gdn.spec_line_index",
+       "offset": 92
       },
       {
-       "i32": 6144
+       "buf": "num_accepted_tokens"
       },
       {
-       "sym": "tokens"
+       "buf": "cu_seqlens_q"
       },
       {
-       "i64": 0
+       "buf": "gdn.spec_slots",
+       "offset": 704
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l46.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
+       "buf": "ba"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.29.linear_attn.A_log"
       },
       {
-       "i32": 48
+       "buf": "model.layers.29.linear_attn.dt_bias"
       },
       {
-       "i32": 96
+       "buf": "core_attn_out"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l46.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.29.linear_attn.norm.weight"
       },
       {
-       "buf": "ba"
+       "buf": "gdn_q"
       },
       {
-       "i32": 48
+       "buf": "gdn_k"
       },
       {
-       "i32": 96
+       "buf": "gdn_v"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l46.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
+       "buf": "g"
+      },
       {
-       "buf": "model.layers.46.linear_attn.A_log"
+       "buf": "beta"
       },
       {
        "buf": "a_c"
@@ -134185,276 +83051,265 @@
        "buf": "b_c"
       },
       {
-       "buf": "model.layers.46.linear_attn.dt_bias"
+       "buf": "z_c"
       },
       {
-       "f32": 1.0
+       "f32": 0.0883883461356163
       },
       {
-       "f32": 20.0
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "gdn_q"
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_k"
+       "i32": 385
       },
       {
-       "buf": "gdn_v"
+       "i32": 1703936
       },
       {
-       "buf": "core_attn_out"
+       "i32": 10
       },
       {
-       "state": "gdn",
-       "offset": 204800
-      },
+       "i32": 851968
+      }
+     ]
+    },
+    {
+     "label": "l29.out_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "x"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 1120
+       "buf": "model.layers.29.linear_attn.out_proj.weight"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "residual"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "model.layers.29.post_attention_layernorm.weight_p1"
       },
       {
-       "i64": 1
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 6144
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l46.z_copy",
-     "kernel": "copy_rows",
+     "label": "l29.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "act"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "x"
       },
       {
-       "i32": 6144
+       "buf": "model.layers.29.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 16384
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 17408
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l46.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l29.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.46.linear_attn.norm.weight"
+       "buf": "model.layers.29.mlp.down_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.30.input_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l46.out_proj",
-     "kernel": "gemm",
+     "label": "l30.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "qkvz"
       },
       {
-       "buf": "model.layers.46.linear_attn.out_proj.weight"
+       "buf": "ba"
       },
       {
-       "buf": "y"
+       "buf": "x"
+      },
+      {
+       "buf": "model.layers.30.linear_attn.in_proj_qkvz.weight"
+      },
+      {
+       "buf": "model.layers.30.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 16384
       },
       {
-       "i32": 6144
+       "i32": 96
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l46.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l30.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "x"
+       "buf": "qkvz"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.30.linear_attn.conv1d.weight"
       },
       {
-       "buf": "residual"
+       "state": "gdn"
       },
       {
-       "buf": "model.layers.46.post_attention_layernorm.weight_p1"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 5120
+       "buf": "gdn.spec_line_index",
+       "offset": 96
+      },
+      {
+       "buf": "num_accepted_tokens"
       },
       {
-       "sym": "tokens"
+       "buf": "cu_seqlens_q"
       },
       {
-       "i32": 5120
+       "buf": "gdn.spec_slots",
+       "offset": 736
       },
       {
-       "i32": 5120
+       "buf": "ba"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l46.gate_up",
-     "kernel": "gemm",
-     "args": [
+       "buf": "model.layers.30.linear_attn.A_log"
+      },
       {
-       "buf": "x"
+       "buf": "model.layers.30.linear_attn.dt_bias"
       },
       {
-       "buf": "model.layers.46.mlp.gate_up_proj.weight"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.30.linear_attn.norm.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "gdn_q"
       },
       {
-       "i32": 34816
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l46.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
+       "buf": "gdn_v"
+      },
       {
-       "buf": "act"
+       "buf": "g"
       },
       {
-       "buf": "gate_up"
+       "buf": "beta"
       },
       {
-       "i32": 17408
+       "buf": "a_c"
       },
       {
-       "i32": 0
+       "buf": "b_c"
       },
       {
-       "f32": 1.0
+       "buf": "z_c"
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l46.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "f32": 0.0883883461356163
+      },
       {
-       "buf": "act"
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "model.layers.46.mlp.down_proj.weight"
+       "sym": "tokens"
       },
       {
-       "buf": "y"
+       "i32": 385
       },
       {
-       "sym": "tokens"
+       "i32": 1703936
       },
       {
-       "i32": 5120
+       "i32": 10
       },
       {
-       "i32": 17408
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l46.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l30.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.30.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.47.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.30.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -134463,7 +83318,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -134471,23 +83326,23 @@
      ]
     },
     {
-     "label": "l47.qkv_proj",
-     "kernel": "gemm",
+     "label": "l30.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.47.self_attn.qkv_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkv"
+       "buf": "model.layers.30.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 14336
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -134495,45 +83350,32 @@
      ]
     },
     {
-     "label": "l47.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l30.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.47.self_attn.q_norm.weight_p1"
+       "buf": "x"
       },
       {
-       "i32": 256
+       "buf": "act"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "buf": "model.layers.30.mlp.down_proj.weight"
       },
       {
-       "i32": 24
+       "buf": "residual"
       },
       {
-       "i32": 14336
+       "buf": "model.layers.31.input_layernorm.weight_p1"
       },
       {
-       "i32": 512
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 5120
       },
       {
-       "i32": 256
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -134541,55 +83383,32 @@
      ]
     },
     {
-     "label": "l47.k_norm",
-     "kernel": "gemma_norm_khead",
+     "label": "l31.qkv_proj",
+     "kernel": "gemm",
      "args": [
       {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.47.self_attn.k_norm.weight_p1"
+       "buf": "x"
       },
       {
-       "i32": 256
+       "buf": "model.layers.31.self_attn.qkv_proj.weight"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
+       "buf": "qkv"
       },
       {
-       "i32": 4
+       "sym": "tokens"
       },
       {
        "i32": 14336
       },
       {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l47.rope",
-     "kernel": "mrope",
+     "label": "l31.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
@@ -134598,82 +83417,70 @@
        "buf": "k_n"
       },
       {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
+       "buf": "qkv"
       },
       {
-       "i32": 0
+       "buf": "model.layers.31.self_attn.q_norm.weight_p1"
       },
       {
-       "i64": 0
+       "buf": "model.layers.31.self_attn.k_norm.weight_p1"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l47.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
+       "buf": "cos_g"
       },
       {
-       "buf": "qkv",
-       "offset": 26624
+       "buf": "sin_g"
       },
       {
        "state": "kv",
-       "offset": 35323904
+       "offset": 22478848
       },
       {
        "state": "kv",
-       "offset": 35324416
+       "offset": 22479360
       },
       {
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l47.attn",
+     "label": "l31.attn",
      "kernel": "attn_prefill",
      "args": [
       {
@@ -134684,11 +83491,11 @@
       },
       {
        "state": "kv",
-       "offset": 35323904
+       "offset": 22478848
       },
       {
        "state": "kv",
-       "offset": 35324416
+       "offset": 22479360
       },
       {
        "buf": "block_table"
@@ -134766,11 +83573,11 @@
      ]
     },
     {
-     "label": "l47.gate",
-     "kernel": "sigmoid_mul",
+     "label": "l31.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "gated"
+       "buf": "x"
       },
       {
        "buf": "attn_out"
@@ -134780,31 +83587,13 @@
        "offset": 512
       },
       {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l47.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
+       "buf": "model.layers.31.self_attn.o_proj.weight"
       },
       {
-       "buf": "model.layers.47.self_attn.o_proj.weight"
+       "buf": "residual"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.31.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -134814,36 +83603,15 @@
       },
       {
        "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l47.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
       },
       {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.47.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
+       "i32": 256
       },
       {
-       "i32": 5120
+       "i32": 14336
       },
       {
-       "i32": 5120
+       "i32": 512
       },
       {
        "f32": 9.999999974752427e-07
@@ -134851,95 +83619,47 @@
      ]
     },
     {
-     "label": "l47.gate_up",
-     "kernel": "gemm",
+     "label": "l31.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.47.mlp.gate_up_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.31.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l47.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
       {
        "i32": 17408
       },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l47.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.47.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
       {
        "i32": 5120
-      },
-      {
-       "i32": 17408
       }
      ]
     },
     {
-     "label": "l47.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l31.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.31.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.48.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.32.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -134948,7 +83668,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -134956,41 +83676,23 @@
      ]
     },
     {
-     "label": "l47.fc_tap",
-     "kernel": "gemm_acc",
+     "label": "l32.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "residual"
-      },
-      {
-       "buf": "draft.fc.3.weight"
-      },
-      {
-       "buf": "fc_out"
-      },
-      {
-       "sym": "tokens"
+       "buf": "qkvz"
       },
       {
-       "i32": 5120
+       "buf": "ba"
       },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l48.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.48.linear_attn.in_proj_qkvz.weight"
+       "buf": "model.layers.32.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.32.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
@@ -134998,27 +83700,6 @@
       {
        "i32": 16384
       },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l48.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.48.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
       {
        "i32": 96
       },
@@ -135028,21 +83709,25 @@
      ]
     },
     {
-     "label": "l48.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l32.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.48.linear_attn.conv1d.weight"
+       "buf": "model.layers.32.linear_attn.conv1d.weight"
       },
       {
        "state": "gdn"
       },
+      {
+       "state": "gdn",
+       "offset": 204800
+      },
       {
        "buf": "gdn.spec_line_index",
-       "offset": 148
+       "offset": 100
       },
       {
        "buf": "num_accepted_tokens"
@@ -135051,47 +83736,23 @@
        "buf": "cu_seqlens_q"
       },
       {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
+       "buf": "gdn.spec_slots",
+       "offset": 768
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l48.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
+       "buf": "ba"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.32.linear_attn.A_log"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.32.linear_attn.dt_bias"
       },
       {
-       "buf": "model.layers.48.linear_attn.A_log"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.48.linear_attn.dt_bias"
+       "buf": "model.layers.32.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -135109,361 +83770,328 @@
        "buf": "beta"
       },
       {
-       "i32": 16384
+       "buf": "a_c"
       },
       {
-       "i32": 96
+       "buf": "b_c"
       },
       {
-       "i32": 96
+       "buf": "z_c"
       },
       {
-       "i32": 2048
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 2048
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 6144
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 385
       },
       {
-       "i64": 0
+       "i32": 1703936
       },
       {
-       "i64": 0
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l48.a_copy",
-     "kernel": "copy_rows",
+     "label": "l32.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "core_attn_out"
       },
       {
-       "i32": 48
+       "buf": "model.layers.32.linear_attn.out_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l48.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.32.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "ba"
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 5120
       },
       {
-       "i32": 96
+       "i32": 6144
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l48.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l32.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "model.layers.48.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.48.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
-      },
-      {
-       "buf": "gdn_q"
+       "buf": "act"
       },
       {
-       "buf": "gdn_k"
+       "buf": "x"
       },
       {
-       "buf": "gdn_v"
+       "buf": "model.layers.32.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "core_attn_out"
+       "sym": "tokens"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "i32": 17408
       },
       {
-       "state": "gdn",
-       "offset": 204800
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l32.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "buf": "cu_seqlens_q"
+       "buf": "x"
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 1152
+       "buf": "act"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "model.layers.32.mlp.down_proj.weight"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "residual"
       },
       {
-       "i64": 1
+       "buf": "model.layers.33.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l48.z_copy",
-     "kernel": "copy_rows",
+     "label": "l33.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "qkvz"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "ba"
       },
       {
-       "i32": 6144
+       "buf": "x"
+      },
+      {
+       "buf": "model.layers.33.linear_attn.in_proj_qkvz.weight"
+      },
+      {
+       "buf": "model.layers.33.linear_attn.in_proj_ba.weight"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 16384
       },
       {
-       "i32": 6144
+       "i32": 96
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l48.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l33.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "qkvz"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.33.linear_attn.conv1d.weight"
       },
       {
-       "buf": "model.layers.48.linear_attn.norm.weight"
+       "state": "gdn"
       },
       {
-       "buf": "z_c"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 128
+       "buf": "gdn.spec_line_index",
+       "offset": 104
       },
       {
-       "i32": 128
+       "buf": "num_accepted_tokens"
       },
       {
-       "i32": 128
+       "buf": "cu_seqlens_q"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "buf": "gdn.spec_slots",
+       "offset": 800
       },
       {
-       "f32": 9.999999974752427e-07
+       "buf": "ba"
       },
       {
-       "i64": 0
+       "buf": "model.layers.33.linear_attn.A_log"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l48.out_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "model.layers.33.linear_attn.dt_bias"
+      },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.48.linear_attn.out_proj.weight"
+       "buf": "model.layers.33.linear_attn.norm.weight"
       },
       {
-       "buf": "y"
+       "buf": "gdn_q"
       },
       {
-       "sym": "tokens"
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
+       "buf": "gdn_v"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l48.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "g"
+      },
       {
-       "buf": "x"
+       "buf": "beta"
       },
       {
-       "buf": "y"
+       "buf": "a_c"
       },
       {
-       "buf": "residual"
+       "buf": "b_c"
       },
       {
-       "buf": "model.layers.48.post_attention_layernorm.weight_p1"
+       "buf": "z_c"
       },
       {
-       "i32": 5120
+       "f32": 0.0883883461356163
+      },
+      {
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
       },
       {
-       "f32": 9.999999974752427e-07
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l48.gate_up",
-     "kernel": "gemm",
+     "label": "l33.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.48.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 34816
+       "buf": "model.layers.33.linear_attn.out_proj.weight"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l48.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "residual"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.33.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 17408
+       "sym": "tokens"
       },
       {
-       "i32": 0
+       "i32": 5120
       },
       {
-       "f32": 1.0
+       "i32": 6144
       },
       {
-       "i32": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l48.down_proj",
-     "kernel": "gemm",
+     "label": "l33.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
        "buf": "act"
       },
       {
-       "buf": "model.layers.48.mlp.down_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.33.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 17408
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l48.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l33.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.33.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.49.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.34.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -135472,7 +84100,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -135480,23 +84108,23 @@
      ]
     },
     {
-     "label": "l49.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l33.fc_tap",
+     "kernel": "gemm_acc",
      "args": [
       {
-       "buf": "x"
+       "buf": "residual"
       },
       {
-       "buf": "model.layers.49.linear_attn.in_proj_qkvz.weight"
+       "buf": "draft.fc.2.weight"
       },
       {
-       "buf": "qkvz"
+       "buf": "fc_out"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 5120
       },
       {
        "i32": 5120
@@ -135504,21 +84132,30 @@
      ]
     },
     {
-     "label": "l49.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l34.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.49.linear_attn.in_proj_ba.weight"
+       "buf": "model.layers.34.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.34.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
+      {
+       "i32": 16384
+      },
       {
        "i32": 96
       },
@@ -135528,21 +84165,25 @@
      ]
     },
     {
-     "label": "l49.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l34.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.49.linear_attn.conv1d.weight"
+       "buf": "model.layers.34.linear_attn.conv1d.weight"
       },
       {
        "state": "gdn"
       },
+      {
+       "state": "gdn",
+       "offset": 204800
+      },
       {
        "buf": "gdn.spec_line_index",
-       "offset": 152
+       "offset": 108
       },
       {
        "buf": "num_accepted_tokens"
@@ -135551,47 +84192,23 @@
        "buf": "cu_seqlens_q"
       },
       {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
+       "buf": "gdn.spec_slots",
+       "offset": 832
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l49.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
+       "buf": "ba"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.34.linear_attn.A_log"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.34.linear_attn.dt_bias"
       },
       {
-       "buf": "model.layers.49.linear_attn.A_log"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.49.linear_attn.dt_bias"
+       "buf": "model.layers.34.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -135609,192 +84226,196 @@
        "buf": "beta"
       },
       {
-       "i32": 16384
+       "buf": "a_c"
       },
       {
-       "i32": 96
+       "buf": "b_c"
       },
       {
-       "i32": 96
+       "buf": "z_c"
       },
       {
-       "i32": 2048
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 2048
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 6144
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 385
       },
       {
-       "i64": 0
+       "i32": 1703936
       },
       {
-       "i64": 0
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l49.a_copy",
-     "kernel": "copy_rows",
+     "label": "l34.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "core_attn_out"
       },
       {
-       "i32": 48
+       "buf": "model.layers.34.linear_attn.out_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l49.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.34.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "ba"
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 5120
       },
       {
-       "i32": 96
+       "i32": 6144
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l49.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l34.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "model.layers.49.linear_attn.A_log"
+       "buf": "act"
       },
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "b_c"
+       "buf": "model.layers.34.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "model.layers.49.linear_attn.dt_bias"
+       "sym": "tokens"
       },
       {
-       "f32": 1.0
+       "i32": 17408
       },
       {
-       "f32": 20.0
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l34.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "buf": "gdn_q"
+       "buf": "x"
       },
       {
-       "buf": "gdn_k"
+       "buf": "act"
       },
       {
-       "buf": "gdn_v"
+       "buf": "model.layers.34.mlp.down_proj.weight"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "residual"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "model.layers.35.input_layernorm.weight_p1"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "sym": "tokens"
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 5120
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 1184
+       "i32": 17408
       },
       {
-       "buf": "num_accepted_tokens"
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l35.qkv_proj",
+     "kernel": "gemm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "model.layers.35.self_attn.qkv_proj.weight"
       },
       {
-       "i64": 1
+       "buf": "qkv"
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l49.z_copy",
-     "kernel": "copy_rows",
+     "label": "l35.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "q_n"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "k_n"
       },
       {
-       "i32": 6144
+       "buf": "qkv"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.35.self_attn.q_norm.weight_p1"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l49.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
+       "buf": "model.layers.35.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "cos_g"
       },
       {
-       "buf": "model.layers.49.linear_attn.norm.weight"
+       "buf": "sin_g"
       },
       {
-       "buf": "z_c"
+       "state": "kv",
+       "offset": 25690112
       },
       {
-       "i32": 128
+       "state": "kv",
+       "offset": 25690624
       },
       {
-       "i32": 128
+       "buf": "slot_mapping"
       },
       {
-       "i32": 128
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
        "expr": {
@@ -135802,168 +84423,208 @@
          {
           "sym": "tokens"
          },
-         48
+         4
         ]
        }
       },
       {
-       "f32": 9.999999974752427e-07
+       "i64": 25690112
       },
       {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 0
+       "i64": 512
+      },
+      {
+       "i32": 784
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l49.out_proj",
-     "kernel": "gemm",
+     "label": "l35.attn",
+     "kernel": "attn_prefill",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "attn_out"
       },
       {
-       "buf": "model.layers.49.linear_attn.out_proj.weight"
+       "buf": "q_n"
       },
       {
-       "buf": "y"
+       "state": "kv",
+       "offset": 25690112
       },
       {
-       "sym": "tokens"
+       "state": "kv",
+       "offset": 25690624
       },
       {
-       "i32": 5120
+       "buf": "block_table"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l49.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "seq_lens"
+      },
       {
-       "buf": "x"
+       "f32": 0.0625
       },
       {
-       "buf": "y"
+       "buf": "kv_scales"
       },
       {
-       "buf": "residual"
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "buf": "model.layers.49.post_attention_layernorm.weight_p1"
+       "f32": 1.0
       },
       {
-       "i32": 5120
+       "f32": 0.0
       },
       {
-       "sym": "tokens"
+       "i64": 8
       },
       {
-       "i32": 5120
+       "i64": 6144
       },
       {
-       "i32": 5120
+       "i64": 256
       },
       {
-       "f32": 9.999999974752427e-07
+       "i64": 6144
+      },
+      {
+       "i64": 256
+      },
+      {
+       "i64": 0
+      },
+      {
+       "buf": "seq_lens"
+      },
+      {
+       "i64": 25690112
+      },
+      {
+       "i64": 2048
+      },
+      {
+       "i64": 512
+      },
+      {
+       "i64": 25690112
+      },
+      {
+       "i64": 2048
+      },
+      {
+       "i64": 512
+      },
+      {
+       "buf": "cu_seqlens_q"
+      },
+      {
+       "i32": 1
+      },
+      {
+       "i64": 0
+      },
+      {
+       "i64": 0
       }
      ]
     },
     {
-     "label": "l49.gate_up",
-     "kernel": "gemm",
+     "label": "l35.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.49.mlp.gate_up_proj.weight"
+       "buf": "attn_out"
       },
       {
-       "buf": "gate_up"
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.35.self_attn.o_proj.weight"
       },
       {
-       "i32": 34816
+       "buf": "residual"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l49.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
+       "buf": "model.layers.35.post_attention_layernorm.weight_p1"
+      },
       {
-       "buf": "act"
+       "sym": "tokens"
       },
       {
-       "buf": "gate_up"
+       "i32": 5120
+      },
+      {
+       "i32": 6144
       },
       {
-       "i32": 17408
+       "i32": 256
       },
       {
-       "i32": 0
+       "i32": 14336
       },
       {
-       "f32": 1.0
+       "i32": 512
       },
       {
-       "i32": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l49.down_proj",
-     "kernel": "gemm",
+     "label": "l35.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
        "buf": "act"
       },
       {
-       "buf": "model.layers.49.mlp.down_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.35.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 17408
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l49.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l35.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.35.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.50.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.36.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -135972,7 +84633,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -135980,45 +84641,30 @@
      ]
     },
     {
-     "label": "l50.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l36.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.50.linear_attn.in_proj_qkvz.weight"
-      },
       {
        "buf": "qkvz"
       },
       {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
+       "buf": "ba"
       },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l50.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.50.linear_attn.in_proj_ba.weight"
+       "buf": "model.layers.36.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.36.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
+      {
+       "i32": 16384
+      },
       {
        "i32": 96
       },
@@ -136028,21 +84674,25 @@
      ]
     },
     {
-     "label": "l50.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l36.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.50.linear_attn.conv1d.weight"
+       "buf": "model.layers.36.linear_attn.conv1d.weight"
       },
       {
        "state": "gdn"
       },
+      {
+       "state": "gdn",
+       "offset": 204800
+      },
       {
        "buf": "gdn.spec_line_index",
-       "offset": 156
+       "offset": 112
       },
       {
        "buf": "num_accepted_tokens"
@@ -136051,314 +84701,362 @@
        "buf": "cu_seqlens_q"
       },
       {
-       "buf": "qkvz"
+       "buf": "gdn.spec_slots",
+       "offset": 864
       },
       {
-       "i32": 1
+       "buf": "ba"
       },
       {
-       "i32": 385
+       "buf": "model.layers.36.linear_attn.A_log"
       },
       {
-       "i64": 16384
+       "buf": "model.layers.36.linear_attn.dt_bias"
       },
       {
-       "i64": 16384
+       "buf": "core_attn_out"
       },
       {
-       "i64": 0
+       "buf": "model.layers.36.linear_attn.norm.weight"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l50.post_conv",
-     "kernel": "post_conv",
-     "args": [
+       "buf": "gdn_q"
+      },
       {
-       "buf": "qkvz"
+       "buf": "gdn_k"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "gdn_v"
       },
       {
-       "buf": "ba"
+       "buf": "g"
       },
       {
-       "buf": "model.layers.50.linear_attn.A_log"
+       "buf": "beta"
       },
       {
-       "buf": "model.layers.50.linear_attn.dt_bias"
+       "buf": "a_c"
       },
       {
-       "buf": "gdn_q"
+       "buf": "b_c"
       },
       {
-       "buf": "gdn_k"
+       "buf": "z_c"
       },
       {
-       "buf": "gdn_v"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "g"
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "beta"
+       "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 385
       },
       {
-       "i32": 96
+       "i32": 1703936
       },
       {
-       "i32": 96
+       "i32": 10
       },
       {
-       "i32": 2048
+       "i32": 851968
+      }
+     ]
+    },
+    {
+     "label": "l36.out_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "i32": 2048
+       "buf": "core_attn_out"
       },
       {
-       "i32": 6144
+       "buf": "model.layers.36.linear_attn.out_proj.weight"
+      },
+      {
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.36.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l50.a_copy",
-     "kernel": "copy_rows",
+     "label": "l36.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "act"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "x"
       },
       {
-       "i32": 48
+       "buf": "model.layers.36.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 96
+       "sym": "tokens"
+      },
+      {
+       "i32": 17408
       },
       {
-       "i32": 48
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l50.b_copy",
-     "kernel": "copy_rows",
+     "label": "l36.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "b_c"
+       "buf": "x"
       },
       {
-       "buf": "ba"
+       "buf": "act"
       },
       {
-       "i32": 48
+       "buf": "model.layers.36.mlp.down_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.37.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 17408
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l50.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l37.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "model.layers.50.linear_attn.A_log"
+       "buf": "qkvz"
       },
       {
-       "buf": "a_c"
+       "buf": "ba"
       },
       {
-       "buf": "b_c"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.50.linear_attn.dt_bias"
+       "buf": "model.layers.37.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "f32": 1.0
+       "buf": "model.layers.37.linear_attn.in_proj_ba.weight"
       },
       {
-       "f32": 20.0
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_q"
+       "i32": 16384
       },
       {
-       "buf": "gdn_k"
+       "i32": 96
       },
       {
-       "buf": "gdn_v"
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l37.gdn",
+     "kernel": "gdn_spec",
+     "args": [
+      {
+       "buf": "qkvz"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.37.linear_attn.conv1d.weight"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "state": "gdn"
       },
       {
        "state": "gdn",
        "offset": 204800
       },
+      {
+       "buf": "gdn.spec_line_index",
+       "offset": 116
+      },
+      {
+       "buf": "num_accepted_tokens"
+      },
       {
        "buf": "cu_seqlens_q"
       },
       {
        "buf": "gdn.spec_slots",
-       "offset": 1216
+       "offset": 896
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "ba"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "model.layers.37.linear_attn.A_log"
       },
       {
-       "i64": 1
+       "buf": "model.layers.37.linear_attn.dt_bias"
       },
       {
-       "sym": "tokens"
+       "buf": "core_attn_out"
       },
       {
-       "i64": 0
+       "buf": "model.layers.37.linear_attn.norm.weight"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l50.z_copy",
-     "kernel": "copy_rows",
-     "args": [
+       "buf": "gdn_q"
+      },
+      {
+       "buf": "gdn_k"
+      },
+      {
+       "buf": "gdn_v"
+      },
+      {
+       "buf": "g"
+      },
+      {
+       "buf": "beta"
+      },
+      {
+       "buf": "a_c"
+      },
+      {
+       "buf": "b_c"
+      },
       {
        "buf": "z_c"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 6144
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 16384
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 385
+      },
+      {
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l50.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l37.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.50.linear_attn.norm.weight"
+       "buf": "model.layers.37.linear_attn.out_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.37.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l50.out_proj",
-     "kernel": "gemm",
+     "label": "l37.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.50.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.37.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l50.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l37.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.37.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.50.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.38.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -136367,7 +85065,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -136375,23 +85073,32 @@
      ]
     },
     {
-     "label": "l50.gate_up",
-     "kernel": "gemm",
+     "label": "l38.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.50.mlp.gate_up_proj.weight"
+       "buf": "model.layers.38.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.38.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -136399,197 +85106,182 @@
      ]
     },
     {
-     "label": "l50.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l38.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.38.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 120
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l50.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "num_accepted_tokens"
       },
       {
-       "buf": "model.layers.50.mlp.down_proj.weight"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "y"
+       "buf": "gdn.spec_slots",
+       "offset": 928
       },
       {
-       "sym": "tokens"
+       "buf": "ba"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.38.linear_attn.A_log"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l50.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "model.layers.38.linear_attn.dt_bias"
+      },
       {
-       "buf": "x"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.38.linear_attn.norm.weight"
       },
       {
-       "buf": "residual"
+       "buf": "gdn_q"
       },
       {
-       "buf": "model.layers.51.input_layernorm.weight_p1"
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
+       "buf": "gdn_v"
       },
       {
-       "sym": "tokens"
+       "buf": "g"
       },
       {
-       "i32": 5120
+       "buf": "beta"
       },
       {
-       "i32": 5120
+       "buf": "a_c"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l51.qkv_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "b_c"
+      },
       {
-       "buf": "x"
+       "buf": "z_c"
       },
       {
-       "buf": "model.layers.51.self_attn.qkv_proj.weight"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "qkv"
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 14336
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l51.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l38.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.51.self_attn.q_norm.weight_p1"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 256
+       "buf": "model.layers.38.linear_attn.out_proj.weight"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "buf": "residual"
       },
       {
-       "i32": 24
+       "buf": "model.layers.38.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 14336
+       "sym": "tokens"
       },
       {
-       "i32": 512
+       "i32": 5120
       },
       {
        "i32": 6144
       },
-      {
-       "i32": 256
-      },
       {
        "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l51.k_norm",
-     "kernel": "gemma_norm_khead",
+     "label": "l38.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "k_n"
+       "buf": "act"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
+       "buf": "x"
       },
       {
-       "buf": "model.layers.51.self_attn.k_norm.weight_p1"
+       "buf": "model.layers.38.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 256
+       "sym": "tokens"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
+       "i32": 17408
+      },
+      {
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l38.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "i32": 4
+       "buf": "act"
       },
       {
-       "i32": 14336
+       "buf": "model.layers.38.mlp.down_proj.weight"
       },
       {
-       "i32": 256
+       "buf": "residual"
       },
       {
-       "i32": 1024
+       "buf": "model.layers.39.input_layernorm.weight_p1"
       },
       {
-       "i32": 256
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -136597,92 +85289,104 @@
      ]
     },
     {
-     "label": "l51.rope",
-     "kernel": "mrope",
+     "label": "l39.qkv_proj",
+     "kernel": "gemm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
+       "buf": "x"
       },
       {
-       "buf": "cos_g"
+       "buf": "model.layers.39.self_attn.qkv_proj.weight"
       },
       {
-       "buf": "sin_g"
+       "buf": "qkv"
       },
       {
-       "i32": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l51.kv_write",
-     "kernel": "reshape_and_cache",
+     "label": "l39.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "k_n"
+       "buf": "q_n"
       },
       {
-       "buf": "qkv",
-       "offset": 26624
+       "buf": "k_n"
       },
       {
-       "state": "kv",
-       "offset": 38535168
+       "buf": "qkv"
       },
       {
-       "state": "kv",
-       "offset": 38535680
+       "buf": "model.layers.39.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "slot_mapping"
+       "buf": "model.layers.39.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "kv_scales"
+       "buf": "cos_g"
       },
       {
-       "buf": "kv_scales",
-       "offset": 4
+       "buf": "sin_g"
       },
       {
-       "i64": 1024
+       "state": "kv",
+       "offset": 28901376
       },
       {
-       "i64": 14336
+       "state": "kv",
+       "offset": 28901888
       },
       {
-       "i64": 25690112
+       "buf": "slot_mapping"
       },
       {
-       "i64": 512
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 0
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i64": 0
+       "i64": 25690112
       },
       {
        "i64": 2048
       },
       {
-       "i64": 0
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l51.attn",
+     "label": "l39.attn",
      "kernel": "attn_prefill",
      "args": [
       {
@@ -136693,11 +85397,11 @@
       },
       {
        "state": "kv",
-       "offset": 38535168
+       "offset": 28901376
       },
       {
        "state": "kv",
-       "offset": 38535680
+       "offset": 28901888
       },
       {
        "buf": "block_table"
@@ -136775,11 +85479,11 @@
      ]
     },
     {
-     "label": "l51.gate",
-     "kernel": "sigmoid_mul",
+     "label": "l39.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "gated"
+       "buf": "x"
       },
       {
        "buf": "attn_out"
@@ -136789,31 +85493,13 @@
        "offset": 512
       },
       {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l51.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
+       "buf": "model.layers.39.self_attn.o_proj.weight"
       },
       {
-       "buf": "model.layers.51.self_attn.o_proj.weight"
+       "buf": "residual"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.39.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -136823,36 +85509,15 @@
       },
       {
        "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l51.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.51.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
+       "i32": 256
       },
       {
-       "i32": 5120
+       "i32": 14336
       },
       {
-       "i32": 5120
+       "i32": 512
       },
       {
        "f32": 9.999999974752427e-07
@@ -136860,95 +85525,47 @@
      ]
     },
     {
-     "label": "l51.gate_up",
-     "kernel": "gemm",
+     "label": "l39.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.51.mlp.gate_up_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.39.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l51.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
       {
        "i32": 17408
       },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l51.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.51.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
       {
        "i32": 5120
-      },
-      {
-       "i32": 17408
       }
      ]
     },
     {
-     "label": "l51.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l39.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.39.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.52.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.40.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -136957,7 +85574,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -136965,45 +85582,30 @@
      ]
     },
     {
-     "label": "l52.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l40.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.52.linear_attn.in_proj_qkvz.weight"
-      },
       {
        "buf": "qkvz"
       },
       {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
+       "buf": "ba"
       },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l52.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.52.linear_attn.in_proj_ba.weight"
+       "buf": "model.layers.40.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.40.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
+      {
+       "i32": 16384
+      },
       {
        "i32": 96
       },
@@ -137013,21 +85615,25 @@
      ]
     },
     {
-     "label": "l52.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l40.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.52.linear_attn.conv1d.weight"
+       "buf": "model.layers.40.linear_attn.conv1d.weight"
       },
       {
        "state": "gdn"
       },
+      {
+       "state": "gdn",
+       "offset": 204800
+      },
       {
        "buf": "gdn.spec_line_index",
-       "offset": 160
+       "offset": 124
       },
       {
        "buf": "num_accepted_tokens"
@@ -137036,47 +85642,23 @@
        "buf": "cu_seqlens_q"
       },
       {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
+       "buf": "gdn.spec_slots",
+       "offset": 960
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l52.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
+       "buf": "ba"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.40.linear_attn.A_log"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.40.linear_attn.dt_bias"
       },
       {
-       "buf": "model.layers.52.linear_attn.A_log"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.52.linear_attn.dt_bias"
+       "buf": "model.layers.40.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -137094,331 +85676,271 @@
        "buf": "beta"
       },
       {
-       "i32": 16384
+       "buf": "a_c"
       },
       {
-       "i32": 96
+       "buf": "b_c"
       },
       {
-       "i32": 96
+       "buf": "z_c"
       },
       {
-       "i32": 2048
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 2048
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 6144
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 385
       },
       {
-       "i64": 0
+       "i32": 1703936
       },
       {
-       "i64": 0
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l52.a_copy",
-     "kernel": "copy_rows",
+     "label": "l40.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "core_attn_out"
       },
       {
-       "i32": 48
+       "buf": "model.layers.40.linear_attn.out_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l52.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.40.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "ba"
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 5120
       },
       {
-       "i32": 96
+       "i32": 6144
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l52.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l40.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "model.layers.52.linear_attn.A_log"
+       "buf": "act"
       },
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "b_c"
+       "buf": "model.layers.40.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "model.layers.52.linear_attn.dt_bias"
+       "sym": "tokens"
       },
       {
-       "f32": 1.0
+       "i32": 17408
       },
       {
-       "f32": 20.0
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l40.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "buf": "gdn_q"
+       "buf": "x"
       },
       {
-       "buf": "gdn_k"
+       "buf": "act"
       },
       {
-       "buf": "gdn_v"
+       "buf": "model.layers.40.mlp.down_proj.weight"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "residual"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "model.layers.41.input_layernorm.weight_p1"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "sym": "tokens"
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 5120
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 1248
+       "i32": 17408
       },
       {
-       "buf": "num_accepted_tokens"
-      },
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l41.in_proj",
+     "kernel": "gemm8_dual",
+     "args": [
       {
-       "f32": 0.0883883461356163
+       "buf": "qkvz"
       },
       {
-       "i64": 1
+       "buf": "ba"
       },
       {
-       "sym": "tokens"
+       "buf": "x"
       },
       {
-       "i64": 0
+       "buf": "model.layers.41.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l52.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
+       "buf": "model.layers.41.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 16384
       },
       {
-       "i32": 16384
+       "i32": 96
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l52.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l41.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
+       "buf": "qkvz"
       },
       {
-       "buf": "model.layers.52.linear_attn.norm.weight"
+       "buf": "model.layers.41.linear_attn.conv1d.weight"
       },
       {
-       "buf": "z_c"
+       "state": "gdn"
       },
       {
-       "i32": 128
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 128
+       "buf": "gdn.spec_line_index",
+       "offset": 128
       },
       {
-       "i32": 128
+       "buf": "num_accepted_tokens"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "buf": "cu_seqlens_q"
       },
       {
-       "f32": 9.999999974752427e-07
+       "buf": "gdn.spec_slots",
+       "offset": 992
       },
       {
-       "i64": 0
+       "buf": "ba"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l52.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
+       "buf": "model.layers.41.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.52.linear_attn.out_proj.weight"
+       "buf": "model.layers.41.linear_attn.dt_bias"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.41.linear_attn.norm.weight"
       },
       {
-       "i32": 5120
+       "buf": "gdn_q"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l52.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
+       "buf": "gdn_k"
       },
       {
-       "buf": "y"
+       "buf": "gdn_v"
       },
       {
-       "buf": "residual"
+       "buf": "g"
       },
       {
-       "buf": "model.layers.52.post_attention_layernorm.weight_p1"
+       "buf": "beta"
       },
       {
-       "i32": 5120
+       "buf": "a_c"
       },
       {
-       "sym": "tokens"
+       "buf": "b_c"
       },
       {
-       "i32": 5120
+       "buf": "z_c"
       },
       {
-       "i32": 5120
+       "f32": 0.0883883461356163
       },
       {
        "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l52.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
       },
       {
-       "buf": "model.layers.52.mlp.gate_up_proj.weight"
+       "sym": "tokens"
       },
       {
-       "buf": "gate_up"
+       "i32": 385
       },
       {
-       "sym": "tokens"
+       "i32": 1703936
       },
       {
-       "i32": 34816
+       "i32": 10
       },
       {
-       "i32": 5120
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l52.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l41.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
+       "buf": "x"
       },
       {
-       "f32": 1.0
+       "buf": "core_attn_out"
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l52.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "model.layers.41.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.52.mlp.down_proj.weight"
+       "buf": "residual"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.41.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -137427,83 +85949,95 @@
        "i32": 5120
       },
       {
-       "i32": 17408
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l52.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l41.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.53.input_layernorm.weight_p1"
+       "buf": "x"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.41.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l53.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l41.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.53.linear_attn.in_proj_qkvz.weight"
+       "buf": "act"
+      },
+      {
+       "buf": "model.layers.41.mlp.down_proj.weight"
+      },
+      {
+       "buf": "residual"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.42.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l53.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l42.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.53.linear_attn.in_proj_ba.weight"
+       "buf": "model.layers.42.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.42.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
+      {
+       "i32": 16384
+      },
       {
        "i32": 96
       },
@@ -137513,21 +86047,25 @@
      ]
     },
     {
-     "label": "l53.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l42.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.53.linear_attn.conv1d.weight"
+       "buf": "model.layers.42.linear_attn.conv1d.weight"
       },
       {
        "state": "gdn"
       },
+      {
+       "state": "gdn",
+       "offset": 204800
+      },
       {
        "buf": "gdn.spec_line_index",
-       "offset": 164
+       "offset": 132
       },
       {
        "buf": "num_accepted_tokens"
@@ -137536,47 +86074,23 @@
        "buf": "cu_seqlens_q"
       },
       {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
+       "buf": "gdn.spec_slots",
+       "offset": 1024
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l53.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
+       "buf": "ba"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.42.linear_attn.A_log"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.42.linear_attn.dt_bias"
       },
       {
-       "buf": "model.layers.53.linear_attn.A_log"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.53.linear_attn.dt_bias"
+       "buf": "model.layers.42.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -137594,192 +86108,196 @@
        "buf": "beta"
       },
       {
-       "i32": 16384
+       "buf": "a_c"
       },
       {
-       "i32": 96
+       "buf": "b_c"
       },
       {
-       "i32": 96
+       "buf": "z_c"
       },
       {
-       "i32": 2048
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 2048
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 6144
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 385
       },
       {
-       "i64": 0
+       "i32": 1703936
       },
       {
-       "i64": 0
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l53.a_copy",
-     "kernel": "copy_rows",
+     "label": "l42.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "core_attn_out"
       },
       {
-       "i32": 48
+       "buf": "model.layers.42.linear_attn.out_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l53.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.42.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "ba"
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 5120
       },
       {
-       "i32": 96
+       "i32": 6144
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l53.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l42.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "model.layers.53.linear_attn.A_log"
+       "buf": "act"
       },
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "b_c"
+       "buf": "model.layers.42.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "model.layers.53.linear_attn.dt_bias"
+       "sym": "tokens"
       },
       {
-       "f32": 1.0
+       "i32": 17408
       },
       {
-       "f32": 20.0
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l42.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "buf": "gdn_q"
+       "buf": "x"
       },
       {
-       "buf": "gdn_k"
+       "buf": "act"
       },
       {
-       "buf": "gdn_v"
+       "buf": "model.layers.42.mlp.down_proj.weight"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "residual"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "model.layers.43.input_layernorm.weight_p1"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "sym": "tokens"
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 5120
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 1280
+       "i32": 17408
       },
       {
-       "buf": "num_accepted_tokens"
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l43.qkv_proj",
+     "kernel": "gemm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "model.layers.43.self_attn.qkv_proj.weight"
       },
       {
-       "i64": 1
+       "buf": "qkv"
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l53.z_copy",
-     "kernel": "copy_rows",
+     "label": "l43.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "q_n"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "k_n"
       },
       {
-       "i32": 6144
+       "buf": "qkv"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.43.self_attn.q_norm.weight_p1"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l53.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
+       "buf": "model.layers.43.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "cos_g"
       },
       {
-       "buf": "model.layers.53.linear_attn.norm.weight"
+       "buf": "sin_g"
       },
       {
-       "buf": "z_c"
+       "state": "kv",
+       "offset": 32112640
       },
       {
-       "i32": 128
+       "state": "kv",
+       "offset": 32113152
       },
       {
-       "i32": 128
+       "buf": "slot_mapping"
       },
       {
-       "i32": 128
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
        "expr": {
@@ -137787,168 +86305,208 @@
          {
           "sym": "tokens"
          },
-         48
+         4
         ]
        }
       },
       {
-       "f32": 9.999999974752427e-07
+       "i64": 25690112
       },
       {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 0
+       "i64": 512
+      },
+      {
+       "i32": 784
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l53.out_proj",
-     "kernel": "gemm",
+     "label": "l43.attn",
+     "kernel": "attn_prefill",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "attn_out"
       },
       {
-       "buf": "model.layers.53.linear_attn.out_proj.weight"
+       "buf": "q_n"
       },
       {
-       "buf": "y"
+       "state": "kv",
+       "offset": 32112640
       },
       {
-       "sym": "tokens"
+       "state": "kv",
+       "offset": 32113152
       },
       {
-       "i32": 5120
+       "buf": "block_table"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l53.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "seq_lens"
+      },
       {
-       "buf": "x"
+       "f32": 0.0625
       },
       {
-       "buf": "y"
+       "buf": "kv_scales"
       },
       {
-       "buf": "residual"
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "buf": "model.layers.53.post_attention_layernorm.weight_p1"
+       "f32": 1.0
       },
       {
-       "i32": 5120
+       "f32": 0.0
       },
       {
-       "sym": "tokens"
+       "i64": 8
       },
       {
-       "i32": 5120
+       "i64": 6144
       },
       {
-       "i32": 5120
+       "i64": 256
       },
       {
-       "f32": 9.999999974752427e-07
+       "i64": 6144
+      },
+      {
+       "i64": 256
+      },
+      {
+       "i64": 0
+      },
+      {
+       "buf": "seq_lens"
+      },
+      {
+       "i64": 25690112
+      },
+      {
+       "i64": 2048
+      },
+      {
+       "i64": 512
+      },
+      {
+       "i64": 25690112
+      },
+      {
+       "i64": 2048
+      },
+      {
+       "i64": 512
+      },
+      {
+       "buf": "cu_seqlens_q"
+      },
+      {
+       "i32": 1
+      },
+      {
+       "i64": 0
+      },
+      {
+       "i64": 0
       }
      ]
     },
     {
-     "label": "l53.gate_up",
-     "kernel": "gemm",
+     "label": "l43.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.53.mlp.gate_up_proj.weight"
+       "buf": "attn_out"
       },
       {
-       "buf": "gate_up"
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.43.self_attn.o_proj.weight"
       },
       {
-       "i32": 34816
+       "buf": "residual"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l53.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
+       "buf": "model.layers.43.post_attention_layernorm.weight_p1"
+      },
       {
-       "buf": "act"
+       "sym": "tokens"
       },
       {
-       "buf": "gate_up"
+       "i32": 5120
       },
       {
-       "i32": 17408
+       "i32": 6144
       },
       {
-       "i32": 0
+       "i32": 256
       },
       {
-       "f32": 1.0
+       "i32": 14336
       },
       {
-       "i32": 0
+       "i32": 512
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l53.down_proj",
-     "kernel": "gemm",
+     "label": "l43.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
        "buf": "act"
       },
       {
-       "buf": "model.layers.53.mlp.down_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.43.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 17408
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l53.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l43.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.43.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.54.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.44.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -137957,7 +86515,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -137965,17 +86523,23 @@
      ]
     },
     {
-     "label": "l54.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l44.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.54.linear_attn.in_proj_qkvz.weight"
+       "buf": "model.layers.44.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.44.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
@@ -137983,367 +86547,398 @@
       {
        "i32": 16384
       },
+      {
+       "i32": 96
+      },
       {
        "i32": 5120
       }
      ]
     },
     {
-     "label": "l54.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l44.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "x"
+       "buf": "qkvz"
       },
       {
-       "buf": "model.layers.54.linear_attn.in_proj_ba.weight"
+       "buf": "model.layers.44.linear_attn.conv1d.weight"
+      },
+      {
+       "state": "gdn"
+      },
+      {
+       "state": "gdn",
+       "offset": 204800
+      },
+      {
+       "buf": "gdn.spec_line_index",
+       "offset": 136
+      },
+      {
+       "buf": "num_accepted_tokens"
+      },
+      {
+       "buf": "cu_seqlens_q"
+      },
+      {
+       "buf": "gdn.spec_slots",
+       "offset": 1056
       },
       {
        "buf": "ba"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.44.linear_attn.A_log"
       },
       {
-       "i32": 96
+       "buf": "model.layers.44.linear_attn.dt_bias"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l54.conv_update",
-     "kernel": "conv_update_spec",
-     "args": [
+       "buf": "core_attn_out"
+      },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.44.linear_attn.norm.weight"
       },
       {
-       "buf": "model.layers.54.linear_attn.conv1d.weight"
+       "buf": "gdn_q"
       },
       {
-       "state": "gdn"
+       "buf": "gdn_k"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 168
+       "buf": "gdn_v"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "g"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "beta"
       },
       {
-       "buf": "qkvz"
+       "buf": "a_c"
       },
       {
-       "i32": 1
+       "buf": "b_c"
       },
       {
-       "i32": 385
+       "buf": "z_c"
       },
       {
-       "i64": 16384
+       "f32": 0.0883883461356163
       },
       {
-       "i64": 16384
+       "f32": 9.999999974752427e-07
       },
       {
-       "i64": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 385
+      },
+      {
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l54.post_conv",
-     "kernel": "post_conv",
+     "label": "l44.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "core_attn_out"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.44.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.54.linear_attn.A_log"
+       "buf": "residual"
       },
       {
-       "buf": "model.layers.54.linear_attn.dt_bias"
+       "buf": "model.layers.44.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "gdn_q"
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_k"
+       "i32": 5120
       },
       {
-       "buf": "gdn_v"
+       "i32": 6144
       },
       {
-       "buf": "g"
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l44.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
+     "args": [
+      {
+       "buf": "act"
       },
       {
-       "buf": "beta"
+       "buf": "x"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.44.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 96
+       "sym": "tokens"
       },
       {
-       "i32": 96
+       "i32": 17408
       },
       {
-       "i32": 2048
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l44.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "i32": 2048
+       "buf": "act"
       },
       {
-       "i32": 6144
+       "buf": "model.layers.44.mlp.down_proj.weight"
+      },
+      {
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.45.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 5120
+      },
+      {
+       "i32": 17408
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l54.a_copy",
-     "kernel": "copy_rows",
+     "label": "l45.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "qkvz"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "ba"
       },
       {
-       "i32": 48
+       "buf": "x"
       },
       {
-       "i32": 96
+       "buf": "model.layers.45.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l54.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.45.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "ba"
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 16384
       },
       {
        "i32": 96
       },
       {
-       "i32": 48
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l54.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l45.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "model.layers.54.linear_attn.A_log"
+       "buf": "qkvz"
       },
       {
-       "buf": "a_c"
+       "buf": "model.layers.45.linear_attn.conv1d.weight"
       },
       {
-       "buf": "b_c"
+       "state": "gdn"
       },
       {
-       "buf": "model.layers.54.linear_attn.dt_bias"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 140
       },
       {
-       "f32": 20.0
+       "buf": "num_accepted_tokens"
       },
       {
-       "buf": "gdn_q"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "gdn_k"
+       "buf": "gdn.spec_slots",
+       "offset": 1088
       },
       {
-       "buf": "gdn_v"
+       "buf": "ba"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.45.linear_attn.A_log"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "model.layers.45.linear_attn.dt_bias"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "core_attn_out"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "model.layers.45.linear_attn.norm.weight"
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 1312
+       "buf": "gdn_q"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "gdn_k"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "gdn_v"
       },
       {
-       "i64": 1
+       "buf": "g"
       },
       {
-       "sym": "tokens"
+       "buf": "beta"
       },
       {
-       "i64": 0
+       "buf": "a_c"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l54.z_copy",
-     "kernel": "copy_rows",
-     "args": [
+       "buf": "b_c"
+      },
       {
        "buf": "z_c"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 6144
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 16384
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 385
+      },
+      {
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l54.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l45.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.54.linear_attn.norm.weight"
+       "buf": "model.layers.45.linear_attn.out_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.45.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l54.out_proj",
-     "kernel": "gemm",
+     "label": "l45.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.54.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.45.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l54.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l45.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.45.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.54.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.46.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -138352,7 +86947,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -138360,23 +86955,32 @@
      ]
     },
     {
-     "label": "l54.gate_up",
-     "kernel": "gemm",
+     "label": "l46.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.54.mlp.gate_up_proj.weight"
+       "buf": "model.layers.46.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.46.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -138384,197 +86988,182 @@
      ]
     },
     {
-     "label": "l54.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l46.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.46.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 144
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l54.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "num_accepted_tokens"
       },
       {
-       "buf": "model.layers.54.mlp.down_proj.weight"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "y"
+       "buf": "gdn.spec_slots",
+       "offset": 1120
       },
       {
-       "sym": "tokens"
+       "buf": "ba"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.46.linear_attn.A_log"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l54.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "model.layers.46.linear_attn.dt_bias"
+      },
       {
-       "buf": "x"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.46.linear_attn.norm.weight"
       },
       {
-       "buf": "residual"
+       "buf": "gdn_q"
       },
       {
-       "buf": "model.layers.55.input_layernorm.weight_p1"
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
+       "buf": "gdn_v"
       },
       {
-       "sym": "tokens"
+       "buf": "g"
       },
       {
-       "i32": 5120
+       "buf": "beta"
       },
       {
-       "i32": 5120
+       "buf": "a_c"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l55.qkv_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "b_c"
+      },
       {
-       "buf": "x"
+       "buf": "z_c"
       },
       {
-       "buf": "model.layers.55.self_attn.qkv_proj.weight"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "qkv"
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 14336
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l55.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l46.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.55.self_attn.q_norm.weight_p1"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 256
+       "buf": "model.layers.46.linear_attn.out_proj.weight"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "buf": "residual"
       },
       {
-       "i32": 24
+       "buf": "model.layers.46.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 14336
+       "sym": "tokens"
       },
       {
-       "i32": 512
+       "i32": 5120
       },
       {
        "i32": 6144
       },
-      {
-       "i32": 256
-      },
       {
        "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l55.k_norm",
-     "kernel": "gemma_norm_khead",
+     "label": "l46.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "k_n"
+       "buf": "act"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
+       "buf": "x"
       },
       {
-       "buf": "model.layers.55.self_attn.k_norm.weight_p1"
+       "buf": "model.layers.46.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 256
+       "sym": "tokens"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
+       "i32": 17408
       },
       {
-       "i32": 4
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l46.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "i32": 14336
+       "buf": "act"
       },
       {
-       "i32": 256
+       "buf": "model.layers.46.mlp.down_proj.weight"
       },
       {
-       "i32": 1024
+       "buf": "residual"
       },
       {
-       "i32": 256
+       "buf": "model.layers.47.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -138582,92 +87171,104 @@
      ]
     },
     {
-     "label": "l55.rope",
-     "kernel": "mrope",
+     "label": "l47.qkv_proj",
+     "kernel": "gemm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
+       "buf": "x"
       },
       {
-       "buf": "cos_g"
+       "buf": "model.layers.47.self_attn.qkv_proj.weight"
       },
       {
-       "buf": "sin_g"
+       "buf": "qkv"
       },
       {
-       "i32": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l55.kv_write",
-     "kernel": "reshape_and_cache",
+     "label": "l47.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "k_n"
+       "buf": "q_n"
       },
       {
-       "buf": "qkv",
-       "offset": 26624
+       "buf": "k_n"
       },
       {
-       "state": "kv",
-       "offset": 41746432
+       "buf": "qkv"
       },
       {
-       "state": "kv",
-       "offset": 41746944
+       "buf": "model.layers.47.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "slot_mapping"
+       "buf": "model.layers.47.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "kv_scales"
+       "buf": "cos_g"
       },
       {
-       "buf": "kv_scales",
-       "offset": 4
+       "buf": "sin_g"
       },
       {
-       "i64": 1024
+       "state": "kv",
+       "offset": 35323904
       },
       {
-       "i64": 14336
+       "state": "kv",
+       "offset": 35324416
       },
       {
-       "i64": 25690112
+       "buf": "slot_mapping"
       },
       {
-       "i64": 512
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 0
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i64": 0
+       "i64": 25690112
       },
       {
        "i64": 2048
       },
       {
-       "i64": 0
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l55.attn",
+     "label": "l47.attn",
      "kernel": "attn_prefill",
      "args": [
       {
@@ -138678,11 +87279,11 @@
       },
       {
        "state": "kv",
-       "offset": 41746432
+       "offset": 35323904
       },
       {
        "state": "kv",
-       "offset": 41746944
+       "offset": 35324416
       },
       {
        "buf": "block_table"
@@ -138760,11 +87361,11 @@
      ]
     },
     {
-     "label": "l55.gate",
-     "kernel": "sigmoid_mul",
+     "label": "l47.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "gated"
+       "buf": "x"
       },
       {
        "buf": "attn_out"
@@ -138774,7 +87375,22 @@
        "offset": 512
       },
       {
-       "i32": 24
+       "buf": "model.layers.47.self_attn.o_proj.weight"
+      },
+      {
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.47.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 6144
       },
       {
        "i32": 256
@@ -138784,51 +87400,54 @@
       },
       {
        "i32": 512
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l55.o_proj",
-     "kernel": "gemm",
+     "label": "l47.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "gated"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.55.self_attn.o_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.47.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l55.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l47.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.47.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.55.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.48.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -138837,7 +87456,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -138845,23 +87464,23 @@
      ]
     },
     {
-     "label": "l55.gate_up",
-     "kernel": "gemm",
+     "label": "l47.fc_tap",
+     "kernel": "gemm_acc",
      "args": [
       {
-       "buf": "x"
+       "buf": "residual"
       },
       {
-       "buf": "model.layers.55.mlp.gate_up_proj.weight"
+       "buf": "draft.fc.3.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "fc_out"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 5120
       },
       {
        "i32": 5120
@@ -138869,128 +87488,182 @@
      ]
     },
     {
-     "label": "l55.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l48.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "ba"
       },
       {
-       "i32": 17408
+       "buf": "x"
       },
       {
-       "i32": 0
+       "buf": "model.layers.48.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "f32": 1.0
+       "buf": "model.layers.48.linear_attn.in_proj_ba.weight"
       },
       {
-       "i32": 0
+       "sym": "tokens"
+      },
+      {
+       "i32": 16384
+      },
+      {
+       "i32": 96
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l55.down_proj",
-     "kernel": "gemm",
+     "label": "l48.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
+      },
+      {
+       "buf": "model.layers.48.linear_attn.conv1d.weight"
+      },
+      {
+       "state": "gdn"
+      },
+      {
+       "state": "gdn",
+       "offset": 204800
+      },
+      {
+       "buf": "gdn.spec_line_index",
+       "offset": 148
+      },
+      {
+       "buf": "num_accepted_tokens"
+      },
+      {
+       "buf": "cu_seqlens_q"
+      },
+      {
+       "buf": "gdn.spec_slots",
+       "offset": 1152
+      },
+      {
+       "buf": "ba"
+      },
+      {
+       "buf": "model.layers.48.linear_attn.A_log"
+      },
+      {
+       "buf": "model.layers.48.linear_attn.dt_bias"
+      },
+      {
+       "buf": "core_attn_out"
+      },
+      {
+       "buf": "model.layers.48.linear_attn.norm.weight"
       },
       {
-       "buf": "model.layers.55.mlp.down_proj.weight"
+       "buf": "gdn_q"
       },
       {
-       "buf": "y"
+       "buf": "gdn_k"
       },
       {
-       "sym": "tokens"
+       "buf": "gdn_v"
       },
       {
-       "i32": 5120
+       "buf": "g"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l55.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "beta"
+      },
       {
-       "buf": "x"
+       "buf": "a_c"
       },
       {
-       "buf": "y"
+       "buf": "b_c"
       },
       {
-       "buf": "residual"
+       "buf": "z_c"
       },
       {
-       "buf": "model.layers.56.input_layernorm.weight_p1"
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 5120
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
       },
       {
-       "f32": 9.999999974752427e-07
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l56.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l48.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.56.linear_attn.in_proj_qkvz.weight"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.48.linear_attn.out_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.48.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l56.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l48.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.56.linear_attn.in_proj_ba.weight"
+       "buf": "x"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.48.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 96
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -138998,85 +87671,59 @@
      ]
     },
     {
-     "label": "l56.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l48.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.56.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 172
-      },
-      {
-       "buf": "num_accepted_tokens"
+       "buf": "x"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "act"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.48.mlp.down_proj.weight"
       },
       {
-       "i32": 1
+       "buf": "residual"
       },
       {
-       "i32": 385
+       "buf": "model.layers.49.input_layernorm.weight_p1"
       },
       {
-       "i64": 16384
+       "sym": "tokens"
       },
       {
-       "i64": 16384
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 17408
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l56.post_conv",
-     "kernel": "post_conv",
+     "label": "l49.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
        "buf": "qkvz"
       },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
       {
        "buf": "ba"
       },
       {
-       "buf": "model.layers.56.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.56.linear_attn.dt_bias"
-      },
-      {
-       "buf": "gdn_q"
-      },
-      {
-       "buf": "gdn_k"
+       "buf": "x"
       },
       {
-       "buf": "gdn_v"
+       "buf": "model.layers.49.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "g"
+       "buf": "model.layers.49.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "beta"
+       "sym": "tokens"
       },
       {
        "i32": 16384
@@ -139085,92 +87732,55 @@
        "i32": 96
       },
       {
-       "i32": 96
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 2048
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l56.a_copy",
-     "kernel": "copy_rows",
+     "label": "l49.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "a_c"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
+       "buf": "qkvz"
       },
       {
-       "i32": 48
+       "buf": "model.layers.49.linear_attn.conv1d.weight"
       },
       {
-       "i32": 96
+       "state": "gdn"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l56.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "buf": "ba"
+       "buf": "gdn.spec_line_index",
+       "offset": 152
       },
       {
-       "i32": 48
+       "buf": "num_accepted_tokens"
       },
       {
-       "i32": 96
+       "buf": "cu_seqlens_q"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l56.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.56.linear_attn.A_log"
+       "buf": "gdn.spec_slots",
+       "offset": 1184
       },
       {
-       "buf": "a_c"
+       "buf": "ba"
       },
       {
-       "buf": "b_c"
+       "buf": "model.layers.49.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.56.linear_attn.dt_bias"
+       "buf": "model.layers.49.linear_attn.dt_bias"
       },
       {
-       "f32": 1.0
+       "buf": "core_attn_out"
       },
       {
-       "f32": 20.0
+       "buf": "model.layers.49.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -139182,153 +87792,118 @@
        "buf": "gdn_v"
       },
       {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "g"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "beta"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "a_c"
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 1344
+       "buf": "b_c"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "z_c"
       },
       {
        "f32": 0.0883883461356163
       },
       {
-       "i64": 1
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l56.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
+       "i32": 385
       },
       {
-       "i32": 6144
+       "i32": 1703936
       },
       {
-       "i32": 16384
+       "i32": 10
       },
       {
-       "i32": 6144
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l56.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l49.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.56.linear_attn.norm.weight"
+       "buf": "model.layers.49.linear_attn.out_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.49.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l56.out_proj",
-     "kernel": "gemm",
+     "label": "l49.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.56.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.49.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l56.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l49.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.49.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.56.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.50.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -139337,7 +87912,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -139345,23 +87920,32 @@
      ]
     },
     {
-     "label": "l56.gate_up",
-     "kernel": "gemm",
+     "label": "l50.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.56.mlp.gate_up_proj.weight"
+       "buf": "model.layers.50.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.50.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -139369,347 +87953,369 @@
      ]
     },
     {
-     "label": "l56.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l50.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.50.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 156
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l56.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "num_accepted_tokens"
       },
       {
-       "buf": "model.layers.56.mlp.down_proj.weight"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "y"
+       "buf": "gdn.spec_slots",
+       "offset": 1216
       },
       {
-       "sym": "tokens"
+       "buf": "ba"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.50.linear_attn.A_log"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l56.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "model.layers.50.linear_attn.dt_bias"
+      },
       {
-       "buf": "x"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.50.linear_attn.norm.weight"
       },
       {
-       "buf": "residual"
+       "buf": "gdn_q"
       },
       {
-       "buf": "model.layers.57.input_layernorm.weight_p1"
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
+       "buf": "gdn_v"
       },
       {
-       "sym": "tokens"
+       "buf": "g"
       },
       {
-       "i32": 5120
+       "buf": "beta"
       },
       {
-       "i32": 5120
+       "buf": "a_c"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l57.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
+       "buf": "b_c"
+      },
       {
-       "buf": "x"
+       "buf": "z_c"
       },
       {
-       "buf": "model.layers.57.linear_attn.in_proj_qkvz.weight"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "qkvz"
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l57.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l50.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.57.linear_attn.in_proj_ba.weight"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.50.linear_attn.out_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 96
+       "buf": "model.layers.50.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l57.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l50.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.57.linear_attn.conv1d.weight"
+       "buf": "x"
       },
       {
-       "state": "gdn"
+       "buf": "model.layers.50.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 176
+       "sym": "tokens"
       },
       {
-       "buf": "num_accepted_tokens"
+       "i32": 17408
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l50.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "buf": "qkvz"
+       "buf": "act"
       },
       {
-       "i32": 1
+       "buf": "model.layers.50.mlp.down_proj.weight"
       },
       {
-       "i32": 385
+       "buf": "residual"
       },
       {
-       "i64": 16384
+       "buf": "model.layers.51.input_layernorm.weight_p1"
       },
       {
-       "i64": 16384
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l57.post_conv",
-     "kernel": "post_conv",
+     "label": "l51.qkv_proj",
+     "kernel": "gemm",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.51.self_attn.qkv_proj.weight"
       },
       {
-       "buf": "ba"
+       "buf": "qkv"
       },
       {
-       "buf": "model.layers.57.linear_attn.A_log"
+       "sym": "tokens"
       },
       {
-       "buf": "model.layers.57.linear_attn.dt_bias"
+       "i32": 14336
       },
       {
-       "buf": "gdn_q"
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l51.prep",
+     "kernel": "attn_prep",
+     "args": [
       {
-       "buf": "gdn_k"
+       "buf": "q_n"
       },
       {
-       "buf": "gdn_v"
+       "buf": "k_n"
       },
       {
-       "buf": "g"
+       "buf": "qkv"
       },
       {
-       "buf": "beta"
+       "buf": "model.layers.51.self_attn.q_norm.weight_p1"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.51.self_attn.k_norm.weight_p1"
       },
       {
-       "i32": 96
+       "buf": "cos_g"
       },
       {
-       "i32": 96
+       "buf": "sin_g"
       },
       {
-       "i32": 2048
+       "state": "kv",
+       "offset": 38535168
       },
       {
-       "i32": 2048
+       "state": "kv",
+       "offset": 38535680
       },
       {
-       "i32": 6144
+       "buf": "slot_mapping"
       },
       {
-       "sym": "tokens"
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 0
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l57.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
+       "i64": 25690112
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "i64": 2048
       },
       {
-       "i32": 48
+       "i64": 512
       },
       {
-       "i32": 96
+       "i32": 784
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l57.b_copy",
-     "kernel": "copy_rows",
+     "label": "l51.attn",
+     "kernel": "attn_prefill",
      "args": [
       {
-       "buf": "b_c"
+       "buf": "attn_out"
       },
       {
-       "buf": "ba"
+       "buf": "q_n"
       },
       {
-       "i32": 48
+       "state": "kv",
+       "offset": 38535168
       },
       {
-       "i32": 96
+       "state": "kv",
+       "offset": 38535680
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l57.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
+       "buf": "block_table"
+      },
       {
-       "buf": "model.layers.57.linear_attn.A_log"
+       "buf": "seq_lens"
       },
       {
-       "buf": "a_c"
+       "f32": 0.0625
       },
       {
-       "buf": "b_c"
+       "buf": "kv_scales"
       },
       {
-       "buf": "model.layers.57.linear_attn.dt_bias"
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
        "f32": 1.0
       },
       {
-       "f32": 20.0
+       "f32": 0.0
       },
       {
-       "buf": "gdn_q"
+       "i64": 8
       },
       {
-       "buf": "gdn_k"
+       "i64": 6144
       },
       {
-       "buf": "gdn_v"
+       "i64": 256
       },
       {
-       "buf": "core_attn_out"
+       "i64": 6144
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "i64": 256
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "i64": 0
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "seq_lens"
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 1376
+       "i64": 25690112
       },
       {
-       "buf": "num_accepted_tokens"
+       "i64": 2048
       },
       {
-       "f32": 0.0883883461356163
+       "i64": 512
+      },
+      {
+       "i64": 25690112
       },
       {
-       "i64": 1
+       "i64": 2048
       },
       {
-       "sym": "tokens"
+       "i64": 512
+      },
+      {
+       "buf": "cu_seqlens_q"
+      },
+      {
+       "i32": 1
       },
       {
        "i64": 0
@@ -139720,115 +88326,93 @@
      ]
     },
     {
-     "label": "l57.z_copy",
-     "kernel": "copy_rows",
+     "label": "l51.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "x"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "attn_out"
       },
       {
-       "i32": 6144
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "i32": 16384
+       "buf": "model.layers.51.self_attn.o_proj.weight"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l57.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
+       "buf": "residual"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.51.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "model.layers.57.linear_attn.norm.weight"
+       "sym": "tokens"
       },
       {
-       "buf": "z_c"
+       "i32": 5120
       },
       {
-       "i32": 128
+       "i32": 6144
       },
       {
-       "i32": 128
+       "i32": 256
       },
       {
-       "i32": 128
+       "i32": 14336
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 512
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l57.out_proj",
-     "kernel": "gemm",
+     "label": "l51.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.57.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.51.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l57.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l51.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.51.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.57.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.52.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -139837,7 +88421,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -139845,23 +88429,32 @@
      ]
     },
     {
-     "label": "l57.gate_up",
-     "kernel": "gemm",
+     "label": "l52.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.57.mlp.gate_up_proj.weight"
+       "buf": "model.layers.52.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.52.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -139869,293 +88462,281 @@
      ]
     },
     {
-     "label": "l57.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l52.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.52.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 160
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l57.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "num_accepted_tokens"
       },
       {
-       "buf": "model.layers.57.mlp.down_proj.weight"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "y"
+       "buf": "gdn.spec_slots",
+       "offset": 1248
       },
       {
-       "sym": "tokens"
+       "buf": "ba"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.52.linear_attn.A_log"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l57.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "model.layers.52.linear_attn.dt_bias"
+      },
       {
-       "buf": "x"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.52.linear_attn.norm.weight"
       },
       {
-       "buf": "residual"
+       "buf": "gdn_q"
       },
       {
-       "buf": "model.layers.58.input_layernorm.weight_p1"
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
+       "buf": "gdn_v"
       },
       {
-       "sym": "tokens"
+       "buf": "g"
       },
       {
-       "i32": 5120
+       "buf": "beta"
       },
       {
-       "i32": 5120
+       "buf": "a_c"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l58.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
+       "buf": "b_c"
+      },
       {
-       "buf": "x"
+       "buf": "z_c"
       },
       {
-       "buf": "model.layers.58.linear_attn.in_proj_qkvz.weight"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "qkvz"
+       "f32": 9.999999974752427e-07
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 385
       },
       {
-       "i32": 5120
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l58.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l52.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.58.linear_attn.in_proj_ba.weight"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.52.linear_attn.out_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 96
+       "buf": "model.layers.52.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l58.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l52.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.58.linear_attn.conv1d.weight"
+       "buf": "x"
       },
       {
-       "state": "gdn"
+       "buf": "model.layers.52.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 180
+       "sym": "tokens"
       },
       {
-       "buf": "num_accepted_tokens"
+       "i32": 17408
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l52.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "buf": "qkvz"
+       "buf": "act"
       },
       {
-       "i32": 1
+       "buf": "model.layers.52.mlp.down_proj.weight"
       },
       {
-       "i32": 385
+       "buf": "residual"
       },
       {
-       "i64": 16384
+       "buf": "model.layers.53.input_layernorm.weight_p1"
       },
       {
-       "i64": 16384
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l58.post_conv",
-     "kernel": "post_conv",
+     "label": "l53.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
        "buf": "qkvz"
       },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
       {
        "buf": "ba"
       },
       {
-       "buf": "model.layers.58.linear_attn.A_log"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.58.linear_attn.dt_bias"
+       "buf": "model.layers.53.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gdn_q"
+       "buf": "model.layers.53.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "gdn_k"
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_v"
+       "i32": 16384
       },
       {
-       "buf": "g"
+       "i32": 96
       },
       {
-       "buf": "beta"
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l53.gdn",
+     "kernel": "gdn_spec",
+     "args": [
       {
-       "i32": 16384
+       "buf": "qkvz"
       },
       {
-       "i32": 96
+       "buf": "model.layers.53.linear_attn.conv1d.weight"
       },
       {
-       "i32": 96
+       "state": "gdn"
       },
       {
-       "i32": 2048
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 2048
+       "buf": "gdn.spec_line_index",
+       "offset": 164
       },
       {
-       "i32": 6144
+       "buf": "num_accepted_tokens"
       },
       {
-       "sym": "tokens"
+       "buf": "cu_seqlens_q"
       },
       {
-       "i64": 0
+       "buf": "gdn.spec_slots",
+       "offset": 1280
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l58.a_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "a_c"
+       "buf": "ba"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.53.linear_attn.A_log"
       },
       {
-       "i32": 48
+       "buf": "model.layers.53.linear_attn.dt_bias"
       },
       {
-       "i32": 96
+       "buf": "core_attn_out"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l58.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.53.linear_attn.norm.weight"
       },
       {
-       "buf": "ba"
+       "buf": "gdn_q"
       },
       {
-       "i32": 48
+       "buf": "gdn_k"
       },
       {
-       "i32": 96
+       "buf": "gdn_v"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l58.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
+       "buf": "g"
+      },
       {
-       "buf": "model.layers.58.linear_attn.A_log"
+       "buf": "beta"
       },
       {
        "buf": "a_c"
@@ -140164,276 +88745,265 @@
        "buf": "b_c"
       },
       {
-       "buf": "model.layers.58.linear_attn.dt_bias"
+       "buf": "z_c"
       },
       {
-       "f32": 1.0
+       "f32": 0.0883883461356163
       },
       {
-       "f32": 20.0
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "gdn_q"
+       "sym": "tokens"
       },
       {
-       "buf": "gdn_k"
+       "i32": 385
       },
       {
-       "buf": "gdn_v"
+       "i32": 1703936
       },
       {
-       "buf": "core_attn_out"
+       "i32": 10
       },
       {
-       "state": "gdn",
-       "offset": 204800
-      },
+       "i32": 851968
+      }
+     ]
+    },
+    {
+     "label": "l53.out_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "x"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 1408
+       "buf": "model.layers.53.linear_attn.out_proj.weight"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "residual"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "model.layers.53.post_attention_layernorm.weight_p1"
       },
       {
-       "i64": 1
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 6144
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l58.z_copy",
-     "kernel": "copy_rows",
+     "label": "l53.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "act"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "x"
       },
       {
-       "i32": 6144
+       "buf": "model.layers.53.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 16384
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 17408
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l58.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l53.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.58.linear_attn.norm.weight"
+       "buf": "model.layers.53.mlp.down_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.54.input_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l58.out_proj",
-     "kernel": "gemm",
+     "label": "l54.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "qkvz"
       },
       {
-       "buf": "model.layers.58.linear_attn.out_proj.weight"
+       "buf": "ba"
       },
       {
-       "buf": "y"
+       "buf": "x"
+      },
+      {
+       "buf": "model.layers.54.linear_attn.in_proj_qkvz.weight"
+      },
+      {
+       "buf": "model.layers.54.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 16384
       },
       {
-       "i32": 6144
+       "i32": 96
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l58.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l54.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "x"
+       "buf": "qkvz"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.54.linear_attn.conv1d.weight"
       },
       {
-       "buf": "residual"
+       "state": "gdn"
       },
       {
-       "buf": "model.layers.58.post_attention_layernorm.weight_p1"
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 5120
+       "buf": "gdn.spec_line_index",
+       "offset": 168
       },
       {
-       "sym": "tokens"
+       "buf": "num_accepted_tokens"
       },
       {
-       "i32": 5120
+       "buf": "cu_seqlens_q"
       },
       {
-       "i32": 5120
+       "buf": "gdn.spec_slots",
+       "offset": 1312
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l58.gate_up",
-     "kernel": "gemm",
-     "args": [
+       "buf": "ba"
+      },
+      {
+       "buf": "model.layers.54.linear_attn.A_log"
+      },
       {
-       "buf": "x"
+       "buf": "model.layers.54.linear_attn.dt_bias"
       },
       {
-       "buf": "model.layers.58.mlp.gate_up_proj.weight"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.54.linear_attn.norm.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "gdn_q"
       },
       {
-       "i32": 34816
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l58.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
+       "buf": "gdn_v"
+      },
       {
-       "buf": "act"
+       "buf": "g"
       },
       {
-       "buf": "gate_up"
+       "buf": "beta"
       },
       {
-       "i32": 17408
+       "buf": "a_c"
       },
       {
-       "i32": 0
+       "buf": "b_c"
       },
       {
-       "f32": 1.0
+       "buf": "z_c"
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l58.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "f32": 0.0883883461356163
+      },
       {
-       "buf": "act"
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "model.layers.58.mlp.down_proj.weight"
+       "sym": "tokens"
       },
       {
-       "buf": "y"
+       "i32": 385
       },
       {
-       "sym": "tokens"
+       "i32": 1703936
       },
       {
-       "i32": 5120
+       "i32": 10
       },
       {
-       "i32": 17408
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l58.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l54.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.54.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.59.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.54.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -140442,7 +89012,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -140450,23 +89020,23 @@
      ]
     },
     {
-     "label": "l59.qkv_proj",
-     "kernel": "gemm",
+     "label": "l54.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.59.self_attn.qkv_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkv"
+       "buf": "model.layers.54.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 14336
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -140474,45 +89044,32 @@
      ]
     },
     {
-     "label": "l59.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l54.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.59.self_attn.q_norm.weight_p1"
+       "buf": "x"
       },
       {
-       "i32": 256
+       "buf": "act"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "buf": "model.layers.54.mlp.down_proj.weight"
       },
       {
-       "i32": 24
+       "buf": "residual"
       },
       {
-       "i32": 14336
+       "buf": "model.layers.55.input_layernorm.weight_p1"
       },
       {
-       "i32": 512
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 5120
       },
       {
-       "i32": 256
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -140520,55 +89077,32 @@
      ]
     },
     {
-     "label": "l59.k_norm",
-     "kernel": "gemma_norm_khead",
+     "label": "l55.qkv_proj",
+     "kernel": "gemm",
      "args": [
       {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.59.self_attn.k_norm.weight_p1"
+       "buf": "x"
       },
       {
-       "i32": 256
+       "buf": "model.layers.55.self_attn.qkv_proj.weight"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
+       "buf": "qkv"
       },
       {
-       "i32": 4
+       "sym": "tokens"
       },
       {
        "i32": 14336
       },
       {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l59.rope",
-     "kernel": "mrope",
+     "label": "l55.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
@@ -140577,82 +89111,70 @@
        "buf": "k_n"
       },
       {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
+       "buf": "qkv"
       },
       {
-       "i32": 0
+       "buf": "model.layers.55.self_attn.q_norm.weight_p1"
       },
       {
-       "i64": 0
+       "buf": "model.layers.55.self_attn.k_norm.weight_p1"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l59.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
+       "buf": "cos_g"
       },
       {
-       "buf": "qkv",
-       "offset": 26624
+       "buf": "sin_g"
       },
       {
        "state": "kv",
-       "offset": 44957696
+       "offset": 41746432
       },
       {
        "state": "kv",
-       "offset": 44958208
+       "offset": 41746944
       },
       {
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l59.attn",
+     "label": "l55.attn",
      "kernel": "attn_prefill",
      "args": [
       {
@@ -140663,11 +89185,11 @@
       },
       {
        "state": "kv",
-       "offset": 44957696
+       "offset": 41746432
       },
       {
        "state": "kv",
-       "offset": 44958208
+       "offset": 41746944
       },
       {
        "buf": "block_table"
@@ -140745,11 +89267,11 @@
      ]
     },
     {
-     "label": "l59.gate",
-     "kernel": "sigmoid_mul",
+     "label": "l55.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "gated"
+       "buf": "x"
       },
       {
        "buf": "attn_out"
@@ -140759,7 +89281,22 @@
        "offset": 512
       },
       {
-       "i32": 24
+       "buf": "model.layers.55.self_attn.o_proj.weight"
+      },
+      {
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.55.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 6144
       },
       {
        "i32": 256
@@ -140769,51 +89306,54 @@
       },
       {
        "i32": 512
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l59.o_proj",
-     "kernel": "gemm",
+     "label": "l55.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "gated"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.59.self_attn.o_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.55.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l59.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l55.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.55.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.59.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.56.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -140822,7 +89362,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -140830,23 +89370,32 @@
      ]
     },
     {
-     "label": "l59.gate_up",
-     "kernel": "gemm",
+     "label": "l56.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.59.mlp.gate_up_proj.weight"
+       "buf": "model.layers.56.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.56.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -140854,466 +89403,389 @@
      ]
     },
     {
-     "label": "l59.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l56.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
+       "buf": "qkvz"
       },
       {
-       "i32": 17408
+       "buf": "model.layers.56.linear_attn.conv1d.weight"
       },
       {
-       "i32": 0
+       "state": "gdn"
       },
       {
-       "f32": 1.0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l59.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "gdn.spec_line_index",
+       "offset": 172
       },
       {
-       "buf": "model.layers.59.mlp.down_proj.weight"
+       "buf": "num_accepted_tokens"
       },
       {
-       "buf": "y"
+       "buf": "cu_seqlens_q"
       },
       {
-       "sym": "tokens"
+       "buf": "gdn.spec_slots",
+       "offset": 1344
       },
       {
-       "i32": 5120
+       "buf": "ba"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l59.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
+       "buf": "model.layers.56.linear_attn.A_log"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.56.linear_attn.dt_bias"
       },
       {
-       "buf": "residual"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.60.input_layernorm.weight_p1"
+       "buf": "model.layers.56.linear_attn.norm.weight"
       },
       {
-       "i32": 5120
+       "buf": "gdn_q"
       },
       {
-       "sym": "tokens"
+       "buf": "gdn_k"
       },
       {
-       "i32": 5120
+       "buf": "gdn_v"
       },
       {
-       "i32": 5120
+       "buf": "g"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l60.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
+       "buf": "beta"
       },
       {
-       "buf": "model.layers.60.linear_attn.in_proj_qkvz.weight"
+       "buf": "a_c"
       },
       {
-       "buf": "qkvz"
+       "buf": "b_c"
       },
       {
-       "sym": "tokens"
+       "buf": "z_c"
       },
       {
-       "i32": 16384
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l60.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "model.layers.60.linear_attn.in_proj_ba.weight"
+       "sym": "tokens"
       },
       {
-       "buf": "ba"
+       "i32": 385
       },
       {
-       "sym": "tokens"
+       "i32": 1703936
       },
       {
-       "i32": 96
+       "i32": 10
       },
       {
-       "i32": 5120
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l60.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l56.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.60.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.spec_line_index",
-       "offset": 184
-      },
-      {
-       "buf": "num_accepted_tokens"
+       "buf": "x"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.56.linear_attn.out_proj.weight"
       },
       {
-       "i32": 1
+       "buf": "residual"
       },
       {
-       "i32": 385
+       "buf": "model.layers.56.post_attention_layernorm.weight_p1"
       },
       {
-       "i64": 16384
+       "sym": "tokens"
       },
       {
-       "i64": 16384
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 6144
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l60.post_conv",
-     "kernel": "post_conv",
+     "label": "l56.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "act"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "x"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.56.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "model.layers.60.linear_attn.A_log"
+       "sym": "tokens"
       },
       {
-       "buf": "model.layers.60.linear_attn.dt_bias"
+       "i32": 17408
       },
       {
-       "buf": "gdn_q"
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l56.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "buf": "gdn_k"
+       "buf": "x"
       },
       {
-       "buf": "gdn_v"
+       "buf": "act"
       },
       {
-       "buf": "g"
+       "buf": "model.layers.56.mlp.down_proj.weight"
       },
       {
-       "buf": "beta"
+       "buf": "residual"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.57.input_layernorm.weight_p1"
       },
       {
-       "i32": 96
+       "sym": "tokens"
       },
       {
-       "i32": 96
+       "i32": 5120
       },
       {
-       "i32": 2048
+       "i32": 17408
       },
       {
-       "i32": 2048
-      },
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l57.in_proj",
+     "kernel": "gemm8_dual",
+     "args": [
       {
-       "i32": 6144
+       "buf": "qkvz"
       },
       {
-       "sym": "tokens"
+       "buf": "ba"
       },
       {
-       "i64": 0
+       "buf": "x"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l60.a_copy",
-     "kernel": "copy_rows",
-     "args": [
+       "buf": "model.layers.57.linear_attn.in_proj_qkvz.weight"
+      },
       {
-       "buf": "a_c"
+       "buf": "model.layers.57.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 16384
       },
       {
        "i32": 96
       },
       {
-       "i32": 48
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l60.b_copy",
-     "kernel": "copy_rows",
+     "label": "l57.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "b_c"
+       "buf": "qkvz"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.57.linear_attn.conv1d.weight"
       },
       {
-       "i32": 48
+       "state": "gdn"
       },
       {
-       "i32": 96
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l60.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.60.linear_attn.A_log"
+       "buf": "gdn.spec_line_index",
+       "offset": 176
       },
       {
-       "buf": "a_c"
+       "buf": "num_accepted_tokens"
       },
       {
-       "buf": "b_c"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "model.layers.60.linear_attn.dt_bias"
+       "buf": "gdn.spec_slots",
+       "offset": 1376
       },
       {
-       "f32": 1.0
+       "buf": "ba"
       },
       {
-       "f32": 20.0
+       "buf": "model.layers.57.linear_attn.A_log"
       },
       {
-       "buf": "gdn_q"
+       "buf": "model.layers.57.linear_attn.dt_bias"
       },
       {
-       "buf": "gdn_k"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "gdn_v"
+       "buf": "model.layers.57.linear_attn.norm.weight"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "gdn_q"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "gdn_k"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "gdn_v"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "g"
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 1440
+       "buf": "beta"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "a_c"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "b_c"
       },
       {
-       "i64": 1
+       "buf": "z_c"
       },
       {
-       "sym": "tokens"
+       "f32": 0.0883883461356163
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l60.z_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "z_c"
+       "sym": "tokens"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "i32": 385
       },
       {
-       "i32": 6144
+       "i32": 1703936
       },
       {
-       "i32": 16384
+       "i32": 10
       },
       {
-       "i32": 6144
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l60.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l57.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.60.linear_attn.norm.weight"
+       "buf": "model.layers.57.linear_attn.out_proj.weight"
       },
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.57.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l60.out_proj",
-     "kernel": "gemm",
+     "label": "l57.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.60.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.57.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l60.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l57.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.57.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.60.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.58.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -141322,7 +89794,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -141330,23 +89802,32 @@
      ]
     },
     {
-     "label": "l60.gate_up",
-     "kernel": "gemm",
+     "label": "l58.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.60.mlp.gate_up_proj.weight"
+       "buf": "model.layers.58.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.58.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -141354,128 +89835,149 @@
      ]
     },
     {
-     "label": "l60.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l58.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.58.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "f32": 1.0
+       "buf": "gdn.spec_line_index",
+       "offset": 180
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l60.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "num_accepted_tokens"
+      },
       {
-       "buf": "act"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "model.layers.60.mlp.down_proj.weight"
+       "buf": "gdn.spec_slots",
+       "offset": 1408
       },
       {
-       "buf": "y"
+       "buf": "ba"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.58.linear_attn.A_log"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.58.linear_attn.dt_bias"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l60.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "core_attn_out"
+      },
       {
-       "buf": "x"
+       "buf": "model.layers.58.linear_attn.norm.weight"
       },
       {
-       "buf": "y"
+       "buf": "gdn_q"
       },
       {
-       "buf": "residual"
+       "buf": "gdn_k"
       },
       {
-       "buf": "model.layers.61.input_layernorm.weight_p1"
+       "buf": "gdn_v"
       },
       {
-       "i32": 5120
+       "buf": "g"
       },
       {
-       "sym": "tokens"
+       "buf": "beta"
       },
       {
-       "i32": 5120
+       "buf": "a_c"
       },
       {
-       "i32": 5120
+       "buf": "b_c"
+      },
+      {
+       "buf": "z_c"
+      },
+      {
+       "f32": 0.0883883461356163
       },
       {
        "f32": 9.999999974752427e-07
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 385
+      },
+      {
+       "i32": 1703936
+      },
+      {
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l61.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l58.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.61.linear_attn.in_proj_qkvz.weight"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.58.linear_attn.out_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.58.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l61.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l58.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.61.linear_attn.in_proj_ba.weight"
+       "buf": "x"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.58.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 96
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -141483,218 +89985,219 @@
      ]
     },
     {
-     "label": "l61.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l58.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.61.linear_attn.conv1d.weight"
+       "buf": "act"
       },
       {
-       "state": "gdn"
+       "buf": "model.layers.58.mlp.down_proj.weight"
       },
       {
-       "buf": "gdn.spec_line_index",
-       "offset": 188
+       "buf": "residual"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "model.layers.59.input_layernorm.weight_p1"
       },
       {
-       "buf": "cu_seqlens_q"
+       "sym": "tokens"
       },
       {
-       "buf": "qkvz"
+       "i32": 5120
       },
       {
-       "i32": 1
+       "i32": 17408
       },
       {
-       "i32": 385
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l59.qkv_proj",
+     "kernel": "gemm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "i64": 16384
+       "buf": "model.layers.59.self_attn.qkv_proj.weight"
       },
       {
-       "i64": 16384
+       "buf": "qkv"
       },
       {
-       "i64": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l61.post_conv",
-     "kernel": "post_conv",
+     "label": "l59.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
+       "buf": "q_n"
       },
       {
-       "buf": "model.layers.61.linear_attn.A_log"
+       "buf": "k_n"
       },
       {
-       "buf": "model.layers.61.linear_attn.dt_bias"
+       "buf": "qkv"
       },
       {
-       "buf": "gdn_q"
+       "buf": "model.layers.59.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "gdn_k"
+       "buf": "model.layers.59.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "gdn_v"
+       "buf": "cos_g"
       },
       {
-       "buf": "g"
+       "buf": "sin_g"
       },
       {
-       "buf": "beta"
+       "state": "kv",
+       "offset": 44957696
       },
       {
-       "i32": 16384
+       "state": "kv",
+       "offset": 44958208
       },
       {
-       "i32": 96
+       "buf": "slot_mapping"
       },
       {
-       "i32": 96
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i32": 2048
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i32": 2048
+       "i64": 25690112
       },
       {
-       "i32": 6144
+       "i64": 2048
       },
       {
-       "sym": "tokens"
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l61.a_copy",
-     "kernel": "copy_rows",
+     "label": "l59.attn",
+     "kernel": "attn_prefill",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "attn_out"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "q_n"
       },
       {
-       "i32": 48
+       "state": "kv",
+       "offset": 44957696
       },
       {
-       "i32": 96
+       "state": "kv",
+       "offset": 44958208
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l61.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "block_table"
       },
       {
-       "buf": "ba"
+       "buf": "seq_lens"
       },
       {
-       "i32": 48
+       "f32": 0.0625
       },
       {
-       "i32": 96
+       "buf": "kv_scales"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l61.recurrent",
-     "kernel": "recurrent_spec",
-     "args": [
-      {
-       "buf": "model.layers.61.linear_attn.A_log"
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "buf": "a_c"
+       "f32": 1.0
       },
       {
-       "buf": "b_c"
+       "f32": 0.0
       },
       {
-       "buf": "model.layers.61.linear_attn.dt_bias"
+       "i64": 8
       },
       {
-       "f32": 1.0
+       "i64": 6144
       },
       {
-       "f32": 20.0
+       "i64": 256
       },
       {
-       "buf": "gdn_q"
+       "i64": 6144
       },
       {
-       "buf": "gdn_k"
+       "i64": 256
       },
       {
-       "buf": "gdn_v"
+       "i64": 0
       },
       {
-       "buf": "core_attn_out"
+       "buf": "seq_lens"
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "i64": 25690112
       },
       {
-       "state": "gdn",
-       "offset": 204800
+       "i64": 2048
       },
       {
-       "buf": "cu_seqlens_q"
+       "i64": 512
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 1472
+       "i64": 25690112
       },
       {
-       "buf": "num_accepted_tokens"
+       "i64": 2048
       },
       {
-       "f32": 0.0883883461356163
+       "i64": 512
       },
       {
-       "i64": 1
+       "buf": "cu_seqlens_q"
       },
       {
-       "sym": "tokens"
+       "i32": 1
       },
       {
        "i64": 0
@@ -141705,115 +90208,93 @@
      ]
     },
     {
-     "label": "l61.z_copy",
-     "kernel": "copy_rows",
+     "label": "l59.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "x"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "attn_out"
       },
       {
-       "i32": 6144
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "i32": 16384
+       "buf": "model.layers.59.self_attn.o_proj.weight"
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l61.gated_norm",
-     "kernel": "gated_norm",
-     "args": [
-      {
-       "buf": "core_attn_out"
+       "buf": "residual"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.59.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "model.layers.61.linear_attn.norm.weight"
+       "sym": "tokens"
       },
       {
-       "buf": "z_c"
+       "i32": 5120
       },
       {
-       "i32": 128
+       "i32": 6144
       },
       {
-       "i32": 128
+       "i32": 256
       },
       {
-       "i32": 128
+       "i32": 14336
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "i32": 512
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l61.out_proj",
-     "kernel": "gemm",
+     "label": "l59.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.61.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.59.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l61.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l59.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.59.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.61.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.60.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -141822,7 +90303,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -141830,23 +90311,32 @@
      ]
     },
     {
-     "label": "l61.gate_up",
-     "kernel": "gemm",
+     "label": "l60.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.61.mlp.gate_up_proj.weight"
+       "buf": "model.layers.60.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.60.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -141854,71 +90344,116 @@
      ]
     },
     {
-     "label": "l61.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l60.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
+      },
+      {
+       "buf": "model.layers.60.linear_attn.conv1d.weight"
+      },
+      {
+       "state": "gdn"
+      },
+      {
+       "state": "gdn",
+       "offset": 204800
+      },
+      {
+       "buf": "gdn.spec_line_index",
+       "offset": 184
+      },
+      {
+       "buf": "num_accepted_tokens"
+      },
+      {
+       "buf": "cu_seqlens_q"
+      },
+      {
+       "buf": "gdn.spec_slots",
+       "offset": 1440
+      },
+      {
+       "buf": "ba"
+      },
+      {
+       "buf": "model.layers.60.linear_attn.A_log"
+      },
+      {
+       "buf": "model.layers.60.linear_attn.dt_bias"
+      },
+      {
+       "buf": "core_attn_out"
+      },
+      {
+       "buf": "model.layers.60.linear_attn.norm.weight"
+      },
+      {
+       "buf": "gdn_q"
+      },
+      {
+       "buf": "gdn_k"
+      },
+      {
+       "buf": "gdn_v"
+      },
+      {
+       "buf": "g"
       },
       {
-       "buf": "gate_up"
+       "buf": "beta"
       },
       {
-       "i32": 17408
+       "buf": "a_c"
       },
       {
-       "i32": 0
+       "buf": "b_c"
       },
       {
-       "f32": 1.0
+       "buf": "z_c"
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l61.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "f32": 0.0883883461356163
+      },
       {
-       "buf": "act"
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "model.layers.61.mlp.down_proj.weight"
+       "sym": "tokens"
       },
       {
-       "buf": "y"
+       "i32": 385
       },
       {
-       "sym": "tokens"
+       "i32": 1703936
       },
       {
-       "i32": 5120
+       "i32": 10
       },
       {
-       "i32": 17408
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l61.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l60.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.60.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.62.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.60.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -141927,7 +90462,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -141935,23 +90470,23 @@
      ]
     },
     {
-     "label": "l61.fc_tap",
-     "kernel": "gemm_acc",
+     "label": "l60.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "residual"
+       "buf": "act"
       },
       {
-       "buf": "draft.fc.4.weight"
+       "buf": "x"
       },
       {
-       "buf": "fc_out"
+       "buf": "model.layers.60.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -141959,45 +90494,63 @@
      ]
     },
     {
-     "label": "l62.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l60.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.62.linear_attn.in_proj_qkvz.weight"
+       "buf": "act"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.60.mlp.down_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.61.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l62.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l61.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.62.linear_attn.in_proj_ba.weight"
+       "buf": "model.layers.61.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.61.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
+      {
+       "i32": 16384
+      },
       {
        "i32": 96
       },
@@ -142007,21 +90560,25 @@
      ]
     },
     {
-     "label": "l62.conv_update",
-     "kernel": "conv_update_spec",
+     "label": "l61.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.62.linear_attn.conv1d.weight"
+       "buf": "model.layers.61.linear_attn.conv1d.weight"
       },
       {
        "state": "gdn"
       },
+      {
+       "state": "gdn",
+       "offset": 204800
+      },
       {
        "buf": "gdn.spec_line_index",
-       "offset": 192
+       "offset": 188
       },
       {
        "buf": "num_accepted_tokens"
@@ -142030,47 +90587,23 @@
        "buf": "cu_seqlens_q"
       },
       {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 385
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 16384
-      },
-      {
-       "i64": 0
+       "buf": "gdn.spec_slots",
+       "offset": 1472
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l62.post_conv",
-     "kernel": "post_conv",
-     "args": [
-      {
-       "buf": "qkvz"
+       "buf": "ba"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.61.linear_attn.A_log"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.61.linear_attn.dt_bias"
       },
       {
-       "buf": "model.layers.62.linear_attn.A_log"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.62.linear_attn.dt_bias"
+       "buf": "model.layers.61.linear_attn.norm.weight"
       },
       {
        "buf": "gdn_q"
@@ -142088,361 +90621,295 @@
        "buf": "beta"
       },
       {
-       "i32": 16384
+       "buf": "a_c"
       },
       {
-       "i32": 96
+       "buf": "b_c"
       },
       {
-       "i32": 96
+       "buf": "z_c"
       },
       {
-       "i32": 2048
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 2048
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 6144
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 385
       },
       {
-       "i64": 0
+       "i32": 1703936
       },
       {
-       "i64": 0
+       "i32": 10
+      },
+      {
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l62.a_copy",
-     "kernel": "copy_rows",
+     "label": "l61.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "a_c"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "core_attn_out"
       },
       {
-       "i32": 48
+       "buf": "model.layers.61.linear_attn.out_proj.weight"
       },
       {
-       "i32": 96
+       "buf": "residual"
       },
       {
-       "i32": 48
-      }
-     ]
-    },
-    {
-     "label": "l62.b_copy",
-     "kernel": "copy_rows",
-     "args": [
-      {
-       "buf": "b_c"
+       "buf": "model.layers.61.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "ba"
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 5120
       },
       {
-       "i32": 96
+       "i32": 6144
       },
       {
-       "i32": 48
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l62.recurrent",
-     "kernel": "recurrent_spec",
+     "label": "l61.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "model.layers.62.linear_attn.A_log"
-      },
-      {
-       "buf": "a_c"
-      },
-      {
-       "buf": "b_c"
-      },
-      {
-       "buf": "model.layers.62.linear_attn.dt_bias"
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 20.0
+       "buf": "act"
       },
       {
-       "buf": "gdn_q"
+       "buf": "x"
       },
       {
-       "buf": "gdn_k"
+       "buf": "model.layers.61.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "gdn_v"
+       "sym": "tokens"
       },
       {
-       "buf": "core_attn_out"
+       "i32": 17408
       },
       {
-       "state": "gdn",
-       "offset": 204800
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l61.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "state": "gdn",
-       "offset": 204800
+       "buf": "x"
       },
       {
-       "buf": "cu_seqlens_q"
+       "buf": "act"
       },
       {
-       "buf": "gdn.spec_slots",
-       "offset": 1504
+       "buf": "model.layers.61.mlp.down_proj.weight"
       },
       {
-       "buf": "num_accepted_tokens"
+       "buf": "residual"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "model.layers.62.input_layernorm.weight_p1"
       },
       {
-       "i64": 1
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 17408
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l62.z_copy",
-     "kernel": "copy_rows",
+     "label": "l61.fc_tap",
+     "kernel": "gemm_acc",
      "args": [
       {
-       "buf": "z_c"
+       "buf": "residual"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "draft.fc.4.weight"
       },
       {
-       "i32": 6144
+       "buf": "fc_out"
       },
       {
-       "i32": 16384
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 5120
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l62.gated_norm",
-     "kernel": "gated_norm",
+     "label": "l62.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.62.linear_attn.norm.weight"
+       "buf": "qkvz"
       },
       {
-       "buf": "z_c"
+       "buf": "ba"
       },
       {
-       "i32": 128
+       "buf": "x"
       },
       {
-       "i32": 128
+       "buf": "model.layers.62.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i32": 128
+       "buf": "model.layers.62.linear_attn.in_proj_ba.weight"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         48
-        ]
-       }
+       "sym": "tokens"
       },
       {
-       "f32": 9.999999974752427e-07
+       "i32": 16384
       },
       {
-       "i64": 0
+       "i32": 96
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l62.out_proj",
-     "kernel": "gemm",
+     "label": "l62.gdn",
+     "kernel": "gdn_spec",
      "args": [
       {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.62.linear_attn.out_proj.weight"
+       "buf": "qkvz"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.62.linear_attn.conv1d.weight"
       },
       {
-       "sym": "tokens"
+       "state": "gdn"
       },
       {
-       "i32": 5120
+       "state": "gdn",
+       "offset": 204800
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l62.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
+       "buf": "gdn.spec_line_index",
+       "offset": 192
       },
       {
-       "buf": "y"
+       "buf": "num_accepted_tokens"
       },
       {
-       "buf": "residual"
+       "buf": "cu_seqlens_q"
       },
       {
-       "buf": "model.layers.62.post_attention_layernorm.weight_p1"
+       "buf": "gdn.spec_slots",
+       "offset": 1504
       },
       {
-       "i32": 5120
+       "buf": "ba"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.62.linear_attn.A_log"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.62.linear_attn.dt_bias"
       },
       {
-       "i32": 5120
+       "buf": "core_attn_out"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l62.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
+       "buf": "model.layers.62.linear_attn.norm.weight"
       },
       {
-       "buf": "model.layers.62.mlp.gate_up_proj.weight"
+       "buf": "gdn_q"
       },
       {
-       "buf": "gate_up"
+       "buf": "gdn_k"
       },
       {
-       "sym": "tokens"
+       "buf": "gdn_v"
       },
       {
-       "i32": 34816
+       "buf": "g"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l62.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "beta"
       },
       {
-       "buf": "gate_up"
+       "buf": "a_c"
       },
       {
-       "i32": 17408
+       "buf": "b_c"
       },
       {
-       "i32": 0
+       "buf": "z_c"
       },
       {
-       "f32": 1.0
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l62.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "model.layers.62.mlp.down_proj.weight"
+       "sym": "tokens"
       },
       {
-       "buf": "y"
+       "i32": 385
       },
       {
-       "sym": "tokens"
+       "i32": 1703936
       },
       {
-       "i32": 5120
+       "i32": 10
       },
       {
-       "i32": 17408
+       "i32": 851968
       }
      ]
     },
     {
-     "label": "l62.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l62.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.62.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.63.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.62.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -142451,7 +90918,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -142459,23 +90926,23 @@
      ]
     },
     {
-     "label": "l63.qkv_proj",
-     "kernel": "gemm",
+     "label": "l62.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.63.self_attn.qkv_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkv"
+       "buf": "model.layers.62.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 14336
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -142483,45 +90950,32 @@
      ]
     },
     {
-     "label": "l63.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l62.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.63.self_attn.q_norm.weight_p1"
+       "buf": "x"
       },
       {
-       "i32": 256
+       "buf": "act"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "buf": "model.layers.62.mlp.down_proj.weight"
       },
       {
-       "i32": 24
+       "buf": "residual"
       },
       {
-       "i32": 14336
+       "buf": "model.layers.63.input_layernorm.weight_p1"
       },
       {
-       "i32": 512
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 5120
       },
       {
-       "i32": 256
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -142529,55 +90983,32 @@
      ]
     },
     {
-     "label": "l63.k_norm",
-     "kernel": "gemma_norm_khead",
+     "label": "l63.qkv_proj",
+     "kernel": "gemm",
      "args": [
       {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.63.self_attn.k_norm.weight_p1"
+       "buf": "x"
       },
       {
-       "i32": 256
+       "buf": "model.layers.63.self_attn.qkv_proj.weight"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
+       "buf": "qkv"
       },
       {
-       "i32": 4
+       "sym": "tokens"
       },
       {
        "i32": 14336
       },
       {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l63.rope",
-     "kernel": "mrope",
+     "label": "l63.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
@@ -142586,32 +91017,19 @@
        "buf": "k_n"
       },
       {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
+       "buf": "qkv"
       },
       {
-       "i32": 0
+       "buf": "model.layers.63.self_attn.q_norm.weight_p1"
       },
       {
-       "i64": 0
+       "buf": "model.layers.63.self_attn.k_norm.weight_p1"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l63.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
+       "buf": "cos_g"
       },
       {
-       "buf": "qkv",
-       "offset": 26624
+       "buf": "sin_g"
       },
       {
        "state": "kv",
@@ -142625,38 +91043,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -142754,11 +91173,11 @@
      ]
     },
     {
-     "label": "l63.gate",
-     "kernel": "sigmoid_mul",
+     "label": "l63.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "gated"
+       "buf": "x"
       },
       {
        "buf": "attn_out"
@@ -142768,31 +91187,13 @@
        "offset": 512
       },
       {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l63.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
+       "buf": "model.layers.63.self_attn.o_proj.weight"
       },
       {
-       "buf": "model.layers.63.self_attn.o_proj.weight"
+       "buf": "residual"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.63.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -142802,36 +91203,15 @@
       },
       {
        "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l63.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
       },
       {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.63.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
+       "i32": 256
       },
       {
-       "i32": 5120
+       "i32": 14336
       },
       {
-       "i32": 5120
+       "i32": 512
       },
       {
        "f32": 9.999999974752427e-07
@@ -142839,23 +91219,23 @@
      ]
     },
     {
-     "label": "l63.gate_up",
-     "kernel": "gemm",
+     "label": "l63.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.63.mlp.gate_up_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.63.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -142863,72 +91243,24 @@
      ]
     },
     {
-     "label": "l63.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l63.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
+       "buf": "x"
       },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l63.down_proj",
-     "kernel": "gemm",
-     "args": [
       {
        "buf": "act"
       },
       {
        "buf": "model.layers.63.mlp.down_proj.weight"
       },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l63.final_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
       {
        "buf": "residual"
       },
       {
        "buf": "model.norm.weight_p1"
       },
-      {
-       "i32": 5120
-      },
       {
        "sym": "tokens"
       },
@@ -142936,7 +91268,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -143559,50 +91891,26 @@
      ]
     },
     {
-     "label": "l0.gate_up",
-     "kernel": "gemm",
+     "label": "l0.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "draft.layers.0.mlp.gate_up_proj.weight"
+       "buf": "y"
       },
       {
-       "buf": "gate_up"
+       "buf": "draft.layers.0.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l0.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
       {
        "i32": 17408
       },
       {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
+       "i32": 5120
       }
      ]
     },
@@ -144204,50 +92512,26 @@
      ]
     },
     {
-     "label": "l1.gate_up",
-     "kernel": "gemm",
+     "label": "l1.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "draft.layers.1.mlp.gate_up_proj.weight"
+       "buf": "y"
       },
       {
-       "buf": "gate_up"
+       "buf": "draft.layers.1.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l1.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
       {
        "i32": 17408
       },
       {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
+       "i32": 5120
       }
      ]
     },
@@ -144849,50 +93133,26 @@
      ]
     },
     {
-     "label": "l2.gate_up",
-     "kernel": "gemm",
+     "label": "l2.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "draft.layers.2.mlp.gate_up_proj.weight"
+       "buf": "y"
       },
       {
-       "buf": "gate_up"
+       "buf": "draft.layers.2.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l2.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
       {
        "i32": 17408
       },
       {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
+       "i32": 5120
       }
      ]
     },
@@ -145494,50 +93754,26 @@
      ]
     },
     {
-     "label": "l3.gate_up",
-     "kernel": "gemm",
+     "label": "l3.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "draft.layers.3.mlp.gate_up_proj.weight"
+       "buf": "y"
       },
       {
-       "buf": "gate_up"
+       "buf": "draft.layers.3.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l3.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
       {
        "i32": 17408
       },
       {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
+       "i32": 5120
       }
      ]
     },
@@ -146139,50 +94375,26 @@
      ]
     },
     {
-     "label": "l4.gate_up",
-     "kernel": "gemm",
+     "label": "l4.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "draft.layers.4.mlp.gate_up_proj.weight"
+       "buf": "y"
       },
       {
-       "buf": "gate_up"
+       "buf": "draft.layers.4.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l4.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
       {
        "i32": 17408
       },
       {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
+       "i32": 5120
       }
      ]
     },
diff --git a/examples/qwen3.8-27b.json b/examples/qwen3.8-27b.json
index eb41957..09c3d0c 100644
--- a/examples/qwen3.8-27b.json
+++ b/examples/qwen3.8-27b.json
@@ -5574,34 +5574,46 @@
     ]
    }
   },
-  "gemma_norm_qhead": {
+  "attn_prep": {
    "params": [
+    "out buffer<bf16>",
     "out buffer<bf16>",
     "in buffer<bf16>",
     "in buffer<f32>",
+    "in buffer<f32>",
+    "in buffer<bf16>",
+    "in buffer<bf16>",
+    "inout ptr",
+    "inout ptr",
+    "in buffer<i64>",
     "i32",
     "i32",
-    "i32",
-    "i32",
-    "i32",
-    "i32",
+    "i64",
+    "i64",
+    "i64",
     "i32",
     "f32"
    ],
    "impl": {
     "steps": [
      {
-      "symbol": "kern_gemma_rms_norm_bf16",
+      "symbol": "kern_attn_prep_bf16",
       "params": [
+       "out buffer<bf16>",
        "out buffer<bf16>",
        "in buffer<bf16>",
        "in buffer<f32>",
+       "in buffer<f32>",
+       "in buffer<bf16>",
+       "in buffer<bf16>",
+       "inout ptr",
+       "inout ptr",
+       "in buffer<i64>",
        "i32",
        "i32",
-       "i32",
-       "i32",
-       "i32",
-       "i32",
+       "i64",
+       "i64",
+       "i64",
        "i32",
        "f32"
       ],
@@ -5616,7 +5628,7 @@
          {
           "sym": "tokens"
          },
-         24
+         28
         ]
        },
        1,
@@ -5655,43 +5667,56 @@
        },
        {
         "arg": 10
+       },
+       {
+        "arg": 11
+       },
+       {
+        "arg": 12
+       },
+       {
+        "arg": 13
+       },
+       {
+        "arg": 14
+       },
+       {
+        "arg": 15
+       },
+       {
+        "arg": 16
        }
       ],
-      "shared_mem": 4096,
-      "cubin": "gemma_rms_norm.cubin"
+      "cubin": "attn_prep.cubin"
      }
     ]
    }
   },
-  "gemma_norm_khead": {
+  "gemma_fused_norm": {
    "params": [
     "out buffer<bf16>",
     "in buffer<bf16>",
+    "inout buffer<bf16>",
     "in buffer<f32>",
     "i32",
     "i32",
     "i32",
     "i32",
-    "i32",
-    "i32",
-    "i32",
     "f32"
    ],
    "impl": {
     "steps": [
      {
-      "symbol": "kern_gemma_rms_norm_bf16",
+      "symbol": "kern_gemma_fused_add_rms_norm_bf16",
       "params": [
        "out buffer<bf16>",
        "in buffer<bf16>",
+       "inout buffer<bf16>",
        "in buffer<f32>",
        "i32",
        "i32",
        "i32",
        "i32",
-       "i32",
-       "i32",
-       "i32",
        "f32"
       ],
       "block": [
@@ -5701,12 +5726,7 @@
       ],
       "grid": [
        {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
+        "sym": "tokens"
        },
        1,
        1
@@ -5738,41 +5758,257 @@
        },
        {
         "arg": 8
+       }
+      ],
+      "shared_mem": 43008,
+      "cubin": "gemma_rms_norm.cubin"
+     }
+    ]
+   }
+  },
+  "silu_mul": {
+   "params": [
+    "out buffer<bf16>",
+    "in buffer<bf16>",
+    "i32",
+    "i32",
+    "f32",
+    "i32"
+   ],
+   "impl": {
+    "steps": [
+     {
+      "symbol": "_ZN4vllm18act_and_mul_kernelIN3c108BFloat16E14__nv_bfloat162XadL_ZNS_11silu_kernelIS2_EET_RKS5_fEEXadL_ZNS_18packed_silu_kernelIS3_EES5_S7_fEELb1ELb1ELb0ELb1EEEvPS5_PS6_ifff",
+      "params": [
+       "out buffer<bf16>",
+       "in buffer<bf16>",
+       "i32",
+       "i32",
+       "f32",
+       "i32"
+      ],
+      "block": [
+       1024,
+       1,
+       1
+      ],
+      "grid": [
+       {
+        "sym": "tokens"
        },
+       1,
+       1
+      ],
+      "args": [
        {
-        "arg": 9
+        "arg": 0
        },
        {
-        "arg": 10
+        "arg": 1
+       },
+       {
+        "arg": 2
+       },
+       {
+        "arg": 3
+       },
+       {
+        "arg": 4
+       },
+       {
+        "arg": 5
        }
+      ]
+     }
+    ]
+   }
+  },
+  "gemm8_gateup_silu": {
+   "params": [
+    "out buffer<bf16>",
+    "in buffer<bf16>",
+    "in buffer<bf16>",
+    "i32",
+    "i32",
+    "i32"
+   ],
+   "impl": {
+    "steps": [
+     {
+      "symbol": "kern_gemm8_gateup_silu_bf16",
+      "params": [
+       "out buffer<bf16>",
+       "in buffer<bf16>",
+       "in buffer<bf16>",
+       "i32",
+       "i32",
+       "i32"
       ],
-      "shared_mem": 4096,
-      "cubin": "gemma_rms_norm.cubin"
+      "block": [
+       288,
+       1,
+       1
+      ],
+      "grid": [
+       152,
+       1,
+       1
+      ],
+      "args": [
+       {
+        "arg": 0
+       },
+       {
+        "arg": 1
+       },
+       {
+        "arg": 2
+       },
+       {
+        "arg": 3
+       },
+       {
+        "arg": 4
+       },
+       {
+        "arg": 5
+       }
+      ],
+      "shared_mem": 224128,
+      "cubin": "gemm8.cubin",
+      "pdl": true
      }
     ]
    }
   },
-  "gemma_fused_norm": {
+  "gemm8_dual": {
+   "params": [
+    "out buffer<bf16>",
+    "out buffer<bf16>",
+    "in buffer<bf16>",
+    "in buffer<bf16>",
+    "in buffer<bf16>",
+    "i32",
+    "i32",
+    "i32",
+    "i32"
+   ],
+   "impl": {
+    "steps": [
+     {
+      "symbol": "kern_gemm8_dual_bf16",
+      "params": [
+       "out buffer<bf16>",
+       "out buffer<bf16>",
+       "in buffer<bf16>",
+       "in buffer<bf16>",
+       "in buffer<bf16>",
+       "i32",
+       "i32",
+       "i32",
+       "i32"
+      ],
+      "block": [
+       288,
+       1,
+       1
+      ],
+      "grid": [
+       152,
+       1,
+       1
+      ],
+      "args": [
+       {
+        "arg": 0
+       },
+       {
+        "arg": 1
+       },
+       {
+        "arg": 2
+       },
+       {
+        "arg": 3
+       },
+       {
+        "arg": 4
+       },
+       {
+        "arg": 5
+       },
+       {
+        "arg": 6
+       },
+       {
+        "arg": 7
+       },
+       {
+        "arg": 8
+       }
+      ],
+      "shared_mem": 224128,
+      "cubin": "gemm8.cubin",
+      "pdl": true
+     }
+    ]
+   }
+  },
+  "gemm8_sgate_add_norm": {
    "params": [
     "out buffer<bf16>",
     "in buffer<bf16>",
+    "in buffer<bf16>",
+    "in buffer<bf16>",
     "inout buffer<bf16>",
     "in buffer<f32>",
     "i32",
     "i32",
     "i32",
     "i32",
+    "i32",
+    "i32",
     "f32"
    ],
    "impl": {
+    "scratch": {
+     "y": {
+      "dtype": "bf16",
+      "shape": [
+       8,
+       5120
+      ]
+     },
+     "partial": {
+      "dtype": "f32",
+      "shape": [
+       8,
+       8,
+       5120
+      ]
+     },
+     "gcnt": {
+      "dtype": "i32",
+      "shape": [
+       2
+      ]
+     }
+    },
     "steps": [
      {
-      "symbol": "kern_gemma_fused_add_rms_norm_bf16",
+      "symbol": "kern_gemm8_sgate_add_norm_bf16",
       "params": [
        "out buffer<bf16>",
        "in buffer<bf16>",
+       "in buffer<bf16>",
+       "in buffer<bf16>",
        "inout buffer<bf16>",
        "in buffer<f32>",
+       "out buffer<bf16>",
+       "out buffer<f32>",
+       "out buffer<i32>",
+       "i32",
+       "i32",
        "i32",
        "i32",
        "i32",
@@ -5780,14 +6016,12 @@
        "f32"
       ],
       "block": [
-       512,
+       288,
        1,
        1
       ],
       "grid": [
-       {
-        "sym": "tokens"
-       },
+       152,
        1,
        1
       ],
@@ -5810,6 +6044,15 @@
        {
         "arg": 5
        },
+       {
+        "scratch": "y"
+       },
+       {
+        "scratch": "partial"
+       },
+       {
+        "scratch": "gcnt"
+       },
        {
         "arg": 6
        },
@@ -5818,44 +6061,87 @@
        },
        {
         "arg": 8
+       },
+       {
+        "arg": 9
+       },
+       {
+        "arg": 10
+       },
+       {
+        "arg": 11
+       },
+       {
+        "arg": 12
        }
       ],
-      "shared_mem": 43008,
-      "cubin": "gemma_rms_norm.cubin"
+      "shared_mem": 224128,
+      "cubin": "gemm8.cubin",
+      "pdl": true
      }
     ]
    }
   },
-  "silu_mul": {
+  "gemm8_add_norm": {
    "params": [
     "out buffer<bf16>",
     "in buffer<bf16>",
+    "in buffer<bf16>",
+    "inout buffer<bf16>",
+    "in buffer<f32>",
     "i32",
     "i32",
-    "f32",
-    "i32"
+    "i32",
+    "f32"
    ],
    "impl": {
+    "scratch": {
+     "y": {
+      "dtype": "bf16",
+      "shape": [
+       8,
+       5120
+      ]
+     },
+     "partial": {
+      "dtype": "f32",
+      "shape": [
+       8,
+       8,
+       5120
+      ]
+     },
+     "gcnt": {
+      "dtype": "i32",
+      "shape": [
+       2
+      ]
+     }
+    },
     "steps": [
      {
-      "symbol": "_ZN4vllm18act_and_mul_kernelIN3c108BFloat16E14__nv_bfloat162XadL_ZNS_11silu_kernelIS2_EET_RKS5_fEEXadL_ZNS_18packed_silu_kernelIS3_EES5_S7_fEELb1ELb1ELb0ELb1EEEvPS5_PS6_ifff",
+      "symbol": "kern_gemm8_add_norm_bf16",
       "params": [
        "out buffer<bf16>",
        "in buffer<bf16>",
+       "in buffer<bf16>",
+       "inout buffer<bf16>",
+       "in buffer<f32>",
+       "out buffer<bf16>",
+       "out buffer<f32>",
+       "out buffer<i32>",
        "i32",
        "i32",
-       "f32",
-       "i32"
+       "i32",
+       "f32"
       ],
       "block": [
-       1024,
+       288,
        1,
        1
       ],
       "grid": [
-       {
-        "sym": "tokens"
-       },
+       152,
        1,
        1
       ],
@@ -5875,10 +6161,31 @@
        {
         "arg": 4
        },
+       {
+        "scratch": "y"
+       },
+       {
+        "scratch": "partial"
+       },
+       {
+        "scratch": "gcnt"
+       },
        {
         "arg": 5
+       },
+       {
+        "arg": 6
+       },
+       {
+        "arg": 7
+       },
+       {
+        "arg": 8
        }
-      ]
+      ],
+      "shared_mem": 224128,
+      "cubin": "gemm8.cubin",
+      "pdl": true
      }
     ]
    }
@@ -6982,385 +7289,54 @@
     ]
    }
   },
-  "conv_update": {
+  "gdn_decode": {
    "params": [
+    "inout buffer<bf16>",
     "in buffer<bf16>",
-    "in buffer<bf16>",
+    "inout ptr",
     "inout ptr",
     "in buffer<i32>",
-    "out buffer<bf16>",
-    "i32",
-    "i32",
-    "i64",
-    "i64",
-    "i64",
-    "i64"
-   ],
-   "impl": {
-    "steps": [
-     {
-      "symbol": "_causal_conv1d_update_kernel",
-      "params": [
-       "in buffer<bf16>",
-       "in buffer<bf16>",
-       "inout ptr",
-       "in buffer<i32>",
-       "out buffer<bf16>",
-       "i32",
-       "i32",
-       "i64",
-       "i64",
-       "i64",
-       "i64"
-      ],
-      "block": [
-       128,
-       1,
-       1
-      ],
-      "grid": [
-       1,
-       40,
-       1
-      ],
-      "args": [
-       {
-        "arg": 0
-       },
-       {
-        "arg": 1
-       },
-       {
-        "arg": 2
-       },
-       {
-        "arg": 3
-       },
-       {
-        "arg": 4
-       },
-       {
-        "arg": 5
-       },
-       {
-        "arg": 6
-       },
-       {
-        "arg": 7
-       },
-       {
-        "arg": 8
-       },
-       {
-        "arg": 9
-       },
-       {
-        "arg": 10
-       }
-      ],
-      "cubin": "module_139.cubin",
-      "sha256": "c2504833f4aa2997dc27f621fa8168695ba8773c41e0ced1c8ffb7d189cf8ae4"
-     }
-    ]
-   }
-  },
-  "recurrent": {
-   "params": [
-    "in buffer<bf16>",
-    "in buffer<bf16>",
     "in buffer<bf16>",
     "in buffer<f32>",
     "in buffer<bf16>",
     "out buffer<bf16>",
-    "inout ptr",
-    "inout ptr",
-    "in buffer<i32>",
-    "f32",
-    "i64",
-    "i64"
-   ],
-   "impl": {
-    "steps": [
-     {
-      "symbol": "fused_recurrent_gated_delta_rule_packed_decode_kernel",
-      "params": [
-       "in buffer<bf16>",
-       "in buffer<bf16>",
-       "in buffer<bf16>",
-       "in buffer<f32>",
-       "in buffer<bf16>",
-       "out buffer<bf16>",
-       "inout ptr",
-       "inout ptr",
-       "in buffer<i32>",
-       "f32",
-       "i64",
-       "i64"
-      ],
-      "block": [
-       32,
-       1,
-       1
-      ],
-      "grid": [
-       4,
-       48,
-       1
-      ],
-      "args": [
-       {
-        "arg": 0
-       },
-       {
-        "arg": 1
-       },
-       {
-        "arg": 2
-       },
-       {
-        "arg": 3
-       },
-       {
-        "arg": 4
-       },
-       {
-        "arg": 5
-       },
-       {
-        "arg": 6
-       },
-       {
-        "arg": 7
-       },
-       {
-        "arg": 8
-       },
-       {
-        "arg": 9
-       },
-       {
-        "arg": 10
-       },
-       {
-        "arg": 11
-       }
-      ],
-      "shared_mem": 64,
-      "cubin": "module_141.cubin",
-      "sha256": "ecc0dc457582a5d0ca3348d366168eb51a68deaaf59de409fb800b2665c903de"
-     }
-    ]
-   }
-  },
-  "gated_norm_decode": {
-   "params": [
-    "inout buffer<bf16>",
-    "out buffer<bf16>",
-    "in buffer<bf16>",
     "in buffer<bf16>",
+    "f32",
+    "f32",
     "i32",
     "i32",
     "i32",
-    "i32",
-    "f32",
-    "i64",
-    "i64"
+    "i32"
    ],
    "impl": {
-    "scratch": {
-     "rstd": {
-      "dtype": "f32",
-      "shape": [
-       1,
-       48
-      ]
-     }
-    },
     "steps": [
      {
-      "symbol": "layer_norm_fwd_kernel",
+      "symbol": "kern_gdn_decode_bf16",
       "params": [
        "inout buffer<bf16>",
-       "out buffer<bf16>",
        "in buffer<bf16>",
+       "inout ptr",
+       "inout ptr",
+       "in buffer<i32>",
        "in buffer<bf16>",
-       "out buffer<f32>",
-       "i32",
-       "i32",
-       "i32",
-       "i32",
-       "f32",
-       "i64",
-       "i64"
-      ],
-      "block": [
-       32,
-       1,
-       1
-      ],
-      "grid": [
-       48,
-       1,
-       1
-      ],
-      "args": [
-       {
-        "arg": 0
-       },
-       {
-        "arg": 1
-       },
-       {
-        "arg": 2
-       },
-       {
-        "arg": 3
-       },
-       {
-        "scratch": "rstd"
-       },
-       {
-        "arg": 4
-       },
-       {
-        "arg": 5
-       },
-       {
-        "arg": 6
-       },
-       {
-        "arg": 7
-       },
-       {
-        "arg": 8
-       },
-       {
-        "arg": 9
-       },
-       {
-        "arg": 10
-       }
-      ],
-      "cubin": "module_143.cubin",
-      "sha256": "ab8921a843b50459d97332df3d691f9b442ef9c4f56e4bdca6fb3689746582e1"
-     }
-    ]
-   }
-  },
-  "mrope": {
-   "params": [
-    "inout buffer<bf16>",
-    "inout buffer<bf16>",
-    "in buffer<bf16>",
-    "in buffer<bf16>",
-    "i32",
-    "i64",
-    "i64"
-   ],
-   "impl": {
-    "steps": [
-     {
-      "symbol": "_triton_mrope_forward",
-      "params": [
-       "inout buffer<bf16>",
-       "inout buffer<bf16>",
+       "in buffer<f32>",
        "in buffer<bf16>",
+       "out buffer<bf16>",
        "in buffer<bf16>",
+       "f32",
+       "f32",
        "i32",
-       "i64",
-       "i64"
-      ],
-      "block": [
-       128,
-       1,
-       1
-      ],
-      "grid": [
-       {
-        "sym": "tokens"
-       },
-       1,
-       1
-      ],
-      "args": [
-       {
-        "arg": 0
-       },
-       {
-        "arg": 1
-       },
-       {
-        "arg": 2
-       },
-       {
-        "arg": 3
-       },
-       {
-        "arg": 4
-       },
-       {
-        "arg": 5
-       },
-       {
-        "arg": 6
-       }
-      ],
-      "cubin": "module_173.cubin",
-      "sha256": "cc5d7bff4bfd36a94e8e43bfb6651908905734c8314019db41d3e469b3517483"
-     }
-    ]
-   }
-  },
-  "reshape_and_cache": {
-   "params": [
-    "in buffer<bf16>",
-    "in buffer<bf16>",
-    "inout ptr",
-    "inout ptr",
-    "in buffer<i64>",
-    "in buffer<f32>",
-    "in buffer<f32>",
-    "i64",
-    "i64",
-    "i64",
-    "i64",
-    "i64",
-    "i64",
-    "i64",
-    "i64",
-    "i64"
-   ],
-   "impl": {
-    "steps": [
-     {
-      "symbol": "reshape_and_cache_kernel_flash",
-      "params": [
-       "in buffer<bf16>",
-       "in buffer<bf16>",
-       "inout ptr",
-       "inout ptr",
-       "in buffer<i64>",
-       "in buffer<f32>",
-       "in buffer<f32>",
-       "i64",
-       "i64",
-       "i64",
-       "i64",
-       "i64",
-       "i64",
-       "i64",
-       "i64",
-       "i64"
+       "i32",
+       "i32",
+       "i32"
       ],
       "block": [
-       512,
+       256,
        1,
        1
       ],
       "grid": [
-       {
-        "sym": "tokens"
-       },
+       96,
        1,
        1
       ],
@@ -7414,8 +7390,7 @@
         "arg": 15
        }
       ],
-      "cubin": "module_119.cubin",
-      "sha256": "93d7768f15ce02e252ad208dca2f6803d8ad65296d57f3444dccb9d1e6b88ac5"
+      "cubin": "gdn_decode.cubin"
      }
     ]
    }
@@ -9828,67 +9803,49 @@
      ]
     },
     {
-     "label": "l3.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l3.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
       {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.3.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "buf": "k_n"
       },
       {
-       "i32": 24
+       "buf": "qkv"
       },
       {
-       "i32": 14336
+       "buf": "model.layers.3.self_attn.q_norm.weight_p1"
       },
       {
-       "i32": 512
+       "buf": "model.layers.3.self_attn.k_norm.weight_p1"
       },
       {
-       "i32": 6144
+       "buf": "cos_g"
       },
       {
-       "i32": 256
+       "buf": "sin_g"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l3.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
+       "state": "kv"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
+       "state": "kv",
+       "offset": 512
       },
       {
-       "buf": "model.layers.3.self_attn.k_norm.weight_p1"
+       "buf": "slot_mapping"
       },
       {
-       "i32": 256
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
        "expr": {
@@ -9900,107 +9857,20 @@
         ]
        }
       },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l3.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l3.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv"
-      },
-      {
-       "state": "kv",
-       "offset": 512
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -12138,107 +12008,23 @@
      ]
     },
     {
-     "label": "l7.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l7.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.7.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l7.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
       {
        "buf": "k_n"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.7.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
+       "buf": "qkv"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l7.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "model.layers.7.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "k_n"
+       "buf": "model.layers.7.self_attn.k_norm.weight_p1"
       },
       {
        "buf": "cos_g"
@@ -12246,28 +12032,6 @@
       {
        "buf": "sin_g"
       },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l7.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
       {
        "state": "kv",
        "offset": 3211264
@@ -12280,38 +12044,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -14450,107 +14215,23 @@
      ]
     },
     {
-     "label": "l11.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l11.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.11.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l11.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
       {
        "buf": "k_n"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.11.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
+       "buf": "qkv"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l11.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "model.layers.11.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "k_n"
+       "buf": "model.layers.11.self_attn.k_norm.weight_p1"
       },
       {
        "buf": "cos_g"
@@ -14558,28 +14239,6 @@
       {
        "buf": "sin_g"
       },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l11.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
       {
        "state": "kv",
        "offset": 6422528
@@ -14592,38 +14251,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -16762,107 +16422,23 @@
      ]
     },
     {
-     "label": "l15.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l15.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.15.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l15.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
       {
        "buf": "k_n"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.15.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
+       "buf": "qkv"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l15.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "model.layers.15.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "k_n"
+       "buf": "model.layers.15.self_attn.k_norm.weight_p1"
       },
       {
        "buf": "cos_g"
@@ -16870,28 +16446,6 @@
       {
        "buf": "sin_g"
       },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l15.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
       {
        "state": "kv",
        "offset": 9633792
@@ -16904,38 +16458,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -19074,107 +18629,23 @@
      ]
     },
     {
-     "label": "l19.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l19.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.19.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l19.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
       {
        "buf": "k_n"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.19.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
+       "buf": "qkv"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l19.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "model.layers.19.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "k_n"
+       "buf": "model.layers.19.self_attn.k_norm.weight_p1"
       },
       {
        "buf": "cos_g"
@@ -19182,28 +18653,6 @@
       {
        "buf": "sin_g"
       },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l19.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
       {
        "state": "kv",
        "offset": 12845056
@@ -19216,38 +18665,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -21386,107 +20836,23 @@
      ]
     },
     {
-     "label": "l23.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l23.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.23.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l23.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
       {
        "buf": "k_n"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.23.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
+       "buf": "qkv"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l23.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "model.layers.23.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "k_n"
+       "buf": "model.layers.23.self_attn.k_norm.weight_p1"
       },
       {
        "buf": "cos_g"
@@ -21494,28 +20860,6 @@
       {
        "buf": "sin_g"
       },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l23.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
       {
        "state": "kv",
        "offset": 16056320
@@ -21528,38 +20872,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -23698,107 +23043,23 @@
      ]
     },
     {
-     "label": "l27.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l27.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.27.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l27.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
       {
        "buf": "k_n"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.27.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
+       "buf": "qkv"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l27.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "model.layers.27.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "k_n"
+       "buf": "model.layers.27.self_attn.k_norm.weight_p1"
       },
       {
        "buf": "cos_g"
@@ -23806,28 +23067,6 @@
       {
        "buf": "sin_g"
       },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l27.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
       {
        "state": "kv",
        "offset": 19267584
@@ -23840,38 +23079,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -26010,107 +25250,23 @@
      ]
     },
     {
-     "label": "l31.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l31.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.31.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l31.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
       {
        "buf": "k_n"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.31.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
+       "buf": "qkv"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l31.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "model.layers.31.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "k_n"
+       "buf": "model.layers.31.self_attn.k_norm.weight_p1"
       },
       {
        "buf": "cos_g"
@@ -26118,28 +25274,6 @@
       {
        "buf": "sin_g"
       },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l31.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
       {
        "state": "kv",
        "offset": 22478848
@@ -26152,38 +25286,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -28322,107 +27457,23 @@
      ]
     },
     {
-     "label": "l35.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l35.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.35.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l35.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
       {
        "buf": "k_n"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.35.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
+       "buf": "qkv"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l35.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "model.layers.35.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "k_n"
+       "buf": "model.layers.35.self_attn.k_norm.weight_p1"
       },
       {
        "buf": "cos_g"
@@ -28430,28 +27481,6 @@
       {
        "buf": "sin_g"
       },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l35.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
       {
        "state": "kv",
        "offset": 25690112
@@ -28464,38 +27493,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -30634,107 +29664,23 @@
      ]
     },
     {
-     "label": "l39.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l39.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.39.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l39.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
       {
        "buf": "k_n"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.39.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
+       "buf": "qkv"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l39.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "model.layers.39.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "k_n"
+       "buf": "model.layers.39.self_attn.k_norm.weight_p1"
       },
       {
        "buf": "cos_g"
@@ -30742,28 +29688,6 @@
       {
        "buf": "sin_g"
       },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l39.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
       {
        "state": "kv",
        "offset": 28901376
@@ -30776,38 +29700,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -32946,107 +31871,23 @@
      ]
     },
     {
-     "label": "l43.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l43.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.43.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l43.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
       {
        "buf": "k_n"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.43.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
+       "buf": "qkv"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l43.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "model.layers.43.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "k_n"
+       "buf": "model.layers.43.self_attn.k_norm.weight_p1"
       },
       {
        "buf": "cos_g"
@@ -33054,28 +31895,6 @@
       {
        "buf": "sin_g"
       },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l43.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
       {
        "state": "kv",
        "offset": 32112640
@@ -33088,38 +31907,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -35258,107 +34078,23 @@
      ]
     },
     {
-     "label": "l47.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l47.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.47.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l47.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
       {
        "buf": "k_n"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.47.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
+       "buf": "qkv"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l47.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "model.layers.47.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "k_n"
+       "buf": "model.layers.47.self_attn.k_norm.weight_p1"
       },
       {
        "buf": "cos_g"
@@ -35366,28 +34102,6 @@
       {
        "buf": "sin_g"
       },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l47.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
       {
        "state": "kv",
        "offset": 35323904
@@ -35400,38 +34114,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -37570,107 +36285,23 @@
      ]
     },
     {
-     "label": "l51.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l51.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.51.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l51.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
       {
        "buf": "k_n"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.51.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
+       "buf": "qkv"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l51.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "model.layers.51.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "k_n"
+       "buf": "model.layers.51.self_attn.k_norm.weight_p1"
       },
       {
        "buf": "cos_g"
@@ -37678,28 +36309,6 @@
       {
        "buf": "sin_g"
       },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l51.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
       {
        "state": "kv",
        "offset": 38535168
@@ -37712,38 +36321,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -39882,107 +38492,23 @@
      ]
     },
     {
-     "label": "l55.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l55.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.55.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l55.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
       {
        "buf": "k_n"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.55.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
+       "buf": "qkv"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l55.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "model.layers.55.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "k_n"
+       "buf": "model.layers.55.self_attn.k_norm.weight_p1"
       },
       {
        "buf": "cos_g"
@@ -39990,28 +38516,6 @@
       {
        "buf": "sin_g"
       },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l55.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
       {
        "state": "kv",
        "offset": 41746432
@@ -40024,38 +38528,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -42194,107 +40699,23 @@
      ]
     },
     {
-     "label": "l59.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l59.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.59.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l59.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
       {
        "buf": "k_n"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.59.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
+       "buf": "qkv"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l59.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "model.layers.59.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "k_n"
+       "buf": "model.layers.59.self_attn.k_norm.weight_p1"
       },
       {
        "buf": "cos_g"
@@ -42302,28 +40723,6 @@
       {
        "buf": "sin_g"
       },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l59.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
       {
        "state": "kv",
        "offset": 44957696
@@ -42336,38 +40735,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -44506,107 +42906,23 @@
      ]
     },
     {
-     "label": "l63.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l63.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.63.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l63.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
       {
        "buf": "k_n"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.63.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
+       "buf": "qkv"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l63.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "model.layers.63.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "k_n"
+       "buf": "model.layers.63.self_attn.k_norm.weight_p1"
       },
       {
        "buf": "cos_g"
@@ -44614,28 +42930,6 @@
       {
        "buf": "sin_g"
       },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l63.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
       {
        "state": "kv",
        "offset": 48168960
@@ -44648,38 +42942,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -45133,44 +43428,29 @@
      ]
     },
     {
-     "label": "l0.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l0.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.0.linear_attn.in_proj_qkvz.weight"
-      },
       {
        "buf": "qkvz"
       },
       {
-       "sym": "tokens"
+       "buf": "ba"
       },
       {
-       "i32": 16384
+       "buf": "x"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l0.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
+       "buf": "model.layers.0.linear_attn.in_proj_qkvz.weight"
       },
       {
        "buf": "model.layers.0.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "ba"
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 16384
       },
       {
        "i32": 96
@@ -45181,8 +43461,8 @@
      ]
     },
     {
-     "label": "l0.conv_update",
-     "kernel": "conv_update",
+     "label": "l0.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
        "buf": "qkvz"
@@ -45194,42 +43474,12 @@
        "state": "gdn"
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 4
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 49
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l0.recurrent",
-     "kernel": "recurrent",
-     "args": [
-      {
-       "buf": "qkvz"
+       "state": "gdn",
+       "offset": 61440
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "gdn.line_index",
+       "offset": 4
       },
       {
        "buf": "ba"
@@ -45244,110 +43494,103 @@
        "buf": "core_attn_out"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "buf": "model.layers.0.linear_attn.norm.weight"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 4
+       "f32": 9.999999974752427e-07
       },
       {
-       "f32": 0.0883883461356163
+       "i32": 49
       },
       {
-       "i64": 0
+       "i32": 1605632
       },
       {
-       "i64": 0
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l0.gated_norm",
-     "kernel": "gated_norm_decode",
+     "label": "l0.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.0.linear_attn.norm.weight"
+       "buf": "model.layers.0.linear_attn.out_proj.weight"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.0.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "i32": 48
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l0.out_proj",
-     "kernel": "gemm",
+     "label": "l0.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.0.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.0.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l0.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l0.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.0.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.0.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.1.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -45356,7 +43599,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -45364,23 +43607,32 @@
      ]
     },
     {
-     "label": "l0.gate_up",
-     "kernel": "gemm",
+     "label": "l1.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.0.mlp.gate_up_proj.weight"
+       "buf": "model.layers.1.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.1.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -45388,71 +43640,79 @@
      ]
     },
     {
-     "label": "l0.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l1.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.1.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 61440
       },
       {
-       "f32": 1.0
+       "buf": "gdn.line_index",
+       "offset": 8
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l0.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "ba"
+      },
       {
-       "buf": "act"
+       "buf": "model.layers.1.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.0.mlp.down_proj.weight"
+       "buf": "model.layers.1.linear_attn.dt_bias"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.1.linear_attn.norm.weight"
       },
       {
-       "i32": 5120
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 17408
+       "f32": 9.999999974752427e-07
+      },
+      {
+       "i32": 49
+      },
+      {
+       "i32": 1605632
+      },
+      {
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l0.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l1.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.1.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.1.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.1.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -45461,7 +43721,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -45469,23 +43729,23 @@
      ]
     },
     {
-     "label": "l1.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l1.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.1.linear_attn.in_proj_qkvz.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.1.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -45493,197 +43753,202 @@
      ]
     },
     {
-     "label": "l1.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l1.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.1.linear_attn.in_proj_ba.weight"
+       "buf": "act"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.1.mlp.down_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 96
+       "buf": "model.layers.2.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l1.conv_update",
-     "kernel": "conv_update",
+     "label": "l2.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.1.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 8
+       "buf": "ba"
       },
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "i32": 1
+       "buf": "model.layers.2.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i32": 49
+       "buf": "model.layers.2.linear_attn.in_proj_ba.weight"
       },
       {
-       "i64": 1
+       "sym": "tokens"
       },
       {
-       "i64": 1
+       "i32": 16384
       },
       {
-       "i64": 0
+       "i32": 96
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l1.recurrent",
-     "kernel": "recurrent",
+     "label": "l2.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.2.linear_attn.conv1d.weight"
+      },
+      {
+       "state": "gdn"
+      },
+      {
+       "state": "gdn",
+       "offset": 61440
+      },
+      {
+       "buf": "gdn.line_index",
+       "offset": 12
       },
       {
        "buf": "ba"
       },
       {
-       "buf": "model.layers.1.linear_attn.A_log"
+       "buf": "model.layers.2.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.1.linear_attn.dt_bias"
+       "buf": "model.layers.2.linear_attn.dt_bias"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "buf": "model.layers.2.linear_attn.norm.weight"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 8
+       "f32": 9.999999974752427e-07
       },
       {
-       "f32": 0.0883883461356163
+       "i32": 49
       },
       {
-       "i64": 0
+       "i32": 1605632
       },
       {
-       "i64": 0
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l1.gated_norm",
-     "kernel": "gated_norm_decode",
+     "label": "l2.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.1.linear_attn.norm.weight"
+       "buf": "model.layers.2.linear_attn.out_proj.weight"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.2.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "i32": 48
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l1.out_proj",
-     "kernel": "gemm",
+     "label": "l2.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.1.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.2.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l1.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l2.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.2.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.1.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.3.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -45692,7 +43957,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -45700,23 +43965,23 @@
      ]
     },
     {
-     "label": "l1.gate_up",
+     "label": "l3.qkv_proj",
      "kernel": "gemm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.1.mlp.gate_up_proj.weight"
+       "buf": "model.layers.3.self_attn.qkv_proj.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "qkv"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 14336
       },
       {
        "i32": 5120
@@ -45724,80 +43989,71 @@
      ]
     },
     {
-     "label": "l1.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l3.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
+       "buf": "q_n"
       },
       {
-       "i32": 0
+       "buf": "k_n"
       },
       {
-       "f32": 1.0
+       "buf": "qkv"
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l1.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "model.layers.3.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "model.layers.1.mlp.down_proj.weight"
+       "buf": "model.layers.3.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "y"
+       "buf": "cos_g"
       },
       {
-       "sym": "tokens"
+       "buf": "sin_g"
       },
       {
-       "i32": 5120
+       "state": "kv"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l1.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
+       "state": "kv",
+       "offset": 512
       },
       {
-       "buf": "y"
+       "buf": "slot_mapping"
       },
       {
-       "buf": "residual"
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "buf": "model.layers.2.input_layernorm.weight_p1"
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i32": 5120
+       "i64": 25690112
       },
       {
-       "sym": "tokens"
+       "i64": 2048
       },
       {
-       "i32": 5120
+       "i64": 512
       },
       {
-       "i32": 5120
+       "i32": 784
       },
       {
        "f32": 9.999999974752427e-07
@@ -45805,84 +44061,88 @@
      ]
     },
     {
-     "label": "l2.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l3.attn",
+     "kernel": "attn",
      "args": [
       {
-       "buf": "x"
+       "buf": "attn_out"
       },
       {
-       "buf": "model.layers.2.linear_attn.in_proj_qkvz.weight"
+       "buf": "q_n"
       },
       {
-       "buf": "qkvz"
+       "state": "kv"
       },
       {
-       "sym": "tokens"
+       "state": "kv",
+       "offset": 512
       },
       {
-       "i32": 16384
+       "buf": "block_table"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l2.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
+       "buf": "seq_lens"
+      },
       {
-       "buf": "x"
+       "f32": 0.0625
       },
       {
-       "buf": "model.layers.2.linear_attn.in_proj_ba.weight"
+       "buf": "kv_scales"
       },
       {
-       "buf": "ba"
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "sym": "tokens"
+       "f32": 1.0
       },
       {
-       "i32": 96
+       "f32": 0.0
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l2.conv_update",
-     "kernel": "conv_update",
-     "args": [
+       "i64": 8
+      },
       {
-       "buf": "qkvz"
+       "i64": 6144
       },
       {
-       "buf": "model.layers.2.linear_attn.conv1d.weight"
+       "i64": 256
       },
       {
-       "state": "gdn"
+       "i64": 6144
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 12
+       "i64": 256
       },
       {
-       "buf": "qkvz"
+       "i64": 0
       },
       {
-       "i32": 1
+       "buf": "seq_lens"
       },
       {
-       "i32": 49
+       "i64": 25690112
+      },
+      {
+       "i64": 2048
+      },
+      {
+       "i64": 512
+      },
+      {
+       "i64": 25690112
+      },
+      {
+       "i64": 2048
+      },
+      {
+       "i64": 512
       },
       {
-       "i64": 1
+       "buf": "cu_seqlens_q"
       },
       {
-       "i64": 1
+       "i32": 1
       },
       {
        "i64": 0
@@ -45893,10696 +44153,45 @@
      ]
     },
     {
-     "label": "l2.recurrent",
-     "kernel": "recurrent",
+     "label": "l3.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "attn_out"
       },
       {
-       "buf": "ba"
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "buf": "model.layers.2.linear_attn.A_log"
+       "buf": "model.layers.3.self_attn.o_proj.weight"
       },
       {
-       "buf": "model.layers.2.linear_attn.dt_bias"
+       "buf": "residual"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.3.post_attention_layernorm.weight_p1"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "sym": "tokens"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "i32": 5120
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 12
+       "i32": 6144
       },
       {
-       "f32": 0.0883883461356163
+       "i32": 256
       },
       {
-       "i64": 0
+       "i32": 14336
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l2.gated_norm",
-     "kernel": "gated_norm_decode",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.2.linear_attn.norm.weight"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 48
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l2.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.2.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l2.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.2.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l2.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.2.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l2.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l2.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.2.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l2.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.3.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l3.qkv_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.3.self_attn.qkv_proj.weight"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l3.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.3.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l3.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.3.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l3.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l3.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv"
-      },
-      {
-       "state": "kv",
-       "offset": 512
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l3.attn",
-     "kernel": "attn",
-     "args": [
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv"
-      },
-      {
-       "state": "kv",
-       "offset": 512
-      },
-      {
-       "buf": "block_table"
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "f32": 0.0625
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 0.0
-      },
-      {
-       "i64": 8
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 0
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l3.gate",
-     "kernel": "sigmoid_mul",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "qkv",
-       "offset": 512
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l3.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "model.layers.3.self_attn.o_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l3.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.3.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l3.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.3.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l3.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l3.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.3.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l3.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.4.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l4.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.4.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l4.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.4.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l4.conv_update",
-     "kernel": "conv_update",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.4.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 16
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 49
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l4.recurrent",
-     "kernel": "recurrent",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.4.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.4.linear_attn.dt_bias"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 16
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l4.gated_norm",
-     "kernel": "gated_norm_decode",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.4.linear_attn.norm.weight"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 48
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l4.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.4.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l4.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.4.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l4.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.4.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l4.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l4.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.4.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l4.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.5.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l5.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.5.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l5.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.5.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l5.conv_update",
-     "kernel": "conv_update",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.5.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 20
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 49
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l5.recurrent",
-     "kernel": "recurrent",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.5.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.5.linear_attn.dt_bias"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 20
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l5.gated_norm",
-     "kernel": "gated_norm_decode",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.5.linear_attn.norm.weight"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 48
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l5.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.5.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l5.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.5.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l5.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.5.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l5.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l5.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.5.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l5.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.6.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l6.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.6.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l6.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.6.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l6.conv_update",
-     "kernel": "conv_update",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.6.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 24
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 49
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l6.recurrent",
-     "kernel": "recurrent",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.6.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.6.linear_attn.dt_bias"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 24
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l6.gated_norm",
-     "kernel": "gated_norm_decode",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.6.linear_attn.norm.weight"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 48
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l6.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.6.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l6.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.6.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l6.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.6.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l6.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l6.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.6.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l6.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.7.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l7.qkv_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.7.self_attn.qkv_proj.weight"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l7.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.7.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l7.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.7.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l7.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l7.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv",
-       "offset": 3211264
-      },
-      {
-       "state": "kv",
-       "offset": 3211776
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l7.attn",
-     "kernel": "attn",
-     "args": [
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv",
-       "offset": 3211264
-      },
-      {
-       "state": "kv",
-       "offset": 3211776
-      },
-      {
-       "buf": "block_table"
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "f32": 0.0625
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 0.0
-      },
-      {
-       "i64": 8
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 0
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l7.gate",
-     "kernel": "sigmoid_mul",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "qkv",
-       "offset": 512
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l7.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "model.layers.7.self_attn.o_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l7.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.7.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l7.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.7.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l7.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l7.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.7.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l7.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.8.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l8.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.8.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l8.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.8.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l8.conv_update",
-     "kernel": "conv_update",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.8.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 28
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 49
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l8.recurrent",
-     "kernel": "recurrent",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.8.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.8.linear_attn.dt_bias"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 28
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l8.gated_norm",
-     "kernel": "gated_norm_decode",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.8.linear_attn.norm.weight"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 48
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l8.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.8.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l8.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.8.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l8.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.8.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l8.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l8.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.8.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l8.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.9.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l9.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.9.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l9.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.9.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l9.conv_update",
-     "kernel": "conv_update",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.9.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 32
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 49
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l9.recurrent",
-     "kernel": "recurrent",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.9.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.9.linear_attn.dt_bias"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 32
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l9.gated_norm",
-     "kernel": "gated_norm_decode",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.9.linear_attn.norm.weight"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 48
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l9.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.9.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l9.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.9.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l9.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.9.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l9.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l9.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.9.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l9.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.10.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l10.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.10.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l10.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.10.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l10.conv_update",
-     "kernel": "conv_update",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.10.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 36
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 49
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l10.recurrent",
-     "kernel": "recurrent",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.10.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.10.linear_attn.dt_bias"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 36
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l10.gated_norm",
-     "kernel": "gated_norm_decode",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.10.linear_attn.norm.weight"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 48
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l10.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.10.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l10.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.10.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l10.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.10.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l10.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l10.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.10.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l10.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.11.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l11.qkv_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.11.self_attn.qkv_proj.weight"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l11.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.11.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l11.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.11.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l11.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l11.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv",
-       "offset": 6422528
-      },
-      {
-       "state": "kv",
-       "offset": 6423040
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l11.attn",
-     "kernel": "attn",
-     "args": [
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv",
-       "offset": 6422528
-      },
-      {
-       "state": "kv",
-       "offset": 6423040
-      },
-      {
-       "buf": "block_table"
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "f32": 0.0625
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 0.0
-      },
-      {
-       "i64": 8
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 0
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l11.gate",
-     "kernel": "sigmoid_mul",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "qkv",
-       "offset": 512
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l11.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "model.layers.11.self_attn.o_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l11.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.11.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l11.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.11.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l11.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l11.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.11.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l11.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.12.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l12.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.12.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l12.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.12.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l12.conv_update",
-     "kernel": "conv_update",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.12.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 40
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 49
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l12.recurrent",
-     "kernel": "recurrent",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.12.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.12.linear_attn.dt_bias"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 40
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l12.gated_norm",
-     "kernel": "gated_norm_decode",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.12.linear_attn.norm.weight"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 48
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l12.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.12.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l12.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.12.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l12.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.12.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l12.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l12.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.12.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l12.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.13.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l13.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.13.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l13.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.13.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l13.conv_update",
-     "kernel": "conv_update",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.13.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 44
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 49
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l13.recurrent",
-     "kernel": "recurrent",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.13.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.13.linear_attn.dt_bias"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 44
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l13.gated_norm",
-     "kernel": "gated_norm_decode",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.13.linear_attn.norm.weight"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 48
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l13.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.13.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l13.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.13.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l13.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.13.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l13.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l13.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.13.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l13.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.14.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l14.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.14.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l14.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.14.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l14.conv_update",
-     "kernel": "conv_update",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.14.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 48
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 49
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l14.recurrent",
-     "kernel": "recurrent",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.14.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.14.linear_attn.dt_bias"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 48
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l14.gated_norm",
-     "kernel": "gated_norm_decode",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.14.linear_attn.norm.weight"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 48
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l14.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.14.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l14.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.14.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l14.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.14.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l14.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l14.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.14.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l14.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.15.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l15.qkv_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.15.self_attn.qkv_proj.weight"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l15.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.15.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l15.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.15.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l15.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l15.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv",
-       "offset": 9633792
-      },
-      {
-       "state": "kv",
-       "offset": 9634304
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l15.attn",
-     "kernel": "attn",
-     "args": [
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv",
-       "offset": 9633792
-      },
-      {
-       "state": "kv",
-       "offset": 9634304
-      },
-      {
-       "buf": "block_table"
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "f32": 0.0625
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 0.0
-      },
-      {
-       "i64": 8
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 0
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l15.gate",
-     "kernel": "sigmoid_mul",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "qkv",
-       "offset": 512
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l15.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "model.layers.15.self_attn.o_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l15.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.15.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l15.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.15.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l15.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l15.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.15.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l15.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.16.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l16.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.16.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l16.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.16.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l16.conv_update",
-     "kernel": "conv_update",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.16.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 52
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 49
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l16.recurrent",
-     "kernel": "recurrent",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.16.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.16.linear_attn.dt_bias"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 52
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l16.gated_norm",
-     "kernel": "gated_norm_decode",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.16.linear_attn.norm.weight"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 48
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l16.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.16.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l16.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.16.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l16.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.16.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l16.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l16.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.16.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l16.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.17.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l17.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.17.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l17.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.17.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l17.conv_update",
-     "kernel": "conv_update",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.17.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 56
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 49
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l17.recurrent",
-     "kernel": "recurrent",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.17.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.17.linear_attn.dt_bias"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 56
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l17.gated_norm",
-     "kernel": "gated_norm_decode",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.17.linear_attn.norm.weight"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 48
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l17.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.17.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l17.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.17.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l17.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.17.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l17.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l17.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.17.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l17.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.18.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l18.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.18.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l18.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.18.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l18.conv_update",
-     "kernel": "conv_update",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.18.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 60
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 49
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l18.recurrent",
-     "kernel": "recurrent",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.18.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.18.linear_attn.dt_bias"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 60
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l18.gated_norm",
-     "kernel": "gated_norm_decode",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.18.linear_attn.norm.weight"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 48
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l18.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.18.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l18.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.18.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l18.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.18.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l18.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l18.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.18.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l18.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.19.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l19.qkv_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.19.self_attn.qkv_proj.weight"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l19.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.19.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l19.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.19.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l19.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l19.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv",
-       "offset": 12845056
-      },
-      {
-       "state": "kv",
-       "offset": 12845568
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l19.attn",
-     "kernel": "attn",
-     "args": [
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv",
-       "offset": 12845056
-      },
-      {
-       "state": "kv",
-       "offset": 12845568
-      },
-      {
-       "buf": "block_table"
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "f32": 0.0625
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 0.0
-      },
-      {
-       "i64": 8
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 0
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l19.gate",
-     "kernel": "sigmoid_mul",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "qkv",
-       "offset": 512
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l19.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "model.layers.19.self_attn.o_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l19.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.19.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l19.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.19.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l19.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l19.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.19.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l19.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.20.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l20.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.20.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l20.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.20.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l20.conv_update",
-     "kernel": "conv_update",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.20.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 64
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 49
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l20.recurrent",
-     "kernel": "recurrent",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.20.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.20.linear_attn.dt_bias"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 64
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l20.gated_norm",
-     "kernel": "gated_norm_decode",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.20.linear_attn.norm.weight"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 48
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l20.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.20.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l20.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.20.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l20.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.20.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l20.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l20.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.20.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l20.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.21.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l21.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.21.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l21.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.21.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l21.conv_update",
-     "kernel": "conv_update",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.21.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 68
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 49
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l21.recurrent",
-     "kernel": "recurrent",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.21.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.21.linear_attn.dt_bias"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 68
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l21.gated_norm",
-     "kernel": "gated_norm_decode",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.21.linear_attn.norm.weight"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 48
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l21.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.21.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l21.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.21.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l21.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.21.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l21.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l21.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.21.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l21.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.22.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l22.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.22.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l22.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.22.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l22.conv_update",
-     "kernel": "conv_update",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.22.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 72
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 49
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l22.recurrent",
-     "kernel": "recurrent",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.22.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.22.linear_attn.dt_bias"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 72
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l22.gated_norm",
-     "kernel": "gated_norm_decode",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.22.linear_attn.norm.weight"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 48
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l22.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.22.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l22.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.22.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l22.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.22.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l22.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l22.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.22.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l22.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.23.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l23.qkv_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.23.self_attn.qkv_proj.weight"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l23.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.23.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l23.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.23.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l23.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l23.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv",
-       "offset": 16056320
-      },
-      {
-       "state": "kv",
-       "offset": 16056832
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l23.attn",
-     "kernel": "attn",
-     "args": [
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv",
-       "offset": 16056320
-      },
-      {
-       "state": "kv",
-       "offset": 16056832
-      },
-      {
-       "buf": "block_table"
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "f32": 0.0625
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 0.0
-      },
-      {
-       "i64": 8
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 0
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l23.gate",
-     "kernel": "sigmoid_mul",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "qkv",
-       "offset": 512
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l23.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "model.layers.23.self_attn.o_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l23.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.23.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l23.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.23.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l23.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l23.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.23.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l23.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.24.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l24.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.24.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l24.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.24.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l24.conv_update",
-     "kernel": "conv_update",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.24.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 76
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 49
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l24.recurrent",
-     "kernel": "recurrent",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.24.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.24.linear_attn.dt_bias"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 76
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l24.gated_norm",
-     "kernel": "gated_norm_decode",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.24.linear_attn.norm.weight"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 48
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l24.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.24.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l24.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.24.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l24.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.24.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l24.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l24.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.24.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l24.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.25.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l25.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.25.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l25.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.25.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l25.conv_update",
-     "kernel": "conv_update",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.25.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 80
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 49
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l25.recurrent",
-     "kernel": "recurrent",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.25.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.25.linear_attn.dt_bias"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 80
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l25.gated_norm",
-     "kernel": "gated_norm_decode",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.25.linear_attn.norm.weight"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 48
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l25.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.25.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l25.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.25.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l25.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.25.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l25.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l25.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.25.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l25.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.26.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l26.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.26.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l26.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.26.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l26.conv_update",
-     "kernel": "conv_update",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.26.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 84
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 49
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l26.recurrent",
-     "kernel": "recurrent",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.26.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.26.linear_attn.dt_bias"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 84
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l26.gated_norm",
-     "kernel": "gated_norm_decode",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.26.linear_attn.norm.weight"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 48
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l26.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.26.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l26.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.26.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l26.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.26.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l26.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l26.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.26.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l26.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.27.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l27.qkv_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.27.self_attn.qkv_proj.weight"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l27.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.27.self_attn.q_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      },
-      {
-       "i32": 6144
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l27.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.27.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l27.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l27.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv",
-       "offset": 19267584
-      },
-      {
-       "state": "kv",
-       "offset": 19268096
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
-      },
-      {
-       "i64": 14336
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l27.attn",
-     "kernel": "attn",
-     "args": [
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv",
-       "offset": 19267584
-      },
-      {
-       "state": "kv",
-       "offset": 19268096
-      },
-      {
-       "buf": "block_table"
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "f32": 0.0625
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 0.0
-      },
-      {
-       "i64": 8
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 0
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
-      },
-      {
-       "i64": 512
-      },
-      {
-       "buf": "cu_seqlens_q"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l27.gate",
-     "kernel": "sigmoid_mul",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "qkv",
-       "offset": 512
-      },
-      {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l27.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
-      },
-      {
-       "buf": "model.layers.27.self_attn.o_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l27.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.27.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l27.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.27.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l27.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l27.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.27.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l27.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.28.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l28.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.28.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l28.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.28.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l28.conv_update",
-     "kernel": "conv_update",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.28.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 88
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 49
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l28.recurrent",
-     "kernel": "recurrent",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.28.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.28.linear_attn.dt_bias"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 88
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l28.gated_norm",
-     "kernel": "gated_norm_decode",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.28.linear_attn.norm.weight"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 48
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l28.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.28.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l28.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.28.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l28.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.28.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l28.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l28.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.28.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l28.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.29.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l29.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.29.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l29.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.29.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l29.conv_update",
-     "kernel": "conv_update",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.29.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 92
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 49
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l29.recurrent",
-     "kernel": "recurrent",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.29.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.29.linear_attn.dt_bias"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 92
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l29.gated_norm",
-     "kernel": "gated_norm_decode",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.29.linear_attn.norm.weight"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 48
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l29.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.29.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l29.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.29.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l29.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.29.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l29.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l29.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.29.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l29.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.30.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l30.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.30.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l30.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.30.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 96
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l30.conv_update",
-     "kernel": "conv_update",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.30.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 96
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 49
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l30.recurrent",
-     "kernel": "recurrent",
-     "args": [
-      {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
-      {
-       "buf": "ba"
-      },
-      {
-       "buf": "model.layers.30.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.30.linear_attn.dt_bias"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "state": "gdn",
-       "offset": 61440
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 96
-      },
-      {
-       "f32": 0.0883883461356163
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l30.gated_norm",
-     "kernel": "gated_norm_decode",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.30.linear_attn.norm.weight"
-      },
-      {
-       "buf": "qkvz",
-       "offset": 20480
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 128
-      },
-      {
-       "i32": 48
-      },
-      {
-       "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l30.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.30.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l30.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.30.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l30.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.30.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l30.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l30.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.30.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l30.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.31.input_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 5120
+       "i32": 512
       },
       {
        "f32": 9.999999974752427e-07
@@ -56590,23 +44199,23 @@
      ]
     },
     {
-     "label": "l31.qkv_proj",
-     "kernel": "gemm",
+     "label": "l3.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.31.self_attn.qkv_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkv"
+       "buf": "model.layers.3.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 14336
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -56614,45 +44223,32 @@
      ]
     },
     {
-     "label": "l31.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l3.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.31.self_attn.q_norm.weight_p1"
+       "buf": "x"
       },
       {
-       "i32": 256
+       "buf": "act"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "buf": "model.layers.3.mlp.down_proj.weight"
       },
       {
-       "i32": 24
+       "buf": "residual"
       },
       {
-       "i32": 14336
+       "buf": "model.layers.4.input_layernorm.weight_p1"
       },
       {
-       "i32": 512
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 5120
       },
       {
-       "i32": 256
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -56660,270 +44256,169 @@
      ]
     },
     {
-     "label": "l31.k_norm",
-     "kernel": "gemma_norm_khead",
+     "label": "l4.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.31.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 1024
+       "buf": "qkvz"
       },
       {
-       "i32": 256
+       "buf": "ba"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l31.rope",
-     "kernel": "mrope",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "x"
       },
       {
-       "buf": "k_n"
+       "buf": "model.layers.4.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "cos_g"
+       "buf": "model.layers.4.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "sin_g"
+       "sym": "tokens"
       },
       {
-       "i32": 0
+       "i32": 16384
       },
       {
-       "i64": 0
+       "i32": 96
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l31.kv_write",
-     "kernel": "reshape_and_cache",
+     "label": "l4.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
-       "buf": "k_n"
+       "buf": "qkvz"
       },
       {
-       "buf": "qkv",
-       "offset": 26624
+       "buf": "model.layers.4.linear_attn.conv1d.weight"
       },
       {
-       "state": "kv",
-       "offset": 22478848
+       "state": "gdn"
       },
       {
-       "state": "kv",
-       "offset": 22479360
+       "state": "gdn",
+       "offset": 61440
       },
       {
-       "buf": "slot_mapping"
+       "buf": "gdn.line_index",
+       "offset": 16
       },
       {
-       "buf": "kv_scales"
+       "buf": "ba"
       },
       {
-       "buf": "kv_scales",
-       "offset": 4
+       "buf": "model.layers.4.linear_attn.A_log"
       },
       {
-       "i64": 1024
+       "buf": "model.layers.4.linear_attn.dt_bias"
       },
       {
-       "i64": 14336
+       "buf": "core_attn_out"
       },
       {
-       "i64": 25690112
+       "buf": "model.layers.4.linear_attn.norm.weight"
       },
       {
-       "i64": 512
+       "f32": 0.0883883461356163
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       },
       {
-       "i64": 0
+       "i32": 49
       },
       {
-       "i64": 2048
+       "i32": 1605632
       },
       {
-       "i64": 0
+       "i32": 10240
       },
       {
-       "i64": 0
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l31.attn",
-     "kernel": "attn",
+     "label": "l4.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv",
-       "offset": 22478848
-      },
-      {
-       "state": "kv",
-       "offset": 22479360
-      },
-      {
-       "buf": "block_table"
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "f32": 0.0625
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "f32": 0.0
-      },
-      {
-       "i64": 8
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 6144
-      },
-      {
-       "i64": 256
-      },
-      {
-       "i64": 0
-      },
-      {
-       "buf": "seq_lens"
-      },
-      {
-       "i64": 25690112
-      },
-      {
-       "i64": 2048
+       "buf": "x"
       },
       {
-       "i64": 512
+       "buf": "core_attn_out"
       },
       {
-       "i64": 25690112
+       "buf": "model.layers.4.linear_attn.out_proj.weight"
       },
       {
-       "i64": 2048
+       "buf": "residual"
       },
       {
-       "i64": 512
+       "buf": "model.layers.4.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "cu_seqlens_q"
+       "sym": "tokens"
       },
       {
-       "i32": 1
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 6144
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l31.gate",
-     "kernel": "sigmoid_mul",
+     "label": "l4.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "gated"
-      },
-      {
-       "buf": "attn_out"
+       "buf": "act"
       },
       {
-       "buf": "qkv",
-       "offset": 512
+       "buf": "x"
       },
       {
-       "i32": 24
+       "buf": "model.layers.4.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 256
+       "sym": "tokens"
       },
       {
-       "i32": 14336
+       "i32": 17408
       },
       {
-       "i32": 512
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l31.o_proj",
-     "kernel": "gemm",
+     "label": "l4.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "gated"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.31.self_attn.o_proj.weight"
+       "buf": "act"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.4.mlp.down_proj.weight"
+      },
+      {
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.5.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -56932,133 +44427,120 @@
        "i32": 5120
       },
       {
-       "i32": 6144
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l31.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l5.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "x"
+       "buf": "qkvz"
       },
       {
-       "buf": "y"
+       "buf": "ba"
       },
       {
-       "buf": "residual"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.31.post_attention_layernorm.weight_p1"
+       "buf": "model.layers.5.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.5.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 16384
       },
       {
-       "i32": 5120
+       "i32": 96
       },
       {
-       "f32": 9.999999974752427e-07
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l31.gate_up",
-     "kernel": "gemm",
+     "label": "l5.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
-       "buf": "x"
+       "buf": "qkvz"
       },
       {
-       "buf": "model.layers.31.mlp.gate_up_proj.weight"
+       "buf": "model.layers.5.linear_attn.conv1d.weight"
       },
       {
-       "buf": "gate_up"
+       "state": "gdn"
       },
       {
-       "sym": "tokens"
+       "state": "gdn",
+       "offset": 61440
       },
       {
-       "i32": 34816
+       "buf": "gdn.line_index",
+       "offset": 20
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l31.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "ba"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.5.linear_attn.A_log"
       },
       {
-       "i32": 17408
+       "buf": "model.layers.5.linear_attn.dt_bias"
       },
       {
-       "i32": 0
+       "buf": "core_attn_out"
       },
       {
-       "f32": 1.0
+       "buf": "model.layers.5.linear_attn.norm.weight"
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l31.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "model.layers.31.mlp.down_proj.weight"
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "y"
+       "i32": 49
       },
       {
-       "sym": "tokens"
+       "i32": 1605632
       },
       {
-       "i32": 5120
+       "i32": 10240
       },
       {
-       "i32": 17408
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l31.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l5.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.5.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.32.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.5.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -57067,7 +44549,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -57075,47 +44557,23 @@
      ]
     },
     {
-     "label": "l32.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l5.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.32.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
+       "buf": "act"
       },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l32.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.32.linear_attn.in_proj_ba.weight"
-      },
-      {
-       "buf": "ba"
+       "buf": "model.layers.5.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 96
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -57123,173 +44581,145 @@
      ]
     },
     {
-     "label": "l32.conv_update",
-     "kernel": "conv_update",
+     "label": "l5.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.32.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
+       "buf": "x"
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 100
+       "buf": "act"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.5.mlp.down_proj.weight"
       },
       {
-       "i32": 1
+       "buf": "residual"
       },
       {
-       "i32": 49
+       "buf": "model.layers.6.input_layernorm.weight_p1"
       },
       {
-       "i64": 1
+       "sym": "tokens"
       },
       {
-       "i64": 1
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 17408
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l32.recurrent",
-     "kernel": "recurrent",
+     "label": "l6.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
        "buf": "qkvz"
       },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
       {
        "buf": "ba"
       },
       {
-       "buf": "model.layers.32.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.32.linear_attn.dt_bias"
-      },
-      {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "buf": "model.layers.6.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "buf": "model.layers.6.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 100
+       "sym": "tokens"
       },
       {
-       "f32": 0.0883883461356163
+       "i32": 16384
       },
       {
-       "i64": 0
+       "i32": 96
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l32.gated_norm",
-     "kernel": "gated_norm_decode",
+     "label": "l6.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "qkvz"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.6.linear_attn.conv1d.weight"
       },
       {
-       "buf": "model.layers.32.linear_attn.norm.weight"
+       "state": "gdn"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "state": "gdn",
+       "offset": 61440
       },
       {
-       "i32": 128
+       "buf": "gdn.line_index",
+       "offset": 24
       },
       {
-       "i32": 128
+       "buf": "ba"
       },
       {
-       "i32": 128
+       "buf": "model.layers.6.linear_attn.A_log"
       },
       {
-       "i32": 48
+       "buf": "model.layers.6.linear_attn.dt_bias"
       },
       {
-       "f32": 9.999999974752427e-07
+       "buf": "core_attn_out"
       },
       {
-       "i64": 0
+       "buf": "model.layers.6.linear_attn.norm.weight"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l32.out_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "core_attn_out"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "model.layers.32.linear_attn.out_proj.weight"
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "y"
+       "i32": 49
       },
       {
-       "sym": "tokens"
+       "i32": 1605632
       },
       {
-       "i32": 5120
+       "i32": 10240
       },
       {
-       "i32": 6144
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l32.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l6.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.6.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.32.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.6.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -57298,7 +44728,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -57306,23 +44736,23 @@
      ]
     },
     {
-     "label": "l32.gate_up",
-     "kernel": "gemm",
+     "label": "l6.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.32.mlp.gate_up_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.6.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -57330,211 +44760,219 @@
      ]
     },
     {
-     "label": "l32.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l6.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
+      {
+       "buf": "x"
+      },
       {
        "buf": "act"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.6.mlp.down_proj.weight"
       },
       {
-       "i32": 17408
+       "buf": "residual"
       },
       {
-       "i32": 0
+       "buf": "model.layers.7.input_layernorm.weight_p1"
       },
       {
-       "f32": 1.0
+       "sym": "tokens"
       },
       {
-       "i32": 0
+       "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l32.down_proj",
+     "label": "l7.qkv_proj",
      "kernel": "gemm",
      "args": [
       {
-       "buf": "act"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.32.mlp.down_proj.weight"
+       "buf": "model.layers.7.self_attn.qkv_proj.weight"
       },
       {
-       "buf": "y"
+       "buf": "qkv"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 14336
       },
       {
-       "i32": 17408
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l32.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l7.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "x"
+       "buf": "q_n"
+      },
+      {
+       "buf": "k_n"
+      },
+      {
+       "buf": "qkv"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.7.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.7.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "model.layers.33.input_layernorm.weight_p1"
+       "buf": "cos_g"
       },
       {
-       "i32": 5120
+       "buf": "sin_g"
       },
       {
-       "sym": "tokens"
+       "state": "kv",
+       "offset": 3211264
       },
       {
-       "i32": 5120
+       "state": "kv",
+       "offset": 3211776
       },
       {
-       "i32": 5120
+       "buf": "slot_mapping"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l33.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
+      },
       {
-       "buf": "x"
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "buf": "model.layers.33.linear_attn.in_proj_qkvz.weight"
+       "i64": 25690112
       },
       {
-       "buf": "qkvz"
+       "i64": 2048
       },
       {
-       "sym": "tokens"
+       "i64": 512
       },
       {
-       "i32": 16384
+       "i32": 784
       },
       {
-       "i32": 5120
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l33.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l7.attn",
+     "kernel": "attn",
      "args": [
       {
-       "buf": "x"
+       "buf": "attn_out"
       },
       {
-       "buf": "model.layers.33.linear_attn.in_proj_ba.weight"
+       "buf": "q_n"
       },
       {
-       "buf": "ba"
+       "state": "kv",
+       "offset": 3211264
       },
       {
-       "sym": "tokens"
+       "state": "kv",
+       "offset": 3211776
       },
       {
-       "i32": 96
+       "buf": "block_table"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l33.conv_update",
-     "kernel": "conv_update",
-     "args": [
+       "buf": "seq_lens"
+      },
       {
-       "buf": "qkvz"
+       "f32": 0.0625
       },
       {
-       "buf": "model.layers.33.linear_attn.conv1d.weight"
+       "buf": "kv_scales"
       },
       {
-       "state": "gdn"
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 104
+       "f32": 1.0
       },
       {
-       "buf": "qkvz"
+       "f32": 0.0
       },
       {
-       "i32": 1
+       "i64": 8
       },
       {
-       "i32": 49
+       "i64": 6144
       },
       {
-       "i64": 1
+       "i64": 256
       },
       {
-       "i64": 1
+       "i64": 6144
       },
       {
-       "i64": 0
+       "i64": 256
       },
       {
        "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l33.recurrent",
-     "kernel": "recurrent",
-     "args": [
-      {
-       "buf": "qkvz"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "seq_lens"
       },
       {
-       "buf": "ba"
+       "i64": 25690112
       },
       {
-       "buf": "model.layers.33.linear_attn.A_log"
+       "i64": 2048
       },
       {
-       "buf": "model.layers.33.linear_attn.dt_bias"
+       "i64": 512
       },
       {
-       "buf": "core_attn_out"
+       "i64": 25690112
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "i64": 2048
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "i64": 512
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 104
+       "buf": "cu_seqlens_q"
       },
       {
-       "f32": 0.0883883461356163
+       "i32": 1
       },
       {
        "i64": 0
@@ -57545,87 +44983,93 @@
      ]
     },
     {
-     "label": "l33.gated_norm",
-     "kernel": "gated_norm_decode",
+     "label": "l7.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "attn_out"
       },
       {
-       "buf": "model.layers.33.linear_attn.norm.weight"
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "model.layers.7.self_attn.o_proj.weight"
       },
       {
-       "i32": 128
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.7.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 5120
       },
       {
-       "f32": 9.999999974752427e-07
+       "i32": 6144
       },
       {
-       "i64": 0
+       "i32": 256
       },
       {
-       "i64": 0
+       "i32": 14336
+      },
+      {
+       "i32": 512
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l33.out_proj",
-     "kernel": "gemm",
+     "label": "l7.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.33.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.7.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l33.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l7.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.7.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.33.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.8.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -57634,7 +45078,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -57642,23 +45086,32 @@
      ]
     },
     {
-     "label": "l33.gate_up",
-     "kernel": "gemm",
+     "label": "l8.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.33.mlp.gate_up_proj.weight"
+       "buf": "model.layers.8.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.8.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -57666,71 +45119,79 @@
      ]
     },
     {
-     "label": "l33.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l8.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.8.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 61440
       },
       {
-       "f32": 1.0
+       "buf": "gdn.line_index",
+       "offset": 28
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l33.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "ba"
+      },
       {
-       "buf": "act"
+       "buf": "model.layers.8.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.33.mlp.down_proj.weight"
+       "buf": "model.layers.8.linear_attn.dt_bias"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.8.linear_attn.norm.weight"
       },
       {
-       "i32": 5120
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 17408
+       "f32": 9.999999974752427e-07
+      },
+      {
+       "i32": 49
+      },
+      {
+       "i32": 1605632
+      },
+      {
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l33.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l8.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.8.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.34.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.8.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -57739,7 +45200,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -57747,23 +45208,23 @@
      ]
     },
     {
-     "label": "l34.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l8.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.34.linear_attn.in_proj_qkvz.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.8.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -57771,197 +45232,202 @@
      ]
     },
     {
-     "label": "l34.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l8.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.34.linear_attn.in_proj_ba.weight"
+       "buf": "act"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.8.mlp.down_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 96
+       "buf": "model.layers.9.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l34.conv_update",
-     "kernel": "conv_update",
+     "label": "l9.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.34.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 108
+       "buf": "ba"
       },
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "i32": 1
+       "buf": "model.layers.9.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i32": 49
+       "buf": "model.layers.9.linear_attn.in_proj_ba.weight"
       },
       {
-       "i64": 1
+       "sym": "tokens"
       },
       {
-       "i64": 1
+       "i32": 16384
       },
       {
-       "i64": 0
+       "i32": 96
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l34.recurrent",
-     "kernel": "recurrent",
+     "label": "l9.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.9.linear_attn.conv1d.weight"
+      },
+      {
+       "state": "gdn"
+      },
+      {
+       "state": "gdn",
+       "offset": 61440
+      },
+      {
+       "buf": "gdn.line_index",
+       "offset": 32
       },
       {
        "buf": "ba"
       },
       {
-       "buf": "model.layers.34.linear_attn.A_log"
+       "buf": "model.layers.9.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.34.linear_attn.dt_bias"
+       "buf": "model.layers.9.linear_attn.dt_bias"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "buf": "model.layers.9.linear_attn.norm.weight"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 108
+       "f32": 9.999999974752427e-07
       },
       {
-       "f32": 0.0883883461356163
+       "i32": 49
       },
       {
-       "i64": 0
+       "i32": 1605632
       },
       {
-       "i64": 0
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l34.gated_norm",
-     "kernel": "gated_norm_decode",
+     "label": "l9.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.34.linear_attn.norm.weight"
+       "buf": "model.layers.9.linear_attn.out_proj.weight"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.9.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "i32": 48
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l34.out_proj",
-     "kernel": "gemm",
+     "label": "l9.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.34.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.9.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l34.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l9.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.9.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.34.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.10.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -57970,7 +45436,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -57978,221 +45444,178 @@
      ]
     },
     {
-     "label": "l34.gate_up",
-     "kernel": "gemm",
+     "label": "l10.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.34.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
+       "buf": "qkvz"
       },
       {
-       "sym": "tokens"
+       "buf": "ba"
       },
       {
-       "i32": 34816
+       "buf": "x"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l34.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "model.layers.10.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.10.linear_attn.in_proj_ba.weight"
       },
       {
-       "i32": 17408
+       "sym": "tokens"
       },
       {
-       "i32": 0
+       "i32": 16384
       },
       {
-       "f32": 1.0
+       "i32": 96
       },
       {
-       "i32": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l34.down_proj",
-     "kernel": "gemm",
+     "label": "l10.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "model.layers.34.mlp.down_proj.weight"
+       "buf": "model.layers.10.linear_attn.conv1d.weight"
       },
       {
-       "buf": "y"
+       "state": "gdn"
       },
       {
-       "sym": "tokens"
+       "state": "gdn",
+       "offset": 61440
       },
       {
-       "i32": 5120
+       "buf": "gdn.line_index",
+       "offset": 36
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l34.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "ba"
+      },
       {
-       "buf": "x"
+       "buf": "model.layers.10.linear_attn.A_log"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.10.linear_attn.dt_bias"
       },
       {
-       "buf": "residual"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.35.input_layernorm.weight_p1"
+       "buf": "model.layers.10.linear_attn.norm.weight"
       },
       {
-       "i32": 5120
+       "f32": 0.0883883461356163
       },
       {
-       "sym": "tokens"
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 5120
+       "i32": 49
       },
       {
-       "i32": 5120
+       "i32": 1605632
       },
       {
-       "f32": 9.999999974752427e-07
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l35.qkv_proj",
-     "kernel": "gemm",
+     "label": "l10.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.35.self_attn.qkv_proj.weight"
-      },
-      {
-       "buf": "qkv"
+       "buf": "core_attn_out"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.10.linear_attn.out_proj.weight"
       },
       {
-       "i32": 14336
+       "buf": "residual"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l35.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "model.layers.10.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "qkv"
+       "sym": "tokens"
       },
       {
-       "buf": "model.layers.35.self_attn.q_norm.weight_p1"
+       "i32": 5120
       },
       {
-       "i32": 256
+       "i32": 6144
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
-      },
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l10.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
+     "args": [
       {
-       "i32": 24
+       "buf": "act"
       },
       {
-       "i32": 14336
+       "buf": "x"
       },
       {
-       "i32": 512
+       "buf": "model.layers.10.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 6144
+       "sym": "tokens"
       },
       {
-       "i32": 256
+       "i32": 17408
       },
       {
-       "f32": 9.999999974752427e-07
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l35.k_norm",
-     "kernel": "gemma_norm_khead",
+     "label": "l10.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.35.self_attn.k_norm.weight_p1"
+       "buf": "x"
       },
       {
-       "i32": 256
+       "buf": "act"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
+       "buf": "model.layers.10.mlp.down_proj.weight"
       },
       {
-       "i32": 4
+       "buf": "residual"
       },
       {
-       "i32": 14336
+       "buf": "model.layers.11.input_layernorm.weight_p1"
       },
       {
-       "i32": 256
+       "sym": "tokens"
       },
       {
-       "i32": 1024
+       "i32": 5120
       },
       {
-       "i32": 256
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -58200,92 +45623,104 @@
      ]
     },
     {
-     "label": "l35.rope",
-     "kernel": "mrope",
+     "label": "l11.qkv_proj",
+     "kernel": "gemm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
+       "buf": "x"
       },
       {
-       "buf": "cos_g"
+       "buf": "model.layers.11.self_attn.qkv_proj.weight"
       },
       {
-       "buf": "sin_g"
+       "buf": "qkv"
       },
       {
-       "i32": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l35.kv_write",
-     "kernel": "reshape_and_cache",
+     "label": "l11.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "k_n"
+       "buf": "q_n"
       },
       {
-       "buf": "qkv",
-       "offset": 26624
+       "buf": "k_n"
       },
       {
-       "state": "kv",
-       "offset": 25690112
+       "buf": "qkv"
       },
       {
-       "state": "kv",
-       "offset": 25690624
+       "buf": "model.layers.11.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "slot_mapping"
+       "buf": "model.layers.11.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "kv_scales"
+       "buf": "cos_g"
       },
       {
-       "buf": "kv_scales",
-       "offset": 4
+       "buf": "sin_g"
       },
       {
-       "i64": 1024
+       "state": "kv",
+       "offset": 6422528
       },
       {
-       "i64": 14336
+       "state": "kv",
+       "offset": 6423040
       },
       {
-       "i64": 25690112
+       "buf": "slot_mapping"
       },
       {
-       "i64": 512
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 0
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i64": 0
+       "i64": 25690112
       },
       {
        "i64": 2048
       },
       {
-       "i64": 0
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l35.attn",
+     "label": "l11.attn",
      "kernel": "attn",
      "args": [
       {
@@ -58296,11 +45731,11 @@
       },
       {
        "state": "kv",
-       "offset": 25690112
+       "offset": 6422528
       },
       {
        "state": "kv",
-       "offset": 25690624
+       "offset": 6423040
       },
       {
        "buf": "block_table"
@@ -58378,11 +45813,11 @@
      ]
     },
     {
-     "label": "l35.gate",
-     "kernel": "sigmoid_mul",
+     "label": "l11.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "gated"
+       "buf": "x"
       },
       {
        "buf": "attn_out"
@@ -58392,7 +45827,22 @@
        "offset": 512
       },
       {
-       "i32": 24
+       "buf": "model.layers.11.self_attn.o_proj.weight"
+      },
+      {
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.11.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 6144
       },
       {
        "i32": 256
@@ -58402,51 +45852,54 @@
       },
       {
        "i32": 512
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l35.o_proj",
-     "kernel": "gemm",
+     "label": "l11.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "gated"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.35.self_attn.o_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.11.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l35.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l11.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.11.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.35.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.12.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -58455,7 +45908,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -58463,23 +45916,32 @@
      ]
     },
     {
-     "label": "l35.gate_up",
-     "kernel": "gemm",
+     "label": "l12.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.35.mlp.gate_up_proj.weight"
+       "buf": "model.layers.12.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.12.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -58487,71 +45949,79 @@
      ]
     },
     {
-     "label": "l35.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l12.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.12.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 61440
       },
       {
-       "f32": 1.0
+       "buf": "gdn.line_index",
+       "offset": 40
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l35.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "ba"
+      },
       {
-       "buf": "act"
+       "buf": "model.layers.12.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.35.mlp.down_proj.weight"
+       "buf": "model.layers.12.linear_attn.dt_bias"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.12.linear_attn.norm.weight"
       },
       {
-       "i32": 5120
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 17408
+       "f32": 9.999999974752427e-07
+      },
+      {
+       "i32": 49
+      },
+      {
+       "i32": 1605632
+      },
+      {
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l35.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l12.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.12.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.36.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.12.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -58560,7 +46030,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -58568,23 +46038,23 @@
      ]
     },
     {
-     "label": "l36.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l12.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.36.linear_attn.in_proj_qkvz.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.12.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -58592,21 +46062,63 @@
      ]
     },
     {
-     "label": "l36.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l12.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.36.linear_attn.in_proj_ba.weight"
+       "buf": "act"
+      },
+      {
+       "buf": "model.layers.12.mlp.down_proj.weight"
+      },
+      {
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.13.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l13.in_proj",
+     "kernel": "gemm8_dual",
+     "args": [
+      {
+       "buf": "qkvz"
       },
       {
        "buf": "ba"
       },
+      {
+       "buf": "x"
+      },
+      {
+       "buf": "model.layers.13.linear_attn.in_proj_qkvz.weight"
+      },
+      {
+       "buf": "model.layers.13.linear_attn.in_proj_ba.weight"
+      },
       {
        "sym": "tokens"
       },
+      {
+       "i32": 16384
+      },
       {
        "i32": 96
       },
@@ -58616,173 +46128,258 @@
      ]
     },
     {
-     "label": "l36.conv_update",
-     "kernel": "conv_update",
+     "label": "l13.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.36.linear_attn.conv1d.weight"
+       "buf": "model.layers.13.linear_attn.conv1d.weight"
       },
       {
        "state": "gdn"
       },
+      {
+       "state": "gdn",
+       "offset": 61440
+      },
       {
        "buf": "gdn.line_index",
-       "offset": 112
+       "offset": 44
       },
       {
-       "buf": "qkvz"
+       "buf": "ba"
       },
       {
-       "i32": 1
+       "buf": "model.layers.13.linear_attn.A_log"
       },
       {
-       "i32": 49
+       "buf": "model.layers.13.linear_attn.dt_bias"
+      },
+      {
+       "buf": "core_attn_out"
       },
       {
-       "i64": 1
+       "buf": "model.layers.13.linear_attn.norm.weight"
       },
       {
-       "i64": 1
+       "f32": 0.0883883461356163
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       },
       {
-       "i64": 0
+       "i32": 49
+      },
+      {
+       "i32": 1605632
+      },
+      {
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l36.recurrent",
-     "kernel": "recurrent",
+     "label": "l13.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "core_attn_out"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.13.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.36.linear_attn.A_log"
+       "buf": "residual"
       },
       {
-       "buf": "model.layers.36.linear_attn.dt_bias"
+       "buf": "model.layers.13.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "core_attn_out"
+       "sym": "tokens"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "i32": 5120
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "i32": 6144
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 112
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l13.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
+     "args": [
+      {
+       "buf": "act"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "x"
       },
       {
-       "i64": 0
+       "buf": "model.layers.13.mlp.gate_up_proj.weight"
       },
       {
-       "i64": 0
+       "sym": "tokens"
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l36.gated_norm",
-     "kernel": "gated_norm_decode",
+     "label": "l13.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.36.linear_attn.norm.weight"
+       "buf": "model.layers.13.mlp.down_proj.weight"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.14.input_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "i32": 48
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l14.in_proj",
+     "kernel": "gemm8_dual",
+     "args": [
+      {
+       "buf": "qkvz"
       },
       {
-       "i64": 0
+       "buf": "ba"
       },
       {
-       "i64": 0
+       "buf": "x"
+      },
+      {
+       "buf": "model.layers.14.linear_attn.in_proj_qkvz.weight"
+      },
+      {
+       "buf": "model.layers.14.linear_attn.in_proj_ba.weight"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 16384
+      },
+      {
+       "i32": 96
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l36.out_proj",
-     "kernel": "gemm",
+     "label": "l14.gdn",
+     "kernel": "gdn_decode",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "model.layers.14.linear_attn.conv1d.weight"
+      },
+      {
+       "state": "gdn"
+      },
+      {
+       "state": "gdn",
+       "offset": 61440
+      },
+      {
+       "buf": "gdn.line_index",
+       "offset": 48
+      },
+      {
+       "buf": "ba"
+      },
+      {
+       "buf": "model.layers.14.linear_attn.A_log"
+      },
+      {
+       "buf": "model.layers.14.linear_attn.dt_bias"
+      },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.36.linear_attn.out_proj.weight"
+       "buf": "model.layers.14.linear_attn.norm.weight"
       },
       {
-       "buf": "y"
+       "f32": 0.0883883461356163
       },
       {
-       "sym": "tokens"
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 5120
+       "i32": 49
       },
       {
-       "i32": 6144
+       "i32": 1605632
+      },
+      {
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l36.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l14.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.14.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.36.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.14.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -58791,7 +46388,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -58799,23 +46396,23 @@
      ]
     },
     {
-     "label": "l36.gate_up",
-     "kernel": "gemm",
+     "label": "l14.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.36.mlp.gate_up_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.14.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -58823,211 +46420,219 @@
      ]
     },
     {
-     "label": "l36.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l14.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
+      {
+       "buf": "x"
+      },
       {
        "buf": "act"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.14.mlp.down_proj.weight"
       },
       {
-       "i32": 17408
+       "buf": "residual"
       },
       {
-       "i32": 0
+       "buf": "model.layers.15.input_layernorm.weight_p1"
       },
       {
-       "f32": 1.0
+       "sym": "tokens"
       },
       {
-       "i32": 0
+       "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l36.down_proj",
+     "label": "l15.qkv_proj",
      "kernel": "gemm",
      "args": [
       {
-       "buf": "act"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.36.mlp.down_proj.weight"
+       "buf": "model.layers.15.self_attn.qkv_proj.weight"
       },
       {
-       "buf": "y"
+       "buf": "qkv"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 14336
       },
       {
-       "i32": 17408
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l36.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l15.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "x"
+       "buf": "q_n"
       },
       {
-       "buf": "y"
+       "buf": "k_n"
       },
       {
-       "buf": "residual"
+       "buf": "qkv"
       },
       {
-       "buf": "model.layers.37.input_layernorm.weight_p1"
+       "buf": "model.layers.15.self_attn.q_norm.weight_p1"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.15.self_attn.k_norm.weight_p1"
+      },
+      {
+       "buf": "cos_g"
+      },
+      {
+       "buf": "sin_g"
       },
       {
-       "sym": "tokens"
+       "state": "kv",
+       "offset": 9633792
       },
       {
-       "i32": 5120
+       "state": "kv",
+       "offset": 9634304
       },
       {
-       "i32": 5120
+       "buf": "slot_mapping"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l37.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
+      },
       {
-       "buf": "x"
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "buf": "model.layers.37.linear_attn.in_proj_qkvz.weight"
+       "i64": 25690112
       },
       {
-       "buf": "qkvz"
+       "i64": 2048
       },
       {
-       "sym": "tokens"
+       "i64": 512
       },
       {
-       "i32": 16384
+       "i32": 784
       },
       {
-       "i32": 5120
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l37.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l15.attn",
+     "kernel": "attn",
      "args": [
       {
-       "buf": "x"
+       "buf": "attn_out"
       },
       {
-       "buf": "model.layers.37.linear_attn.in_proj_ba.weight"
+       "buf": "q_n"
       },
       {
-       "buf": "ba"
+       "state": "kv",
+       "offset": 9633792
       },
       {
-       "sym": "tokens"
+       "state": "kv",
+       "offset": 9634304
       },
       {
-       "i32": 96
+       "buf": "block_table"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l37.conv_update",
-     "kernel": "conv_update",
-     "args": [
+       "buf": "seq_lens"
+      },
       {
-       "buf": "qkvz"
+       "f32": 0.0625
       },
       {
-       "buf": "model.layers.37.linear_attn.conv1d.weight"
+       "buf": "kv_scales"
       },
       {
-       "state": "gdn"
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 116
+       "f32": 1.0
       },
       {
-       "buf": "qkvz"
+       "f32": 0.0
       },
       {
-       "i32": 1
+       "i64": 8
       },
       {
-       "i32": 49
+       "i64": 6144
       },
       {
-       "i64": 1
+       "i64": 256
       },
       {
-       "i64": 1
+       "i64": 6144
       },
       {
-       "i64": 0
+       "i64": 256
       },
       {
        "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l37.recurrent",
-     "kernel": "recurrent",
-     "args": [
-      {
-       "buf": "qkvz"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "seq_lens"
       },
       {
-       "buf": "ba"
+       "i64": 25690112
       },
       {
-       "buf": "model.layers.37.linear_attn.A_log"
+       "i64": 2048
       },
       {
-       "buf": "model.layers.37.linear_attn.dt_bias"
+       "i64": 512
       },
       {
-       "buf": "core_attn_out"
+       "i64": 25690112
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "i64": 2048
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "i64": 512
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 116
+       "buf": "cu_seqlens_q"
       },
       {
-       "f32": 0.0883883461356163
+       "i32": 1
       },
       {
        "i64": 0
@@ -59038,87 +46643,93 @@
      ]
     },
     {
-     "label": "l37.gated_norm",
-     "kernel": "gated_norm_decode",
+     "label": "l15.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "attn_out"
       },
       {
-       "buf": "model.layers.37.linear_attn.norm.weight"
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "model.layers.15.self_attn.o_proj.weight"
       },
       {
-       "i32": 128
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.15.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 5120
       },
       {
-       "f32": 9.999999974752427e-07
+       "i32": 6144
       },
       {
-       "i64": 0
+       "i32": 256
       },
       {
-       "i64": 0
+       "i32": 14336
+      },
+      {
+       "i32": 512
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l37.out_proj",
-     "kernel": "gemm",
+     "label": "l15.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.37.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.15.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l37.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l15.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.15.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.37.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.16.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -59127,7 +46738,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -59135,23 +46746,32 @@
      ]
     },
     {
-     "label": "l37.gate_up",
-     "kernel": "gemm",
+     "label": "l16.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.37.mlp.gate_up_proj.weight"
+       "buf": "model.layers.16.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.16.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -59159,71 +46779,79 @@
      ]
     },
     {
-     "label": "l37.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l16.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.16.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 61440
       },
       {
-       "f32": 1.0
+       "buf": "gdn.line_index",
+       "offset": 52
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l37.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "ba"
+      },
       {
-       "buf": "act"
+       "buf": "model.layers.16.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.37.mlp.down_proj.weight"
+       "buf": "model.layers.16.linear_attn.dt_bias"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.16.linear_attn.norm.weight"
       },
       {
-       "i32": 5120
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 17408
+       "f32": 9.999999974752427e-07
+      },
+      {
+       "i32": 49
+      },
+      {
+       "i32": 1605632
+      },
+      {
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l37.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l16.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.16.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.38.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.16.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -59232,7 +46860,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -59240,23 +46868,23 @@
      ]
     },
     {
-     "label": "l38.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l16.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.38.linear_attn.in_proj_qkvz.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.16.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -59264,197 +46892,202 @@
      ]
     },
     {
-     "label": "l38.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l16.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.38.linear_attn.in_proj_ba.weight"
+       "buf": "act"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.16.mlp.down_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 96
+       "buf": "model.layers.17.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l38.conv_update",
-     "kernel": "conv_update",
+     "label": "l17.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.38.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 120
+       "buf": "ba"
       },
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "i32": 1
+       "buf": "model.layers.17.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i32": 49
+       "buf": "model.layers.17.linear_attn.in_proj_ba.weight"
       },
       {
-       "i64": 1
+       "sym": "tokens"
       },
       {
-       "i64": 1
+       "i32": 16384
       },
       {
-       "i64": 0
+       "i32": 96
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l38.recurrent",
-     "kernel": "recurrent",
+     "label": "l17.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.17.linear_attn.conv1d.weight"
+      },
+      {
+       "state": "gdn"
+      },
+      {
+       "state": "gdn",
+       "offset": 61440
+      },
+      {
+       "buf": "gdn.line_index",
+       "offset": 56
       },
       {
        "buf": "ba"
       },
       {
-       "buf": "model.layers.38.linear_attn.A_log"
+       "buf": "model.layers.17.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.38.linear_attn.dt_bias"
+       "buf": "model.layers.17.linear_attn.dt_bias"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "buf": "model.layers.17.linear_attn.norm.weight"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 120
+       "f32": 9.999999974752427e-07
       },
       {
-       "f32": 0.0883883461356163
+       "i32": 49
       },
       {
-       "i64": 0
+       "i32": 1605632
       },
       {
-       "i64": 0
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l38.gated_norm",
-     "kernel": "gated_norm_decode",
+     "label": "l17.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.38.linear_attn.norm.weight"
+       "buf": "model.layers.17.linear_attn.out_proj.weight"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.17.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "i32": 48
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l38.out_proj",
-     "kernel": "gemm",
+     "label": "l17.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.38.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.17.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l38.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l17.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.17.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.38.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.18.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -59463,7 +47096,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -59471,221 +47104,178 @@
      ]
     },
     {
-     "label": "l38.gate_up",
-     "kernel": "gemm",
+     "label": "l18.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.38.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
+       "buf": "qkvz"
       },
       {
-       "sym": "tokens"
+       "buf": "ba"
       },
       {
-       "i32": 34816
+       "buf": "x"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l38.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "model.layers.18.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.18.linear_attn.in_proj_ba.weight"
       },
       {
-       "i32": 17408
+       "sym": "tokens"
       },
       {
-       "i32": 0
+       "i32": 16384
       },
       {
-       "f32": 1.0
+       "i32": 96
       },
       {
-       "i32": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l38.down_proj",
-     "kernel": "gemm",
+     "label": "l18.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.38.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
+       "buf": "qkvz"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.18.linear_attn.conv1d.weight"
       },
       {
-       "i32": 5120
+       "state": "gdn"
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l38.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
+       "state": "gdn",
+       "offset": 61440
       },
       {
-       "buf": "y"
+       "buf": "gdn.line_index",
+       "offset": 60
       },
       {
-       "buf": "residual"
+       "buf": "ba"
       },
       {
-       "buf": "model.layers.39.input_layernorm.weight_p1"
+       "buf": "model.layers.18.linear_attn.A_log"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.18.linear_attn.dt_bias"
       },
       {
-       "sym": "tokens"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.18.linear_attn.norm.weight"
       },
       {
-       "i32": 5120
+       "f32": 0.0883883461356163
       },
       {
        "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l39.qkv_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.39.self_attn.qkv_proj.weight"
       },
       {
-       "buf": "qkv"
+       "i32": 49
       },
       {
-       "sym": "tokens"
+       "i32": 1605632
       },
       {
-       "i32": 14336
+       "i32": 10240
       },
       {
-       "i32": 5120
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l39.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l18.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.39.self_attn.q_norm.weight_p1"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 256
+       "buf": "model.layers.18.linear_attn.out_proj.weight"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "buf": "residual"
       },
       {
-       "i32": 24
+       "buf": "model.layers.18.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 14336
+       "sym": "tokens"
       },
       {
-       "i32": 512
+       "i32": 5120
       },
       {
        "i32": 6144
       },
-      {
-       "i32": 256
-      },
       {
        "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l39.k_norm",
-     "kernel": "gemma_norm_khead",
+     "label": "l18.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "k_n"
+       "buf": "act"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
+       "buf": "x"
+      },
+      {
+       "buf": "model.layers.18.mlp.gate_up_proj.weight"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 17408
       },
       {
-       "buf": "model.layers.39.self_attn.k_norm.weight_p1"
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l18.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "i32": 256
+       "buf": "act"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
+       "buf": "model.layers.18.mlp.down_proj.weight"
       },
       {
-       "i32": 4
+       "buf": "residual"
       },
       {
-       "i32": 14336
+       "buf": "model.layers.19.input_layernorm.weight_p1"
       },
       {
-       "i32": 256
+       "sym": "tokens"
       },
       {
-       "i32": 1024
+       "i32": 5120
       },
       {
-       "i32": 256
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -59693,92 +47283,104 @@
      ]
     },
     {
-     "label": "l39.rope",
-     "kernel": "mrope",
+     "label": "l19.qkv_proj",
+     "kernel": "gemm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
+       "buf": "x"
       },
       {
-       "buf": "cos_g"
+       "buf": "model.layers.19.self_attn.qkv_proj.weight"
       },
       {
-       "buf": "sin_g"
+       "buf": "qkv"
       },
       {
-       "i32": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 14336
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l39.kv_write",
-     "kernel": "reshape_and_cache",
+     "label": "l19.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "k_n"
+       "buf": "q_n"
       },
       {
-       "buf": "qkv",
-       "offset": 26624
+       "buf": "k_n"
       },
       {
-       "state": "kv",
-       "offset": 28901376
+       "buf": "qkv"
       },
       {
-       "state": "kv",
-       "offset": 28901888
+       "buf": "model.layers.19.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "slot_mapping"
+       "buf": "model.layers.19.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "kv_scales"
+       "buf": "cos_g"
       },
       {
-       "buf": "kv_scales",
-       "offset": 4
+       "buf": "sin_g"
       },
       {
-       "i64": 1024
+       "state": "kv",
+       "offset": 12845056
       },
       {
-       "i64": 14336
+       "state": "kv",
+       "offset": 12845568
       },
       {
-       "i64": 25690112
+       "buf": "slot_mapping"
       },
       {
-       "i64": 512
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 0
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i64": 0
+       "i64": 25690112
       },
       {
        "i64": 2048
       },
       {
-       "i64": 0
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l39.attn",
+     "label": "l19.attn",
      "kernel": "attn",
      "args": [
       {
@@ -59789,11 +47391,11 @@
       },
       {
        "state": "kv",
-       "offset": 28901376
+       "offset": 12845056
       },
       {
        "state": "kv",
-       "offset": 28901888
+       "offset": 12845568
       },
       {
        "buf": "block_table"
@@ -59871,11 +47473,11 @@
      ]
     },
     {
-     "label": "l39.gate",
-     "kernel": "sigmoid_mul",
+     "label": "l19.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "gated"
+       "buf": "x"
       },
       {
        "buf": "attn_out"
@@ -59885,7 +47487,22 @@
        "offset": 512
       },
       {
-       "i32": 24
+       "buf": "model.layers.19.self_attn.o_proj.weight"
+      },
+      {
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.19.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 6144
       },
       {
        "i32": 256
@@ -59895,51 +47512,54 @@
       },
       {
        "i32": 512
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l39.o_proj",
-     "kernel": "gemm",
+     "label": "l19.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "gated"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.39.self_attn.o_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.19.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l39.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l19.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.19.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.39.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.20.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -59948,7 +47568,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -59956,23 +47576,32 @@
      ]
     },
     {
-     "label": "l39.gate_up",
-     "kernel": "gemm",
+     "label": "l20.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.39.mlp.gate_up_proj.weight"
+       "buf": "model.layers.20.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.20.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -59980,71 +47609,79 @@
      ]
     },
     {
-     "label": "l39.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l20.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.20.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 61440
       },
       {
-       "f32": 1.0
+       "buf": "gdn.line_index",
+       "offset": 64
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l39.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "ba"
+      },
       {
-       "buf": "act"
+       "buf": "model.layers.20.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.39.mlp.down_proj.weight"
+       "buf": "model.layers.20.linear_attn.dt_bias"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.20.linear_attn.norm.weight"
       },
       {
-       "i32": 5120
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 17408
+       "f32": 9.999999974752427e-07
+      },
+      {
+       "i32": 49
+      },
+      {
+       "i32": 1605632
+      },
+      {
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l39.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l20.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.20.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.40.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.20.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -60053,7 +47690,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -60061,23 +47698,23 @@
      ]
     },
     {
-     "label": "l40.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l20.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.40.linear_attn.in_proj_qkvz.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.20.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -60085,21 +47722,63 @@
      ]
     },
     {
-     "label": "l40.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l20.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.40.linear_attn.in_proj_ba.weight"
+       "buf": "act"
+      },
+      {
+       "buf": "model.layers.20.mlp.down_proj.weight"
+      },
+      {
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.21.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l21.in_proj",
+     "kernel": "gemm8_dual",
+     "args": [
+      {
+       "buf": "qkvz"
       },
       {
        "buf": "ba"
       },
+      {
+       "buf": "x"
+      },
+      {
+       "buf": "model.layers.21.linear_attn.in_proj_qkvz.weight"
+      },
+      {
+       "buf": "model.layers.21.linear_attn.in_proj_ba.weight"
+      },
       {
        "sym": "tokens"
       },
+      {
+       "i32": 16384
+      },
       {
        "i32": 96
       },
@@ -60109,173 +47788,258 @@
      ]
     },
     {
-     "label": "l40.conv_update",
-     "kernel": "conv_update",
+     "label": "l21.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.40.linear_attn.conv1d.weight"
+       "buf": "model.layers.21.linear_attn.conv1d.weight"
       },
       {
        "state": "gdn"
       },
+      {
+       "state": "gdn",
+       "offset": 61440
+      },
       {
        "buf": "gdn.line_index",
-       "offset": 124
+       "offset": 68
       },
       {
-       "buf": "qkvz"
+       "buf": "ba"
       },
       {
-       "i32": 1
+       "buf": "model.layers.21.linear_attn.A_log"
       },
       {
-       "i32": 49
+       "buf": "model.layers.21.linear_attn.dt_bias"
+      },
+      {
+       "buf": "core_attn_out"
       },
       {
-       "i64": 1
+       "buf": "model.layers.21.linear_attn.norm.weight"
       },
       {
-       "i64": 1
+       "f32": 0.0883883461356163
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       },
       {
-       "i64": 0
+       "i32": 49
+      },
+      {
+       "i32": 1605632
+      },
+      {
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l40.recurrent",
-     "kernel": "recurrent",
+     "label": "l21.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "core_attn_out"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.21.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.40.linear_attn.A_log"
+       "buf": "residual"
       },
       {
-       "buf": "model.layers.40.linear_attn.dt_bias"
+       "buf": "model.layers.21.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "core_attn_out"
+       "sym": "tokens"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "i32": 5120
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "i32": 6144
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 124
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l21.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
+     "args": [
+      {
+       "buf": "act"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "x"
       },
       {
-       "i64": 0
+       "buf": "model.layers.21.mlp.gate_up_proj.weight"
       },
       {
-       "i64": 0
+       "sym": "tokens"
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l40.gated_norm",
-     "kernel": "gated_norm_decode",
+     "label": "l21.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.40.linear_attn.norm.weight"
+       "buf": "model.layers.21.mlp.down_proj.weight"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.22.input_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "i32": 48
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l22.in_proj",
+     "kernel": "gemm8_dual",
+     "args": [
+      {
+       "buf": "qkvz"
       },
       {
-       "i64": 0
+       "buf": "ba"
       },
       {
-       "i64": 0
+       "buf": "x"
+      },
+      {
+       "buf": "model.layers.22.linear_attn.in_proj_qkvz.weight"
+      },
+      {
+       "buf": "model.layers.22.linear_attn.in_proj_ba.weight"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 16384
+      },
+      {
+       "i32": 96
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l40.out_proj",
-     "kernel": "gemm",
+     "label": "l22.gdn",
+     "kernel": "gdn_decode",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "model.layers.22.linear_attn.conv1d.weight"
+      },
+      {
+       "state": "gdn"
+      },
+      {
+       "state": "gdn",
+       "offset": 61440
+      },
+      {
+       "buf": "gdn.line_index",
+       "offset": 72
+      },
+      {
+       "buf": "ba"
+      },
+      {
+       "buf": "model.layers.22.linear_attn.A_log"
+      },
+      {
+       "buf": "model.layers.22.linear_attn.dt_bias"
+      },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.40.linear_attn.out_proj.weight"
+       "buf": "model.layers.22.linear_attn.norm.weight"
       },
       {
-       "buf": "y"
+       "f32": 0.0883883461356163
       },
       {
-       "sym": "tokens"
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 5120
+       "i32": 49
       },
       {
-       "i32": 6144
+       "i32": 1605632
+      },
+      {
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l40.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l22.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.22.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.40.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.22.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -60284,7 +48048,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -60292,23 +48056,23 @@
      ]
     },
     {
-     "label": "l40.gate_up",
-     "kernel": "gemm",
+     "label": "l22.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.40.mlp.gate_up_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.22.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -60316,211 +48080,219 @@
      ]
     },
     {
-     "label": "l40.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l22.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
+      {
+       "buf": "x"
+      },
       {
        "buf": "act"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.22.mlp.down_proj.weight"
       },
       {
-       "i32": 17408
+       "buf": "residual"
       },
       {
-       "i32": 0
+       "buf": "model.layers.23.input_layernorm.weight_p1"
       },
       {
-       "f32": 1.0
+       "sym": "tokens"
       },
       {
-       "i32": 0
+       "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l40.down_proj",
+     "label": "l23.qkv_proj",
      "kernel": "gemm",
      "args": [
       {
-       "buf": "act"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.40.mlp.down_proj.weight"
+       "buf": "model.layers.23.self_attn.qkv_proj.weight"
       },
       {
-       "buf": "y"
+       "buf": "qkv"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 14336
       },
       {
-       "i32": 17408
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l40.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l23.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "x"
+       "buf": "q_n"
       },
       {
-       "buf": "y"
+       "buf": "k_n"
       },
       {
-       "buf": "residual"
+       "buf": "qkv"
       },
       {
-       "buf": "model.layers.41.input_layernorm.weight_p1"
+       "buf": "model.layers.23.self_attn.q_norm.weight_p1"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.23.self_attn.k_norm.weight_p1"
       },
       {
-       "sym": "tokens"
+       "buf": "cos_g"
+      },
+      {
+       "buf": "sin_g"
+      },
+      {
+       "state": "kv",
+       "offset": 16056320
+      },
+      {
+       "state": "kv",
+       "offset": 16056832
       },
       {
-       "i32": 5120
+       "buf": "slot_mapping"
       },
       {
-       "i32": 5120
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l41.in_proj_qkvz",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "buf": "model.layers.41.linear_attn.in_proj_qkvz.weight"
+       "i64": 25690112
       },
       {
-       "buf": "qkvz"
+       "i64": 2048
       },
       {
-       "sym": "tokens"
+       "i64": 512
       },
       {
-       "i32": 16384
+       "i32": 784
       },
       {
-       "i32": 5120
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l41.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l23.attn",
+     "kernel": "attn",
      "args": [
       {
-       "buf": "x"
+       "buf": "attn_out"
       },
       {
-       "buf": "model.layers.41.linear_attn.in_proj_ba.weight"
+       "buf": "q_n"
       },
       {
-       "buf": "ba"
+       "state": "kv",
+       "offset": 16056320
       },
       {
-       "sym": "tokens"
+       "state": "kv",
+       "offset": 16056832
       },
       {
-       "i32": 96
+       "buf": "block_table"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l41.conv_update",
-     "kernel": "conv_update",
-     "args": [
+       "buf": "seq_lens"
+      },
       {
-       "buf": "qkvz"
+       "f32": 0.0625
       },
       {
-       "buf": "model.layers.41.linear_attn.conv1d.weight"
+       "buf": "kv_scales"
       },
       {
-       "state": "gdn"
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 128
+       "f32": 1.0
       },
       {
-       "buf": "qkvz"
+       "f32": 0.0
       },
       {
-       "i32": 1
+       "i64": 8
       },
       {
-       "i32": 49
+       "i64": 6144
       },
       {
-       "i64": 1
+       "i64": 256
       },
       {
-       "i64": 1
+       "i64": 6144
       },
       {
-       "i64": 0
+       "i64": 256
       },
       {
        "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l41.recurrent",
-     "kernel": "recurrent",
-     "args": [
-      {
-       "buf": "qkvz"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "seq_lens"
       },
       {
-       "buf": "ba"
+       "i64": 25690112
       },
       {
-       "buf": "model.layers.41.linear_attn.A_log"
+       "i64": 2048
       },
       {
-       "buf": "model.layers.41.linear_attn.dt_bias"
+       "i64": 512
       },
       {
-       "buf": "core_attn_out"
+       "i64": 25690112
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "i64": 2048
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "i64": 512
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 128
+       "buf": "cu_seqlens_q"
       },
       {
-       "f32": 0.0883883461356163
+       "i32": 1
       },
       {
        "i64": 0
@@ -60531,87 +48303,93 @@
      ]
     },
     {
-     "label": "l41.gated_norm",
-     "kernel": "gated_norm_decode",
+     "label": "l23.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "attn_out"
       },
       {
-       "buf": "model.layers.41.linear_attn.norm.weight"
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "model.layers.23.self_attn.o_proj.weight"
       },
       {
-       "i32": 128
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.23.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 5120
       },
       {
-       "f32": 9.999999974752427e-07
+       "i32": 6144
       },
       {
-       "i64": 0
+       "i32": 256
       },
       {
-       "i64": 0
+       "i32": 14336
+      },
+      {
+       "i32": 512
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l41.out_proj",
-     "kernel": "gemm",
+     "label": "l23.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.41.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.23.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l41.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l23.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.23.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.41.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.24.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -60620,7 +48398,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -60628,23 +48406,32 @@
      ]
     },
     {
-     "label": "l41.gate_up",
-     "kernel": "gemm",
+     "label": "l24.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.41.mlp.gate_up_proj.weight"
+       "buf": "model.layers.24.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.24.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -60652,71 +48439,79 @@
      ]
     },
     {
-     "label": "l41.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l24.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.24.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 61440
       },
       {
-       "f32": 1.0
+       "buf": "gdn.line_index",
+       "offset": 76
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l41.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "ba"
+      },
       {
-       "buf": "act"
+       "buf": "model.layers.24.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.41.mlp.down_proj.weight"
+       "buf": "model.layers.24.linear_attn.dt_bias"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.24.linear_attn.norm.weight"
       },
       {
-       "i32": 5120
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 17408
+       "f32": 9.999999974752427e-07
+      },
+      {
+       "i32": 49
+      },
+      {
+       "i32": 1605632
+      },
+      {
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l41.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l24.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.24.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.42.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.24.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -60725,7 +48520,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -60733,23 +48528,23 @@
      ]
     },
     {
-     "label": "l42.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l24.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.42.linear_attn.in_proj_qkvz.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.24.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -60757,197 +48552,202 @@
      ]
     },
     {
-     "label": "l42.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l24.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.42.linear_attn.in_proj_ba.weight"
+       "buf": "act"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.24.mlp.down_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 96
+       "buf": "model.layers.25.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l42.conv_update",
-     "kernel": "conv_update",
+     "label": "l25.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.42.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 132
+       "buf": "ba"
       },
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "i32": 1
+       "buf": "model.layers.25.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i32": 49
+       "buf": "model.layers.25.linear_attn.in_proj_ba.weight"
       },
       {
-       "i64": 1
+       "sym": "tokens"
       },
       {
-       "i64": 1
+       "i32": 16384
       },
       {
-       "i64": 0
+       "i32": 96
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l42.recurrent",
-     "kernel": "recurrent",
+     "label": "l25.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.25.linear_attn.conv1d.weight"
+      },
+      {
+       "state": "gdn"
+      },
+      {
+       "state": "gdn",
+       "offset": 61440
+      },
+      {
+       "buf": "gdn.line_index",
+       "offset": 80
       },
       {
        "buf": "ba"
       },
       {
-       "buf": "model.layers.42.linear_attn.A_log"
+       "buf": "model.layers.25.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.42.linear_attn.dt_bias"
+       "buf": "model.layers.25.linear_attn.dt_bias"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "buf": "model.layers.25.linear_attn.norm.weight"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 132
+       "f32": 9.999999974752427e-07
       },
       {
-       "f32": 0.0883883461356163
+       "i32": 49
       },
       {
-       "i64": 0
+       "i32": 1605632
       },
       {
-       "i64": 0
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l42.gated_norm",
-     "kernel": "gated_norm_decode",
+     "label": "l25.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.42.linear_attn.norm.weight"
+       "buf": "model.layers.25.linear_attn.out_proj.weight"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.25.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "i32": 48
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l42.out_proj",
-     "kernel": "gemm",
+     "label": "l25.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.42.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.25.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l42.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l25.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.25.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.42.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.26.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -60956,7 +48756,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -60964,23 +48764,32 @@
      ]
     },
     {
-     "label": "l42.gate_up",
-     "kernel": "gemm",
+     "label": "l26.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.42.mlp.gate_up_proj.weight"
+       "buf": "model.layers.26.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.26.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -60988,71 +48797,79 @@
      ]
     },
     {
-     "label": "l42.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l26.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.26.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 61440
       },
       {
-       "f32": 1.0
+       "buf": "gdn.line_index",
+       "offset": 84
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l42.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "ba"
+      },
       {
-       "buf": "act"
+       "buf": "model.layers.26.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.42.mlp.down_proj.weight"
+       "buf": "model.layers.26.linear_attn.dt_bias"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.26.linear_attn.norm.weight"
       },
       {
-       "i32": 5120
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 17408
+       "f32": 9.999999974752427e-07
+      },
+      {
+       "i32": 49
+      },
+      {
+       "i32": 1605632
+      },
+      {
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l42.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l26.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.26.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.43.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.26.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -61061,7 +48878,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -61069,23 +48886,23 @@
      ]
     },
     {
-     "label": "l43.qkv_proj",
-     "kernel": "gemm",
+     "label": "l26.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.43.self_attn.qkv_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkv"
+       "buf": "model.layers.26.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 14336
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -61093,45 +48910,32 @@
      ]
     },
     {
-     "label": "l43.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l26.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.43.self_attn.q_norm.weight_p1"
+       "buf": "x"
       },
       {
-       "i32": 256
+       "buf": "act"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "buf": "model.layers.26.mlp.down_proj.weight"
       },
       {
-       "i32": 24
+       "buf": "residual"
       },
       {
-       "i32": 14336
+       "buf": "model.layers.27.input_layernorm.weight_p1"
       },
       {
-       "i32": 512
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 5120
       },
       {
-       "i32": 256
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -61139,55 +48943,32 @@
      ]
     },
     {
-     "label": "l43.k_norm",
-     "kernel": "gemma_norm_khead",
+     "label": "l27.qkv_proj",
+     "kernel": "gemm",
      "args": [
       {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.43.self_attn.k_norm.weight_p1"
-      },
-      {
-       "i32": 256
-      },
-      {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
-      },
-      {
-       "i32": 4
+       "buf": "x"
       },
       {
-       "i32": 14336
+       "buf": "model.layers.27.self_attn.qkv_proj.weight"
       },
       {
-       "i32": 256
+       "buf": "qkv"
       },
       {
-       "i32": 1024
+       "sym": "tokens"
       },
       {
-       "i32": 256
+       "i32": 14336
       },
       {
-       "f32": 9.999999974752427e-07
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l43.rope",
-     "kernel": "mrope",
+     "label": "l27.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
@@ -61196,82 +48977,70 @@
        "buf": "k_n"
       },
       {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
+       "buf": "qkv"
       },
       {
-       "i32": 0
+       "buf": "model.layers.27.self_attn.q_norm.weight_p1"
       },
       {
-       "i64": 0
+       "buf": "model.layers.27.self_attn.k_norm.weight_p1"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l43.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
+       "buf": "cos_g"
       },
       {
-       "buf": "qkv",
-       "offset": 26624
+       "buf": "sin_g"
       },
       {
        "state": "kv",
-       "offset": 32112640
+       "offset": 19267584
       },
       {
        "state": "kv",
-       "offset": 32113152
+       "offset": 19268096
       },
       {
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l43.attn",
+     "label": "l27.attn",
      "kernel": "attn",
      "args": [
       {
@@ -61282,11 +49051,11 @@
       },
       {
        "state": "kv",
-       "offset": 32112640
+       "offset": 19267584
       },
       {
        "state": "kv",
-       "offset": 32113152
+       "offset": 19268096
       },
       {
        "buf": "block_table"
@@ -61364,11 +49133,11 @@
      ]
     },
     {
-     "label": "l43.gate",
-     "kernel": "sigmoid_mul",
+     "label": "l27.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "gated"
+       "buf": "x"
       },
       {
        "buf": "attn_out"
@@ -61378,7 +49147,22 @@
        "offset": 512
       },
       {
-       "i32": 24
+       "buf": "model.layers.27.self_attn.o_proj.weight"
+      },
+      {
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.27.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 6144
       },
       {
        "i32": 256
@@ -61388,51 +49172,54 @@
       },
       {
        "i32": 512
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l43.o_proj",
-     "kernel": "gemm",
+     "label": "l27.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "gated"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.43.self_attn.o_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.27.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l43.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l27.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.27.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.43.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.28.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -61441,7 +49228,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -61449,23 +49236,32 @@
      ]
     },
     {
-     "label": "l43.gate_up",
-     "kernel": "gemm",
+     "label": "l28.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.43.mlp.gate_up_proj.weight"
+       "buf": "model.layers.28.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.28.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -61473,71 +49269,79 @@
      ]
     },
     {
-     "label": "l43.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l28.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.28.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 61440
       },
       {
-       "f32": 1.0
+       "buf": "gdn.line_index",
+       "offset": 88
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l43.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "ba"
+      },
       {
-       "buf": "act"
+       "buf": "model.layers.28.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.43.mlp.down_proj.weight"
+       "buf": "model.layers.28.linear_attn.dt_bias"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.28.linear_attn.norm.weight"
       },
       {
-       "i32": 5120
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 17408
+       "f32": 9.999999974752427e-07
+      },
+      {
+       "i32": 49
+      },
+      {
+       "i32": 1605632
+      },
+      {
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l43.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l28.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.28.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.44.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.28.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -61546,7 +49350,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -61554,23 +49358,23 @@
      ]
     },
     {
-     "label": "l44.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l28.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.44.linear_attn.in_proj_qkvz.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.28.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -61578,197 +49382,202 @@
      ]
     },
     {
-     "label": "l44.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l28.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.44.linear_attn.in_proj_ba.weight"
+       "buf": "act"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.28.mlp.down_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 96
+       "buf": "model.layers.29.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l44.conv_update",
-     "kernel": "conv_update",
+     "label": "l29.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.44.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 136
+       "buf": "ba"
       },
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "i32": 1
+       "buf": "model.layers.29.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i32": 49
+       "buf": "model.layers.29.linear_attn.in_proj_ba.weight"
       },
       {
-       "i64": 1
+       "sym": "tokens"
       },
       {
-       "i64": 1
+       "i32": 16384
       },
       {
-       "i64": 0
+       "i32": 96
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l44.recurrent",
-     "kernel": "recurrent",
+     "label": "l29.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.29.linear_attn.conv1d.weight"
+      },
+      {
+       "state": "gdn"
+      },
+      {
+       "state": "gdn",
+       "offset": 61440
+      },
+      {
+       "buf": "gdn.line_index",
+       "offset": 92
       },
       {
        "buf": "ba"
       },
       {
-       "buf": "model.layers.44.linear_attn.A_log"
+       "buf": "model.layers.29.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.44.linear_attn.dt_bias"
+       "buf": "model.layers.29.linear_attn.dt_bias"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "buf": "model.layers.29.linear_attn.norm.weight"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 136
+       "f32": 9.999999974752427e-07
       },
       {
-       "f32": 0.0883883461356163
+       "i32": 49
       },
       {
-       "i64": 0
+       "i32": 1605632
       },
       {
-       "i64": 0
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l44.gated_norm",
-     "kernel": "gated_norm_decode",
+     "label": "l29.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.44.linear_attn.norm.weight"
+       "buf": "model.layers.29.linear_attn.out_proj.weight"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.29.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "i32": 48
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l44.out_proj",
-     "kernel": "gemm",
+     "label": "l29.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.44.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.29.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l44.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l29.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.29.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.44.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.30.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -61777,7 +49586,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -61785,23 +49594,32 @@
      ]
     },
     {
-     "label": "l44.gate_up",
-     "kernel": "gemm",
+     "label": "l30.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.44.mlp.gate_up_proj.weight"
+       "buf": "model.layers.30.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.30.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -61809,41 +49627,79 @@
      ]
     },
     {
-     "label": "l44.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l30.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.30.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 61440
       },
       {
-       "f32": 1.0
+       "buf": "gdn.line_index",
+       "offset": 96
       },
       {
-       "i32": 0
+       "buf": "ba"
+      },
+      {
+       "buf": "model.layers.30.linear_attn.A_log"
+      },
+      {
+       "buf": "model.layers.30.linear_attn.dt_bias"
+      },
+      {
+       "buf": "core_attn_out"
+      },
+      {
+       "buf": "model.layers.30.linear_attn.norm.weight"
+      },
+      {
+       "f32": 0.0883883461356163
+      },
+      {
+       "f32": 9.999999974752427e-07
+      },
+      {
+       "i32": 49
+      },
+      {
+       "i32": 1605632
+      },
+      {
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l44.down_proj",
-     "kernel": "gemm",
+     "label": "l30.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "act"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.44.mlp.down_proj.weight"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.30.linear_attn.out_proj.weight"
+      },
+      {
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.30.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -61852,28 +49708,55 @@
        "i32": 5120
       },
       {
-       "i32": 17408
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l44.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l30.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
+      {
+       "buf": "act"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.30.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "residual"
+       "sym": "tokens"
       },
       {
-       "buf": "model.layers.45.input_layernorm.weight_p1"
+       "i32": 17408
       },
       {
        "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l30.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
+      },
+      {
+       "buf": "act"
+      },
+      {
+       "buf": "model.layers.30.mlp.down_proj.weight"
+      },
+      {
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.31.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -61882,7 +49765,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -61890,23 +49773,23 @@
      ]
     },
     {
-     "label": "l45.in_proj_qkvz",
+     "label": "l31.qkv_proj",
      "kernel": "gemm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.45.linear_attn.in_proj_qkvz.weight"
+       "buf": "model.layers.31.self_attn.qkv_proj.weight"
       },
       {
-       "buf": "qkvz"
+       "buf": "qkv"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 14336
       },
       {
        "i32": 5120
@@ -61914,106 +49797,162 @@
      ]
     },
     {
-     "label": "l45.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l31.prep",
+     "kernel": "attn_prep",
+     "args": [
+      {
+       "buf": "q_n"
+      },
+      {
+       "buf": "k_n"
+      },
+      {
+       "buf": "qkv"
+      },
+      {
+       "buf": "model.layers.31.self_attn.q_norm.weight_p1"
+      },
+      {
+       "buf": "model.layers.31.self_attn.k_norm.weight_p1"
+      },
+      {
+       "buf": "cos_g"
+      },
+      {
+       "buf": "sin_g"
+      },
+      {
+       "state": "kv",
+       "offset": 22478848
+      },
+      {
+       "state": "kv",
+       "offset": 22479360
+      },
+      {
+       "buf": "slot_mapping"
+      },
+      {
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
+      },
+      {
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
+      },
+      {
+       "i64": 25690112
+      },
+      {
+       "i64": 2048
+      },
+      {
+       "i64": 512
+      },
+      {
+       "i32": 784
+      },
+      {
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l31.attn",
+     "kernel": "attn",
      "args": [
       {
-       "buf": "x"
+       "buf": "attn_out"
       },
       {
-       "buf": "model.layers.45.linear_attn.in_proj_ba.weight"
+       "buf": "q_n"
       },
       {
-       "buf": "ba"
+       "state": "kv",
+       "offset": 22478848
       },
       {
-       "sym": "tokens"
+       "state": "kv",
+       "offset": 22479360
       },
       {
-       "i32": 96
+       "buf": "block_table"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l45.conv_update",
-     "kernel": "conv_update",
-     "args": [
+       "buf": "seq_lens"
+      },
       {
-       "buf": "qkvz"
+       "f32": 0.0625
       },
       {
-       "buf": "model.layers.45.linear_attn.conv1d.weight"
+       "buf": "kv_scales"
       },
       {
-       "state": "gdn"
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 140
+       "f32": 1.0
       },
       {
-       "buf": "qkvz"
+       "f32": 0.0
       },
       {
-       "i32": 1
+       "i64": 8
       },
       {
-       "i32": 49
+       "i64": 6144
       },
       {
-       "i64": 1
+       "i64": 256
       },
       {
-       "i64": 1
+       "i64": 6144
       },
       {
-       "i64": 0
+       "i64": 256
       },
       {
        "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l45.recurrent",
-     "kernel": "recurrent",
-     "args": [
-      {
-       "buf": "qkvz"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "seq_lens"
       },
       {
-       "buf": "ba"
+       "i64": 25690112
       },
       {
-       "buf": "model.layers.45.linear_attn.A_log"
+       "i64": 2048
       },
       {
-       "buf": "model.layers.45.linear_attn.dt_bias"
+       "i64": 512
       },
       {
-       "buf": "core_attn_out"
+       "i64": 25690112
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "i64": 2048
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "i64": 512
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 140
+       "buf": "cu_seqlens_q"
       },
       {
-       "f32": 0.0883883461356163
+       "i32": 1
       },
       {
        "i64": 0
@@ -62024,87 +49963,93 @@
      ]
     },
     {
-     "label": "l45.gated_norm",
-     "kernel": "gated_norm_decode",
+     "label": "l31.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "attn_out"
       },
       {
-       "buf": "model.layers.45.linear_attn.norm.weight"
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "model.layers.31.self_attn.o_proj.weight"
       },
       {
-       "i32": 128
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.31.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 5120
       },
       {
-       "f32": 9.999999974752427e-07
+       "i32": 6144
       },
       {
-       "i64": 0
+       "i32": 256
       },
       {
-       "i64": 0
+       "i32": 14336
+      },
+      {
+       "i32": 512
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l45.out_proj",
-     "kernel": "gemm",
+     "label": "l31.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.45.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.31.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l45.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l31.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.31.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.45.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.32.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -62113,7 +50058,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -62121,23 +50066,32 @@
      ]
     },
     {
-     "label": "l45.gate_up",
-     "kernel": "gemm",
+     "label": "l32.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.45.mlp.gate_up_proj.weight"
+       "buf": "model.layers.32.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.32.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -62145,71 +50099,79 @@
      ]
     },
     {
-     "label": "l45.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l32.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.32.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 61440
       },
       {
-       "f32": 1.0
+       "buf": "gdn.line_index",
+       "offset": 100
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l45.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "ba"
+      },
       {
-       "buf": "act"
+       "buf": "model.layers.32.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.45.mlp.down_proj.weight"
+       "buf": "model.layers.32.linear_attn.dt_bias"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.32.linear_attn.norm.weight"
       },
       {
-       "i32": 5120
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 17408
+       "f32": 9.999999974752427e-07
+      },
+      {
+       "i32": 49
+      },
+      {
+       "i32": 1605632
+      },
+      {
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l45.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l32.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.32.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.46.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.32.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -62218,7 +50180,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -62226,23 +50188,23 @@
      ]
     },
     {
-     "label": "l46.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l32.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.46.linear_attn.in_proj_qkvz.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.32.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -62250,197 +50212,202 @@
      ]
     },
     {
-     "label": "l46.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l32.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.46.linear_attn.in_proj_ba.weight"
+       "buf": "act"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.32.mlp.down_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 96
+       "buf": "model.layers.33.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l46.conv_update",
-     "kernel": "conv_update",
+     "label": "l33.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.46.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 144
+       "buf": "ba"
       },
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "i32": 1
+       "buf": "model.layers.33.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i32": 49
+       "buf": "model.layers.33.linear_attn.in_proj_ba.weight"
       },
       {
-       "i64": 1
+       "sym": "tokens"
       },
       {
-       "i64": 1
+       "i32": 16384
       },
       {
-       "i64": 0
+       "i32": 96
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l46.recurrent",
-     "kernel": "recurrent",
+     "label": "l33.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.33.linear_attn.conv1d.weight"
+      },
+      {
+       "state": "gdn"
+      },
+      {
+       "state": "gdn",
+       "offset": 61440
+      },
+      {
+       "buf": "gdn.line_index",
+       "offset": 104
       },
       {
        "buf": "ba"
       },
       {
-       "buf": "model.layers.46.linear_attn.A_log"
+       "buf": "model.layers.33.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.46.linear_attn.dt_bias"
+       "buf": "model.layers.33.linear_attn.dt_bias"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "buf": "model.layers.33.linear_attn.norm.weight"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 144
+       "f32": 9.999999974752427e-07
       },
       {
-       "f32": 0.0883883461356163
+       "i32": 49
       },
       {
-       "i64": 0
+       "i32": 1605632
       },
       {
-       "i64": 0
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l46.gated_norm",
-     "kernel": "gated_norm_decode",
+     "label": "l33.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.46.linear_attn.norm.weight"
+       "buf": "model.layers.33.linear_attn.out_proj.weight"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.33.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "i32": 48
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l46.out_proj",
-     "kernel": "gemm",
+     "label": "l33.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.46.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.33.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l46.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l33.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.33.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.46.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.34.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -62449,7 +50416,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -62457,23 +50424,32 @@
      ]
     },
     {
-     "label": "l46.gate_up",
-     "kernel": "gemm",
+     "label": "l34.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.46.mlp.gate_up_proj.weight"
+       "buf": "model.layers.34.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.34.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -62481,71 +50457,79 @@
      ]
     },
     {
-     "label": "l46.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l34.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.34.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 61440
       },
       {
-       "f32": 1.0
+       "buf": "gdn.line_index",
+       "offset": 108
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l46.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "ba"
+      },
       {
-       "buf": "act"
+       "buf": "model.layers.34.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.46.mlp.down_proj.weight"
+       "buf": "model.layers.34.linear_attn.dt_bias"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.34.linear_attn.norm.weight"
       },
       {
-       "i32": 5120
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 17408
+       "f32": 9.999999974752427e-07
+      },
+      {
+       "i32": 49
+      },
+      {
+       "i32": 1605632
+      },
+      {
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l46.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l34.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.34.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.47.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.34.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -62554,7 +50538,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -62562,23 +50546,23 @@
      ]
     },
     {
-     "label": "l47.qkv_proj",
-     "kernel": "gemm",
+     "label": "l34.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.47.self_attn.qkv_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkv"
+       "buf": "model.layers.34.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 14336
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -62586,45 +50570,32 @@
      ]
     },
     {
-     "label": "l47.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l34.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.47.self_attn.q_norm.weight_p1"
+       "buf": "x"
       },
       {
-       "i32": 256
+       "buf": "act"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "buf": "model.layers.34.mlp.down_proj.weight"
       },
       {
-       "i32": 24
+       "buf": "residual"
       },
       {
-       "i32": 14336
+       "buf": "model.layers.35.input_layernorm.weight_p1"
       },
       {
-       "i32": 512
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 5120
       },
       {
-       "i32": 256
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -62632,55 +50603,32 @@
      ]
     },
     {
-     "label": "l47.k_norm",
-     "kernel": "gemma_norm_khead",
+     "label": "l35.qkv_proj",
+     "kernel": "gemm",
      "args": [
       {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.47.self_attn.k_norm.weight_p1"
+       "buf": "x"
       },
       {
-       "i32": 256
+       "buf": "model.layers.35.self_attn.qkv_proj.weight"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
+       "buf": "qkv"
       },
       {
-       "i32": 4
+       "sym": "tokens"
       },
       {
        "i32": 14336
       },
       {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l47.rope",
-     "kernel": "mrope",
+     "label": "l35.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
@@ -62689,82 +50637,70 @@
        "buf": "k_n"
       },
       {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
-      },
-      {
-       "i32": 0
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l47.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
+       "buf": "qkv"
       },
       {
-       "state": "kv",
-       "offset": 35323904
+       "buf": "model.layers.35.self_attn.q_norm.weight_p1"
       },
       {
-       "state": "kv",
-       "offset": 35324416
+       "buf": "model.layers.35.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "slot_mapping"
+       "buf": "cos_g"
       },
       {
-       "buf": "kv_scales"
+       "buf": "sin_g"
       },
       {
-       "buf": "kv_scales",
-       "offset": 4
+       "state": "kv",
+       "offset": 25690112
       },
       {
-       "i64": 1024
+       "state": "kv",
+       "offset": 25690624
       },
       {
-       "i64": 14336
+       "buf": "slot_mapping"
       },
       {
-       "i64": 25690112
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 512
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i64": 0
+       "i64": 25690112
       },
       {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l47.attn",
+     "label": "l35.attn",
      "kernel": "attn",
      "args": [
       {
@@ -62775,11 +50711,11 @@
       },
       {
        "state": "kv",
-       "offset": 35323904
+       "offset": 25690112
       },
       {
        "state": "kv",
-       "offset": 35324416
+       "offset": 25690624
       },
       {
        "buf": "block_table"
@@ -62857,11 +50793,11 @@
      ]
     },
     {
-     "label": "l47.gate",
-     "kernel": "sigmoid_mul",
+     "label": "l35.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "gated"
+       "buf": "x"
       },
       {
        "buf": "attn_out"
@@ -62871,31 +50807,13 @@
        "offset": 512
       },
       {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l47.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
+       "buf": "model.layers.35.self_attn.o_proj.weight"
       },
       {
-       "buf": "model.layers.47.self_attn.o_proj.weight"
+       "buf": "residual"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.35.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -62905,36 +50823,15 @@
       },
       {
        "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l47.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
       },
       {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.47.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "sym": "tokens"
+       "i32": 256
       },
       {
-       "i32": 5120
+       "i32": 14336
       },
       {
-       "i32": 5120
+       "i32": 512
       },
       {
        "f32": 9.999999974752427e-07
@@ -62942,95 +50839,47 @@
      ]
     },
     {
-     "label": "l47.gate_up",
-     "kernel": "gemm",
+     "label": "l35.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.47.mlp.gate_up_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.35.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
-      {
-       "i32": 34816
-      },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l47.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
       {
        "i32": 17408
       },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
-      },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l47.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
-      },
-      {
-       "buf": "model.layers.47.mlp.down_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
       {
        "i32": 5120
-      },
-      {
-       "i32": 17408
       }
      ]
     },
     {
-     "label": "l47.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l35.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.35.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.48.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.36.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -63039,7 +50888,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -63047,45 +50896,30 @@
      ]
     },
     {
-     "label": "l48.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l36.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.48.linear_attn.in_proj_qkvz.weight"
-      },
       {
        "buf": "qkvz"
       },
       {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
+       "buf": "ba"
       },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l48.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.48.linear_attn.in_proj_ba.weight"
+       "buf": "model.layers.36.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.36.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
+      {
+       "i32": 16384
+      },
       {
        "i32": 96
       },
@@ -63095,173 +50929,136 @@
      ]
     },
     {
-     "label": "l48.conv_update",
-     "kernel": "conv_update",
+     "label": "l36.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.48.linear_attn.conv1d.weight"
+       "buf": "model.layers.36.linear_attn.conv1d.weight"
       },
       {
        "state": "gdn"
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 148
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 49
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l48.recurrent",
-     "kernel": "recurrent",
-     "args": [
-      {
-       "buf": "qkvz"
+       "state": "gdn",
+       "offset": 61440
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "gdn.line_index",
+       "offset": 112
       },
       {
        "buf": "ba"
       },
       {
-       "buf": "model.layers.48.linear_attn.A_log"
+       "buf": "model.layers.36.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.48.linear_attn.dt_bias"
+       "buf": "model.layers.36.linear_attn.dt_bias"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "buf": "model.layers.36.linear_attn.norm.weight"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 148
+       "f32": 9.999999974752427e-07
       },
       {
-       "f32": 0.0883883461356163
+       "i32": 49
       },
       {
-       "i64": 0
+       "i32": 1605632
       },
       {
-       "i64": 0
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l48.gated_norm",
-     "kernel": "gated_norm_decode",
+     "label": "l36.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.48.linear_attn.norm.weight"
+       "buf": "model.layers.36.linear_attn.out_proj.weight"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.36.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "i32": 48
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l48.out_proj",
-     "kernel": "gemm",
+     "label": "l36.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.48.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.36.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l48.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l36.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.36.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.48.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.37.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -63270,7 +51067,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -63278,152 +51075,145 @@
      ]
     },
     {
-     "label": "l48.gate_up",
-     "kernel": "gemm",
+     "label": "l37.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.48.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
+       "buf": "qkvz"
       },
       {
-       "sym": "tokens"
+       "buf": "ba"
       },
       {
-       "i32": 34816
+       "buf": "x"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l48.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "model.layers.37.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.37.linear_attn.in_proj_ba.weight"
       },
       {
-       "i32": 17408
+       "sym": "tokens"
       },
       {
-       "i32": 0
+       "i32": 16384
       },
       {
-       "f32": 1.0
+       "i32": 96
       },
       {
-       "i32": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l48.down_proj",
-     "kernel": "gemm",
+     "label": "l37.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "model.layers.48.mlp.down_proj.weight"
+       "buf": "model.layers.37.linear_attn.conv1d.weight"
       },
       {
-       "buf": "y"
+       "state": "gdn"
       },
       {
-       "sym": "tokens"
+       "state": "gdn",
+       "offset": 61440
       },
       {
-       "i32": 5120
+       "buf": "gdn.line_index",
+       "offset": 116
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l48.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "buf": "ba"
+      },
       {
-       "buf": "x"
+       "buf": "model.layers.37.linear_attn.A_log"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.37.linear_attn.dt_bias"
       },
       {
-       "buf": "residual"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.49.input_layernorm.weight_p1"
+       "buf": "model.layers.37.linear_attn.norm.weight"
       },
       {
-       "i32": 5120
+       "f32": 0.0883883461356163
       },
       {
-       "sym": "tokens"
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 5120
+       "i32": 49
       },
       {
-       "i32": 5120
+       "i32": 1605632
       },
       {
-       "f32": 9.999999974752427e-07
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l49.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l37.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.49.linear_attn.in_proj_qkvz.weight"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.37.linear_attn.out_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 16384
+       "buf": "model.layers.37.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l49.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l37.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.49.linear_attn.in_proj_ba.weight"
+       "buf": "x"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.37.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 96
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -63431,278 +51221,202 @@
      ]
     },
     {
-     "label": "l49.conv_update",
-     "kernel": "conv_update",
+     "label": "l37.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "qkvz"
-      },
-      {
-       "buf": "model.layers.49.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
+       "buf": "x"
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 152
+       "buf": "act"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.37.mlp.down_proj.weight"
       },
       {
-       "i32": 1
+       "buf": "residual"
       },
       {
-       "i32": 49
+       "buf": "model.layers.38.input_layernorm.weight_p1"
       },
       {
-       "i64": 1
+       "sym": "tokens"
       },
       {
-       "i64": 1
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 17408
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l49.recurrent",
-     "kernel": "recurrent",
+     "label": "l38.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
        "buf": "qkvz"
       },
-      {
-       "buf": "ba",
-       "offset": 96
-      },
       {
        "buf": "ba"
       },
       {
-       "buf": "model.layers.49.linear_attn.A_log"
-      },
-      {
-       "buf": "model.layers.49.linear_attn.dt_bias"
-      },
-      {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "buf": "model.layers.38.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "buf": "model.layers.38.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 152
+       "sym": "tokens"
       },
       {
-       "f32": 0.0883883461356163
+       "i32": 16384
       },
       {
-       "i64": 0
+       "i32": 96
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l49.gated_norm",
-     "kernel": "gated_norm_decode",
+     "label": "l38.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "core_attn_out"
-      },
-      {
-       "buf": "model.layers.49.linear_attn.norm.weight"
+       "buf": "qkvz"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "model.layers.38.linear_attn.conv1d.weight"
       },
       {
-       "i32": 128
+       "state": "gdn"
       },
       {
-       "i32": 128
+       "state": "gdn",
+       "offset": 61440
       },
       {
-       "i32": 128
+       "buf": "gdn.line_index",
+       "offset": 120
       },
       {
-       "i32": 48
+       "buf": "ba"
       },
       {
-       "f32": 9.999999974752427e-07
+       "buf": "model.layers.38.linear_attn.A_log"
       },
       {
-       "i64": 0
+       "buf": "model.layers.38.linear_attn.dt_bias"
       },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l49.out_proj",
-     "kernel": "gemm",
-     "args": [
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.49.linear_attn.out_proj.weight"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l49.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
+       "buf": "model.layers.38.linear_attn.norm.weight"
       },
       {
-       "buf": "model.layers.49.post_attention_layernorm.weight_p1"
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 5120
+       "f32": 9.999999974752427e-07
       },
       {
-       "sym": "tokens"
+       "i32": 49
       },
       {
-       "i32": 5120
+       "i32": 1605632
       },
       {
-       "i32": 5120
+       "i32": 10240
       },
       {
-       "f32": 9.999999974752427e-07
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l49.gate_up",
-     "kernel": "gemm",
+     "label": "l38.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.49.mlp.gate_up_proj.weight"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "sym": "tokens"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 34816
+       "buf": "model.layers.38.linear_attn.out_proj.weight"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l49.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "residual"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.38.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 17408
+       "sym": "tokens"
       },
       {
-       "i32": 0
+       "i32": 5120
       },
       {
-       "f32": 1.0
+       "i32": 6144
       },
       {
-       "i32": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l49.down_proj",
-     "kernel": "gemm",
+     "label": "l38.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
        "buf": "act"
       },
       {
-       "buf": "model.layers.49.mlp.down_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.38.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 17408
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l49.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l38.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.38.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.50.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.39.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -63711,7 +51425,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -63719,23 +51433,23 @@
      ]
     },
     {
-     "label": "l50.in_proj_qkvz",
+     "label": "l39.qkv_proj",
      "kernel": "gemm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.50.linear_attn.in_proj_qkvz.weight"
+       "buf": "model.layers.39.self_attn.qkv_proj.weight"
       },
       {
-       "buf": "qkvz"
+       "buf": "qkv"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 14336
       },
       {
        "i32": 5120
@@ -63743,272 +51457,259 @@
      ]
     },
     {
-     "label": "l50.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l39.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "x"
+       "buf": "q_n"
       },
       {
-       "buf": "model.layers.50.linear_attn.in_proj_ba.weight"
+       "buf": "k_n"
       },
       {
-       "buf": "ba"
+       "buf": "qkv"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.39.self_attn.q_norm.weight_p1"
       },
       {
-       "i32": 96
+       "buf": "model.layers.39.self_attn.k_norm.weight_p1"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l50.conv_update",
-     "kernel": "conv_update",
-     "args": [
+       "buf": "cos_g"
+      },
       {
-       "buf": "qkvz"
+       "buf": "sin_g"
       },
       {
-       "buf": "model.layers.50.linear_attn.conv1d.weight"
+       "state": "kv",
+       "offset": 28901376
       },
       {
-       "state": "gdn"
+       "state": "kv",
+       "offset": 28901888
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 156
+       "buf": "slot_mapping"
       },
       {
-       "buf": "qkvz"
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i32": 1
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i32": 49
+       "i64": 25690112
       },
       {
-       "i64": 1
+       "i64": 2048
       },
       {
-       "i64": 1
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l50.recurrent",
-     "kernel": "recurrent",
+     "label": "l39.attn",
+     "kernel": "attn",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "attn_out"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "q_n"
       },
       {
-       "buf": "ba"
+       "state": "kv",
+       "offset": 28901376
       },
       {
-       "buf": "model.layers.50.linear_attn.A_log"
+       "state": "kv",
+       "offset": 28901888
       },
       {
-       "buf": "model.layers.50.linear_attn.dt_bias"
+       "buf": "block_table"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "seq_lens"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "f32": 0.0625
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "buf": "kv_scales"
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 156
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "f32": 0.0883883461356163
+       "f32": 1.0
       },
       {
-       "i64": 0
+       "f32": 0.0
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l50.gated_norm",
-     "kernel": "gated_norm_decode",
-     "args": [
-      {
-       "buf": "core_attn_out"
+       "i64": 8
       },
       {
-       "buf": "core_attn_out"
+       "i64": 6144
       },
       {
-       "buf": "model.layers.50.linear_attn.norm.weight"
+       "i64": 256
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "i64": 6144
       },
       {
-       "i32": 128
+       "i64": 256
       },
       {
-       "i32": 128
+       "i64": 0
       },
       {
-       "i32": 128
+       "buf": "seq_lens"
       },
       {
-       "i32": 48
+       "i64": 25690112
       },
       {
-       "f32": 9.999999974752427e-07
+       "i64": 2048
       },
       {
-       "i64": 0
+       "i64": 512
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l50.out_proj",
-     "kernel": "gemm",
-     "args": [
+       "i64": 25690112
+      },
       {
-       "buf": "core_attn_out"
+       "i64": 2048
       },
       {
-       "buf": "model.layers.50.linear_attn.out_proj.weight"
+       "i64": 512
       },
       {
-       "buf": "y"
+       "buf": "cu_seqlens_q"
       },
       {
-       "sym": "tokens"
+       "i32": 1
       },
       {
-       "i32": 5120
+       "i64": 0
       },
       {
-       "i32": 6144
+       "i64": 0
       }
      ]
     },
     {
-     "label": "l50.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l39.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "buf": "model.layers.50.post_attention_layernorm.weight_p1"
+       "buf": "model.layers.39.self_attn.o_proj.weight"
       },
       {
-       "i32": 5120
+       "buf": "residual"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.39.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 5120
+       "sym": "tokens"
       },
       {
        "i32": 5120
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l50.gate_up",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.50.mlp.gate_up_proj.weight"
+       "i32": 6144
       },
       {
-       "buf": "gate_up"
+       "i32": 256
       },
       {
-       "sym": "tokens"
+       "i32": 14336
       },
       {
-       "i32": 34816
+       "i32": 512
       },
       {
-       "i32": 5120
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l50.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l39.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
        "buf": "act"
       },
       {
-       "buf": "gate_up"
+       "buf": "x"
       },
       {
-       "i32": 17408
+       "buf": "model.layers.39.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 0
+       "sym": "tokens"
       },
       {
-       "f32": 1.0
+       "i32": 17408
       },
       {
-       "i32": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l50.down_proj",
-     "kernel": "gemm",
+     "label": "l39.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
+      {
+       "buf": "x"
+      },
       {
        "buf": "act"
       },
       {
-       "buf": "model.layers.50.mlp.down_proj.weight"
+       "buf": "model.layers.39.mlp.down_proj.weight"
       },
       {
-       "buf": "y"
+       "buf": "residual"
+      },
+      {
+       "buf": "model.layers.40.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -64018,153 +51719,128 @@
       },
       {
        "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l50.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l40.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "x"
+       "buf": "qkvz"
       },
       {
-       "buf": "y"
+       "buf": "ba"
       },
       {
-       "buf": "residual"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.51.input_layernorm.weight_p1"
+       "buf": "model.layers.40.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.40.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 16384
       },
       {
-       "i32": 5120
+       "i32": 96
       },
       {
-       "f32": 9.999999974752427e-07
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l51.qkv_proj",
-     "kernel": "gemm",
+     "label": "l40.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
-       "buf": "x"
+       "buf": "qkvz"
       },
       {
-       "buf": "model.layers.51.self_attn.qkv_proj.weight"
+       "buf": "model.layers.40.linear_attn.conv1d.weight"
       },
       {
-       "buf": "qkv"
+       "state": "gdn"
       },
       {
-       "sym": "tokens"
+       "state": "gdn",
+       "offset": 61440
       },
       {
-       "i32": 14336
+       "buf": "gdn.line_index",
+       "offset": 124
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l51.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
-      {
-       "buf": "q_n"
+       "buf": "ba"
       },
       {
-       "buf": "qkv"
+       "buf": "model.layers.40.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.51.self_attn.q_norm.weight_p1"
+       "buf": "model.layers.40.linear_attn.dt_bias"
       },
       {
-       "i32": 256
+       "buf": "core_attn_out"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "buf": "model.layers.40.linear_attn.norm.weight"
       },
       {
-       "i32": 24
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 14336
+       "f32": 9.999999974752427e-07
       },
       {
-       "i32": 512
+       "i32": 49
       },
       {
-       "i32": 6144
+       "i32": 1605632
       },
       {
-       "i32": 256
+       "i32": 10240
       },
       {
-       "f32": 9.999999974752427e-07
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l51.k_norm",
-     "kernel": "gemma_norm_khead",
+     "label": "l40.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.51.self_attn.k_norm.weight_p1"
+       "buf": "x"
       },
       {
-       "i32": 256
+       "buf": "core_attn_out"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
+       "buf": "model.layers.40.linear_attn.out_proj.weight"
       },
       {
-       "i32": 4
+       "buf": "residual"
       },
       {
-       "i32": 14336
+       "buf": "model.layers.40.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 256
+       "sym": "tokens"
       },
       {
-       "i32": 1024
+       "i32": 5120
       },
       {
-       "i32": 256
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -64172,253 +51848,226 @@
      ]
     },
     {
-     "label": "l51.rope",
-     "kernel": "mrope",
+     "label": "l40.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "k_n"
+       "buf": "act"
       },
       {
-       "buf": "cos_g"
+       "buf": "x"
       },
       {
-       "buf": "sin_g"
+       "buf": "model.layers.40.mlp.gate_up_proj.weight"
       },
       {
-       "i32": 0
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 17408
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l51.kv_write",
-     "kernel": "reshape_and_cache",
+     "label": "l40.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
-      },
-      {
-       "state": "kv",
-       "offset": 38535168
-      },
-      {
-       "state": "kv",
-       "offset": 38535680
-      },
-      {
-       "buf": "slot_mapping"
-      },
-      {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "buf": "x"
       },
       {
-       "i64": 14336
+       "buf": "act"
       },
       {
-       "i64": 25690112
+       "buf": "model.layers.40.mlp.down_proj.weight"
       },
       {
-       "i64": 512
+       "buf": "residual"
       },
       {
-       "i64": 0
+       "buf": "model.layers.41.input_layernorm.weight_p1"
       },
       {
-       "i64": 0
+       "sym": "tokens"
       },
       {
-       "i64": 2048
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 17408
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l51.attn",
-     "kernel": "attn",
+     "label": "l41.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv",
-       "offset": 38535168
-      },
-      {
-       "state": "kv",
-       "offset": 38535680
+       "buf": "qkvz"
       },
       {
-       "buf": "block_table"
+       "buf": "ba"
       },
       {
-       "buf": "seq_lens"
+       "buf": "x"
       },
       {
-       "f32": 0.0625
+       "buf": "model.layers.41.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "kv_scales"
+       "buf": "model.layers.41.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "kv_scales",
-       "offset": 4
+       "sym": "tokens"
       },
       {
-       "f32": 1.0
+       "i32": 16384
       },
       {
-       "f32": 0.0
+       "i32": 96
       },
       {
-       "i64": 8
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l41.gdn",
+     "kernel": "gdn_decode",
+     "args": [
       {
-       "i64": 6144
+       "buf": "qkvz"
       },
       {
-       "i64": 256
+       "buf": "model.layers.41.linear_attn.conv1d.weight"
       },
       {
-       "i64": 6144
+       "state": "gdn"
       },
       {
-       "i64": 256
+       "state": "gdn",
+       "offset": 61440
       },
       {
-       "i64": 0
+       "buf": "gdn.line_index",
+       "offset": 128
       },
       {
-       "buf": "seq_lens"
+       "buf": "ba"
       },
       {
-       "i64": 25690112
+       "buf": "model.layers.41.linear_attn.A_log"
       },
       {
-       "i64": 2048
+       "buf": "model.layers.41.linear_attn.dt_bias"
       },
       {
-       "i64": 512
+       "buf": "core_attn_out"
       },
       {
-       "i64": 25690112
+       "buf": "model.layers.41.linear_attn.norm.weight"
       },
       {
-       "i64": 2048
+       "f32": 0.0883883461356163
       },
       {
-       "i64": 512
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 49
       },
       {
-       "i32": 1
+       "i32": 1605632
       },
       {
-       "i64": 0
+       "i32": 10240
       },
       {
-       "i64": 0
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l51.gate",
-     "kernel": "sigmoid_mul",
+     "label": "l41.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "gated"
+       "buf": "x"
       },
       {
-       "buf": "attn_out"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "qkv",
-       "offset": 512
+       "buf": "model.layers.41.linear_attn.out_proj.weight"
       },
       {
-       "i32": 24
+       "buf": "residual"
       },
       {
-       "i32": 256
+       "buf": "model.layers.41.post_attention_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
       },
       {
-       "i32": 14336
+       "i32": 6144
       },
       {
-       "i32": 512
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l51.o_proj",
-     "kernel": "gemm",
+     "label": "l41.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "gated"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.51.self_attn.o_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.41.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l51.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l41.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.41.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.51.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.42.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -64427,7 +52076,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -64435,23 +52084,32 @@
      ]
     },
     {
-     "label": "l51.gate_up",
-     "kernel": "gemm",
+     "label": "l42.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.51.mlp.gate_up_proj.weight"
+       "buf": "model.layers.42.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.42.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -64459,71 +52117,79 @@
      ]
     },
     {
-     "label": "l51.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l42.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.42.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 61440
       },
       {
-       "f32": 1.0
+       "buf": "gdn.line_index",
+       "offset": 132
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l51.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "ba"
+      },
       {
-       "buf": "act"
+       "buf": "model.layers.42.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.51.mlp.down_proj.weight"
+       "buf": "model.layers.42.linear_attn.dt_bias"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.42.linear_attn.norm.weight"
       },
       {
-       "i32": 5120
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 17408
+       "f32": 9.999999974752427e-07
+      },
+      {
+       "i32": 49
+      },
+      {
+       "i32": 1605632
+      },
+      {
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l51.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l42.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.42.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.52.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.42.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -64532,7 +52198,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -64540,23 +52206,23 @@
      ]
     },
     {
-     "label": "l52.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l42.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.52.linear_attn.in_proj_qkvz.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.42.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -64564,302 +52230,316 @@
      ]
     },
     {
-     "label": "l52.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l42.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.52.linear_attn.in_proj_ba.weight"
+       "buf": "act"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.42.mlp.down_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 96
+       "buf": "model.layers.43.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l52.conv_update",
-     "kernel": "conv_update",
+     "label": "l43.qkv_proj",
+     "kernel": "gemm",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.52.linear_attn.conv1d.weight"
+       "buf": "model.layers.43.self_attn.qkv_proj.weight"
       },
       {
-       "state": "gdn"
+       "buf": "qkv"
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 160
+       "sym": "tokens"
       },
       {
-       "buf": "qkvz"
+       "i32": 14336
       },
       {
-       "i32": 1
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l43.prep",
+     "kernel": "attn_prep",
+     "args": [
       {
-       "i32": 49
+       "buf": "q_n"
       },
       {
-       "i64": 1
+       "buf": "k_n"
       },
       {
-       "i64": 1
+       "buf": "qkv"
       },
       {
-       "i64": 0
+       "buf": "model.layers.43.self_attn.q_norm.weight_p1"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l52.recurrent",
-     "kernel": "recurrent",
-     "args": [
+       "buf": "model.layers.43.self_attn.k_norm.weight_p1"
+      },
       {
-       "buf": "qkvz"
+       "buf": "cos_g"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "sin_g"
       },
       {
-       "buf": "ba"
+       "state": "kv",
+       "offset": 32112640
       },
       {
-       "buf": "model.layers.52.linear_attn.A_log"
+       "state": "kv",
+       "offset": 32113152
       },
       {
-       "buf": "model.layers.52.linear_attn.dt_bias"
+       "buf": "slot_mapping"
       },
       {
-       "buf": "core_attn_out"
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "i64": 25690112
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 160
+       "i64": 2048
       },
       {
-       "f32": 0.0883883461356163
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l52.gated_norm",
-     "kernel": "gated_norm_decode",
+     "label": "l43.attn",
+     "kernel": "attn",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "attn_out"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "q_n"
       },
       {
-       "buf": "model.layers.52.linear_attn.norm.weight"
+       "state": "kv",
+       "offset": 32112640
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "state": "kv",
+       "offset": 32113152
       },
       {
-       "i32": 128
+       "buf": "block_table"
       },
       {
-       "i32": 128
+       "buf": "seq_lens"
       },
       {
-       "i32": 128
+       "f32": 0.0625
       },
       {
-       "i32": 48
+       "buf": "kv_scales"
       },
       {
-       "f32": 9.999999974752427e-07
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "i64": 0
+       "f32": 1.0
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l52.out_proj",
-     "kernel": "gemm",
-     "args": [
+       "f32": 0.0
+      },
       {
-       "buf": "core_attn_out"
+       "i64": 8
       },
       {
-       "buf": "model.layers.52.linear_attn.out_proj.weight"
+       "i64": 6144
       },
       {
-       "buf": "y"
+       "i64": 256
       },
       {
-       "sym": "tokens"
+       "i64": 6144
       },
       {
-       "i32": 5120
+       "i64": 256
       },
       {
-       "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l52.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
+       "i64": 0
+      },
       {
-       "buf": "x"
+       "buf": "seq_lens"
       },
       {
-       "buf": "y"
+       "i64": 25690112
       },
       {
-       "buf": "residual"
+       "i64": 2048
       },
       {
-       "buf": "model.layers.52.post_attention_layernorm.weight_p1"
+       "i64": 512
       },
       {
-       "i32": 5120
+       "i64": 25690112
       },
       {
-       "sym": "tokens"
+       "i64": 2048
       },
       {
-       "i32": 5120
+       "i64": 512
       },
       {
-       "i32": 5120
+       "buf": "cu_seqlens_q"
       },
       {
-       "f32": 9.999999974752427e-07
+       "i32": 1
+      },
+      {
+       "i64": 0
+      },
+      {
+       "i64": 0
       }
      ]
     },
     {
-     "label": "l52.gate_up",
-     "kernel": "gemm",
+     "label": "l43.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.52.mlp.gate_up_proj.weight"
+       "buf": "attn_out"
       },
       {
-       "buf": "gate_up"
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.43.self_attn.o_proj.weight"
       },
       {
-       "i32": 34816
+       "buf": "residual"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l52.silu_mul",
-     "kernel": "silu_mul",
-     "args": [
+       "buf": "model.layers.43.post_attention_layernorm.weight_p1"
+      },
       {
-       "buf": "act"
+       "sym": "tokens"
       },
       {
-       "buf": "gate_up"
+       "i32": 5120
       },
       {
-       "i32": 17408
+       "i32": 6144
       },
       {
-       "i32": 0
+       "i32": 256
       },
       {
-       "f32": 1.0
+       "i32": 14336
       },
       {
-       "i32": 0
+       "i32": 512
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l52.down_proj",
-     "kernel": "gemm",
+     "label": "l43.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
        "buf": "act"
       },
       {
-       "buf": "model.layers.52.mlp.down_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.43.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 17408
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l52.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l43.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.43.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.53.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.44.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -64868,7 +52548,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -64876,45 +52556,30 @@
      ]
     },
     {
-     "label": "l53.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l44.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.53.linear_attn.in_proj_qkvz.weight"
-      },
       {
        "buf": "qkvz"
       },
       {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
+       "buf": "ba"
       },
-      {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l53.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.53.linear_attn.in_proj_ba.weight"
+       "buf": "model.layers.44.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.44.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
+      {
+       "i32": 16384
+      },
       {
        "i32": 96
       },
@@ -64924,173 +52589,136 @@
      ]
     },
     {
-     "label": "l53.conv_update",
-     "kernel": "conv_update",
+     "label": "l44.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.53.linear_attn.conv1d.weight"
+       "buf": "model.layers.44.linear_attn.conv1d.weight"
       },
       {
        "state": "gdn"
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 164
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "i32": 1
-      },
-      {
-       "i32": 49
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 1
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l53.recurrent",
-     "kernel": "recurrent",
-     "args": [
-      {
-       "buf": "qkvz"
+       "state": "gdn",
+       "offset": 61440
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "gdn.line_index",
+       "offset": 136
       },
       {
        "buf": "ba"
       },
       {
-       "buf": "model.layers.53.linear_attn.A_log"
+       "buf": "model.layers.44.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.53.linear_attn.dt_bias"
+       "buf": "model.layers.44.linear_attn.dt_bias"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "buf": "model.layers.44.linear_attn.norm.weight"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 164
+       "f32": 9.999999974752427e-07
       },
       {
-       "f32": 0.0883883461356163
+       "i32": 49
       },
       {
-       "i64": 0
+       "i32": 1605632
       },
       {
-       "i64": 0
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l53.gated_norm",
-     "kernel": "gated_norm_decode",
+     "label": "l44.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.53.linear_attn.norm.weight"
+       "buf": "model.layers.44.linear_attn.out_proj.weight"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.44.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "i32": 48
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l53.out_proj",
-     "kernel": "gemm",
+     "label": "l44.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.53.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.44.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l53.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l44.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.44.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.53.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.45.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -65099,7 +52727,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -65107,23 +52735,32 @@
      ]
     },
     {
-     "label": "l53.gate_up",
-     "kernel": "gemm",
+     "label": "l45.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.53.mlp.gate_up_proj.weight"
+       "buf": "model.layers.45.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.45.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -65131,71 +52768,79 @@
      ]
     },
     {
-     "label": "l53.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l45.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
+      },
+      {
+       "buf": "model.layers.45.linear_attn.conv1d.weight"
+      },
+      {
+       "state": "gdn"
+      },
+      {
+       "state": "gdn",
+       "offset": 61440
+      },
+      {
+       "buf": "gdn.line_index",
+       "offset": 140
       },
       {
-       "buf": "gate_up"
+       "buf": "ba"
       },
       {
-       "i32": 17408
+       "buf": "model.layers.45.linear_attn.A_log"
       },
       {
-       "i32": 0
+       "buf": "model.layers.45.linear_attn.dt_bias"
       },
       {
-       "f32": 1.0
+       "buf": "core_attn_out"
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l53.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "model.layers.45.linear_attn.norm.weight"
+      },
       {
-       "buf": "act"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "model.layers.53.mlp.down_proj.weight"
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "y"
+       "i32": 49
       },
       {
-       "sym": "tokens"
+       "i32": 1605632
       },
       {
-       "i32": 5120
+       "i32": 10240
       },
       {
-       "i32": 17408
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l53.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l45.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.45.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.54.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.45.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -65204,7 +52849,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -65212,23 +52857,23 @@
      ]
     },
     {
-     "label": "l54.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l45.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.54.linear_attn.in_proj_qkvz.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.45.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -65236,197 +52881,202 @@
      ]
     },
     {
-     "label": "l54.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l45.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.54.linear_attn.in_proj_ba.weight"
+       "buf": "act"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.45.mlp.down_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 96
+       "buf": "model.layers.46.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l54.conv_update",
-     "kernel": "conv_update",
+     "label": "l46.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.54.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 168
+       "buf": "ba"
       },
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "i32": 1
+       "buf": "model.layers.46.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i32": 49
+       "buf": "model.layers.46.linear_attn.in_proj_ba.weight"
       },
       {
-       "i64": 1
+       "sym": "tokens"
       },
       {
-       "i64": 1
+       "i32": 16384
       },
       {
-       "i64": 0
+       "i32": 96
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l54.recurrent",
-     "kernel": "recurrent",
+     "label": "l46.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.46.linear_attn.conv1d.weight"
+      },
+      {
+       "state": "gdn"
+      },
+      {
+       "state": "gdn",
+       "offset": 61440
+      },
+      {
+       "buf": "gdn.line_index",
+       "offset": 144
       },
       {
        "buf": "ba"
       },
       {
-       "buf": "model.layers.54.linear_attn.A_log"
+       "buf": "model.layers.46.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.54.linear_attn.dt_bias"
+       "buf": "model.layers.46.linear_attn.dt_bias"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "buf": "model.layers.46.linear_attn.norm.weight"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 168
+       "f32": 9.999999974752427e-07
       },
       {
-       "f32": 0.0883883461356163
+       "i32": 49
       },
       {
-       "i64": 0
+       "i32": 1605632
       },
       {
-       "i64": 0
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l54.gated_norm",
-     "kernel": "gated_norm_decode",
+     "label": "l46.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.54.linear_attn.norm.weight"
+       "buf": "model.layers.46.linear_attn.out_proj.weight"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.46.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "i32": 48
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l54.out_proj",
-     "kernel": "gemm",
+     "label": "l46.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.54.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.46.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l54.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l46.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.46.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.54.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.47.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -65435,7 +53085,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -65443,23 +53093,23 @@
      ]
     },
     {
-     "label": "l54.gate_up",
+     "label": "l47.qkv_proj",
      "kernel": "gemm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.54.mlp.gate_up_proj.weight"
+       "buf": "model.layers.47.self_attn.qkv_proj.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "qkv"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 14336
       },
       {
        "i32": 5120
@@ -65467,80 +53117,72 @@
      ]
     },
     {
-     "label": "l54.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l47.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
+       "buf": "q_n"
       },
       {
-       "i32": 0
+       "buf": "k_n"
       },
       {
-       "f32": 1.0
+       "buf": "qkv"
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l54.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "model.layers.47.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "model.layers.54.mlp.down_proj.weight"
+       "buf": "model.layers.47.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "y"
+       "buf": "cos_g"
       },
       {
-       "sym": "tokens"
+       "buf": "sin_g"
       },
       {
-       "i32": 5120
+       "state": "kv",
+       "offset": 35323904
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l54.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
+       "state": "kv",
+       "offset": 35324416
       },
       {
-       "buf": "y"
+       "buf": "slot_mapping"
       },
       {
-       "buf": "residual"
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "buf": "model.layers.55.input_layernorm.weight_p1"
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i32": 5120
+       "i64": 25690112
       },
       {
-       "sym": "tokens"
+       "i64": 2048
       },
       {
-       "i32": 5120
+       "i64": 512
       },
       {
-       "i32": 5120
+       "i32": 784
       },
       {
        "f32": 9.999999974752427e-07
@@ -65548,370 +53190,365 @@
      ]
     },
     {
-     "label": "l55.qkv_proj",
-     "kernel": "gemm",
+     "label": "l47.attn",
+     "kernel": "attn",
      "args": [
       {
-       "buf": "x"
+       "buf": "attn_out"
       },
       {
-       "buf": "model.layers.55.self_attn.qkv_proj.weight"
+       "buf": "q_n"
       },
       {
-       "buf": "qkv"
+       "state": "kv",
+       "offset": 35323904
       },
       {
-       "sym": "tokens"
+       "state": "kv",
+       "offset": 35324416
       },
       {
-       "i32": 14336
+       "buf": "block_table"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l55.q_norm",
-     "kernel": "gemma_norm_qhead",
-     "args": [
+       "buf": "seq_lens"
+      },
       {
-       "buf": "q_n"
+       "f32": 0.0625
       },
       {
-       "buf": "qkv"
+       "buf": "kv_scales"
       },
       {
-       "buf": "model.layers.55.self_attn.q_norm.weight_p1"
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "i32": 256
+       "f32": 1.0
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "f32": 0.0
       },
       {
-       "i32": 24
+       "i64": 8
       },
       {
-       "i32": 14336
+       "i64": 6144
       },
       {
-       "i32": 512
+       "i64": 256
       },
       {
-       "i32": 6144
+       "i64": 6144
       },
       {
-       "i32": 256
+       "i64": 256
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l55.k_norm",
-     "kernel": "gemma_norm_khead",
-     "args": [
+       "i64": 0
+      },
       {
-       "buf": "k_n"
+       "buf": "seq_lens"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
+       "i64": 25690112
       },
       {
-       "buf": "model.layers.55.self_attn.k_norm.weight_p1"
+       "i64": 2048
       },
       {
-       "i32": 256
+       "i64": 512
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
+       "i64": 25690112
       },
       {
-       "i32": 4
+       "i64": 2048
       },
       {
-       "i32": 14336
+       "i64": 512
       },
       {
-       "i32": 256
+       "buf": "cu_seqlens_q"
       },
       {
-       "i32": 1024
+       "i32": 1
       },
       {
-       "i32": 256
+       "i64": 0
       },
       {
-       "f32": 9.999999974752427e-07
+       "i64": 0
       }
      ]
     },
     {
-     "label": "l55.rope",
-     "kernel": "mrope",
+     "label": "l47.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "q_n"
+       "buf": "x"
       },
       {
-       "buf": "k_n"
+       "buf": "attn_out"
       },
       {
-       "buf": "cos_g"
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "buf": "sin_g"
+       "buf": "model.layers.47.self_attn.o_proj.weight"
       },
       {
-       "i32": 0
+       "buf": "residual"
       },
       {
-       "i64": 0
+       "buf": "model.layers.47.post_attention_layernorm.weight_p1"
       },
       {
-       "i64": 0
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 6144
+      },
+      {
+       "i32": 256
+      },
+      {
+       "i32": 14336
+      },
+      {
+       "i32": 512
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l55.kv_write",
-     "kernel": "reshape_and_cache",
+     "label": "l47.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 26624
+       "buf": "act"
       },
       {
-       "state": "kv",
-       "offset": 41746432
+       "buf": "x"
       },
       {
-       "state": "kv",
-       "offset": 41746944
+       "buf": "model.layers.47.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "slot_mapping"
+       "sym": "tokens"
       },
       {
-       "buf": "kv_scales"
+       "i32": 17408
       },
       {
-       "buf": "kv_scales",
-       "offset": 4
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l47.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
       {
-       "i64": 1024
+       "buf": "x"
       },
       {
-       "i64": 14336
+       "buf": "act"
       },
       {
-       "i64": 25690112
+       "buf": "model.layers.47.mlp.down_proj.weight"
       },
       {
-       "i64": 512
+       "buf": "residual"
       },
       {
-       "i64": 0
+       "buf": "model.layers.48.input_layernorm.weight_p1"
       },
       {
-       "i64": 0
+       "sym": "tokens"
       },
       {
-       "i64": 2048
+       "i32": 5120
       },
       {
-       "i64": 0
+       "i32": 17408
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l55.attn",
-     "kernel": "attn",
+     "label": "l48.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "attn_out"
-      },
-      {
-       "buf": "q_n"
-      },
-      {
-       "state": "kv",
-       "offset": 41746432
-      },
-      {
-       "state": "kv",
-       "offset": 41746944
+       "buf": "qkvz"
       },
       {
-       "buf": "block_table"
+       "buf": "ba"
       },
       {
-       "buf": "seq_lens"
+       "buf": "x"
       },
       {
-       "f32": 0.0625
+       "buf": "model.layers.48.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "kv_scales"
+       "buf": "model.layers.48.linear_attn.in_proj_ba.weight"
       },
       {
-       "buf": "kv_scales",
-       "offset": 4
+       "sym": "tokens"
       },
       {
-       "f32": 1.0
+       "i32": 16384
       },
       {
-       "f32": 0.0
+       "i32": 96
       },
       {
-       "i64": 8
-      },
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l48.gdn",
+     "kernel": "gdn_decode",
+     "args": [
       {
-       "i64": 6144
+       "buf": "qkvz"
       },
       {
-       "i64": 256
+       "buf": "model.layers.48.linear_attn.conv1d.weight"
       },
       {
-       "i64": 6144
+       "state": "gdn"
       },
       {
-       "i64": 256
+       "state": "gdn",
+       "offset": 61440
       },
       {
-       "i64": 0
+       "buf": "gdn.line_index",
+       "offset": 148
       },
       {
-       "buf": "seq_lens"
+       "buf": "ba"
       },
       {
-       "i64": 25690112
+       "buf": "model.layers.48.linear_attn.A_log"
       },
       {
-       "i64": 2048
+       "buf": "model.layers.48.linear_attn.dt_bias"
       },
       {
-       "i64": 512
+       "buf": "core_attn_out"
       },
       {
-       "i64": 25690112
+       "buf": "model.layers.48.linear_attn.norm.weight"
       },
       {
-       "i64": 2048
+       "f32": 0.0883883461356163
       },
       {
-       "i64": 512
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 49
       },
       {
-       "i32": 1
+       "i32": 1605632
       },
       {
-       "i64": 0
+       "i32": 10240
       },
       {
-       "i64": 0
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l55.gate",
-     "kernel": "sigmoid_mul",
+     "label": "l48.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "gated"
+       "buf": "x"
       },
       {
-       "buf": "attn_out"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "qkv",
-       "offset": 512
+       "buf": "model.layers.48.linear_attn.out_proj.weight"
       },
       {
-       "i32": 24
+       "buf": "residual"
       },
       {
-       "i32": 256
+       "buf": "model.layers.48.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 14336
+       "sym": "tokens"
       },
       {
-       "i32": 512
+       "i32": 5120
+      },
+      {
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l55.o_proj",
-     "kernel": "gemm",
+     "label": "l48.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "gated"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.55.self_attn.o_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.48.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l55.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l48.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.48.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.55.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.49.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -65920,7 +53557,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -65928,23 +53565,32 @@
      ]
     },
     {
-     "label": "l55.gate_up",
-     "kernel": "gemm",
+     "label": "l49.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.55.mlp.gate_up_proj.weight"
+       "buf": "model.layers.49.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.49.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -65952,71 +53598,79 @@
      ]
     },
     {
-     "label": "l55.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l49.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.49.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 61440
       },
       {
-       "f32": 1.0
+       "buf": "gdn.line_index",
+       "offset": 152
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l55.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "ba"
+      },
       {
-       "buf": "act"
+       "buf": "model.layers.49.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.55.mlp.down_proj.weight"
+       "buf": "model.layers.49.linear_attn.dt_bias"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.49.linear_attn.norm.weight"
       },
       {
-       "i32": 5120
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 17408
+       "f32": 9.999999974752427e-07
+      },
+      {
+       "i32": 49
+      },
+      {
+       "i32": 1605632
+      },
+      {
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l55.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l49.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.49.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.56.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.49.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -66025,7 +53679,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -66033,23 +53687,23 @@
      ]
     },
     {
-     "label": "l56.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l49.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.56.linear_attn.in_proj_qkvz.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.49.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -66057,197 +53711,202 @@
      ]
     },
     {
-     "label": "l56.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l49.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.56.linear_attn.in_proj_ba.weight"
+       "buf": "act"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.49.mlp.down_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 96
+       "buf": "model.layers.50.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l56.conv_update",
-     "kernel": "conv_update",
+     "label": "l50.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.56.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 172
+       "buf": "ba"
       },
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "i32": 1
+       "buf": "model.layers.50.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i32": 49
+       "buf": "model.layers.50.linear_attn.in_proj_ba.weight"
       },
       {
-       "i64": 1
+       "sym": "tokens"
       },
       {
-       "i64": 1
+       "i32": 16384
       },
       {
-       "i64": 0
+       "i32": 96
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l56.recurrent",
-     "kernel": "recurrent",
+     "label": "l50.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.50.linear_attn.conv1d.weight"
+      },
+      {
+       "state": "gdn"
+      },
+      {
+       "state": "gdn",
+       "offset": 61440
+      },
+      {
+       "buf": "gdn.line_index",
+       "offset": 156
       },
       {
        "buf": "ba"
       },
       {
-       "buf": "model.layers.56.linear_attn.A_log"
+       "buf": "model.layers.50.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.56.linear_attn.dt_bias"
+       "buf": "model.layers.50.linear_attn.dt_bias"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "buf": "model.layers.50.linear_attn.norm.weight"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 172
+       "f32": 9.999999974752427e-07
       },
       {
-       "f32": 0.0883883461356163
+       "i32": 49
       },
       {
-       "i64": 0
+       "i32": 1605632
       },
       {
-       "i64": 0
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l56.gated_norm",
-     "kernel": "gated_norm_decode",
+     "label": "l50.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.56.linear_attn.norm.weight"
+       "buf": "model.layers.50.linear_attn.out_proj.weight"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.50.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "i32": 48
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l56.out_proj",
-     "kernel": "gemm",
+     "label": "l50.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.56.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.50.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l56.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l50.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.50.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.56.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.51.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -66256,7 +53915,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -66264,23 +53923,23 @@
      ]
     },
     {
-     "label": "l56.gate_up",
+     "label": "l51.qkv_proj",
      "kernel": "gemm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.56.mlp.gate_up_proj.weight"
+       "buf": "model.layers.51.self_attn.qkv_proj.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "qkv"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 14336
       },
       {
        "i32": 5120
@@ -66288,80 +53947,72 @@
      ]
     },
     {
-     "label": "l56.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l51.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
+       "buf": "q_n"
       },
       {
-       "i32": 0
+       "buf": "k_n"
       },
       {
-       "f32": 1.0
+       "buf": "qkv"
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l56.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "model.layers.51.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "model.layers.56.mlp.down_proj.weight"
+       "buf": "model.layers.51.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "y"
+       "buf": "cos_g"
       },
       {
-       "sym": "tokens"
+       "buf": "sin_g"
       },
       {
-       "i32": 5120
+       "state": "kv",
+       "offset": 38535168
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l56.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
+       "state": "kv",
+       "offset": 38535680
       },
       {
-       "buf": "y"
+       "buf": "slot_mapping"
       },
       {
-       "buf": "residual"
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "buf": "model.layers.57.input_layernorm.weight_p1"
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i32": 5120
+       "i64": 25690112
       },
       {
-       "sym": "tokens"
+       "i64": 2048
       },
       {
-       "i32": 5120
+       "i64": 512
       },
       {
-       "i32": 5120
+       "i32": 784
       },
       {
        "f32": 9.999999974752427e-07
@@ -66369,84 +54020,89 @@
      ]
     },
     {
-     "label": "l57.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l51.attn",
+     "kernel": "attn",
      "args": [
       {
-       "buf": "x"
+       "buf": "attn_out"
       },
       {
-       "buf": "model.layers.57.linear_attn.in_proj_qkvz.weight"
+       "buf": "q_n"
       },
       {
-       "buf": "qkvz"
+       "state": "kv",
+       "offset": 38535168
       },
       {
-       "sym": "tokens"
+       "state": "kv",
+       "offset": 38535680
       },
       {
-       "i32": 16384
+       "buf": "block_table"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l57.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
+       "buf": "seq_lens"
+      },
       {
-       "buf": "x"
+       "f32": 0.0625
       },
       {
-       "buf": "model.layers.57.linear_attn.in_proj_ba.weight"
+       "buf": "kv_scales"
       },
       {
-       "buf": "ba"
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "sym": "tokens"
+       "f32": 1.0
       },
       {
-       "i32": 96
+       "f32": 0.0
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l57.conv_update",
-     "kernel": "conv_update",
-     "args": [
+       "i64": 8
+      },
       {
-       "buf": "qkvz"
+       "i64": 6144
       },
       {
-       "buf": "model.layers.57.linear_attn.conv1d.weight"
+       "i64": 256
       },
       {
-       "state": "gdn"
+       "i64": 6144
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 176
+       "i64": 256
       },
       {
-       "buf": "qkvz"
+       "i64": 0
       },
       {
-       "i32": 1
+       "buf": "seq_lens"
       },
       {
-       "i32": 49
+       "i64": 25690112
+      },
+      {
+       "i64": 2048
+      },
+      {
+       "i64": 512
+      },
+      {
+       "i64": 25690112
+      },
+      {
+       "i64": 2048
+      },
+      {
+       "i64": 512
       },
       {
-       "i64": 1
+       "buf": "cu_seqlens_q"
       },
       {
-       "i64": 1
+       "i32": 1
       },
       {
        "i64": 0
@@ -66457,238 +54113,272 @@
      ]
     },
     {
-     "label": "l57.recurrent",
-     "kernel": "recurrent",
+     "label": "l51.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "attn_out"
       },
       {
-       "buf": "ba"
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "buf": "model.layers.57.linear_attn.A_log"
+       "buf": "model.layers.51.self_attn.o_proj.weight"
       },
       {
-       "buf": "model.layers.57.linear_attn.dt_bias"
+       "buf": "residual"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "model.layers.51.post_attention_layernorm.weight_p1"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "sym": "tokens"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "i32": 5120
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 176
+       "i32": 6144
       },
       {
-       "f32": 0.0883883461356163
+       "i32": 256
       },
       {
-       "i64": 0
+       "i32": 14336
       },
       {
-       "i64": 0
+       "i32": 512
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l57.gated_norm",
-     "kernel": "gated_norm_decode",
+     "label": "l51.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
-       "buf": "model.layers.57.linear_attn.norm.weight"
+       "buf": "model.layers.51.mlp.gate_up_proj.weight"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 17408
       },
       {
-       "i32": 128
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l51.down_norm",
+     "kernel": "gemm8_add_norm",
+     "args": [
+      {
+       "buf": "x"
       },
       {
-       "i32": 128
+       "buf": "act"
       },
       {
-       "i32": 48
+       "buf": "model.layers.51.mlp.down_proj.weight"
       },
       {
-       "f32": 9.999999974752427e-07
+       "buf": "residual"
       },
       {
-       "i64": 0
+       "buf": "model.layers.52.input_layernorm.weight_p1"
       },
       {
-       "i64": 0
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l57.out_proj",
-     "kernel": "gemm",
+     "label": "l52.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "qkvz"
       },
       {
-       "buf": "model.layers.57.linear_attn.out_proj.weight"
+       "buf": "ba"
       },
       {
-       "buf": "y"
+       "buf": "x"
+      },
+      {
+       "buf": "model.layers.52.linear_attn.in_proj_qkvz.weight"
+      },
+      {
+       "buf": "model.layers.52.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 16384
       },
       {
-       "i32": 6144
+       "i32": 96
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l57.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l52.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
-       "buf": "x"
+       "buf": "qkvz"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.52.linear_attn.conv1d.weight"
       },
       {
-       "buf": "residual"
+       "state": "gdn"
       },
       {
-       "buf": "model.layers.57.post_attention_layernorm.weight_p1"
+       "state": "gdn",
+       "offset": 61440
       },
       {
-       "i32": 5120
+       "buf": "gdn.line_index",
+       "offset": 160
       },
       {
-       "sym": "tokens"
+       "buf": "ba"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.52.linear_attn.A_log"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.52.linear_attn.dt_bias"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l57.gate_up",
-     "kernel": "gemm",
-     "args": [
+       "buf": "core_attn_out"
+      },
       {
-       "buf": "x"
+       "buf": "model.layers.52.linear_attn.norm.weight"
       },
       {
-       "buf": "model.layers.57.mlp.gate_up_proj.weight"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "gate_up"
+       "f32": 9.999999974752427e-07
       },
       {
-       "sym": "tokens"
+       "i32": 49
       },
       {
-       "i32": 34816
+       "i32": 1605632
+      },
+      {
+       "i32": 10240
       },
       {
-       "i32": 5120
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l57.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l52.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "act"
+       "buf": "x"
       },
       {
-       "buf": "gate_up"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 17408
+       "buf": "model.layers.52.linear_attn.out_proj.weight"
       },
       {
-       "i32": 0
+       "buf": "residual"
       },
       {
-       "f32": 1.0
+       "buf": "model.layers.52.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 0
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l57.down_proj",
-     "kernel": "gemm",
+     "label": "l52.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
        "buf": "act"
       },
       {
-       "buf": "model.layers.57.mlp.down_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.52.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 17408
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l57.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l52.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.52.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.58.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.53.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -66697,7 +54387,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -66705,17 +54395,23 @@
      ]
     },
     {
-     "label": "l58.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l53.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.58.linear_attn.in_proj_qkvz.weight"
+       "buf": "model.layers.53.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.53.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
@@ -66723,308 +54419,324 @@
       {
        "i32": 16384
       },
+      {
+       "i32": 96
+      },
       {
        "i32": 5120
       }
      ]
     },
     {
-     "label": "l58.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l53.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
-       "buf": "x"
+       "buf": "qkvz"
       },
       {
-       "buf": "model.layers.58.linear_attn.in_proj_ba.weight"
+       "buf": "model.layers.53.linear_attn.conv1d.weight"
       },
       {
-       "buf": "ba"
+       "state": "gdn"
       },
       {
-       "sym": "tokens"
+       "state": "gdn",
+       "offset": 61440
       },
       {
-       "i32": 96
+       "buf": "gdn.line_index",
+       "offset": 164
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l58.conv_update",
-     "kernel": "conv_update",
-     "args": [
-      {
-       "buf": "qkvz"
+       "buf": "ba"
       },
       {
-       "buf": "model.layers.58.linear_attn.conv1d.weight"
+       "buf": "model.layers.53.linear_attn.A_log"
       },
       {
-       "state": "gdn"
+       "buf": "model.layers.53.linear_attn.dt_bias"
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 180
+       "buf": "core_attn_out"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.53.linear_attn.norm.weight"
       },
       {
-       "i32": 1
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 49
+       "f32": 9.999999974752427e-07
       },
       {
-       "i64": 1
+       "i32": 49
       },
       {
-       "i64": 1
+       "i32": 1605632
       },
       {
-       "i64": 0
+       "i32": 10240
       },
       {
-       "i64": 0
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l58.recurrent",
-     "kernel": "recurrent",
+     "label": "l53.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "core_attn_out"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.53.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.58.linear_attn.A_log"
+       "buf": "residual"
       },
       {
-       "buf": "model.layers.58.linear_attn.dt_bias"
+       "buf": "model.layers.53.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "core_attn_out"
+       "sym": "tokens"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "i32": 5120
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "i32": 6144
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 180
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l53.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
+     "args": [
+      {
+       "buf": "act"
       },
       {
-       "f32": 0.0883883461356163
+       "buf": "x"
       },
       {
-       "i64": 0
+       "buf": "model.layers.53.mlp.gate_up_proj.weight"
       },
       {
-       "i64": 0
+       "sym": "tokens"
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l58.gated_norm",
-     "kernel": "gated_norm_decode",
+     "label": "l53.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.58.linear_attn.norm.weight"
+       "buf": "model.layers.53.mlp.down_proj.weight"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.54.input_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "i32": 48
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l58.out_proj",
-     "kernel": "gemm",
+     "label": "l54.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "qkvz"
       },
       {
-       "buf": "model.layers.58.linear_attn.out_proj.weight"
+       "buf": "ba"
       },
       {
-       "buf": "y"
+       "buf": "x"
+      },
+      {
+       "buf": "model.layers.54.linear_attn.in_proj_qkvz.weight"
+      },
+      {
+       "buf": "model.layers.54.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 16384
       },
       {
-       "i32": 6144
+       "i32": 96
+      },
+      {
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l58.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l54.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
-       "buf": "x"
+       "buf": "qkvz"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.54.linear_attn.conv1d.weight"
       },
       {
-       "buf": "residual"
+       "state": "gdn"
       },
       {
-       "buf": "model.layers.58.post_attention_layernorm.weight_p1"
+       "state": "gdn",
+       "offset": 61440
       },
       {
-       "i32": 5120
+       "buf": "gdn.line_index",
+       "offset": 168
       },
       {
-       "sym": "tokens"
+       "buf": "ba"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.54.linear_attn.A_log"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.54.linear_attn.dt_bias"
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l58.gate_up",
-     "kernel": "gemm",
-     "args": [
+       "buf": "core_attn_out"
+      },
       {
-       "buf": "x"
+       "buf": "model.layers.54.linear_attn.norm.weight"
       },
       {
-       "buf": "model.layers.58.mlp.gate_up_proj.weight"
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "gate_up"
+       "f32": 9.999999974752427e-07
       },
       {
-       "sym": "tokens"
+       "i32": 49
       },
       {
-       "i32": 34816
+       "i32": 1605632
       },
       {
-       "i32": 5120
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l58.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l54.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "act"
+       "buf": "x"
       },
       {
-       "buf": "gate_up"
+       "buf": "core_attn_out"
       },
       {
-       "i32": 17408
+       "buf": "model.layers.54.linear_attn.out_proj.weight"
       },
       {
-       "i32": 0
+       "buf": "residual"
       },
       {
-       "f32": 1.0
+       "buf": "model.layers.54.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 0
+       "sym": "tokens"
+      },
+      {
+       "i32": 5120
+      },
+      {
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l58.down_proj",
-     "kernel": "gemm",
+     "label": "l54.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
        "buf": "act"
       },
       {
-       "buf": "model.layers.58.mlp.down_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.54.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 17408
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l58.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l54.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.54.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.59.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.55.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -67033,7 +54745,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -67041,14 +54753,14 @@
      ]
     },
     {
-     "label": "l59.qkv_proj",
+     "label": "l55.qkv_proj",
      "kernel": "gemm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.59.self_attn.qkv_proj.weight"
+       "buf": "model.layers.55.self_attn.qkv_proj.weight"
       },
       {
        "buf": "qkv"
@@ -67065,20 +54777,40 @@
      ]
     },
     {
-     "label": "l59.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l55.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
       },
+      {
+       "buf": "k_n"
+      },
       {
        "buf": "qkv"
       },
       {
-       "buf": "model.layers.59.self_attn.q_norm.weight_p1"
+       "buf": "model.layers.55.self_attn.q_norm.weight_p1"
       },
       {
-       "i32": 256
+       "buf": "model.layers.55.self_attn.k_norm.weight_p1"
+      },
+      {
+       "buf": "cos_g"
+      },
+      {
+       "buf": "sin_g"
+      },
+      {
+       "state": "kv",
+       "offset": 41746432
+      },
+      {
+       "state": "kv",
+       "offset": 41746944
+      },
+      {
+       "buf": "slot_mapping"
       },
       {
        "expr": {
@@ -67091,19 +54823,26 @@
        }
       },
       {
-       "i32": 24
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i32": 14336
+       "i64": 25690112
       },
       {
-       "i32": 512
+       "i64": 2048
       },
       {
-       "i32": 6144
+       "i64": 512
       },
       {
-       "i32": 256
+       "i32": 784
       },
       {
        "f32": 9.999999974752427e-07
@@ -67111,70 +54850,89 @@
      ]
     },
     {
-     "label": "l59.k_norm",
-     "kernel": "gemma_norm_khead",
+     "label": "l55.attn",
+     "kernel": "attn",
      "args": [
       {
-       "buf": "k_n"
+       "buf": "attn_out"
       },
       {
-       "buf": "qkv",
-       "offset": 24576
+       "buf": "q_n"
       },
       {
-       "buf": "model.layers.59.self_attn.k_norm.weight_p1"
+       "state": "kv",
+       "offset": 41746432
       },
       {
-       "i32": 256
+       "state": "kv",
+       "offset": 41746944
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
+       "buf": "block_table"
       },
       {
-       "i32": 4
+       "buf": "seq_lens"
       },
       {
-       "i32": 14336
+       "f32": 0.0625
       },
       {
-       "i32": 256
+       "buf": "kv_scales"
       },
       {
-       "i32": 1024
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "i32": 256
+       "f32": 1.0
       },
       {
-       "f32": 9.999999974752427e-07
-      }
-     ]
-    },
-    {
-     "label": "l59.rope",
-     "kernel": "mrope",
-     "args": [
+       "f32": 0.0
+      },
       {
-       "buf": "q_n"
+       "i64": 8
       },
       {
-       "buf": "k_n"
+       "i64": 6144
       },
       {
-       "buf": "cos_g"
+       "i64": 256
       },
       {
-       "buf": "sin_g"
+       "i64": 6144
       },
       {
-       "i32": 0
+       "i64": 256
+      },
+      {
+       "i64": 0
+      },
+      {
+       "buf": "seq_lens"
+      },
+      {
+       "i64": 25690112
+      },
+      {
+       "i64": 2048
+      },
+      {
+       "i64": 512
+      },
+      {
+       "i64": 25690112
+      },
+      {
+       "i64": 2048
+      },
+      {
+       "i64": 512
+      },
+      {
+       "buf": "cu_seqlens_q"
+      },
+      {
+       "i32": 1
       },
       {
        "i64": 0
@@ -67185,226 +54943,272 @@
      ]
     },
     {
-     "label": "l59.kv_write",
-     "kernel": "reshape_and_cache",
+     "label": "l55.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "k_n"
+       "buf": "x"
+      },
+      {
+       "buf": "attn_out"
       },
       {
        "buf": "qkv",
-       "offset": 26624
+       "offset": 512
       },
       {
-       "state": "kv",
-       "offset": 44957696
+       "buf": "model.layers.55.self_attn.o_proj.weight"
       },
       {
-       "state": "kv",
-       "offset": 44958208
+       "buf": "residual"
       },
       {
-       "buf": "slot_mapping"
+       "buf": "model.layers.55.post_attention_layernorm.weight_p1"
       },
       {
-       "buf": "kv_scales"
+       "sym": "tokens"
       },
       {
-       "buf": "kv_scales",
-       "offset": 4
+       "i32": 5120
+      },
+      {
+       "i32": 6144
       },
       {
-       "i64": 1024
+       "i32": 256
       },
       {
-       "i64": 14336
+       "i32": 14336
       },
       {
-       "i64": 25690112
+       "i32": 512
       },
       {
-       "i64": 512
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l55.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
+     "args": [
+      {
+       "buf": "act"
       },
       {
-       "i64": 0
+       "buf": "x"
       },
       {
-       "i64": 0
+       "buf": "model.layers.55.mlp.gate_up_proj.weight"
       },
       {
-       "i64": 2048
+       "sym": "tokens"
       },
       {
-       "i64": 0
+       "i32": 17408
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l59.attn",
-     "kernel": "attn",
+     "label": "l55.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "attn_out"
+       "buf": "x"
       },
       {
-       "buf": "q_n"
+       "buf": "act"
       },
       {
-       "state": "kv",
-       "offset": 44957696
+       "buf": "model.layers.55.mlp.down_proj.weight"
       },
       {
-       "state": "kv",
-       "offset": 44958208
+       "buf": "residual"
       },
       {
-       "buf": "block_table"
+       "buf": "model.layers.56.input_layernorm.weight_p1"
       },
       {
-       "buf": "seq_lens"
+       "sym": "tokens"
       },
       {
-       "f32": 0.0625
+       "i32": 5120
       },
       {
-       "buf": "kv_scales"
+       "i32": 17408
       },
       {
-       "buf": "kv_scales",
-       "offset": 4
+       "f32": 9.999999974752427e-07
+      }
+     ]
+    },
+    {
+     "label": "l56.in_proj",
+     "kernel": "gemm8_dual",
+     "args": [
+      {
+       "buf": "qkvz"
       },
       {
-       "f32": 1.0
+       "buf": "ba"
       },
       {
-       "f32": 0.0
+       "buf": "x"
       },
       {
-       "i64": 8
+       "buf": "model.layers.56.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i64": 6144
+       "buf": "model.layers.56.linear_attn.in_proj_ba.weight"
       },
       {
-       "i64": 256
+       "sym": "tokens"
       },
       {
-       "i64": 6144
+       "i32": 16384
       },
       {
-       "i64": 256
+       "i32": 96
       },
       {
-       "i64": 0
+       "i32": 5120
+      }
+     ]
+    },
+    {
+     "label": "l56.gdn",
+     "kernel": "gdn_decode",
+     "args": [
+      {
+       "buf": "qkvz"
       },
       {
-       "buf": "seq_lens"
+       "buf": "model.layers.56.linear_attn.conv1d.weight"
       },
       {
-       "i64": 25690112
+       "state": "gdn"
+      },
+      {
+       "state": "gdn",
+       "offset": 61440
+      },
+      {
+       "buf": "gdn.line_index",
+       "offset": 172
+      },
+      {
+       "buf": "ba"
+      },
+      {
+       "buf": "model.layers.56.linear_attn.A_log"
       },
       {
-       "i64": 2048
+       "buf": "model.layers.56.linear_attn.dt_bias"
       },
       {
-       "i64": 512
+       "buf": "core_attn_out"
       },
       {
-       "i64": 25690112
+       "buf": "model.layers.56.linear_attn.norm.weight"
       },
       {
-       "i64": 2048
+       "f32": 0.0883883461356163
       },
       {
-       "i64": 512
+       "f32": 9.999999974752427e-07
       },
       {
-       "buf": "cu_seqlens_q"
+       "i32": 49
       },
       {
-       "i32": 1
+       "i32": 1605632
       },
       {
-       "i64": 0
+       "i32": 10240
       },
       {
-       "i64": 0
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l59.gate",
-     "kernel": "sigmoid_mul",
+     "label": "l56.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "gated"
+       "buf": "x"
       },
       {
-       "buf": "attn_out"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "qkv",
-       "offset": 512
+       "buf": "model.layers.56.linear_attn.out_proj.weight"
       },
       {
-       "i32": 24
+       "buf": "residual"
       },
       {
-       "i32": 256
+       "buf": "model.layers.56.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 14336
+       "sym": "tokens"
       },
       {
-       "i32": 512
+       "i32": 5120
+      },
+      {
+       "i32": 6144
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l59.o_proj",
-     "kernel": "gemm",
+     "label": "l56.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "gated"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.59.self_attn.o_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.56.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l59.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l56.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.56.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.59.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.57.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -67413,7 +55217,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -67421,23 +55225,32 @@
      ]
     },
     {
-     "label": "l59.gate_up",
-     "kernel": "gemm",
+     "label": "l57.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.59.mlp.gate_up_proj.weight"
+       "buf": "model.layers.57.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.57.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -67445,71 +55258,79 @@
      ]
     },
     {
-     "label": "l59.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l57.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.57.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 61440
       },
       {
-       "f32": 1.0
+       "buf": "gdn.line_index",
+       "offset": 176
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l59.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "ba"
+      },
       {
-       "buf": "act"
+       "buf": "model.layers.57.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.59.mlp.down_proj.weight"
+       "buf": "model.layers.57.linear_attn.dt_bias"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.57.linear_attn.norm.weight"
       },
       {
-       "i32": 5120
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 17408
+       "f32": 9.999999974752427e-07
+      },
+      {
+       "i32": 49
+      },
+      {
+       "i32": 1605632
+      },
+      {
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l59.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l57.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.57.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.60.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.57.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -67518,7 +55339,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -67526,23 +55347,23 @@
      ]
     },
     {
-     "label": "l60.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l57.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.60.linear_attn.in_proj_qkvz.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.57.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -67550,197 +55371,202 @@
      ]
     },
     {
-     "label": "l60.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l57.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.60.linear_attn.in_proj_ba.weight"
+       "buf": "act"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.57.mlp.down_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 96
+       "buf": "model.layers.58.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l60.conv_update",
-     "kernel": "conv_update",
+     "label": "l58.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.60.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 184
+       "buf": "ba"
       },
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "i32": 1
+       "buf": "model.layers.58.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i32": 49
+       "buf": "model.layers.58.linear_attn.in_proj_ba.weight"
       },
       {
-       "i64": 1
+       "sym": "tokens"
       },
       {
-       "i64": 1
+       "i32": 16384
       },
       {
-       "i64": 0
+       "i32": 96
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l60.recurrent",
-     "kernel": "recurrent",
+     "label": "l58.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.58.linear_attn.conv1d.weight"
+      },
+      {
+       "state": "gdn"
+      },
+      {
+       "state": "gdn",
+       "offset": 61440
+      },
+      {
+       "buf": "gdn.line_index",
+       "offset": 180
       },
       {
        "buf": "ba"
       },
       {
-       "buf": "model.layers.60.linear_attn.A_log"
+       "buf": "model.layers.58.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.60.linear_attn.dt_bias"
+       "buf": "model.layers.58.linear_attn.dt_bias"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "buf": "model.layers.58.linear_attn.norm.weight"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 184
+       "f32": 9.999999974752427e-07
       },
       {
-       "f32": 0.0883883461356163
+       "i32": 49
       },
       {
-       "i64": 0
+       "i32": 1605632
       },
       {
-       "i64": 0
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l60.gated_norm",
-     "kernel": "gated_norm_decode",
+     "label": "l58.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.60.linear_attn.norm.weight"
+       "buf": "model.layers.58.linear_attn.out_proj.weight"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.58.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "i32": 48
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l60.out_proj",
-     "kernel": "gemm",
+     "label": "l58.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.60.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.58.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l60.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l58.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.58.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.60.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.59.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -67749,7 +55575,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -67757,23 +55583,23 @@
      ]
     },
     {
-     "label": "l60.gate_up",
+     "label": "l59.qkv_proj",
      "kernel": "gemm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.60.mlp.gate_up_proj.weight"
+       "buf": "model.layers.59.self_attn.qkv_proj.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "qkv"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 14336
       },
       {
        "i32": 5120
@@ -67781,80 +55607,72 @@
      ]
     },
     {
-     "label": "l60.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l59.prep",
+     "kernel": "attn_prep",
      "args": [
       {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
+       "buf": "q_n"
       },
       {
-       "i32": 0
+       "buf": "k_n"
       },
       {
-       "f32": 1.0
+       "buf": "qkv"
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l60.down_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "act"
+       "buf": "model.layers.59.self_attn.q_norm.weight_p1"
       },
       {
-       "buf": "model.layers.60.mlp.down_proj.weight"
+       "buf": "model.layers.59.self_attn.k_norm.weight_p1"
       },
       {
-       "buf": "y"
+       "buf": "cos_g"
       },
       {
-       "sym": "tokens"
+       "buf": "sin_g"
       },
       {
-       "i32": 5120
+       "state": "kv",
+       "offset": 44957696
       },
       {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l60.next_input_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
+       "state": "kv",
+       "offset": 44958208
       },
       {
-       "buf": "y"
+       "buf": "slot_mapping"
       },
       {
-       "buf": "residual"
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "buf": "model.layers.61.input_layernorm.weight_p1"
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
-       "i32": 5120
+       "i64": 25690112
       },
       {
-       "sym": "tokens"
+       "i64": 2048
       },
       {
-       "i32": 5120
+       "i64": 512
       },
       {
-       "i32": 5120
+       "i32": 784
       },
       {
        "f32": 9.999999974752427e-07
@@ -67862,130 +55680,89 @@
      ]
     },
     {
-     "label": "l61.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l59.attn",
+     "kernel": "attn",
      "args": [
       {
-       "buf": "x"
-      },
-      {
-       "buf": "model.layers.61.linear_attn.in_proj_qkvz.weight"
-      },
-      {
-       "buf": "qkvz"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 16384
+       "buf": "attn_out"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l61.in_proj_ba",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "x"
+       "buf": "q_n"
       },
       {
-       "buf": "model.layers.61.linear_attn.in_proj_ba.weight"
+       "state": "kv",
+       "offset": 44957696
       },
       {
-       "buf": "ba"
+       "state": "kv",
+       "offset": 44958208
       },
       {
-       "sym": "tokens"
+       "buf": "block_table"
       },
       {
-       "i32": 96
+       "buf": "seq_lens"
       },
       {
-       "i32": 5120
-      }
-     ]
-    },
-    {
-     "label": "l61.conv_update",
-     "kernel": "conv_update",
-     "args": [
-      {
-       "buf": "qkvz"
+       "f32": 0.0625
       },
       {
-       "buf": "model.layers.61.linear_attn.conv1d.weight"
+       "buf": "kv_scales"
       },
       {
-       "state": "gdn"
+       "buf": "kv_scales",
+       "offset": 4
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 188
+       "f32": 1.0
       },
       {
-       "buf": "qkvz"
+       "f32": 0.0
       },
       {
-       "i32": 1
+       "i64": 8
       },
       {
-       "i32": 49
+       "i64": 6144
       },
       {
-       "i64": 1
+       "i64": 256
       },
       {
-       "i64": 1
+       "i64": 6144
       },
       {
-       "i64": 0
+       "i64": 256
       },
       {
        "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l61.recurrent",
-     "kernel": "recurrent",
-     "args": [
-      {
-       "buf": "qkvz"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "seq_lens"
       },
       {
-       "buf": "ba"
+       "i64": 25690112
       },
       {
-       "buf": "model.layers.61.linear_attn.A_log"
+       "i64": 2048
       },
       {
-       "buf": "model.layers.61.linear_attn.dt_bias"
+       "i64": 512
       },
       {
-       "buf": "core_attn_out"
+       "i64": 25690112
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "i64": 2048
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "i64": 512
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 188
+       "buf": "cu_seqlens_q"
       },
       {
-       "f32": 0.0883883461356163
+       "i32": 1
       },
       {
        "i64": 0
@@ -67996,87 +55773,93 @@
      ]
     },
     {
-     "label": "l61.gated_norm",
-     "kernel": "gated_norm_decode",
+     "label": "l59.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
-       "buf": "core_attn_out"
+       "buf": "attn_out"
       },
       {
-       "buf": "model.layers.61.linear_attn.norm.weight"
+       "buf": "qkv",
+       "offset": 512
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "model.layers.59.self_attn.o_proj.weight"
       },
       {
-       "i32": 128
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.59.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 48
+       "i32": 5120
       },
       {
-       "f32": 9.999999974752427e-07
+       "i32": 6144
       },
       {
-       "i64": 0
+       "i32": 256
       },
       {
-       "i64": 0
+       "i32": 14336
+      },
+      {
+       "i32": 512
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l61.out_proj",
-     "kernel": "gemm",
+     "label": "l59.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.61.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.59.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l61.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l59.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.59.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.61.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.60.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -68085,7 +55868,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -68093,23 +55876,32 @@
      ]
     },
     {
-     "label": "l61.gate_up",
-     "kernel": "gemm",
+     "label": "l60.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.61.mlp.gate_up_proj.weight"
+       "buf": "model.layers.60.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.60.linear_attn.in_proj_ba.weight"
+      },
+      {
+       "sym": "tokens"
       },
       {
-       "sym": "tokens"
+       "i32": 16384
       },
       {
-       "i32": 34816
+       "i32": 96
       },
       {
        "i32": 5120
@@ -68117,71 +55909,79 @@
      ]
     },
     {
-     "label": "l61.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l60.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.60.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 61440
       },
       {
-       "f32": 1.0
+       "buf": "gdn.line_index",
+       "offset": 184
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l61.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "ba"
+      },
       {
-       "buf": "act"
+       "buf": "model.layers.60.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.61.mlp.down_proj.weight"
+       "buf": "model.layers.60.linear_attn.dt_bias"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.60.linear_attn.norm.weight"
       },
       {
-       "i32": 5120
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 17408
+       "f32": 9.999999974752427e-07
+      },
+      {
+       "i32": 49
+      },
+      {
+       "i32": 1605632
+      },
+      {
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l61.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l60.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.60.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.62.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.60.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -68190,7 +55990,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -68198,23 +55998,23 @@
      ]
     },
     {
-     "label": "l62.in_proj_qkvz",
-     "kernel": "gemm",
+     "label": "l60.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.62.linear_attn.in_proj_qkvz.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkvz"
+       "buf": "model.layers.60.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 16384
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -68222,197 +56022,202 @@
      ]
     },
     {
-     "label": "l62.in_proj_ba",
-     "kernel": "gemm",
+     "label": "l60.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.62.linear_attn.in_proj_ba.weight"
+       "buf": "act"
       },
       {
-       "buf": "ba"
+       "buf": "model.layers.60.mlp.down_proj.weight"
       },
       {
-       "sym": "tokens"
+       "buf": "residual"
       },
       {
-       "i32": 96
+       "buf": "model.layers.61.input_layernorm.weight_p1"
+      },
+      {
+       "sym": "tokens"
       },
       {
        "i32": 5120
+      },
+      {
+       "i32": 17408
+      },
+      {
+       "f32": 9.999999974752427e-07
       }
      ]
     },
     {
-     "label": "l62.conv_update",
-     "kernel": "conv_update",
+     "label": "l61.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "model.layers.62.linear_attn.conv1d.weight"
-      },
-      {
-       "state": "gdn"
-      },
-      {
-       "buf": "gdn.line_index",
-       "offset": 192
+       "buf": "ba"
       },
       {
-       "buf": "qkvz"
+       "buf": "x"
       },
       {
-       "i32": 1
+       "buf": "model.layers.61.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "i32": 49
+       "buf": "model.layers.61.linear_attn.in_proj_ba.weight"
       },
       {
-       "i64": 1
+       "sym": "tokens"
       },
       {
-       "i64": 1
+       "i32": 16384
       },
       {
-       "i64": 0
+       "i32": 96
       },
       {
-       "i64": 0
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l62.recurrent",
-     "kernel": "recurrent",
+     "label": "l61.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
        "buf": "qkvz"
       },
       {
-       "buf": "ba",
-       "offset": 96
+       "buf": "model.layers.61.linear_attn.conv1d.weight"
+      },
+      {
+       "state": "gdn"
+      },
+      {
+       "state": "gdn",
+       "offset": 61440
+      },
+      {
+       "buf": "gdn.line_index",
+       "offset": 188
       },
       {
        "buf": "ba"
       },
       {
-       "buf": "model.layers.62.linear_attn.A_log"
+       "buf": "model.layers.61.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.62.linear_attn.dt_bias"
+       "buf": "model.layers.61.linear_attn.dt_bias"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "buf": "model.layers.61.linear_attn.norm.weight"
       },
       {
-       "state": "gdn",
-       "offset": 61440
+       "f32": 0.0883883461356163
       },
       {
-       "buf": "gdn.line_index",
-       "offset": 192
+       "f32": 9.999999974752427e-07
       },
       {
-       "f32": 0.0883883461356163
+       "i32": 49
       },
       {
-       "i64": 0
+       "i32": 1605632
       },
       {
-       "i64": 0
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l62.gated_norm",
-     "kernel": "gated_norm_decode",
+     "label": "l61.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "x"
       },
       {
        "buf": "core_attn_out"
       },
       {
-       "buf": "model.layers.62.linear_attn.norm.weight"
+       "buf": "model.layers.61.linear_attn.out_proj.weight"
       },
       {
-       "buf": "qkvz",
-       "offset": 20480
+       "buf": "residual"
       },
       {
-       "i32": 128
+       "buf": "model.layers.61.post_attention_layernorm.weight_p1"
       },
       {
-       "i32": 128
+       "sym": "tokens"
       },
       {
-       "i32": 128
+       "i32": 5120
       },
       {
-       "i32": 48
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
       }
      ]
     },
     {
-     "label": "l62.out_proj",
-     "kernel": "gemm",
+     "label": "l61.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "core_attn_out"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.62.linear_attn.out_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.61.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
-       "i32": 6144
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l62.post_attn_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l61.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "act"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.61.mlp.down_proj.weight"
       },
       {
-       "buf": "model.layers.62.post_attention_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.62.input_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -68421,7 +56226,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -68429,23 +56234,32 @@
      ]
     },
     {
-     "label": "l62.gate_up",
-     "kernel": "gemm",
+     "label": "l62.in_proj",
+     "kernel": "gemm8_dual",
      "args": [
+      {
+       "buf": "qkvz"
+      },
+      {
+       "buf": "ba"
+      },
       {
        "buf": "x"
       },
       {
-       "buf": "model.layers.62.mlp.gate_up_proj.weight"
+       "buf": "model.layers.62.linear_attn.in_proj_qkvz.weight"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.62.linear_attn.in_proj_ba.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 16384
+      },
+      {
+       "i32": 96
       },
       {
        "i32": 5120
@@ -68453,71 +56267,79 @@
      ]
     },
     {
-     "label": "l62.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l62.gdn",
+     "kernel": "gdn_decode",
      "args": [
       {
-       "buf": "act"
+       "buf": "qkvz"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.62.linear_attn.conv1d.weight"
       },
       {
-       "i32": 17408
+       "state": "gdn"
       },
       {
-       "i32": 0
+       "state": "gdn",
+       "offset": 61440
       },
       {
-       "f32": 1.0
+       "buf": "gdn.line_index",
+       "offset": 192
       },
       {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l62.down_proj",
-     "kernel": "gemm",
-     "args": [
+       "buf": "ba"
+      },
       {
-       "buf": "act"
+       "buf": "model.layers.62.linear_attn.A_log"
       },
       {
-       "buf": "model.layers.62.mlp.down_proj.weight"
+       "buf": "model.layers.62.linear_attn.dt_bias"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "sym": "tokens"
+       "buf": "model.layers.62.linear_attn.norm.weight"
       },
       {
-       "i32": 5120
+       "f32": 0.0883883461356163
       },
       {
-       "i32": 17408
+       "f32": 9.999999974752427e-07
+      },
+      {
+       "i32": 49
+      },
+      {
+       "i32": 1605632
+      },
+      {
+       "i32": 10240
+      },
+      {
+       "i32": 802816
       }
      ]
     },
     {
-     "label": "l62.next_input_norm",
-     "kernel": "gemma_fused_norm",
+     "label": "l62.out_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
        "buf": "x"
       },
       {
-       "buf": "y"
+       "buf": "core_attn_out"
       },
       {
-       "buf": "residual"
+       "buf": "model.layers.62.linear_attn.out_proj.weight"
       },
       {
-       "buf": "model.layers.63.input_layernorm.weight_p1"
+       "buf": "residual"
       },
       {
-       "i32": 5120
+       "buf": "model.layers.62.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -68526,7 +56348,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 6144
       },
       {
        "f32": 9.999999974752427e-07
@@ -68534,23 +56356,23 @@
      ]
     },
     {
-     "label": "l63.qkv_proj",
-     "kernel": "gemm",
+     "label": "l62.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.63.self_attn.qkv_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "qkv"
+       "buf": "model.layers.62.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 14336
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -68558,45 +56380,32 @@
      ]
     },
     {
-     "label": "l63.q_norm",
-     "kernel": "gemma_norm_qhead",
+     "label": "l62.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "q_n"
-      },
-      {
-       "buf": "qkv"
-      },
-      {
-       "buf": "model.layers.63.self_attn.q_norm.weight_p1"
+       "buf": "x"
       },
       {
-       "i32": 256
+       "buf": "act"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         24
-        ]
-       }
+       "buf": "model.layers.62.mlp.down_proj.weight"
       },
       {
-       "i32": 24
+       "buf": "residual"
       },
       {
-       "i32": 14336
+       "buf": "model.layers.63.input_layernorm.weight_p1"
       },
       {
-       "i32": 512
+       "sym": "tokens"
       },
       {
-       "i32": 6144
+       "i32": 5120
       },
       {
-       "i32": 256
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
@@ -68604,55 +56413,32 @@
      ]
     },
     {
-     "label": "l63.k_norm",
-     "kernel": "gemma_norm_khead",
+     "label": "l63.qkv_proj",
+     "kernel": "gemm",
      "args": [
       {
-       "buf": "k_n"
-      },
-      {
-       "buf": "qkv",
-       "offset": 24576
-      },
-      {
-       "buf": "model.layers.63.self_attn.k_norm.weight_p1"
+       "buf": "x"
       },
       {
-       "i32": 256
+       "buf": "model.layers.63.self_attn.qkv_proj.weight"
       },
       {
-       "expr": {
-        "mul": [
-         {
-          "sym": "tokens"
-         },
-         4
-        ]
-       }
+       "buf": "qkv"
       },
       {
-       "i32": 4
+       "sym": "tokens"
       },
       {
        "i32": 14336
       },
       {
-       "i32": 256
-      },
-      {
-       "i32": 1024
-      },
-      {
-       "i32": 256
-      },
-      {
-       "f32": 9.999999974752427e-07
+       "i32": 5120
       }
      ]
     },
     {
-     "label": "l63.rope",
-     "kernel": "mrope",
+     "label": "l63.prep",
+     "kernel": "attn_prep",
      "args": [
       {
        "buf": "q_n"
@@ -68661,32 +56447,19 @@
        "buf": "k_n"
       },
       {
-       "buf": "cos_g"
-      },
-      {
-       "buf": "sin_g"
+       "buf": "qkv"
       },
       {
-       "i32": 0
+       "buf": "model.layers.63.self_attn.q_norm.weight_p1"
       },
       {
-       "i64": 0
+       "buf": "model.layers.63.self_attn.k_norm.weight_p1"
       },
       {
-       "i64": 0
-      }
-     ]
-    },
-    {
-     "label": "l63.kv_write",
-     "kernel": "reshape_and_cache",
-     "args": [
-      {
-       "buf": "k_n"
+       "buf": "cos_g"
       },
       {
-       "buf": "qkv",
-       "offset": 26624
+       "buf": "sin_g"
       },
       {
        "state": "kv",
@@ -68700,38 +56473,39 @@
        "buf": "slot_mapping"
       },
       {
-       "buf": "kv_scales"
-      },
-      {
-       "buf": "kv_scales",
-       "offset": 4
-      },
-      {
-       "i64": 1024
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         24
+        ]
+       }
       },
       {
-       "i64": 14336
+       "expr": {
+        "mul": [
+         {
+          "sym": "tokens"
+         },
+         4
+        ]
+       }
       },
       {
        "i64": 25690112
       },
       {
-       "i64": 512
-      },
-      {
-       "i64": 0
-      },
-      {
-       "i64": 0
+       "i64": 2048
       },
       {
-       "i64": 2048
+       "i64": 512
       },
       {
-       "i64": 0
+       "i32": 784
       },
       {
-       "i64": 0
+       "f32": 9.999999974752427e-07
       }
      ]
     },
@@ -68829,11 +56603,11 @@
      ]
     },
     {
-     "label": "l63.gate",
-     "kernel": "sigmoid_mul",
+     "label": "l63.o_norm",
+     "kernel": "gemm8_sgate_add_norm",
      "args": [
       {
-       "buf": "gated"
+       "buf": "x"
       },
       {
        "buf": "attn_out"
@@ -68843,31 +56617,13 @@
        "offset": 512
       },
       {
-       "i32": 24
-      },
-      {
-       "i32": 256
-      },
-      {
-       "i32": 14336
-      },
-      {
-       "i32": 512
-      }
-     ]
-    },
-    {
-     "label": "l63.o_proj",
-     "kernel": "gemm",
-     "args": [
-      {
-       "buf": "gated"
+       "buf": "model.layers.63.self_attn.o_proj.weight"
       },
       {
-       "buf": "model.layers.63.self_attn.o_proj.weight"
+       "buf": "residual"
       },
       {
-       "buf": "y"
+       "buf": "model.layers.63.post_attention_layernorm.weight_p1"
       },
       {
        "sym": "tokens"
@@ -68877,36 +56633,15 @@
       },
       {
        "i32": 6144
-      }
-     ]
-    },
-    {
-     "label": "l63.post_attn_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
-      {
-       "buf": "residual"
-      },
-      {
-       "buf": "model.layers.63.post_attention_layernorm.weight_p1"
-      },
-      {
-       "i32": 5120
       },
       {
-       "sym": "tokens"
+       "i32": 256
       },
       {
-       "i32": 5120
+       "i32": 14336
       },
       {
-       "i32": 5120
+       "i32": 512
       },
       {
        "f32": 9.999999974752427e-07
@@ -68914,23 +56649,23 @@
      ]
     },
     {
-     "label": "l63.gate_up",
-     "kernel": "gemm",
+     "label": "l63.gate_up_silu",
+     "kernel": "gemm8_gateup_silu",
      "args": [
       {
-       "buf": "x"
+       "buf": "act"
       },
       {
-       "buf": "model.layers.63.mlp.gate_up_proj.weight"
+       "buf": "x"
       },
       {
-       "buf": "gate_up"
+       "buf": "model.layers.63.mlp.gate_up_proj.weight"
       },
       {
        "sym": "tokens"
       },
       {
-       "i32": 34816
+       "i32": 17408
       },
       {
        "i32": 5120
@@ -68938,72 +56673,24 @@
      ]
     },
     {
-     "label": "l63.silu_mul",
-     "kernel": "silu_mul",
+     "label": "l63.down_norm",
+     "kernel": "gemm8_add_norm",
      "args": [
       {
-       "buf": "act"
-      },
-      {
-       "buf": "gate_up"
-      },
-      {
-       "i32": 17408
-      },
-      {
-       "i32": 0
-      },
-      {
-       "f32": 1.0
+       "buf": "x"
       },
-      {
-       "i32": 0
-      }
-     ]
-    },
-    {
-     "label": "l63.down_proj",
-     "kernel": "gemm",
-     "args": [
       {
        "buf": "act"
       },
       {
        "buf": "model.layers.63.mlp.down_proj.weight"
       },
-      {
-       "buf": "y"
-      },
-      {
-       "sym": "tokens"
-      },
-      {
-       "i32": 5120
-      },
-      {
-       "i32": 17408
-      }
-     ]
-    },
-    {
-     "label": "l63.final_norm",
-     "kernel": "gemma_fused_norm",
-     "args": [
-      {
-       "buf": "x"
-      },
-      {
-       "buf": "y"
-      },
       {
        "buf": "residual"
       },
       {
        "buf": "model.norm.weight_p1"
       },
-      {
-       "i32": 5120
-      },
       {
        "sym": "tokens"
       },
@@ -69011,7 +56698,7 @@
        "i32": 5120
       },
       {
-       "i32": 5120
+       "i32": 17408
       },
       {
        "f32": 9.999999974752427e-07
diff --git a/schema/manifest-v2.schema.json b/schema/manifest-v2.schema.json
index 4a3b70f..4d14d0e 100644
--- a/schema/manifest-v2.schema.json
+++ b/schema/manifest-v2.schema.json
@@ -564,6 +564,10 @@
           },
           "type": "array"
         },
+        "pdl": {
+          "description": "Programmatic dependent launch: the runtime launches this step with\n`CU_LAUNCH_ATTRIBUTE_PROGRAMMATIC_STREAM_SERIALIZATION`, so it may\nstart while the preceding launch is still draining. Only for kernels\nthat put every read of an upstream product and every global write\nbehind `griddepcontrol.wait` (their own inputs — weights, state —\nmay stream ahead of it).",
+          "type": "boolean"
+        },
         "sha256": {
           "description": "sha256 of that cubin file, checked by the runtime when both are set.",
           "type": [
diff --git a/tools/gen_qwen35.py b/tools/gen_qwen35.py
index c6ee5eb..dc3d0a8 100644
--- a/tools/gen_qwen35.py
+++ b/tools/gen_qwen35.py
@@ -103,6 +103,9 @@
 NT_MAX = CHUNK_MAX // FLA_CHUNK          # 32 chunks
 # attention
 BLOCK_SIZE = 784                         # vLLM block_size (constexpr in the kernels)
+GEMM8_GRID = 152                         # one persistent CTA per GB300 SM
+GEMM8_THREADS = 288                      # 1 producer + 8 compute warps
+GEMM8_SMEM = 224128                      # sizeof(Smem) + alignment (gemm8.cu)
 BLOCK_Q = 5                              # unified 2D: query rows per block
 NUM_SEGMENTS = 16                        # unified 3D: grid.z
 BLOCK_TABLE_LEN = 8                      # vLLM block_table_stride
@@ -511,7 +514,7 @@ def scr(name, off=0):
     return {"scratch": name, "offset": off} if off else {"scratch": name}
 
 
-def step(symbol, params, block, grid, args, shared_mem=None, cubin=None, sha256=None):
+def step(symbol, params, block, grid, args, shared_mem=None, cubin=None, sha256=None, pdl=False):
     s = {"symbol": symbol, "params": params, "block": block, "grid": grid, "args": args}
     if shared_mem is not None:
         s["shared_mem"] = shared_mem
@@ -519,13 +522,15 @@ def step(symbol, params, block, grid, args, shared_mem=None, cubin=None, sha256=
         s["cubin"] = cubin
     if sha256 is not None:
         s["sha256"] = sha256
+    if pdl:  # the kernel puts its dependent reads / all writes behind griddepcontrol.wait
+        s["pdl"] = True
     return s
 
 
-def single(symbol, params, block, grid, shared_mem=None, cubin=None, sha256=None):
+def single(symbol, params, block, grid, shared_mem=None, cubin=None, sha256=None, pdl=False):
     return {"params": params,
             "impl": {"steps": [step(symbol, params, block, grid, [a(i) for i in range(len(params))],
-                                    shared_mem, cubin, sha256)]}}
+                                    shared_mem, cubin, sha256, pdl)]}}
 
 
 TOKEN_DOMAIN = {"index_into": "model.embed_tokens.weight"}
@@ -700,6 +705,15 @@ def tri(tag, params, grid, src=pre, **kw):
         return single(TRITON[tag], params, blk(tag, src), grid,
                       shared_mem=smem(tag, src) or None, cubin=cubin, sha256=sha, **kw)
 
+    # gcnt is declared `out`: the kernel does read it (atomic task counter),
+    # but its contents are zero at rest -- alloc_zeros at load, and the tail
+    # resets it before the dispatch ends
+    GEMM8_AN_PARAMS = ["out buffer<bf16>", "in buffer<bf16>", "in buffer<bf16>", "inout buffer<bf16>",
+                       "in buffer<f32>", "out buffer<bf16>", "out buffer<f32>", "out buffer<i32>",
+                       "i32", "i32", "i32", "f32"]
+    GEMM8_SG_PARAMS = ["out buffer<bf16>", "in buffer<bf16>", "in buffer<bf16>", "in buffer<bf16>",
+                       "inout buffer<bf16>", "in buffer<f32>", "out buffer<bf16>", "out buffer<f32>",
+                       "out buffer<i32>", "i32", "i32", "i32", "i32", "i32", "i32", "f32"]
     GEMMA_PARAMS = ["out buffer<bf16>", "in buffer<bf16>", "in buffer<f32>",
                     "i32", "i32", "i32", "i32", "i32", "i32", "i32", "f32"]
     GEMMA_FUSED_PARAMS = ["out buffer<bf16>", "in buffer<bf16>", "inout buffer<bf16>",
@@ -745,17 +759,65 @@ def layer_norm_kernel(src, grid, rows_shape):
         # reduction width depends on the row count, so `rows` is an arg).
         "gemma_norm": single("kern_gemma_rms_norm_bf16", GEMMA_PARAMS, [512, 1, 1], [T, 1, 1],
                              shared_mem=GEMMA_SMEM, cubin="gemma_rms_norm.cubin"),
-        "gemma_norm_qhead": single("kern_gemma_rms_norm_bf16", GEMMA_PARAMS, [512, 1, 1],
-                                   [mul(T, HEADS), 1, 1], shared_mem=GEMMA_HEAD_SMEM,
-                                   cubin="gemma_rms_norm.cubin"),
-        "gemma_norm_khead": single("kern_gemma_rms_norm_bf16", GEMMA_PARAMS, [512, 1, 1],
-                                   [mul(T, KV_HEADS), 1, 1], shared_mem=GEMMA_HEAD_SMEM,
-                                   cubin="gemma_rms_norm.cubin"),
+        # per-head q/k gemma norm + partial rope + kv cache write of one
+        # attention layer (tools/kernels-src/attn_prep.cu): CTA = (token, head)
+        "attn_prep": single("kern_attn_prep_bf16",
+                            ["out buffer<bf16>", "out buffer<bf16>", "in buffer<bf16>",
+                             "in buffer<f32>", "in buffer<f32>", "in buffer<bf16>",
+                             "in buffer<bf16>", "inout ptr", "inout ptr", "in buffer<i64>",
+                             "i32", "i32", "i64", "i64", "i64", "i32", "f32"],
+                            [512, 1, 1], [mul(T, HEADS + KV_HEADS), 1, 1],
+                            cubin="attn_prep.cubin"),
         "gemma_fused_norm": single("kern_gemma_fused_add_rms_norm_bf16", GEMMA_FUSED_PARAMS,
                                    [512, 1, 1], [T, 1, 1], shared_mem=GEMMA_SMEM,
                                    cubin="gemma_rms_norm.cubin"),
         "silu_mul": single(silu_sym, ["out buffer<bf16>", "in buffer<bf16>", "i32", "i32", "f32", "i32"],
                            blk("silu"), [T, 1, 1]),
+        # streaming M<=8 GEMM (tools/kernels-src/gemm8.cu): one persistent CTA
+        # per GB300 SM, weight rows via TMA ring + mma; grid/block/smem are
+        # compile-time constants of that kernel
+        "gemm8_gateup_silu": single("kern_gemm8_gateup_silu_bf16",
+                                    ["out buffer<bf16>", "in buffer<bf16>", "in buffer<bf16>",
+                                     "i32", "i32", "i32"],
+                                    [GEMM8_THREADS, 1, 1], [GEMM8_GRID, 1, 1],
+                                    shared_mem=GEMM8_SMEM, cubin="gemm8.cubin", pdl=True),
+        # down GEMM + residual add + Gemma fused_add_rms_norm of the next
+        # layer input (gemm8's split-K partials, y and the task counters are
+        # impl scratch; the norm tail reproduces kern_gemma_rms_norm.cu)
+        # two weights sharing one x: in_proj_qkvz + in_proj_ba per GDN layer
+        "gemm8_dual": single("kern_gemm8_dual_bf16",
+                             ["out buffer<bf16>", "out buffer<bf16>", "in buffer<bf16>",
+                              "in buffer<bf16>", "in buffer<bf16>", "i32", "i32", "i32", "i32"],
+                             [GEMM8_THREADS, 1, 1], [GEMM8_GRID, 1, 1],
+                             shared_mem=GEMM8_SMEM, cubin="gemm8.cubin", pdl=True),
+        # o_proj + the sigmoid gate on its input + residual add + post-attention
+        # norm: x = attn * sigmoid(gate) is built inside the GEMM
+        "gemm8_sgate_add_norm": {
+            "params": GEMM8_SG_PARAMS[:6] + GEMM8_SG_PARAMS[9:],
+            "impl": {
+                "scratch": {"y": {"dtype": "bf16", "shape": [8, HIDDEN]},
+                            "partial": {"dtype": "f32", "shape": [8, 8, HIDDEN]},
+                            "gcnt": {"dtype": "i32", "shape": [2]}},
+                "steps": [step("kern_gemm8_sgate_add_norm_bf16", GEMM8_SG_PARAMS,
+                               [GEMM8_THREADS, 1, 1], [GEMM8_GRID, 1, 1],
+                               [a(0), a(1), a(2), a(3), a(4), a(5), scr("y"), scr("partial"), scr("gcnt"),
+                                a(6), a(7), a(8), a(9), a(10), a(11), a(12)],
+                               shared_mem=GEMM8_SMEM, cubin="gemm8.cubin", pdl=True)],
+            },
+        },
+        "gemm8_add_norm": {
+            "params": GEMM8_AN_PARAMS[:5] + GEMM8_AN_PARAMS[8:],
+            "impl": {
+                "scratch": {"y": {"dtype": "bf16", "shape": [8, HIDDEN]},
+                            "partial": {"dtype": "f32", "shape": [8, 8, HIDDEN]},
+                            "gcnt": {"dtype": "i32", "shape": [2]}},
+                "steps": [step("kern_gemm8_add_norm_bf16", GEMM8_AN_PARAMS,
+                               [GEMM8_THREADS, 1, 1], [GEMM8_GRID, 1, 1],
+                               [a(0), a(1), a(2), a(3), a(4), scr("y"), scr("partial"), scr("gcnt"),
+                                a(5), a(6), a(7), a(8)],
+                               shared_mem=GEMM8_SMEM, cubin="gemm8.cubin", pdl=True)],
+            },
+        },
         "copy_rows": single("kern_copy_rows_bf16",
                             ["out buffer<bf16>", "in buffer<bf16>", "i32", "i32", "i32"],
                             [256, 1, 1], [T, 1, 1], cubin="copy_rows.cubin"),
@@ -825,26 +887,15 @@ def layer_norm_kernel(src, grid, rows_shape):
                        [GDN_D // 64, cdiv(T, FLA_CHUNK), GDN_V_HEADS]),
         "gated_norm": layer_norm_kernel(pre, [mul(T, GDN_V_HEADS // LN_ROWS_PER_BLOCK), 1, 1],
                                         ["tokens", GDN_V_HEADS]),
-        # --- GDN decode
-        "conv_update": tri("conv_update",
-                           ["in buffer<bf16>", "in buffer<bf16>", "inout ptr", "in buffer<i32>",
-                            "out buffer<bf16>", "i32", "i32", "i64", "i64"] + I2,
-                           [1, CONV_DIM // 256, 1], src=dec),
-        "recurrent": tri("recurrent",
-                         ["in buffer<bf16>", "in buffer<bf16>", "in buffer<bf16>", "in buffer<f32>",
-                          "in buffer<bf16>", "out buffer<bf16>", "inout ptr", "inout ptr",
-                          "in buffer<i32>", "f32"] + I2,
-                         [GDN_D // 32, GDN_V_HEADS, 1], src=dec),
-        "gated_norm_decode": layer_norm_kernel(dec, [GDN_V_HEADS, 1, 1], [1, GDN_V_HEADS]),
+        # --- GDN decode: the whole chain in one launch, one thread-block
+        # cluster per q/k head (tools/kernels-src/gdn_decode.cu)
+        "gdn_decode": single("kern_gdn_decode_bf16",
+                             ["inout buffer<bf16>", "in buffer<bf16>", "inout ptr", "inout ptr",
+                              "in buffer<i32>", "in buffer<bf16>", "in buffer<f32>",
+                              "in buffer<bf16>", "out buffer<bf16>", "in buffer<bf16>",
+                              "f32", "f32", "i32", "i32", "i32", "i32"],
+                             [256, 1, 1], [2 * GDN_V_HEADS, 1, 1], cubin="gdn_decode.cubin"),
         # --- attention
-        # one generic instance for both programs (decode's launch in vLLM is
-        # the num_tokens==1 specialization: same arithmetic, one arg fewer)
-        "mrope": tri("mrope", ["inout buffer<bf16>", "inout buffer<bf16>", "in buffer<bf16>",
-                               "in buffer<bf16>", "i32"] + I2, [T, 1, 1]),
-        "reshape_and_cache": tri("cache",
-                                 ["in buffer<bf16>", "in buffer<bf16>", "inout ptr", "inout ptr",
-                                  "in buffer<i64>", "in buffer<f32>", "in buffer<f32>"] + ["i64"] * 9,
-                                 [T, 1, 1]),
         "attn_prefill": tri("unified", ATTN_IFACE, [cdiv(T, BLOCK_Q), KV_HEADS, 1]),
         "attn": {
             "params": ATTN_IFACE,
@@ -875,6 +926,23 @@ def layer_norm_kernel(src, grid, rows_shape):
     Z2 = [i64(0), i64(0)]
 
     if spec:
+        # the whole spec-path GDN chain of a layer in one launch
+        # (tools/kernels-src/gdn_spec.cu): conv update + post_conv split/
+        # l2norm/gating + a/b/z copies + T-row delta rule with per-token
+        # SSM checkpoints + gated RMSNorm.  Counter scratch is zero at
+        # rest (the last CTA resets); hpart holds data, write-before-read.
+        GDN_SPEC_PARAMS = [
+            "inout buffer<bf16>",                                    # qkvz (conv writeback)
+            "in buffer<bf16>", "inout ptr", "inout ptr",             # conv weight, conv state, ssm state
+            "in buffer<i32>", "in buffer<i32>", "in buffer<i32>",    # line, num_accepted, cu_seqlens_q
+            "in buffer<i32>",                                        # spec_slots row
+            "in buffer<bf16>", "in buffer<f32>", "in buffer<bf16>",  # ba, A_log, dt_bias
+            "out buffer<bf16>", "in buffer<bf16>",                   # core_attn_out, norm.weight
+            "out buffer<bf16>", "out buffer<bf16>", "out buffer<bf16>",  # gdn_q/k/v
+            "out buffer<f32>", "out buffer<f32>",                    # g, beta
+            "out buffer<bf16>", "out buffer<bf16>", "out buffer<bf16>",  # a_c, b_c, z_c
+            "out buffer<i32>", "out buffer<f32>", "out buffer<i32>", "out buffer<i32>",  # scratch
+            "f32", "f32", "i32", "i32", "i32", "i32", "i32"]         # scale, eps, tokens, n_pages, cls, cds, sls
         RMS_PARAMS = ["out buffer<bf16>", "in buffer<bf16>", "i64", "i64", "i64", "i64", "i64",
                       "in buffer<bf16>", "i64", "f32", "i32", "i32"]
         ad = spec["attn_draft"]
@@ -882,24 +950,25 @@ def layer_norm_kernel(src, grid, rows_shape):
             "gemm_acc": single("extern:cublaslt_bf16_tn_acc",
                                ["in buffer<bf16>", "in buffer<bf16>", "inout buffer<bf16>", "i32", "i32", "i32"],
                                [1, 1, 1], [1, 1, 1]),
-            # --- target GDN under speculation (vLLM's spec-decode kernels)
-            # conv update: seqlen 8 / state_len 10 are constexprs; reads the
-            # history taps at conv-line offset num_accepted-1, writes the
-            # new 10-wide window (in place on the qkvz rows)
-            "conv_update_spec": spec_kernel("conv_update_spec",
-                                            ["in buffer<bf16>", "in buffer<bf16>", "inout ptr", "in buffer<i32>",
-                                             "in buffer<i32>", "in buffer<i32>", "out buffer<bf16>",
-                                             "i32", "i32", "i64", "i64"] + I2,
-                                            [1, CONV_DIM // 256, 1]),
+            # --- target GDN under speculation: the whole chain is fused
+            # into gdn_spec below (kern_gdn_spec_bf16)
+            "gdn_spec": {
+                "params": GDN_SPEC_PARAMS[:21] + GDN_SPEC_PARAMS[25:],
+                "impl": {
+                    "scratch": {"xcnt": {"dtype": "i32", "shape": [GDN_Q // GDN_D]},
+                                "hpart": {"dtype": "f32", "shape": [SPEC_BLOCK, GDN_V_HEADS, 4]},
+                                "hcnt": {"dtype": "i32", "shape": [GDN_V_HEADS]},
+                                "gcnt": {"dtype": "i32", "shape": [1]}},
+                    "steps": [step("kern_gdn_spec_bf16", GDN_SPEC_PARAMS, [256, 1, 1], [192, 1, 1],
+                                   [a(i) for i in range(21)]
+                                   + [scr("xcnt"), scr("hpart"), scr("hcnt"), scr("gcnt")]
+                                   + [a(i) for i in range(21, 28)],
+                                   cubin="gdn_spec.cubin")],
+                },
+            },
             # recurrent delta rule over T rows with the sigmoid gating fused:
             # initial state from SSM slot num_accepted-1, every row's state
             # checkpointed to its own slot (ssm_state_indices)
-            "recurrent_spec": spec_kernel("recurrent_spec",
-                                          ["in buffer<f32>", "in buffer<bf16>", "in buffer<bf16>", "in buffer<bf16>",
-                                           "f32", "f32", "in buffer<bf16>", "in buffer<bf16>", "in buffer<bf16>",
-                                           "out buffer<bf16>", "inout ptr", "inout ptr", "in buffer<i32>",
-                                           "in buffer<i32>", "in buffer<i32>", "f32", "i64", "i64"] + I2,
-                                          [1, GDN_D // 32, GDN_V_HEADS]),
             "argmax": {
                 "params": ["in buffer<bf16>", "out buffer<i64>", "i32"],
                 "impl": {
@@ -962,7 +1031,7 @@ def fused(label, x_in, w):
                  [buf("x"), buf(x_in), buf("residual"), buf(w), i32(HIDDEN), T,
                   i32(HIDDEN), i32(HIDDEN), f32(eps)])
 
-    def gdn_layer(i, decode, nacc=None):
+    def gdn_layer(i, decode, nacc=None, small=False):
         """decode=False: the chunked FLA prefill chain.  decode=True: the
         recurrent chain — Stage 1's packed kernels, or, when `nacc` names
         the num_accepted_tokens buffer, vLLM's speculative kernels over the
@@ -973,10 +1042,16 @@ def gdn_layer(i, decode, nacc=None):
         line = g + 1
         idx = buf(line_table, 4 * line)
         page = gdn_page(i)
-        ds = [
-            gemm(l + "in_proj_qkvz", "x", p + "in_proj_qkvz.weight", "qkvz", T, QKVZ_DIM, HIDDEN),
-            gemm(l + "in_proj_ba", "x", p + "in_proj_ba.weight", "ba", T, BA_DIM, HIDDEN),
-        ]
+        if small:
+            # one weight stream for both input projections
+            ds = [d(l + "in_proj", "gemm8_dual",
+                    [buf("qkvz"), buf("ba"), buf("x"), buf(p + "in_proj_qkvz.weight"),
+                     buf(p + "in_proj_ba.weight"), T, i32(QKVZ_DIM), i32(BA_DIM), i32(HIDDEN)])]
+        else:
+            ds = [
+                gemm(l + "in_proj_qkvz", "x", p + "in_proj_qkvz.weight", "qkvz", T, QKVZ_DIM, HIDDEN),
+                gemm(l + "in_proj_ba", "x", p + "in_proj_ba.weight", "ba", T, BA_DIM, HIDDEN),
+            ]
         a_ = buf("ba", GDN_V_HEADS * BF16)   # ba row = [b | a]
         b_ = buf("ba")
         if not decode and nacc is None:
@@ -1017,70 +1092,52 @@ def gdn_layer(i, decode, nacc=None):
                    i32(GDN_D), i32(GDN_D), i32(GDN_D), expr(mul(T, GDN_V_HEADS)), f32(eps)] + Z2),
             ]
         elif nacc is None:
+            # the decode chain (conv update + delta rule step + gated norm),
+            # fused into one launch -- see tools/kernels-src/gdn_decode.cu
             ds += [
-                d(l + "conv_update", "conv_update",
-                  [buf("qkvz"), buf(p + "conv1d.weight"), state("gdn"), idx, buf("qkvz"),
-                   i32(1), i32(n_pages), i64(1), i64(1)] + Z2),
-                d(l + "recurrent", "recurrent",
-                  [buf("qkvz"), a_, b_, buf(p + "A_log"), buf(p + "dt_bias"), buf("core_attn_out"),
-                   state("gdn", ssm_off), state("gdn", ssm_off), idx,
-                   f32(gdn_scale)] + Z2),
-                d(l + "gated_norm", "gated_norm_decode",
-                  [buf("core_attn_out"), buf("core_attn_out"), buf(p + "norm.weight"),
-                   buf("qkvz", Z_OFF * BF16), i32(GDN_D), i32(GDN_D), i32(GDN_D), i32(GDN_V_HEADS),
-                   f32(eps)] + Z2),
+                d(l + "gdn", "gdn_decode",
+                  [buf("qkvz"), buf(p + "conv1d.weight"), state("gdn"), state("gdn", ssm_off),
+                   idx, buf("ba"), buf(p + "A_log"), buf(p + "dt_bias"), buf("core_attn_out"),
+                   buf(p + "norm.weight"), f32(gdn_scale), f32(eps), i32(n_pages),
+                   i32(page_bytes // BF16), i32(CONV_DIM), i32(page_bytes // 4)]),
             ]
         else:
-            # vLLM's spec-decode GDN path: conv update with the accepted-token
-            # offset (seqlen 8 baked: it reads/writes 8 qkvz rows, rows past
-            # `tokens` are scratch), post_conv split, contiguous a/b, the
-            # recurrent kernel over T rows checkpointing every row's state.
+            # vLLM's spec-decode GDN path, fused into one launch (the chain
+            # was: conv update with the accepted-token offset, post_conv
+            # split + l2norm + gating, contiguous a/b copies, the recurrent
+            # kernel over T rows checkpointing every row's state, z copy,
+            # gated RMSNorm).  The fused kernel reproduces every buffer
+            # write of that chain, including the qkvz conv writeback and
+            # the per-token SSM checkpoints.
             ds += [
-                d(l + "conv_update", "conv_update_spec",
-                  [buf("qkvz"), buf(p + "conv1d.weight"), state("gdn"), idx, nacc, buf("cu_seqlens_q"),
-                   buf("qkvz"), i32(1), i32(n_pages), i64(QKVZ_DIM), i64(QKVZ_DIM)] + Z2),
-                d(l + "post_conv", "post_conv",
-                  [buf("qkvz"), a_, b_, buf(p + "A_log"), buf(p + "dt_bias"),
-                   buf("gdn_q"), buf("gdn_k"), buf("gdn_v"), buf("g"), buf("beta"),
-                   i32(QKVZ_DIM), i32(BA_DIM), i32(BA_DIM), i32(GDN_Q), i32(GDN_Q), i32(GDN_V), T] + Z2),
-                d(l + "a_copy", "copy_rows", [buf("a_c"), a_, i32(GDN_V_HEADS), i32(BA_DIM), i32(GDN_V_HEADS)]),
-                d(l + "b_copy", "copy_rows", [buf("b_c"), b_, i32(GDN_V_HEADS), i32(BA_DIM), i32(GDN_V_HEADS)]),
-                d(l + "recurrent", "recurrent_spec",
-                  [buf(p + "A_log"), buf("a_c"), buf("b_c"), buf(p + "dt_bias"), f32(1.0), f32(20.0),
-                   buf("gdn_q"), buf("gdn_k"), buf("gdn_v"), buf("core_attn_out"),
-                   state("gdn", ssm_off), state("gdn", ssm_off), buf("cu_seqlens_q"),
-                   buf("gdn.spec_slots", 4 * SPEC_BLOCK * g), nacc, f32(gdn_scale), i64(1), T] + Z2),
-                d(l + "z_copy", "copy_rows",
-                  [buf("z_c"), buf("qkvz", Z_OFF * BF16), i32(GDN_V), i32(QKVZ_DIM), i32(GDN_V)]),
-                d(l + "gated_norm", "gated_norm",
-                  [buf("core_attn_out"), buf("core_attn_out"), buf(p + "norm.weight"), buf("z_c"),
-                   i32(GDN_D), i32(GDN_D), i32(GDN_D), expr(mul(T, GDN_V_HEADS)), f32(eps)] + Z2),
+                d(l + "gdn", "gdn_spec",
+                  [buf("qkvz"), buf(p + "conv1d.weight"), state("gdn"), state("gdn", ssm_off),
+                   idx, nacc, buf("cu_seqlens_q"), buf("gdn.spec_slots", 4 * SPEC_BLOCK * g),
+                   buf("ba"), buf(p + "A_log"), buf(p + "dt_bias"), buf("core_attn_out"),
+                   buf(p + "norm.weight"), buf("gdn_q"), buf("gdn_k"), buf("gdn_v"),
+                   buf("g"), buf("beta"), buf("a_c"), buf("b_c"), buf("z_c"),
+                   f32(gdn_scale), f32(eps), T, i32(n_pages),
+                   i32(page_bytes // BF16), i32(ssm_off // (CONV_DIM * BF16)), i32(page_bytes // 4)]),
             ]
         ds.append(gemm(l + "out_proj", "core_attn_out", p + "out_proj.weight", "y", T, HIDDEN, GDN_V))
         return ds
 
-    def attn_layer(i, decode):
+    def attn_layer(i, decode, small=False):
         p = f"model.layers.{i}.self_attn."
         l = f"l{i}."
         koff = ATTN_LAYERS.index(i) * LAYER_KV_BYTES
         ks, vs = buf("kv_scales"), buf("kv_scales", 4)
         kv_k, kv_v = state("kv", koff), state("kv", koff + V_BYTE_OFF)
-        return [
+        ds = [
             gemm(l + "qkv_proj", "x", p + "qkv_proj.weight", "qkv", T, QKV_DIM, HIDDEN),
-            d(l + "q_norm", "gemma_norm_qhead",
-              [buf("q_n"), buf("qkv"), buf(p + "q_norm.weight_p1"), i32(HEAD_DIM),
-               expr(mul(T, HEADS)), i32(HEADS), i32(QKV_DIM), i32(2 * HEAD_DIM), i32(Q_DIM),
-               i32(HEAD_DIM), f32(eps)]),
-            d(l + "k_norm", "gemma_norm_khead",
-              [buf("k_n"), buf("qkv", HEADS * 2 * HEAD_DIM * BF16), buf(p + "k_norm.weight_p1"),
-               i32(HEAD_DIM), expr(mul(T, KV_HEADS)), i32(KV_HEADS), i32(QKV_DIM), i32(HEAD_DIM),
-               i32(KV_DIM), i32(HEAD_DIM), f32(eps)]),
-            d(l + "rope", "mrope", [buf("q_n"), buf("k_n"), buf("cos_g"), buf("sin_g"), i32(0)] + Z2),
-            d(l + "kv_write", "reshape_and_cache",
-              [buf("k_n"), buf("qkv", (HEADS * 2 * HEAD_DIM + KV_DIM) * BF16), kv_k, kv_v,
-               buf("slot_mapping"), ks, vs,
-               i64(KV_DIM), i64(QKV_DIM), i64(BLOCK_STRIDE), i64(2 * HEAD_DIM), i64(0), i64(0),
-               i64(KV_HEADS * 2 * HEAD_DIM), i64(0), i64(0)]),
+            # q/k gemma norm + partial rope + kv cache write, one CTA per
+            # (token, head) -- see tools/kernels-src/attn_prep.cu
+            d(l + "prep", "attn_prep",
+              [buf("q_n"), buf("k_n"), buf("qkv"), buf(p + "q_norm.weight_p1"),
+               buf(p + "k_norm.weight_p1"), buf("cos_g"), buf("sin_g"), kv_k, kv_v,
+               buf("slot_mapping"), expr(mul(T, HEADS)), expr(mul(T, KV_HEADS)),
+               i64(BLOCK_STRIDE), i64(KV_HEADS * 2 * HEAD_DIM), i64(2 * HEAD_DIM),
+               i32(BLOCK_SIZE), f32(eps)]),
             d(l + "attn", "attn" if decode else "attn_prefill",
               [buf("attn_out"), buf("q_n"), kv_k, kv_v, buf("block_table"), buf("seq_lens"),
                f32(attn_scale), ks, vs, f32(1.0), f32(0.0),
@@ -1094,6 +1151,14 @@ def attn_layer(i, decode):
                i32(QKV_DIM), i32(2 * HEAD_DIM)]),
             gemm(l + "o_proj", "gated", p + "o_proj.weight", "y", T, HIDDEN, Q_DIM),
         ]
+        if small:
+            # sigmoid gate + o_proj + residual add + post-attention norm
+            ds[-2:] = [d(l + "o_norm", "gemm8_sgate_add_norm",
+                         [buf("x"), buf("attn_out"), buf("qkv", GATE_OFF * BF16), buf(p + "o_proj.weight"),
+                          buf("residual"), buf(f"model.layers.{i}.post_attention_layernorm.weight_p1"),
+                          T, i32(HIDDEN), i32(Q_DIM), i32(HEAD_DIM), i32(QKV_DIM), i32(2 * HEAD_DIM),
+                          f32(eps)])]
+        return ds
 
     def forward(decode, taps=False, nacc=None, tail=None):
         """Target forward.  decode: recurrent GDN + split-KV attention (else
@@ -1103,6 +1168,9 @@ def forward(decode, taps=False, nacc=None, tail=None):
         next_token), "decode" (row 0 -> next_token), "verify" (all rows ->
         verify_tokens)."""
         tail = tail or ("decode" if decode else "prefill")
+        # bs<=8 forwards (decode / decode_spec / verify) take the streaming
+        # gemm8 kernels; chunked prefill keeps cuBLAS
+        small = decode or tail == "verify"
         ds = [
             d("embed", "embedding",
               [buf("token_ids"), buf("model.embed_tokens.weight"), buf("residual"), T, i32(HIDDEN)]),
@@ -1117,17 +1185,37 @@ def forward(decode, taps=False, nacc=None, tail=None):
         for i in range(LAYERS):
             p = f"model.layers.{i}."
             l = f"l{i}."
-            ds += attn_layer(i, decode) if i in ATTN_LAYERS else gdn_layer(i, decode, nacc)
-            ds += [
-                fused(l + "post_attn_norm", "y", p + "post_attention_layernorm.weight_p1"),
-                gemm(l + "gate_up", "x", p + "mlp.gate_up_proj.weight", "gate_up", T, 2 * FFN, HIDDEN),
-                d(l + "silu_mul", "silu_mul",
-                  [buf("act"), buf("gate_up"), i32(FFN), i32(0), f32(1.0), i32(0)]),
-                gemm(l + "down_proj", "act", p + "mlp.down_proj.weight", "y", T, HIDDEN, FFN),
-            ]
+            lay = attn_layer(i, decode, small) if i in ATTN_LAYERS else gdn_layer(i, decode, nacc, small)
+            if small and i not in ATTN_LAYERS:
+                # out_proj GEMM + residual add + post-attention norm (the
+                # attention layers fold their sigmoid gate in too: attn_layer)
+                lay[-1] = d(l + "out_norm", "gemm8_add_norm",
+                            [buf("x"), buf("core_attn_out"), buf(p + "linear_attn.out_proj.weight"),
+                             buf("residual"), buf(p + "post_attention_layernorm.weight_p1"),
+                             T, i32(HIDDEN), i32(GDN_V), f32(eps)])
+            ds += lay
+            if small:
+                mlp = [d(l + "gate_up_silu", "gemm8_gateup_silu",
+                         [buf("act"), buf("x"), buf(p + "mlp.gate_up_proj.weight"),
+                          T, i32(FFN), i32(HIDDEN)])]
+            else:
+                mlp = [gemm(l + "gate_up", "x", p + "mlp.gate_up_proj.weight", "gate_up", T, 2 * FFN, HIDDEN),
+                       d(l + "silu_mul", "silu_mul",
+                         [buf("act"), buf("gate_up"), i32(FFN), i32(0), f32(1.0), i32(0)])]
             last = i + 1 == LAYERS
-            ds.append(fused(l + ("final_norm" if last else "next_input_norm"), "y",
-                            "model.norm.weight_p1" if last else f"model.layers.{i + 1}.input_layernorm.weight_p1"))
+            wnorm = "model.norm.weight_p1" if last else f"model.layers.{i + 1}.input_layernorm.weight_p1"
+            nlabel = l + ("final_norm" if last else "next_input_norm")
+            if not small:
+                ds.append(fused(l + "post_attn_norm", "y", p + "post_attention_layernorm.weight_p1"))
+            ds += mlp
+            if small:
+                # down GEMM + residual add + next input norm in one dispatch
+                ds.append(d(l + "down_norm", "gemm8_add_norm",
+                            [buf("x"), buf("act"), buf(p + "mlp.down_proj.weight"), buf("residual"),
+                             buf(wnorm), T, i32(HIDDEN), i32(FFN), f32(eps)]))
+            else:
+                ds.append(gemm(l + "down_proj", "act", p + "mlp.down_proj.weight", "y", T, HIDDEN, FFN))
+                ds.append(fused(nlabel, "y", wnorm))
             if taps and i in TAPS:
                 # residual now holds hidden + residual = the input of layer
                 # i+1, vLLM's aux hidden state; fc's column block j, β=1
@@ -1215,8 +1303,8 @@ def draft():
                 gemm(l + "mlp_conv_proj", "y", p + "mlp_conv.kernel_projection.weight", "d_coef",
                      T, D_CONV_PROJ, HIDDEN),
                 d_conv(l + "mlp_conv_pre", "y", p + "mlp_conv", 0),
-                gemm(l + "gate_up", "y", p + "mlp.gate_up_proj.weight", "gate_up", T, 2 * FFN, HIDDEN),
-                d(l + "silu_mul", "silu_mul", [buf("act"), buf("gate_up"), i32(FFN), i32(0), f32(1.0), i32(0)]),
+                d(l + "gate_up_silu", "gemm8_gateup_silu",
+                  [buf("act"), buf("y"), buf(p + "mlp.gate_up_proj.weight"), T, i32(FFN), i32(HIDDEN)]),
                 gemm(l + "down_proj", "act", p + "mlp.down_proj.weight", "x", T, HIDDEN, FFN),
                 d_conv(l + "mlp_conv_post", "x", p + "mlp_conv", 1),
                 d_fused(l + ("final_norm" if last else "next_input_norm"), "x",
diff --git a/tools/kernels-src/attn_prep.cu b/tools/kernels-src/attn_prep.cu
new file mode 100644
index 0000000..eba81a2
--- /dev/null
+++ b/tools/kernels-src/attn_prep.cu
@@ -0,0 +1,149 @@
+// The attention prep chain of one layer in one launch:
+//
+//   per-head Gemma RMSNorm of q and k   kern_gemma_rms_norm_bf16 x2
+//   partial rotary (dim 64 of 256)      _triton_mrope_forward
+//   KV cache write                      reshape_and_cache_kernel_flash
+//
+// One CTA per (token, head): 24 q heads norm + rope into q_n, 4 k heads
+// norm + rope into k_n and copy the k/v pair into the paged KV cache.
+// No cross-CTA traffic at all -- the fusion removes three graph nodes
+// per layer and the q_n/k_n round trips between norm, rope and the
+// cache write.
+//
+// Rounding is bit-exact against the chain: the norm reproduces ATen's
+// reduction order (same code as gemma_rms_norm.cu, W from the original
+// per-kernel `rows`), the rope operates on the bf16-rounded norm output
+// with the Triton kernel's instruction shapes (product of the far
+// element rounded to bf16, then a single-rounding bf16 fma with the
+// near element -- HMUL2/HFMA2 in the pinned cubin), and the cache write
+// is a plain bf16 copy (FP8 off, scales unused).
+//
+//   nvcc -cubin -arch=sm_103a -o kernels-qwen38/attn_prep.cubin tools/kernels-src/attn_prep.cu
+#include <cuda_bf16.h>
+#include <cstdint>
+
+namespace {
+constexpr int NT = 512;
+constexpr int HEADS = 24, KV_HEADS = 4, HD = 256, ROT = 32;  // rotary_dim 64
+constexpr int QKV_ROW = 14336;               // q|gate 24*512 | k 4*256 | v 4*256
+constexpr int K_OFF = HEADS * 2 * HD;        // 12288
+constexpr int V_OFF = K_OFF + KV_HEADS * HD; // 13312
+constexpr int GROUPS = HEADS + KV_HEADS;     // CTAs per token
+
+__device__ inline int last_pow2(int n) {
+  n |= (n >> 1); n |= (n >> 2); n |= (n >> 4); n |= (n >> 8); n |= (n >> 16);
+  return (n - (n >> 1)) > 0 ? (n - (n >> 1)) : 1;
+}
+// ATen ReduceConfig::set_block_dimension(dim0 = N/vec, dim1 = rows), mnt=512.
+__device__ inline int aten_block_width(int dim0, int dim1) {
+  int dim0_pow2 = dim0 < NT ? last_pow2(dim0) : NT;
+  int dim1_pow2 = dim1 < NT ? last_pow2(dim1) : NT;
+  int block_width = min(dim0_pow2, 32);
+  int block_height = min(dim1_pow2, NT / block_width);
+  block_width = min(dim0_pow2, NT / block_height);
+  return block_width;
+}
+// Sum of sq[0..N) in ATen's order for W lanes (see gemma_rms_norm.cu).
+__device__ float aten_row_sum(const float* __restrict__ sq, int N, int W,
+                              float* __restrict__ s) {
+  const int t = threadIdx.x;
+  float value = 0.f;
+  if (t < W) {
+    float acc[4] = {0.f, 0.f, 0.f, 0.f};
+    for (int idx = t; idx * 4 + 3 < N; idx += W) {
+      const float4 v = *reinterpret_cast<const float4*>(sq + idx * 4);
+      acc[0] += v.x; acc[1] += v.y; acc[2] += v.z; acc[3] += v.w;
+    }
+    value = ((acc[0] + acc[1]) + acc[2]) + acc[3];
+  }
+  if (W > 32) {
+    s[t] = value;
+    for (int off = W / 2; off >= 32; off >>= 1) {
+      __syncthreads();
+      if (t < off && t + off < W) { value += s[t + off]; s[t] = value; }
+    }
+  }
+  __syncthreads();
+  if (t < 32) {
+    for (int off = 16; off > 0; off >>= 1)
+      value += __shfl_down_sync(0xffffffffu, value, off);
+    if (t == 0) s[0] = value;
+  }
+  __syncthreads();
+  return s[0];
+}
+}  // namespace
+
+extern "C" __global__ void __launch_bounds__(NT) kern_attn_prep_bf16(
+    __nv_bfloat16* __restrict__ q_n,          // [tokens, 6144] out (post rope)
+    __nv_bfloat16* __restrict__ k_n,          // [tokens, 1024] out (post rope)
+    const __nv_bfloat16* __restrict__ qkv,    // [tokens, 14336]
+    const float* __restrict__ qw1,            // q_norm weight + 1
+    const float* __restrict__ kw1,            // k_norm weight + 1
+    const __nv_bfloat16* __restrict__ cos_g,  // [tokens, 32]
+    const __nv_bfloat16* __restrict__ sin_g,  // [tokens, 32]
+    __nv_bfloat16* __restrict__ kv_k,         // paged cache, k plane of this layer
+    __nv_bfloat16* __restrict__ kv_v,         // paged cache, v plane (k + 256)
+    const int64_t* __restrict__ slot_mapping, // [tokens]
+    int rows_q, int rows_k,                   // T*24 / T*4 (ATen width inputs)
+    int64_t block_stride, int64_t page_stride, int64_t head_stride,
+    int block_size, float eps) {
+  __shared__ float z[HD], s[NT], sq[HD];
+  __shared__ __nv_bfloat16 nb[HD];
+  const int tok = blockIdx.x / GROUPS, h = blockIdx.x % GROUPS;
+  const bool isq = h < HEADS;
+  const int kh = h - HEADS;
+  const __nv_bfloat16* in =
+      qkv + (int64_t)tok * QKV_ROW + (isq ? h * 2 * HD : K_OFF + kh * HD);
+  const float* w1 = isq ? qw1 : kw1;
+
+  // ---- Gemma RMSNorm, ATen reduction order, rounded to bf16 ----
+  for (int j = threadIdx.x; j < HD; j += NT) {
+    const float v = __bfloat162float(in[j]);
+    z[j] = v;
+    sq[j] = __fmul_rn(v, v);
+  }
+  __syncthreads();
+  const int rows = isq ? rows_q : rows_k;
+  const int W = aten_block_width(HD / 4, rows);
+  const float sum = aten_row_sum(sq, HD, W, s);
+  const float factor = (float)rows / (float)((long long)rows * (long long)HD);
+  const float var = __fmul_rn(sum, factor);
+  const float r = rsqrtf(__fadd_rn(var, eps));
+  for (int j = threadIdx.x; j < HD; j += NT)
+    nb[j] = __float2bfloat16_rn(__fmul_rn(__fmul_rn(z[j], r), w1[j]));
+  __syncthreads();
+
+  // ---- partial neox rotary on dims [0, 64): pair (d, d+32) ----
+  // Triton emits (per SASS): y1 = bf16fma(x1, cos, -bf16(x2 * sin))
+  //                           y2 = bf16fma(x1, sin,  bf16(x2 * cos))
+  // -- the x1 product is HFMA2-fused in both outputs, the x2 product
+  // pre-rounded.
+  if (threadIdx.x < ROT) {
+    const int j = threadIdx.x;
+    const __nv_bfloat16 c = cos_g[(int64_t)tok * ROT + j];
+    const __nv_bfloat16 sn = sin_g[(int64_t)tok * ROT + j];
+    const __nv_bfloat16 x1 = nb[j], x2 = nb[j + ROT];
+    nb[j] = __hfma(x1, c, __hneg(__hmul(x2, sn)));
+    nb[j + ROT] = __hfma(x1, sn, __hmul(x2, c));
+  }
+  __syncthreads();
+
+  // ---- stores: q_n / k_n, and the paged KV cache for k heads ----
+  if (isq) {
+    for (int j = threadIdx.x; j < HD; j += NT)
+      q_n[(int64_t)tok * (HEADS * HD) + h * HD + j] = nb[j];
+    return;
+  }
+  for (int j = threadIdx.x; j < HD; j += NT)
+    k_n[(int64_t)tok * (KV_HEADS * HD) + kh * HD + j] = nb[j];
+  const int64_t slot = slot_mapping[tok];
+  if (slot < 0) return;  // padding token
+  const int64_t base = (slot / block_size) * block_stride +
+                       (slot % block_size) * page_stride + kh * head_stride;
+  const __nv_bfloat16* v_src = qkv + (int64_t)tok * QKV_ROW + V_OFF + kh * HD;
+  for (int j = threadIdx.x; j < HD; j += NT) {
+    kv_k[base + j] = nb[j];
+    kv_v[base + j] = v_src[j];
+  }
+}
diff --git a/tools/kernels-src/gdn_decode.cu b/tools/kernels-src/gdn_decode.cu
new file mode 100644
index 0000000..ad80034
--- /dev/null
+++ b/tools/kernels-src/gdn_decode.cu
@@ -0,0 +1,238 @@
+// The whole GDN decode chain of one layer in one launch (tokens == 1):
+//
+//   causal_conv1d_update  (width 4, silu)       _causal_conv1d_update_kernel
+//   gated delta rule step (l2norm q/k)          fused_recurrent_gated_delta_rule_packed_decode_kernel
+//   gated RMSNorm         (norm, then *silu(z)) layer_norm_fwd_kernel
+//
+// The real work is one 48-head recurrent state update (48 x 128 x 128 f32,
+// 3 MB each way); everything else is bandwidth-free.  One thread-block
+// cluster per q/k head: GDN_SPLIT CTAs per v head, 3 v heads, so the q/k
+// row is convolved and l2-normalized once (cluster rank 0) and picked up
+// by the siblings over distributed shared memory after one cluster.sync()
+// -- no global scratch, no flags, no spinning, nothing to reset.  Every
+// CTA owns its v channels and state rows exclusively: it convolves them,
+// shifts their conv state, runs its rows of the delta rule, and applies
+// the gated norm (GDN_SPLIT == 2 exchanges the two variance partials over
+// the same cluster).  The sibling CTAs' state loads are issued before the
+// sync, so rank 0's extra conv work hides under them.
+//
+// Rounding chains reproduce the Triton kernels bit-for-bit where the
+// order is defined: bf16 x*w products rounded before the f32 conv
+// accumulate; division is rcp.approx + mul and sqrt is sqrt.approx, the
+// instructions the pinned cubins actually contain (MUFU.RCP / MUFU.SQRT
+// -- Triton never emits IEEE division); silu -> bf16 store; the recurrent
+// re-loads that bf16 value; beta goes through bf16; the norm input is the
+// bf16-rounded recurrent output; rstd is rsqrtf (MUFU.RSQ); libdevice
+// expf/logf match tl.exp / tl.log.  Only the sum orders of the wide
+// reductions (l2norm, variance) differ.
+//
+// State layout: a cache line holds the conv state [3, CONV_DIM] bf16
+// (token-major: tap j of channel ch at j*cds + ch, cds = CONV_DIM) at
+// offset 0 and the SSM state [HV, V, K] f32 at the ssm offset; line
+// strides are passed in elements of the respective dtype.
+//
+//   nvcc -cubin -arch=sm_103a -o kernels-qwen38/gdn_decode.cubin tools/kernels-src/gdn_decode.cu
+#include <cooperative_groups.h>
+#include <cuda_bf16.h>
+#include <cstdint>
+
+namespace cg = cooperative_groups;
+
+#ifndef GDN_SPLIT
+#define GDN_SPLIT 2
+#endif
+
+namespace {
+constexpr int HV = 48, H = 16, K = 128, V = 128;
+constexpr int CONV_DIM = 10240;          // q | k | v channels
+constexpr int Z_OFF = CONV_DIM;          // z columns after q|k|v in the qkvz row
+constexpr int QOFF = 0, KOFF = 2048, VOFF = 4096;
+constexpr int WIDTH = 4;                 // conv kernel width; state_len = 3
+constexpr int NT = 256;
+constexpr int SPLIT = GDN_SPLIT, ROWS = V / SPLIT;  // CTAs per v head, state rows per CTA
+constexpr int CL = 3 * SPLIT;            // CTAs per cluster (= per q/k head)
+constexpr float SOFTPLUS_THRESHOLD = 20.0f;
+
+__device__ __forceinline__ float bf16r(float v) { return __bfloat162float(__float2bfloat16_rn(v)); }
+__device__ __forceinline__ float rcpa(float x) {
+  float r;
+  asm("rcp.approx.f32 %0, %1;" : "=f"(r) : "f"(x));
+  return r;
+}
+__device__ __forceinline__ float sqrta(float x) {
+  float r;
+  asm("sqrt.approx.f32 %0, %1;" : "=f"(r) : "f"(x));
+  return r;
+}
+__device__ __forceinline__ float sigmoidf_(float x) { return rcpa(1.0f + expf(-x)); }
+
+struct SmemT {
+  float qk[2 * K];      // rank 0: normalized q|k of the head; others: local copy
+  float v[ROWS], o[ROWS];
+  float red[8];
+};
+}  // namespace
+
+extern "C" __global__ void __launch_bounds__(NT) __cluster_dims__(CL, 1, 1)
+kern_gdn_decode_bf16(__nv_bfloat16* __restrict__ qkvz, const __nv_bfloat16* __restrict__ w,
+                     __nv_bfloat16* __restrict__ conv_state, float* __restrict__ ssm_state,
+                     const int* __restrict__ idx, const __nv_bfloat16* __restrict__ ba,
+                     const float* __restrict__ A_log, const __nv_bfloat16* __restrict__ dt_bias,
+                     __nv_bfloat16* __restrict__ out, const __nv_bfloat16* __restrict__ nw,
+                     float scale, float eps, int nlines, int cls, int cds, int sls) {
+  __shared__ SmemT s;
+  const int hv = blockIdx.x / SPLIT, sp = blockIdx.x % SPLIT;
+  const int ih = hv / (HV / H), t = threadIdx.x;
+  const int r0 = sp * ROWS;                       // this CTA's first state row
+  const int coord = idx[0];
+  cg::cluster_group cluster = cg::this_cluster();
+
+  if (coord <= 0) {  // null cache line: conv skips, the recurrent zeroes o,
+                     // and the norm output is (0 * w) * (z * sigmoid(z)) --
+                     // signed zeros (or NaN from an inf z) exactly as A emits
+    if (t < ROWS) {
+      const int j = r0 + t;
+      const float z = __bfloat162float(qkvz[Z_OFF + hv * V + j]);
+      const float y = __fmul_rn(__fmul_rn(0.0f, __bfloat162float(nw[j])), __fmul_rn(z, sigmoidf_(z)));
+      out[hv * V + j] = __float2bfloat16_rn(y);
+    }
+    return;  // uniform across the cluster: no CTA reaches a cluster.sync
+
+  }
+
+  // ---- issue the SSM tile loads of the first row group now: thread t owns
+  // state row r0 + t/8 (+32 per pass), k columns [(t&7)*16, +16)
+  const int rl = t >> 3, c0 = (t & 7) * 16;
+  float* hb0 = ssm_state + (size_t)coord * sls + ((size_t)hv * V + r0 + rl) * K + c0;
+  float hh[ROWS / 32][16], h[16];
+#pragma unroll
+  for (int pass = 0; pass < ROWS / 32; pass++)
+#pragma unroll
+    for (int j = 0; j < 16; j += 4)
+      *reinterpret_cast<float4*>(&hh[pass][j]) =
+          *reinterpret_cast<const float4*>(hb0 + (size_t)pass * 32 * K + j);
+
+  // ---- conv: silu(w0 s0 + w1 s1 + w2 s2 + w3 x) per channel, every value
+  // rounded to bf16 exactly as the in-place Triton store does
+  __nv_bfloat16* cbase = conv_state + (size_t)coord * cls;
+  const bool valid_line = coord < nlines;   // Triton load mask (reads 0 beyond the pool)
+  auto conv1 = [&](int ch) {
+    const __nv_bfloat16* cs = cbase + ch;
+    const float s0 = valid_line ? __bfloat162float(cs[0]) : 0.f;
+    const float s1 = valid_line ? __bfloat162float(cs[cds]) : 0.f;
+    const float s2 = valid_line ? __bfloat162float(cs[2 * cds]) : 0.f;
+    const float x = __bfloat162float(qkvz[ch]);
+    // Triton's `acc += x * w` multiplies two bf16 tiles: the product is
+    // rounded to bf16 before the f32 accumulate
+    float acc = 0.f;
+    acc = __fadd_rn(acc, bf16r(__fmul_rn(s0, __bfloat162float(w[ch * WIDTH + 0]))));
+    acc = __fadd_rn(acc, bf16r(__fmul_rn(s1, __bfloat162float(w[ch * WIDTH + 1]))));
+    acc = __fadd_rn(acc, bf16r(__fmul_rn(s2, __bfloat162float(w[ch * WIDTH + 2]))));
+    acc = __fadd_rn(acc, bf16r(__fmul_rn(x, __bfloat162float(w[ch * WIDTH + 3]))));
+    const float y = bf16r(__fmul_rn(acc, sigmoidf_(acc)));  // silu, division as rcp.approx
+    // the channel is exclusive to its convolving CTA: shift the conv state
+    // and store the conv output over the qkvz column right away
+    __nv_bfloat16* csw = cbase + ch;
+    csw[0] = __float2bfloat16_rn(s1); csw[cds] = __float2bfloat16_rn(s2); csw[2 * cds] = __float2bfloat16_rn(x);
+    qkvz[ch] = __float2bfloat16_rn(y);
+    return y;
+  };
+  // own v channels
+  for (int j = t; j < ROWS; j += NT) s.v[j] = conv1(VOFF + hv * V + r0 + j);
+
+  if (cluster.block_rank() == 0) {
+    // q/k conv + l2norm.  q in threads 0..127, k in 128..255.
+    const float y = conv1(t < K ? QOFF + ih * K + t : KOFF + ih * K + (t - K));
+    s.qk[t] = y;
+    __syncthreads();
+    if (t < 64) {  // warps 0/1 reduce q, k
+      const float* src = s.qk + (t < 32 ? 0 : K);
+      const int lane = t & 31;
+      float p = 0.f;
+      for (int j = lane; j < K; j += 32) p += __fmul_rn(src[j], src[j]);
+      for (int off = 16; off > 0; off >>= 1) p += __shfl_down_sync(0xffffffffu, p, off);
+      if (lane == 0) s.red[t >> 5] = p;
+    }
+    __syncthreads();
+    // Triton: b_q / tl.sqrt(sum + 1e-6) lowers to sqrt.approx + rcp.approx + mul
+    const float iq = rcpa(sqrta(s.red[0] + 1e-6f)), ik = rcpa(sqrta(s.red[1] + 1e-6f));
+    s.qk[t] = t < K ? __fmul_rn(__fmul_rn(s.qk[t], iq), scale) : __fmul_rn(s.qk[t], ik);
+  }
+  cluster.sync();
+  if (cluster.block_rank() != 0) {  // copy the head's q/k rows to local smem
+    const float* remote = cluster.map_shared_rank(s.qk, 0);
+    s.qk[t] = remote[t];
+    __syncthreads();
+  }
+
+  // ---- delta rule step over this CTA's ROWS state rows, 32 per pass
+  const float a_val = __bfloat162float(ba[HV + hv]);
+  const float b_val = __bfloat162float(ba[hv]);
+  const float xg = a_val + __bfloat162float(dt_bias[hv]);
+  const float sp_ = xg <= SOFTPLUS_THRESHOLD ? logf(1.0f + expf(xg)) : xg;
+  const float g_e = expf(-expf(A_log[hv]) * sp_);
+  const float beta = bf16r(sigmoidf_(b_val));  // Triton rounds beta through bf16
+
+  const float* kk = s.qk + K;
+  // all row groups' state tiles are already prefetched into hh (issued
+  // before the conv/cluster.sync so the loads run under them)
+#pragma unroll
+  for (int pass = 0; pass < ROWS / 32; pass++) {
+    const int row = pass * 32 + rl;
+    float* hb = hb0 + (size_t)pass * 32 * K;
+#pragma unroll
+    for (int j = 0; j < 16; j++) h[j] = hh[pass][j];
+    float dot = 0.f;
+#pragma unroll
+    for (int j = 0; j < 16; j++) {
+      h[j] *= g_e;
+      dot = __fadd_rn(dot, __fmul_rn(h[j], kk[c0 + j]));
+    }
+    dot += __shfl_xor_sync(0xffffffffu, dot, 1);
+    dot += __shfl_xor_sync(0xffffffffu, dot, 2);
+    dot += __shfl_xor_sync(0xffffffffu, dot, 4);
+    const float vn = (s.v[pass * 32 + rl] - dot) * beta;
+    float op = 0.f;
+#pragma unroll
+    for (int j = 0; j < 16; j++) {
+      h[j] = __fmaf_rn(vn, kk[c0 + j], h[j]);
+      op = __fadd_rn(op, __fmul_rn(h[j], s.qk[c0 + j]));
+    }
+    op += __shfl_xor_sync(0xffffffffu, op, 1);
+    op += __shfl_xor_sync(0xffffffffu, op, 2);
+    op += __shfl_xor_sync(0xffffffffu, op, 4);
+#pragma unroll
+    for (int j = 0; j < 16; j += 4) *reinterpret_cast<float4*>(hb + j) = *reinterpret_cast<const float4*>(&h[j]);
+    if ((t & 7) == 0) s.o[row] = bf16r(op);  // Triton stores o as bf16; the norm re-loads it
+  }
+  __syncthreads();
+
+  // ---- gated RMSNorm: y = (x * rstd) * w, then y *= z * sigmoid(z)
+  if (t < 32) {
+    float p = 0.f;
+    for (int j = t; j < ROWS; j += 32) p += __fmul_rn(s.o[j], s.o[j]);
+    for (int off = 16; off > 0; off >>= 1) p += __shfl_down_sync(0xffffffffu, p, off);
+    if (t == 0) s.red[0] = p;
+  }
+#if GDN_SPLIT == 2
+  cluster.sync();  // exchange the two variance partials of this v head
+  if (t == 0) {
+    const float* other = cluster.map_shared_rank(s.red, cluster.block_rank() ^ 1);
+    const float p0 = (sp == 0) ? s.red[0] : other[0];
+    const float p1 = (sp == 0) ? other[0] : s.red[0];
+    s.red[1] = rsqrtf(__fadd_rn(p0, p1) / (float)V + eps);
+  }
+  __syncthreads();
+  const float rstd = s.red[1];
+#else
+  __syncthreads();
+  const float rstd = rsqrtf(s.red[0] / (float)V + eps);
+#endif
+  for (int j = t; j < ROWS; j += NT) {
+    const int col = r0 + j;
+    const float z = __bfloat162float(qkvz[Z_OFF + hv * V + col]);
+    float y = __fmul_rn(__fmul_rn(s.o[j], rstd), __bfloat162float(nw[col]));
+    y = __fmul_rn(y, __fmul_rn(z, sigmoidf_(z)));
+    out[hv * V + col] = __float2bfloat16_rn(y);
+  }
+}
diff --git a/tools/kernels-src/gdn_spec.cu b/tools/kernels-src/gdn_spec.cu
new file mode 100644
index 0000000..9857390
--- /dev/null
+++ b/tools/kernels-src/gdn_spec.cu
@@ -0,0 +1,369 @@
+// The speculative-path GDN chain of one layer in one launch (tokens <= 8):
+//
+//   causal_conv1d_update      (IS_SPEC_DECODING,    _causal_conv1d_update_kernel
+//                              width 4, silu, state
+//                              roll from slot nacc-1)
+//   fused post-conv prep      (split + l2norm +     _fused_post_conv_kernel
+//                              gating)
+//   a/b contiguous copies                           kern_copy_rows_bf16 x2
+//   gated delta rule, T rows  (l2norm again, per-   fused_sigmoid_gating_delta_rule_update_kernel
+//                              token SSM checkpoint
+//                              to spec_slots[t])
+//   z copy                                          kern_copy_rows_bf16
+//   gated RMSNorm             (norm, then *silu(z)) layer_norm_fwd_kernel
+//
+// The pipeline the manifest pins normalizes q/k twice: the post_conv
+// instance is the prefill one (APPLY_L2NORM, 1/sqrt form) and the
+// recurrent kernel was captured with USE_QK_L2NORM_IN_KERNEL (rsqrt
+// form); the fused kernel reproduces both, feeding the bf16-rounded
+// first-pass values into the second, like the buffers did.
+//
+// Four CTAs per v head (192 total, co-resident) stream 32 state rows
+// each and redundantly convolve + normalize the 256 q/k channels of
+// their k head from registers/smem -- no producer/consumer chain.  The
+// only synchronization: a per-k-head staging counter before the conv
+// writeback may overwrite qkvz (twelve sibling CTAs read those raw
+// columns), and the fixed-order variance merge of the gated norm.
+//
+// Rounding chains reproduce the Triton kernels bit-for-bit where the
+// order is defined (bf16 x*w conv products rounded before the f32
+// accumulate; silu -> bf16 store; the post_conv l2norm divides by
+// tl.sqrt and rounds to bf16; the recurrent reloads that value,
+// multiplies by tl.rsqrt of its own sum, then by scale; its softplus is
+// the one-sided log(1+exp) form while the g/beta buffers keep the
+// two-sided post_conv form; state updates go through fmaf like the
+// Triton fma; the norm input is the bf16-rounded recurrent output).
+// Only the sum orders of the wide reductions differ.
+//
+// Buffers past the cut carry every write of the chain: qkvz conv
+// writeback, the conv state roll, the per-token SSM checkpoints,
+// gdn_q/k/v, g/beta, a_c/b_c/z_c and the normed core_attn_out.
+//
+// All counter scratch is zero at rest: alloc_zeros at load, the last
+// CTA resets (hpart holds data, write-before-read behind the counters).
+//
+//   nvcc -cubin -arch=sm_103a -o kernels-qwen38-dflash2/gdn_spec.cubin tools/kernels-src/gdn_spec.cu
+#include <cuda_bf16.h>
+#include <cstdint>
+
+namespace {
+constexpr int HV = 48, K = 128, V = 128;
+constexpr int CONV_DIM = 10240;             // q | k | v channels
+constexpr int Z_OFF = CONV_DIM;             // z columns after q|k|v
+constexpr int QKVZ = 16384, BA = 96;
+constexpr int KOFF = 2048, VOFF = 4096;
+constexpr int NT = 256;
+constexpr int SPLIT = 4, ROWS = V / SPLIT;  // CTAs per v head, state rows per CTA
+constexpr int NB = SPLIT * HV;
+constexpr int TMAX = 8;                     // SPEC_BLOCK
+constexpr float L2EPS = 1e-6f;
+
+__device__ __forceinline__ int ld_acquire(const int* p) {
+  int v;
+  asm volatile("ld.acquire.gpu.global.s32 %0, [%1];" : "=r"(v) : "l"(p) : "memory");
+  return v;
+}
+__device__ __forceinline__ float bf16r(float v) { return __bfloat162float(__float2bfloat16_rn(v)); }
+// Triton lowers float division to rcp.approx + mul and tl.sqrt to
+// sqrt.approx (see the pinned cubins' SASS); replicate the instructions.
+__device__ __forceinline__ float rcpa(float x) {
+  float y;
+  asm("rcp.approx.f32 %0, %1;" : "=f"(y) : "f"(x));
+  return y;
+}
+__device__ __forceinline__ float sqrta(float x) {
+  float y;
+  asm("sqrt.approx.f32 %0, %1;" : "=f"(y) : "f"(x));
+  return y;
+}
+__device__ __forceinline__ float sigmoidf_(float x) { return rcpa(1.0f + expf(-x)); }
+__device__ __forceinline__ float ldcg(const float* p) { return __ldcg(p); }
+
+struct SmemT {
+  __nv_bfloat16 cq[TMAX][K], ck[TMAX][K];   // conv outputs (raw x rows past T)
+  __nv_bfloat16 nq[TMAX][K], nk[TMAX][K];   // post_conv l2norm'd values
+  __nv_bfloat16 cv[TMAX][ROWS];             // v conv outputs (this CTA's rows)
+  __nv_bfloat16 oarr[TMAX][ROWS];           // recurrent output rows
+  float s1q[TMAX], s1k[TMAX];               // post_conv sums of squares
+  float s2q[TMAX], s2k[TMAX];               // recurrent sums of the rounded values
+  float red[TMAX][8];                       // warp partials for the sums
+  float gg[TMAX], bb[TMAX];                 // exp(g) and beta, recurrent form
+};
+}  // namespace
+
+// scratch: xcnt i32 [16] per-k-head staging counters, hpart f32
+// [TMAX*HV*SPLIT] variance partials, hcnt i32 [HV] arrive counters,
+// gcnt i32 [1] exit counter for the reset.
+extern "C" __global__ void __launch_bounds__(NT) kern_gdn_spec_bf16(
+    __nv_bfloat16* __restrict__ qkvz,          // [tokens, 16384] inout
+    const __nv_bfloat16* __restrict__ w,       // [10240, 4]
+    __nv_bfloat16* __restrict__ conv_state,    // state pool, conv area
+    float* __restrict__ ssm_state,             // state pool, ssm area
+    const int* __restrict__ line_p,            // conv line page id
+    const int* __restrict__ nacc_p,            // num accepted (resume slot + 1)
+    const int* __restrict__ cu,                // cu_seqlens_q [2]
+    const int* __restrict__ slots,             // 8 checkpoint page ids
+    const __nv_bfloat16* __restrict__ ba,      // [tokens, 96] b | a
+    const float* __restrict__ A_log,           // [48]
+    const __nv_bfloat16* __restrict__ dt_bias, // [48]
+    __nv_bfloat16* __restrict__ out,           // core_attn_out [tokens, 6144]
+    const __nv_bfloat16* __restrict__ nw,      // norm.weight [128]
+    __nv_bfloat16* __restrict__ gdn_q,         // [tokens, 2048]
+    __nv_bfloat16* __restrict__ gdn_k,         // [tokens, 2048]
+    __nv_bfloat16* __restrict__ gdn_v,         // [tokens, 6144]
+    float* __restrict__ g_buf,                 // [tokens, 48]
+    float* __restrict__ beta_buf,              // [tokens, 48]
+    __nv_bfloat16* __restrict__ a_c,           // [tokens, 48]
+    __nv_bfloat16* __restrict__ b_c,           // [tokens, 48]
+    __nv_bfloat16* __restrict__ z_c,           // [tokens, 6144]
+    int* __restrict__ xcnt, float* __restrict__ hpart,
+    int* __restrict__ hcnt, int* __restrict__ gcnt,
+    float scale, float eps, int tokens,
+    int nlines, int cls, int cds, int sls) {
+  __shared__ SmemT s;
+  const int tid = threadIdx.x;
+  const int hv = blockIdx.x >> 2, sp = blockIdx.x & 3;
+  const int ih = hv / 3;
+  const bool qkwriter = (hv % 3) == 0;
+  const int line = line_p[0], nacc = nacc_p[0];
+  const int cu0 = cu[0];
+  const int T = cu[1] - cu0;              // conv/recurrent extent
+  const int tt = tokens;                  // post_conv/copies/norm extent
+  const bool live = line > 0;
+
+  // ---- stage: conv history + raw x for this thread's channel(s) ----
+  // threads 0..127: q channel c of head ih; 128..255: k channel c.
+  const int c = tid & 127;
+  const int qk_ch = (tid < 128 ? ih * K : KOFF + ih * K) + c;
+  const int v_ch = VOFF + hv * V + sp * ROWS + (tid < ROWS ? tid : 0);
+  float wq[4], wv[4], hq[3], hvv[3];
+  __nv_bfloat16 xq[TMAX], xv[TMAX];
+  for (int j = 0; j < 4; j++) {
+    wq[j] = __bfloat162float(w[qk_ch * 4 + j]);
+    wv[j] = __bfloat162float(w[v_ch * 4 + j]);
+  }
+  // spec-pool conv state layout is token-major: [conv_len rows][CONV_DIM]
+  const int64_t cbase_qk = (int64_t)line * cls + qk_ch;
+  const int64_t cbase_v = (int64_t)line * cls + v_ch;
+  for (int j = 0; j < 3; j++) {
+    hq[j] = live ? __bfloat162float(conv_state[cbase_qk + (int64_t)(nacc - 1 + j) * CONV_DIM]) : 0.f;
+    hvv[j] = live ? __bfloat162float(conv_state[cbase_v + (int64_t)(nacc - 1 + j) * CONV_DIM]) : 0.f;
+  }
+#pragma unroll
+  for (int t = 0; t < TMAX; t++) {
+    if (t >= tt) break;
+    xq[t] = qkvz[(int64_t)(cu0 + t) * QKVZ + qk_ch];
+    xv[t] = qkvz[(int64_t)(cu0 + t) * QKVZ + v_ch];
+  }
+  __threadfence();
+  __syncthreads();
+  if (tid == 0) atomicAdd(&xcnt[ih], 1);
+
+  // initial SSM state, prefetched early so the cold load overlaps the conv
+  const int row = tid >> 3, lane8 = tid & 7, c0 = lane8 * 16;
+  const int page0 = slots[nacc - 1];
+  const bool rec = page0 > 0;
+  const int64_t hb = (int64_t)hv * V * K + (int64_t)(sp * ROWS + row) * K + c0;
+  float bh[16];
+  if (rec) {
+    const float4* h0 = (const float4*)(ssm_state + (int64_t)page0 * sls + hb);
+#pragma unroll
+    for (int j = 0; j < 4; j++) *(float4*)&bh[j * 4] = h0[j];
+  }
+
+  // ---- conv (Triton: f32 acc of bf16-rounded products, then silu) ----
+  auto conv1 = [&](const float* h, const float* wt, const __nv_bfloat16* x, int t) {
+    float acc = 0.f;
+#pragma unroll
+    for (int j = 0; j < 4; j++) {
+      const int i = t + j;  // window over [h0 h1 h2 x0 x1 ...]
+      const float xf = i < 3 ? h[i] : __bfloat162float(x[i - 3]);
+      acc = __fadd_rn(acc, bf16r(__fmul_rn(xf, wt[j])));
+    }
+    return __float2bfloat16_rn(__fmul_rn(acc, rcpa(1.0f + expf(-acc))));
+  };
+  {
+    __nv_bfloat16* dst = tid < 128 ? &s.cq[0][0] : &s.ck[0][0];
+#pragma unroll
+    for (int t = 0; t < TMAX; t++) {
+      if (t >= tt) break;
+      dst[t * K + c] = (live && t < T) ? conv1(hq, wq, xq, t) : xq[t];
+      if (tid < ROWS) s.cv[t][tid] = (live && t < T) ? conv1(hvv, wv, xv, t) : xv[t];
+    }
+  }
+  // gating scalars for this v head (both softplus forms)
+  if (tid < tt) {
+    const int t = tid;
+    const float bv = __bfloat162float(ba[(int64_t)t * BA + hv]);
+    const float av = __bfloat162float(ba[(int64_t)t * BA + 48 + hv]);
+    const float x = av + __bfloat162float(dt_bias[hv]);
+    const float ea = expf(A_log[hv]);
+    // recurrent form: (1/beta) * log(1 + exp(beta*x)), beta = 1
+    const float spr = x <= 20.f ? 1.0f * logf(1.0f + expf(x)) : x;
+    s.gg[t] = expf(-ea * spr);
+    s.bb[t] = sigmoidf_(bv);
+    if (sp == 0) {
+      // post_conv form: two-sided stable softplus
+      float spp = x > 0.f ? x + logf(1.0f + expf(-x)) : logf(1.0f + expf(x));
+      if (x > 20.f) spp = x;
+      g_buf[(int64_t)t * HV + hv] = -ea * spp;
+      beta_buf[(int64_t)t * HV + hv] = sigmoidf_(bv);
+      a_c[(int64_t)t * HV + hv] = ba[(int64_t)t * BA + 48 + hv];
+      b_c[(int64_t)t * HV + hv] = ba[(int64_t)t * BA + hv];
+    }
+  }
+  __syncthreads();
+
+  // ---- sums of squares of the conv outputs (post_conv l2norm) ----
+  auto sumsq = [&](const __nv_bfloat16* v128, int t, int part) {
+    // 128 threads (4 warps): warp-tree then fixed-order partial merge
+    const float x = __bfloat162float(v128[c]);
+    float p = __fmul_rn(x, x);
+    for (int d = 16; d > 0; d >>= 1) p += __shfl_xor_sync(0xffffffffu, p, d);
+    if ((tid & 31) == 0) s.red[t][part * 4 + ((tid >> 5) & 3)] = p;
+  };
+  for (int t = 0; t < tt; t++) sumsq(tid < 128 ? s.cq[t] : s.ck[t], t, tid < 128 ? 0 : 1);
+  __syncthreads();
+  if (tid < 2 * tt) {
+    const int t = tid >> 1;
+    const float* r = &s.red[t][(tid & 1) * 4];
+    (tid & 1 ? s.s1k : s.s1q)[t] = (r[0] + r[1]) + (r[2] + r[3]);
+  }
+  __syncthreads();
+  {
+    __nv_bfloat16* dst = tid < 128 ? &s.nq[0][0] : &s.nk[0][0];
+    const __nv_bfloat16* src = tid < 128 ? &s.cq[0][0] : &s.ck[0][0];
+    const float* ss = tid < 128 ? s.s1q : s.s1k;
+    for (int t = 0; t < tt; t++) {
+      const float inv = rcpa(sqrta(ss[t] + L2EPS));
+      dst[t * K + c] = __float2bfloat16_rn(__bfloat162float(src[t * K + c]) * inv);
+    }
+  }
+  __syncthreads();
+  // sums of squares of the rounded values (recurrent l2norm)
+  for (int t = 0; t < tt; t++) sumsq(tid < 128 ? s.nq[t] : s.nk[t], t, tid < 128 ? 0 : 1);
+  __syncthreads();
+  if (tid < 2 * tt) {
+    const int t = tid >> 1;
+    const float* r = &s.red[t][(tid & 1) * 4];
+    (tid & 1 ? s.s2k : s.s2q)[t] = (r[0] + r[1]) + (r[2] + r[3]);
+  }
+  __syncthreads();
+
+  // ---- writebacks that only this CTA touches ----
+  for (int t = 0; t < tt; t++) {
+    if (tid < ROWS) {
+      const int col = hv * V + sp * ROWS + tid;
+      gdn_v[(int64_t)t * 6144 + col] = s.cv[t][tid];
+      z_c[(int64_t)t * 6144 + col] = qkvz[(int64_t)t * QKVZ + Z_OFF + col];
+    }
+    if (tid < 128) {
+      gdn_q[(int64_t)t * KOFF + ih * K + c] = s.nq[t][c];
+    } else {
+      gdn_k[(int64_t)t * KOFF + ih * K + c] = s.nk[t][c];
+    }
+  }
+  // conv state roll + qkvz writeback (v channels: this CTA is the only
+  // reader; q/k channels: wait for the twelve sibling CTAs to stage)
+  auto roll = [&](int64_t cbase, const float* h, const __nv_bfloat16* x) {
+    const bool ok = line < nlines;  // A masks the shifted loads by this
+    for (int i = 0; i < T + 2 && i < cds; i++) {
+      __nv_bfloat16 val;
+      if (i < 2)
+        val = ok ? __float2bfloat16_rn(h[1 + i]) : __float2bfloat16_rn(0.f);
+      else
+        val = x[i - 2];
+      conv_state[cbase + (int64_t)i * CONV_DIM] = val;
+    }
+  };
+  if (live && tid < ROWS) {
+    roll(cbase_v, hvv, xv);
+    for (int t = 0; t < T; t++) qkvz[(int64_t)(cu0 + t) * QKVZ + v_ch] = s.cv[t][tid];
+  }
+  if (live && qkwriter && (c >> 5) == sp) {
+    if (tid == (sp << 5)) while (ld_acquire(&xcnt[ih]) < 3 * SPLIT) {}
+    __syncwarp();
+    roll(cbase_qk, hq, xq);
+    for (int t = 0; t < T; t++)
+      qkvz[(int64_t)(cu0 + t) * QKVZ + qk_ch] = (tid < 128 ? s.cq : s.ck)[t][c];
+  }
+
+  // ---- gated delta rule over T rows, checkpoint each row's state ----
+  if (rec) {
+    for (int t = 0; t < T; t++) {
+      const float i2q = rsqrtf(s.s2q[t] + L2EPS), i2k = rsqrtf(s.s2k[t] + L2EPS);
+      float qv[16], kv[16];
+      uint4 qraw[2], kraw[2];
+      qraw[0] = ((const uint4*)&s.nq[t][c0])[0]; qraw[1] = ((const uint4*)&s.nq[t][c0])[1];
+      kraw[0] = ((const uint4*)&s.nk[t][c0])[0]; kraw[1] = ((const uint4*)&s.nk[t][c0])[1];
+#pragma unroll
+      for (int j = 0; j < 16; j++) {
+        qv[j] = __bfloat162float(((const __nv_bfloat16*)qraw)[j]) * i2q * scale;
+        kv[j] = __bfloat162float(((const __nv_bfloat16*)kraw)[j]) * i2k;
+      }
+      float dot = 0.f;
+      for (int j = 0; j < 16; j++) {
+        bh[j] *= s.gg[t];
+        dot += __fmul_rn(bh[j], kv[j]);
+      }
+      for (int d = 1; d < 8; d <<= 1) dot += __shfl_xor_sync(0xffffffffu, dot, d);
+      float bv = __bfloat162float(s.cv[t][row]) - dot;
+      bv *= s.bb[t];
+      float o = 0.f;
+      for (int j = 0; j < 16; j++) {
+        bh[j] = fmaf(bv, kv[j], bh[j]);
+        o += __fmul_rn(bh[j], qv[j]);
+      }
+      for (int d = 1; d < 8; d <<= 1) o += __shfl_xor_sync(0xffffffffu, o, d);
+      if (lane8 == 0) s.oarr[t][row] = __float2bfloat16_rn(o);
+      const int st = slots[t];
+      if (st > 0) {
+        float4* ht = (float4*)(ssm_state + (int64_t)st * sls + hb);
+        for (int j = 0; j < 4; j++) ht[j] = *(float4*)&bh[j * 4];
+      }
+    }
+  }
+  __syncthreads();
+
+  // ---- gated RMSNorm, variance merged across the four sibling CTAs ----
+  if (tid < ROWS) {
+    for (int t = 0; t < tt; t++) {
+      float x;
+      if (rec && t < T)
+        x = __bfloat162float(s.oarr[t][tid]);
+      else
+        x = __bfloat162float(out[(int64_t)t * 6144 + hv * V + sp * ROWS + tid]);
+      s.oarr[t][tid] = __float2bfloat16_rn(x);  // keep the norm input
+      float p = __fmul_rn(x, x);
+      for (int d = 16; d > 0; d >>= 1) p += __shfl_xor_sync(0xffffffffu, p, d);
+      if (tid == 0) hpart[((int64_t)t * HV + hv) * SPLIT + sp] = p;
+    }
+  }
+  __threadfence();
+  __syncthreads();
+  if (tid == 0) atomicAdd(&hcnt[hv], 1);
+  if (tid == 0) while (ld_acquire(&hcnt[hv]) < SPLIT) {}
+  __syncthreads();
+  if (tid < ROWS) {
+    for (int t = 0; t < tt; t++) {
+      const float* p = hpart + ((int64_t)t * HV + hv) * SPLIT;
+      const float var = ((ldcg(p) + ldcg(p + 1)) + (ldcg(p + 2) + ldcg(p + 3))) / (float)V;
+      const float rstd = rsqrtf(var + eps);
+      const int col = hv * V + sp * ROWS + tid;
+      float y = __bfloat162float(s.oarr[t][tid]) * rstd;
+      y *= __bfloat162float(nw[sp * ROWS + tid]);
+      const float z = __bfloat162float(qkvz[(int64_t)t * QKVZ + Z_OFF + col]);
+      y *= z * sigmoidf_(z);
+      out[(int64_t)t * 6144 + col] = __float2bfloat16_rn(y);
+    }
+  }
+
+  // ---- last CTA resets the counter scratch ----
+  __syncthreads();
+  if (tid == 0 && atomicAdd(gcnt, 1) == NB - 1) {
+    for (int i = 0; i < 16; i++) xcnt[i] = 0;
+    for (int i = 0; i < HV; i++) hcnt[i] = 0;
+    *gcnt = 0;
+    __threadfence();
+  }
+}
diff --git a/tools/kernels-src/gemm8.cu b/tools/kernels-src/gemm8.cu
new file mode 100644
index 0000000..cb7a8d1
--- /dev/null
+++ b/tools/kernels-src/gemm8.cu
@@ -0,0 +1,676 @@
+// Streaming bf16 GEMM for M <= 8 token rows (bs=1 decode and the 8-row
+// draft / verify blocks of DFlash2), Blackwell sm_103a.  At these shapes the
+// weight matrix is the only real traffic: one persistent CTA per SM pulls its
+// weight rows through a smem ring with cp.async.bulk (TMA 1D, plain pointers,
+// 40 KB stages of 8 rows) and 8 compute warps multiply each stage with
+// mma.m16n8k16 via ldmatrix.x4 (bf16 in, f32 accumulate; the 8 weight rows
+// are the n8 side, the <= 8 tokens fill half of m16).  x is staged once into
+// a dedicated smem area (<= 40960 elements); when M*K exceeds it -- or N is
+// too small to balance the grid -- K is cut into ranges, tasks are ordered
+// range-major so a CTA reloads x only on a range switch, and the f32 range
+// partials go to impl scratch with no atomics: the tail that already reads
+// every output element sums them there.  Smem rows are padded by 16 B so the
+// ldmatrix reads are bank-conflict-free (measured 3.5x slower without).
+//
+// Tasks: whole 8-row blocks, a multiple of the grid so every CTA streams the
+// same number, plus the leftover rows spread over the grid in tasks one
+// granule apart -- scheduled first, where their latency-bound stages hide
+// under the blocks behind them.  Every task walks the same K stages, so the
+// per-row accumulation order does not depend on the geometry.  The ring/x
+// split of the shared area is per launch, so bs=1 gets 5-6 stages in flight.
+//
+// The tail runs after a single end-of-CTA fence + task-count atomic.  With
+// grid <= %nsmid (one ~200 KB-smem CTA per SM, the launch geometry of every
+// call site) all CTAs are co-resident, so CTAs 0..M-1 spin on the counter and
+// each finishes one token row in parallel; on any other geometry the CTA that
+// finished last does all rows.  Counters are back to zero at kernel end.
+// The tail is pure latency: the norm weight is fetched under the spin and
+// each global round trip of a row is one batch of loads.
+//
+// Programmatic dependent launch (manifest step `pdl`): the ring is filled
+// with the first task's weights before griddepcontrol.wait, everything that
+// reads an upstream product or writes comes after it, and the CTA signals
+// launch_dependents once its stream is issued, so the next launch's ramp
+// overlaps this launch's drain and tail.
+//
+// Entry points differ in what is fused around the GEMM (rounding chains
+// reproduce the vLLM / ATen kernels they replace):
+//
+//   kern_gemm8_bf16               y[M,N] = x W^T
+//   kern_gemm8_gateup_silu_bf16   act = silu(gate) * up over the fused [2F,K]
+//                                 gate_up weight (vLLM act_and_mul: bf16(gate),
+//                                 bf16(up), bf16(silu), bf16(product))
+//   kern_gemm8_dual_bf16          y1 = x W1^T, y2 = x W2^T (one weight stream)
+//   kern_gemm8_add_norm_bf16      y = x W^T, then per token row the Gemma
+//                                 fused_add_rms_norm (kern_gemma_rms_norm.cu
+//                                 arithmetic: z = f32(bf16(y)) + f32(res),
+//                                 res = bf16(z), ATen-order mean of z^2,
+//                                 out = bf16((z * rsqrt(var + eps)) * w1))
+//   kern_gemm8_sgate_add_norm_bf16  the add_norm chain with x computed on
+//                                 the fly as attn * sigmoid(gate) (the
+//                                 sigmoid_mul.cu chain: g = bf16(sigmoid
+//                                 f32), x = bf16(f32(attn) * g)) -- o_proj
+//                                 of the gated-attention layers
+//
+//   nvcc -cubin -arch=sm_103a -o kernels-qwen38/gemm8.cubin tools/kernels-src/gemm8.cu
+#include <cuda_bf16.h>
+#include <cstdint>
+
+namespace {
+constexpr int MMAX = 8;                  // token rows per launch
+constexpr int R = 8;                     // weight rows per task (one n8 tile)
+constexpr int SEG_MAX = 2560;            // K elements per weight row per stage
+constexpr int PAD = 8;                   // 16 B row padding: bank shift 4
+constexpr int X_ELEMS = 40960;           // x area capacity (M * kseg elements)
+constexpr int X_SLOT = X_ELEMS + MMAX * PAD;
+constexpr int RING_ELEMS = 69888;        // ring minimum: 3 stages of 8 rows at SEG_MAX (x at capacity)
+constexpr int BUF_ELEMS = RING_ELEMS + X_SLOT;  // ring + x share one area, split per launch by M * kseg
+constexpr int STAGES_MAX = 6;
+constexpr int NCW = 8, CTHREADS = NCW * 32, NTHREADS = CTHREADS + 32;
+constexpr int BAR_C = 1;                 // named barrier of the compute warps
+constexpr int NT_ATEN = 512;             // ATen reduce block (emulated)
+
+__device__ __forceinline__ uint32_t smem_u32(const void* p) { return (uint32_t)__cvta_generic_to_shared(p); }
+__device__ __forceinline__ void mbar_init(uint32_t bar, uint32_t count) {
+  asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;" ::"r"(bar), "r"(count));
+}
+__device__ __forceinline__ void mbar_expect_tx(uint32_t bar, uint32_t bytes) {
+  asm volatile("mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;" ::"r"(bar), "r"(bytes) : "memory");
+}
+__device__ __forceinline__ void mbar_arrive(uint32_t bar) {
+  asm volatile("mbarrier.arrive.shared::cta.b64 _, [%0];" ::"r"(bar) : "memory");
+}
+__device__ __forceinline__ void mbar_wait(uint32_t bar, uint32_t parity) {
+  asm volatile(
+      "{\n.reg .pred p;\nWAIT_%=:\nmbarrier.try_wait.parity.shared::cta.b64 p, [%0], %1;\n@!p bra WAIT_%=;\n}" ::"r"(bar),
+      "r"(parity)
+      : "memory");
+}
+__device__ __forceinline__ void fence_barrier_init() { asm volatile("fence.mbarrier_init.release.cluster;" ::: "memory"); }
+__device__ __forceinline__ void bulk_copy_1d(uint32_t dst, const void* src, uint32_t bytes, uint32_t bar) {
+  asm volatile("cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" ::"r"(dst), "l"(src),
+               "r"(bytes), "r"(bar)
+               : "memory");
+}
+__device__ __forceinline__ void named_bar_sync(int id, int n) { asm volatile("bar.sync %0, %1;" ::"r"(id), "r"(n) : "memory"); }
+__device__ __forceinline__ void mma_bf16_16816(float& c0, float& c1, float& c2, float& c3, uint32_t a0, uint32_t a1, uint32_t a2,
+                                               uint32_t a3, uint32_t b0, uint32_t b1) {
+  asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%0,%1,%2,%3};"
+               : "+f"(c0), "+f"(c1), "+f"(c2), "+f"(c3)
+               : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(b0), "r"(b1));
+}
+__device__ __forceinline__ void ldsm_x4(uint32_t& r0, uint32_t& r1, uint32_t& r2, uint32_t& r3, uint32_t addr) {
+  asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];" : "=r"(r0), "=r"(r1), "=r"(r2), "=r"(r3) : "r"(addr));
+}
+__device__ __forceinline__ int ld_acquire(const int* p) {
+  int v;
+  asm volatile("ld.acquire.gpu.global.s32 %0, [%1];" : "=r"(v) : "l"(p) : "memory");
+  return v;
+}
+__device__ __forceinline__ uint32_t nsmid() {
+  uint32_t n;
+  asm("mov.u32 %0, %%nsmid;" : "=r"(n));
+  return n;
+}
+__device__ __forceinline__ float bf16r(float v) { return __bfloat162float(__float2bfloat16_rn(v)); }
+// Programmatic dependent launch: the caller may launch this kernel with
+// CU_LAUNCH_ATTRIBUTE_PROGRAMMATIC_STREAM_SERIALIZATION.  Every global read of
+// an upstream product and every global write sit behind pdl_wait(); only the
+// weight stream (this launch's own inputs) runs ahead of it.  Without the
+// attribute both instructions are no-ops.
+__device__ __forceinline__ void pdl_wait() { asm volatile("griddepcontrol.wait;" ::: "memory"); }
+__device__ __forceinline__ void pdl_trigger() { asm volatile("griddepcontrol.launch_dependents;" ::: "memory"); }
+
+struct Smem {
+  alignas(128) __nv_bfloat16 buf[BUF_ELEMS];  // [ring: stage s at s*8*(seg+PAD)] ... [x: M rows, stride kseg+PAD]
+  alignas(8) uint64_t full[STAGES_MAX], empty[STAGES_MAX], xfull, xempty;
+  float red[NCW][MMAX][R];
+  int flag;
+};
+static_assert(sizeof(Smem) + 128 <= 224128, "manifest shared_mem (GEMM8_SMEM) covers Smem");
+
+enum Mode { PLAIN = 0, GATEUP = 1, DUAL = 2, NORM = 3, SGATE = 4 };
+
+struct Args {
+  const __nv_bfloat16* x;     // [M, ldx]
+  const __nv_bfloat16* W;     // [N, K] (GATEUP: [2F, K])
+  const __nv_bfloat16* W2;    // DUAL: rows >= N1
+  __nv_bfloat16* y;           // PLAIN out [M, N]; DUAL y1; GATEUP act [M, F]; NORM: y scratch [M, N]
+  __nv_bfloat16* y2;          // DUAL y2 [M, N2]
+  float* partial;             // [nranges, MMAX, N] f32 scratch, or nullptr (GATEUP / DUAL: never split)
+  int* gcnt;                  // [2] i32 scratch: finished tasks / finished rows, zero at rest
+  int M, N, N1, K, ldx;
+  const __nv_bfloat16* gate;  // SGATE: strided gate view; x is then attn [M, ldx]
+  int head_dim, gate_ts, gate_hs;
+};
+
+// Tasks of one K range: nfull whole 8-row blocks, a multiple of the grid so
+// every CTA streams the same number, plus the leftover rows spread over the
+// grid in tasks of `rem_g` rows (GATEUP: whole gate/up pairs).  The small
+// tasks come first: their stages are latency-bound, and at the head of a
+// CTA's queue that latency hides under the full blocks streaming behind
+// them instead of being the launch's last few microseconds.  The per-row
+// accumulation order is the same either way.
+struct Geom {
+  int nrows, nfull, nsmall, na, hi, lo, nps;  // rows; full blocks; small tasks: na of hi rows then hi-gran rows; tasks per range
+};
+template <int MODE>
+__device__ __forceinline__ Geom task_geom(const Args& a) {
+  Geom g;
+  g.nrows = MODE == GATEUP ? 2 * a.N : a.N;
+  const int nblk = (g.nrows + R - 1) / R, grid = gridDim.x;
+  g.nfull = nblk / grid * grid;
+  // leftover rows over the whole grid in two task widths differing by one
+  // granule, so no CTA streams more than a granule beyond another
+  const int rem = g.nrows - g.nfull * R, gran = MODE == GATEUP ? 2 : 1;
+  g.hi = min(R, ((rem + grid - 1) / grid + gran - 1) / gran * gran);
+  g.lo = g.hi - gran;
+  if (rem == 0) { g.na = g.nsmall = 0; }
+  else if (a.M > 1) { g.hi = R; g.lo = 0; g.na = g.nsmall = (rem + R - 1) / R; }  // whole blocks: a narrow task costs full MMA rounds
+  else if (g.lo == 0) { g.na = g.nsmall = (rem + g.hi - 1) / g.hi; }
+  else { g.na = (rem - g.lo * grid + gran - 1) / gran; g.nsmall = grid; }
+  g.nps = g.nfull + g.nsmall;
+  return g;
+}
+__device__ __forceinline__ void task_rows(const Geom& g, int u, int& r0, int& rows) {
+  if (u < g.nsmall) {
+    r0 = g.nfull * R + (u < g.na ? u * g.hi : g.na * g.hi + (u - g.na) * g.lo);
+    rows = min(u < g.na ? g.hi : g.lo, g.nrows - r0);
+  } else { r0 = (u - g.nsmall) * R; rows = R; }
+}
+
+
+// K ranges: as many as x capacity demands, and at least enough tasks to
+// balance the grid when the mode can split (partial != nullptr).
+__device__ __forceinline__ int pick_nranges(const Args& a, int nblk) {
+  int ks = 1;
+  while ((long long)a.M * ((a.K + ks - 1) / ks + 31 & ~31) > X_ELEMS) ks++;
+  const int kseg = (a.K + ks - 1) / ks + 31 & ~31;   // K % 32 == 0 keeps every stage in whole ldmatrix pairs
+  return (a.K + kseg - 1) / kseg;
+}
+
+template <int MODE>
+__device__ __forceinline__ const __nv_bfloat16* row_ptr(const Args& a, int row) {
+  if (MODE == GATEUP) {  // logical rows are (gate j, up j) pairs
+    const int j = row >> 1;
+    return a.W + (size_t)((row & 1) ? a.N + j : j) * a.K;
+  }
+  if (MODE == DUAL) return row < a.N1 ? a.W + (size_t)row * a.K : a.W2 + (size_t)(row - a.N1) * a.K;
+  return a.W + (size_t)row * a.K;
+}
+
+// Output (m, row) of one range.  Called by 64 threads (m = ct/8, r = ct%8) of
+// two full warps, so the pair shuffle of GATEUP is convergent.
+template <int MODE>
+__device__ __forceinline__ void emit(const Args& a, int split, int nranges, int m, int row, bool valid, float v) {
+  if (nranges > 1) {  // f32 range partial; the tail folds them
+    if (valid) a.partial[((size_t)split * MMAX + m) * a.N + row] = v;
+    return;
+  }
+  if (MODE == GATEUP) {
+    const float other = __shfl_down_sync(0xffffffffu, v, 1);
+    if (valid && (row & 1) == 0) {
+      const float g = bf16r(v), u = bf16r(other);
+      const float sg = bf16r(g / (1.0f + expf(-g)));
+      a.y[(size_t)m * a.N + (row >> 1)] = __float2bfloat16_rn(sg * u);
+    }
+    return;
+  }
+  if (!valid) return;
+  if (MODE == DUAL) {
+    if (row < a.N1) a.y[(size_t)m * a.N1 + row] = __float2bfloat16_rn(v);
+    else a.y2[(size_t)m * (a.N - a.N1) + row - a.N1] = __float2bfloat16_rn(v);
+  } else {
+    a.y[(size_t)m * a.N + row] = __float2bfloat16_rn(v);
+  }
+}
+
+// The streaming GEMM proper.  Returns (in every thread) whether this CTA
+// finished the grid's last task (tail duty when the grid is oversubscribed).
+template <int MODE>
+__device__ __forceinline__ bool gemm8_core(const Args& a, Smem& s) {
+  const int warp = threadIdx.x / 32, lane = threadIdx.x % 32;
+  const Geom g = task_geom<MODE>(a);
+  const int nrows = g.nrows, nps = g.nps;
+  const int nblk = (nrows + R - 1) / R;
+  const int nranges = pick_nranges(a, nblk);
+  const int kseg = (a.K + nranges - 1) / nranges + 31 & ~31;
+  const int nst0 = (kseg + SEG_MAX - 1) / SEG_MAX;
+  const int seg = (kseg + nst0 - 1) / nst0 + 31 & ~31;          // stage slot width, whole ldmatrix pairs
+  const int srow = seg + PAD, xrow = kseg + PAD;                // smem row strides
+  // x sits at the top of the shared area; whatever it leaves (never less
+  // than RING_ELEMS) is ring depth -- at M = 1 that is 5-6 stages
+  const int xbeg = (BUF_ELEMS - a.M * xrow) & ~63;
+  __nv_bfloat16* const ring = s.buf;
+  __nv_bfloat16* const xs = s.buf + xbeg;
+  const int nstages = min(STAGES_MAX, xbeg / (R * srow));
+  const int ntasks = nps * nranges;
+  if (threadIdx.x == 0) {
+    for (int i = 0; i < nstages; i++) {
+      mbar_init(smem_u32(&s.full[i]), 1);
+      mbar_init(smem_u32(&s.empty[i]), NCW);
+    }
+    mbar_init(smem_u32(&s.xfull), 1);
+    mbar_init(smem_u32(&s.xempty), NCW);
+    fence_barrier_init();
+    s.flag = 0;
+  }
+  __syncthreads();
+  if (warp == 0) {
+    if (lane == 0) {
+      int stage = 0;
+      uint32_t phase = 0, xphase = 0;
+      int cur_split = -1;
+      // Weight stages of this CTA's first task fill the ring before the
+      // upstream dependency resolves; the main loop skips those `pre` stages.
+      int pre = 0;
+      if ((int)blockIdx.x < ntasks) {
+        const int t = blockIdx.x;
+        const int split = t / nps;
+        int r0, rows;
+        task_rows(g, t - split * nps, r0, rows);
+        const int k0 = split * kseg, klen = min(kseg, a.K - k0);
+        for (int off = 0; off < klen && pre < nstages; off += seg, pre++) {
+          const int len = min(seg, klen - off);
+          mbar_wait(smem_u32(&s.empty[stage]), phase ^ 1);
+          mbar_expect_tx(smem_u32(&s.full[stage]), rows * len * 2);
+          __nv_bfloat16* dst = &ring[stage * R * srow];
+          for (int r = 0; r < rows; r++)
+            bulk_copy_1d(smem_u32(dst + r * srow), row_ptr<MODE>(a, r0 + r) + k0 + off, len * 2, smem_u32(&s.full[stage]));
+          if (++stage == nstages) { stage = 0; phase ^= 1; }
+        }
+      }
+      pdl_wait();
+      for (int t = blockIdx.x; t < ntasks; t += gridDim.x) {
+        const int split = t / nps;                                // range-major
+        int r0, rows;
+        task_rows(g, t - split * nps, r0, rows);
+        const int k0 = split * kseg, klen = min(kseg, a.K - k0);
+        if (MODE != SGATE && split != cur_split) {  // stage this range of x (consumers have released the old one)
+          mbar_wait(smem_u32(&s.xempty), xphase);
+          mbar_expect_tx(smem_u32(&s.xfull), a.M * klen * 2);
+          for (int m = 0; m < a.M; m++)
+            bulk_copy_1d(smem_u32(&xs[m * xrow]), a.x + (size_t)m * a.ldx + k0, klen * 2, smem_u32(&s.xfull));
+          xphase ^= 1;
+          cur_split = split;
+        }
+        for (int off = 0; off < klen; off += seg) {
+          if (pre > 0) { pre--; continue; }  // already in flight (first task only)
+          const int len = min(seg, klen - off);
+          mbar_wait(smem_u32(&s.empty[stage]), phase ^ 1);
+          mbar_expect_tx(smem_u32(&s.full[stage]), rows * len * 2);
+          __nv_bfloat16* dst = &ring[stage * R * srow];
+          for (int r = 0; r < rows; r++)
+            bulk_copy_1d(smem_u32(dst + r * srow), row_ptr<MODE>(a, r0 + r) + k0 + off, len * 2, smem_u32(&s.full[stage]));
+          if (++stage == nstages) { stage = 0; phase ^= 1; }
+        }
+      }
+      pdl_trigger();  // our weight stream is issued: the dependent launch may take the SMs we vacate
+    }
+    __syncthreads();  // matches the compute warps' final barrier
+    return s.flag != 0;
+  }
+  const int ct = threadIdx.x - 32, cw = warp - 1;
+  pdl_wait();  // everything from here on touches upstream products or writes
+  int stage = 0;
+  uint32_t phase = 0, xphase = 0;
+  int cur_split = -1;
+  for (int t = blockIdx.x; t < ntasks; t += gridDim.x) {
+    const int split = t / nps;
+    int r0, rows;
+    task_rows(g, t - split * nps, r0, rows);
+    const int klen = min(kseg, a.K - split * kseg);
+    if (split != cur_split) {  // release the old x range, wait for the new one
+      if (MODE == SGATE) {  // the compute warps make this range of x themselves
+        const int k0 = split * kseg, total = a.M * klen;
+        named_bar_sync(BAR_C, CTHREADS);  // everyone is done reading the old range
+        // 8 consecutive elements per thread: klen, head_dim and the batch
+        // stride are all multiples of 8, so a batch never crosses a token row
+        // or a head boundary and both loads and the smem store are one 16 B
+        // vector each (elementwise scalar code here is latency-bound)
+        for (int i0 = ct * 8; i0 < total; i0 += CTHREADS * 8) {
+          const int m = i0 / klen, kk = i0 - m * klen, k = k0 + kk;
+          const int h = k / a.head_dim, hd = k - h * a.head_dim;
+          const uint4 gq = *reinterpret_cast<const uint4*>(a.gate + (size_t)m * a.gate_ts + (size_t)h * a.gate_hs + hd);
+          const uint4 aq = *reinterpret_cast<const uint4*>(a.x + (size_t)m * a.ldx + k);
+          const __nv_bfloat16* gv = reinterpret_cast<const __nv_bfloat16*>(&gq);
+          const __nv_bfloat16* av = reinterpret_cast<const __nv_bfloat16*>(&aq);
+          uint4 oq;
+          __nv_bfloat16* ov = reinterpret_cast<__nv_bfloat16*>(&oq);
+#pragma unroll
+          for (int j = 0; j < 8; j++) {
+            const float g = bf16r(1.0f / (1.0f + expf(-__bfloat162float(gv[j]))));
+            ov[j] = __float2bfloat16_rn(__bfloat162float(av[j]) * g);
+          }
+          *reinterpret_cast<uint4*>(&xs[m * xrow + kk]) = oq;
+        }
+        named_bar_sync(BAR_C, CTHREADS);
+      } else {
+        if (lane == 0) mbar_arrive(smem_u32(&s.xempty));
+        mbar_wait(smem_u32(&s.xfull), xphase);
+        xphase ^= 1;
+      }
+      cur_split = split;
+    }
+    // ldmatrix.x4 row addresses: lane l supplies row l%8 of the 8x8 matrix
+    // (l/8) = k tile-half; matrices (k 0-7, 8-15, 16-23, 24-31) of a 32-k
+    // pair of tiles come back as (b0, b1, b0', b1') / (a0, a2, a0', a2').
+    // x rows beyond M are never emitted: clamp them onto row M-1 (broadcast).
+    const uint32_t xbase = smem_u32(&xs[min(lane & 7, a.M - 1) * xrow + (lane >> 3) * 8]);
+    float c[4][4];
+#pragma unroll
+    for (int j = 0; j < 4; j++) c[j][0] = c[j][1] = c[j][2] = c[j][3] = 0.f;
+    const int wrow = min(lane & 7, rows - 1);  // rows beyond the task: clamp (never emitted)
+    for (int off = 0; off < klen; off += seg) {
+      const int npairs = min(seg, klen - off) / 32;              // len % 32 == 0 (every K is)
+      mbar_wait(smem_u32(&s.full[stage]), phase);
+      const uint32_t wbase = smem_u32(&ring[stage * R * srow + wrow * srow + (lane >> 3) * 8]);
+      const uint32_t xoff = xbase + off * 2;
+      int pr = cw;
+      for (; pr + NCW < npairs; pr += 2 * NCW) {
+        uint32_t a0[4], a2[4], b0[4], b1[4];
+#pragma unroll
+        for (int j = 0; j < 2; j++) {
+          const uint32_t o = (pr + j * NCW) * 64;                // 32 elements * 2 B
+          ldsm_x4(a0[2 * j], a2[2 * j], a0[2 * j + 1], a2[2 * j + 1], xoff + o);
+          ldsm_x4(b0[2 * j], b1[2 * j], b0[2 * j + 1], b1[2 * j + 1], wbase + o);
+        }
+#pragma unroll
+        for (int j = 0; j < 4; j++) mma_bf16_16816(c[j][0], c[j][1], c[j][2], c[j][3], a0[j], 0u, a2[j], 0u, b0[j], b1[j]);
+      }
+      if (pr < npairs) {
+        uint32_t a0[2], a2[2], b0[2], b1[2];
+        ldsm_x4(a0[0], a2[0], a0[1], a2[1], xoff + pr * 64);
+        ldsm_x4(b0[0], b1[0], b0[1], b1[1], wbase + pr * 64);
+        mma_bf16_16816(c[0][0], c[0][1], c[0][2], c[0][3], a0[0], 0u, a2[0], 0u, b0[0], b1[0]);
+        mma_bf16_16816(c[1][0], c[1][1], c[1][2], c[1][3], a0[1], 0u, a2[1], 0u, b0[1], b1[1]);
+      }
+      __syncwarp();
+      if (lane == 0) mbar_arrive(smem_u32(&s.empty[stage]));
+      if (++stage == nstages) { stage = 0; phase ^= 1; }
+    }
+    // (c0, c1) = out[token q = lane/4][rows 2p, 2p+1], p = lane%4
+    s.red[cw][lane >> 2][2 * (lane & 3)] = (c[0][0] + c[1][0]) + (c[2][0] + c[3][0]);
+    s.red[cw][lane >> 2][2 * (lane & 3) + 1] = (c[0][1] + c[1][1]) + (c[2][1] + c[3][1]);
+    named_bar_sync(BAR_C, CTHREADS);
+    if (ct < MMAX * R) {
+      const int m = ct >> 3, r = ct & 7;
+      float v = 0.f;
+#pragma unroll
+      for (int w = 0; w < NCW; w++) v += s.red[w][m][r];
+      emit<MODE>(a, split, nranges, m, r0 + r, m < a.M && r < rows, v);
+    }
+    named_bar_sync(BAR_C, CTHREADS);
+  }
+  pdl_trigger();
+  if (ct == 0) {  // one thread of the compute warps
+    const int my = (ntasks - (int)blockIdx.x + (int)gridDim.x - 1) / (int)gridDim.x;
+    if (my > 0 && a.gcnt) {
+      __threadfence();
+      const int old = atomicAdd(a.gcnt, my);
+      if (old + my == ntasks) { __threadfence(); s.flag = 1; }
+    }
+  }
+  __syncthreads();
+  return s.flag != 0;
+}
+
+// ---- tail: fold range partials, optionally Gemma fused_add_rms_norm ----
+__device__ __forceinline__ int last_pow2(int n) {
+  n |= (n >> 1); n |= (n >> 2); n |= (n >> 4); n |= (n >> 8); n |= (n >> 16);
+  return (n - (n >> 1)) > 0 ? (n - (n >> 1)) : 1;
+}
+// ATen ReduceConfig::set_block_dimension(dim0 = N/4, dim1 = rows), mnt=512.
+__device__ __forceinline__ int aten_block_width(int dim0, int dim1) {
+  const int max_threads = NT_ATEN;
+  int dim0_pow2 = dim0 < max_threads ? last_pow2(dim0) : max_threads;
+  int dim1_pow2 = dim1 < max_threads ? last_pow2(dim1) : max_threads;
+  int block_width = min(dim0_pow2, 32);
+  int block_height = min(dim1_pow2, max_threads / block_width);
+  block_width = min(dim0_pow2, max_threads / block_height);
+  return block_width;
+}
+// Sum of z[j]^2 over j in [0, N) in ATen's order (sq[j] = fmul_rn(z, z), then
+// the vectorized-by-4 reduction of kern_gemma_rms_norm.cu) for W virtual
+// lanes; the result is returned in every thread.  (Folding the W lanes in
+// the registers of one warp instead costs 40 registers and a local-memory
+// tree in every add_norm variant, and the M = 8 verify pass pays 3 us per
+// launch for it; the smem tree is the same additions in the same order.)
+__device__ float aten_row_sumsq(const float* __restrict__ z, int N, int W, float* __restrict__ s) {
+  const int t = threadIdx.x;
+  constexpr int VL = (NT_ATEN + NTHREADS - 1) / NTHREADS;
+  if (t < 32) s[t] = 0.f;
+  __syncthreads();
+#pragma unroll
+  for (int v = 0; v < VL; v++) {
+    const int l = t + v * NTHREADS;
+    if (l < W) {
+      float acc[4] = {0.f, 0.f, 0.f, 0.f};
+      for (int idx = l; idx * 4 + 3 < N; idx += W) {
+        const float4 x4 = *reinterpret_cast<const float4*>(z + idx * 4);
+        acc[0] += __fmul_rn(x4.x, x4.x); acc[1] += __fmul_rn(x4.y, x4.y);
+        acc[2] += __fmul_rn(x4.z, x4.z); acc[3] += __fmul_rn(x4.w, x4.w);
+      }
+      s[l] = ((acc[0] + acc[1]) + acc[2]) + acc[3];
+    }
+  }
+  __syncthreads();
+  for (int off = W / 2; off >= 32; off >>= 1) {
+#pragma unroll
+    for (int v = 0; v < VL; v++) {
+      const int l = t + v * NTHREADS;
+      if (l < off && l + off < W) s[l] += s[l + off];
+    }
+    __syncthreads();
+  }
+  float value = t < 32 ? s[t] : 0.f;
+  if (t < 32) {
+    for (int off = 16; off > 0; off >>= 1) value += __shfl_down_sync(0xffffffffu, value, off);
+    if (t == 0) s[0] = value;
+  }
+  __syncthreads();
+  return s[0];
+}
+
+// The f32 GEMM value of elements [j8, j8+4) of row m: the folded range
+// partials, or the y scratch when the GEMM ran unsplit.
+__device__ __forceinline__ float4 ysum4(const Args& a, int nranges, int m, int j) {
+  if (nranges == 1) {
+    const __nv_bfloat162 lo = __ldcg(reinterpret_cast<const __nv_bfloat162*>(a.y + (size_t)m * a.N + j));
+    const __nv_bfloat162 hi = __ldcg(reinterpret_cast<const __nv_bfloat162*>(a.y + (size_t)m * a.N + j + 2));
+    const float2 l = __bfloat1622float2(lo), h = __bfloat1622float2(hi);
+    return make_float4(l.x, l.y, h.x, h.y);
+  }
+  // guarded full unroll: all range loads issue before the adds need them
+  float4 p[8];
+#pragma unroll
+  for (int sp = 0; sp < 8; sp++)
+    if (sp < nranges) p[sp] = __ldcg(reinterpret_cast<const float4*>(a.partial + ((size_t)sp * MMAX + m) * a.N + j));
+  float4 v = p[0];
+#pragma unroll
+  for (int sp = 1; sp < 8; sp++)
+    if (sp < nranges) { v.x += p[sp].x; v.y += p[sp].y; v.z += p[sp].z; v.w += p[sp].w; }
+  return v;
+}
+
+// Rows [m0, m1) of the tail.  NORM: vLLM's Gemma fused_add_rms_norm with
+// ATen's reduction width for the launch's M rows; z lives in smem (ring).
+// PLAIN: just fold the partials to bf16 y.
+// The tail is latency, not bandwidth: every global round trip of a row is
+// issued for all of the thread's groups at once (TG = 6 float4 per thread
+// covers N <= 6144), and w1 is already in smem (loaded during the wait).
+constexpr int TG = (6144 / 4 + NTHREADS - 1) / NTHREADS;
+template <int MODE>
+__device__ void tail_rows(const Args& a, int nranges, int m0, int m1, const float* __restrict__ w1s, __nv_bfloat16* res,
+                          __nv_bfloat16* out, float eps, float* __restrict__ z, float* __restrict__ s) {
+  const int N = a.N;
+  for (int m = m0; m < m1; m++) {
+    float4 v[TG];
+    float2 r0[TG], r1[TG];
+    if (MODE >= NORM) {  // the residual pair of every group, issued as one batch
+      uint32_t rw[TG][2];
+#pragma unroll
+      for (int gi = 0; gi < TG; gi++) {
+        const int j = (threadIdx.x + gi * NTHREADS) * 4;
+        const uint32_t* rr = reinterpret_cast<const uint32_t*>(res + (size_t)m * N + j);
+        rw[gi][0] = j < N ? rr[0] : 0u;
+        rw[gi][1] = j < N ? rr[1] : 0u;
+      }
+#pragma unroll
+      for (int gi = 0; gi < TG; gi++) {
+        r0[gi] = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&rw[gi][0]));
+        r1[gi] = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&rw[gi][1]));
+      }
+    }
+    if (nranges == 1) {  // y scratch, bf16: one batch of plain loads
+      uint32_t yw[TG][2];
+#pragma unroll
+      for (int gi = 0; gi < TG; gi++) {
+        const int j = (threadIdx.x + gi * NTHREADS) * 4;
+        const uint32_t* yy = reinterpret_cast<const uint32_t*>(a.y + (size_t)m * N + j);
+        yw[gi][0] = j < N ? __ldcg(yy) : 0u;
+        yw[gi][1] = j < N ? __ldcg(yy + 1) : 0u;
+      }
+#pragma unroll
+      for (int gi = 0; gi < TG; gi++) {
+        const float2 l = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&yw[gi][0]));
+        const float2 h = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&yw[gi][1]));
+        v[gi] = make_float4(l.x, l.y, h.x, h.y);
+      }
+    } else {
+#pragma unroll
+      for (int gi = 0; gi < TG; gi++) {
+        const int j = (threadIdx.x + gi * NTHREADS) * 4;
+        if (j < N) v[gi] = ysum4(a, nranges, m, j);
+      }
+    }
+#pragma unroll
+    for (int gi = 0; gi < TG; gi++) {
+      const int j = (threadIdx.x + gi * NTHREADS) * 4;
+      if (j >= N) continue;
+      float4 y = v[gi];
+      if (MODE >= NORM) {
+        __nv_bfloat16* rr = res + (size_t)m * N + j;
+        y.x = bf16r(y.x); y.y = bf16r(y.y); y.z = bf16r(y.z); y.w = bf16r(y.w);   // the bf16 y rounding
+        y.x = __fadd_rn(y.x, r0[gi].x); y.y = __fadd_rn(y.y, r0[gi].y);
+        y.z = __fadd_rn(y.z, r1[gi].x); y.w = __fadd_rn(y.w, r1[gi].y);
+        *reinterpret_cast<__nv_bfloat162*>(rr) = __floats2bfloat162_rn(y.x, y.y);
+        *reinterpret_cast<__nv_bfloat162*>(rr + 2) = __floats2bfloat162_rn(y.z, y.w);
+        *reinterpret_cast<float4*>(z + j) = y;
+      } else {
+        __nv_bfloat16* yy = a.y + (size_t)m * N + j;
+        *reinterpret_cast<__nv_bfloat162*>(yy) = __floats2bfloat162_rn(y.x, y.y);
+        *reinterpret_cast<__nv_bfloat162*>(yy + 2) = __floats2bfloat162_rn(y.z, y.w);
+      }
+    }
+    if (MODE < NORM) continue;
+    __syncthreads();
+    const int W = aten_block_width(N / 4, a.M);
+    const float sum = aten_row_sumsq(z, N, W, s);
+    const float factor = (float)a.M / (float)((long long)a.M * (long long)N);
+    const float var = __fmul_rn(sum, factor);
+    const float r = rsqrtf(__fadd_rn(var, eps));
+#pragma unroll
+    for (int gi = 0; gi < TG; gi++) {
+      const int j = (threadIdx.x + gi * NTHREADS) * 4;
+      if (j >= N) continue;
+      const float4 zv = *reinterpret_cast<const float4*>(z + j);
+      const float4 w0 = *reinterpret_cast<const float4*>(w1s + j);
+      __nv_bfloat162 o0 = __floats2bfloat162_rn(__fmul_rn(__fmul_rn(zv.x, r), w0.x), __fmul_rn(__fmul_rn(zv.y, r), w0.y));
+      __nv_bfloat162 o1 = __floats2bfloat162_rn(__fmul_rn(__fmul_rn(zv.z, r), w0.z), __fmul_rn(__fmul_rn(zv.w, r), w0.w));
+      *reinterpret_cast<__nv_bfloat162*>(out + (size_t)m * N + j) = o0;
+      *reinterpret_cast<__nv_bfloat162*>(out + (size_t)m * N + j + 2) = o1;
+    }
+    __syncthreads();
+  }
+}
+
+// Tail driver: cooperative one-row-per-CTA when the whole grid is resident,
+// else all rows in the CTA that finished last.
+template <int MODE>
+__device__ __forceinline__ void run_tail(const Args& a, Smem& s, bool last, __nv_bfloat16* res, const float* w1,
+                                         __nv_bfloat16* out, float eps) {
+  const Geom g = task_geom<MODE>(a);
+  const int nblk = (g.nrows + R - 1) / R;
+  const int nranges = pick_nranges(a, nblk);
+  const int ntasks = g.nps * nranges;
+  if (MODE == PLAIN && nranges == 1) return;  // nothing left to do
+  static_assert(RING_ELEMS * 2 >= 2 * 6144 * 4, "z row + w1 copy: N <= 6144");
+  float* z = reinterpret_cast<float*>(&s.buf[0]);
+  float* w1s = z + 6144;
+  if (gridDim.x <= nsmid()) {
+    if ((int)blockIdx.x >= a.M) return;
+    if (MODE >= NORM)  // the norm weight is not waiting on anyone: fetch it under the spin
+      for (int j = threadIdx.x * 4; j < a.N; j += NTHREADS * 4) *reinterpret_cast<float4*>(w1s + j) = *reinterpret_cast<const float4*>(w1 + j);
+    if (threadIdx.x == 0)
+      while (ld_acquire(a.gcnt) < ntasks) __nanosleep(64);
+    __syncthreads();
+    tail_rows<MODE>(a, nranges, blockIdx.x, blockIdx.x + 1, w1s, res, out, eps, z, &s.red[0][0][0]);
+    __syncthreads();
+    if (threadIdx.x == 0 && atomicAdd(a.gcnt + 1, 1) == a.M - 1) {
+      a.gcnt[1] = 0;
+      a.gcnt[0] = 0;
+      __threadfence();
+    }
+  } else if (last) {
+    __threadfence();
+    if (MODE >= NORM)
+      for (int j = threadIdx.x * 4; j < a.N; j += NTHREADS * 4) *reinterpret_cast<float4*>(w1s + j) = *reinterpret_cast<const float4*>(w1 + j);
+    __syncthreads();
+    tail_rows<MODE>(a, nranges, 0, a.M, w1s, res, out, eps, z, &s.red[0][0][0]);
+    __syncthreads();
+    if (threadIdx.x == 0) { *a.gcnt = 0; __threadfence(); }
+  }
+}
+}  // namespace
+
+#define GEMM8_SMEM                                                                                     \
+  extern __shared__ char smem_raw[];                                                                  \
+  Smem& s = *reinterpret_cast<Smem*>((reinterpret_cast<uintptr_t>(smem_raw) + 127) & ~uintptr_t(127))
+
+extern "C" __global__ void __launch_bounds__(NTHREADS, 1)
+kern_gemm8_bf16(__nv_bfloat16* __restrict__ y, const __nv_bfloat16* __restrict__ x, const __nv_bfloat16* __restrict__ W,
+                float* __restrict__ partial, int* __restrict__ gcnt, int M, int N, int K) {
+  GEMM8_SMEM;
+  Args a{x, W, nullptr, y, nullptr, partial, gcnt, M, N, 0, K, K};
+  const bool last = gemm8_core<PLAIN>(a, s);
+  run_tail<PLAIN>(a, s, last, nullptr, nullptr, nullptr, 0.f);
+}
+
+extern "C" __global__ void __launch_bounds__(NTHREADS, 1)
+kern_gemm8_gateup_silu_bf16(__nv_bfloat16* __restrict__ act, const __nv_bfloat16* __restrict__ x,
+                            const __nv_bfloat16* __restrict__ W, int M, int F, int K) {
+  GEMM8_SMEM;
+  Args a{x, W, nullptr, act, nullptr, nullptr, nullptr, M, F, 0, K, K};   // M*K <= X_ELEMS: never splits
+  gemm8_core<GATEUP>(a, s);
+}
+
+extern "C" __global__ void __launch_bounds__(NTHREADS, 1)
+kern_gemm8_dual_bf16(__nv_bfloat16* __restrict__ y1, __nv_bfloat16* __restrict__ y2, const __nv_bfloat16* __restrict__ x,
+                     const __nv_bfloat16* __restrict__ W1, const __nv_bfloat16* __restrict__ W2, int M, int N1, int N2, int K) {
+  GEMM8_SMEM;
+  Args a{x, W1, W2, y1, y2, nullptr, nullptr, M, N1 + N2, N1, K, K};      // M*K <= X_ELEMS: never splits
+  gemm8_core<DUAL>(a, s);
+}
+
+extern "C" __global__ void __launch_bounds__(NTHREADS, 1)
+kern_gemm8_sgate_add_norm_bf16(__nv_bfloat16* __restrict__ out, const __nv_bfloat16* __restrict__ attn,
+                               const __nv_bfloat16* __restrict__ gate, const __nv_bfloat16* __restrict__ W,
+                               __nv_bfloat16* __restrict__ res, const float* __restrict__ w1, __nv_bfloat16* __restrict__ y,
+                               float* __restrict__ partial, int* __restrict__ gcnt, int M, int N, int K, int head_dim,
+                               int gate_tstride, int gate_hstride, float eps) {
+  GEMM8_SMEM;
+  Args a{attn, W, nullptr, y, nullptr, partial, gcnt, M, N, 0, K, K, gate, head_dim, gate_tstride, gate_hstride};
+  const bool last = gemm8_core<SGATE>(a, s);
+  run_tail<SGATE>(a, s, last, res, w1, out, eps);
+}
+
+extern "C" __global__ void __launch_bounds__(NTHREADS, 1)
+kern_gemm8_add_norm_bf16(__nv_bfloat16* __restrict__ out, const __nv_bfloat16* __restrict__ x, const __nv_bfloat16* __restrict__ W,
+                         __nv_bfloat16* __restrict__ res, const float* __restrict__ w1, __nv_bfloat16* __restrict__ y,
+                         float* __restrict__ partial, int* __restrict__ gcnt, int M, int N, int K, float eps) {
+  GEMM8_SMEM;
+  Args a{x, W, nullptr, y, nullptr, partial, gcnt, M, N, 0, K, K};
+  const bool last = gemm8_core<NORM>(a, s);
+  run_tail<NORM>(a, s, last, res, w1, out, eps);
+}