fix(ci): repair lazy CUDA carrier dependencies and reuse audited compilation

.github/model-e2e/release_contracts.py

@@ -21,6 +21,14 @@ def verify_release(root, expected_digest):
         "Invalid attempt",
     )
     sha(manifest["workflow_sha"])
+    if "assembly_workflow_sha" in manifest:
+        sha(manifest["assembly_workflow_sha"])
+        recovery = manifest["source_lock"].get("recovery")
+        if recovery:
+            require(recovery["assembly_workflow_sha"] == manifest["assembly_workflow_sha"], "Recovery assembler mismatch")
+            require(recovery["workflow_sha"] == manifest["workflow_sha"], "Recovery source mismatch")
+        else:
+            require(manifest["assembly_workflow_sha"] == manifest["workflow_sha"], "Different assembler without recovery provenance")
     lock = manifest["source_lock"]
     require(lock["workflow_sha"] == manifest["workflow_sha"], "Workflow lock mismatch")
     require(set(lock["sources"]) == set(REPOSITORIES), "Need four main sources")

.github/release/README.md

@@ -1,4 +1,4 @@
-# 四仓 main 一键发布（Draft，尚未启用）
+# 四仓 main 一键发布
 
 入口：**KTransformers → Actions → Release four-main stack → Run workflow**。
 选择 `main`，`target=build` 只产出候选 artifact；`target=candidate` 构建和验收；
@@ -10,6 +10,23 @@
 `KT_RELEASE_WORK_ROOT` 为可执行的空闲内存盘目录，避免填满 runner 磁盘；
 构建和临时文件都进入该目录下本次创建、带 run ID 标记的独立子目录。
 
+### 仅打包失败时，复用已经完成的原生编译
+
+如果原生编译和源码/哈希审计已成功，但后续修复或打包失败，使用
+`target=build`、`reuse_raw_run_id=<原构建 run ID>`、`reuse_raw_attempt=<原 attempt>`。
+CI 会从该 run 的不可变 artifacts 恢复六个 raw wheels，逐个核对原审计的
+SHA256、版本和依赖，再重新修复、打包与封存，不重新编译 CUDA。
+
+恢复保留原来的四仓 main 源码快照，单独记录新的 `assembly_workflow_sha`。
+仅允许原构建是官方 main 的已完成 run，且 KT 后续差异全部在 `.github/` 下；
+若模型、内核或包元数据发生变化，则必须重新构建。取消、编译失败、混合文件、
+已修改的 raw wheels 和恢复链均会拒绝。重新封存后的最终 wheels 必须重新进行
+完整验收，原 raw wheels 永远不能直接发布。
+
+CUDA 大库由固定版本的 Torch/NVIDIA 依赖提供，不重复嵌入。auditwheel 携带的
+小型系统库同时放入延迟加载缓存，ELF RPATH 指向缓存内的 `.libs`；代码、SASS
+架构和 Python loader 不变，重定位前后哈希及最终二进制架构均记录在构建证据中。
+
 ### 模型验收 runner 未就绪时的人工接管
 
 `Promote manually validated CI wheels` 只接管上传，不重新编译、不执行 Kimi

.github/release/carriers.py

@@ -1,8 +1,8 @@
 """Assemble fresh five-project carriers without version changes or source overlays.
 
-Only WHEEL/RECORD and the generated CUDA payload manifest are rewritten. Runtime
-files (including SM90 and licenses) are retained. Reject oversized wheels instead
-of deleting architectures. Native inputs must first pass auditwheel repair.
+Only wheel packaging metadata and cache-relative ELF RPATHs are rewritten.
+Python runtime files, CUDA code, SM90 and licenses are retained. Reject oversized
+wheels instead of deleting architectures. Native inputs require auditwheel repair.
 """
 
 from __future__ import annotations
@@ -137,17 +137,18 @@ def retag(root, python, abi):
     )
 
 
-def archive_payload(sgl, output):
+def archive_payload(sgl, output, libraries=None):
     hashes = {}
+    sources = {name: sgl / "sgl_kernel" / name for name in LARGE}
+    sources.update({".libs/" + name: path for name, path in (libraries or {}).items()})
     with (
         output.open("wb") as raw,
         gzip.GzipFile(
             filename="", mode="wb", fileobj=raw, mtime=0, compresslevel=9
         ) as compressed,
     ):
         with tarfile.open(fileobj=compressed, mode="w|") as archive:
-            for name in LARGE:
-                source = sgl / "sgl_kernel" / name
+            for name, source in sources.items():
                 hashes[name] = sha256(source)
                 info = archive.gettarinfo(str(source), arcname=name)
                 info.uid = info.gid = info.mtime = 0
@@ -157,6 +158,33 @@ def archive_payload(sgl, output):
     return hashes
 
 
+def relocate_lazy_dependencies(sgl):
+    """Keep auditwheel's private small dependencies next to cached ELF files.
+
+    The main loader already materializes all manifest files. No Python loader
+    overlay is needed: only adjust ELF RPATH as a normal wheel-packaging step.
+    Directly loaded SM90/deep_gemm objects keep auditwheel's original layout.
+    """
+    libraries = {}
+    for path in sgl.rglob("*.so*"):
+        if path.is_file() and any(part.endswith(".libs") for part in path.parts):
+            require(path.name not in libraries, "Colliding bundled library names")
+            libraries[path.name] = path
+    changes = []
+    require(not any(name.startswith(("libcublas", "libnvrtc", "libnvJitLink")) for name in libraries),
+            "NVIDIA libraries must be supplied by the pinned Torch dependencies")
+    for name in LARGE:
+        path = sgl / "sgl_kernel" / name
+        dynamic = subprocess.check_output(["readelf", "--dynamic", str(path)], text=True)
+        needed = set(re.findall(r"Shared library: \[([^]]+)\]", dynamic))
+        if needed & libraries.keys():
+            before = sha256(path)
+            rpath = "$ORIGIN/../.libs" if "/" in name else "$ORIGIN/.libs"
+            subprocess.run(["patchelf", "--set-rpath", rpath, str(path)], check=True)
+            changes.append({"file": name, "operation": "set-rpath", "rpath": rpath, "before_sha256": before, "after_sha256": sha256(path)})
+    return libraries, changes
+
+
 def binary_evidence(roots):
     evidence = {}
     for package, root in roots.items():
@@ -222,33 +250,16 @@ def assemble(raw, output, evidence_dir):
         roots = {name: temp / name for name in by_name}
         for name, entry in by_name.items():
             unpack(entry["path"], roots[name])
-        save_json(evidence_dir / "cuda-binaries.json", binary_evidence(roots))
         sgl = roots["sgl-kernel-kt"]
-        # Lazy objects are extracted into a cache, not site-packages. An
-        # auditwheel-renamed dependency resolved via $ORIGIN would break there.
-        # Torch/CUDA-runtime libraries are supplied by the pinned torch wheel;
-        # reject any other bundled dependency rather than emitting a broken wheel.
-        bundled = {
-            path.name
-            for path in sgl.rglob("*.so*")
-            if any(part.endswith(".libs") for part in path.parts)
-        }
-        for name in LARGE:
-            dynamic = subprocess.check_output(
-                ["readelf", "--dynamic", str(sgl / "sgl_kernel" / name)], text=True
-            )
-            needed = set(re.findall(r"Shared library: \[([^]]+)\]", dynamic))
-            require(
-                not (needed & bundled),
-                "Lazy payload depends on a relocated auditwheel library; fix the main build linkage",
-            )
+        libraries, relocations = relocate_lazy_dependencies(sgl)
+        save_json(evidence_dir / "cuda-binaries.json", binary_evidence(roots))
         for name in ("payload_runtime.py", "load_utils.py", "flash_attn.py"):
             require(
                 (sgl / "sgl_kernel" / name).is_file(),
                 "Locked SGL main lacks the carrier loader: " + name,
             )
         archive = temp / "payload.tar.gz"
-        hashes = archive_payload(sgl, archive)
+        hashes = archive_payload(sgl, archive, libraries)
         # Only the two objects supported by main's lazy loader are externalized.
         # No SM90 deletion or copied Python implementation from another checkout.
         for name in LARGE:
@@ -374,6 +385,8 @@ def assemble(raw, output, evidence_dir):
                 "payload_files": hashes,
                 "parts": parts,
                 "direct_native_carriers": {"ktransformers": direct_files},
+                "elf_packaging_relocations": relocations,
+                "lazy_dependency_files": sorted(".libs/" + name for name in libraries),
                 "runtime_overlays": [],
                 "version_overrides": [],
                 "dependency_overrides": [],

.github/release/manual_promote.py

@@ -40,6 +40,7 @@ def check_attestation(data, manifest, digest):
     require(data.get("wheels") == manifest["wheels"], "Different accepted wheel bytes")
     require(data.get("candidate_run_id") == manifest["run_id"], "Different build run")
     require(data.get("candidate_attempt") == manifest["run_attempt"], "Different build attempt")
+    require(data.get("assembly_workflow_sha", data["source_lock"].get("workflow_sha")) == manifest.get("assembly_workflow_sha", manifest.get("workflow_sha")), "Different assembler revision")
     for host in ("sap4", "qj5090"):
         for extra in ("sglang", "sglang,sft"):
             verify_install_report(data["install_reports"][host][extra], manifest, extra, public=False)
@@ -89,7 +90,7 @@ def fetch():
     require(hashlib.sha256(content).hexdigest() == digest, "Acceptance record changed")
     data = json.loads(content)
     require(data["candidate_run_id"] == int(run_id) and data["candidate_attempt"] == int(attempt), "Acceptance belongs to another build")
-    require(data["source_lock"]["workflow_sha"] == run["head_sha"], "Different source revision")
+    require(data.get("assembly_workflow_sha", data["source_lock"]["workflow_sha"]) == run["head_sha"], "Different assembler revision")
     save_json(Path("accepted.json"), data)
     save_json(Path("build-run.json"), run)
 

.github/release/recover_native.py

@@ -0,0 +1,96 @@
+"""Reuse hash-verified CI raw wheels after a packaging-only failure, never rebuild.
+
+The original four-main source snapshot remains frozen. A different trusted CI
+assembler revision is explicit; recovery is refused after any KT runtime change.
+"""
+
+import argparse
+import json
+import os
+from pathlib import Path
+import re
+import urllib.request
+
+from four_main import inspect_wheel, read_lock, save_json
+
+REPO = "kvcache-ai/ktransformers"
+
+
+def require(value, message):
+    if not value:
+        raise ValueError(message)
+
+
+def api(path):
+    request = urllib.request.Request("https://api.github.com/repos/" + REPO + path,
+                                     headers={"Authorization": "Bearer " + os.environ["GH_TOKEN"], "Accept": "application/vnd.github+json"})
+    with urllib.request.urlopen(request, timeout=60) as response:
+        return json.load(response)
+
+
+def check_run(run, jobs, comparison):
+    require(run["event"] == "workflow_dispatch" and run["head_branch"] == "main", "Only official main builds may be recovered")
+    require(run["path"] == ".github/workflows/release-four-main.yml" and run["status"] == "completed", "Original build must be complete")
+    require(run["conclusion"] in ("success", "failure"), "Do not recover canceled or unfinished compilation")
+    completed = {step["name"] for job in jobs for step in job.get("steps", []) if step["conclusion"] == "success"}
+    required = {"Compile all KT CPU variants and CUDA architectures", "Compile SGL CUDA payload from the independently locked SGLang main", "Inspect wheel metadata and source provenance"}
+    require(required <= completed, "Original native compilation and source audit must have succeeded")
+    require(comparison["status"] in ("ahead", "identical"), "Original build is not a main ancestor")
+    files = comparison["files"]
+    require(len(files) < 300, "Refuse a truncated compare response")
+    require(all(item["filename"].startswith(".github/") and item.get("previous_filename", item["filename"]).startswith(".github/") for item in files), "Runtime sources changed; compile a fresh stack instead")
+
+
+def preflight(run_id, attempt):
+    require(os.environ.get("GITHUB_REPOSITORY") == REPO and os.environ.get("GITHUB_REF") == "refs/heads/main", "Official main only")
+    require(run_id.isdecimal() and attempt.isdecimal() and int(attempt) > 0, "Invalid original run")
+    run = api("/actions/runs/" + run_id)
+    require(run["run_attempt"] == int(attempt), "Original attempt changed")
+    jobs = api(f"/actions/runs/{run_id}/attempts/{attempt}/jobs?per_page=100")
+    require(jobs["total_count"] <= 100, "Too many jobs to validate")
+    compare = api("/compare/" + run["head_sha"] + "..." + os.environ["GITHUB_SHA"])
+    check_run(run, jobs["jobs"], compare)
+    save_json(Path("recovery-source.json"), {"run_id": int(run_id), "run_attempt": int(attempt), "workflow_sha": run["head_sha"], "assembly_workflow_sha": os.environ["GITHUB_SHA"], "ci_only_diff": [item["filename"] for item in compare["files"]]})
+
+
+def check_lock(path):
+    lock = read_lock(path)
+    recovery = json.loads(Path("recovery-source.json").read_text())
+    require(lock["workflow_sha"] == lock["sources"]["ktransformers"]["sha"] == recovery["workflow_sha"], "Original source lock differs from the compiled workflow")
+    require("recovery" not in lock, "Recover only original compile runs, not recovery chains")
+    lock["recovery"] = recovery
+    save_json(path, lock)
+
+
+def verify_inputs(lock, report, wheels):
+    recovery = lock["recovery"]
+    expected_lock = {key: value for key, value in lock.items() if key != "recovery"}
+    require(report["source_lock"] == expected_lock, "Raw wheels belong to another four-main snapshot")
+    require(report["stage"] == "raw-native-wheels" and report["metadata_consistent"] is True and not report["errors"], "Original source audit failed")
+    require(re.fullmatch(r"[0-9a-f]{40}", recovery["assembly_workflow_sha"]), "Invalid assembler SHA")
+    expected = {item["filename"]: item for item in report["wheels"]}
+    actual = {item["filename"]: item for item in wheels}
+    require(len(expected) == len(report["wheels"]) == 6 and set(actual) == set(expected), "Require exactly the six original raw wheels")
+    for name, item in actual.items():
+        require(all(item[key] == expected[name][key] for key in ("name", "version", "sha256", "size", "requires_dist")), "Raw wheel changed: " + name)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("command", choices=("preflight", "lock", "inputs"))
+    parser.add_argument("--run-id", default="")
+    parser.add_argument("--attempt", default="1")
+    parser.add_argument("--lock", type=Path, default=Path("source-lock.json"))
+    parser.add_argument("--report", type=Path)
+    parser.add_argument("--wheels", type=Path)
+    args = parser.parse_args()
+    if args.command == "preflight":
+        preflight(args.run_id, args.attempt)
+    elif args.command == "lock":
+        check_lock(args.lock)
+    else:
+        verify_inputs(read_lock(args.lock), json.loads(args.report.read_text()), [inspect_wheel(path) for path in sorted(args.wheels.glob("*.whl"))])
+        import torch
+        original_torch = (args.report.parent / "torch.txt").read_text().strip()
+        require(original_torch == f"{torch.__version__} {torch.version.cuda}", "Recovery Torch/CUDA ABI differs from original compilation")
+        require("release 12.8" in (args.report.parent / "nvcc.txt").read_text(), "Original CUDA toolkit is not 12.8")

.github/release/release_stack.py

@@ -118,6 +118,9 @@ def make_release(final, root, lock_path, evidence):
         "run_id": int(os.environ["GITHUB_RUN_ID"]),
         "run_attempt": int(os.environ["GITHUB_RUN_ATTEMPT"]),
         "workflow_sha": lock["workflow_sha"],
+        # A packaging-only recovery can use a newer trusted CI implementation
+        # while preserving the original compiled four-main runtime snapshot.
+        "assembly_workflow_sha": os.environ["GITHUB_SHA"],
         "source_lock": lock,
         "wheels": {
             entry["name"]: {

.github/release/repair_native.py

@@ -0,0 +1,41 @@
+"""Auditwheel repair with CUDA dependencies provided by the pinned Torch stack."""
+
+import argparse
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+
+from four_main import inspect_wheel, save_json
+
+# Do not embed another copy of NVIDIA's large libraries or give them private
+# auditwheel SONAMEs. Torch 2.9.1 pins their corresponding nvidia-* wheels.
+EXTERNAL = (
+    "libcuda.so.1", "libtorch.so", "libtorch_cpu.so", "libtorch_cuda.so",
+    "libtorch_python.so", "libc10.so", "libc10_cuda.so", "libcudart.so.12",
+    "libcublas.so.12", "libcublasLt.so.12", "libnvrtc.so.12",
+    "libnvrtc-builtins.so.12.8", "libnvJitLink.so.12",
+)
+
+
+def repair(raw, destination, evidence):
+    destination.mkdir(exist_ok=False)
+    for path in sorted(raw.glob("*.whl")):
+        entry = inspect_wheel(path)
+        if entry["name"] in {"kt-kernel", "sgl-kernel-kt"}:
+            command = [sys.executable, "-m", "auditwheel", "repair", str(path), "--plat", "manylinux_2_35_x86_64"]
+            for soname in EXTERNAL:
+                command += ["--exclude", soname]
+            subprocess.run(command + ["-w", str(destination)], check=True)
+        else:
+            shutil.copyfile(path, destination / path.name)
+    save_json(evidence / "native-repair.json", {"external_libraries": list(EXTERNAL), "provider": "pinned torch==2.9.1 CUDA 12.8 dependencies; NVIDIA driver supplies libcuda"})
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--raw", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("--evidence", type=Path, required=True)
+    args = parser.parse_args()
+    repair(args.raw, args.output, args.evidence)