From df73c179b6f3426158e8b6503dd6931c28c949b8 Mon Sep 17 00:00:00 2001 From: yyj Date: Fri, 11 Sep 2026 03:11:43 +0800 Subject: [PATCH] fix(ci): repair lazy native dependencies and recover verified build artifacts --- .github/model-e2e/release_contracts.py | 8 +++ .github/release/README.md | 19 ++++- .github/release/carriers.py | 65 ++++++++++------- .github/release/manual_promote.py | 3 +- .github/release/recover_native.py | 96 +++++++++++++++++++++++++ .github/release/release_stack.py | 3 + .github/release/repair_native.py | 41 +++++++++++ .github/release/test_carriers.py | 69 ++++++++++++++++++ .github/release/test_manual_promote.py | 9 +++ .github/release/test_recover_native.py | 55 ++++++++++++++ .github/workflows/release-four-main.yml | 81 +++++++++++++++------ 11 files changed, 399 insertions(+), 50 deletions(-) create mode 100644 .github/release/recover_native.py create mode 100644 .github/release/repair_native.py create mode 100644 .github/release/test_recover_native.py diff --git a/.github/model-e2e/release_contracts.py b/.github/model-e2e/release_contracts.py index 088c92b07..74beb4532 100644 --- a/.github/model-e2e/release_contracts.py +++ b/.github/model-e2e/release_contracts.py @@ -21,6 +21,14 @@ def verify_release(root, expected_digest): "Invalid attempt", ) sha(manifest["workflow_sha"]) + if "assembly_workflow_sha" in manifest: + sha(manifest["assembly_workflow_sha"]) + recovery = manifest["source_lock"].get("recovery") + if recovery: + require(recovery["assembly_workflow_sha"] == manifest["assembly_workflow_sha"], "Recovery assembler mismatch") + require(recovery["workflow_sha"] == manifest["workflow_sha"], "Recovery source mismatch") + else: + require(manifest["assembly_workflow_sha"] == manifest["workflow_sha"], "Different assembler without recovery provenance") lock = manifest["source_lock"] require(lock["workflow_sha"] == manifest["workflow_sha"], "Workflow lock mismatch") require(set(lock["sources"]) == set(REPOSITORIES), "Need four main sources") diff --git a/.github/release/README.md b/.github/release/README.md index 09ab17d82..4aee0c78d 100644 --- a/.github/release/README.md +++ b/.github/release/README.md @@ -1,4 +1,4 @@ -# 四仓 main 一键发布（Draft，尚未启用） +# 四仓 main 一键发布 入口：**KTransformers → Actions → Release four-main stack → Run workflow**。 选择 `main`，`target=build` 只产出候选 artifact；`target=candidate` 构建和验收； @@ -10,6 +10,23 @@ `KT_RELEASE_WORK_ROOT` 为可执行的空闲内存盘目录，避免填满 runner 磁盘； 构建和临时文件都进入该目录下本次创建、带 run ID 标记的独立子目录。 +### 仅打包失败时，复用已经完成的原生编译 + +如果原生编译和源码/哈希审计已成功，但后续修复或打包失败，使用 +`target=build`、`reuse_raw_run_id=<原构建 run ID>`、`reuse_raw_attempt=<原 attempt>`。 +CI 会从该 run 的不可变 artifacts 恢复六个 raw wheels，逐个核对原审计的 +SHA256、版本和依赖，再重新修复、打包与封存，不重新编译 CUDA。 + +恢复保留原来的四仓 main 源码快照，单独记录新的 `assembly_workflow_sha`。 +仅允许原构建是官方 main 的已完成 run，且 KT 后续差异全部在 `.github/` 下； +若模型、内核或包元数据发生变化，则必须重新构建。取消、编译失败、混合文件、 +已修改的 raw wheels 和恢复链均会拒绝。重新封存后的最终 wheels 必须重新进行 +完整验收，原 raw wheels 永远不能直接发布。 + +CUDA 大库由固定版本的 Torch/NVIDIA 依赖提供，不重复嵌入。auditwheel 携带的 +小型系统库同时放入延迟加载缓存，ELF RPATH 指向缓存内的 `.libs`；代码、SASS +架构和 Python loader 不变，重定位前后哈希及最终二进制架构均记录在构建证据中。 + ### 模型验收 runner 未就绪时的人工接管 `Promote manually validated CI wheels` 只接管上传，不重新编译、不执行 Kimi diff --git a/.github/release/carriers.py b/.github/release/carriers.py index 0df30a894..28e16fe5b 100644 --- a/.github/release/carriers.py +++ b/.github/release/carriers.py @@ -1,8 +1,8 @@ """Assemble fresh five-project carriers without version changes or source overlays. -Only WHEEL/RECORD and the generated CUDA payload manifest are rewritten. Runtime -files (including SM90 and licenses) are retained. Reject oversized wheels instead -of deleting architectures. Native inputs must first pass auditwheel repair. +Only wheel packaging metadata and cache-relative ELF RPATHs are rewritten. +Python runtime files, CUDA code, SM90 and licenses are retained. Reject oversized +wheels instead of deleting architectures. Native inputs require auditwheel repair. """ from __future__ import annotations @@ -137,8 +137,10 @@ def retag(root, python, abi): ) -def archive_payload(sgl, output): +def archive_payload(sgl, output, libraries=None): hashes = {} + sources = {name: sgl / "sgl_kernel" / name for name in LARGE} + sources.update({".libs/" + name: path for name, path in (libraries or {}).items()}) with ( output.open("wb") as raw, gzip.GzipFile( @@ -146,8 +148,7 @@ def archive_payload(sgl, output): ) as compressed, ): with tarfile.open(fileobj=compressed, mode="w|") as archive: - for name in LARGE: - source = sgl / "sgl_kernel" / name + for name, source in sources.items(): hashes[name] = sha256(source) info = archive.gettarinfo(str(source), arcname=name) info.uid = info.gid = info.mtime = 0 @@ -157,6 +158,33 @@ def archive_payload(sgl, output): return hashes +def relocate_lazy_dependencies(sgl): + """Keep auditwheel's private small dependencies next to cached ELF files. + + The main loader already materializes all manifest files. No Python loader + overlay is needed: only adjust ELF RPATH as a normal wheel-packaging step. + Directly loaded SM90/deep_gemm objects keep auditwheel's original layout. + """ + libraries = {} + for path in sgl.rglob("*.so*"): + if path.is_file() and any(part.endswith(".libs") for part in path.parts): + require(path.name not in libraries, "Colliding bundled library names") + libraries[path.name] = path + changes = [] + require(not any(name.startswith(("libcublas", "libnvrtc", "libnvJitLink")) for name in libraries), + "NVIDIA libraries must be supplied by the pinned Torch dependencies") + for name in LARGE: + path = sgl / "sgl_kernel" / name + dynamic = subprocess.check_output(["readelf", "--dynamic", str(path)], text=True) + needed = set(re.findall(r"Shared library: \[([^]]+)\]", dynamic)) + if needed & libraries.keys(): + before = sha256(path) + rpath = "$ORIGIN/../.libs" if "/" in name else "$ORIGIN/.libs" + subprocess.run(["patchelf", "--set-rpath", rpath, str(path)], check=True) + changes.append({"file": name, "operation": "set-rpath", "rpath": rpath, "before_sha256": before, "after_sha256": sha256(path)}) + return libraries, changes + + def binary_evidence(roots): evidence = {} for package, root in roots.items(): @@ -222,33 +250,16 @@ def assemble(raw, output, evidence_dir): roots = {name: temp / name for name in by_name} for name, entry in by_name.items(): unpack(entry["path"], roots[name]) - save_json(evidence_dir / "cuda-binaries.json", binary_evidence(roots)) sgl = roots["sgl-kernel-kt"] - # Lazy objects are extracted into a cache, not site-packages. An - # auditwheel-renamed dependency resolved via $ORIGIN would break there. - # Torch/CUDA-runtime libraries are supplied by the pinned torch wheel; - # reject any other bundled dependency rather than emitting a broken wheel. - bundled = { - path.name - for path in sgl.rglob("*.so*") - if any(part.endswith(".libs") for part in path.parts) - } - for name in LARGE: - dynamic = subprocess.check_output( - ["readelf", "--dynamic", str(sgl / "sgl_kernel" / name)], text=True - ) - needed = set(re.findall(r"Shared library: \[([^]]+)\]", dynamic)) - require( - not (needed & bundled), - "Lazy payload depends on a relocated auditwheel library; fix the main build linkage", - ) + libraries, relocations = relocate_lazy_dependencies(sgl) + save_json(evidence_dir / "cuda-binaries.json", binary_evidence(roots)) for name in ("payload_runtime.py", "load_utils.py", "flash_attn.py"): require( (sgl / "sgl_kernel" / name).is_file(), "Locked SGL main lacks the carrier loader: " + name, ) archive = temp / "payload.tar.gz" - hashes = archive_payload(sgl, archive) + hashes = archive_payload(sgl, archive, libraries) # Only the two objects supported by main's lazy loader are externalized. # No SM90 deletion or copied Python implementation from another checkout. for name in LARGE: @@ -374,6 +385,8 @@ def assemble(raw, output, evidence_dir): "payload_files": hashes, "parts": parts, "direct_native_carriers": {"ktransformers": direct_files}, + "elf_packaging_relocations": relocations, + "lazy_dependency_files": sorted(".libs/" + name for name in libraries), "runtime_overlays": [], "version_overrides": [], "dependency_overrides": [], diff --git a/.github/release/manual_promote.py b/.github/release/manual_promote.py index d8d01843e..e2565ace7 100644 --- a/.github/release/manual_promote.py +++ b/.github/release/manual_promote.py @@ -40,6 +40,7 @@ def check_attestation(data, manifest, digest): require(data.get("wheels") == manifest["wheels"], "Different accepted wheel bytes") require(data.get("candidate_run_id") == manifest["run_id"], "Different build run") require(data.get("candidate_attempt") == manifest["run_attempt"], "Different build attempt") + require(data.get("assembly_workflow_sha", data["source_lock"].get("workflow_sha")) == manifest.get("assembly_workflow_sha", manifest.get("workflow_sha")), "Different assembler revision") for host in ("sap4", "qj5090"): for extra in ("sglang", "sglang,sft"): verify_install_report(data["install_reports"][host][extra], manifest, extra, public=False) @@ -89,7 +90,7 @@ def fetch(): require(hashlib.sha256(content).hexdigest() == digest, "Acceptance record changed") data = json.loads(content) require(data["candidate_run_id"] == int(run_id) and data["candidate_attempt"] == int(attempt), "Acceptance belongs to another build") - require(data["source_lock"]["workflow_sha"] == run["head_sha"], "Different source revision") + require(data.get("assembly_workflow_sha", data["source_lock"]["workflow_sha"]) == run["head_sha"], "Different assembler revision") save_json(Path("accepted.json"), data) save_json(Path("build-run.json"), run) diff --git a/.github/release/recover_native.py b/.github/release/recover_native.py new file mode 100644 index 000000000..01ee63df5 --- /dev/null +++ b/.github/release/recover_native.py @@ -0,0 +1,96 @@ +"""Reuse hash-verified CI raw wheels after a packaging-only failure, never rebuild. + +The original four-main source snapshot remains frozen. A different trusted CI +assembler revision is explicit; recovery is refused after any KT runtime change. +""" + +import argparse +import json +import os +from pathlib import Path +import re +import urllib.request + +from four_main import inspect_wheel, read_lock, save_json + +REPO = "kvcache-ai/ktransformers" + + +def require(value, message): + if not value: + raise ValueError(message) + + +def api(path): + request = urllib.request.Request("https://api.github.com/repos/" + REPO + path, + headers={"Authorization": "Bearer " + os.environ["GH_TOKEN"], "Accept": "application/vnd.github+json"}) + with urllib.request.urlopen(request, timeout=60) as response: + return json.load(response) + + +def check_run(run, jobs, comparison): + require(run["event"] == "workflow_dispatch" and run["head_branch"] == "main", "Only official main builds may be recovered") + require(run["path"] == ".github/workflows/release-four-main.yml" and run["status"] == "completed", "Original build must be complete") + require(run["conclusion"] in ("success", "failure"), "Do not recover canceled or unfinished compilation") + completed = {step["name"] for job in jobs for step in job.get("steps", []) if step["conclusion"] == "success"} + required = {"Compile all KT CPU variants and CUDA architectures", "Compile SGL CUDA payload from the independently locked SGLang main", "Inspect wheel metadata and source provenance"} + require(required <= completed, "Original native compilation and source audit must have succeeded") + require(comparison["status"] in ("ahead", "identical"), "Original build is not a main ancestor") + files = comparison["files"] + require(len(files) < 300, "Refuse a truncated compare response") + require(all(item["filename"].startswith(".github/") and item.get("previous_filename", item["filename"]).startswith(".github/") for item in files), "Runtime sources changed; compile a fresh stack instead") + + +def preflight(run_id, attempt): + require(os.environ.get("GITHUB_REPOSITORY") == REPO and os.environ.get("GITHUB_REF") == "refs/heads/main", "Official main only") + require(run_id.isdecimal() and attempt.isdecimal() and int(attempt) > 0, "Invalid original run") + run = api("/actions/runs/" + run_id) + require(run["run_attempt"] == int(attempt), "Original attempt changed") + jobs = api(f"/actions/runs/{run_id}/attempts/{attempt}/jobs?per_page=100") + require(jobs["total_count"] <= 100, "Too many jobs to validate") + compare = api("/compare/" + run["head_sha"] + "..." + os.environ["GITHUB_SHA"]) + check_run(run, jobs["jobs"], compare) + save_json(Path("recovery-source.json"), {"run_id": int(run_id), "run_attempt": int(attempt), "workflow_sha": run["head_sha"], "assembly_workflow_sha": os.environ["GITHUB_SHA"], "ci_only_diff": [item["filename"] for item in compare["files"]]}) + + +def check_lock(path): + lock = read_lock(path) + recovery = json.loads(Path("recovery-source.json").read_text()) + require(lock["workflow_sha"] == lock["sources"]["ktransformers"]["sha"] == recovery["workflow_sha"], "Original source lock differs from the compiled workflow") + require("recovery" not in lock, "Recover only original compile runs, not recovery chains") + lock["recovery"] = recovery + save_json(path, lock) + + +def verify_inputs(lock, report, wheels): + recovery = lock["recovery"] + expected_lock = {key: value for key, value in lock.items() if key != "recovery"} + require(report["source_lock"] == expected_lock, "Raw wheels belong to another four-main snapshot") + require(report["stage"] == "raw-native-wheels" and report["metadata_consistent"] is True and not report["errors"], "Original source audit failed") + require(re.fullmatch(r"[0-9a-f]{40}", recovery["assembly_workflow_sha"]), "Invalid assembler SHA") + expected = {item["filename"]: item for item in report["wheels"]} + actual = {item["filename"]: item for item in wheels} + require(len(expected) == len(report["wheels"]) == 6 and set(actual) == set(expected), "Require exactly the six original raw wheels") + for name, item in actual.items(): + require(all(item[key] == expected[name][key] for key in ("name", "version", "sha256", "size", "requires_dist")), "Raw wheel changed: " + name) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=("preflight", "lock", "inputs")) + parser.add_argument("--run-id", default="") + parser.add_argument("--attempt", default="1") + parser.add_argument("--lock", type=Path, default=Path("source-lock.json")) + parser.add_argument("--report", type=Path) + parser.add_argument("--wheels", type=Path) + args = parser.parse_args() + if args.command == "preflight": + preflight(args.run_id, args.attempt) + elif args.command == "lock": + check_lock(args.lock) + else: + verify_inputs(read_lock(args.lock), json.loads(args.report.read_text()), [inspect_wheel(path) for path in sorted(args.wheels.glob("*.whl"))]) + import torch + original_torch = (args.report.parent / "torch.txt").read_text().strip() + require(original_torch == f"{torch.__version__} {torch.version.cuda}", "Recovery Torch/CUDA ABI differs from original compilation") + require("release 12.8" in (args.report.parent / "nvcc.txt").read_text(), "Original CUDA toolkit is not 12.8") diff --git a/.github/release/release_stack.py b/.github/release/release_stack.py index 3d87ac42b..b5b7ac6a8 100644 --- a/.github/release/release_stack.py +++ b/.github/release/release_stack.py @@ -118,6 +118,9 @@ def make_release(final, root, lock_path, evidence): "run_id": int(os.environ["GITHUB_RUN_ID"]), "run_attempt": int(os.environ["GITHUB_RUN_ATTEMPT"]), "workflow_sha": lock["workflow_sha"], + # A packaging-only recovery can use a newer trusted CI implementation + # while preserving the original compiled four-main runtime snapshot. + "assembly_workflow_sha": os.environ["GITHUB_SHA"], "source_lock": lock, "wheels": { entry["name"]: { diff --git a/.github/release/repair_native.py b/.github/release/repair_native.py new file mode 100644 index 000000000..b06c315ce --- /dev/null +++ b/.github/release/repair_native.py @@ -0,0 +1,41 @@ +"""Auditwheel repair with CUDA dependencies provided by the pinned Torch stack.""" + +import argparse +from pathlib import Path +import shutil +import subprocess +import sys + +from four_main import inspect_wheel, save_json + +# Do not embed another copy of NVIDIA's large libraries or give them private +# auditwheel SONAMEs. Torch 2.9.1 pins their corresponding nvidia-* wheels. +EXTERNAL = ( + "libcuda.so.1", "libtorch.so", "libtorch_cpu.so", "libtorch_cuda.so", + "libtorch_python.so", "libc10.so", "libc10_cuda.so", "libcudart.so.12", + "libcublas.so.12", "libcublasLt.so.12", "libnvrtc.so.12", + "libnvrtc-builtins.so.12.8", "libnvJitLink.so.12", +) + + +def repair(raw, destination, evidence): + destination.mkdir(exist_ok=False) + for path in sorted(raw.glob("*.whl")): + entry = inspect_wheel(path) + if entry["name"] in {"kt-kernel", "sgl-kernel-kt"}: + command = [sys.executable, "-m", "auditwheel", "repair", str(path), "--plat", "manylinux_2_35_x86_64"] + for soname in EXTERNAL: + command += ["--exclude", soname] + subprocess.run(command + ["-w", str(destination)], check=True) + else: + shutil.copyfile(path, destination / path.name) + save_json(evidence / "native-repair.json", {"external_libraries": list(EXTERNAL), "provider": "pinned torch==2.9.1 CUDA 12.8 dependencies; NVIDIA driver supplies libcuda"}) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--raw", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--evidence", type=Path, required=True) + args = parser.parse_args() + repair(args.raw, args.output, args.evidence) diff --git a/.github/release/test_carriers.py b/.github/release/test_carriers.py index d69ad9f81..c99422d5b 100644 --- a/.github/release/test_carriers.py +++ b/.github/release/test_carriers.py @@ -1,6 +1,9 @@ """Exercise carrier assembly with tiny synthetic wheels, not CUDA execution.""" import sys +import tarfile +import shutil +import subprocess import zipfile from pathlib import Path @@ -96,3 +99,69 @@ def test_size_limit_fails_without_dropping_architectures(tmp_path, monkeypatch): evidence.mkdir() with pytest.raises(ValueError, match="capacity"): carriers.assemble(raw, tmp_path / "final", evidence) + + +def test_lazy_libraries_are_relocated_and_hashed_without_python_overlay(tmp_path, monkeypatch): + sgl = tmp_path / "sgl" + library = sgl / "sgl_kernel_kt.libs/libnuma-test.so.1" + library.parent.mkdir(parents=True) + library.write_bytes(b"small-system-library") + for name in carriers.LARGE: + path = sgl / "sgl_kernel" / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"synthetic-elf") + calls = [] + monkeypatch.setattr(carriers.subprocess, "check_output", lambda *a, **kw: "Shared library: [libnuma-test.so.1]") + monkeypatch.setattr(carriers.subprocess, "run", lambda command, **kw: calls.append(command)) + libraries, changes = carriers.relocate_lazy_dependencies(sgl) + assert {call[2] for call in calls} == {"$ORIGIN/.libs", "$ORIGIN/../.libs"} + assert len(changes) == 2 + archive = tmp_path / "payload.tar.gz" + hashes = carriers.archive_payload(sgl, archive, libraries) + assert set(hashes) == {*carriers.LARGE, ".libs/libnuma-test.so.1"} + with tarfile.open(archive, "r:gz") as bundle: + assert set(bundle.getnames()) == set(hashes) + assert bundle.extractfile(".libs/libnuma-test.so.1").read() == library.read_bytes() + + +def test_cuda_vendor_libraries_must_not_be_bundled_again(tmp_path): + library = tmp_path / "sgl_kernel_kt.libs/libcublas-bad.so.12" + library.parent.mkdir() + library.touch() + with pytest.raises(ValueError, match="NVIDIA libraries"): + carriers.relocate_lazy_dependencies(tmp_path) + + +@pytest.mark.skipif(not shutil.which("gcc") or not shutil.which("patchelf"), reason="Native ELF packaging test needs gcc and patchelf") +def test_real_elf_dependency_loads_after_cache_relocation(tmp_path): + sgl = tmp_path / "site-packages" + libs = sgl / "sgl_kernel_kt.libs" + libs.mkdir(parents=True) + dependency = tmp_path / "dependency.c" + dependency.write_text("int kt_dependency_value(void) { return 42; }\n") + consumer = tmp_path / "consumer.c" + consumer.write_text("extern int kt_dependency_value(void); int kt_fixture_value(void) { return kt_dependency_value(); }\n") + soname = "libkt_fixture_dependency-private.so.1" + subprocess.run(["gcc", "-shared", "-fPIC", str(dependency), "-Wl,-soname," + soname, "-o", str(libs / soname)], check=True) + for name in carriers.LARGE: + target = sgl / "sgl_kernel" / name + target.parent.mkdir(parents=True, exist_ok=True) + rpath = "$ORIGIN/../../sgl_kernel_kt.libs" if "/" in name else "$ORIGIN/../sgl_kernel_kt.libs" + subprocess.run(["gcc", "-shared", "-fPIC", str(consumer), "-L" + str(libs), "-l:" + soname, "-Wl,-rpath," + rpath, "-o", str(target)], check=True) + broken = tmp_path / "broken-cache" + shutil.copytree(sgl / "sgl_kernel", broken) + probe = "import ctypes,sys; assert ctypes.CDLL(sys.argv[1]).kt_fixture_value() == 42" + failed = subprocess.run([sys.executable, "-c", probe, str(broken / carriers.LARGE[1])], capture_output=True) + assert failed.returncode != 0 + libraries, changes = carriers.relocate_lazy_dependencies(sgl) + assert len(changes) == 2 + archive = tmp_path / "payload.tar.gz" + hashes = carriers.archive_payload(sgl, archive, libraries) + cached = tmp_path / "valid-cache" + with tarfile.open(archive, "r:gz") as bundle: + bundle.extractall(cached, filter="data") + for name, expected in hashes.items(): + assert carriers.sha256(cached / name) == expected + # Separate interpreters prevent preloaded libraries from hiding missing files. + for name in carriers.LARGE: + subprocess.run([sys.executable, "-c", probe, str(cached / name)], check=True) diff --git a/.github/release/test_manual_promote.py b/.github/release/test_manual_promote.py index a81478a17..1cfb017fe 100644 --- a/.github/release/test_manual_promote.py +++ b/.github/release/test_manual_promote.py @@ -52,3 +52,12 @@ def test_incomplete_or_changed_acceptance_cannot_publish(monkeypatch, mutation): mutation(changed) with pytest.raises(ValueError): module.check_attestation(changed, manifest, "b" * 64) + + +def test_recovery_assembler_identity_must_match(monkeypatch): + data, manifest, _ = fixture(monkeypatch) + manifest["assembly_workflow_sha"] = "d" * 40 + with pytest.raises(ValueError, match="assembler"): + module.check_attestation(data, manifest, "b" * 64) + data["assembly_workflow_sha"] = "d" * 40 + module.check_attestation(data, manifest, "b" * 64) diff --git a/.github/release/test_recover_native.py b/.github/release/test_recover_native.py new file mode 100644 index 000000000..196323fae --- /dev/null +++ b/.github/release/test_recover_native.py @@ -0,0 +1,55 @@ +"""Recovery only consumes completed, audited CI inputs from unchanged runtime sources.""" + +import copy + +import pytest + +import recover_native as module + + +def run_fixture(): + run = {"event": "workflow_dispatch", "head_branch": "main", "path": ".github/workflows/release-four-main.yml", "status": "completed", "conclusion": "failure"} + jobs = [{"steps": [{"name": name, "conclusion": "success"} for name in ("Compile all KT CPU variants and CUDA architectures", "Compile SGL CUDA payload from the independently locked SGLang main", "Inspect wheel metadata and source provenance")]}] + comparison = {"status": "ahead", "files": [{"filename": ".github/release/carriers.py"}]} + return run, jobs, comparison + + +def test_completed_compilation_allows_ci_only_repackaging(): + module.check_run(*run_fixture()) + + +@pytest.mark.parametrize("change", ("canceled", "compile_failed", "model_changed", "renamed_model", "diverged", "untrusted_branch", "truncated")) +def test_recovery_rejects_unverified_inputs(change): + run, jobs, comparison = run_fixture() + if change == "canceled": + run["conclusion"] = "cancelled" + elif change == "compile_failed": + jobs[0]["steps"][0]["conclusion"] = "failure" + elif change == "model_changed": + comparison["files"][0]["filename"] = "kt-kernel/python/sft/wrapper.py" + elif change == "renamed_model": + comparison["files"][0]["previous_filename"] = "kt-kernel/model.py" + elif change == "diverged": + comparison["status"] = "diverged" + elif change == "untrusted_branch": + run["head_branch"] = "feature" + else: + comparison["files"] *= 300 + with pytest.raises(ValueError): + module.check_run(run, jobs, comparison) + + +def test_raw_input_hashes_and_original_snapshot_are_mandatory(): + original = {"sources": {"unit": "fixture"}} + lock = original | {"recovery": {"assembly_workflow_sha": "a" * 40}} + wheels = [{"filename": f"fixture{i}.whl", "name": f"fixture{i}", "version": "1", "sha256": str(i) * 64, "size": 10, "requires_dist": []} for i in range(6)] + report = {"source_lock": original, "stage": "raw-native-wheels", "metadata_consistent": True, "errors": [], "wheels": wheels} + module.verify_inputs(lock, report, wheels) + altered = copy.deepcopy(wheels) + altered[0]["sha256"] = "f" * 64 + with pytest.raises(ValueError, match="changed"): + module.verify_inputs(lock, report, altered) + with pytest.raises(ValueError, match="six"): + module.verify_inputs(lock, report, wheels[:-1]) + with pytest.raises(ValueError, match="snapshot"): + module.verify_inputs(lock, report | {"source_lock": {}}, wheels) diff --git a/.github/workflows/release-four-main.yml b/.github/workflows/release-four-main.yml index bb200e291..6643ed31e 100644 --- a/.github/workflows/release-four-main.yml +++ b/.github/workflows/release-four-main.yml @@ -10,9 +10,18 @@ on: required: true default: candidate options: [build, candidate, pypi] + reuse_raw_run_id: + description: 'Optional completed original compile run; packaging-only recovery, target must be build' + type: string + default: '' + reuse_raw_attempt: + description: 'Exact original compile attempt for recovery' + type: string + default: '1' permissions: contents: read + actions: read concurrency: # Share the existing release lock: native builds must not overlap a release @@ -41,12 +50,15 @@ jobs: RELEASE_ENABLED: ${{ vars.KT_FOUR_MAIN_RELEASE_ENABLED }} E2E_ENABLED: ${{ vars.KT_MODEL_E2E_ENABLED }} TARGET: ${{ inputs.target }} + REUSE_RAW_RUN_ID: ${{ inputs.reuse_raw_run_id }} run: | test "$RELEASE_ENABLED" = true if [ "$TARGET" != build ]; then test "$E2E_ENABLED" = true; fi + if [ -n "$REUSE_RAW_RUN_ID" ]; then test "$TARGET" = build; fi test -f .github/workflows/model-e2e-release.yml test -f .github/model-e2e/release_contracts.py - name: Resolve latest main heads once + if: inputs.reuse_raw_run_id == '' env: WORKFLOW_SHA: ${{ github.sha }} run: | @@ -57,6 +69,24 @@ jobs: lock = json.load(open('source-lock.json')) assert lock['sources']['ktransformers']['sha'] == os.environ['WORKFLOW_SHA'], 'KT main moved after dispatch; start a fresh run' PY + - name: Verify packaging-only recovery of completed CI compilation + if: inputs.reuse_raw_run_id != '' + env: + GH_TOKEN: ${{ github.token }} + SOURCE_RUN: ${{ inputs.reuse_raw_run_id }} + SOURCE_ATTEMPT: ${{ inputs.reuse_raw_attempt }} + run: | + python .github/release/recover_native.py preflight --run-id "$SOURCE_RUN" --attempt "$SOURCE_ATTEMPT" + - uses: actions/download-artifact@v4 + if: inputs.reuse_raw_run_id != '' + with: + github-token: ${{ github.token }} + run-id: ${{ inputs.reuse_raw_run_id }} + name: four-main-source-lock-${{ inputs.reuse_raw_run_id }}-${{ inputs.reuse_raw_attempt }} + path: . + - name: Retain the original four-main snapshot and explicit assembler identity + if: inputs.reuse_raw_run_id != '' + run: python .github/release/recover_native.py lock --lock source-lock.json - name: Name this snapshot id: names env: @@ -157,6 +187,7 @@ jobs: python ci-tools/.github/release/four_main.py checkout \ --lock lock/source-lock.json --destination "$RELEASE_WORK/sources" - name: Build Python package wheels from main source + if: inputs.reuse_raw_run_id == '' run: | set -euo pipefail for source in accelerate transformers ktransformers; do @@ -169,6 +200,7 @@ jobs: SGLANG_KT_VERSION="$kt_version" python -m build --wheel --no-isolation \ --outdir "$RELEASE_WORK/raw-wheels" "$RELEASE_WORK/sources/sglang/python" - name: Fail on incompatible source pins before expensive native builds + if: inputs.reuse_raw_run_id == '' run: | python ci-tools/.github/release/four_main.py audit --python-only \ --lock lock/source-lock.json --sources "$RELEASE_WORK/sources" \ @@ -177,6 +209,7 @@ jobs: python ci-tools/.github/release/release_stack.py unused \ --wheels "$RELEASE_WORK/raw-wheels" - name: Compile all KT CPU variants and CUDA architectures + if: inputs.reuse_raw_run_id == '' env: CPUINFER_BUILD_ALL_VARIANTS: '1' CPUINFER_ENABLE_CPPTRACE: '0' @@ -190,6 +223,7 @@ jobs: python -m build --wheel --no-isolation \ --outdir "$RELEASE_WORK/raw-wheels" "$RELEASE_WORK/sources/ktransformers/kt-kernel" - name: Compile SGL CUDA payload from the independently locked SGLang main + if: inputs.reuse_raw_run_id == '' env: # qj5090 has enough RAM for these template-heavy CUDA translation # units. Keep the CPU variant build conservative; scale SGL separately. @@ -202,6 +236,28 @@ jobs: run: | python -m build --wheel --no-isolation \ --outdir "$RELEASE_WORK/raw-wheels" "$RELEASE_WORK/sources/sglang/sgl-kernel" + - name: Recover the exact original CI raw wheel bytes + if: inputs.reuse_raw_run_id != '' + uses: actions/download-artifact@v4 + with: + github-token: ${{ github.token }} + run-id: ${{ inputs.reuse_raw_run_id }} + name: raw-wheels-not-for-pypi-${{ inputs.reuse_raw_run_id }}-${{ inputs.reuse_raw_attempt }} + path: ${{ env.RELEASE_WORK }}/raw-wheels + - name: Recover the original source and wheel-hash audit + if: inputs.reuse_raw_run_id != '' + uses: actions/download-artifact@v4 + with: + github-token: ${{ github.token }} + run-id: ${{ inputs.reuse_raw_run_id }} + name: four-main-evidence-${{ inputs.reuse_raw_run_id }}-${{ inputs.reuse_raw_attempt }} + path: ${{ env.RELEASE_WORK }}/evidence/original-build + - name: Reject altered or mixed raw inputs before repackaging + if: inputs.reuse_raw_run_id != '' + run: | + python ci-tools/.github/release/recover_native.py inputs \ + --lock lock/source-lock.json --wheels "$RELEASE_WORK/raw-wheels" \ + --report "$RELEASE_WORK/evidence/original-build/raw-wheel-report.json" - name: Inspect wheel metadata and source provenance run: | set -euo pipefail @@ -212,28 +268,9 @@ jobs: --output "$RELEASE_WORK/evidence/raw-wheel-report.json" - name: Repair native inputs without changing runtime source run: | - set -euo pipefail - mkdir "$RELEASE_WORK/repaired" - python - <<'PY' - import os, pathlib, shutil, subprocess, sys - sys.path.insert(0, 'ci-tools/.github/release') - from four_main import inspect_wheel - work = pathlib.Path(os.environ['RELEASE_WORK']) - for path in sorted((work / 'raw-wheels').glob('*.whl')): - entry = inspect_wheel(path) - if entry['name'] in {'kt-kernel', 'sgl-kernel-kt'}: - subprocess.run([ - sys.executable, '-m', 'auditwheel', 'repair', str(path), - '--plat', 'manylinux_2_35_x86_64', '--exclude', 'libcuda.so.1', - '--exclude', 'libtorch.so', '--exclude', 'libtorch_cpu.so', - '--exclude', 'libtorch_cuda.so', '--exclude', 'libtorch_python.so', - '--exclude', 'libc10.so', '--exclude', 'libc10_cuda.so', - '--exclude', 'libcudart.so.12', - '-w', str(work / 'repaired'), - ], check=True) - else: - shutil.copyfile(path, work / 'repaired' / path.name) - PY + python ci-tools/.github/release/repair_native.py \ + --raw "$RELEASE_WORK/raw-wheels" --output "$RELEASE_WORK/repaired" \ + --evidence "$RELEASE_WORK/evidence" - name: Assemble final carriers and inspect CUDA SASS coverage run: | python ci-tools/.github/release/carriers.py \