diff --git a/.github/workflows/native-ci.yml b/.github/workflows/native-ci.yml new file mode 100644 index 00000000..f40dff29 --- /dev/null +++ b/.github/workflows/native-ci.yml @@ -0,0 +1,281 @@ +name: native-ci + +# The EVERY-PR gates for the NATIVE library — binding-agnostic on purpose: +# these lanes certify the C/C++ contracts every binding (Python today; +# Rust/TypeScript/Swift next) relies on, so they live apart from any one +# binding's workflow. Moved here from python-bindings.yml (2026-06-13, +# bindings-shared-infra) with job names unchanged. +# +# - cpp-tests: the full C++ white-box suite against a static +# build, then `cmake --install` + an EXTERNAL toy C +# consumer linked purely from the installed +# lib/transcribe-link.json manifest (static posture). +# - cpp-tests-sanitized: ASan+UBSan over the white-box suite — certifies +# the C lifetime/ABI contracts FFI bindings rely on. +# - provider-dl-vulkan: the Vulkan degradation contract on a +# hand-assembled provider directory, plus the +# link-smoke in the shared/DL posture (the toy +# consumer drives transcribe_init_backends on the +# installed module dir — the Rust dylib-mode shape). +# - posture-lint: the wheel-preset <-> pyproject-lane mirror gate +# (scripts/ci/check_lane_mirror.py): anchored lane +# regexes + settled-posture equality. The 2026-06-12 +# wrong-posture wheel bug, made structural. +# +# Python-binding lanes (FFI drift gate, pytest suite) stay in +# python-bindings.yml. NOTE: provider-dl-vulkan drives its checks through +# the Python binding (the conformance driver), so a bindings/python change +# that breaks the driver surfaces on the next native-path PR — accepted +# coupling, kept out of this workflow's path filters to keep them native. + +on: + push: + branches: [main] + paths: + - "src/**" + - "include/**" + - "tests/**" + - "ggml/**" + - "CMakeLists.txt" + - "CMakePresets.json" + - "cmake/**" + - "pyproject.toml" + - ".github/workflows/native-ci.yml" + - ".github/actions/**" + - "scripts/ci/vulkan_degradation_check.py" + - "scripts/ci/link_smoke.c" + - "scripts/ci/link_smoke.py" + - "scripts/ci/check_lane_mirror.py" + pull_request: + paths: + - "src/**" + - "include/**" + - "tests/**" + - "ggml/**" + - "CMakeLists.txt" + - "CMakePresets.json" + - "cmake/**" + - "pyproject.toml" + - ".github/workflows/native-ci.yml" + - ".github/actions/**" + - "scripts/ci/vulkan_degradation_check.py" + - "scripts/ci/link_smoke.c" + - "scripts/ci/link_smoke.py" + - "scripts/ci/check_lane_mirror.py" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + cpp-tests: + name: cpp-tests (${{ matrix.label }}) + strategy: + fail-fast: false + matrix: + include: + - label: linux + runner: blacksmith-2vcpu-ubuntu-2404 + # macOS arm64 on the bare-metal M4 mini, not a GitHub VM (keep all + # macOS arm64 CI on owned hardware; the white-box suite builds with + # Metal on, and real hardware is the trustworthy place to run it). + - label: macos-arm64 + runner: [self-hosted, macOS, ARM64] + runs-on: ${{ matrix.runner }} + timeout-minutes: 40 # bound the run if the self-hosted mini is offline + steps: + - uses: actions/checkout@v6 + - uses: astral-sh/setup-uv@v8.2.0 # fixtures are generated via uv + - name: Install build deps (Linux) + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y cmake ninja-build zlib1g-dev ccache + - name: Install build deps (macOS) + if: runner.os == 'macOS' + run: | + brew install ninja + command -v ccache >/dev/null || brew install ccache + - name: CPU ISA signature (segregates ccache across the heterogeneous fleet) + # ggml builds with -march=native here (the dev posture under test); + # ccache hashes the literal flag, not the ISA it resolves to, so a + # cache compiled on a richer CPU SIGILLs on a weaker one. Key the + # cache by the CPU's feature flags instead. + if: runner.os == 'Linux' + run: echo "CPU_SIG=$(grep -m1 '^flags' /proc/cpuinfo | sha256sum | cut -c1-8)" >> "$GITHUB_ENV" + - name: ccache (compile cache across runs) + if: runner.os == 'Linux' # the mini is persistent; its local cache suffices + uses: actions/cache@v5 + with: + path: ~/.cache/ccache + key: ccache-cpp-tests-${{ matrix.label }}-${{ env.CPU_SIG }}-${{ github.sha }} + restore-keys: ccache-cpp-tests-${{ matrix.label }}-${{ env.CPU_SIG }}- + - name: Configure (static white-box build) + # On macOS the Metal shader library is embedded: it makes the + # INSTALLED tree the link-smoke consumes below self-contained (no + # default.metallib sidecar to locate) and matches how the shipped + # macOS wheel is built. Compute is identical either way. + run: | + extra="" + [ "$RUNNER_OS" = "macOS" ] && extra="-DGGML_METAL_EMBED_LIBRARY=ON" + cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release $extra \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + - name: Build + run: cmake --build build -j + - name: Test (full white-box suite) + run: ctest --test-dir build --output-on-failure + - name: Install + external-consumer link smoke (static posture) + # `cmake --install` of the SAME tree (zero extra compile), then a toy + # C program is compiled against the staged prefix with a link line + # built ONLY from lib/transcribe-link.json — the manifest the Rust + # -sys crate's build.rs will trust. See cmake/transcribe-install.cmake. + run: | + cmake --install build --prefix "$RUNNER_TEMP/staging-install" > /dev/null + python3 scripts/ci/link_smoke.py --prefix "$RUNNER_TEMP/staging-install" + - name: ccache stats + run: ccache -s | head -8 + + cpp-tests-sanitized: + # ASan+UBSan over the white-box suite. This is the lane that certifies + # the C lifetime/ABI contracts the FFI bindings rely on — including the + # params copy-out regression (stream_dispatch_unit), which reproduces a + # ctypes-shaped caller freeing its params right after stream_begin. + runs-on: blacksmith-2vcpu-ubuntu-2404 + steps: + - uses: actions/checkout@v6 + - uses: astral-sh/setup-uv@v8.2.0 # fixtures are generated via uv + - name: Install build deps + run: sudo apt-get update && sudo apt-get install -y cmake ninja-build zlib1g-dev ccache + - name: CPU ISA signature (segregates ccache across the heterogeneous fleet) + # ggml builds with -march=native here (the dev posture under test); + # ccache hashes the literal flag, not the ISA it resolves to, so a + # cache compiled on a richer CPU SIGILLs on a weaker one. Key the + # cache by the CPU's feature flags instead. + run: echo "CPU_SIG=$(grep -m1 '^flags' /proc/cpuinfo | sha256sum | cut -c1-8)" >> "$GITHUB_ENV" + - name: ccache (compile cache across runs) + uses: actions/cache@v5 + with: + path: ~/.cache/ccache + key: ccache-asan-${{ env.CPU_SIG }}-${{ github.sha }} + restore-keys: ccache-asan-${{ env.CPU_SIG }}- + - name: Configure (static, sanitized) + run: | + cmake -B build-asan -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DTRANSCRIBE_SANITIZE=ON \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + - name: Build + run: cmake --build build-asan -j + - name: Test (white-box suite under ASan+UBSan) + run: ctest --test-dir build-asan --output-on-failure + - name: ccache stats + run: ccache -s | head -8 + + provider-dl-vulkan: + # Builds the default Linux provider shape — CPU + Vulkan as dynamic + # backend modules (GGML_BACKEND_DL) — assembles it into a flat + # wheel-like directory with $ORIGIN rpaths, deletes the build tree so + # nothing can resolve outside the directory, and proves the Vulkan + # degradation contract on the real runner via + # scripts/ci/vulkan_degradation_check.py: + # 1. loader REMOVED (no libvulkan) -> import + CPU devices work, + # vulkan answers unavailable; the module-load failure is quiet. + # 2. mesa lavapipe installed -> the SAME artifacts discover a + # software Vulkan device, and (with HF_TOKEN for the canary model) + # actually transcribe on it. + # Before the build tree is deleted, the shared/DL leg of the link smoke + # runs against `cmake --install` output: the toy consumer links + # libtranscribe alone and drives transcribe_init_backends() on the + # installed module directory — the exact shape a Rust dylib-feature + # consumer has. + runs-on: blacksmith-2vcpu-ubuntu-2404 + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + steps: + - uses: actions/checkout@v6 + - uses: astral-sh/setup-uv@v8.2.0 + - name: Install build deps (Vulkan SDK pieces + patchelf) + run: | + sudo apt-get update + sudo apt-get install -y cmake ninja-build zlib1g-dev \ + libvulkan-dev glslc patchelf ccache + - name: CPU ISA signature (segregates ccache across the heterogeneous fleet) + # ggml builds with -march=native here (the dev posture under test); + # ccache hashes the literal flag, not the ISA it resolves to, so a + # cache compiled on a richer CPU SIGILLs on a weaker one. Key the + # cache by the CPU's feature flags instead. + run: echo "CPU_SIG=$(grep -m1 '^flags' /proc/cpuinfo | sha256sum | cut -c1-8)" >> "$GITHUB_ENV" + - name: ccache (compile cache across runs) + uses: actions/cache@v5 + with: + path: ~/.cache/ccache + key: ccache-dl-vulkan-${{ env.CPU_SIG }}-${{ github.sha }} + restore-keys: ccache-dl-vulkan-${{ env.CPU_SIG }}- + - name: Configure (DL provider, CPU + Vulkan modules, wheel posture) + run: | + cmake -B build-dl -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DTRANSCRIBE_BUILD_SHARED=ON \ + -DTRANSCRIBE_GGML_BACKEND_DL=ON \ + -DTRANSCRIBE_VULKAN=ON \ + -DTRANSCRIBE_USE_OPENMP=OFF \ + -DTRANSCRIBE_USE_SYSTEM_BLAS=OFF \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + - name: Build + run: cmake --build build-dl -j + - name: Install + external-consumer link smoke (shared/DL posture) + run: | + cmake --install build-dl --prefix "$RUNNER_TEMP/staging-install" > /dev/null + python3 scripts/ci/link_smoke.py --prefix "$RUNNER_TEMP/staging-install" + - name: Assemble flat provider directory ($ORIGIN rpaths, build tree deleted) + run: | + mkdir -p provider + cp -L build-dl/src/libtranscribe.so provider/ + cp -L build-dl/ggml/src/libggml.so.* build-dl/ggml/src/libggml-base.so.* provider/ 2>/dev/null || \ + cp -L build-dl/ggml/src/libggml.so build-dl/ggml/src/libggml-base.so provider/ + cp build-dl/bin/libggml-*.so provider/ + for f in provider/*.so*; do patchelf --set-rpath '$ORIGIN' "$f"; done + ls -la provider/ + rm -rf build-dl # isolation: nothing may resolve outside provider/ + - name: "Tier 1: no Vulkan loader — CPU works, vulkan cleanly unavailable" + run: | + sudo apt-get remove -y libvulkan-dev libvulkan1 || \ + sudo rm -f /usr/lib/x86_64-linux-gnu/libvulkan.so.1* + export TRANSCRIBE_LIBRARY="$PWD/provider/libtranscribe.so" + uv run --project bindings/python python \ + scripts/ci/vulkan_degradation_check.py --tier no-loader + - name: "Tier 2: lavapipe installed — same artifacts discover Vulkan" + run: | + sudo apt-get install -y libvulkan1 mesa-vulkan-drivers + export TRANSCRIBE_LIBRARY="$PWD/provider/libtranscribe.so" + # lavapipe reports VK_PHYSICAL_DEVICE_TYPE_CPU, and ggml-vulkan's + # default selection takes only discrete/integrated GPUs. The env + # override bypasses the type filter so the software device counts — + # CI-only; real GPUs need no override. + export GGML_VK_VISIBLE_DEVICES=0 + uv run --project bindings/python python \ + scripts/ci/vulkan_degradation_check.py --tier loader + - uses: ./.github/actions/fetch-canary + with: + hf-token: ${{ secrets.HF_TOKEN }} + - name: "Tier 2b: real transcription on the Vulkan device (needs HF_TOKEN)" + if: env.HF_TOKEN != '' + run: | + export TRANSCRIBE_LIBRARY="$PWD/provider/libtranscribe.so" + export GGML_VK_VISIBLE_DEVICES=0 + uv run --project bindings/python python \ + scripts/ci/vulkan_degradation_check.py --tier loader \ + --model canary/whisper-tiny-Q5_K_M.gguf --audio samples/jfk.wav + + posture-lint: + # The wheel-preset <-> pyproject-lane mirror, made structural. Tiny and + # idle-friendly: the Hetzner box (free); offline fails bounded via the + # timeout. Same pre-public caveat as python-bindings lint: gate + # PR-triggered self-hosted jobs before the repo goes public. + runs-on: [self-hosted, Linux, X64, hetzner] + timeout-minutes: 10 + steps: + - uses: actions/checkout@v6 + - uses: astral-sh/setup-uv@v8.2.0 + - name: Lane/preset mirror gate + run: uv run --no-project python scripts/ci/check_lane_mirror.py diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0f0e1b4d..079c52c7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -176,6 +176,15 @@ jobs: pattern: cuda-dist-* merge-multiple: true path: cu12 + # The canonical native bundles (extracted from the repaired wheels by + # python-wheels.yml). On the release they become the distribution home + # for non-PyPI ecosystems: npm platform packages and prebuilt-Rust + # fetch these exact bytes by tag instead of rebuilding. + - uses: actions/download-artifact@v8 + with: + pattern: native-* + merge-multiple: true + path: native-bundles - name: Create the release for the tag and attach the cu12 wheels env: GH_TOKEN: ${{ github.token }} @@ -186,6 +195,21 @@ jobs: gh release create "$tag" --repo "$GITHUB_REPOSITORY" \ --title "$tag" --notes "transcribe.cpp $tag" --verify-tag gh release upload "$tag" cu12/*.whl --repo "$GITHUB_REPOSITORY" --clobber + - name: Attach the native bundles (versioned names) + env: + GH_TOKEN: ${{ github.token }} + run: | + set -e + tag="${GITHUB_REF#refs/tags/}" + ver="${tag#v}" + mkdir -p upload + for f in native-bundles/transcribe-native-*.tar.gz; do + base="$(basename "$f" .tar.gz)" + tuple="${base#transcribe-native-}" + cp "$f" "upload/transcribe-native-${ver}-${tuple}.tar.gz" + done + ls -la upload/ + gh release upload "$tag" upload/*.tar.gz --repo "$GITHUB_REPOSITORY" --clobber - name: Refresh the PEP 503 index (requires Pages enabled on the repo) env: GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/python-bindings.yml b/.github/workflows/python-bindings.yml index 5cbd3016..36f8ca27 100644 --- a/.github/workflows/python-bindings.yml +++ b/.github/workflows/python-bindings.yml @@ -1,18 +1,22 @@ name: python-bindings -# The EVERY-PR correctness gate: -# - lint: generated-FFI drift gate + version-sync across the three -# sources of truth (header / pyproject / __init__). -# - cpp-tests: the full C++ white-box suite against a static build. +# The EVERY-PR gate for the PYTHON binding: +# - lint: generated-FFI drift gate (also regenerates the +# binding-neutral include/transcribe.abihash) + +# version-sync across the sources of truth. # - python-shared: a shared libtranscribe, the pure-C api_smoke as an # exported-symbol canary, and the Python test suite (model # tests un-skip when the canary GGUFs are fetchable). -# - provider-dl-vulkan: the Vulkan degradation contract on a hand-assembled -# provider directory (the wheel-shaped artifact). +# +# The binding-agnostic native gates (cpp-tests, cpp-tests-sanitized, +# provider-dl-vulkan, link-smoke, posture-lint) moved to native-ci.yml +# (2026-06-13, bindings-shared-infra): they certify the C contracts EVERY +# binding relies on, not just Python's. Job names were preserved across +# the move. # # The full WHEEL matrix (python-wheels.yml) deliberately does NOT run per PR: # it runs on workflow_dispatch and on every publish (rehearsal or release). -# This workflow is what must stay green on every change. +# This workflow plus native-ci.yml is what must stay green on every change. on: push: @@ -73,181 +77,6 @@ jobs: - name: Version sync (header / pyproject / __init__) run: uv run --no-project bindings/python/_generate/check_version_sync.py - cpp-tests: - name: cpp-tests (${{ matrix.label }}) - strategy: - fail-fast: false - matrix: - include: - - label: linux - runner: blacksmith-2vcpu-ubuntu-2404 - # macOS arm64 on the bare-metal M4 mini, not a GitHub VM (keep all - # macOS arm64 CI on owned hardware; the white-box suite builds with - # Metal on, and real hardware is the trustworthy place to run it). - - label: macos-arm64 - runner: [self-hosted, macOS, ARM64] - runs-on: ${{ matrix.runner }} - timeout-minutes: 30 # bound the run if the self-hosted mini is offline - steps: - - uses: actions/checkout@v6 - - uses: astral-sh/setup-uv@v8.2.0 # fixtures are generated via uv - - name: Install build deps (Linux) - if: runner.os == 'Linux' - run: sudo apt-get update && sudo apt-get install -y cmake ninja-build zlib1g-dev ccache - - name: Install build deps (macOS) - if: runner.os == 'macOS' - run: | - brew install ninja - command -v ccache >/dev/null || brew install ccache - - name: CPU ISA signature (segregates ccache across the heterogeneous fleet) - # ggml builds with -march=native here (the dev posture under test); - # ccache hashes the literal flag, not the ISA it resolves to, so a - # cache compiled on a richer CPU SIGILLs on a weaker one. Key the - # cache by the CPU's feature flags instead. - if: runner.os == 'Linux' - run: echo "CPU_SIG=$(grep -m1 '^flags' /proc/cpuinfo | sha256sum | cut -c1-8)" >> "$GITHUB_ENV" - - name: ccache (compile cache across runs) - if: runner.os == 'Linux' # the mini is persistent; its local cache suffices - uses: actions/cache@v5 - with: - path: ~/.cache/ccache - key: ccache-cpp-tests-${{ matrix.label }}-${{ env.CPU_SIG }}-${{ github.sha }} - restore-keys: ccache-cpp-tests-${{ matrix.label }}-${{ env.CPU_SIG }}- - - name: Configure (static white-box build) - run: | - cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache - - name: Build - run: cmake --build build -j - - name: Test (full white-box suite) - run: ctest --test-dir build --output-on-failure - - name: ccache stats - run: ccache -s | head -8 - - cpp-tests-sanitized: - # ASan+UBSan over the white-box suite. This is the lane that certifies - # the C lifetime/ABI contracts the FFI bindings rely on — including the - # params copy-out regression (stream_dispatch_unit), which reproduces a - # ctypes-shaped caller freeing its params right after stream_begin. - runs-on: blacksmith-2vcpu-ubuntu-2404 - steps: - - uses: actions/checkout@v6 - - uses: astral-sh/setup-uv@v8.2.0 # fixtures are generated via uv - - name: Install build deps - run: sudo apt-get update && sudo apt-get install -y cmake ninja-build zlib1g-dev ccache - - name: CPU ISA signature (segregates ccache across the heterogeneous fleet) - # ggml builds with -march=native here (the dev posture under test); - # ccache hashes the literal flag, not the ISA it resolves to, so a - # cache compiled on a richer CPU SIGILLs on a weaker one. Key the - # cache by the CPU's feature flags instead. - run: echo "CPU_SIG=$(grep -m1 '^flags' /proc/cpuinfo | sha256sum | cut -c1-8)" >> "$GITHUB_ENV" - - name: ccache (compile cache across runs) - uses: actions/cache@v5 - with: - path: ~/.cache/ccache - key: ccache-asan-${{ env.CPU_SIG }}-${{ github.sha }} - restore-keys: ccache-asan-${{ env.CPU_SIG }}- - - name: Configure (static, sanitized) - run: | - cmake -B build-asan -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DTRANSCRIBE_SANITIZE=ON \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache - - name: Build - run: cmake --build build-asan -j - - name: Test (white-box suite under ASan+UBSan) - run: ctest --test-dir build-asan --output-on-failure - - name: ccache stats - run: ccache -s | head -8 - - provider-dl-vulkan: - # Builds the default Linux provider shape — CPU + Vulkan as dynamic - # backend modules (GGML_BACKEND_DL) — assembles it into a flat - # wheel-like directory with $ORIGIN rpaths, deletes the build tree so - # nothing can resolve outside the directory, and proves the Vulkan - # degradation contract on the real runner via - # scripts/ci/vulkan_degradation_check.py: - # 1. loader REMOVED (no libvulkan) -> import + CPU devices work, - # vulkan answers unavailable; the module-load failure is quiet. - # 2. mesa lavapipe installed -> the SAME artifacts discover a - # software Vulkan device, and (with HF_TOKEN for the canary model) - # actually transcribe on it. - runs-on: blacksmith-2vcpu-ubuntu-2404 - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} - steps: - - uses: actions/checkout@v6 - - uses: astral-sh/setup-uv@v8.2.0 - - name: Install build deps (Vulkan SDK pieces + patchelf) - run: | - sudo apt-get update - sudo apt-get install -y cmake ninja-build zlib1g-dev \ - libvulkan-dev glslc patchelf ccache - - name: CPU ISA signature (segregates ccache across the heterogeneous fleet) - # ggml builds with -march=native here (the dev posture under test); - # ccache hashes the literal flag, not the ISA it resolves to, so a - # cache compiled on a richer CPU SIGILLs on a weaker one. Key the - # cache by the CPU's feature flags instead. - run: echo "CPU_SIG=$(grep -m1 '^flags' /proc/cpuinfo | sha256sum | cut -c1-8)" >> "$GITHUB_ENV" - - name: ccache (compile cache across runs) - uses: actions/cache@v5 - with: - path: ~/.cache/ccache - key: ccache-dl-vulkan-${{ env.CPU_SIG }}-${{ github.sha }} - restore-keys: ccache-dl-vulkan-${{ env.CPU_SIG }}- - - name: Configure (DL provider, CPU + Vulkan modules, wheel posture) - run: | - cmake -B build-dl -G Ninja -DCMAKE_BUILD_TYPE=Release \ - -DTRANSCRIBE_BUILD_SHARED=ON \ - -DTRANSCRIBE_GGML_BACKEND_DL=ON \ - -DTRANSCRIBE_VULKAN=ON \ - -DTRANSCRIBE_USE_OPENMP=OFF \ - -DTRANSCRIBE_USE_SYSTEM_BLAS=OFF \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache - - name: Build - run: cmake --build build-dl -j - - name: Assemble flat provider directory ($ORIGIN rpaths, build tree deleted) - run: | - mkdir -p provider - cp -L build-dl/src/libtranscribe.so provider/ - cp -L build-dl/ggml/src/libggml.so.* build-dl/ggml/src/libggml-base.so.* provider/ 2>/dev/null || \ - cp -L build-dl/ggml/src/libggml.so build-dl/ggml/src/libggml-base.so provider/ - cp build-dl/bin/libggml-*.so provider/ - for f in provider/*.so*; do patchelf --set-rpath '$ORIGIN' "$f"; done - ls -la provider/ - rm -rf build-dl # isolation: nothing may resolve outside provider/ - - name: "Tier 1: no Vulkan loader — CPU works, vulkan cleanly unavailable" - run: | - sudo apt-get remove -y libvulkan-dev libvulkan1 || \ - sudo rm -f /usr/lib/x86_64-linux-gnu/libvulkan.so.1* - export TRANSCRIBE_LIBRARY="$PWD/provider/libtranscribe.so" - uv run --project bindings/python python \ - scripts/ci/vulkan_degradation_check.py --tier no-loader - - name: "Tier 2: lavapipe installed — same artifacts discover Vulkan" - run: | - sudo apt-get install -y libvulkan1 mesa-vulkan-drivers - export TRANSCRIBE_LIBRARY="$PWD/provider/libtranscribe.so" - # lavapipe reports VK_PHYSICAL_DEVICE_TYPE_CPU, and ggml-vulkan's - # default selection takes only discrete/integrated GPUs. The env - # override bypasses the type filter so the software device counts — - # CI-only; real GPUs need no override. - export GGML_VK_VISIBLE_DEVICES=0 - uv run --project bindings/python python \ - scripts/ci/vulkan_degradation_check.py --tier loader - - uses: ./.github/actions/fetch-canary - with: - hf-token: ${{ secrets.HF_TOKEN }} - - name: "Tier 2b: real transcription on the Vulkan device (needs HF_TOKEN)" - if: env.HF_TOKEN != '' - run: | - export TRANSCRIBE_LIBRARY="$PWD/provider/libtranscribe.so" - export GGML_VK_VISIBLE_DEVICES=0 - uv run --project bindings/python python \ - scripts/ci/vulkan_degradation_check.py --tier loader \ - --model canary/whisper-tiny-Q5_K_M.gguf --audio samples/jfk.wav - python-shared: runs-on: blacksmith-2vcpu-ubuntu-2404 env: diff --git a/.github/workflows/python-wheels.yml b/.github/workflows/python-wheels.yml index 6b9b6ea8..b5f7f1fc 100644 --- a/.github/workflows/python-wheels.yml +++ b/.github/workflows/python-wheels.yml @@ -19,6 +19,15 @@ name: python-wheels # - clean-install: bare container, wheels only; Vulkan degradation # tiers incl. a REAL lavapipe transcription (the # capability-positive gate) +# - bundle-smoke: the extracted native bundle (see below) loaded +# OUTSIDE any wheel via TRANSCRIBE_LIBRARY — the +# posture npm/prebuilt-Rust consumers have +# +# Every native wheel lane also EXTRACTS its repaired wheel's _native/ dir +# (+ contract.json + licenses) into a transcribe-native-.tar.gz +# artifact (native-): the canonical native bundle non-Python +# ecosystems repackage instead of rebuilding. publish.yml attaches these to +# the GitHub release on tags. # - vulkan-hw: shipped linux wheel on the T14 (RADV Renoir): # real-GPU Vulkan + the fat-CPU tier assertion # - co-import: numpy/torch coexistence with a real transcription, @@ -103,6 +112,18 @@ jobs: with: name: dist-native-linux-${{ matrix.arch }} path: wheelhouse/*.whl + # The canonical native bundle for non-Python ecosystems (npm platform + # packages, prebuilt-Rust, ...): the REPAIRED wheel's _native/ dir + + # contract.json + licenses, re-containered byte-for-byte. One build + # per tuple, every ecosystem ships the same bytes. + - name: Extract the native bundle + run: | + python3 scripts/ci/extract_native_bundle.py --wheel-dir wheelhouse \ + --tuple linux-${{ matrix.arch }}-cpu-vulkan --out bundles + - uses: actions/upload-artifact@v7 + with: + name: native-linux-${{ matrix.arch }}-cpu-vulkan + path: bundles/*.tar.gz wheel-macos: # Bare-metal M4 Mac mini (self-hosted, runner "m4-mini"): builds the @@ -134,6 +155,14 @@ jobs: with: name: dist-native-macos-arm64 path: wheelhouse/*.whl + - name: Extract the native bundle + run: | + python3 scripts/ci/extract_native_bundle.py --wheel-dir wheelhouse \ + --tuple macos-arm64-metal --out bundles + - uses: actions/upload-artifact@v7 + with: + name: native-macos-arm64-metal + path: bundles/*.tar.gz wheel-macos-x86: # Intel macOS: the CPU-ONLY x86_64 wheel, CROSS-COMPILED on the M4 mini @@ -174,6 +203,14 @@ jobs: with: name: dist-native-macos-x86_64 path: wheelhouse/*.whl + - name: Extract the native bundle + run: | + python3 scripts/ci/extract_native_bundle.py --wheel-dir wheelhouse \ + --tuple macos-x86_64-cpu --out bundles + - uses: actions/upload-artifact@v7 + with: + name: native-macos-x86_64-cpu + path: bundles/*.tar.gz wheel-windows: # Blacksmith Windows Server 2025 (public beta) — same image family as @@ -232,6 +269,14 @@ jobs: with: name: dist-native-windows-amd64 path: wheelhouse/*.whl + - name: Extract the native bundle + run: | + python scripts/ci/extract_native_bundle.py --wheel-dir wheelhouse ` + --tuple windows-x86_64-cpu-vulkan --out bundles + - uses: actions/upload-artifact@v7 + with: + name: native-windows-x86_64-cpu-vulkan + path: bundles/*.tar.gz sdist: # The universal fallback must actually work: build the sdist, audit what @@ -345,6 +390,10 @@ jobs: # Named OUTSIDE the dist-* pattern on purpose: co-import merges dist-* # and must not pick up the cu12 provider (it would outrank the default # provider and change what that job tests). + # NOTE: no native-bundle extraction here (or in cuda-windows.yml) — + # the mechanism would work identically (the cu12 wheel has the same + # _native/ + contract.json shape), but no ecosystem consumes a CUDA + # bundle yet. Add the extract step the day one does. - uses: actions/upload-artifact@v7 if: env.MODAL_TOKEN_ID != '' with: @@ -428,6 +477,73 @@ jobs: --tier loader --expect-provider transcribe-cpp-native \ --model canary/whisper-tiny-Q5_K_M.gguf --audio assets/samples/jfk.wav + bundle-smoke: + # The extracted native BUNDLE works outside any wheel — the exact + # posture npm platform packages / prebuilt-Rust consume. One tuple + # suffices (the other tuples' bytes are already validated inside their + # wheels; the bundle step only re-containers them): bare container, no + # provider package installed, the API package dlopens the bundle's + # libtranscribe via TRANSCRIBE_LIBRARY, backend modules load from the + # bundle dir, CPU transcribes, vulkan quietly unavailable. contract.json + # is asserted against the lane posture — the capability-positive gate + # that catches a wrong-posture bundle (the 2026-06-12 bug class). + needs: [wheel-linux, api-wheel] + runs-on: blacksmith-2vcpu-ubuntu-2404 + container: python:3.12-slim + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + steps: + - uses: actions/download-artifact@v8 + with: + name: native-linux-x86_64-cpu-vulkan + path: bundles + - uses: actions/download-artifact@v8 + with: + name: dist-api + path: dist-api + - uses: actions/download-artifact@v8 + with: + name: smoke-assets + path: assets + - name: Canary model cache (avoid HF rate limits) + if: env.HF_TOKEN != '' + uses: actions/cache@v5 + with: + path: canary + key: canary-models-v1 + - name: Fetch canary model (cache miss only; pip — no uv in this container) + if: env.HF_TOKEN != '' + run: | + pip install -q huggingface_hub + [ -f canary/whisper-tiny-Q5_K_M.gguf ] || \ + hf download handy-computer/whisper-tiny-gguf \ + whisper-tiny-Q5_K_M.gguf --local-dir canary + - name: Unpack the bundle + assert the contract posture + run: | + tar xzf bundles/transcribe-native-linux-x86_64-cpu-vulkan.tar.gz + python - <<'EOF' + import json + c = json.load(open( + "transcribe-native-linux-x86_64-cpu-vulkan/contract.json")) + print("contract:", c) + assert c["lane"] == "cpu-vulkan", c + assert set(c["backends"]) == {"vulkan", "cpu"}, c + EOF + - name: API package only (no provider) + the bundle via TRANSCRIBE_LIBRARY + run: | + # --no-deps: the resolver pin would pull the provider wheel; the + # point here is that the BUNDLE is the only native artifact. + pip install --no-deps --no-index --find-links dist-api transcribe-cpp + export TRANSCRIBE_LIBRARY="$PWD/transcribe-native-linux-x86_64-cpu-vulkan/libtranscribe.so" + if [ -f canary/whisper-tiny-Q5_K_M.gguf ]; then + python assets/scripts/ci/vulkan_degradation_check.py \ + --tier no-loader \ + --model canary/whisper-tiny-Q5_K_M.gguf --audio assets/samples/jfk.wav + else + echo "!! no canary (fork without HF_TOKEN) — load-only check" + python assets/scripts/ci/vulkan_degradation_check.py --tier no-loader + fi + vulkan-hw: # Real-GPU Vulkan truth lane — the Linux mirror of wheel-macos on the # mini: installs the SHIPPED linux x86_64 wheel on the self-hosted T14 diff --git a/CMakeLists.txt b/CMakeLists.txt index 76deb18d..504422f7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -160,6 +160,21 @@ option(TRANSCRIBE_GGML_BACKEND_DL "Build ggml backends as loadable modules" OFF) option(TRANSCRIBE_X86_CONSERVATIVE "Default GGML_NATIVE and every x86 SIMD tier to OFF (SIGILL-safe baseline; explicit -DGGML_* still wins)" OFF) +# Install rules for non-Python consumers: headers + the ABI digest + +# libtranscribe + the transcribe-link.json link manifest (ggml's own install +# rules supply the ggml libs/headers). This is what the Rust -sys crate's +# build.rs consumes (`cmake --install` into a staging prefix, then link per +# the manifest) — see cmake/transcribe-install.cmake. Default ON for +# top-level builds; OFF as a subproject, and never active under SKBUILD +# (the wheel ships its own component-filtered install rules instead). +if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND NOT SKBUILD) + set(_transcribe_install_default ON) +else() + set(_transcribe_install_default OFF) +endif() +option(TRANSCRIBE_INSTALL + "Install headers, libtranscribe, and the link manifest" ${_transcribe_install_default}) + # Real-model gated tests. OFF by default because the tests need a # converted Parakeet GGUF on disk (~2.4 GB) and CI can't ship one. # When ON, the test reads TRANSCRIBE_REAL_PARAKEET_GGUF from the @@ -332,3 +347,8 @@ endif() if(SKBUILD) include(cmake/python-wheel-install.cmake) endif() + +# Consumer install rules (same placement constraint as above). +if(TRANSCRIBE_INSTALL AND NOT SKBUILD) + include(cmake/transcribe-install.cmake) +endif() diff --git a/bindings/python/_generate/check_version_sync.py b/bindings/python/_generate/check_version_sync.py index 0ed719da..6b723efe 100644 --- a/bindings/python/_generate/check_version_sync.py +++ b/bindings/python/_generate/check_version_sync.py @@ -31,6 +31,20 @@ PYPROJECT = REPO / "bindings" / "python" / "pyproject.toml" INIT = REPO / "bindings" / "python" / "src" / "transcribe_cpp" / "__init__.py" +# Binding package manifests (requirements doc §2: every manifest is derived +# from or gated against the header). Gated by the `active` flag: a 0.0.0 +# name-reservation placeholder is NOT version-locked — flip its entry to True +# in the PR that lands the real binding. Inactive manifests are still parsed +# (file must exist and carry a readable version) so the mechanism itself +# stays exercised. Package.swift has no entry: SwiftPM versions via git tags, +# so its gate is the tag itself (release-workflow concern, not this script). +BINDING_MANIFESTS = [ + # (relative path, extractor name, active) + ("bindings/rust/Cargo.toml", "cargo", False), + ("bindings/rust/sys/Cargo.toml", "cargo", False), + ("bindings/typescript/package.json", "npm", False), +] + def base_version(version: str) -> str: """The leading dotted-numeric release segment (suffix stripped).""" @@ -60,6 +74,21 @@ def init_version(text: str) -> str | None: return m.group(1) if m else None +def cargo_version(text: str) -> str | None: + # First `version = "..."` in the file: [package] leads a Cargo.toml by + # convention, and dependency tables spell it `name = { version = ... }`. + m = re.search(r'(?m)^version\s*=\s*"([^"]+)"', text) + return m.group(1) if m else None + + +def npm_version(text: str) -> str | None: + m = re.search(r'"version"\s*:\s*"([^"]+)"', text) + return m.group(1) if m else None + + +_BINDING_EXTRACTORS = {"cargo": cargo_version, "npm": npm_version} + + def native_pin_versions(text: str) -> "dict[str, str | None]": # Every native-provider pin (the hard dependency AND accelerator extras) # is the pre-1.0 base-version contract at resolver level: @@ -83,6 +112,25 @@ def main() -> int: } sources.update(native_pin_versions(pyproject_text)) + # Binding manifests: active ones join the equality set; inactive ones + # must merely exist and parse (placeholder versions are reported, not + # compared). + inactive: dict[str, str] = {} + for rel, kind, active in BINDING_MANIFESTS: + path = REPO / rel + version = ( + _BINDING_EXTRACTORS[kind](path.read_text()) if path.exists() else None + ) + if active: + sources[rel] = version + elif version is None: + sources[rel] = None # missing/unparseable is an error either way + else: + inactive[rel] = version + if inactive: + detail = ", ".join(f"{name}={v}" for name, v in inactive.items()) + print(f"inactive binding manifests (parsed, not compared): {detail}") + missing = [name for name, v in sources.items() if v is None] if missing: for name in missing: diff --git a/bindings/python/_generate/generate.py b/bindings/python/_generate/generate.py index 5cbe09b0..93644448 100644 --- a/bindings/python/_generate/generate.py +++ b/bindings/python/_generate/generate.py @@ -32,6 +32,13 @@ INCLUDE = REPO / "include" HEADER = INCLUDE / "transcribe" / "extensions.h" OUTPUT = REPO / "bindings" / "python" / "src" / "transcribe_cpp" / "_generated.py" +# The binding-neutral home of the ABI digest (PUBLIC_HEADER_HASH). This +# generator is the oracle that computes it, but the checked-in file lives with +# the headers it digests so every consumer — CMake's provider-contract stamp, +# the Rust/TS/Swift drift gates — reads it WITHOUT depending on the Python +# binding's _generated.py. Same drift discipline as _generated.py: --check +# fails when either file is stale. +ABIHASH = INCLUDE / "transcribe.abihash" # Fixed-width / size typedefs map to fixed-width ctypes so the generated layout # is correct on every target (never c_long, whose width differs LP64 vs LLP64). @@ -222,7 +229,7 @@ def order_by_value_deps(structs: dict, order: list[str]) -> list[str]: return out -def render(s: Surface, libclang_version: str) -> str: +def render(s: Surface, libclang_version: str) -> tuple[str, str]: # The structural payload (everything below the imports) is built first so a # stable digest can be hashed over it and emitted as PUBLIC_HEADER_HASH. That # digest is the coarse cross-language ABI tag a native provider echoes back @@ -316,7 +323,7 @@ def render(s: Surface, libclang_version: str) -> str: w(f'PUBLIC_HEADER_HASH = "{digest}"') w("") out.extend(body) - return "\n".join(out) + return "\n".join(out), digest def main() -> int: @@ -342,19 +349,29 @@ def main() -> int: print(f"clang error: {d.spelling} at {d.location}", file=sys.stderr) return 2 - text = render(collect(tu), libclang_version) + text, digest = render(collect(tu), libclang_version) + hash_text = digest + "\n" if args.check: + stale = [] current = OUTPUT.read_text() if OUTPUT.exists() else "" if current != text: - print(f"{OUTPUT} is out of date — regenerate with " - "_generate/generate.py", file=sys.stderr) + stale.append(str(OUTPUT)) + current_hash = ABIHASH.read_text() if ABIHASH.exists() else "" + if current_hash != hash_text: + stale.append(str(ABIHASH)) + if stale: + for path in stale: + print(f"{path} is out of date — regenerate with " + "_generate/generate.py", file=sys.stderr) return 1 - print(f"{OUTPUT.name} is up to date") + print(f"{OUTPUT.name} and {ABIHASH.name} are up to date") return 0 OUTPUT.write_text(text) + ABIHASH.write_text(hash_text) print(f"wrote {OUTPUT}") + print(f"wrote {ABIHASH} ({digest})") return 0 diff --git a/bindings/rust/sys/.gitignore b/bindings/rust/sys/.gitignore new file mode 100644 index 00000000..96ef6c0b --- /dev/null +++ b/bindings/rust/sys/.gitignore @@ -0,0 +1,2 @@ +/target +Cargo.lock diff --git a/bindings/rust/sys/Cargo.toml b/bindings/rust/sys/Cargo.toml new file mode 100644 index 00000000..35b5e8d8 --- /dev/null +++ b/bindings/rust/sys/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "transcribe-cpp-sys" +version = "0.0.0" +edition = "2021" +description = "Native FFI bindings for transcribe.cpp (pre-release name reservation placeholder)" +license = "MIT" +readme = "README.md" +repository = "https://github.com/handy-computer/transcribe.cpp" +homepage = "https://github.com/handy-computer/transcribe.cpp" +keywords = ["transcription", "speech", "asr", "stt", "ggml"] +categories = ["multimedia::audio", "external-ffi-bindings"] + +[lib] +name = "transcribe_cpp_sys" +path = "src/lib.rs" diff --git a/bindings/rust/sys/LICENSE b/bindings/rust/sys/LICENSE new file mode 100644 index 00000000..38c7ccc7 --- /dev/null +++ b/bindings/rust/sys/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 The transcribe.cpp authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/bindings/rust/sys/README.md b/bindings/rust/sys/README.md new file mode 100644 index 00000000..ebc36c69 --- /dev/null +++ b/bindings/rust/sys/README.md @@ -0,0 +1,11 @@ +# transcribe-cpp-sys + +Native FFI bindings for [transcribe.cpp](https://github.com/handy-computer/transcribe.cpp), +a C/C++ speech-to-text library built on ggml. + +> **Status: placeholder.** This release reserves the `transcribe-cpp-sys` name +> on crates.io while the first-party bindings are developed. It ships no +> functionality yet. Watch the repository for the first functional release. + +- Crate: `transcribe-cpp-sys` (raw FFI; the safe API will be `transcribe-cpp`) +- License: MIT diff --git a/bindings/rust/sys/src/lib.rs b/bindings/rust/sys/src/lib.rs new file mode 100644 index 00000000..38d81860 --- /dev/null +++ b/bindings/rust/sys/src/lib.rs @@ -0,0 +1,8 @@ +//! Native FFI bindings for [transcribe.cpp](https://github.com/handy-computer/transcribe.cpp). +//! +//! This is a placeholder crate that reserves the `transcribe-cpp-sys` name on +//! crates.io while first-party bindings are developed. It ships no +//! functionality yet. + +/// Version of this placeholder crate. +pub const VERSION: &str = env!("CARGO_PKG_VERSION"); diff --git a/cmake/python-wheel-install.cmake b/cmake/python-wheel-install.cmake index 71965458..a7b26de7 100644 --- a/cmake/python-wheel-install.cmake +++ b/cmake/python-wheel-install.cmake @@ -15,13 +15,11 @@ # --- the library set -------------------------------------------------------- # ggml maintains GGML_AVAILABLE_BACKENDS (internal cache) as backends register; # in GGML_BACKEND_DL builds these are MODULE libraries (the provider-directory -# shape), otherwise ordinary shared libraries linked into libggml. -set(_wheel_targets transcribe ggml ggml-base) -foreach(_backend IN LISTS GGML_AVAILABLE_BACKENDS) - if(TARGET ${_backend}) - list(APPEND _wheel_targets ${_backend}) - endif() -endforeach() +# shape), otherwise ordinary shared libraries linked into libggml. The +# backend-target/kind classification is shared with transcribe-install.cmake. +include("${CMAKE_CURRENT_LIST_DIR}/transcribe-backend-kinds.cmake") +transcribe_backend_kinds(_backend_kinds _backend_targets) +set(_wheel_targets transcribe ggml ggml-base ${_backend_targets}) list(REMOVE_DUPLICATES _wheel_targets) foreach(_tgt IN LISTS _wheel_targets) @@ -50,42 +48,25 @@ endforeach() # --- the build-time contract stamp (_contract.py) --------------------------- # The binding hard-fails on a provider whose version or public-header hash # disagrees with it (_library.validate_contract). Version comes from the -# header macros via PROJECT_VERSION; the header hash is the digest the FFI -# generator emitted into the committed _generated.py (drift-gated in CI), so -# wheel and binding are stamped from the same source. -file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/bindings/python/src/transcribe_cpp/_generated.py" - _hash_line REGEX "^PUBLIC_HEADER_HASH = ") -string(REGEX MATCH "\"([0-9a-f]+)\"" _ "${_hash_line}") -set(_public_header_hash "${CMAKE_MATCH_1}") +# header macros via PROJECT_VERSION; the header hash comes from the +# binding-neutral include/transcribe.abihash — emitted by the FFI generator +# (bindings/python/_generate/generate.py, the hash oracle) and drift-gated in +# CI alongside _generated.py, so wheel and binding are stamped from the same +# source without this file depending on a Python binding artifact. +file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/include/transcribe.abihash" + _public_header_hash LIMIT_COUNT 1) +string(STRIP "${_public_header_hash}" _public_header_hash) if(NOT _public_header_hash MATCHES "^[0-9a-f]+$") message(FATAL_ERROR - "python-wheel-install: failed to parse PUBLIC_HEADER_HASH from " - "bindings/python/src/transcribe_cpp/_generated.py") + "python-wheel-install: failed to read the ABI digest from " + "include/transcribe.abihash (regenerate with " + "bindings/python/_generate/generate.py)") endif() # Backend kinds this artifact supports, for the descriptor's `backends` field -# (and the binding's accelerated-first provider ranking). Derived from the -# ggml backend target names; CPU ISA variants (ggml-cpu-haswell, ...) all -# collapse to "cpu". -set(_kinds "") -foreach(_backend IN LISTS GGML_AVAILABLE_BACKENDS) - string(REGEX REPLACE "^ggml-" "" _kind "${_backend}") - string(REGEX REPLACE "^cpu-.*$" "cpu" _kind "${_kind}") - list(APPEND _kinds "${_kind}") -endforeach() -list(REMOVE_DUPLICATES _kinds) -# Accelerated kinds first, cpu last — readability only; ranking is the -# binding's job. -set(_backend_kinds "") -foreach(_kind IN ITEMS cuda metal vulkan) - if(_kind IN_LIST _kinds) - list(APPEND _backend_kinds "${_kind}") - list(REMOVE_ITEM _kinds "${_kind}") - endif() -endforeach() -list(REMOVE_ITEM _kinds cpu) -list(APPEND _backend_kinds ${_kinds} cpu) - +# (and the binding's accelerated-first provider ranking). Classified by +# transcribe_backend_kinds above: "ggml-" stripped, CPU ISA variants +# (ggml-cpu-haswell, ...) collapsed to "cpu", accelerated kinds first. list(JOIN _backend_kinds "\", \"" _backends_joined) set(TRANSCRIBE_WHEEL_BACKENDS_PY "\"${_backends_joined}\"") set(TRANSCRIBE_PUBLIC_HEADER_HASH "${_public_header_hash}") @@ -97,6 +78,29 @@ configure_file( install(FILES "${CMAKE_CURRENT_BINARY_DIR}/python-wheel/_contract.py" DESTINATION . COMPONENT wheel) +# --- the language-neutral contract twin (contract.json) --------------------- +# Same stamping pass as _contract.py, but JSON and INSIDE _native/, so it +# travels with the native bytes when CI extracts the directory into a +# transcribe-native- bundle (the canonical artifact non-Python +# ecosystems consume). Any binding validates version + header_hash before +# dlopen — the same contract the Python provider enforces. `lane` records +# which official wheel lane built the artifact (null for sdist/dev builds). +if(DEFINED ENV{TRANSCRIBE_WHEEL_LANE} AND NOT "$ENV{TRANSCRIBE_WHEEL_LANE}" STREQUAL "") + set(_contract_lane_json "\"$ENV{TRANSCRIBE_WHEEL_LANE}\"") +else() + set(_contract_lane_json "null") +endif() +file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/python-wheel/contract.json" +"{ + \"version\": \"${PROJECT_VERSION}\", + \"header_hash\": \"${_public_header_hash}\", + \"backends\": [\"${_backends_joined}\"], + \"lane\": ${_contract_lane_json} +} +") +install(FILES "${CMAKE_CURRENT_BINARY_DIR}/python-wheel/contract.json" + DESTINATION _native COMPONENT wheel) + message(STATUS "python wheel: transcribe-cpp-native ${PROJECT_VERSION} " "(header ${_public_header_hash}, backends: ${_backend_kinds})") diff --git a/cmake/transcribe-backend-kinds.cmake b/cmake/transcribe-backend-kinds.cmake new file mode 100644 index 00000000..cabafeed --- /dev/null +++ b/cmake/transcribe-backend-kinds.cmake @@ -0,0 +1,50 @@ +# Shared helper: classify ggml's backend target list into the collapsed +# backend-kind list every artifact contract uses (provider descriptor, +# contract.json, transcribe-link.json). +# +# transcribe_backend_kinds( ) +# : collapsed kind names — "ggml-" prefix stripped, every CPU +# ISA variant (ggml-cpu-haswell, ...) collapsed to "cpu", +# accelerated kinds first, cpu last (readability only; +# ranking is the consumer's job). +# : the GGML_AVAILABLE_BACKENDS entries that exist as targets +# (the libraries/modules an install or wheel must carry). +# +# Reads ggml's GGML_AVAILABLE_BACKENDS internal cache, maintained as backends +# register. Call after add_subdirectory(ggml). +# +# IMPORTANT: that cache is APPEND-ONLY across reconfigures (ggml never +# resets it), so a long-lived build tree that once enabled a backend keeps +# the entry forever. Everything here is therefore filtered to entries that +# exist as TARGETS in the CURRENT configure — kinds included — so a stale +# cache can never stamp a backend into a contract that the artifact does +# not actually contain. (Found the hard way: a local build dir reported +# "blas" with no libggml-blas in the install.) + +function(transcribe_backend_kinds out_kinds out_targets) + set(_targets "") + set(_kinds "") + foreach(_backend IN LISTS GGML_AVAILABLE_BACKENDS) + if(NOT TARGET ${_backend}) + continue() + endif() + list(APPEND _targets ${_backend}) + string(REGEX REPLACE "^ggml-" "" _kind "${_backend}") + string(REGEX REPLACE "^cpu-.*$" "cpu" _kind "${_kind}") + list(APPEND _kinds "${_kind}") + endforeach() + list(REMOVE_DUPLICATES _kinds) + + set(_ordered "") + foreach(_kind IN ITEMS cuda metal vulkan) + if(_kind IN_LIST _kinds) + list(APPEND _ordered "${_kind}") + list(REMOVE_ITEM _kinds "${_kind}") + endif() + endforeach() + list(REMOVE_ITEM _kinds cpu) + list(APPEND _ordered ${_kinds} cpu) + + set(${out_kinds} "${_ordered}" PARENT_SCOPE) + set(${out_targets} "${_targets}" PARENT_SCOPE) +endfunction() diff --git a/cmake/transcribe-install.cmake b/cmake/transcribe-install.cmake new file mode 100644 index 00000000..a4152a51 --- /dev/null +++ b/cmake/transcribe-install.cmake @@ -0,0 +1,182 @@ +# Install rules for consuming the C library OUTSIDE Python packaging — +# the foundation the Rust -sys crate's build.rs (and any plain C/C++ +# consumer) builds on. Included from the top-level CMakeLists.txt when +# TRANSCRIBE_INSTALL is ON (default for top-level non-SKBUILD builds; the +# Python wheel has its own component-filtered install in +# python-wheel-install.cmake). +# +# What `cmake --install` produces: +# include/ transcribe.h, transcribe.abihash, transcribe/*.h +# (plus ggml's public headers via ggml's own install rules) +# lib/ libtranscribe (static or shared per TRANSCRIBE_BUILD_SHARED) +# + ggml/ggml-base/backend libs (ggml's own install rules) +# + transcribe-link.json (the link manifest, see below) +# +# transcribe-link.json is the machine-readable link interface for non-CMake +# consumers: which archives to link in which order, plus the system +# libraries/frameworks/flags they drag in. It is generated from the +# configured build (backend set, OpenMP/BLAS posture), not hardcoded by the +# consumer — the whisper-rs drift class this avoids. The link-smoke CI lane +# (scripts/ci/link_smoke.py) compiles a toy C consumer from NOTHING but this +# manifest, in both postures, so a wrong manifest is a red check, not a +# downstream surprise. +# +# Windows note: the static-posture system-library translation below is +# provisional (zlib via vcpkg spells differently, MSVC has no -framework/ +# -lstdc++); it gets exercised and completed when the Rust branch does its +# MSVC shakeout. The smoke lanes cover linux + macos. + +include(GNUInstallDirs) +include("${CMAKE_CURRENT_LIST_DIR}/transcribe-backend-kinds.cmake") + +# --- headers + the ABI digest ------------------------------------------------ +install(FILES + "${CMAKE_CURRENT_SOURCE_DIR}/include/transcribe.h" + "${CMAKE_CURRENT_SOURCE_DIR}/include/transcribe.abihash" + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) +install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/include/transcribe" + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + FILES_MATCHING PATTERN "*.h") + +# --- libtranscribe ------------------------------------------------------------ +# ggml's own install rules already cover ggml / ggml-base / the backend libs. +# In shared mode, give every library a self-referential rpath so the +# installed lib dir is self-contained: under RUNPATH semantics the LOADING +# object's rpath resolves its deps, so libtranscribe needs $ORIGIN to find +# its sibling libggml*, not just the consumer binary. +transcribe_backend_kinds(_kinds _backend_targets) +if(TRANSCRIBE_BUILD_SHARED) + foreach(_tgt transcribe ggml ggml-base ${_backend_targets}) + if(APPLE) + set_property(TARGET ${_tgt} PROPERTY INSTALL_RPATH "@loader_path") + else() + set_property(TARGET ${_tgt} PROPERTY INSTALL_RPATH "$ORIGIN") + endif() + endforeach() +endif() +install(TARGETS transcribe + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) + +# --- the link manifest (transcribe-link.json) -------------------------------- +set(_libraries transcribe) +set(_library_paths "") +set(_system_libs "") +set(_frameworks "") +set(_link_flags "") + +if(NOT TRANSCRIBE_BUILD_SHARED) + # Static: the consumer links the whole archive set. Order is + # single-pass-ld safe: each archive's undefined refs resolve in a later + # one (transcribe -> ggml -> backends -> ggml-base). + list(APPEND _libraries ggml ${_backend_targets} ggml-base) + + # The archives are C++; the consumer may be C or Rust. + if(APPLE) + list(APPEND _system_libs c++ m) + elseif(UNIX) + list(APPEND _system_libs stdc++ m pthread dl) + endif() + + # Translate libtranscribe's PRIVATE link list (the configure-time truth + # for zlib / OpenMP / Accelerate / system BLAS) into consumer terms. + get_target_property(_transcribe_links transcribe LINK_LIBRARIES) + foreach(_dep IN LISTS _transcribe_links) + if(_dep STREQUAL "ZLIB::ZLIB") + list(APPEND _system_libs z) + elseif(_dep STREQUAL "OpenMP::OpenMP_CXX") + list(APPEND _link_flags -fopenmp) + elseif(_dep MATCHES "^-framework (.+)$") + list(APPEND _frameworks "${CMAKE_MATCH_1}") + elseif(_dep MATCHES "\\.(a|so[.0-9]*|dylib|tbd|lib)$") + # Absolute paths (e.g. find_package(BLAS) results). + list(APPEND _library_paths "${_dep}") + endif() + endforeach() + + # Per-backend system dependencies the ggml archives drag in. + if("metal" IN_LIST _kinds) + list(APPEND _frameworks Foundation Metal MetalKit) + endif() + if("blas" IN_LIST _kinds AND APPLE) + list(APPEND _frameworks Accelerate) + endif() + if("vulkan" IN_LIST _kinds) + list(APPEND _system_libs vulkan) + endif() + if(_frameworks) + list(REMOVE_DUPLICATES _frameworks) + endif() + if(_system_libs) + list(REMOVE_DUPLICATES _system_libs) + endif() +endif() +# Shared: the consumer links libtranscribe alone; the ggml libraries are +# runtime dependencies resolved through the rpaths set above. + +set(_metal_embed false) +if("metal" IN_LIST _kinds AND GGML_METAL_EMBED_LIBRARY) + set(_metal_embed true) +endif() +if(TRANSCRIBE_BUILD_SHARED) + set(_shared_json true) +else() + set(_shared_json false) +endif() +if(TRANSCRIBE_GGML_BACKEND_DL) + set(_backend_dl_json true) + # ggml installs backend MODULES to GGML_BACKEND_DIR when set, else to + # bin/. Record where, so a consumer knows what directory to hand to + # transcribe_init_backends() (DL builds compile in NO backends — without + # that call the process has zero devices). + if(GGML_BACKEND_DIR) + set(_module_dir_json "\"${GGML_BACKEND_DIR}\"") + else() + set(_module_dir_json "\"${CMAKE_INSTALL_BINDIR}\"") + endif() +else() + set(_backend_dl_json false) + set(_module_dir_json null) +endif() + +# JSON list helper: emits `"a", "b"` (empty list -> empty string -> []). +function(_transcribe_json_strings out) + set(_quoted "") + foreach(_item IN LISTS ARGN) + list(APPEND _quoted "\"${_item}\"") + endforeach() + list(JOIN _quoted ", " _joined) + set(${out} "${_joined}" PARENT_SCOPE) +endfunction() +_transcribe_json_strings(_kinds_json ${_kinds}) +_transcribe_json_strings(_libraries_json ${_libraries}) +_transcribe_json_strings(_library_paths_json ${_library_paths}) +_transcribe_json_strings(_system_libs_json ${_system_libs}) +_transcribe_json_strings(_frameworks_json ${_frameworks}) +_transcribe_json_strings(_link_flags_json ${_link_flags}) + +file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/transcribe-link.json" +"{ + \"version\": \"${PROJECT_VERSION}\", + \"commit\": \"${TRANSCRIBE_BUILD_COMMIT}\", + \"shared\": ${_shared_json}, + \"backend_dl\": ${_backend_dl_json}, + \"module_dir\": ${_module_dir_json}, + \"backends\": [${_kinds_json}], + \"metal_embed\": ${_metal_embed}, + \"include_dir\": \"${CMAKE_INSTALL_INCLUDEDIR}\", + \"lib_dir\": \"${CMAKE_INSTALL_LIBDIR}\", + \"libraries\": [${_libraries_json}], + \"library_paths\": [${_library_paths_json}], + \"system_libs\": [${_system_libs_json}], + \"frameworks\": [${_frameworks_json}], + \"link_flags\": [${_link_flags_json}] +} +") +install(FILES "${CMAKE_CURRENT_BINARY_DIR}/transcribe-link.json" + DESTINATION ${CMAKE_INSTALL_LIBDIR}) + +message(STATUS + "transcribe install: ${PROJECT_VERSION} shared=${TRANSCRIBE_BUILD_SHARED} " + "backends: ${_kinds} (link manifest: lib/transcribe-link.json)") diff --git a/docs/bindings.md b/docs/bindings.md index 428f0f5e..6d993359 100644 --- a/docs/bindings.md +++ b/docs/bindings.md @@ -13,6 +13,34 @@ family headers include `transcribe.h`, do not depend on each other, and are flattened normally by C preprocessors, bindgen, CFFI API-mode builds, cgo preambles, and Swift/ObjC module maps. +## ABI digest, contracts, and the link manifest + +Three binding-neutral artifacts exist so no binding depends on another +binding's generated files: + +- **`include/transcribe.abihash`** — the public-ABI digest (sha256/16 over + the normalized FFI surface: structs, enums, macros, layout, prototypes). + Emitted by `bindings/python/_generate/generate.py` (the hash oracle) and + drift-gated in CI alongside `_generated.py`. Every first-class binding + pins this value: when the header's ABI changes, the hash moves and the + binding's CI goes red until its FFI layer is regenerated or consciously + reviewed. Comment-only header edits do not move it. +- **`contract.json`** — stamped into every native artifact directory + (`_native/` in provider wheels; the root of extracted + `transcribe-native-` bundles): `version`, `header_hash`, + `backends`, `lane`. A binding validates `version` (pre-1.0: exact base + match) and `header_hash` (must equal the hash its FFI layer was generated + against) BEFORE dlopen. This is the same contract the Python provider + enforces via `_contract.py`. +- **`lib/transcribe-link.json`** — installed by `cmake --install` (the + `TRANSCRIBE_INSTALL` rules): the machine-readable link interface for + non-CMake consumers building from source (the Rust `-sys` crate's + `build.rs`). Archive order, system libs, frameworks, flags, and — for + `GGML_BACKEND_DL` installs — `module_dir`, the directory to hand to + `transcribe_init_backends()`. Proven per push by the link-smoke CI lane, + which compiles a toy C consumer from nothing but this manifest in both + static and shared postures. + ## Result text pointers: copy at the FFI boundary Every accessor that returns a `const char *` (`transcribe_full_text`, diff --git a/include/transcribe.abihash b/include/transcribe.abihash new file mode 100644 index 00000000..516731e7 --- /dev/null +++ b/include/transcribe.abihash @@ -0,0 +1 @@ +0007af60bfcecf7e diff --git a/scripts/ci/check_lane_mirror.py b/scripts/ci/check_lane_mirror.py new file mode 100644 index 00000000..0cc76646 --- /dev/null +++ b/scripts/ci/check_lane_mirror.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Fail if the wheel-* CMake presets and the pyproject wheel-lane overrides +drift apart. + +The same official build postures are defined twice, on purpose: + + - CMakePresets.json `wheel-*` presets — how a human hand-builds a + provider-shaped artifact outside Python packaging (and, soon, how other + bindings' docs describe the official postures). + - the repo-root pyproject `[[tool.scikit-build.overrides]]` lanes keyed by + TRANSCRIBE_WHEEL_LANE — what cibuildwheel actually ships. + +Both files say "keep in sync" in comments; the 2026-06-12 lane-override +regex bug (unanchored "cpu" matched "cpu-vulkan"; wheels silently shipped +the wrong posture) is what discipline-by-comment buys. This script makes the +mirror structural: + + 1. every lane regex must be anchored (^...$) — the regex bug class itself; + 2. every non-hidden wheel-* preset must be claimed by a lane mapping; + 3. per lane, the EFFECTIVE posture (preset cacheVariables resolved through + `inherits`, vs lane cmake.args on top of the SKBUILD-implied base from + CMakeLists.txt) must agree after applying the documented TRANSCRIBE_* -> + GGML_* derivations. + + uv run --no-project python scripts/ci/check_lane_mirror.py + +Exit 0 on agreement; 1 on drift (with a per-key diff). +""" + +from __future__ import annotations + +import json +import re +import sys +import tomllib +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +PRESETS = REPO / "CMakePresets.json" +PYPROJECT = REPO / "pyproject.toml" + +# lane value -> the presets that ship that posture. A new preset or lane +# must be entered here (check 2 enforces the preset side). +LANE_PRESETS = { + "cpu-vulkan": ["wheel-linux-cpu-vulkan", "wheel-windows-cpu-vulkan"], + "metal": ["wheel-macos-metal"], + # The macOS x86_64 lane shares the conservative-CPU floor the Linux + # preset defines; TRANSCRIBE_METAL=OFF is its only (default-restating) + # addition — see DEFAULT_RESTATEMENTS. + "cpu": ["wheel-linux-cpu"], +} + +# The SKBUILD posture block in CMakeLists.txt: every lane builds on top of +# this implicitly; the presets restate it explicitly (minus the build-shape +# keys stripped by IGNORED_KEYS). +SKBUILD_IMPLIED = { + "TRANSCRIBE_BUILD_SHARED": "ON", + "TRANSCRIBE_USE_OPENMP": "OFF", + "TRANSCRIBE_USE_SYSTEM_BLAS": "OFF", + "GGML_METAL_EMBED_LIBRARY": "ON", +} + +# Not posture: build type and which targets get built. +IGNORED_KEYS = { + "CMAKE_BUILD_TYPE", + "TRANSCRIBE_BUILD_TESTS", + "TRANSCRIBE_BUILD_EXAMPLES", + "TRANSCRIBE_BUILD_TOOLS", +} + +# Keys that one side may pin to a value that is already the platform +# default while the other side omits it (pinned for intent, not effect). +DEFAULT_RESTATEMENTS = { + ("TRANSCRIBE_METAL", "OFF"), # default everywhere but Apple Silicon + ("GGML_METAL", "OFF"), +} + + +def normalize(value: object) -> str: + s = str(value).strip().upper() + return {"TRUE": "ON", "FALSE": "OFF", "1": "ON", "0": "OFF"}.get(s, s) + + +def derive(posture: dict[str, str]) -> dict[str, str]: + """Apply the TRANSCRIBE_* -> GGML_* fan-outs CMakeLists.txt performs, so + a side that states only the TRANSCRIBE_ knob equals a side that also + restates the GGML_ effect.""" + out = dict(posture) + fanout = { + "TRANSCRIBE_METAL": [("GGML_METAL", None)], + "TRANSCRIBE_VULKAN": [("GGML_VULKAN", None)], + "TRANSCRIBE_CUDA": [("GGML_CUDA", None)], + # BACKEND_DL forces GGML_BACKEND_DL and GGML_NATIVE=OFF. + "TRANSCRIBE_GGML_BACKEND_DL": [("GGML_BACKEND_DL", None), + ("GGML_NATIVE", "OFF")], + # The conservative floor defaults GGML_NATIVE (and the SIMD tiers, + # which neither file lists) to OFF. + "TRANSCRIBE_X86_CONSERVATIVE": [("GGML_NATIVE", "OFF")], + } + for knob, effects in fanout.items(): + if out.get(knob) == "ON": + for key, forced in effects: + out.setdefault(key, forced if forced else "ON") + return out + + +def resolved_preset(presets: dict[str, dict], name: str) -> dict[str, str]: + node = presets[name] + base: dict[str, str] = {} + inherits = node.get("inherits", []) + for parent in [inherits] if isinstance(inherits, str) else inherits: + base.update(resolved_preset(presets, parent)) + base.update({k: normalize(v) for k, v in node.get("cacheVariables", {}).items()}) + return base + + +def main() -> int: + presets_doc = json.loads(PRESETS.read_text()) + presets = {p["name"]: p for p in presets_doc["configurePresets"]} + + pyproject = tomllib.loads(PYPROJECT.read_text()) + overrides = pyproject["tool"]["scikit-build"].get("overrides", []) + + failures: list[str] = [] + + # Lane overrides: anchored regexes, literal lane names, parsed -D args. + lanes: dict[str, dict[str, str]] = {} + for ov in overrides: + pattern = ov.get("if", {}).get("env", {}).get("TRANSCRIBE_WHEEL_LANE") + if pattern is None: + continue + m = re.fullmatch(r"\^([A-Za-z0-9_-]+)\$", pattern) + if not m: + failures.append( + f"lane override pattern {pattern!r} must be an anchored " + f"literal (^lane$) — unanchored patterns are the 2026-06-12 " + f"wrong-posture bug") + continue + lane = m.group(1) + flags: dict[str, str] = {} + for arg in ov.get("cmake", {}).get("args", []): + dm = re.fullmatch(r"-D([A-Za-z0-9_]+)=(.+)", arg) + if not dm: + failures.append(f"lane {lane}: unparseable cmake arg {arg!r}") + continue + flags[dm.group(1)] = normalize(dm.group(2)) + lanes[lane] = flags + + if set(lanes) != set(LANE_PRESETS): + failures.append( + f"lane set mismatch: pyproject has {sorted(lanes)}, " + f"LANE_PRESETS maps {sorted(LANE_PRESETS)} — update the mapping") + + claimed = {p for names in LANE_PRESETS.values() for p in names} + visible_wheel_presets = { + name for name, node in presets.items() + if name.startswith("wheel-") and not node.get("hidden", False) + } + unclaimed = visible_wheel_presets - claimed + if unclaimed: + failures.append( + f"presets not claimed by any lane: {sorted(unclaimed)} — a new " + f"wheel preset needs a lane (or an entry here explaining why not)") + + def strip_and_settle(posture: dict[str, str]) -> dict[str, str]: + eff = derive({k: v for k, v in posture.items() if k not in IGNORED_KEYS}) + # The embed knob is dead without Metal: SKBUILD implies it ON for + # every lane (a no-op off-macOS) while only the metal preset states + # it. Compare it only where Metal is actually on. + if eff.get("GGML_METAL") != "ON" and eff.get("TRANSCRIBE_METAL") != "ON": + eff.pop("GGML_METAL_EMBED_LIBRARY", None) + return eff + + for lane, preset_names in LANE_PRESETS.items(): + if lane not in lanes: + continue + lane_eff = strip_and_settle({**SKBUILD_IMPLIED, **lanes[lane]}) + for preset_name in preset_names: + preset_eff = strip_and_settle(resolved_preset(presets, preset_name)) + keys = set(lane_eff) | set(preset_eff) + for key in sorted(keys): + lv, pv = lane_eff.get(key), preset_eff.get(key) + if lv == pv: + continue + missing_side = "lane" if lv is None else "preset" + present_value = pv if lv is None else lv + if (key, present_value) in DEFAULT_RESTATEMENTS and None in (lv, pv): + continue # one side pins a platform default for intent + failures.append( + f"{lane} vs {preset_name}: {key}: lane={lv} preset={pv} " + f"({missing_side} side missing or different)") + + if failures: + print("lane/preset mirror drift:", file=sys.stderr) + for f in failures: + print(f" - {f}", file=sys.stderr) + return 1 + + print(f"lane mirror ok: {sorted(lanes)} match " + f"{sorted(visible_wheel_presets)} (postures settled + compared)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci/extract_native_bundle.py b/scripts/ci/extract_native_bundle.py new file mode 100644 index 00000000..e86f2888 --- /dev/null +++ b/scripts/ci/extract_native_bundle.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +"""Extract the canonical native bundle from a repaired provider wheel. + + python3 scripts/ci/extract_native_bundle.py \ + --wheel-dir wheelhouse --tuple linux-x86_64-cpu-vulkan --out bundles + +Produces /transcribe-native-.tar.gz containing: + + transcribe-native-/ + libtranscribe.* + libggml* (+ backend modules) <- the wheel's + contract.json (version, header_hash, backends, lane) _native/ dir + licenses/ (the wheel's .dist-info/licenses tree) + +The REPAIRED wheel is the most-validated native artifact this project +produces (auditwheel/delvewheel/delocate + cibuildwheel's test phase + +the hardware-truth lanes all ran against it), so the bundle is those exact +bytes re-containered — the decided mechanism (2026-06-13) by which npm +platform packages, prebuilt-Rust, and any future ecosystem share one +canonical build per tuple instead of rebuilding. "Same native bytes" holds +by construction. + +Notes: + - Windows wheels carry no import .lib (the wheel install filters the + wheel-dev component out); dlopen-style consumers need none, and + compiled consumers are served by the TRANSCRIBE_INSTALL path. Revisit + if an ecosystem needs the .lib inside bundles. + - The tar is deterministic-ish (sorted names, zeroed mtimes) so re-runs + of the same wheel produce byte-identical bundles. + +Stdlib only. +""" + +from __future__ import annotations + +import argparse +import io +import json +import sys +import tarfile +import zipfile +from pathlib import Path + + +def fail(msg: str) -> "NoReturn": # noqa: F821 - py3.9 compat, comment only + print(f"error: {msg}", file=sys.stderr) + raise SystemExit(1) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--wheel-dir", required=True, + help="directory containing exactly one repaired *.whl") + ap.add_argument("--tuple", required=True, dest="tuple_name", + help="build tuple, e.g. linux-x86_64-cpu-vulkan") + ap.add_argument("--out", required=True, help="output directory") + args = ap.parse_args() + + wheels = sorted(Path(args.wheel_dir).glob("*.whl")) + if len(wheels) != 1: + fail(f"expected exactly one wheel in {args.wheel_dir}, found " + f"{[w.name for w in wheels]}") + wheel = wheels[0] + + bundle_name = f"transcribe-native-{args.tuple_name}" + out_dir = Path(args.out) + out_dir.mkdir(parents=True, exist_ok=True) + out_tar = out_dir / f"{bundle_name}.tar.gz" + + # bundle-relative path -> bytes + members: dict[str, bytes] = {} + with zipfile.ZipFile(wheel) as zf: + for info in zf.infolist(): + if info.is_dir(): + continue + parts = Path(info.filename).parts + # /_native/ -> flattened bundle root + if len(parts) >= 3 and parts[1] == "_native" and \ + parts[0].startswith("transcribe_cpp_native"): + members["/".join(parts[2:])] = zf.read(info) + # .dist-info/licenses/ -> licenses/ + elif len(parts) >= 3 and parts[0].endswith(".dist-info") and \ + parts[1] == "licenses": + members["licenses/" + "/".join(parts[2:])] = zf.read(info) + + native_files = [n for n in members if "/" not in n] + if not any(n.startswith("libtranscribe.") or n == "transcribe.dll" + for n in native_files): + fail(f"no libtranscribe in the wheel's _native/ (got {native_files})") + if "contract.json" not in members: + fail("contract.json missing from _native/ — wheel built before the " + "contract stamp, or the install rules regressed") + if not any(n.startswith("licenses/") for n in members): + fail("no license files found in the wheel's dist-info") + + contract = json.loads(members["contract.json"]) + for key in ("version", "header_hash", "backends", "lane"): + if key not in contract: + fail(f"contract.json missing key {key!r}: {contract}") + + with tarfile.open(out_tar, "w:gz") as tf: + for rel in sorted(members): + data = members[rel] + info = tarfile.TarInfo(name=f"{bundle_name}/{rel}") + info.size = len(data) + info.mtime = 0 + info.mode = 0o755 if "/" not in rel and rel != "contract.json" \ + else 0o644 + tf.addfile(info, io.BytesIO(data)) + + total = sum(len(v) for v in members.values()) + print(f"{out_tar}: {len(members)} files, {total / 1e6:.1f} MB uncompressed," + f" {out_tar.stat().st_size / 1e6:.1f} MB compressed") + print(f"contract: version={contract['version']} " + f"hash={contract['header_hash']} backends={contract['backends']} " + f"lane={contract['lane']}") + print(f"native files: {sorted(native_files)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci/link_smoke.c b/scripts/ci/link_smoke.c new file mode 100644 index 00000000..710117bd --- /dev/null +++ b/scripts/ci/link_smoke.c @@ -0,0 +1,57 @@ +/* Minimal EXTERNAL consumer of an installed transcribe tree. + * + * Compiled by scripts/ci/link_smoke.py against `cmake --install` output, + * with a link line constructed ONLY from lib/transcribe-link.json — this + * program plus that manifest are the contract a non-CMake consumer (the + * Rust -sys crate's build.rs, above all) relies on. No model is loaded: + * the assertions are link + runtime-registry health, which is exactly what + * a wrong archive order / missing system lib / broken rpath breaks. + */ +#include +#include + +int main(int argc, char ** argv) { + const char * version = transcribe_version(); + printf("version=%s commit=%s\n", version, transcribe_version_commit()); + if (version == NULL || version[0] == '\0') { + fprintf(stderr, "link-smoke: empty version\n"); + return 1; + } + + /* Compiled-in backends register without transcribe_init_backends(). + * GGML_BACKEND_DL builds compile in NONE — the driver passes the + * installed module directory (manifest `module_dir`) as argv[1], the + * same call a real DL-posture consumer must make. */ + if (argc > 1) { + transcribe_status st = transcribe_init_backends(argv[1]); + printf("init_backends(%s) -> %d\n", argv[1], (int) st); + if (st != TRANSCRIBE_OK) { + fprintf(stderr, "link-smoke: init_backends failed\n"); + return 1; + } + } + + int n = transcribe_backend_device_count(); + printf("devices=%d\n", n); + if (n < 1) { + fprintf(stderr, "link-smoke: no registered compute devices\n"); + return 1; + } + for (int i = 0; i < n; i++) { + struct transcribe_backend_device dev; + transcribe_backend_device_init(&dev); + if (transcribe_get_backend_device(i, &dev) != TRANSCRIBE_OK) { + fprintf(stderr, "link-smoke: device %d query failed\n", i); + return 1; + } + printf("device[%d]=%s kind=%s\n", i, dev.name, dev.kind); + } + + if (!transcribe_backend_available(TRANSCRIBE_BACKEND_CPU)) { + fprintf(stderr, "link-smoke: CPU backend unavailable\n"); + return 1; + } + + printf("link-smoke ok\n"); + return 0; +} diff --git a/scripts/ci/link_smoke.py b/scripts/ci/link_smoke.py new file mode 100644 index 00000000..6f2a4323 --- /dev/null +++ b/scripts/ci/link_smoke.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Compile and run a toy C consumer against an INSTALLED transcribe tree. + + python3 scripts/ci/link_smoke.py --prefix [--cc cc] + +The link line is constructed from NOTHING but the installed +lib/transcribe-link.json — the manifest is the artifact under test. If the +manifest's archive order, system-library list, frameworks, or flags are +wrong, this fails at link or run time, which is the point: the manifest is +what the Rust -sys crate's build.rs (and any non-CMake consumer) will trust. + +Covers both postures from one entry point: the manifest says whether the +install is static or shared; shared adds an rpath to the installed lib dir +and asserts the binary runs without LD_LIBRARY_PATH/DYLD_* help. + +Stdlib only. +""" + +from __future__ import annotations + +import argparse +import json +import platform +import subprocess +import sys +import tempfile +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +SOURCE = REPO / "scripts" / "ci" / "link_smoke.c" + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--prefix", required=True, help="cmake --install prefix") + ap.add_argument("--cc", default="cc", help="C compiler driver (default cc)") + args = ap.parse_args() + + prefix = Path(args.prefix).resolve() + manifest_path = prefix / "lib" / "transcribe-link.json" + manifest = json.loads(manifest_path.read_text()) + print(f"manifest: {manifest_path}") + print(json.dumps(manifest, indent=2)) + + include_dir = prefix / manifest["include_dir"] + lib_dir = prefix / manifest["lib_dir"] + out = Path(tempfile.mkdtemp(prefix="link-smoke-")) / "link_smoke" + + cmd = [args.cc, str(SOURCE), f"-I{include_dir}", "-o", str(out)] + cmd += manifest["link_flags"] + cmd += [f"-L{lib_dir}"] + + libs = [f"-l{name}" for name in manifest["libraries"]] + if manifest["libraries"][1:] and platform.system() == "Linux": + # Static archive sets are order-sensitive under single-pass GNU ld; + # group them so the manifest's content (not its luck) is what's + # being tested. macOS ld64 resolves regardless of order. + cmd += ["-Wl,--start-group", *libs, *manifest["library_paths"], + "-Wl,--end-group"] + else: + cmd += [*libs, *manifest["library_paths"]] + + cmd += [f"-l{name}" for name in manifest["system_libs"]] + for framework in manifest["frameworks"]: + cmd += ["-framework", framework] + if manifest["shared"]: + cmd += [f"-Wl,-rpath,{lib_dir}"] + + print("compile:", " ".join(cmd)) + subprocess.run(cmd, check=True) + + # Run with a clean environment posture: no loader-path help. The rpaths + # (binary -> lib_dir; installed libs -> $ORIGIN/@loader_path) must carry + # the shared case on their own. DL installs compile in no backends: + # hand the toy the installed module directory, the call a real + # DL-posture consumer makes. + run_cmd = [str(out)] + if manifest["backend_dl"]: + run_cmd.append(str(prefix / manifest["module_dir"])) + res = subprocess.run(run_cmd, capture_output=True, text=True) + sys.stdout.write(res.stdout) + sys.stderr.write(res.stderr) + if res.returncode != 0: + print(f"link-smoke FAILED (exit {res.returncode})", file=sys.stderr) + return 1 + if "link-smoke ok" not in res.stdout: + print("link-smoke FAILED (missing ok marker)", file=sys.stderr) + return 1 + posture = "shared" if manifest["shared"] else "static" + print(f"link-smoke ok ({posture}, backends: {manifest['backends']})") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())