Skip to content
publish

typescript bindings (#31) #3

Sign in to view logs

typescript bindings (#31)

typescript bindings (#31) #3

Workflow file for this run

.github/workflows/publish.yml at 37a159f
name: publish
# Release pipeline (plan M5 tail). Auth is PyPI Trusted Publishing (OIDC) —
# no tokens anywhere; the publishers are registered against THIS filename
# (publish.yml) and the `testpypi` / `pypi` environments.
#
# REHEARSAL (workflow_dispatch): builds + validates the FULL matrix by
# calling python-wheels.yml and cuda-windows.yml, publishes the
# PyPI-bound set (api wheel, native wheels, sdist — the dist-*
# artifacts) to TestPyPI, then proves a clean-machine install FROM
# TestPyPI with a real transcription. cu12 wheels are NOT rehearsed on
# TestPyPI (197 MB > its 100 MB file cap); their home is GitHub release
# assets + the /whl/cu12 index, exercised on real tags.
# NOTE: (Test)PyPI filenames are write-once — a failed rehearsal
# iteration means bumping the patch version in include/transcribe.h
# (the single version source) and re-dispatching.
#
# RELEASE (push tag v0.0.X): same full build, then with the `pypi`
# environment (add a required-reviewer rule once the repo is public):
# dist-* → PyPI, cu12 wheels → draft GitHub release assets for the tag,
# Rust crates → crates.io, Swift → the draft release, then the draft is
# published and wheel-index.yml is dispatched so /whl/cu12 picks it up.
# cu12 → PyPI additionally when the repo variable CU12_ON_PYPI is "true"
# (set it once the PyPI file-size request is granted).
on:
push:
tags: ["v[0-9]*"]
workflow_dispatch:
# Never two publishes at once; never cancel one mid-upload.
concurrency:
group: publish
cancel-in-progress: false
jobs:
# Full build + validation matrix; the artifacts published below are the
# exact files these workflows tested post-repair.
wheels:
uses: ./.github/workflows/python-wheels.yml
secrets: inherit
cuda-windows:
# Tags only: the heaviest build in the project (nvcc over ggml-cuda x 5
# arches under MSVC, ~3 h on 16vcpu), and the TestPyPI rehearsal neither
# publishes nor smokes its output (cu12 wheels exceed TestPyPI's file
# cap). Validate it on demand with its own workflow_dispatch.
if: startsWith(github.ref, 'refs/tags/v')
uses: ./.github/workflows/cuda-windows.yml
secrets: inherit
create-release:
# Tags only. Owns creation of the GitHub Release object. Keep it draft
# until the mandatory publishers have succeeded and assets are verified.
if: startsWith(github.ref, 'refs/tags/v')
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- name: Create the draft release for the tag
env:
GH_TOKEN: ${{ github.token }}
run: |
set -e
tag="${GITHUB_REF_NAME}"
gh release view "$tag" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1 || \
gh release create "$tag" --repo "$GITHUB_REPOSITORY" \
--title "$tag" --notes "transcribe.cpp $tag" --verify-tag --draft
publish-testpypi:
# Rehearsal target (dispatch only — real tags go to PyPI).
# Blacksmith, not the Hetzner box: gh-action-pypi-publish is a Docker
# container action and needs docker on the runner.
if: github.event_name == 'workflow_dispatch'
needs: [wheels]
runs-on: blacksmith-2vcpu-ubuntu-2404
timeout-minutes: 30
environment: testpypi
permissions:
id-token: write # OIDC for trusted publishing
steps:
- uses: actions/download-artifact@v8
with:
pattern: dist-*
merge-multiple: true
path: dist
- run: ls -la dist/
- uses: pypa/gh-action-pypi-publish@release/v1
with:
repository-url: https://test.pypi.org/legacy/
packages-dir: dist
smoke-testpypi:
# The point of the rehearsal: a clean machine installs from TestPyPI
# alone (resolver pulls the native provider through the hard pin) and
# transcribes. Mirrors clean-install, but the index is the real thing.
needs: [publish-testpypi]
runs-on: blacksmith-2vcpu-ubuntu-2404
container: python:3.12-slim
timeout-minutes: 30
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
steps:
- uses: actions/download-artifact@v8
with:
name: dist-api
path: dist-api
- uses: actions/download-artifact@v8
with:
# samples/jfk.wav (+ the degradation script), uploaded by api-wheel;
# files keep their repo-relative paths inside the artifact.
name: smoke-assets
path: smoke-assets
- name: Install from TestPyPI (exact just-published version) and transcribe
run: |
set -e
ver=$(ls dist-api/transcribe_cpp-*.whl | sed -E 's/.*transcribe_cpp-([0-9][^-]*)-.*/\1/')
echo "published version: $ver"
# TestPyPI can lag a few seconds behind a fresh upload; retry.
for i in $(seq 1 10); do
pip install -q \
--index-url https://test.pypi.org/simple/ \
--extra-index-url https://pypi.org/simple/ \
"transcribe-cpp==$ver" && break
echo "index not ready yet ($i); retrying in 30s" && sleep 30
done
python - <<'EOF'
import transcribe_cpp as t
print("provider:", t.native_provider())
print("devices:", [(d.name, d.kind) for d in t.backends()])
assert t.backend_available("cpu")
EOF
if [ -z "$HF_TOKEN" ]; then
echo "no HF_TOKEN (fork?) — install-only check passed, skipping the model smoke"
exit 0
fi
pip install -q huggingface_hub
hf download handy-computer/whisper-tiny-gguf \
whisper-tiny-Q5_K_M.gguf --local-dir canary
python - <<'EOF'
import array, wave
import transcribe_cpp as t
with wave.open("smoke-assets/samples/jfk.wav", "rb") as w:
pcm16 = array.array("h"); pcm16.frombytes(w.readframes(w.getnframes()))
pcm = array.array("f", (s / 32768.0 for s in pcm16))
with t.Model("canary/whisper-tiny-Q5_K_M.gguf") as m, m.session() as s:
text = s.run(pcm).text
print("text:", text.strip())
assert "country" in text.lower(), text
print("ok: clean machine installed from TestPyPI and transcribed")
EOF
publish-pypi:
# The real thing. Gate: only version tags reach this, and the `pypi`
# environment carries the approval rule once the repo is public.
if: startsWith(github.ref, 'refs/tags/v')
needs: [wheels, cuda-windows]
# Blacksmith for the same docker-container-action reason as testpypi.
runs-on: blacksmith-2vcpu-ubuntu-2404
timeout-minutes: 30
environment: pypi
permissions:
id-token: write
steps:
- uses: actions/download-artifact@v8
with:
pattern: dist-*
merge-multiple: true
path: dist
- name: Add cu12 wheels (only once the PyPI size request is granted)
if: vars.CU12_ON_PYPI == 'true'
uses: actions/download-artifact@v8
with:
pattern: cuda-dist-*
merge-multiple: true
path: dist
- run: ls -la dist/
- uses: pypa/gh-action-pypi-publish@release/v1
with:
packages-dir: dist
release-assets:
# cu12's primary distribution home: wheels as GitHub release assets,
# served to pip through the PEP 503 index on Pages (wheel-index.yml).
if: startsWith(github.ref, 'refs/tags/v')
needs: [create-release, wheels, cuda-windows]
runs-on: [self-hosted, Linux, X64, hetzner]
timeout-minutes: 30
permissions:
contents: write # upload release assets
steps:
- uses: actions/download-artifact@v8
with:
pattern: cuda-dist-*
merge-multiple: true
path: cu12
# The canonical native bundles (extracted from the repaired wheels by
# python-wheels.yml). On the release they become the distribution home
# for non-PyPI ecosystems: npm platform packages and prebuilt-Rust
# fetch these exact bytes by tag instead of rebuilding.
- uses: actions/download-artifact@v8
with:
pattern: native-*
merge-multiple: true
path: native-bundles
- name: Attach the cu12 wheels
env:
GH_TOKEN: ${{ github.token }}
run: |
set -e
tag="${GITHUB_REF#refs/tags/}"
gh release upload "$tag" cu12/*.whl --repo "$GITHUB_REPOSITORY" --clobber
- name: Attach the native bundles (versioned names)
env:
GH_TOKEN: ${{ github.token }}
run: |
set -e
tag="${GITHUB_REF#refs/tags/}"
ver="${tag#v}"
mkdir -p upload
for f in native-bundles/transcribe-native-*.tar.gz; do
base="$(basename "$f" .tar.gz)"
tuple="${base#transcribe-native-}"
cp "$f" "upload/transcribe-native-${ver}-${tuple}.tar.gz"
done
ls -la upload/
gh release upload "$tag" upload/*.tar.gz --repo "$GITHUB_REPOSITORY" --clobber
# ---- Rust crates (crates.io) -------------------------------------------------
# The Rust release path mirrors the Python one: a dispatch REHEARSAL that
# validates without uploading, and a tag-gated RELEASE cut from CI. crates.io
# has no TestPyPI analog, so the rehearsal is `cargo publish --dry-run` plus
# the packed-crate smoke (build libtranscribe from the shipped tarball and
# transcribe) — the Rust "test the shipped artifact" gate (requirements §4).
rust-rehearsal:
# Dispatch only. Independent of the wheel jobs (no `needs`): the Rust and
# Python release artifacts don't share build steps.
if: github.event_name == 'workflow_dispatch'
runs-on: blacksmith-2vcpu-ubuntu-2404
timeout-minutes: 40
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
steps:
- uses: actions/checkout@v6
- uses: dtolnay/rust-toolchain@stable
- uses: astral-sh/setup-uv@v8.2.0 # fetch-canary + the packed-smoke script
- name: Install build deps
run: sudo apt-get update && sudo apt-get install -y cmake ninja-build zlib1g-dev
- name: Dry-run transcribe-cpp-sys (full verify build, as cargo would publish it)
run: cargo publish --dry-run --allow-dirty -p transcribe-cpp-sys
# The safe crate cannot be dry-run-verified pre-publish: it depends on
# transcribe-cpp-sys = "0.0.1", and the registry only has the 0.0.0
# name-reservation placeholder until sys actually publishes (sys-first
# ordering). Its packaging is verified at release time below; here the
# packed-crate smoke exercises it against the packed sys instead.
- uses: ./.github/actions/fetch-canary
with:
hf-token: ${{ secrets.HF_TOKEN }}
- name: Packed-crate smoke (build the native lib from the shipped tarball, transcribe)
run: uv run --no-project python scripts/ci/rust_packed_smoke.py
rust-release:
# Tags only. Releases are cut from CI, never a laptop (requirements §5).
# sys publishes FIRST (the safe crate resolves it by version), then the safe
# crate. The `crates-io` environment carries the approval gate (CJ approves)
# and holds CARGO_REGISTRY_TOKEN. cargo waits for the index to carry sys
# before returning, so the safe publish that follows resolves it.
if: startsWith(github.ref, 'refs/tags/v')
runs-on: blacksmith-2vcpu-ubuntu-2404
timeout-minutes: 40
environment: crates-io
env:
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
steps:
- uses: actions/checkout@v6
- uses: dtolnay/rust-toolchain@stable
- name: Install build deps
run: sudo apt-get update && sudo apt-get install -y cmake ninja-build zlib1g-dev
- name: Publish transcribe-cpp-sys (the native-carrying crate; verifies the build)
run: cargo publish -p transcribe-cpp-sys
- name: Publish transcribe-cpp (the safe wrapper; resolves the just-published sys)
run: cargo publish -p transcribe-cpp
# ---------------------------------------------------------------------------
# Swift binding (TranscribeCpp) — xcframework release.
#
# The Swift package is consumed as a prebuilt static `.xcframework`
# binaryTarget (notes/swift-bindings-plan.md; requirements §5). Releasing it
# means: build the four Apple slices, zip + checksum the xcframework, attach
# the zip as a release asset, then point the mirror repo's Package.swift
# `binaryTarget(url:checksum:)` at it. "Releases are cut from CI, never a
# laptop." macOS runner: the xcframework needs Xcode (libtool/xcodebuild).
# ---------------------------------------------------------------------------
swift-rehearsal:
# The shipped-artifact gate (§4): build the real xcframework and run the
# suite against it (transcribes the canary through the published shape).
if: github.event_name == 'workflow_dispatch'
runs-on: macos-15
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
steps:
- uses: actions/checkout@v4
- uses: astral-sh/setup-uv@v8.2.0
- run: brew install ninja
- name: Build the full xcframework (macOS + iOS device + iOS simulator)
run: scripts/ci/build_xcframework.sh
- name: Package + checksum (proves the release artifact + licenses)
run: scripts/ci/package_xcframework.sh
- uses: ./.github/actions/fetch-canary
with:
hf-token: ${{ secrets.HF_TOKEN }}
- name: swift test against the built xcframework
working-directory: bindings/swift
run: swift test
swift-release:
# Tags only. Builds the artifact, attaches the zip to the tag's release, and
# emits the checksum. The mirror-repo Package.swift bump is the final step
# (CJ-gated — needs the dedicated SwiftPM repo + a deploy key; see the plan).
if: startsWith(github.ref, 'refs/tags/v')
needs: [create-release]
runs-on: macos-15
permissions:
contents: write # attach the release asset
steps:
- uses: actions/checkout@v4
- run: brew install ninja
- name: Build the full xcframework
run: scripts/ci/build_xcframework.sh
- name: Package + checksum
id: pkg
run: |
scripts/ci/package_xcframework.sh | tee pkg.txt
echo "checksum=$(awk '/checksum:/ {print $2}' pkg.txt)" >> "$GITHUB_OUTPUT"
- name: Attach the xcframework zip to the release
env:
GH_TOKEN: ${{ github.token }}
run: |
set -e
tag="${GITHUB_REF_NAME}"
gh release upload "$tag" \
bindings/swift/build-apple/TranscribeCpp.xcframework.zip \
--repo "$GITHUB_REPOSITORY" --clobber
- name: Checksum for the mirror repo's Package.swift
run: |
echo "binaryTarget(url: .../TranscribeCpp.xcframework.zip,"
echo " checksum: \"${{ steps.pkg.outputs.checksum }}\")"
# TODO(CJ): push the thin Swift sources + the url/checksum-bearing
# Package.swift to the dedicated mirror repo (transcribe-cpp-swift) and tag
# it, so `swift package add` resolves the release. Needs the mirror repo +
# a deploy key secret. Until then, the asset + checksum above are produced
# but not wired into a resolvable SwiftPM tag.
# ---- npm (TypeScript) --------------------------------------------------------
# Mirrors the Python/Rust path: a dispatch REHEARSAL that builds + installs the
# shipped tarballs and transcribes the canary (the §4 shipped-artifact gate),
# and a tag-gated RELEASE that turns the native bundles into per-platform npm
# packages and publishes them with the pure-TS API package.
ts-rehearsal:
# Pack the API + a platform package, install from the tarballs into a clean
# dir, and transcribe the canary — exactly the clean-machine path, no
# TRANSCRIBE_LIBRARY (the install stands on the platform package alone).
if: github.event_name == 'workflow_dispatch'
runs-on: [self-hosted, macOS, ARM64]
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 22
- uses: astral-sh/setup-uv@v8.2.0
- run: command -v ninja >/dev/null || brew install ninja
- name: Build + install a shared libtranscribe (the native-bundle twin)
run: |
cmake -B build-shared -G Ninja -DTRANSCRIBE_BUILD_SHARED=ON \
-DGGML_METAL=ON -DGGML_METAL_EMBED_LIBRARY=ON
cmake --build build-shared --target transcribe
cmake --install build-shared --prefix "$PWD/ts-native"
- name: Pack API + platform tarballs
working-directory: bindings/typescript
run: |
npm install --no-audit --no-fund
npm run build
ver="$(node -p "require('./package.json').version")"
node scripts/pack-platform.mjs --lib-dir "$GITHUB_WORKSPACE/ts-native/lib" \
--tuple darwin-arm64-metal --version "$ver" --out /tmp/ts-platform
rm -rf /tmp/ts-tarballs && mkdir -p /tmp/ts-tarballs
npm pack --pack-destination /tmp/ts-tarballs
(cd /tmp/ts-platform/darwin-arm64-metal && npm pack --pack-destination /tmp/ts-tarballs)
- uses: ./.github/actions/fetch-canary
with:
hf-token: ${{ secrets.HF_TOKEN }}
- name: Install from tarballs (clean dir) and transcribe the canary
run: |
rm -rf /tmp/ts-consumer && mkdir -p /tmp/ts-consumer
(cd /tmp/ts-consumer && npm init -y >/dev/null && \
npm install --no-audit --no-fund /tmp/ts-tarballs/*.tgz)
TS_PKG_ENTRY=/tmp/ts-consumer/node_modules/transcribe-cpp/dist/index.js \
TRANSCRIBE_SMOKE_AUDIO="$GITHUB_WORKSPACE/samples/jfk.wav" \
node scripts/ci/ts_packed_smoke.mjs
ts-release:
# Tags only. Build each @transcribe-cpp/<platform> package from its native
# bundle (the exact bytes the wheels shipped) and publish them + the pure-TS
# API package to npm. The `npm` environment carries the approval gate.
if: startsWith(github.ref, 'refs/tags/v')
needs: [create-release, wheels]
runs-on: blacksmith-2vcpu-ubuntu-2404
environment: npm
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 22
registry-url: "https://registry.npmjs.org"
- uses: actions/download-artifact@v8
with:
pattern: native-*
merge-multiple: true
path: native-bundles
- name: Build platform packages from the native bundles
working-directory: bindings/typescript
run: |
set -e
npm install --no-audit --no-fund
npm run build
ver="$(node -p "require('./package.json').version")"
# Sync the API package's @transcribe-cpp/* platform pins to the version
# we publish, so the tarball pins the exact bytes we ship (the static
# gate in check_version_sync.py enforces the base match; this nails the
# exact pin). Without this the pins go stale on the first version bump
# and the install resolves a mismatched native -> load-time AbiError.
node -e '
const fs = require("node:fs");
const v = process.argv[1];
const p = JSON.parse(fs.readFileSync("package.json", "utf8"));
for (const k of Object.keys(p.optionalDependencies || {})) p.optionalDependencies[k] = v;
fs.writeFileSync("package.json", JSON.stringify(p, null, 2) + "\n");
' "$ver"
out=/tmp/ts-platform
# build-tuple (bundle) -> npm tuple (Node platform-arch convention)
map() { case "$1" in
macos-arm64-metal) echo darwin-arm64-metal ;;
macos-x86_64-cpu) echo darwin-x64-cpu ;;
linux-x86_64-cpu-vulkan) echo linux-x64-cpu-vulkan ;;
linux-aarch64-cpu-vulkan) echo linux-arm64-cpu-vulkan ;;
windows-x86_64-cpu-vulkan) echo win32-x64-cpu-vulkan ;;
*) echo "" ;; esac; }
for f in "$GITHUB_WORKSPACE"/native-bundles/transcribe-native-*.tar.gz; do
base="$(basename "$f" .tar.gz)"; build_tuple="${base#transcribe-native-}"
npm_tuple="$(map "$build_tuple")"
[ -n "$npm_tuple" ] || { echo "::warning::no npm tuple for $build_tuple"; continue; }
ex="/tmp/extract/$build_tuple"; rm -rf "$ex"; mkdir -p "$ex"; tar xzf "$f" -C "$ex"
libdir="$(dirname "$(find "$ex" \( -name 'libtranscribe.*' -o -name 'transcribe.dll' \) | head -1)")"
node scripts/pack-platform.mjs --lib-dir "$libdir" --tuple "$npm_tuple" --version "$ver" --out "$out"
done
echo "TS_VER=$ver" >> "$GITHUB_ENV"
- name: Publish platform packages, then the API package
working-directory: bindings/typescript
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
run: |
set -e
for d in /tmp/ts-platform/*/; do
echo "publishing $(node -p "require('${d}package.json').name")"
npm publish "$d" --access public
done
npm publish --access public # the API package (transcribe-cpp)
finalize-release:
# Tags only. Publish the draft GitHub Release only after its hosted assets
# and the core PyPI publish have completed. crates.io has its own approval
# gate and does not host artifacts on this release.
if: startsWith(github.ref, 'refs/tags/v')
needs: [publish-pypi, release-assets, swift-release]
runs-on: ubuntu-latest
permissions:
contents: write # publish the draft release
actions: write # dispatch wheel-index
steps:
- name: Verify assets and publish the release
env:
GH_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
tag="${GITHUB_REF_NAME}"
ver="${tag#v}"
assets_file="$(mktemp)"
gh release view "$tag" --repo "$GITHUB_REPOSITORY" \
--json assets --jq '.assets[].name' | sort > "$assets_file"
echo "Release assets:"
sed 's/^/ /' "$assets_file"
require_asset() {
name="$1"
if ! grep -Fxq "$name" "$assets_file"; then
echo "::error::missing release asset: $name"
exit 1
fi
}
require_asset "TranscribeCpp.xcframework.zip"
require_asset "transcribe-native-${ver}-linux-x86_64-cpu-vulkan.tar.gz"
require_asset "transcribe-native-${ver}-linux-aarch64-cpu-vulkan.tar.gz"
require_asset "transcribe-native-${ver}-macos-arm64-metal.tar.gz"
require_asset "transcribe-native-${ver}-macos-x86_64-cpu.tar.gz"
require_asset "transcribe-native-${ver}-windows-x86_64-cpu-vulkan.tar.gz"
cu12_count="$(grep -Ec '^transcribe_cpp_native_cu12-.*\.whl$' "$assets_file" || true)"
if [ "$cu12_count" -lt 2 ]; then
echo "::error::expected at least two cu12 provider wheels, found $cu12_count"
exit 1
fi
gh release edit "$tag" --repo "$GITHUB_REPOSITORY" --draft=false
- name: Refresh the PEP 503 index (requires Pages enabled on the repo)
env:
GH_TOKEN: ${{ github.token }}
run: |
gh workflow run wheel-index.yml --repo "$GITHUB_REPOSITORY" || \
echo "::warning::wheel-index dispatch failed — is the workflow on the default branch and Pages enabled?"