python bindings #10

Workflow file for this run

.github/workflows/python-bindings.yml at 1a993ee

	name: python-bindings

	# Locks in what the Python bindings already guarantee (see
	# docs/python-bindings-distribution-plan.md, Milestone 1):
	# - lint: generated-FFI drift gate + version-sync across the three
	# sources of truth (header / pyproject / __init__).
	# - cpp-tests: the full C++ white-box suite against a static build.
	# - python-shared: a shared libtranscribe, the pure-C api_smoke as an
	# exported-symbol canary, and the Python no-model tests
	# (import, ABI layout, version gate, status/enum agreement).
	#
	# Real-model tests are intentionally not run here: they need multi-GB GGUFs CI
	# cannot ship. They live behind env vars and skip cleanly. A tiny redistributable
	# canary GGUF (plan M1) will later let one real transcription run in CI.

	on:
	push:
	branches: [main]
	paths:
	- "bindings/python/**"
	- "include/**"
	- "src/**"
	- "tests/**"
	- "ggml/**"
	- "CMakeLists.txt"
	- ".github/workflows/python-bindings.yml"
	pull_request:
	paths:
	- "bindings/python/**"
	- "include/**"
	- "src/**"
	- "tests/**"
	- "ggml/**"
	- "CMakeLists.txt"
	- ".github/workflows/python-bindings.yml"
	workflow_dispatch:

	# The generated FFI layer is pinned to one libclang so the drift gate's embedded
	# `libclang: <version>` line is reproducible across runs.
	env:
	LIBCLANG_PIN: "libclang==18.1.1"

	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: true

	jobs:
	lint:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- name: Install uv
	run: \|
	curl -LsSf https://astral.sh/uv/install.sh \| sh
	echo "$HOME/.local/bin" >> "$GITHUB_PATH"
	- name: Install clang (libclang resource-dir headers)
	run: sudo apt-get update && sudo apt-get install -y clang
	- name: Generated-FFI drift gate
	run: \|
	uv run --no-project --with "$LIBCLANG_PIN" \
	bindings/python/_generate/generate.py --check
	- name: Version sync (header / pyproject / __init__)
	run: uv run --no-project bindings/python/_generate/check_version_sync.py

	cpp-tests:
	name: cpp-tests (${{ matrix.os }})
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest, macos-14]
	runs-on: ${{ matrix.os }}
	steps:
	- uses: actions/checkout@v4
	- name: Install uv (fixtures are generated via uv)
	run: \|
	curl -LsSf https://astral.sh/uv/install.sh \| sh
	echo "$HOME/.local/bin" >> "$GITHUB_PATH"
	- name: Install build deps (Linux)
	if: runner.os == 'Linux'
	run: sudo apt-get update && sudo apt-get install -y cmake ninja-build zlib1g-dev
	- name: Install build deps (macOS)
	if: runner.os == 'macOS'
	run: brew install ninja
	- name: Configure (static white-box build)
	run: cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release
	- name: Build
	run: cmake --build build -j
	- name: Test (full white-box suite)
	run: ctest --test-dir build --output-on-failure

	cpp-tests-sanitized:
	# ASan+UBSan over the white-box suite. This is the lane that certifies
	# the C lifetime/ABI contracts the FFI bindings rely on — including the
	# params copy-out regression (stream_dispatch_unit), which reproduces a
	# ctypes-shaped caller freeing its params right after stream_begin.
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- name: Install uv (fixtures are generated via uv)
	run: \|
	curl -LsSf https://astral.sh/uv/install.sh \| sh
	echo "$HOME/.local/bin" >> "$GITHUB_PATH"
	- name: Install build deps
	run: sudo apt-get update && sudo apt-get install -y cmake ninja-build zlib1g-dev
	- name: Configure (static, sanitized)
	run: \|
	cmake -B build-asan -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo \
	-DTRANSCRIBE_SANITIZE=ON
	- name: Build
	run: cmake --build build-asan -j
	- name: Test (white-box suite under ASan+UBSan)
	run: ctest --test-dir build-asan --output-on-failure

	provider-dl-vulkan:
	# Builds the default Linux provider shape — CPU + Vulkan as dynamic
	# backend modules (GGML_BACKEND_DL) — assembles it into a flat
	# wheel-like directory with $ORIGIN rpaths, deletes the build tree so
	# nothing can resolve outside the directory, and proves the Vulkan
	# degradation contract on the real runner:
	# 1. loader REMOVED (no libvulkan) -> import + CPU devices work,
	# vulkan answers unavailable; the module-load failure is quiet.
	# 2. mesa lavapipe installed -> the SAME artifacts discover a
	# software Vulkan device, and (with HF_TOKEN for the canary model)
	# actually transcribe on it.
	runs-on: ubuntu-latest
	env:
	HF_TOKEN: ${{ secrets.HF_TOKEN }}
	steps:
	- uses: actions/checkout@v4
	- name: Install uv
	run: \|
	curl -LsSf https://astral.sh/uv/install.sh \| sh
	echo "$HOME/.local/bin" >> "$GITHUB_PATH"
	- name: Install build deps (Vulkan SDK pieces + patchelf)
	run: \|
	sudo apt-get update
	sudo apt-get install -y cmake ninja-build zlib1g-dev \
	libvulkan-dev glslc patchelf
	- name: Configure (DL provider, CPU + Vulkan modules, wheel posture)
	run: \|
	cmake -B build-dl -G Ninja -DCMAKE_BUILD_TYPE=Release \
	-DTRANSCRIBE_BUILD_SHARED=ON \
	-DTRANSCRIBE_GGML_BACKEND_DL=ON \
	-DTRANSCRIBE_VULKAN=ON \
	-DTRANSCRIBE_USE_OPENMP=OFF \
	-DTRANSCRIBE_USE_SYSTEM_BLAS=OFF
	- name: Build
	run: cmake --build build-dl -j
	- name: Assemble flat provider directory ($ORIGIN rpaths, build tree deleted)
	run: \|
	mkdir -p provider
	cp -L build-dl/src/libtranscribe.so provider/
	cp -L build-dl/ggml/src/libggml.so.* build-dl/ggml/src/libggml-base.so.* provider/ 2>/dev/null \|\| \
	cp -L build-dl/ggml/src/libggml.so build-dl/ggml/src/libggml-base.so provider/
	cp build-dl/bin/libggml-*.so provider/
	for f in provider/.so; do patchelf --set-rpath '$ORIGIN' "$f"; done
	ls -la provider/
	rm -rf build-dl # isolation: nothing may resolve outside provider/
	- name: "Tier 1: no Vulkan loader — CPU works, vulkan cleanly unavailable"
	run: \|
	sudo apt-get remove -y libvulkan-dev libvulkan1 \|\| \
	sudo rm -f /usr/lib/x86_64-linux-gnu/libvulkan.so.1*
	export TRANSCRIBE_LIBRARY="$PWD/provider/libtranscribe.so"
	cat > /tmp/check_no_vulkan.py <<'EOF'
	import transcribe_cpp as t
	devs = t.backends()
	print("devices (no loader):", [(d.name, d.kind) for d in devs])
	assert any(d.kind == "cpu" for d in devs), devs
	assert t.backend_available("cpu")
	assert not t.backend_available("vulkan"), devs
	print("ok: import + CPU fine without a Vulkan loader; vulkan answers False")
	EOF
	uv run --project bindings/python python /tmp/check_no_vulkan.py
	- name: "Tier 2: lavapipe installed — same artifacts discover Vulkan"
	run: \|
	sudo apt-get install -y libvulkan1 mesa-vulkan-drivers
	export TRANSCRIBE_LIBRARY="$PWD/provider/libtranscribe.so"
	# lavapipe reports VK_PHYSICAL_DEVICE_TYPE_CPU, and ggml-vulkan's
	# default selection takes only discrete/integrated GPUs. The env
	# override bypasses the type filter so the software device counts —
	# CI-only; real GPUs need no override.
	export GGML_VK_VISIBLE_DEVICES=0
	cat > /tmp/check_vulkan.py <<'EOF'
	import transcribe_cpp as t
	devs = t.backends()
	print("devices (lavapipe):", [(d.name, d.kind) for d in devs])
	assert t.backend_available("vulkan"), devs
	assert any(d.kind == "vulkan" for d in devs), devs
	print("ok: the same provider directory discovers a Vulkan device")
	EOF
	uv run --project bindings/python python /tmp/check_vulkan.py
	- name: Canary model cache (avoid HF rate limits)
	if: env.HF_TOKEN != ''
	uses: actions/cache@v4
	with:
	path: canary
	key: canary-models-v1
	- name: "Tier 2b: real transcription on the Vulkan device (needs HF_TOKEN)"
	if: env.HF_TOKEN != ''
	run: \|
	[ -f canary/whisper-tiny-Q5_K_M.gguf ] \|\| \
	uv run --no-project --with huggingface_hub \
	hf download handy-computer/whisper-tiny-gguf \
	whisper-tiny-Q5_K_M.gguf --local-dir canary
	export TRANSCRIBE_LIBRARY="$PWD/provider/libtranscribe.so"
	export GGML_VK_VISIBLE_DEVICES=0
	cat > /tmp/check_vk_transcribe.py <<'EOF'
	import array, wave
	import transcribe_cpp as t
	assert t.backend_available("vulkan")
	with wave.open("samples/jfk.wav", "rb") as w:
	pcm16 = array.array("h")
	pcm16.frombytes(w.readframes(w.getnframes()))
	pcm = array.array("f", (s / 32768.0 for s in pcm16))
	with t.Model("canary/whisper-tiny-Q5_K_M.gguf", backend="vulkan") as m:
	print("model backend:", m.backend)
	with m.session() as s:
	text = s.run(pcm, language="en").text
	print("text:", text.strip())
	assert "country" in text.lower(), text
	print("ok: transcribed end-to-end on the Vulkan (lavapipe) device")
	EOF
	uv run --project bindings/python python /tmp/check_vk_transcribe.py

	python-shared:
	runs-on: ubuntu-latest
	env:
	HF_TOKEN: ${{ secrets.HF_TOKEN }}
	steps:
	- uses: actions/checkout@v4
	- name: Install uv
	run: \|
	curl -LsSf https://astral.sh/uv/install.sh \| sh
	echo "$HOME/.local/bin" >> "$GITHUB_PATH"
	- name: Install build deps
	run: sudo apt-get update && sudo apt-get install -y cmake ninja-build zlib1g-dev
	# Shared build in the official-wheel posture (no OpenMP / system BLAS), so
	# the canary exercises the same configuration the provider wheels ship.
	- name: Configure (shared, wheel posture)
	run: \|
	cmake -B build-shared -G Ninja -DCMAKE_BUILD_TYPE=Release \
	-DTRANSCRIBE_BUILD_SHARED=ON \
	-DTRANSCRIBE_USE_OPENMP=OFF \
	-DTRANSCRIBE_USE_SYSTEM_BLAS=OFF
	- name: Build
	run: cmake --build build-shared -j
	- name: api_smoke (exported-symbol canary) + extension umbrella
	run: ctest --test-dir build-shared --output-on-failure
	# The canary model upgrades this lane from "imports and ABI" to a real
	# end-to-end transcription. Guarded on HF_TOKEN (the model repo is
	# private for now; forks have no token) — without it the model tests
	# skip cleanly, exactly as before.
	- name: Canary model cache (avoid HF rate limits — 429s were real)
	if: env.HF_TOKEN != ''
	uses: actions/cache@v4
	with:
	path: canary
	key: canary-models-v1
	- name: Download canary models (cache miss only; needs HF_TOKEN)
	if: env.HF_TOKEN != ''
	run: \|
	[ -f canary/whisper-tiny-Q5_K_M.gguf ] \|\| \
	uv run --no-project --with huggingface_hub \
	hf download handy-computer/whisper-tiny-gguf \
	whisper-tiny-Q5_K_M.gguf --local-dir canary
	[ -f canary/moonshine-streaming-tiny-Q8_0.gguf ] \|\| \
	uv run --no-project --with huggingface_hub \
	hf download handy-computer/moonshine-streaming-tiny-gguf \
	moonshine-streaming-tiny-Q8_0.gguf --local-dir canary
	echo "TRANSCRIBE_SMOKE_MODEL=$PWD/canary/whisper-tiny-Q5_K_M.gguf" >> "$GITHUB_ENV"
	echo "TRANSCRIBE_SMOKE_STREAMING_MODEL=$PWD/canary/moonshine-streaming-tiny-Q8_0.gguf" >> "$GITHUB_ENV"
	- name: Python tests (real transcription when the canary is present)
	run: \|
	# libtranscribe.so is dlopen'd by ctypes; help it resolve its sibling
	# ggml shared libs in the build tree.
	export LD_LIBRARY_PATH="$(find "$PWD/build-shared" -name '*.so' \
	-printf '%h\n' \| sort -u \| tr '\n' ':')$LD_LIBRARY_PATH"
	# The loader auto-discovers build-shared/src/libtranscribe.so from the
	# repo root; the TRANSCRIBE_SMOKE_* vars (set above when HF_TOKEN
	# exists) un-skip the offline AND streaming model tests. The prompted
	# (nemotron) streaming test still skips here — its regression is
	# pinned in CI by the sanitized C-level test instead.
	uv run --project bindings/python --extra test \
	pytest bindings/python/tests -q -rs

Navigation Menu

Uh oh!

python bindings #10

Workflow file

python bindings #10

Uh oh!

Workflow file for this run