use ggml native threadpool (#49)

Author: cjpais

.github/workflows/typescript-ci.yml

@@ -166,16 +166,14 @@ jobs:
       # transcription/streaming/cancel/extension coverage. jfk.wav ships in-repo;
       # only the model paths need exporting (fetch-canary handles that).
       #
-      # Windows is no-model-only for now: that is the koffi DLL-resolution proof
-      # this leg was added for, and it is unaffected. The model tier is deferred
-      # because loading a model spins up the ggml CPU threadpool, and on MSVC with
-      # OpenMP (vcomp) the runtime crashes (0xC0000005) at process exit under
-      # node/koffi, after the run itself succeeds. Skipping the canary makes the
-      # model-gated tests and the (model-loading) examples self-skip cleanly. Drop
-      # this guard to re-enable the Windows model tier once that teardown crash in
-      # the MSVC OpenMP runtime is fixed.
+      # Windows now runs the full model tier too. It was previously no-model-only
+      # because loading a model spun up ggml's CPU threadpool, and on MSVC with
+      # OpenMP (vcomp) the runtime crashed (0xC0000005) at process exit under
+      # node/koffi. The build now defaults to ggml's native threadpool (no OpenMP
+      # — see the OpenMP CENTRAL POLICY in CMakeLists.txt), so there is no vcomp
+      # pool to crash at teardown, and the native barrier is MSVC-correct (the
+      # transcribe_threadpool_oversubscription test guards it). Re-enabled here.
       - uses: ./.github/actions/fetch-canary
-        if: runner.os != 'Windows'
         with:
           hf-token: ${{ secrets.HF_TOKEN }}
       - name: Conformance (no-model tier always; model tier when canary present)

CMakeLists.txt

@@ -125,12 +125,15 @@ option(TRANSCRIBE_LTO            "Enable LTO in Release"    OFF)
 # and bench workflow relies on. The Python wheel/provider build sets it ON to
 # produce a shared libtranscribe the FFI layer can dlopen.
 #
-# TRANSCRIBE_USE_OPENMP / TRANSCRIBE_USE_SYSTEM_BLAS default ON to keep the
-# developer build fast (ggml OpenMP, Accelerate/system BLAS host decoder).
-# Official provider wheels pass them OFF so no libgomp/libiomp or OpenBLAS/MKL
-# runtime is vendored into the wheel where it can collide with PyTorch/NumPy.
+# TRANSCRIBE_USE_OPENMP defaults OFF: we use ggml's native CPU threadpool
+# everywhere (see the OpenMP CENTRAL POLICY below — OpenMP's process-global pool
+# is a teardown/coexistence liability for an embeddable library, and the native
+# pool is now correct on MSVC and under oversubscription). Opt in with
+# -DTRANSCRIBE_USE_OPENMP=ON. TRANSCRIBE_USE_SYSTEM_BLAS stays ON for the
+# Accelerate/system-BLAS host decoder; official provider wheels pass it OFF so no
+# OpenBLAS/MKL runtime is vendored where it can collide with PyTorch/NumPy.
 option(TRANSCRIBE_BUILD_SHARED    "Build libtranscribe + ggml as shared libraries"  OFF)
-option(TRANSCRIBE_USE_OPENMP      "Use OpenMP (ggml + Parakeet host-decoder TU)"     ON)
+option(TRANSCRIBE_USE_OPENMP      "Use OpenMP for ggml's CPU threadpool"             OFF)
 option(TRANSCRIBE_USE_SYSTEM_BLAS "Link non-Apple system BLAS for the host decoder"  ON)
 
 # Dynamic ggml backends: each backend (CPU, Vulkan, CUDA, ...) becomes a
@@ -265,6 +268,9 @@ set(GGML_METAL    ${TRANSCRIBE_METAL}  CACHE BOOL "" FORCE)
 set(GGML_VULKAN   ${TRANSCRIBE_VULKAN} CACHE BOOL "" FORCE)
 set(GGML_CUDA     ${TRANSCRIBE_CUDA}   CACHE BOOL "" FORCE)
 set(GGML_BLAS OFF CACHE BOOL "" FORCE)
+# tinyBLAS (Justine Tunney's llamafile_sgemm CPU kernels): ~29% faster encoder on
+# CPU (q8_0 GEMM), numerically WER-equivalent. On by default; CPU-backend only.
+set(GGML_LLAMAFILE ON CACHE BOOL "" FORCE)
 
 # Conservative x86 floor (see the option's comment above): one switch fans
 # out to GGML_NATIVE plus the full x86 SIMD tier list. Placed BEFORE
@@ -283,34 +289,36 @@ if(TRANSCRIBE_X86_CONSERVATIVE)
     endforeach()
 endif()
 
-# OpenMP — CENTRAL POLICY (read before changing the Windows/MSVC threading).
+# OpenMP — CENTRAL POLICY.
 #
-# ggml defaults GGML_OPENMP ON and probes for it gracefully, so leave ggml's own
-# default in place when we want OpenMP. Only force it OFF (matching the
-# GGML_OPENMP=OFF posture official wheels use) when OpenMP is switched off, so
-# one TRANSCRIBE_USE_OPENMP knob covers ggml and our host-decoder TU.
+# DEFAULT: OFF. We use ggml's native CPU threadpool everywhere; OpenMP is an
+# opt-in (-DTRANSCRIBE_USE_OPENMP=ON), not the default. The TRANSCRIBE_USE_OPENMP
+# knob simply drives ggml's GGML_OPENMP.
 #
-# The catch: ggml's NON-OpenMP CPU threadpool barrier (ggml_barrier's custom
-# spin path) DEADLOCKS under MSVC codegen on Windows — any multi-threaded CPU
-# run wedges its workers in the barrier spin (it works on every other compiler).
-# OpenMP's `#pragma omp barrier` is the only working multi-threaded CPU path on
-# MSVC. But OpenMP is a per-CONSUMER tradeoff, not a global flag, so this is not
-# one switch — it is a deliberate, documented split:
+# Why native, not OpenMP — OpenMP's runtime pool is process-global and outlives
+# any single compute, which is a liability for an embeddable/dlopen'd library:
+#   - Teardown crash: a binding (node/koffi, ctypes, ...) calls in on a worker
+#     thread, libgomp/vcomp spawns its pool there, and at process teardown the
+#     loader unmaps the runtime out from under those still-live pool threads ->
+#     SIGSEGV / 0xC0000005. The native pool joins its threads per graph_compute,
+#     so nothing outlives the call and there is nothing to unmap-race.
+#   - Coexistence: a vendored libgomp/libiomp can collide with numpy/torch/MKL's
+#     own OpenMP in one process. The native pool vendors no second runtime.
 #
-#   - Rust binding (CPU is its default backend, standalone process): MUST have
-#     OpenMP on Windows. Its build.rs passes -DGGML_OPENMP=ON; the guard below
-#     honors that explicit value. MSVC auto-links vcomp via the /openmp pragma in
-#     the ggml objects (no -fopenmp flag, so the link manifest is untouched);
-#     vcomp140.dll ships in the VC++ runtime the binary already needs.
-#   - Python wheels (default to the Vulkan GPU backend; deliberately vendor NO
-#     OpenMP so ggml's runtime cannot collide with numpy/torch's own OpenMP/MKL
-#     in one process — see python-wheels.yml): leave GGML_OPENMP unset and get
-#     the force-off below. KNOWN LIMITATION as a consequence: multi-threaded CPU
-#     compute on Windows is unsupported for the wheels (they use Vulkan). Lifting
-#     it needs a non-OpenMP fix (e.g. single-threaded CPU on Windows, or patching
-#     ggml's barrier) rather than forcing OpenMP and the coexistence hazard back.
+# This used to be a per-consumer SPLIT because ggml's native barrier deadlocked
+# under MSVC (its spin `relax` was a no-op on MSVC, so a waiter starved an
+# un-arrived worker under oversubscription) — OpenMP was the only working
+# multi-threaded CPU path on Windows. That barrier is now fixed (ggml-cpu.c:
+# YieldProcessor() relax + a bounded-spin ggml_thread_yield fallback), so the
+# native pool is correct on MSVC and under oversubscription. The split is gone:
+#   - Rust binding: no longer needs -DGGML_OPENMP=ON on Windows; native pool.
+#   - Python wheels: already vendored no OpenMP; multi-threaded CPU now works on
+#     every platform (the old "CPU-on-Windows unsupported" limitation is lifted).
+#   - perf: native persistent/ephemeral pool is on par with OpenMP (it is
+#     llama.cpp's default). Thread count is sized affinity-aware in
+#     transcribe-batch-util (default_n_threads) to avoid needless oversubscription.
 #
-# So: honor an explicit -DGGML_OPENMP=... (the Rust opt-in); otherwise force OFF.
+# So: honor an explicit -DGGML_OPENMP=... (opt-in); otherwise force OFF.
 if(NOT TRANSCRIBE_USE_OPENMP AND NOT DEFINED CACHE{GGML_OPENMP})
     set(GGML_OPENMP OFF CACHE BOOL "" FORCE)
 endif()

bindings/rust/sys/build.rs

@@ -157,29 +157,27 @@ fn main() {
     if feature("CUDA") {
         cfg.define("TRANSCRIBE_CUDA", "ON");
     }
-    // Force OpenMP OFF unless explicitly opted in. TRANSCRIBE_USE_OPENMP
-    // defaults ON and auto-detects, but its `-fopenmp` shows up only as a
-    // manifest link_flag → a `cargo:rustc-link-arg` that does NOT propagate to
-    // downstream binaries, so a static consumer link fails with undefined
-    // GOMP_*/omp_* symbols. A self-contained static build is the default;
-    // `--features openmp` opts in (and then owns providing the OpenMP runtime).
+    // Keep OpenMP OFF unless explicitly opted in. TRANSCRIBE_USE_OPENMP already
+    // defaults OFF in CMake (the native ggml threadpool is the default path); we
+    // set it explicitly here so `--features openmp` is the single switch. We keep
+    // it OFF by default because OpenMP's `-fopenmp` shows up only as a manifest
+    // link_flag → a `cargo:rustc-link-arg` that does NOT propagate to downstream
+    // binaries, so a static consumer link would fail with undefined GOMP_*/omp_*
+    // symbols. A self-contained static build is the default; `--features openmp`
+    // opts in (and then owns providing the OpenMP runtime).
     cfg.define(
         "TRANSCRIBE_USE_OPENMP",
         if feature("OPENMP") { "ON" } else { "OFF" },
     );
-    // ggml's non-OpenMP CPU threadpool barrier deadlocks under MSVC on Windows;
-    // OpenMP's `#pragma omp barrier` is the only working multi-threaded CPU path
-    // there. The Rust binding is CPU-default and standalone (no numpy/torch
-    // OpenMP-coexistence concern), so it opts ggml into OpenMP on Windows.
-    // GGML-internal only: TRANSCRIBE_USE_OPENMP stays off, so the link manifest
-    // emits no (GNU-only) -fopenmp and the Parakeet host-decoder TU is unchanged;
-    // MSVC auto-links vcomp via the ggml objects' /openmp pragma. CMakeLists
-    // honors this explicit GGML_OPENMP and skips its force-off. See the full
-    // per-consumer policy (incl. why the Python wheels stay OpenMP-free) in the
-    // "OpenMP — CENTRAL POLICY" block of the root CMakeLists.txt.
-    if target_os == "windows" {
-        cfg.define("GGML_OPENMP", "ON");
-    }
+    // Windows no longer needs ggml's OpenMP. ggml's native CPU threadpool barrier
+    // used to deadlock under MSVC (its spin `relax` compiled to a no-op, starving
+    // an un-arrived worker), so OpenMP was the only multi-threaded CPU path there.
+    // That barrier is fixed upstream (ggml-cpu.c: YieldProcessor relax + a
+    // bounded-spin ggml_thread_yield fallback), so the native pool is correct on
+    // MSVC — and avoids the vcomp pool that crashes at teardown when a host
+    // dlopen/dlcloses this lib. We therefore leave GGML_OPENMP at its
+    // CMakeLists-forced OFF on every platform; `--features openmp` is the only
+    // opt-in. See the "OpenMP — CENTRAL POLICY" block in the root CMakeLists.txt.
 
     // Escape hatch: forward arbitrary configure args so the curated features are
     // never a hard ceiling. Anything CMake accepts (-DGGML_*, a -DTRANSCRIBE_*

docs/build-windows.md

@@ -93,10 +93,12 @@ cmake --build build --target transcribe-cli --config Release
 Notes:
 - The Visual Studio generator is **multi-config**, so pass `--config Release`
   at build time (omitting it yields a Debug build).
-- A successful configure prints `Found OpenMP`. **OpenMP matters on
-  Windows**: ggml's non-OpenMP CPU threadpool barrier deadlocks under MSVC,
-  so OpenMP is the working multi-threaded CPU path. MSVC auto-links it
-  (`vcomp`) — no action needed, just don't disable it.
+- OpenMP is **not** required on Windows. ggml's native CPU threadpool is the
+  default everywhere (see the OpenMP CENTRAL POLICY in the top-level
+  `CMakeLists.txt`); the MSVC barrier bug that once forced OpenMP here is fixed,
+  so the native pool is correct under MSVC and oversubscription. Build with the
+  defaults; opt into OpenMP only with `-DTRANSCRIBE_USE_OPENMP=ON` if you have a
+  specific reason.
 - `no BLAS found — decoder uses scalar fallback` is fine; it only affects
   host-side decoder speed, not correctness.
 - `uv not found on PATH` is harmless — it only skips an optional test.

src/CMakeLists.txt

@@ -124,27 +124,6 @@ add_library(transcribe
     third_party/miniz/miniz.c
 )
 
-# The parakeet host decoder's matmul fallback (linear()) parallelizes its
-# large output projection across rows. Compile ONLY this TU with OpenMP so
-# the `#pragma omp parallel for` in linear() activates; libgomp is already
-# linked transitively via ggml. Scoped to one TU and numerically identical
-# (rows summed in the same order). The primary out_w path is the ggml graph
-# in joint_step; this row-parallelism covers the host fallback and the
-# remaining host-side matmuls.
-#
-# Gated on TRANSCRIBE_USE_OPENMP so official provider wheels (which pass it OFF,
-# alongside GGML_OPENMP=OFF) do not link an OpenMP runtime into this TU. When
-# off, the `#pragma omp` is an ignored no-op and linear() runs serially — the
-# same path taken when OpenMP simply is not found.
-if(TRANSCRIBE_USE_OPENMP)
-    find_package(OpenMP QUIET)
-    if(OpenMP_CXX_FOUND)
-        set_source_files_properties(arch/parakeet/decoder.cpp PROPERTIES
-            COMPILE_OPTIONS "${OpenMP_CXX_FLAGS}")
-        target_link_libraries(transcribe PRIVATE OpenMP::OpenMP_CXX)
-    endif()
-endif()
-
 target_include_directories(transcribe
     PUBLIC
         $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/include>

src/arch/canary/model.cpp

@@ -902,22 +902,7 @@ transcribe_status run(
     }
 
     // Thread count.
-    {
-        int n_threads = cc->n_threads;
-        if (n_threads <= 0) {
-            n_threads = std::min(8, std::max(1, static_cast<int>(
-                std::thread::hardware_concurrency())));
-        }
-        for (int i = 0; i < ggml_backend_sched_get_n_backends(cc->sched); ++i) {
-            ggml_backend_t be = ggml_backend_sched_get_backend(cc->sched, i);
-            ggml_backend_dev_t dev = ggml_backend_get_device(be);
-            ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr;
-            if (reg == nullptr) continue;
-            auto * fn = reinterpret_cast<ggml_backend_set_n_threads_t>(
-                ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads"));
-            if (fn != nullptr) fn(be, n_threads);
-        }
-    }
+    transcribe::configure_sched_n_threads(cc->sched, cc->n_threads);
 
     const int64_t t_enc_start = ggml_time_us();
     if (const ggml_status gs = ggml_backend_sched_graph_compute(cc->sched, eb.graph);
@@ -1570,20 +1555,7 @@ transcribe_status encode_one_to_host(
                                 pos_buf.size() * sizeof(float));
     }
 
-    {
-        int n_threads = cc->n_threads;
-        if (n_threads <= 0) n_threads = std::min(8, std::max(1,
-            static_cast<int>(std::thread::hardware_concurrency())));
-        for (int i = 0; i < ggml_backend_sched_get_n_backends(cc->sched); ++i) {
-            ggml_backend_t be = ggml_backend_sched_get_backend(cc->sched, i);
-            ggml_backend_dev_t dev = ggml_backend_get_device(be);
-            ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr;
-            if (reg == nullptr) continue;
-            auto * fn = reinterpret_cast<ggml_backend_set_n_threads_t>(
-                ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads"));
-            if (fn != nullptr) fn(be, n_threads);
-        }
-    }
+    transcribe::configure_sched_n_threads(cc->sched, cc->n_threads);
 
     const int64_t t0 = ggml_time_us();
     if (ggml_backend_sched_graph_compute(cc->sched, eb.graph) != GGML_STATUS_SUCCESS)
@@ -1672,8 +1644,7 @@ transcribe_status run_batch(
     std::vector<std::vector<float>> mel_bufs(n);
     std::vector<int> mel_nf(n, 0);
     int n_threads = cc->n_threads;
-    if (n_threads <= 0) n_threads = std::min(8, std::max(1,
-        static_cast<int>(std::thread::hardware_concurrency())));
+    if (n_threads <= 0) n_threads = transcribe::default_n_threads();
     int64_t mel_us = 0, enc_us = 0;
     const int64_t t_mel0 = ggml_time_us();
     transcribe::parallel_for_all(n, n_threads, [&](int b) {

src/arch/canary_qwen/model.cpp

@@ -830,20 +830,7 @@ transcribe_status init_context(
 // ---------------------------------------------------------------------------
 
 void apply_thread_policy(CanaryQwenSession * cc) {
-    int n_threads = cc->n_threads;
-    if (n_threads <= 0) {
-        n_threads = std::min(8, std::max(1, static_cast<int>(
-            std::thread::hardware_concurrency())));
-    }
-    for (int i = 0; i < ggml_backend_sched_get_n_backends(cc->sched); ++i) {
-        ggml_backend_t be       = ggml_backend_sched_get_backend(cc->sched, i);
-        ggml_backend_dev_t dev  = ggml_backend_get_device(be);
-        ggml_backend_reg_t reg  = dev ? ggml_backend_dev_backend_reg(dev) : nullptr;
-        if (reg == nullptr) continue;
-        auto * fn = reinterpret_cast<ggml_backend_set_n_threads_t>(
-            ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads"));
-        if (fn != nullptr) fn(be, n_threads);
-    }
+    transcribe::configure_sched_n_threads(cc->sched, cc->n_threads);
 }
 
 void build_relpos_emb_host(std::vector<float> & pos_buf,
@@ -1559,8 +1546,7 @@ transcribe_status run_batch(
     std::vector<int> mel_nf(n, 0);
     int n_mel_threads = cc->n_threads;
     if (n_mel_threads <= 0)
-        n_mel_threads = std::min(8, std::max(1, static_cast<int>(
-            std::thread::hardware_concurrency())));
+        n_mel_threads = transcribe::default_n_threads();
     const int64_t t_mel0 = ggml_time_us();
     transcribe::parallel_for_all(n, n_mel_threads, [&](int b) {
         if (pcm[b] == nullptr || n_samples[b] <= 0) return true;

src/arch/cohere/model.cpp

@@ -941,24 +941,7 @@ transcribe_status run(
     }
 
     // Set thread count.
-    {
-        int n_threads = cc->n_threads;
-        if (n_threads <= 0) {
-            n_threads = std::min(8, std::max(1, static_cast<int>(
-                std::thread::hardware_concurrency())));
-        }
-        for (int i = 0; i < ggml_backend_sched_get_n_backends(cc->sched); ++i) {
-            ggml_backend_t be = ggml_backend_sched_get_backend(cc->sched, i);
-            ggml_backend_dev_t dev = ggml_backend_get_device(be);
-            ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr;
-            if (reg == nullptr) continue;
-            auto * fn = reinterpret_cast<ggml_backend_set_n_threads_t>(
-                ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads"));
-            if (fn != nullptr) {
-                fn(be, n_threads);
-            }
-        }
-    }
+    transcribe::configure_sched_n_threads(cc->sched, cc->n_threads);
 
     // Compute encoder graph.
     const int64_t t_enc_start = ggml_time_us();
@@ -1686,20 +1669,7 @@ transcribe_status encode_one_to_host(
                                 pos_buf.size() * sizeof(float));
     }
 
-    {
-        int n_threads = cc->n_threads;
-        if (n_threads <= 0) n_threads = std::min(8, std::max(1,
-            static_cast<int>(std::thread::hardware_concurrency())));
-        for (int i = 0; i < ggml_backend_sched_get_n_backends(cc->sched); ++i) {
-            ggml_backend_t be = ggml_backend_sched_get_backend(cc->sched, i);
-            ggml_backend_dev_t dev = ggml_backend_get_device(be);
-            ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr;
-            if (reg == nullptr) continue;
-            auto * fn = reinterpret_cast<ggml_backend_set_n_threads_t>(
-                ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads"));
-            if (fn != nullptr) fn(be, n_threads);
-        }
-    }
+    transcribe::configure_sched_n_threads(cc->sched, cc->n_threads);
 
     const int64_t t0 = ggml_time_us();
     if (ggml_backend_sched_graph_compute(cc->sched, eb.graph) != GGML_STATUS_SUCCESS)
@@ -1786,8 +1756,7 @@ transcribe_status run_batch(
     std::vector<std::vector<float>> mel_bufs(n);
     std::vector<int> mel_nf(n, 0);
     int n_threads = cc->n_threads;
-    if (n_threads <= 0) n_threads = std::min(8, std::max(1,
-        static_cast<int>(std::thread::hardware_concurrency())));
+    if (n_threads <= 0) n_threads = transcribe::default_n_threads();
     int64_t mel_us = 0, enc_us = 0;
     const int64_t t_mel0 = ggml_time_us();
     transcribe::parallel_for_all(n, n_threads, [&](int b) {