From e4736b69b6ee3ae204792c242e3dc5b56a245ae4 Mon Sep 17 00:00:00 2001 From: Andreas Fehlner Date: Sat, 21 Mar 2026 06:35:10 +0100 Subject: [PATCH 1/9] update Signed-off-by: Andreas Fehlner --- .github/workflows/benchmark-backends.yml | 33 ++- backends/onnx2c/backend.py | 278 +++++++++++++++++++++++ runtimes/onnx2c/stable/Dockerfile | 48 ++++ setup/config.json | 6 + 4 files changed, 364 insertions(+), 1 deletion(-) create mode 100644 backends/onnx2c/backend.py create mode 100644 runtimes/onnx2c/stable/Dockerfile diff --git a/.github/workflows/benchmark-backends.yml b/.github/workflows/benchmark-backends.yml index 0bb0f4233c..0ab462055d 100644 --- a/.github/workflows/benchmark-backends.yml +++ b/.github/workflows/benchmark-backends.yml @@ -272,11 +272,36 @@ jobs: name: results-tvm-stable path: results/tvm/stable/ + onnx2c_stable: + runs-on: ubuntu-latest + timeout-minutes: 90 + permissions: + contents: read + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Build docker image + run: docker build -t scoreboard/onnx2c -f runtimes/onnx2c/stable/Dockerfile . + + - name: Prepare results directory + run: mkdir -p ${{ github.workspace }}/results/onnx2c/stable && chmod 777 ${{ github.workspace }}/results/onnx2c/stable + + - name: Run docker container + run: docker run --name onnx2c --env-file setup/env.list -v ${{ github.workspace }}/results/onnx2c/stable:/root/results scoreboard/onnx2c || true + + - name: Upload results + if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} + uses: actions/upload-artifact@v7 + with: + name: results-onnx2c-stable + path: results/onnx2c/stable/ + collect_and_deploy: runs-on: ubuntu-latest permissions: contents: write - needs: [onnxruntime_stable, onnxruntime_dev, onnx_tf_stable, onnx_tf_dev, jaxonnxruntime_stable, jaxonnxruntime_dev, emx_onnx_cgen_stable, tract_stable, onnx_reference_stable, opencv_stable, tvm_stable] + needs: [onnxruntime_stable, onnxruntime_dev, onnx_tf_stable, onnx_tf_dev, jaxonnxruntime_stable, jaxonnxruntime_dev, emx_onnx_cgen_stable, tract_stable, onnx_reference_stable, opencv_stable, tvm_stable, onnx2c_stable] if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} steps: - name: Checkout code @@ -360,6 +385,12 @@ jobs: name: results-tvm-stable path: results/tvm/stable/ + - name: Download onnx2c stable results + uses: actions/download-artifact@v8 + with: + name: results-onnx2c-stable + path: results/onnx2c/stable/ + - name: Deploy results run: | git add results diff --git a/backends/onnx2c/backend.py b/backends/onnx2c/backend.py new file mode 100644 index 0000000000..c722f2984c --- /dev/null +++ b/backends/onnx2c/backend.py @@ -0,0 +1,278 @@ +# SPDX-License-Identifier: Apache-2.0 + +"""ONNX backend wrapper for onnx2c (ONNX to C code generator).""" + +import os +import re +import shutil +import subprocess +import tempfile + +import numpy as np +from onnx import TensorProto, shape_inference +from onnx.backend.base import Backend, BackendRep +from onnx.backend.test.runner import BackendIsNotSupposedToImplementIt + + +def _c_name(name): + """Convert ONNX tensor name to valid C identifier (matching onnx2c's naming).""" + name = re.sub(r"[^a-zA-Z0-9_]", "_", name) + if name and name[0].isdigit(): + name = "_" + name + return name + + +_ONNX_TO_C_TYPE = { + TensorProto.FLOAT: "float", + TensorProto.DOUBLE: "double", + TensorProto.INT8: "int8_t", + TensorProto.INT16: "int16_t", + TensorProto.INT32: "int32_t", + TensorProto.INT64: "int64_t", + TensorProto.UINT8: "uint8_t", + TensorProto.UINT16: "uint16_t", + TensorProto.UINT32: "uint32_t", + TensorProto.UINT64: "uint64_t", + TensorProto.BOOL: "bool", +} + +_ONNX_TO_NUMPY = { + TensorProto.FLOAT: np.float32, + TensorProto.DOUBLE: np.float64, + TensorProto.INT8: np.int8, + TensorProto.INT16: np.int16, + TensorProto.INT32: np.int32, + TensorProto.INT64: np.int64, + TensorProto.UINT8: np.uint8, + TensorProto.UINT16: np.uint16, + TensorProto.UINT32: np.uint32, + TensorProto.UINT64: np.uint64, + TensorProto.BOOL: np.bool_, +} + + +def _parse_tensor_info(value_info): + """Return (c_name, c_type, numpy_dtype, shape) or None if unsupported.""" + t = value_info.type.tensor_type + elem_type = t.elem_type + c_type = _ONNX_TO_C_TYPE.get(elem_type) + np_dtype = _ONNX_TO_NUMPY.get(elem_type) + if c_type is None or np_dtype is None: + return None + + shape = [] + if t.HasField("shape"): + for dim in t.shape.dim: + if dim.HasField("dim_value") and dim.dim_value > 0: + shape.append(dim.dim_value) + else: + # Dynamic or unknown dimension — not supported by onnx2c + return None + else: + return None + + return _c_name(value_info.name), c_type, np_dtype, shape + + +def _generate_harness(inputs_info, outputs_info): + """Return C harness source that feeds inputs, calls entry(), writes outputs.""" + lines = [ + "#include ", + "#include ", + "#include ", + "#include ", + "", + ] + + # extern declarations for input/output tensors (defined in generated model.c) + for c_nm, c_type, _, shape in inputs_info: + n = 1 + for d in shape: + n *= d + lines.append(f"extern {c_type} {c_nm}[{n}];") + + for c_nm, c_type, _, shape in outputs_info: + n = 1 + for d in shape: + n *= d + lines.append(f"extern {c_type} {c_nm}[{n}];") + + lines += ["", "void entry(void);", "", "int main(int argc, char** argv) {"] + lines.append(" FILE* f;") + lines.append("") + + # Read inputs from files passed as argv[1..n_inputs] + for i, (c_nm, c_type, _, shape) in enumerate(inputs_info): + n = 1 + for d in shape: + n *= d + lines.append(f" f = fopen(argv[{i + 1}], \"rb\");") + lines.append( + f" if (!f) {{ fprintf(stderr, \"Cannot open input {i}\\n\"); return 1; }}" + ) + lines.append(f" fread({c_nm}, sizeof({c_type}), {n}, f);") + lines.append(" fclose(f);") + + lines += ["", " entry();", ""] + + # Write outputs to files passed as argv[n_inputs+1..n_inputs+n_outputs] + n_in = len(inputs_info) + for i, (c_nm, c_type, _, shape) in enumerate(outputs_info): + n = 1 + for d in shape: + n *= d + lines.append(f" f = fopen(argv[{n_in + i + 1}], \"wb\");") + lines.append( + f" if (!f) {{ fprintf(stderr, \"Cannot open output {i}\\n\"); return 1; }}" + ) + lines.append(f" fwrite({c_nm}, sizeof({c_type}), {n}, f);") + lines.append(" fclose(f);") + + lines += ["", " return 0;", "}"] + return "\n".join(lines) + + +class Onnx2cBackendRep(BackendRep): + """Holds a compiled onnx2c binary and runs it for each inference call.""" + + def __init__(self, binary_path, workdir, inputs_info, outputs_info): + self._binary = binary_path + self._workdir = workdir + # Store only dtype and shape needed at run time + self._input_dtypes = [np_dtype for _, _, np_dtype, _ in inputs_info] + self._output_info = [(np_dtype, shape) for _, _, np_dtype, shape in outputs_info] + + def __del__(self): + if self._workdir and os.path.isdir(self._workdir): + shutil.rmtree(self._workdir, ignore_errors=True) + + def run(self, inputs, **kwargs): + run_dir = tempfile.mkdtemp() + try: + # Write each input as raw binary with the expected dtype + input_files = [] + for i, (inp, dtype) in enumerate(zip(inputs, self._input_dtypes)): + path = os.path.join(run_dir, f"input_{i}.bin") + np.asarray(inp, dtype=dtype).flatten().tofile(path) + input_files.append(path) + + output_files = [] + for i in range(len(self._output_info)): + path = os.path.join(run_dir, f"output_{i}.bin") + output_files.append(path) + + result = subprocess.run( + [self._binary] + input_files + output_files, + capture_output=True, + timeout=60, + ) + if result.returncode != 0: + raise BackendIsNotSupposedToImplementIt( + f"onnx2c binary failed: {result.stderr.decode()}" + ) + + outputs = [] + for path, (dtype, shape) in zip(output_files, self._output_info): + arr = np.fromfile(path, dtype=dtype).reshape(shape) + outputs.append(arr) + return outputs + finally: + shutil.rmtree(run_dir, ignore_errors=True) + + +class Onnx2cBackend(Backend): + """ONNX backend that compiles models to native code via onnx2c.""" + + @classmethod + def is_compatible(cls, model, device="CPU", **kwargs): + return True + + @classmethod + def prepare(cls, model, device="CPU", **kwargs): + # Run ONNX shape inference so output shapes are concrete where possible + try: + model = shape_inference.infer_shapes(model) + except Exception: + pass + + initializer_names = {init.name for init in model.graph.initializer} + + # Parse inputs (exclude initializers / weights) + inputs_info = [] + for vi in model.graph.input: + if vi.name in initializer_names: + continue + info = _parse_tensor_info(vi) + if info is None: + raise BackendIsNotSupposedToImplementIt( + f"Input '{vi.name}' has unsupported type or dynamic shape" + ) + inputs_info.append(info) + + # Parse outputs + outputs_info = [] + for vi in model.graph.output: + info = _parse_tensor_info(vi) + if info is None: + raise BackendIsNotSupposedToImplementIt( + f"Output '{vi.name}' has unsupported type or dynamic shape" + ) + outputs_info.append(info) + + workdir = tempfile.mkdtemp() + try: + # Serialize model + model_path = os.path.join(workdir, "model.onnx") + with open(model_path, "wb") as f: + f.write(model.SerializeToString()) + + # Generate C code with onnx2c + result = subprocess.run( + ["onnx2c", model_path], capture_output=True, timeout=120 + ) + if result.returncode != 0: + raise BackendIsNotSupposedToImplementIt( + f"onnx2c failed: {result.stderr.decode()}" + ) + model_c_path = os.path.join(workdir, "model.c") + with open(model_c_path, "wb") as f: + f.write(result.stdout) + + # Write harness + harness_path = os.path.join(workdir, "harness.c") + with open(harness_path, "w") as f: + f.write(_generate_harness(inputs_info, outputs_info)) + + # Compile + binary_path = os.path.join(workdir, "model_exec") + result = subprocess.run( + ["gcc", "-O2", "-o", binary_path, harness_path, model_c_path, "-lm"], + capture_output=True, + timeout=120, + ) + if result.returncode != 0: + raise BackendIsNotSupposedToImplementIt( + f"Compilation failed: {result.stderr.decode()}" + ) + + return Onnx2cBackendRep(binary_path, workdir, inputs_info, outputs_info) + + except BackendIsNotSupposedToImplementIt: + shutil.rmtree(workdir, ignore_errors=True) + raise + except Exception as e: + shutil.rmtree(workdir, ignore_errors=True) + raise BackendIsNotSupposedToImplementIt(str(e)) from e + + @classmethod + def run_model(cls, model, inputs, device="CPU", **kwargs): + return cls.prepare(model, device, **kwargs).run(inputs) + + @classmethod + def supports_device(cls, device): + return device == "CPU" + + +prepare = Onnx2cBackend.prepare +run_model = Onnx2cBackend.run_model +supports_device = Onnx2cBackend.supports_device diff --git a/runtimes/onnx2c/stable/Dockerfile b/runtimes/onnx2c/stable/Dockerfile new file mode 100644 index 0000000000..57b3aed268 --- /dev/null +++ b/runtimes/onnx2c/stable/Dockerfile @@ -0,0 +1,48 @@ +FROM ubuntu:24.04 + +# Disable interactive installation mode +ENV DEBIAN_FRONTEND=noninteractive +ENV PIP_BREAK_SYSTEM_PACKAGES=1 + +# Install Python, build tools, and protobuf dependencies +RUN apt-get update && apt-get install -y \ + python3 \ + python3-dev \ + python3-pip \ + cmake \ + gcc \ + g++ \ + git \ + libprotobuf-dev \ + protobuf-compiler \ + && apt-get clean autoclean && apt-get autoremove -y + +# Copy local directories +COPY ./test /root/test +COPY ./setup /root/setup +COPY ./backends/onnx2c/backend.py /root/onnx2c_backend.py + +# Install test report dependencies +RUN pip3 install --no-cache-dir -r /root/setup/requirements_report.txt + +############## ONNX Backend dependencies ########### +ENV ONNX_BACKEND="onnx2c_backend" +ENV PYTHONPATH="/root" + +# Install onnx (needed for Python backend wrapper and onnx2c build) +RUN pip3 install onnx + +# Build onnx2c from source +RUN git clone https://github.com/kraiskil/onnx2c /opt/onnx2c && \ + cd /opt/onnx2c && \ + mkdir build && cd build && \ + cmake -DCMAKE_BUILD_TYPE=Release .. && \ + make -j$(nproc) && \ + cp onnx2c /usr/local/bin/onnx2c +#################################################### + +RUN useradd -m runner && chown -R runner:runner /root +USER runner +WORKDIR /root + +CMD ["sh", "-c", ". /root/setup/docker-setup.sh && pytest /root/test/test_backend.py --onnx_backend=${ONNX_BACKEND} -k 'not _cuda' -v"] diff --git a/setup/config.json b/setup/config.json index 8eca259b97..17d5eec5d5 100644 --- a/setup/config.json +++ b/setup/config.json @@ -47,6 +47,12 @@ "results_dir": "./results/tvm/stable", "core_packages": ["apache-tvm"], "dockerfile_link": "runtimes/tvm/stable/Dockerfile" + }, + "onnx2c": { + "name": "onnx2c", + "results_dir": "./results/onnx2c/stable", + "core_packages": ["onnx2c"], + "dockerfile_link": "runtimes/onnx2c/stable/Dockerfile" } }, "development": { From f1e57bdf5667bb59214cc20ba97c58e81dd336a0 Mon Sep 17 00:00:00 2001 From: Andreas Fehlner Date: Sat, 21 Mar 2026 19:43:10 +0100 Subject: [PATCH 2/9] Fix ruff lint errors in onnx2c backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Extract _io_lines() and _parse_graph_io() and _build_binary() helpers to reduce McCabe complexity of _generate_harness (9→3) and prepare (11→4) - Use math.prod() instead of manual shape product loops - Fix Q003: use single outer quotes in f-strings containing double quotes - Add missing docstrings (D102, D105, D107) - Add strict=False to zip() calls (B905) - Suppress BLE001 with noqa on intentional broad except clauses Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Andreas Fehlner --- backends/onnx2c/backend.py | 201 +++++++++++++++++++------------------ 1 file changed, 102 insertions(+), 99 deletions(-) diff --git a/backends/onnx2c/backend.py b/backends/onnx2c/backend.py index c722f2984c..a38660084b 100644 --- a/backends/onnx2c/backend.py +++ b/backends/onnx2c/backend.py @@ -2,6 +2,7 @@ """ONNX backend wrapper for onnx2c (ONNX to C code generator).""" +import math import os import re import shutil @@ -74,6 +75,24 @@ def _parse_tensor_info(value_info): return _c_name(value_info.name), c_type, np_dtype, shape +def _io_lines(tensors_info, arg_offset, mode): + """Return C lines that open and read/write tensor binary files via argv.""" + rw_func = "fread" if mode == "rb" else "fwrite" + direction = "input" if mode == "rb" else "output" + lines = [] + for i, (c_nm, c_type, _, shape) in enumerate(tensors_info): + n = math.prod(shape) + arg_idx = arg_offset + i + 1 + lines.append(f' f = fopen(argv[{arg_idx}], "{mode}");') + lines.append( + f' if (!f) {{ fprintf(stderr, "Cannot open {direction} {i}\\n"); ' + f"return 1; }}" + ) + lines.append(f" {rw_func}({c_nm}, sizeof({c_type}), {n}, f);") + lines.append(" fclose(f);") + return lines + + def _generate_harness(inputs_info, outputs_info): """Return C harness source that feeds inputs, calls entry(), writes outputs.""" lines = [ @@ -84,74 +103,109 @@ def _generate_harness(inputs_info, outputs_info): "", ] - # extern declarations for input/output tensors (defined in generated model.c) for c_nm, c_type, _, shape in inputs_info: - n = 1 - for d in shape: - n *= d - lines.append(f"extern {c_type} {c_nm}[{n}];") + lines.append(f"extern {c_type} {c_nm}[{math.prod(shape)}];") for c_nm, c_type, _, shape in outputs_info: - n = 1 - for d in shape: - n *= d - lines.append(f"extern {c_type} {c_nm}[{n}];") + lines.append(f"extern {c_type} {c_nm}[{math.prod(shape)}];") lines += ["", "void entry(void);", "", "int main(int argc, char** argv) {"] lines.append(" FILE* f;") lines.append("") + lines += _io_lines(inputs_info, 0, "rb") + lines += ["", " entry();", ""] + lines += _io_lines(outputs_info, len(inputs_info), "wb") + lines += ["", " return 0;", "}"] + return "\n".join(lines) - # Read inputs from files passed as argv[1..n_inputs] - for i, (c_nm, c_type, _, shape) in enumerate(inputs_info): - n = 1 - for d in shape: - n *= d - lines.append(f" f = fopen(argv[{i + 1}], \"rb\");") - lines.append( - f" if (!f) {{ fprintf(stderr, \"Cannot open input {i}\\n\"); return 1; }}" - ) - lines.append(f" fread({c_nm}, sizeof({c_type}), {n}, f);") - lines.append(" fclose(f);") - lines += ["", " entry();", ""] +def _parse_graph_io(graph): + """Parse graph inputs/outputs; return (inputs_info, outputs_info) or raise.""" + initializer_names = {init.name for init in graph.initializer} + inputs_info = [] + for vi in graph.input: + if vi.name in initializer_names: + continue + info = _parse_tensor_info(vi) + if info is None: + raise BackendIsNotSupposedToImplementIt( + f"Input '{vi.name}' has unsupported type or dynamic shape" + ) + inputs_info.append(info) + + outputs_info = [] + for vi in graph.output: + info = _parse_tensor_info(vi) + if info is None: + raise BackendIsNotSupposedToImplementIt( + f"Output '{vi.name}' has unsupported type or dynamic shape" + ) + outputs_info.append(info) - # Write outputs to files passed as argv[n_inputs+1..n_inputs+n_outputs] - n_in = len(inputs_info) - for i, (c_nm, c_type, _, shape) in enumerate(outputs_info): - n = 1 - for d in shape: - n *= d - lines.append(f" f = fopen(argv[{n_in + i + 1}], \"wb\");") - lines.append( - f" if (!f) {{ fprintf(stderr, \"Cannot open output {i}\\n\"); return 1; }}" - ) - lines.append(f" fwrite({c_nm}, sizeof({c_type}), {n}, f);") - lines.append(" fclose(f);") + return inputs_info, outputs_info - lines += ["", " return 0;", "}"] - return "\n".join(lines) + +def _build_binary(workdir, model, inputs_info, outputs_info): + """Compile the ONNX model to a native binary; return the binary path.""" + model_path = os.path.join(workdir, "model.onnx") + with open(model_path, "wb") as f: + f.write(model.SerializeToString()) + + result = subprocess.run( + ["onnx2c", model_path], capture_output=True, timeout=120 + ) + if result.returncode != 0: + raise BackendIsNotSupposedToImplementIt( + f"onnx2c failed: {result.stderr.decode()}" + ) + model_c_path = os.path.join(workdir, "model.c") + with open(model_c_path, "wb") as f: + f.write(result.stdout) + + harness_path = os.path.join(workdir, "harness.c") + with open(harness_path, "w") as f: + f.write(_generate_harness(inputs_info, outputs_info)) + + binary_path = os.path.join(workdir, "model_exec") + result = subprocess.run( + ["gcc", "-O2", "-o", binary_path, harness_path, model_c_path, "-lm"], + capture_output=True, + timeout=120, + ) + if result.returncode != 0: + raise BackendIsNotSupposedToImplementIt( + f"Compilation failed: {result.stderr.decode()}" + ) + return binary_path class Onnx2cBackendRep(BackendRep): """Holds a compiled onnx2c binary and runs it for each inference call.""" def __init__(self, binary_path, workdir, inputs_info, outputs_info): + """Store compiled binary path and run-time type/shape information.""" self._binary = binary_path self._workdir = workdir # Store only dtype and shape needed at run time self._input_dtypes = [np_dtype for _, _, np_dtype, _ in inputs_info] - self._output_info = [(np_dtype, shape) for _, _, np_dtype, shape in outputs_info] + self._output_info = [ + (np_dtype, shape) for _, _, np_dtype, shape in outputs_info + ] def __del__(self): + """Remove the temporary working directory on garbage collection.""" if self._workdir and os.path.isdir(self._workdir): shutil.rmtree(self._workdir, ignore_errors=True) def run(self, inputs, **kwargs): + """Write inputs as binary files, invoke compiled binary, return outputs.""" run_dir = tempfile.mkdtemp() try: # Write each input as raw binary with the expected dtype input_files = [] - for i, (inp, dtype) in enumerate(zip(inputs, self._input_dtypes)): + for i, (inp, dtype) in enumerate( + zip(inputs, self._input_dtypes, strict=False) + ): path = os.path.join(run_dir, f"input_{i}.bin") np.asarray(inp, dtype=dtype).flatten().tofile(path) input_files.append(path) @@ -172,7 +226,9 @@ def run(self, inputs, **kwargs): ) outputs = [] - for path, (dtype, shape) in zip(output_files, self._output_info): + for path, (dtype, shape) in zip( + output_files, self._output_info, strict=False + ): arr = np.fromfile(path, dtype=dtype).reshape(shape) outputs.append(arr) return outputs @@ -185,91 +241,38 @@ class Onnx2cBackend(Backend): @classmethod def is_compatible(cls, model, device="CPU", **kwargs): + """Return whether this backend can attempt to handle the model.""" return True @classmethod def prepare(cls, model, device="CPU", **kwargs): - # Run ONNX shape inference so output shapes are concrete where possible + """Compile the ONNX model to native code and return a runnable rep.""" try: model = shape_inference.infer_shapes(model) - except Exception: + except Exception: # noqa: BLE001 pass - initializer_names = {init.name for init in model.graph.initializer} - - # Parse inputs (exclude initializers / weights) - inputs_info = [] - for vi in model.graph.input: - if vi.name in initializer_names: - continue - info = _parse_tensor_info(vi) - if info is None: - raise BackendIsNotSupposedToImplementIt( - f"Input '{vi.name}' has unsupported type or dynamic shape" - ) - inputs_info.append(info) - - # Parse outputs - outputs_info = [] - for vi in model.graph.output: - info = _parse_tensor_info(vi) - if info is None: - raise BackendIsNotSupposedToImplementIt( - f"Output '{vi.name}' has unsupported type or dynamic shape" - ) - outputs_info.append(info) + inputs_info, outputs_info = _parse_graph_io(model.graph) workdir = tempfile.mkdtemp() try: - # Serialize model - model_path = os.path.join(workdir, "model.onnx") - with open(model_path, "wb") as f: - f.write(model.SerializeToString()) - - # Generate C code with onnx2c - result = subprocess.run( - ["onnx2c", model_path], capture_output=True, timeout=120 - ) - if result.returncode != 0: - raise BackendIsNotSupposedToImplementIt( - f"onnx2c failed: {result.stderr.decode()}" - ) - model_c_path = os.path.join(workdir, "model.c") - with open(model_c_path, "wb") as f: - f.write(result.stdout) - - # Write harness - harness_path = os.path.join(workdir, "harness.c") - with open(harness_path, "w") as f: - f.write(_generate_harness(inputs_info, outputs_info)) - - # Compile - binary_path = os.path.join(workdir, "model_exec") - result = subprocess.run( - ["gcc", "-O2", "-o", binary_path, harness_path, model_c_path, "-lm"], - capture_output=True, - timeout=120, - ) - if result.returncode != 0: - raise BackendIsNotSupposedToImplementIt( - f"Compilation failed: {result.stderr.decode()}" - ) - + binary_path = _build_binary(workdir, model, inputs_info, outputs_info) return Onnx2cBackendRep(binary_path, workdir, inputs_info, outputs_info) - except BackendIsNotSupposedToImplementIt: shutil.rmtree(workdir, ignore_errors=True) raise - except Exception as e: + except Exception as e: # noqa: BLE001 shutil.rmtree(workdir, ignore_errors=True) raise BackendIsNotSupposedToImplementIt(str(e)) from e @classmethod def run_model(cls, model, inputs, device="CPU", **kwargs): + """Prepare then run a model in one call.""" return cls.prepare(model, device, **kwargs).run(inputs) @classmethod def supports_device(cls, device): + """Return whether the backend supports the given device.""" return device == "CPU" From 5cd2f8ed3d444e25920576e0fc83f882e1be241f Mon Sep 17 00:00:00 2001 From: Timo Stripf Date: Sat, 21 Mar 2026 14:52:33 +0100 Subject: [PATCH 3/9] Fix TVM runtime package compatibility (#102) * Fix TVM runtime package compatibility Signed-off-by: Timo Stripf * Fix TVM backend lint issues Signed-off-by: Timo Stripf --------- Signed-off-by: Timo Stripf Signed-off-by: Andreas Fehlner --- backends/tvm/backend.py | 22 ++++++++++++++++++---- runtimes/tvm/stable/Dockerfile | 3 ++- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/backends/tvm/backend.py b/backends/tvm/backend.py index 5c65e61a80..043c858b88 100644 --- a/backends/tvm/backend.py +++ b/backends/tvm/backend.py @@ -40,9 +40,16 @@ def _native_ops_only(model): def _tvm_worker(model_bytes, inputs, input_names, output_count, result_queue): """Compile and run an ONNX model via TVM Relay in an isolated subprocess.""" import onnx - import tvm - from tvm import relay - from tvm.contrib import graph_executor + + try: + import tvm + from tvm import relay + from tvm.contrib import graph_executor + except (ImportError, AttributeError, OSError, RuntimeError) as e: + msg = f"tvm import failed: {type(e).__name__}: {e}" + print(f"[tvm] SKIP {msg}", file=sys.stderr, flush=True) + result_queue.put(("error", msg)) + return model = onnx.ModelProto() model.ParseFromString(model_bytes) @@ -68,7 +75,14 @@ def _tvm_worker(model_bytes, inputs, input_names, output_count, result_queue): module.run() outputs = [module.get_output(i).numpy() for i in range(output_count)] result_queue.put(("ok", outputs)) - except (tvm.TVMError, RuntimeError, ValueError, TypeError, OSError) as e: + except ( + tvm.TVMError, + RuntimeError, + ValueError, + TypeError, + OSError, + ImportError, + ) as e: ops = sorted({n.op_type for n in model.graph.node}) msg = f"ops={ops} error={type(e).__name__}: {e}" print(f"[tvm] SKIP {msg}", file=sys.stderr, flush=True) diff --git a/runtimes/tvm/stable/Dockerfile b/runtimes/tvm/stable/Dockerfile index 3fe5af1658..36ea57d532 100644 --- a/runtimes/tvm/stable/Dockerfile +++ b/runtimes/tvm/stable/Dockerfile @@ -27,7 +27,8 @@ ENV PYTHONPATH="/root" # Set locale and install TVM and onnx RUN locale-gen en_US.UTF-8 && update-locale LANG=en_US.UTF-8 && \ pip3 install \ - onnx \ + "numpy<2" \ + "onnx<1.19" \ apache-tvm #################################################### From fdb70338b1a2eabec21316352dea257dbae6c3c5 Mon Sep 17 00:00:00 2001 From: Andreas Fehlner Date: Sat, 21 Mar 2026 19:56:07 +0100 Subject: [PATCH 4/9] Fix ruff format: collapse subprocess.run call to single line Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Andreas Fehlner --- backends/onnx2c/backend.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/backends/onnx2c/backend.py b/backends/onnx2c/backend.py index a38660084b..9888bc441e 100644 --- a/backends/onnx2c/backend.py +++ b/backends/onnx2c/backend.py @@ -151,9 +151,7 @@ def _build_binary(workdir, model, inputs_info, outputs_info): with open(model_path, "wb") as f: f.write(model.SerializeToString()) - result = subprocess.run( - ["onnx2c", model_path], capture_output=True, timeout=120 - ) + result = subprocess.run(["onnx2c", model_path], capture_output=True, timeout=120) if result.returncode != 0: raise BackendIsNotSupposedToImplementIt( f"onnx2c failed: {result.stderr.decode()}" From 0b538acc3c334fab53f4f8add498c824b3399c63 Mon Sep 17 00:00:00 2001 From: Andreas Fehlner Date: Sat, 21 Mar 2026 20:12:55 +0100 Subject: [PATCH 5/9] Add path-filtered push/PR triggers to benchmark workflow On push to main or on pull_request, a 'changes' job detects which backends' files were modified using dorny/paths-filter. Each backend job then only runs when its own files changed (runtimes/, backends/, test/, or setup/), keeping CI fast for unrelated changes. On schedule and workflow_dispatch, all jobs run unconditionally as before. Results are only uploaded and deployed on schedule/workflow_dispatch. Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Andreas Fehlner --- .github/workflows/benchmark-backends.yml | 98 ++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/.github/workflows/benchmark-backends.yml b/.github/workflows/benchmark-backends.yml index 0ab462055d..1f5834236f 100644 --- a/.github/workflows/benchmark-backends.yml +++ b/.github/workflows/benchmark-backends.yml @@ -4,11 +4,87 @@ on: schedule: - cron: '0 0 * * *' workflow_dispatch: + push: + branches: [main] + paths: + - 'runtimes/**' + - 'backends/**' + - 'test/**' + - 'setup/**' + pull_request: + paths: + - 'runtimes/**' + - 'backends/**' + - 'test/**' + - 'setup/**' jobs: + changes: + runs-on: ubuntu-latest + if: github.event_name == 'push' || github.event_name == 'pull_request' + outputs: + onnxruntime: ${{ steps.filter.outputs.onnxruntime }} + onnx_tf: ${{ steps.filter.outputs.onnx_tf }} + jaxonnxruntime: ${{ steps.filter.outputs.jaxonnxruntime }} + emx_onnx_cgen: ${{ steps.filter.outputs.emx_onnx_cgen }} + tract: ${{ steps.filter.outputs.tract }} + onnx_reference: ${{ steps.filter.outputs.onnx_reference }} + opencv: ${{ steps.filter.outputs.opencv }} + tvm: ${{ steps.filter.outputs.tvm }} + onnx2c: ${{ steps.filter.outputs.onnx2c }} + steps: + - uses: actions/checkout@v6 + - uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + onnxruntime: + - 'runtimes/ort/**' + - 'test/**' + - 'setup/**' + onnx_tf: + - 'runtimes/onnx-tf/**' + - 'test/**' + - 'setup/**' + jaxonnxruntime: + - 'runtimes/jaxonnxruntime/**' + - 'test/**' + - 'setup/**' + emx_onnx_cgen: + - 'runtimes/emx-onnx-cgen/**' + - 'test/**' + - 'setup/**' + tract: + - 'runtimes/tract/**' + - 'backends/tract/**' + - 'test/**' + - 'setup/**' + onnx_reference: + - 'runtimes/onnx-reference/**' + - 'backends/onnx_reference/**' + - 'test/**' + - 'setup/**' + opencv: + - 'runtimes/opencv/**' + - 'backends/opencv/**' + - 'test/**' + - 'setup/**' + tvm: + - 'runtimes/tvm/**' + - 'backends/tvm/**' + - 'test/**' + - 'setup/**' + onnx2c: + - 'runtimes/onnx2c/**' + - 'backends/onnx2c/**' + - 'test/**' + - 'setup/**' + onnxruntime_stable: runs-on: ubuntu-latest timeout-minutes: 90 + needs: changes + if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.changes.outputs.onnxruntime == 'true') permissions: contents: read steps: @@ -31,6 +107,8 @@ jobs: onnx_tf_stable: runs-on: ubuntu-latest timeout-minutes: 90 + needs: changes + if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.changes.outputs.onnx_tf == 'true') permissions: contents: read steps: @@ -54,6 +132,8 @@ jobs: onnx_tf_dev: runs-on: ubuntu-latest timeout-minutes: 90 + needs: changes + if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.changes.outputs.onnx_tf == 'true') permissions: contents: read steps: @@ -77,6 +157,8 @@ jobs: jaxonnxruntime_stable: runs-on: ubuntu-latest timeout-minutes: 90 + needs: changes + if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.changes.outputs.jaxonnxruntime == 'true') permissions: contents: read steps: @@ -99,6 +181,8 @@ jobs: jaxonnxruntime_dev: runs-on: ubuntu-latest timeout-minutes: 90 + needs: changes + if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.changes.outputs.jaxonnxruntime == 'true') permissions: contents: read steps: @@ -125,6 +209,8 @@ jobs: onnxruntime_dev: runs-on: ubuntu-latest timeout-minutes: 90 + needs: changes + if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.changes.outputs.onnxruntime == 'true') permissions: contents: read steps: @@ -150,6 +236,8 @@ jobs: emx_onnx_cgen_stable: runs-on: ubuntu-latest timeout-minutes: 90 + needs: changes + if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.changes.outputs.emx_onnx_cgen == 'true') permissions: contents: read steps: @@ -175,6 +263,8 @@ jobs: tract_stable: runs-on: ubuntu-latest timeout-minutes: 90 + needs: changes + if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.changes.outputs.tract == 'true') permissions: contents: read steps: @@ -200,6 +290,8 @@ jobs: onnx_reference_stable: runs-on: ubuntu-latest timeout-minutes: 90 + needs: changes + if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.changes.outputs.onnx_reference == 'true') permissions: contents: read steps: @@ -225,6 +317,8 @@ jobs: opencv_stable: runs-on: ubuntu-latest timeout-minutes: 90 + needs: changes + if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.changes.outputs.opencv == 'true') permissions: contents: read steps: @@ -250,6 +344,8 @@ jobs: tvm_stable: runs-on: ubuntu-latest timeout-minutes: 90 + needs: changes + if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.changes.outputs.tvm == 'true') permissions: contents: read steps: @@ -275,6 +371,8 @@ jobs: onnx2c_stable: runs-on: ubuntu-latest timeout-minutes: 90 + needs: changes + if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.changes.outputs.onnx2c == 'true') permissions: contents: read steps: From 78622b87908f432ccde523b212bc5ed417cddca4 Mon Sep 17 00:00:00 2001 From: Andreas Fehlner Date: Sat, 21 Mar 2026 20:17:37 +0100 Subject: [PATCH 6/9] Fix onnx2c build: clone with --recurse-submodules onnx2c uses git submodules (cmake_timestamp, benchmark) which were missing, causing CMake to fail. Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Andreas Fehlner --- runtimes/onnx2c/stable/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtimes/onnx2c/stable/Dockerfile b/runtimes/onnx2c/stable/Dockerfile index 57b3aed268..dd3f1912be 100644 --- a/runtimes/onnx2c/stable/Dockerfile +++ b/runtimes/onnx2c/stable/Dockerfile @@ -33,7 +33,7 @@ ENV PYTHONPATH="/root" RUN pip3 install onnx # Build onnx2c from source -RUN git clone https://github.com/kraiskil/onnx2c /opt/onnx2c && \ +RUN git clone --recurse-submodules https://github.com/kraiskil/onnx2c /opt/onnx2c && \ cd /opt/onnx2c && \ mkdir build && cd build && \ cmake -DCMAKE_BUILD_TYPE=Release .. && \ From 790e117d895ede5d1b9a1e869f35c9cda7806e32 Mon Sep 17 00:00:00 2001 From: Andreas Fehlner Date: Sat, 21 Mar 2026 20:37:41 +0100 Subject: [PATCH 7/9] Pin onnx2c to last known-good commit The latest commit (f963de9) introduced QLinearConv with a broken `override` declaration that fails to compile. Pin to the commit immediately before it (e6308a7) until upstream is fixed. Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Andreas Fehlner --- runtimes/onnx2c/stable/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/runtimes/onnx2c/stable/Dockerfile b/runtimes/onnx2c/stable/Dockerfile index dd3f1912be..a9034b7d25 100644 --- a/runtimes/onnx2c/stable/Dockerfile +++ b/runtimes/onnx2c/stable/Dockerfile @@ -35,6 +35,8 @@ RUN pip3 install onnx # Build onnx2c from source RUN git clone --recurse-submodules https://github.com/kraiskil/onnx2c /opt/onnx2c && \ cd /opt/onnx2c && \ + git checkout e6308a729d067ea3be729c840558ee7832e10b6c && \ + git submodule update --init --recursive && \ mkdir build && cd build && \ cmake -DCMAKE_BUILD_TYPE=Release .. && \ make -j$(nproc) && \ From 1044a0b172c35d299cfd4fbef160dfd2e42923f1 Mon Sep 17 00:00:00 2001 From: Andreas Fehlner Date: Sat, 21 Mar 2026 21:40:03 +0100 Subject: [PATCH 8/9] Narrow path filters: exclude setup/config.json from triggers config.json is only used by the website generator and does not affect test execution. Using setup/** caused all backends to re-run whenever a new backend was added (since adding one requires editing config.json). Now only the files that actually affect runtime behaviour trigger jobs: docker-setup.sh, env.list, and requirements_report.txt. Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Andreas Fehlner --- .github/workflows/benchmark-backends.yml | 44 ++++++++++++++++++------ 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/.github/workflows/benchmark-backends.yml b/.github/workflows/benchmark-backends.yml index 1f5834236f..f4663e4c56 100644 --- a/.github/workflows/benchmark-backends.yml +++ b/.github/workflows/benchmark-backends.yml @@ -10,13 +10,17 @@ on: - 'runtimes/**' - 'backends/**' - 'test/**' - - 'setup/**' + - 'setup/docker-setup.sh' + - 'setup/env.list' + - 'setup/requirements_report.txt' pull_request: paths: - 'runtimes/**' - 'backends/**' - 'test/**' - - 'setup/**' + - 'setup/docker-setup.sh' + - 'setup/env.list' + - 'setup/requirements_report.txt' jobs: changes: @@ -41,44 +45,62 @@ jobs: onnxruntime: - 'runtimes/ort/**' - 'test/**' - - 'setup/**' + - 'setup/docker-setup.sh' + - 'setup/env.list' + - 'setup/requirements_report.txt' onnx_tf: - 'runtimes/onnx-tf/**' - 'test/**' - - 'setup/**' + - 'setup/docker-setup.sh' + - 'setup/env.list' + - 'setup/requirements_report.txt' jaxonnxruntime: - 'runtimes/jaxonnxruntime/**' - 'test/**' - - 'setup/**' + - 'setup/docker-setup.sh' + - 'setup/env.list' + - 'setup/requirements_report.txt' emx_onnx_cgen: - 'runtimes/emx-onnx-cgen/**' - 'test/**' - - 'setup/**' + - 'setup/docker-setup.sh' + - 'setup/env.list' + - 'setup/requirements_report.txt' tract: - 'runtimes/tract/**' - 'backends/tract/**' - 'test/**' - - 'setup/**' + - 'setup/docker-setup.sh' + - 'setup/env.list' + - 'setup/requirements_report.txt' onnx_reference: - 'runtimes/onnx-reference/**' - 'backends/onnx_reference/**' - 'test/**' - - 'setup/**' + - 'setup/docker-setup.sh' + - 'setup/env.list' + - 'setup/requirements_report.txt' opencv: - 'runtimes/opencv/**' - 'backends/opencv/**' - 'test/**' - - 'setup/**' + - 'setup/docker-setup.sh' + - 'setup/env.list' + - 'setup/requirements_report.txt' tvm: - 'runtimes/tvm/**' - 'backends/tvm/**' - 'test/**' - - 'setup/**' + - 'setup/docker-setup.sh' + - 'setup/env.list' + - 'setup/requirements_report.txt' onnx2c: - 'runtimes/onnx2c/**' - 'backends/onnx2c/**' - 'test/**' - - 'setup/**' + - 'setup/docker-setup.sh' + - 'setup/env.list' + - 'setup/requirements_report.txt' onnxruntime_stable: runs-on: ubuntu-latest From 6b09801b35d1f8aed08f24226f1175b8069af717 Mon Sep 17 00:00:00 2001 From: Andreas Fehlner Date: Sat, 21 Mar 2026 22:08:28 +0100 Subject: [PATCH 9/9] update Signed-off-by: Andreas Fehlner --- backends/onnx2c/backend.py | 64 ++++++++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 24 deletions(-) diff --git a/backends/onnx2c/backend.py b/backends/onnx2c/backend.py index 9888bc441e..df02052c4c 100644 --- a/backends/onnx2c/backend.py +++ b/backends/onnx2c/backend.py @@ -72,15 +72,42 @@ def _parse_tensor_info(value_info): else: return None + if not shape: + # Scalar tensors use pointer parameters in onnx2c — not supported here + return None + return _c_name(value_info.name), c_type, np_dtype, shape -def _io_lines(tensors_info, arg_offset, mode): - """Return C lines that open and read/write tensor binary files via argv.""" +def _c_tensor_dims(shape): + """Return C array dimension string e.g. '[3][4][5]' for shape [3, 4, 5].""" + return "".join(f"[{d}]" for d in shape) + + +def _decl_lines(inputs_info, outputs_info): + """Return buffer declarations and entry() forward declaration.""" + lines = [] + for i, (_, c_type, _, shape) in enumerate(inputs_info): + lines.append(f"{c_type} inp_{i}{_c_tensor_dims(shape)};") + for i, (_, c_type, _, shape) in enumerate(outputs_info): + lines.append(f"{c_type} out_{i}{_c_tensor_dims(shape)};") + lines.append("") + params = [] + for i, (_, c_type, _, shape) in enumerate(inputs_info): + params.append(f"const {c_type} inp_{i}{_c_tensor_dims(shape)}") + for i, (_, c_type, _, shape) in enumerate(outputs_info): + params.append(f"{c_type} out_{i}{_c_tensor_dims(shape)}") + lines.append(f"void entry({', '.join(params)});") + return lines + + +def _file_io_lines(tensors_info, arg_offset, mode): + """Return C lines to read/write tensor binary files using positional buffers.""" rw_func = "fread" if mode == "rb" else "fwrite" direction = "input" if mode == "rb" else "output" + prefix = "inp" if mode == "rb" else "out" lines = [] - for i, (c_nm, c_type, _, shape) in enumerate(tensors_info): + for i, (_, c_type, _, shape) in enumerate(tensors_info): n = math.prod(shape) arg_idx = arg_offset + i + 1 lines.append(f' f = fopen(argv[{arg_idx}], "{mode}");') @@ -88,33 +115,22 @@ def _io_lines(tensors_info, arg_offset, mode): f' if (!f) {{ fprintf(stderr, "Cannot open {direction} {i}\\n"); ' f"return 1; }}" ) - lines.append(f" {rw_func}({c_nm}, sizeof({c_type}), {n}, f);") + lines.append(f" {rw_func}({prefix}_{i}, sizeof({c_type}), {n}, f);") lines.append(" fclose(f);") return lines def _generate_harness(inputs_info, outputs_info): - """Return C harness source that feeds inputs, calls entry(), writes outputs.""" - lines = [ - "#include ", - "#include ", - "#include ", - "#include ", - "", - ] - - for c_nm, c_type, _, shape in inputs_info: - lines.append(f"extern {c_type} {c_nm}[{math.prod(shape)}];") - - for c_nm, c_type, _, shape in outputs_info: - lines.append(f"extern {c_type} {c_nm}[{math.prod(shape)}];") - - lines += ["", "void entry(void);", "", "int main(int argc, char** argv) {"] - lines.append(" FILE* f;") + """Return C harness that calls onnx2c entry() with array parameters.""" + lines = ["#include ", "#include ", "#include ", ""] + lines += _decl_lines(inputs_info, outputs_info) + lines += ["", "int main(int argc, char** argv) {", " FILE* f;", ""] + lines += _file_io_lines(inputs_info, 0, "rb") + in_args = [f"inp_{i}" for i in range(len(inputs_info))] + out_args = [f"out_{i}" for i in range(len(outputs_info))] + lines.append(f"\n entry({', '.join(in_args + out_args)});") lines.append("") - lines += _io_lines(inputs_info, 0, "rb") - lines += ["", " entry();", ""] - lines += _io_lines(outputs_info, len(inputs_info), "wb") + lines += _file_io_lines(outputs_info, len(inputs_info), "wb") lines += ["", " return 0;", "}"] return "\n".join(lines)