diff --git a/.github/workflows/benchmark-backends.yml b/.github/workflows/benchmark-backends.yml index 0bb0f4233c..f4663e4c56 100644 --- a/.github/workflows/benchmark-backends.yml +++ b/.github/workflows/benchmark-backends.yml @@ -4,11 +4,109 @@ on: schedule: - cron: '0 0 * * *' workflow_dispatch: + push: + branches: [main] + paths: + - 'runtimes/**' + - 'backends/**' + - 'test/**' + - 'setup/docker-setup.sh' + - 'setup/env.list' + - 'setup/requirements_report.txt' + pull_request: + paths: + - 'runtimes/**' + - 'backends/**' + - 'test/**' + - 'setup/docker-setup.sh' + - 'setup/env.list' + - 'setup/requirements_report.txt' jobs: + changes: + runs-on: ubuntu-latest + if: github.event_name == 'push' || github.event_name == 'pull_request' + outputs: + onnxruntime: ${{ steps.filter.outputs.onnxruntime }} + onnx_tf: ${{ steps.filter.outputs.onnx_tf }} + jaxonnxruntime: ${{ steps.filter.outputs.jaxonnxruntime }} + emx_onnx_cgen: ${{ steps.filter.outputs.emx_onnx_cgen }} + tract: ${{ steps.filter.outputs.tract }} + onnx_reference: ${{ steps.filter.outputs.onnx_reference }} + opencv: ${{ steps.filter.outputs.opencv }} + tvm: ${{ steps.filter.outputs.tvm }} + onnx2c: ${{ steps.filter.outputs.onnx2c }} + steps: + - uses: actions/checkout@v6 + - uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + onnxruntime: + - 'runtimes/ort/**' + - 'test/**' + - 'setup/docker-setup.sh' + - 'setup/env.list' + - 'setup/requirements_report.txt' + onnx_tf: + - 'runtimes/onnx-tf/**' + - 'test/**' + - 'setup/docker-setup.sh' + - 'setup/env.list' + - 'setup/requirements_report.txt' + jaxonnxruntime: + - 'runtimes/jaxonnxruntime/**' + - 'test/**' + - 'setup/docker-setup.sh' + - 'setup/env.list' + - 'setup/requirements_report.txt' + emx_onnx_cgen: + - 'runtimes/emx-onnx-cgen/**' + - 'test/**' + - 'setup/docker-setup.sh' + - 'setup/env.list' + - 'setup/requirements_report.txt' + tract: + - 'runtimes/tract/**' + - 'backends/tract/**' + - 'test/**' + - 'setup/docker-setup.sh' + - 'setup/env.list' + - 'setup/requirements_report.txt' + onnx_reference: + - 'runtimes/onnx-reference/**' + - 'backends/onnx_reference/**' + - 'test/**' + - 'setup/docker-setup.sh' + - 'setup/env.list' + - 'setup/requirements_report.txt' + opencv: + - 'runtimes/opencv/**' + - 'backends/opencv/**' + - 'test/**' + - 'setup/docker-setup.sh' + - 'setup/env.list' + - 'setup/requirements_report.txt' + tvm: + - 'runtimes/tvm/**' + - 'backends/tvm/**' + - 'test/**' + - 'setup/docker-setup.sh' + - 'setup/env.list' + - 'setup/requirements_report.txt' + onnx2c: + - 'runtimes/onnx2c/**' + - 'backends/onnx2c/**' + - 'test/**' + - 'setup/docker-setup.sh' + - 'setup/env.list' + - 'setup/requirements_report.txt' + onnxruntime_stable: runs-on: ubuntu-latest timeout-minutes: 90 + needs: changes + if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.changes.outputs.onnxruntime == 'true') permissions: contents: read steps: @@ -31,6 +129,8 @@ jobs: onnx_tf_stable: runs-on: ubuntu-latest timeout-minutes: 90 + needs: changes + if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.changes.outputs.onnx_tf == 'true') permissions: contents: read steps: @@ -54,6 +154,8 @@ jobs: onnx_tf_dev: runs-on: ubuntu-latest timeout-minutes: 90 + needs: changes + if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.changes.outputs.onnx_tf == 'true') permissions: contents: read steps: @@ -77,6 +179,8 @@ jobs: jaxonnxruntime_stable: runs-on: ubuntu-latest timeout-minutes: 90 + needs: changes + if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.changes.outputs.jaxonnxruntime == 'true') permissions: contents: read steps: @@ -99,6 +203,8 @@ jobs: jaxonnxruntime_dev: runs-on: ubuntu-latest timeout-minutes: 90 + needs: changes + if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.changes.outputs.jaxonnxruntime == 'true') permissions: contents: read steps: @@ -125,6 +231,8 @@ jobs: onnxruntime_dev: runs-on: ubuntu-latest timeout-minutes: 90 + needs: changes + if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.changes.outputs.onnxruntime == 'true') permissions: contents: read steps: @@ -150,6 +258,8 @@ jobs: emx_onnx_cgen_stable: runs-on: ubuntu-latest timeout-minutes: 90 + needs: changes + if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.changes.outputs.emx_onnx_cgen == 'true') permissions: contents: read steps: @@ -175,6 +285,8 @@ jobs: tract_stable: runs-on: ubuntu-latest timeout-minutes: 90 + needs: changes + if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.changes.outputs.tract == 'true') permissions: contents: read steps: @@ -200,6 +312,8 @@ jobs: onnx_reference_stable: runs-on: ubuntu-latest timeout-minutes: 90 + needs: changes + if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.changes.outputs.onnx_reference == 'true') permissions: contents: read steps: @@ -225,6 +339,8 @@ jobs: opencv_stable: runs-on: ubuntu-latest timeout-minutes: 90 + needs: changes + if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.changes.outputs.opencv == 'true') permissions: contents: read steps: @@ -250,6 +366,8 @@ jobs: tvm_stable: runs-on: ubuntu-latest timeout-minutes: 90 + needs: changes + if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.changes.outputs.tvm == 'true') permissions: contents: read steps: @@ -272,11 +390,38 @@ jobs: name: results-tvm-stable path: results/tvm/stable/ + onnx2c_stable: + runs-on: ubuntu-latest + timeout-minutes: 90 + needs: changes + if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.changes.outputs.onnx2c == 'true') + permissions: + contents: read + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Build docker image + run: docker build -t scoreboard/onnx2c -f runtimes/onnx2c/stable/Dockerfile . + + - name: Prepare results directory + run: mkdir -p ${{ github.workspace }}/results/onnx2c/stable && chmod 777 ${{ github.workspace }}/results/onnx2c/stable + + - name: Run docker container + run: docker run --name onnx2c --env-file setup/env.list -v ${{ github.workspace }}/results/onnx2c/stable:/root/results scoreboard/onnx2c || true + + - name: Upload results + if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} + uses: actions/upload-artifact@v7 + with: + name: results-onnx2c-stable + path: results/onnx2c/stable/ + collect_and_deploy: runs-on: ubuntu-latest permissions: contents: write - needs: [onnxruntime_stable, onnxruntime_dev, onnx_tf_stable, onnx_tf_dev, jaxonnxruntime_stable, jaxonnxruntime_dev, emx_onnx_cgen_stable, tract_stable, onnx_reference_stable, opencv_stable, tvm_stable] + needs: [onnxruntime_stable, onnxruntime_dev, onnx_tf_stable, onnx_tf_dev, jaxonnxruntime_stable, jaxonnxruntime_dev, emx_onnx_cgen_stable, tract_stable, onnx_reference_stable, opencv_stable, tvm_stable, onnx2c_stable] if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} steps: - name: Checkout code @@ -360,6 +505,12 @@ jobs: name: results-tvm-stable path: results/tvm/stable/ + - name: Download onnx2c stable results + uses: actions/download-artifact@v8 + with: + name: results-onnx2c-stable + path: results/onnx2c/stable/ + - name: Deploy results run: | git add results diff --git a/backends/onnx2c/backend.py b/backends/onnx2c/backend.py new file mode 100644 index 0000000000..df02052c4c --- /dev/null +++ b/backends/onnx2c/backend.py @@ -0,0 +1,295 @@ +# SPDX-License-Identifier: Apache-2.0 + +"""ONNX backend wrapper for onnx2c (ONNX to C code generator).""" + +import math +import os +import re +import shutil +import subprocess +import tempfile + +import numpy as np +from onnx import TensorProto, shape_inference +from onnx.backend.base import Backend, BackendRep +from onnx.backend.test.runner import BackendIsNotSupposedToImplementIt + + +def _c_name(name): + """Convert ONNX tensor name to valid C identifier (matching onnx2c's naming).""" + name = re.sub(r"[^a-zA-Z0-9_]", "_", name) + if name and name[0].isdigit(): + name = "_" + name + return name + + +_ONNX_TO_C_TYPE = { + TensorProto.FLOAT: "float", + TensorProto.DOUBLE: "double", + TensorProto.INT8: "int8_t", + TensorProto.INT16: "int16_t", + TensorProto.INT32: "int32_t", + TensorProto.INT64: "int64_t", + TensorProto.UINT8: "uint8_t", + TensorProto.UINT16: "uint16_t", + TensorProto.UINT32: "uint32_t", + TensorProto.UINT64: "uint64_t", + TensorProto.BOOL: "bool", +} + +_ONNX_TO_NUMPY = { + TensorProto.FLOAT: np.float32, + TensorProto.DOUBLE: np.float64, + TensorProto.INT8: np.int8, + TensorProto.INT16: np.int16, + TensorProto.INT32: np.int32, + TensorProto.INT64: np.int64, + TensorProto.UINT8: np.uint8, + TensorProto.UINT16: np.uint16, + TensorProto.UINT32: np.uint32, + TensorProto.UINT64: np.uint64, + TensorProto.BOOL: np.bool_, +} + + +def _parse_tensor_info(value_info): + """Return (c_name, c_type, numpy_dtype, shape) or None if unsupported.""" + t = value_info.type.tensor_type + elem_type = t.elem_type + c_type = _ONNX_TO_C_TYPE.get(elem_type) + np_dtype = _ONNX_TO_NUMPY.get(elem_type) + if c_type is None or np_dtype is None: + return None + + shape = [] + if t.HasField("shape"): + for dim in t.shape.dim: + if dim.HasField("dim_value") and dim.dim_value > 0: + shape.append(dim.dim_value) + else: + # Dynamic or unknown dimension — not supported by onnx2c + return None + else: + return None + + if not shape: + # Scalar tensors use pointer parameters in onnx2c — not supported here + return None + + return _c_name(value_info.name), c_type, np_dtype, shape + + +def _c_tensor_dims(shape): + """Return C array dimension string e.g. '[3][4][5]' for shape [3, 4, 5].""" + return "".join(f"[{d}]" for d in shape) + + +def _decl_lines(inputs_info, outputs_info): + """Return buffer declarations and entry() forward declaration.""" + lines = [] + for i, (_, c_type, _, shape) in enumerate(inputs_info): + lines.append(f"{c_type} inp_{i}{_c_tensor_dims(shape)};") + for i, (_, c_type, _, shape) in enumerate(outputs_info): + lines.append(f"{c_type} out_{i}{_c_tensor_dims(shape)};") + lines.append("") + params = [] + for i, (_, c_type, _, shape) in enumerate(inputs_info): + params.append(f"const {c_type} inp_{i}{_c_tensor_dims(shape)}") + for i, (_, c_type, _, shape) in enumerate(outputs_info): + params.append(f"{c_type} out_{i}{_c_tensor_dims(shape)}") + lines.append(f"void entry({', '.join(params)});") + return lines + + +def _file_io_lines(tensors_info, arg_offset, mode): + """Return C lines to read/write tensor binary files using positional buffers.""" + rw_func = "fread" if mode == "rb" else "fwrite" + direction = "input" if mode == "rb" else "output" + prefix = "inp" if mode == "rb" else "out" + lines = [] + for i, (_, c_type, _, shape) in enumerate(tensors_info): + n = math.prod(shape) + arg_idx = arg_offset + i + 1 + lines.append(f' f = fopen(argv[{arg_idx}], "{mode}");') + lines.append( + f' if (!f) {{ fprintf(stderr, "Cannot open {direction} {i}\\n"); ' + f"return 1; }}" + ) + lines.append(f" {rw_func}({prefix}_{i}, sizeof({c_type}), {n}, f);") + lines.append(" fclose(f);") + return lines + + +def _generate_harness(inputs_info, outputs_info): + """Return C harness that calls onnx2c entry() with array parameters.""" + lines = ["#include ", "#include ", "#include ", ""] + lines += _decl_lines(inputs_info, outputs_info) + lines += ["", "int main(int argc, char** argv) {", " FILE* f;", ""] + lines += _file_io_lines(inputs_info, 0, "rb") + in_args = [f"inp_{i}" for i in range(len(inputs_info))] + out_args = [f"out_{i}" for i in range(len(outputs_info))] + lines.append(f"\n entry({', '.join(in_args + out_args)});") + lines.append("") + lines += _file_io_lines(outputs_info, len(inputs_info), "wb") + lines += ["", " return 0;", "}"] + return "\n".join(lines) + + +def _parse_graph_io(graph): + """Parse graph inputs/outputs; return (inputs_info, outputs_info) or raise.""" + initializer_names = {init.name for init in graph.initializer} + inputs_info = [] + for vi in graph.input: + if vi.name in initializer_names: + continue + info = _parse_tensor_info(vi) + if info is None: + raise BackendIsNotSupposedToImplementIt( + f"Input '{vi.name}' has unsupported type or dynamic shape" + ) + inputs_info.append(info) + + outputs_info = [] + for vi in graph.output: + info = _parse_tensor_info(vi) + if info is None: + raise BackendIsNotSupposedToImplementIt( + f"Output '{vi.name}' has unsupported type or dynamic shape" + ) + outputs_info.append(info) + + return inputs_info, outputs_info + + +def _build_binary(workdir, model, inputs_info, outputs_info): + """Compile the ONNX model to a native binary; return the binary path.""" + model_path = os.path.join(workdir, "model.onnx") + with open(model_path, "wb") as f: + f.write(model.SerializeToString()) + + result = subprocess.run(["onnx2c", model_path], capture_output=True, timeout=120) + if result.returncode != 0: + raise BackendIsNotSupposedToImplementIt( + f"onnx2c failed: {result.stderr.decode()}" + ) + model_c_path = os.path.join(workdir, "model.c") + with open(model_c_path, "wb") as f: + f.write(result.stdout) + + harness_path = os.path.join(workdir, "harness.c") + with open(harness_path, "w") as f: + f.write(_generate_harness(inputs_info, outputs_info)) + + binary_path = os.path.join(workdir, "model_exec") + result = subprocess.run( + ["gcc", "-O2", "-o", binary_path, harness_path, model_c_path, "-lm"], + capture_output=True, + timeout=120, + ) + if result.returncode != 0: + raise BackendIsNotSupposedToImplementIt( + f"Compilation failed: {result.stderr.decode()}" + ) + return binary_path + + +class Onnx2cBackendRep(BackendRep): + """Holds a compiled onnx2c binary and runs it for each inference call.""" + + def __init__(self, binary_path, workdir, inputs_info, outputs_info): + """Store compiled binary path and run-time type/shape information.""" + self._binary = binary_path + self._workdir = workdir + # Store only dtype and shape needed at run time + self._input_dtypes = [np_dtype for _, _, np_dtype, _ in inputs_info] + self._output_info = [ + (np_dtype, shape) for _, _, np_dtype, shape in outputs_info + ] + + def __del__(self): + """Remove the temporary working directory on garbage collection.""" + if self._workdir and os.path.isdir(self._workdir): + shutil.rmtree(self._workdir, ignore_errors=True) + + def run(self, inputs, **kwargs): + """Write inputs as binary files, invoke compiled binary, return outputs.""" + run_dir = tempfile.mkdtemp() + try: + # Write each input as raw binary with the expected dtype + input_files = [] + for i, (inp, dtype) in enumerate( + zip(inputs, self._input_dtypes, strict=False) + ): + path = os.path.join(run_dir, f"input_{i}.bin") + np.asarray(inp, dtype=dtype).flatten().tofile(path) + input_files.append(path) + + output_files = [] + for i in range(len(self._output_info)): + path = os.path.join(run_dir, f"output_{i}.bin") + output_files.append(path) + + result = subprocess.run( + [self._binary] + input_files + output_files, + capture_output=True, + timeout=60, + ) + if result.returncode != 0: + raise BackendIsNotSupposedToImplementIt( + f"onnx2c binary failed: {result.stderr.decode()}" + ) + + outputs = [] + for path, (dtype, shape) in zip( + output_files, self._output_info, strict=False + ): + arr = np.fromfile(path, dtype=dtype).reshape(shape) + outputs.append(arr) + return outputs + finally: + shutil.rmtree(run_dir, ignore_errors=True) + + +class Onnx2cBackend(Backend): + """ONNX backend that compiles models to native code via onnx2c.""" + + @classmethod + def is_compatible(cls, model, device="CPU", **kwargs): + """Return whether this backend can attempt to handle the model.""" + return True + + @classmethod + def prepare(cls, model, device="CPU", **kwargs): + """Compile the ONNX model to native code and return a runnable rep.""" + try: + model = shape_inference.infer_shapes(model) + except Exception: # noqa: BLE001 + pass + + inputs_info, outputs_info = _parse_graph_io(model.graph) + + workdir = tempfile.mkdtemp() + try: + binary_path = _build_binary(workdir, model, inputs_info, outputs_info) + return Onnx2cBackendRep(binary_path, workdir, inputs_info, outputs_info) + except BackendIsNotSupposedToImplementIt: + shutil.rmtree(workdir, ignore_errors=True) + raise + except Exception as e: # noqa: BLE001 + shutil.rmtree(workdir, ignore_errors=True) + raise BackendIsNotSupposedToImplementIt(str(e)) from e + + @classmethod + def run_model(cls, model, inputs, device="CPU", **kwargs): + """Prepare then run a model in one call.""" + return cls.prepare(model, device, **kwargs).run(inputs) + + @classmethod + def supports_device(cls, device): + """Return whether the backend supports the given device.""" + return device == "CPU" + + +prepare = Onnx2cBackend.prepare +run_model = Onnx2cBackend.run_model +supports_device = Onnx2cBackend.supports_device diff --git a/backends/tvm/backend.py b/backends/tvm/backend.py index 5c65e61a80..043c858b88 100644 --- a/backends/tvm/backend.py +++ b/backends/tvm/backend.py @@ -40,9 +40,16 @@ def _native_ops_only(model): def _tvm_worker(model_bytes, inputs, input_names, output_count, result_queue): """Compile and run an ONNX model via TVM Relay in an isolated subprocess.""" import onnx - import tvm - from tvm import relay - from tvm.contrib import graph_executor + + try: + import tvm + from tvm import relay + from tvm.contrib import graph_executor + except (ImportError, AttributeError, OSError, RuntimeError) as e: + msg = f"tvm import failed: {type(e).__name__}: {e}" + print(f"[tvm] SKIP {msg}", file=sys.stderr, flush=True) + result_queue.put(("error", msg)) + return model = onnx.ModelProto() model.ParseFromString(model_bytes) @@ -68,7 +75,14 @@ def _tvm_worker(model_bytes, inputs, input_names, output_count, result_queue): module.run() outputs = [module.get_output(i).numpy() for i in range(output_count)] result_queue.put(("ok", outputs)) - except (tvm.TVMError, RuntimeError, ValueError, TypeError, OSError) as e: + except ( + tvm.TVMError, + RuntimeError, + ValueError, + TypeError, + OSError, + ImportError, + ) as e: ops = sorted({n.op_type for n in model.graph.node}) msg = f"ops={ops} error={type(e).__name__}: {e}" print(f"[tvm] SKIP {msg}", file=sys.stderr, flush=True) diff --git a/runtimes/onnx2c/stable/Dockerfile b/runtimes/onnx2c/stable/Dockerfile new file mode 100644 index 0000000000..a9034b7d25 --- /dev/null +++ b/runtimes/onnx2c/stable/Dockerfile @@ -0,0 +1,50 @@ +FROM ubuntu:24.04 + +# Disable interactive installation mode +ENV DEBIAN_FRONTEND=noninteractive +ENV PIP_BREAK_SYSTEM_PACKAGES=1 + +# Install Python, build tools, and protobuf dependencies +RUN apt-get update && apt-get install -y \ + python3 \ + python3-dev \ + python3-pip \ + cmake \ + gcc \ + g++ \ + git \ + libprotobuf-dev \ + protobuf-compiler \ + && apt-get clean autoclean && apt-get autoremove -y + +# Copy local directories +COPY ./test /root/test +COPY ./setup /root/setup +COPY ./backends/onnx2c/backend.py /root/onnx2c_backend.py + +# Install test report dependencies +RUN pip3 install --no-cache-dir -r /root/setup/requirements_report.txt + +############## ONNX Backend dependencies ########### +ENV ONNX_BACKEND="onnx2c_backend" +ENV PYTHONPATH="/root" + +# Install onnx (needed for Python backend wrapper and onnx2c build) +RUN pip3 install onnx + +# Build onnx2c from source +RUN git clone --recurse-submodules https://github.com/kraiskil/onnx2c /opt/onnx2c && \ + cd /opt/onnx2c && \ + git checkout e6308a729d067ea3be729c840558ee7832e10b6c && \ + git submodule update --init --recursive && \ + mkdir build && cd build && \ + cmake -DCMAKE_BUILD_TYPE=Release .. && \ + make -j$(nproc) && \ + cp onnx2c /usr/local/bin/onnx2c +#################################################### + +RUN useradd -m runner && chown -R runner:runner /root +USER runner +WORKDIR /root + +CMD ["sh", "-c", ". /root/setup/docker-setup.sh && pytest /root/test/test_backend.py --onnx_backend=${ONNX_BACKEND} -k 'not _cuda' -v"] diff --git a/runtimes/tvm/stable/Dockerfile b/runtimes/tvm/stable/Dockerfile index 3fe5af1658..36ea57d532 100644 --- a/runtimes/tvm/stable/Dockerfile +++ b/runtimes/tvm/stable/Dockerfile @@ -27,7 +27,8 @@ ENV PYTHONPATH="/root" # Set locale and install TVM and onnx RUN locale-gen en_US.UTF-8 && update-locale LANG=en_US.UTF-8 && \ pip3 install \ - onnx \ + "numpy<2" \ + "onnx<1.19" \ apache-tvm #################################################### diff --git a/setup/config.json b/setup/config.json index 8eca259b97..17d5eec5d5 100644 --- a/setup/config.json +++ b/setup/config.json @@ -47,6 +47,12 @@ "results_dir": "./results/tvm/stable", "core_packages": ["apache-tvm"], "dockerfile_link": "runtimes/tvm/stable/Dockerfile" + }, + "onnx2c": { + "name": "onnx2c", + "results_dir": "./results/onnx2c/stable", + "core_packages": ["onnx2c"], + "dockerfile_link": "runtimes/onnx2c/stable/Dockerfile" } }, "development": {