From d79e763d035c45f3eb43f2224dc4b81c15b3fc1d Mon Sep 17 00:00:00 2001 From: "Tybulewicz, Tomasz" Date: Wed, 1 Jul 2026 15:26:46 +0200 Subject: [PATCH 1/2] chore(tests): move accuracy_tests to functional tests as test_inference --- .github/workflows/pre_commit.yml | 68 ++------ CONTRIBUTING.md | 5 +- model_api/examples/asynchronous_api/README.md | 2 +- model_api/examples/metrics/README.md | 2 +- model_api/examples/serving_api/README.md | 2 +- model_api/examples/visual_prompting/README.md | 2 +- model_api/examples/visualization/README.md | 2 +- .../examples/zsl_visual_prompting/README.md | 2 +- model_api/tests/accuracy/__init__.py | 2 - model_api/tests/accuracy/conftest.py | 55 ------ model_api/tests/accuracy/test_YOLOv8.py | 160 ------------------ model_api/tests/functional/conftest.py | 47 ++++- .../download_models.py | 0 .../{accuracy => functional}/examples.json | 0 .../geti_2.13.0_models.json | 0 .../{accuracy => functional}/images.json | 0 .../otx_2.6.0_models.json | 0 .../public_scope.json | 0 model_api/tests/functional/serving_scope.json | 6 + .../test_inference.py} | 0 model_api/tests/precommit/__init__.py | 2 - model_api/tests/precommit/public_scope.json | 38 ----- 22 files changed, 72 insertions(+), 323 deletions(-) delete mode 100644 model_api/tests/accuracy/__init__.py delete mode 100644 model_api/tests/accuracy/conftest.py delete mode 100644 model_api/tests/accuracy/test_YOLOv8.py rename model_api/tests/{accuracy => functional}/download_models.py (100%) rename model_api/tests/{accuracy => functional}/examples.json (100%) rename model_api/tests/{accuracy => functional}/geti_2.13.0_models.json (100%) rename model_api/tests/{accuracy => functional}/images.json (100%) rename model_api/tests/{accuracy => functional}/otx_2.6.0_models.json (100%) rename model_api/tests/{accuracy => functional}/public_scope.json (100%) create mode 100644 model_api/tests/functional/serving_scope.json rename model_api/tests/{accuracy/test_accuracy.py => functional/test_inference.py} (100%) delete mode 100644 model_api/tests/precommit/__init__.py delete mode 100644 model_api/tests/precommit/public_scope.json diff --git a/.github/workflows/pre_commit.yml b/.github/workflows/pre_commit.yml index 9f3f14426..dce919499 100644 --- a/.github/workflows/pre_commit.yml +++ b/.github/workflows/pre_commit.yml @@ -57,7 +57,7 @@ jobs: - name: Run pre-commit checks run: uvx pre-commit run --all-files - accuracy-tests: + unit-functional-tests: strategy: fail-fast: false matrix: @@ -85,10 +85,15 @@ jobs: download.pytorch.org:443 files.pythonhosted.org:443 github.com:443 + incoming.telemetry.mozilla.org:443 + ocsp.digicert.com:443 + ocsp.sectigo.com:443 pypi.org:443 raw.githubusercontent.com:443 releases.astral.sh:443 + settings-win.data.microsoft.com:443 storage.geti.intel.com:443 + telemetry-incoming.r53-2.services.mozilla.com:443 - *checkout @@ -100,68 +105,18 @@ jobs: python-version: ${{ matrix.python-version }} version: "0.11.13" - - &install-dependencies - name: Install dependencies + - name: Install dependencies run: uv --directory model_api sync --locked --group tests --extra-index-url https://download.pytorch.org/whl/cpu - - name: Prepare test data - run: uv --directory model_api run python tests/accuracy/download_models.py -d data -j tests/accuracy/public_scope.json -l - - - name: Run Python Test - run: uv --directory model_api run pytest --data=./data tests/accuracy/test_accuracy.py - - unit-functional-tests: - strategy: - fail-fast: false - matrix: - os: - - "ubuntu-24.04" - - "windows-2022" - python-version: - - "3.11" - - "3.12" - - "3.13" - - "3.14" - name: unit & functional tests (${{ matrix.os }}, Python ${{ matrix.python-version }}) - runs-on: ${{ matrix.os }} - steps: - - name: Harden the runner - uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 - with: - disable-sudo: true - disable-telemetry: true - egress-policy: block - allowed-endpoints: > - dc.services.visualstudio.com:443 - download.pytorch.org:443 - files.pythonhosted.org:443 - github.com:443 - incoming.telemetry.mozilla.org:443 - ocsp.digicert.com:443 - ocsp.sectigo.com:443 - pypi.org:443 - raw.githubusercontent.com:443 - releases.astral.sh:443 - settings-win.data.microsoft.com:443 - storage.geti.intel.com:443 - telemetry-incoming.r53-2.services.mozilla.com:443 - - - *checkout - - - *matrix-setup-uv - - - *install-dependencies - - name: Run model_api unit tests run: uv --directory model_api run pytest tests/unit --cov - name: Run model_converter unit tests run: uv --directory model_converter run --group tests pytest tests/unit --cov - - &prepare-test-data - name: Prepare test data + - name: Prepare test data run: | - uv --directory model_api run python tests/accuracy/download_models.py -d data -j tests/precommit/public_scope.json -l + uv --directory model_api run python tests/functional/download_models.py -d data -j tests/functional/public_scope.json -l - name: Run test run: | @@ -213,7 +168,9 @@ jobs: - name: Install dependencies run: uv --directory model_api sync --locked --group tests --extra ovms --extra-index-url https://download.pytorch.org/whl/cpu - - *prepare-test-data + - name: Prepare test data + run: | + uv --directory model_api run python tests/functional/download_models.py -d data -j tests/functional/serving_scope.json -l - name: serving_api run: | @@ -226,7 +183,6 @@ jobs: pre-commit-result: runs-on: ubuntu-latest needs: - - accuracy-tests - code_quality_checks - unit-functional-tests - serving_api-tests diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7a5319394..5f929bba1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -70,10 +70,9 @@ Install only the subproject(s) you are working on. Each step is independent. ```bash uvx pre-commit run --all-files uv --directory model_api run pytest tests/unit - uv --directory model_api run python tests/accuracy/download_models.py -d data -j tests/precommit/public_scope.json -l + uv --directory model_api run python tests/functional/download_models.py -d data -j tests/precommit/public_scope.json -l + uv --directory model_api run python tests/functional/download_models.py -d data -j tests/functional/public_scope.json -l uv --directory model_api run pytest --data=./data tests/functional - uv --directory model_api run python tests/accuracy/download_models.py -d data -j tests/accuracy/public_scope.json -l - uv --directory model_api run pytest --data=./data tests/accuracy/test_accuracy.py uv --directory model_converter run model-converter examples/config.json --list ``` diff --git a/model_api/examples/asynchronous_api/README.md b/model_api/examples/asynchronous_api/README.md index 445769ce5..b5970c49f 100644 --- a/model_api/examples/asynchronous_api/README.md +++ b/model_api/examples/asynchronous_api/README.md @@ -27,5 +27,5 @@ uv run python run.py In the root directory of the repository: -- download sample models and images by running `uv run python tests/accuracy/download_models.py -d data -j tests/accuracy/examples.json -l` +- download sample models and images by running `uv run python tests/functional/download_models.py -d data -j tests/functional/examples.json -l` - run the example with the following command: `uv run python examples/asynchronous_api/run.py data/otx_models/ssd-card-detection.xml data/cards.png` diff --git a/model_api/examples/metrics/README.md b/model_api/examples/metrics/README.md index 355a313f4..969c21d44 100644 --- a/model_api/examples/metrics/README.md +++ b/model_api/examples/metrics/README.md @@ -54,7 +54,7 @@ uv run python benchmark.py --help In the root directory of the repository: -- download sample models and images by running `uv run python tests/accuracy/download_models.py -d data -j tests/accuracy/examples.json -l` +- download sample models and images by running `uv run python tests/functional/download_models.py -d data -j tests/functional/examples.json -l` - run the example with the following command: `uv run python examples/metrics/benchmark.py data/otx_models/ssd-card-detection.xml data/coco128/images/train2017` ## Expected Output diff --git a/model_api/examples/serving_api/README.md b/model_api/examples/serving_api/README.md index b4b77b577..1b49324a4 100644 --- a/model_api/examples/serving_api/README.md +++ b/model_api/examples/serving_api/README.md @@ -17,7 +17,7 @@ This example demonstrates how to use a Python API of OpenVINO Model API for a re - Install Docker. Please refer to the [official documentation](https://docs.docker.com/get-docker/) for details. -- Download sample models and images by running `uv run python tests/accuracy/download_models.py -d data -j tests/accuracy/examples.json -l` and resave a configured model at OVMS friendly folder layout: +- Download sample models and images by running `uv run python tests/functional/download_models.py -d data -j tests/functional/examples.json -l` and resave a configured model at OVMS friendly folder layout: ```bash mkdir -p data/ovms/ssd-card-detection/1 diff --git a/model_api/examples/visual_prompting/README.md b/model_api/examples/visual_prompting/README.md index e6d627446..14afa02aa 100644 --- a/model_api/examples/visual_prompting/README.md +++ b/model_api/examples/visual_prompting/README.md @@ -28,7 +28,7 @@ where prompts are in X Y format. To run the pipeline out-of-the box you can download the test data by running the following command from the repo root: ```bash -uv run python tests/accuracy/download_models.py -d data -j tests/accuracy/examples.json -l +uv run python tests/functional/download_models.py -d data -j tests/functional/examples.json -l ``` and then run diff --git a/model_api/examples/visualization/README.md b/model_api/examples/visualization/README.md index c3438de6e..565dc4861 100644 --- a/model_api/examples/visualization/README.md +++ b/model_api/examples/visualization/README.md @@ -21,7 +21,7 @@ uv run python run.py --image --model .xml --outpu To run the pipeline out-of-the box you can download the test data by running the following command from the repo root: ```bash -uv run python tests/accuracy/download_models.py -d data -j tests/accuracy/examples.json -l +uv run python tests/functional/download_models.py -d data -j tests/functional/examples.json -l ``` and then run diff --git a/model_api/examples/zsl_visual_prompting/README.md b/model_api/examples/zsl_visual_prompting/README.md index 1e97e7138..727a144de 100644 --- a/model_api/examples/zsl_visual_prompting/README.md +++ b/model_api/examples/zsl_visual_prompting/README.md @@ -31,7 +31,7 @@ Higher thresholds leads to lower mask recall in the final prediction, i.e. low c To run the pipeline out-of-the box you can download the test data by running the following command from the repo root: ```bash -uv run python tests/accuracy/download_models.py -d data -j tests/accuracy/examples.json -l +uv run python tests/functional/download_models.py -d data -j tests/functional/examples.json -l ``` and then run diff --git a/model_api/tests/accuracy/__init__.py b/model_api/tests/accuracy/__init__.py deleted file mode 100644 index 916f3a44b..000000000 --- a/model_api/tests/accuracy/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright (C) 2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 diff --git a/model_api/tests/accuracy/conftest.py b/model_api/tests/accuracy/conftest.py deleted file mode 100644 index 9b7b45711..000000000 --- a/model_api/tests/accuracy/conftest.py +++ /dev/null @@ -1,55 +0,0 @@ -# -# Copyright (C) 2020-2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# -import json -from pathlib import Path - -import pytest - - -def pytest_addoption(parser): - parser.addoption("--data", action="store", help="data folder with dataset") - parser.addoption( - "--model_data", - action="store", - default="public_scope.json", - help="path to model data JSON file for test parameterization", - ) - parser.addoption( - "--device", - action="store", - default="CPU", - help="device to run tests on (in case of OpenvinoAdapter)", - ) - parser.addoption( - "--dump", - action="store_true", - default=False, - help="whether to dump results into json file", - ) - parser.addoption( - "--results-dir", - action="store", - default="", - help="directory to store inference result", - ) - - -def pytest_configure(config): - config.test_results = [] - - -@pytest.hookimpl(tryfirst=True, hookwrapper=True) -def pytest_runtest_makereport(item, call): - outcome = yield - result = outcome.get_result() - - if result.when == "call": - test_results = item.config.test_results - - if not test_results: - return - - with Path("test_scope.json").open("w") as outfile: - json.dump(test_results, outfile, indent=4) diff --git a/model_api/tests/accuracy/test_YOLOv8.py b/model_api/tests/accuracy/test_YOLOv8.py deleted file mode 100644 index 2a689f207..000000000 --- a/model_api/tests/accuracy/test_YOLOv8.py +++ /dev/null @@ -1,160 +0,0 @@ -# -# Copyright (C) 2020-2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# -import functools -import os -import types -from pathlib import Path - -import cv2 -import numpy as np -import openvino as ov -import pytest -import torch -import ultralytics -from model_api.models import YOLOv5 -from ultralytics.data import utils -from ultralytics.models import yolo - - -def _init_predictor(yolo): - yolo.predict(np.empty([1, 1, 3], np.uint8)) - - -@functools.lru_cache(maxsize=1) -def _cached_alignment(pt): - export_dir = Path( - ultralytics.YOLO( - Path(os.environ["DATA"]) / "ultralytics" / pt, - "detect", - ).export(format="openvino", half=True), - ) - impl_wrapper = YOLOv5.create_model(export_dir / (pt.stem + ".xml"), device="CPU") - ref_wrapper = ultralytics.YOLO(export_dir, "detect") - ref_wrapper.overrides["imgsz"] = (impl_wrapper.w, impl_wrapper.h) - _init_predictor(ref_wrapper) - ref_wrapper.predictor.model.ov_compiled_model = ov.Core().compile_model( - ref_wrapper.predictor.model.ov_model, - "CPU", - ) - ref_dir = export_dir / "ref" - ref_dir.mkdir(exist_ok=True) - return impl_wrapper, ref_wrapper, ref_dir - - -def _impaths(): - """ - It's impossible to pass fixture as argument for - @pytest.mark.parametrize, so it can't take a cmd arg. Use env var - instead. Another solution was to define - pytest_generate_tests(metafunc) in conftest.py - """ - impaths = sorted( - file - for file in (Path(os.environ["DATA"]) / "coco128/images/train2017/").iterdir() - if file.name - not in { # This images fail because image preprocessing is imbedded into the model - "000000000143.jpg", - "000000000491.jpg", - "000000000536.jpg", - "000000000581.jpg", - } - ) - if not impaths: - error_message = f"{Path(os.environ['DATA']) / 'coco128/images/train2017/'} is empty" - raise RuntimeError(error_message) - return impaths - - -@pytest.mark.parametrize("impath", _impaths()) -@pytest.mark.parametrize("pt", [Path("yolov5mu.pt"), Path("yolov8l.pt")]) -def test_alignment(impath, pt): - impl_wrapper, ref_wrapper, ref_dir = _cached_alignment(pt) - im = cv2.imread(str(impath)) - assert im is not None - impl_preds = impl_wrapper(im) - pred_boxes = impl_preds.bboxes.astype(np.float32) - pred_scores = impl_preds.scores.astype(np.float32) - pred_labels = impl_preds.labels - ref_predictions = ref_wrapper.predict(im) - assert len(ref_predictions) == 1 - ref_boxes = ref_predictions[0].boxes.data.numpy() - if 0 == len(pred_boxes) == len(ref_boxes): - return # np.isclose() doesn't work for empty arrays - ref_boxes[:, :4] = np.round(ref_boxes[:, :4], out=ref_boxes[:, :4]) - # Allow one pixel deviation because image preprocessing is imbedded into the model - assert np.isclose(pred_boxes, ref_boxes[:, :4], 0, 1).all() - assert np.isclose(pred_scores, ref_boxes[:, 4], 0.0, 0.02).all() - assert (pred_labels == ref_boxes[:, 5]).all() - with Path.open(ref_dir / impath.with_suffix(".txt").name, "w") as file: - print(impl_preds, end="", file=file) - - -class Metrics(yolo.detect.DetectionValidator): - @torch.inference_mode() - def evaluate(self, wrapper): - self.data = utils.check_det_dataset("coco128.yaml") - dataloader = self.get_dataloader(self.data[self.args.split], batch_size=1) - dataloader.dataset.transforms.transforms = ( - lambda di: { - "batch_idx": torch.zeros(len(di["instances"])), - "bboxes": torch.from_numpy(di["instances"].bboxes), - "cls": torch.from_numpy(di["cls"]), - "img": torch.empty(1, 1, 1), - "im_file": di["im_file"], - "ori_shape": di["ori_shape"], - "ratio_pad": [(1.0, 1.0), (0, 0)], - }, - ) - self.init_metrics( - types.SimpleNamespace( - names=dict(enumerate(wrapper.labels)), - ), - ) - for batch in dataloader: - im = cv2.imread(batch["im_file"][0]) - result = wrapper(im) - bboxes = torch.from_numpy(result.bboxes) / torch.tile( - torch.tensor([im.shape[1], im.shape[0]], dtype=torch.float32), - (1, 2), - ) - scores = torch.from_numpy(result.scores) - labels = torch.from_numpy(result.labels) - - pred = torch.cat( - [bboxes, scores[:, None], labels[:, None].float()], - dim=1, - ).unsqueeze(0) - if not pred.numel(): - pred = pred.view(1, 0, 6) - self.update_metrics(pred, batch) - return self.get_stats() - - -@pytest.mark.parametrize( - ("pt", "ref_mAP50_95"), - [ - ( - Path("yolov8n.pt"), - 0.439413760605130543357432770790182985365390777587890625, - ), - ( - Path("yolov5n6u.pt"), - 0.48720141594764942993833756190724670886993408203125, - ), - ], -) -def test_metric(pt, ref_mAP50_95): - mAP50_95 = Metrics().evaluate( - YOLOv5.create_model( - ultralytics.YOLO( - Path(os.environ["DATA"]) / "ultralytics" / pt, - "detect", - ).export(format="openvino", half=True) - / pt.with_suffix(".xml"), - device="CPU", - configuration={"confidence_threshold": 0.001}, - ), - )["metrics/mAP50-95(B)"] - assert abs(mAP50_95 - ref_mAP50_95) <= 0.01 * ref_mAP50_95 or mAP50_95 >= ref_mAP50_95 diff --git a/model_api/tests/functional/conftest.py b/model_api/tests/functional/conftest.py index 2ef51d3da..b1eadd931 100644 --- a/model_api/tests/functional/conftest.py +++ b/model_api/tests/functional/conftest.py @@ -1,13 +1,58 @@ # -# Copyright (C) 2024 Intel Corporation +# Copyright (C) 2020-2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 # +import json +from pathlib import Path import pytest def pytest_addoption(parser): parser.addoption("--data", action="store", help="data folder with dataset") + parser.addoption( + "--model_data", + action="store", + default="public_scope.json", + help="path to model data JSON file for test parameterization", + ) + parser.addoption( + "--device", + action="store", + default="CPU", + help="device to run tests on (in case of OpenvinoAdapter)", + ) + parser.addoption( + "--dump", + action="store_true", + default=False, + help="whether to dump results into json file", + ) + parser.addoption( + "--results-dir", + action="store", + default="", + help="directory to store inference result", + ) + + +def pytest_configure(config): + config.test_results = [] + + +@pytest.hookimpl(tryfirst=True, hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + result = outcome.get_result() + + if result.when == "call": + test_results = item.config.test_results + + if not test_results: + return + + with Path("test_scope.json").open("w") as outfile: + json.dump(test_results, outfile, indent=4) @pytest.fixture(scope="session") diff --git a/model_api/tests/accuracy/download_models.py b/model_api/tests/functional/download_models.py similarity index 100% rename from model_api/tests/accuracy/download_models.py rename to model_api/tests/functional/download_models.py diff --git a/model_api/tests/accuracy/examples.json b/model_api/tests/functional/examples.json similarity index 100% rename from model_api/tests/accuracy/examples.json rename to model_api/tests/functional/examples.json diff --git a/model_api/tests/accuracy/geti_2.13.0_models.json b/model_api/tests/functional/geti_2.13.0_models.json similarity index 100% rename from model_api/tests/accuracy/geti_2.13.0_models.json rename to model_api/tests/functional/geti_2.13.0_models.json diff --git a/model_api/tests/accuracy/images.json b/model_api/tests/functional/images.json similarity index 100% rename from model_api/tests/accuracy/images.json rename to model_api/tests/functional/images.json diff --git a/model_api/tests/accuracy/otx_2.6.0_models.json b/model_api/tests/functional/otx_2.6.0_models.json similarity index 100% rename from model_api/tests/accuracy/otx_2.6.0_models.json rename to model_api/tests/functional/otx_2.6.0_models.json diff --git a/model_api/tests/accuracy/public_scope.json b/model_api/tests/functional/public_scope.json similarity index 100% rename from model_api/tests/accuracy/public_scope.json rename to model_api/tests/functional/public_scope.json diff --git a/model_api/tests/functional/serving_scope.json b/model_api/tests/functional/serving_scope.json new file mode 100644 index 000000000..1c6279d34 --- /dev/null +++ b/model_api/tests/functional/serving_scope.json @@ -0,0 +1,6 @@ +[ + { + "name": "otx_models/ssd-card-detection.xml", + "type": "DetectionModel" + } +] diff --git a/model_api/tests/accuracy/test_accuracy.py b/model_api/tests/functional/test_inference.py similarity index 100% rename from model_api/tests/accuracy/test_accuracy.py rename to model_api/tests/functional/test_inference.py diff --git a/model_api/tests/precommit/__init__.py b/model_api/tests/precommit/__init__.py deleted file mode 100644 index 916f3a44b..000000000 --- a/model_api/tests/precommit/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright (C) 2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 diff --git a/model_api/tests/precommit/public_scope.json b/model_api/tests/precommit/public_scope.json deleted file mode 100644 index 001203aa3..000000000 --- a/model_api/tests/precommit/public_scope.json +++ /dev/null @@ -1,38 +0,0 @@ -[ - { - "name": "otx_models/detection_model_with_xai_head.xml", - "type": "DetectionModel" - }, - { - "name": "otx_models/mlc_mobilenetv3_large_voc.xml", - "type": "ClassificationModel" - }, - { - "name": "otx_models/Lite-hrnet-18_mod2.xml", - "type": "SegmentationModel" - }, - { - "name": "otx_models/cls_mobilenetv3_large_cars.onnx", - "type": "ClassificationModel" - }, - { - "name": "otx_models/tinynet_imagenet.xml", - "type": "ClassificationModel" - }, - { - "name": "anomalib_models/padim.xml", - "type": "AnomalyDetection" - }, - { - "name": "anomalib_models/stfpm.xml", - "type": "AnomalyDetection" - }, - { - "name": "anomalib_models/uflow.xml", - "type": "AnomalyDetection" - }, - { - "name": "otx_models/ssd-card-detection.xml", - "type": "DetectionModel" - } -] From 4d5420a0c4ac7a3f4b348c4362ffddd88a144224 Mon Sep 17 00:00:00 2001 From: "Tybulewicz, Tomasz" Date: Thu, 2 Jul 2026 09:51:07 +0200 Subject: [PATCH 2/2] Remove not needed line --- CONTRIBUTING.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5f929bba1..fe3ae3ef2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -70,7 +70,6 @@ Install only the subproject(s) you are working on. Each step is independent. ```bash uvx pre-commit run --all-files uv --directory model_api run pytest tests/unit - uv --directory model_api run python tests/functional/download_models.py -d data -j tests/precommit/public_scope.json -l uv --directory model_api run python tests/functional/download_models.py -d data -j tests/functional/public_scope.json -l uv --directory model_api run pytest --data=./data tests/functional uv --directory model_converter run model-converter examples/config.json --list