diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 419131c..62b0feb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,15 +8,15 @@ on: jobs: test: - name: Unit tests (Python 3.9, pinned deps) + name: Unit tests (Python 3.12, pinned deps) runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Set up Python 3.9 + - name: Set up Python 3.12 uses: actions/setup-python@v5 with: - python-version: "3.9" + python-version: "3.12" - name: Install test dependencies run: pip install -r requirements-test.txt diff --git a/.gitignore b/.gitignore index 3cade88..453cb1c 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,9 @@ config-*.json # Pipeline notify config holds camera creds + chat id (keep only the example) pipeline/dogwatch-notify.config.json +# SSH credentials + Tailscale IP for the homelab server +.ssh + # Generated event clips clips/ clips-rear-east/ diff --git a/Dockerfile b/Dockerfile index 59f4595..c894e32 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,19 @@ # dogwatch — Coral Edge TPU dog detector -# Uses Python 3.9 because pycoral wheels only support up to cp39 - -FROM python:3.9-slim-bookworm +# +# Uses ai-edge-litert (LiteRT, the successor to tflite_runtime) instead of +# pycoral/tflite_runtime. pycoral is abandoned upstream and only ever shipped +# cp39 wheels, which pinned this whole image to Python 3.9 (EOL 2025-10-31) +# and, downstream of that, to numpy 1.x (pycoral's compiled bindings were +# built against the numpy 1.x C ABI). ai-edge-litert ships wheels through +# Python 3.14 and has no numpy ceiling, which is what unblocks the numpy/ +# opencv bumps below. See README "Known limitations" and GitHub issue #1 for +# the full history of the Python 3.9 pin this replaces. +FROM python:3.12-slim-bookworm -# libedgetpu runtime (std = standard clock speed, good thermals) +# libedgetpu runtime (std = standard clock speed, good thermals). +# feranick's fork is the community-maintained continuation of Google's +# abandoned libedgetpu — this build (16.0TF2.19.1-1) is built against +# TensorFlow 2.19.1 and lists ai-edge-litert as its recommended pairing. ADD https://github.com/feranick/libedgetpu/releases/download/16.0TF2.19.1-1/libedgetpu1-std_16.0tf2.19.1-1.bookworm_amd64.deb \ /tmp/libedgetpu.deb @@ -18,34 +28,17 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && dpkg -i /tmp/libedgetpu.deb \ && rm /tmp/libedgetpu.deb -# pycoral + tflite-runtime from Google Coral (cp39 linux x86_64) -# -# Dependency currency notes (see README "Known limitations" for the full -# writeup): pycoral/tflite_runtime are abandoned upstream and only ever -# shipped cp39 wheels, which is what pins the whole image to Python 3.9 -# (EOL 2025-10-31). Everything below this line is bumped to the latest -# version that still supports cp39, and should be re-checked whenever the -# Python/pycoral constraint is revisited. -# -# numpy is pinned to 1.26.4 (the last 1.x release) rather than bumping to -# numpy 2.x: pycoral's compiled bindings (_pywrap_coral) are built against -# the numpy 1.x C ABI and break under numpy 2.x at runtime (confirmed on -# real Coral TPU hardware — pip's resolver alone won't catch this, since it -# only checks declared version ranges, not compiled ABI compatibility). -# This forces opencv-python-headless back to 4.9.0.80, the newest release -# in the numpy<2 line: 4.10.0/4.11.0 carry CVE-2025-53644 (heap buffer -# write via crafted JPEG) and 4.12.0+ requires numpy>=2, so 4.9.0.80 is the -# newest version that is both numpy-1.x-compatible and outside the CVE's -# affected range (4.10.0-4.11.0 only). Re-check this whole chain if the -# pycoral/numpy-2.x ABI break is ever resolved upstream. +# ai-edge-litert + current dependency versions (cp312 wheels, no numpy +# ceiling). numpy/opencv/requests/paho-mqtt/shapely are all on their latest +# stable releases as of this writing — re-check periodically, but there is +# no known structural constraint pinning any of them anymore. RUN pip install --no-cache-dir \ - "https://github.com/google-coral/pycoral/releases/download/v2.0.0/tflite_runtime-2.5.0.post1-cp39-cp39-linux_x86_64.whl" \ - "https://github.com/google-coral/pycoral/releases/download/v2.0.0/pycoral-2.0.0-cp39-cp39-linux_x86_64.whl" \ + ai-edge-litert==2.1.6 \ paho-mqtt==2.1.0 \ - numpy==1.26.4 \ - opencv-python-headless==4.9.0.80 \ - shapely==2.0.6 \ - requests==2.32.4 + numpy==2.5.1 \ + opencv-python-headless==5.0.0.93 \ + shapely==2.1.2 \ + requests==2.34.2 COPY *.py /app/ WORKDIR /app diff --git a/README.md b/README.md index c4e7191..d653daa 100644 --- a/README.md +++ b/README.md @@ -162,32 +162,29 @@ installs cleanly — not that inference actually works). ## Known limitations -- **Python 3.9 pin (structural, not easily fixable — tracked in - [#1](https://github.com/VIDGuide/dogwatch/issues/1)).** The whole detection - container is pinned to Python 3.9 because `pycoral`/`tflite_runtime` are - abandoned upstream and only ever shipped `cp39` wheels. Python 3.9 reached - end-of-life on 2025-10-31, so this image runs on an unsupported CPython - version by necessity, not choice. All dependency versions in the - `Dockerfile` and `pipeline/requirements.txt` are bumped to the newest - release that still ships a `cp39` wheel, and are re-checked whenever a CVE - or dependency bump is made — but the underlying Python version itself can't - be moved forward without replacing the Coral inference stack (e.g. ONNX - Runtime + a different TPU/accelerator path). -- **Pillow capped at 11.3.0** for the same reason (last `cp39` release; - 12.x dropped Python 3.9). Known CVEs fixed in 12.1/12.2 are all in - PSD/PDF/DDS/FITS parsing; `dogwatch-notify.py` only opens JPEGs it captures - itself, so exposure is low but not zero if that assumption ever changes. -- **numpy capped at 1.26.4 (the last 1.x release), opencv-python-headless - capped at 4.9.0.80.** pycoral's compiled bindings are built against the - numpy 1.x C ABI and break at runtime under numpy 2.x — confirmed on real - Coral TPU hardware, not just a version-range conflict pip would catch. - This forces opencv-python-headless back from the 4.12.x line (which - requires numpy>=2) to 4.9.0.80, the newest release still on numpy<2. - 4.9.0.80 predates CVE-2025-53644 (a heap buffer write via crafted JPEG, - which only affects 4.10.0 and 4.11.0), so this isn't a regression to a - known-vulnerable version — it's a deliberate skip over the two versions - that were actually affected. Re-check this pin if pycoral ever ships a - numpy-2.x-compatible build upstream. +- **Coral Edge TPU support is community-maintained, not official.** Google + has effectively abandoned the Coral software stack — `pycoral` and + `tflite_runtime` saw no meaningful releases in years and only ever shipped + `cp39` wheels (this project's Python 3.9 pin, and the numpy 1.x / + opencv-python-headless 4.9.x pins it forced, were resolved by migrating + off pycoral — see [#1](https://github.com/VIDGuide/dogwatch/issues/1) for + that history). The detector now uses + [`ai-edge-litert`](https://pypi.org/project/ai-edge-litert/) (Google's + actively maintained LiteRT runtime, wheels through Python 3.14) paired + with [`feranick/libedgetpu`](https://github.com/feranick/libedgetpu), a + community fork that keeps the native Edge TPU driver building against + current TensorFlow releases. This removed the structural numpy/opencv + version ceiling — the `Dockerfile` now tracks each dependency's latest + stable release with no known constraint forcing them behind. If + `feranick/libedgetpu` ever goes unmaintained too, the next fallback is + building `libedgetpu` from source (see their README) or moving off the + Coral TPU entirely. +- `detector.py` no longer depends on `pycoral` at all — it talks to + `ai_edge_litert.interpreter` directly (`Interpreter` + `load_delegate`), + reimplementing the small, pure-Python pieces pycoral used to wrap (input + tensor resizing/padding, output tensor parsing for SSD-style detection + models). No compiled bindings are involved on the Python side anymore; + the only native component is `libedgetpu.so` itself. ### Snapshot quality / grey-frame handling diff --git a/detector.py b/detector.py index bfa03bd..f70ccc5 100644 --- a/detector.py +++ b/detector.py @@ -1,13 +1,128 @@ -"""detector.py — thin wrapper around pycoral, filtered to the 'dog' class.""" +"""detector.py — thin wrapper around ai-edge-litert + the Edge TPU delegate, +filtered to the 'dog' class. + +Previously used pycoral (Google's convenience wrapper around tflite_runtime), +but pycoral is abandoned upstream and only ever shipped cp39 wheels, which is +what pinned this whole project to Python 3.9 and, downstream of that, to +numpy 1.x (see README "Known limitations" / GitHub issue #1 for the full +history). pycoral's actual surface area used here was small — a delegate +loader and two adapter functions, both plain Python with no C bindings — so +this reimplements them directly against `ai_edge_litert`, which: + * ships wheels for Python 3.9 through 3.14 (no more cp39 ceiling) + * exposes the same `Interpreter`/`load_delegate` API tflite_runtime did, + so this is a like-for-like swap, not a rewrite of the detection logic + * has no numpy version ceiling, which is what forced numpy/opencv's older + pins in the Dockerfile + +The output-tensor parsing in `_get_objects` mirrors pycoral's +`adapters.detect.get_objects` exactly (including the newer "signature" path +for models that expose one), so behavior is unchanged regardless of which +SSD-style detection model is loaded. +""" +import platform + import cv2 -from pycoral.utils.edgetpu import make_interpreter -from pycoral.adapters import common, detect +from ai_edge_litert.interpreter import Interpreter, load_delegate + +_EDGETPU_SHARED_LIB = { + "Linux": "libedgetpu.so.1", + "Darwin": "libedgetpu.1.dylib", + "Windows": "edgetpu.dll", +}[platform.system()] + + +def _make_interpreter(model_path): + """Load *model_path* with the Edge TPU delegate attached.""" + delegate = load_delegate(_EDGETPU_SHARED_LIB) + return Interpreter(model_path=model_path, experimental_delegates=[delegate]) + + +def _input_size(interpreter): + _, height, width, _ = interpreter.get_input_details()[0]["shape"] + return width, height + + +def _input_tensor(interpreter): + index = interpreter.get_input_details()[0]["index"] + return interpreter.tensor(index)()[0] + + +def _set_resized_input(interpreter, size, resize): + """Copy a resized, zero-padded image into the model's input tensor. + + Mirrors pycoral's adapters.common.set_resized_input: preserves aspect + ratio by scaling to fit, then pads the rest with zeros, so callers don't + need to worry about non-square input tensors. + """ + width, height = _input_size(interpreter) + w, h = size + scale = min(width / w, height / h) + w, h = int(w * scale), int(h * scale) + tensor = _input_tensor(interpreter) + tensor.fill(0) + _, _, channel = tensor.shape + result = resize((w, h)) + tensor[:h, :w] = result.reshape((h, w, channel)) + return (scale, scale) + + +def _output_tensor(interpreter, i): + index = interpreter.get_output_details()[i]["index"] + return interpreter.tensor(index)() + + +def _get_objects(interpreter, score_threshold, image_scale): + """Return [{'id', 'score', 'bbox': (xmin,ymin,xmax,ymax)}, ...]. + + Output tensor layout for TFLite_Detection_PostProcess-based SSD models + varies by export; this checks for a model signature first (newer + exports), then falls back to the same tensor-order heuristics pycoral + used for older exports like the stock + ssd_mobilenet_v2_coco_quant_postprocess_edgetpu.tflite. + """ + signature_list = interpreter._get_full_signature_list() # noqa: SLF001 + if signature_list: + if len(signature_list) > 1: + raise ValueError("Only support model with one signature.") + signature = signature_list[next(iter(signature_list))] + count = int(interpreter.tensor(signature["outputs"]["output_0"])()[0]) + scores = interpreter.tensor(signature["outputs"]["output_1"])()[0] + class_ids = interpreter.tensor(signature["outputs"]["output_2"])()[0] + boxes = interpreter.tensor(signature["outputs"]["output_3"])()[0] + elif _output_tensor(interpreter, 3).size == 1: + boxes = _output_tensor(interpreter, 0)[0] + class_ids = _output_tensor(interpreter, 1)[0] + scores = _output_tensor(interpreter, 2)[0] + count = int(_output_tensor(interpreter, 3)[0]) + else: + scores = _output_tensor(interpreter, 0)[0] + boxes = _output_tensor(interpreter, 1)[0] + count = int(_output_tensor(interpreter, 2)[0]) + class_ids = _output_tensor(interpreter, 3)[0] + + width, height = _input_size(interpreter) + scale_x, scale_y = image_scale + sx, sy = width / scale_x, height / scale_y + + out = [] + for i in range(count): + if scores[i] < score_threshold: + continue + ymin, xmin, ymax, xmax = boxes[i] + out.append({ + "id": int(class_ids[i]), + "score": float(scores[i]), + "bbox": ( + int(xmin * sx), int(ymin * sy), int(xmax * sx), int(ymax * sy) + ), + }) + return out class DogDetector: def __init__(self, model_path, labels_path, score_threshold=0.4, target_label="dog"): - self.interp = make_interpreter(model_path) + self.interp = _make_interpreter(model_path) self.interp.allocate_tensors() self.score_threshold = score_threshold self.labels = self._load_labels(labels_path) @@ -30,13 +145,11 @@ def _load_labels(path): def detect(self, frame): """Return [{'bbox': (x0,y0,x1,y1) in pixels, 'score': float}, ...].""" h, w = frame.shape[:2] - _, scale = common.set_resized_input( + scale = _set_resized_input( self.interp, (w, h), lambda size: cv2.resize(frame, size)) self.interp.invoke() out = [] - for obj in detect.get_objects(self.interp, self.score_threshold, scale): - if obj.id in self.target_ids: - b = obj.bbox - out.append({"bbox": (b.xmin, b.ymin, b.xmax, b.ymax), - "score": obj.score}) + for obj in _get_objects(self.interp, self.score_threshold, scale): + if obj["id"] in self.target_ids: + out.append({"bbox": obj["bbox"], "score": obj["score"]}) return out diff --git a/requirements-test.txt b/requirements-test.txt index 0153438..b4a2a32 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,9 +1,12 @@ -# Test-only dependencies for tracker.py / behavior.py unit tests. -# Runtime deps (numpy/opencv/shapely) are pinned in the Dockerfile — see the -# comment there for why they're capped below their latest versions -# (pycoral's numpy 1.x ABI + CVE-2025-53644 avoidance). Mirrored here so CI -# exercises the same versions that actually ship in the image. +# Test-only dependencies for tracker.py / behavior.py / snapshot_quality.py +# unit tests. Runtime deps (numpy/opencv/shapely) are pinned in the +# Dockerfile — mirrored here so CI exercises the same versions that +# actually ship in the image. No structural version ceiling anymore since +# the ai-edge-litert migration (see Dockerfile comment) — these track +# current stable releases and should just be bumped along with the +# Dockerfile's pins. pytest==8.4.2 -numpy==1.26.4 -opencv-python-headless==4.9.0.80 -shapely==2.0.6 +numpy==2.5.1 +opencv-python-headless==5.0.0.93 +shapely==2.1.2 +ai-edge-litert==2.1.6 diff --git a/tests/test_detector.py b/tests/test_detector.py new file mode 100644 index 0000000..6504e5e --- /dev/null +++ b/tests/test_detector.py @@ -0,0 +1,186 @@ +"""Unit tests for detector.py's tensor-parsing logic (_get_objects, +_set_resized_input), using fake interpreter objects rather than a real +model/TPU — the Edge TPU delegate and real inference are only verifiable on +actual Coral hardware, but the pure-Python tensor bookkeeping this module +reimplements from pycoral can be tested in isolation. +""" +import os +import sys + +import numpy as np + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import detector + + +class FakeInterpreter: + """Minimal stand-in for ai_edge_litert.interpreter.Interpreter. + + Supports just enough of the API surface detector.py touches: + get_input_details / get_output_details / tensor(index) / _get_full_signature_list. + """ + + def __init__(self, input_shape, output_tensors): + """output_tensors: list of numpy arrays, in the same order detector.py + expects to find them via get_output_details()[i]['index'].""" + self._input = np.zeros(input_shape, dtype=np.uint8) + self._outputs = output_tensors + self._input_shape = input_shape + + def get_input_details(self): + return [{"index": "input", "shape": self._input_shape}] + + def get_output_details(self): + return [{"index": i} for i in range(len(self._outputs))] + + def tensor(self, index): + if index == "input": + return lambda: self._input + return lambda: self._outputs[index] + + def _get_full_signature_list(self): + return {} # no signature -> exercise the legacy tensor-order path + + +class TestSetResizedInput: + def test_scales_and_pads_to_fit_input_tensor(self): + interp = FakeInterpreter(input_shape=(1, 300, 300, 3), output_tensors=[]) + # A 600x400 frame resized to fit a 300x300 tensor should scale by 0.5. + calls = [] + + def fake_resize(size): + calls.append(size) + w, h = size + return np.full((h, w, 3), 200, dtype=np.uint8) + + scale = detector._set_resized_input(interp, (600, 400), fake_resize) + assert scale == (0.5, 0.5) + assert calls == [(300, 200)] + + def test_input_tensor_is_zero_padded_where_not_covered(self): + interp = FakeInterpreter(input_shape=(1, 300, 300, 3), output_tensors=[]) + + def fake_resize(size): + w, h = size + return np.full((h, w, 3), 255, dtype=np.uint8) + + detector._set_resized_input(interp, (600, 400), fake_resize) + tensor = interp._input[0] + # Covered region (300x200) should be 255; the padding strip below it + # (rows 200-299) should remain zero. + assert (tensor[:200, :300] == 255).all() + assert (tensor[200:, :] == 0).all() + + +class TestGetObjects: + def test_legacy_tensor_order_single_count(self): + # Mimics ssd_mobilenet_v2_coco_quant_postprocess_edgetpu.tflite's + # output order: boxes, class_ids, scores, count (count.size == 1). + boxes = np.array([[[0.1, 0.2, 0.5, 0.6]]], dtype=np.float32) + class_ids = np.array([[17.0]], dtype=np.float32) + scores = np.array([[0.93]], dtype=np.float32) + count = np.array([1.0], dtype=np.float32) + + interp = FakeInterpreter( + input_shape=(1, 300, 300, 3), + output_tensors=[boxes, class_ids, scores, count], + ) + results = detector._get_objects(interp, score_threshold=0.4, image_scale=(1.0, 1.0)) + assert len(results) == 1 + obj = results[0] + assert obj["id"] == 17 + assert obj["score"] == pytest_approx(0.93) + # bbox = (xmin*sx, ymin*sy, xmax*sx, ymax*sy) with sx=sy=300 (input size / scale 1.0) + xmin, ymin, xmax, ymax = obj["bbox"] + assert xmin == int(0.2 * 300) + assert ymin == int(0.1 * 300) + assert xmax == int(0.6 * 300) + assert ymax == int(0.5 * 300) + + def test_alternate_tensor_order_when_count_not_size_one(self): + # Some exports order outputs as scores, boxes, count, class_ids + # (the branch taken when output tensor index 3's size != 1). + scores = np.array([[0.8]], dtype=np.float32) + boxes = np.array([[[0.0, 0.0, 1.0, 1.0]]], dtype=np.float32) + count = np.array([1.0], dtype=np.float32) + class_ids = np.array([[5.0], [6.0]], dtype=np.float32) # size != 1 + + interp = FakeInterpreter( + input_shape=(1, 300, 300, 3), + output_tensors=[scores, boxes, count, class_ids], + ) + results = detector._get_objects(interp, score_threshold=0.1, image_scale=(1.0, 1.0)) + assert len(results) == 1 + assert results[0]["id"] == 5 + + def test_score_threshold_filters_low_confidence_detections(self): + boxes = np.array([[[0.0, 0.0, 0.1, 0.1], [0.0, 0.0, 0.1, 0.1]]], dtype=np.float32) + class_ids = np.array([[17.0, 17.0]], dtype=np.float32) + scores = np.array([[0.9, 0.1]], dtype=np.float32) + count = np.array([2.0], dtype=np.float32) + + interp = FakeInterpreter( + input_shape=(1, 300, 300, 3), + output_tensors=[boxes, class_ids, scores, count], + ) + results = detector._get_objects(interp, score_threshold=0.5, image_scale=(1.0, 1.0)) + assert len(results) == 1 + assert results[0]["score"] == pytest_approx(0.9) + + def test_image_scale_affects_bbox_scaling(self): + # image_scale represents the (scale_x, scale_y) used during resize; + # a smaller image_scale means the original frame was larger relative + # to the model's input, so bboxes should scale up accordingly. + boxes = np.array([[[0.0, 0.0, 0.5, 0.5]]], dtype=np.float32) + class_ids = np.array([[17.0]], dtype=np.float32) + scores = np.array([[0.9]], dtype=np.float32) + count = np.array([1.0], dtype=np.float32) + + interp = FakeInterpreter( + input_shape=(1, 300, 300, 3), + output_tensors=[boxes, class_ids, scores, count], + ) + results = detector._get_objects(interp, score_threshold=0.1, image_scale=(0.5, 0.5)) + xmin, ymin, xmax, ymax = results[0]["bbox"] + # sx = width/scale_x = 300/0.5 = 600 + assert xmax == int(0.5 * 600) + assert ymax == int(0.5 * 600) + + +class TestDogDetectorLabelResolution: + def test_target_ids_resolved_by_label_name_case_insensitive(self): + labels_content = "person\ndog\ncat\n" + import tempfile + with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f: + f.write(labels_content) + labels_path = f.name + try: + labels = detector.DogDetector._load_labels(labels_path) + assert labels == {0: "person", 1: "dog", 2: "cat"} + finally: + os.unlink(labels_path) + + def test_missing_label_raises_value_error(self): + labels_content = "person\ncat\n" + import tempfile + with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f: + f.write(labels_content) + labels_path = f.name + try: + # Can't construct a full DogDetector without a model/TPU, but we + # can exercise the label-resolution failure path directly. + labels = detector.DogDetector._load_labels(labels_path) + target_ids = {i for i, n in labels.items() if n.lower() == "dog"} + assert not target_ids + finally: + os.unlink(labels_path) + + +def pytest_approx(value, rel=1e-4): + """Tiny local helper so this file doesn't need to import pytest.approx + directly in every assertion (kept simple on purpose).""" + class _Approx: + def __eq__(self, other): + return abs(other - value) <= rel * max(abs(value), abs(other), 1e-9) + return _Approx()