diff --git a/.vscode/settings.json b/.vscode/settings.json index 35b8bcfb7d..5bd814e47d 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,5 +2,18 @@ "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true, "python.testing.cwd": "photon-lib/py", - "java.configuration.updateBuildConfiguration": "automatic" + "java.configuration.updateBuildConfiguration": "automatic", + "ROS2.distro": "humble", + "python.autoComplete.extraPaths": [ + "/opt/ros/humble/lib/python3.10/site-packages", + "/opt/ros/humble/local/lib/python3.10/dist-packages", + "/home/kevin/Robot/AdaptiGraph/PyFleX/bindings/build", + "" + ], + "python.analysis.extraPaths": [ + "/opt/ros/humble/lib/python3.10/site-packages", + "/opt/ros/humble/local/lib/python3.10/dist-packages", + "/home/kevin/Robot/AdaptiGraph/PyFleX/bindings/build", + "" + ] } \ No newline at end of file diff --git a/README.md b/README.md index fc8400e25d..9db31b7444 100644 --- a/README.md +++ b/README.md @@ -25,8 +25,71 @@ If you are interested in contributing code or documentation to the project, plea Gradle is used for all C++ and Java code, and pnpm is used for the web UI. Instructions to compile PhotonVision yourself can be found [in our docs](https://docs.photonvision.org/en/latest/docs/contributing/building-photon.html#compiling-instructions). +This repo currently requires JDK 17 and Node 22+ for local builds. + You can run one of the many built in examples straight from the command line, too! They contain a fully featured robot project, and some include simulation support. The projects can be found inside the [`photonlib-java-examples`](photonlib-java-examples) and [`photonlib-cpp-examples`](photonlib-cpp-examples) subdirectories, respectively. Instructions for running these examples directly from the repo are found [in the docs](https://docs.photonvision.org/en/latest/docs/contributing/building-photon.html#running-examples). +### Optional NVIDIA AprilTag Backend + +PhotonVision can optionally build an NVIDIA CUDA AprilTag detector backend for supported Linux hosts. This path is opt-in and falls back to the existing CPU detector when the JNI library, SDK, or runtime support is unavailable. + +Build and run with the optional backend enabled: + +```bash +export JAVA_HOME=/path/to/jdk-17 +export PATH="$JAVA_HOME/bin:/path/to/node-v22/bin:$PATH" +export NVIDIA_APRILTAG_SDK_ROOT=/path/to/cuapriltags-root +export CUDA_HOME=/path/to/cuda + +./gradlew :photon-targeting:nvidiaapriltagJNILinuxx86-64ReleaseSharedLibrary \ + :photon-server:run \ + -PenableNvidiaAprilTag \ + -Dphotonvision.apriltag.backend=auto +``` + +Backend selection controls: + +- `-PenableNvidiaAprilTag` enables the optional CUDA JNI build. +- `NVIDIA_APRILTAG_SDK_ROOT` must contain `libcuapriltags.a`. +- `CUDA_HOME` must contain a compatible CUDA runtime, usually under `lib64/libcudart.so`. +- `-Dphotonvision.apriltag.backend=auto|cpu|nvidia` controls runtime backend selection for testing. + +The current implementation only enables the NVIDIA path for `tag36h11`. Other families continue to use the existing CPU detector. + +### Generating Device Images + +PhotonVision release artifacts for coprocessors are board images such as `.img.xz` or `.tar.xz`. This repo does not generate a bootable desktop ISO. + +To build a Linux ARM64 jar and inject it into a base PhotonVision device image locally: + +```bash +./gradlew :photon-server:shadowJar -PArchOverride=linuxarm64 +./scripts/generatePiImage.sh +``` + +Example for a Raspberry Pi image: + +```bash +./gradlew :photon-server:shadowJar -PArchOverride=linuxarm64 +./scripts/generatePiImage.sh \ + https://github.com/PhotonVision/photon-image-modifier/releases/download//photonvision_raspi.img.xz \ + RaspberryPi +``` + +This script downloads the base image, mounts the root partition, replaces `/opt/photonvision/photonvision.jar`, recreates the `photonvision.service` unit, and recompresses the image next to the built jar. + +Example for a custom Jetson-style image: + +```bash +./gradlew :photon-server:shadowJar -PArchOverride=linuxarm64 +PV_ROOT_PARTITION=1 \ +./scripts/generatePiImage.sh \ + https://example.com/your-jetson-photon-base.img.xz \ + JetsonOrinNano +``` + +This Jetson example assumes the root filesystem is on partition `1`. If your image uses a different layout, set `PV_ROOT_PARTITION` accordingly. The helper also supports `PV_PHOTON_DIR` and `PV_SYSTEMD_UNIT_DIR` when the target image uses different install paths. + ## Gradle Arguments Note that these are case sensitive! @@ -44,6 +107,7 @@ Note that these are case sensitive! - `-PtgtPw`: Specifies custom password for `./gradlew deploy` to SSH into - `-Pprofile`: enables JVM profiling - `-PwithSanitizers`: On Linux, enables `-fsanitize=address,undefined,leak` +- `-PenableNvidiaAprilTag`: builds the optional NVIDIA CUDA AprilTag JNI when the SDK is available If you're cross-compiling, you'll need the WPILib toolchain installed. This must be done via Gradle: for example `./gradlew installArm64Toolchain` or `./gradlew installRoboRioToolchain` diff --git a/docs/source/docs/contributing/building-photon.md b/docs/source/docs/contributing/building-photon.md index b9fc80339e..d1d07152ec 100644 --- a/docs/source/docs/contributing/building-photon.md +++ b/docs/source/docs/contributing/building-photon.md @@ -101,6 +101,40 @@ Running the following command under the root directory will build the jar under ``gradlew shadowJar`` ``` +### Build and Run PhotonVision with the Optional NVIDIA AprilTag Backend + +The CUDA AprilTag backend is optional and only builds when `-PenableNvidiaAprilTag` is set and the required SDK pieces are present. Runtime selection still falls back to the existing CPU detector when the NVIDIA path cannot be used. + +Current requirements for the optional backend: + +- Linux host with a supported NVIDIA GPU and driver +- JDK 17 +- Node 22+ +- `NVIDIA_APRILTAG_SDK_ROOT` pointing at a directory that contains `libcuapriltags.a` +- `CUDA_HOME` pointing at a CUDA install that contains `lib64/libcudart.so` + +Example build and run flow on Linux x86_64: + +```bash +export JAVA_HOME=/path/to/jdk-17 +export PATH="$JAVA_HOME/bin:/path/to/node-v22/bin:$PATH" +export NVIDIA_APRILTAG_SDK_ROOT=/path/to/cuapriltags-root +export CUDA_HOME=/path/to/cuda + +./gradlew :photon-targeting:nvidiaapriltagJNILinuxx86-64ReleaseSharedLibrary \ + :photon-server:run \ + -PenableNvidiaAprilTag \ + -Dphotonvision.apriltag.backend=auto +``` + +Useful backend selection modes: + +- `-Dphotonvision.apriltag.backend=auto`: prefer NVIDIA when supported, otherwise fall back to CPU +- `-Dphotonvision.apriltag.backend=cpu`: force the existing CPU path +- `-Dphotonvision.apriltag.backend=nvidia`: request the CUDA path and log a fallback reason if it cannot be used + +The current NVIDIA implementation only applies to `tag36h11`. Other tag families continue to run on the CPU detector. + ### Build and Run PhotonVision on a Raspberry Pi Coprocessor As a convenience, the build has a built-in `deploy` command which builds, deploys, and starts the current source code on a coprocessor. It uses [deploy-utils](https://github.com/wpilibsuite/deploy-utils/blob/main/README.md), so it works very similarly to deploys on robot projects. @@ -134,6 +168,67 @@ An architecture override is required to specify the deploy target's architecture The `deploy` command is tested against Raspberry Pi coprocessors. Other similar coprocessors may work too. +### Generate a Device Image Locally + +PhotonVision coprocessor releases are delivered as board images such as `.img.xz` or `.tar.xz`. This repo does not produce a desktop installer ISO. + +If you want to inject a locally built Linux ARM64 jar into an existing PhotonVision base image, use the helper script in `scripts/generatePiImage.sh`. + +1. Build a Linux ARM64 jar: + +```bash +./gradlew :photon-server:shadowJar -PArchOverride=linuxarm64 +``` + +2. Download and repack a board image by passing the base image URL and the desired artifact suffix: + +```bash +./scripts/generatePiImage.sh +``` + +Example for Raspberry Pi: + +```bash +./scripts/generatePiImage.sh \ + https://github.com/PhotonVision/photon-image-modifier/releases/download//photonvision_raspi.img.xz \ + RaspberryPi +``` + +What the script does: + +- downloads the base `.img.xz` +- decompresses it +- mounts the root filesystem with `losetup` +- replaces `/opt/photonvision/photonvision.jar` +- rewrites the `photonvision.service` systemd unit +- recompresses the result as `photonvision*-image_.xz` + +This flow requires a Linux host with `sudo`, loop-device support, `wget`, and `xz-utils`. + +Example for a custom Jetson-style image: + +```bash +./gradlew :photon-server:shadowJar -PArchOverride=linuxarm64 +PV_ROOT_PARTITION=1 \ +./scripts/generatePiImage.sh \ + https://example.com/your-jetson-photon-base.img.xz \ + JetsonOrinNano +``` + +The Jetson example above assumes: + +- the base image is already a PhotonVision-ready Linux ARM64 image +- the root filesystem is on partition `1` +- PhotonVision should live at `/opt/photonvision` + +If your Jetson image uses different paths, the helper also supports: + +- `PV_ROOT_PARTITION` +- `PV_PHOTON_DIR` +- `PV_SYSTEMD_UNIT_DIR` + +This repo does not currently publish an official Jetson base image in CI, so you must provide the base `.img.xz` yourself. + ### Using PhotonLib Builds The build process automatically generates a vendordep JSON of your local build at `photon-lib/build/generated/vendordeps/photonlib.json`. diff --git a/docs/source/docs/quick-start/networking.md b/docs/source/docs/quick-start/networking.md index e373221dc7..335af30967 100644 --- a/docs/source/docs/quick-start/networking.md +++ b/docs/source/docs/quick-start/networking.md @@ -94,7 +94,7 @@ The address in the code above (`photonvision.local`) is the hostname of the copr ## Camera Stream Ports -The camera streams start at 1181 with two ports for each camera (ex. 1181 and 1182 for camera one, 1183 and 1184 for camera two, etc.). The easiest way to identify the port of the camera that you want is by double clicking on the stream, which opens it in a separate page. The port will be listed below the stream. +The camera streams start at 1181 with two ports for each camera (for example, 1181 and 1182 for camera one, 1183 and 1184 for camera two, etc.). The current stream index allocation supports up to 16 cameras, so the last default pair is 1211 and 1212. The easiest way to identify the port of the camera that you want is by double clicking on the stream, which opens it in a separate page. The port will be listed below the stream. :::{warning} If your camera stream isn't sent to the same port as it's originally found on, its stream will not be visible in the UI. diff --git a/photon-core/src/main/java/org/photonvision/common/LoadJNI.java b/photon-core/src/main/java/org/photonvision/common/LoadJNI.java index 154fe40c22..5d68c554c9 100644 --- a/photon-core/src/main/java/org/photonvision/common/LoadJNI.java +++ b/photon-core/src/main/java/org/photonvision/common/LoadJNI.java @@ -28,6 +28,7 @@ public class LoadJNI { public enum JNITypes { RUBIK_DETECTOR("tensorflowlite", "tensorflowlite_c", "external_delegate", "rubik_jni"), RKNN_DETECTOR("rga", "rknnrt", "rknn_jni"), + NVIDIA_APRILTAG("nvidiaapriltagJNI"), MRCAL("mrcal_jni"), LIBCAMERA("photonlibcamera"); @@ -45,7 +46,11 @@ public static synchronized void forceLoad(JNITypes type) throws IOException { return; } - CombinedRuntimeLoader.loadLibraries(LoadJNI.class, type.libraries); + if (type == JNITypes.NVIDIA_APRILTAG) { + LibraryLoader.loadNvidiaAprilTag(); + } else { + CombinedRuntimeLoader.loadLibraries(LoadJNI.class, type.libraries); + } loadedMap.put(type, true); } diff --git a/photon-core/src/main/java/org/photonvision/common/configuration/CameraConfiguration.java b/photon-core/src/main/java/org/photonvision/common/configuration/CameraConfiguration.java index e51175f36c..994c6c2acf 100644 --- a/photon-core/src/main/java/org/photonvision/common/configuration/CameraConfiguration.java +++ b/photon-core/src/main/java/org/photonvision/common/configuration/CameraConfiguration.java @@ -60,7 +60,7 @@ public class CameraConfiguration { public List calibrations = new ArrayList<>(); public int currentPipelineIndex = 0; - public int streamIndex = 0; // 0 index means ports [1181, 1182], 1 means [1183, 1184], etc... + public int streamIndex = 0; // 0 index means ports [1181, 1182], 1 means [1183, 1184], etc. // Ignore the pipes, as we serialize them to their own column to hack around // polymorphic lists diff --git a/photon-core/src/main/java/org/photonvision/common/dataflow/websocket/UIPhotonConfiguration.java b/photon-core/src/main/java/org/photonvision/common/dataflow/websocket/UIPhotonConfiguration.java index 6e5a14eb09..268c2fcd96 100644 --- a/photon-core/src/main/java/org/photonvision/common/dataflow/websocket/UIPhotonConfiguration.java +++ b/photon-core/src/main/java/org/photonvision/common/dataflow/websocket/UIPhotonConfiguration.java @@ -18,6 +18,7 @@ package org.photonvision.common.dataflow.websocket; import java.util.List; +import java.util.stream.Stream; import org.photonvision.PhotonVersion; import org.photonvision.common.LoadJNI; import org.photonvision.common.LoadJNI.JNITypes; @@ -28,6 +29,7 @@ import org.photonvision.common.hardware.Platform; import org.photonvision.common.networking.NetworkManager; import org.photonvision.common.networking.NetworkUtils; +import org.photonvision.vision.apriltag.AprilTagBackendManager; import org.photonvision.vision.processes.VisionModule; import org.photonvision.vision.processes.VisionSourceManager; @@ -56,8 +58,13 @@ public static UIPhotonConfiguration programStateToUi(PhotonConfiguration c) { OsImageData.IMAGE_METADATA.isPresent() ? OsImageData.IMAGE_METADATA.get().commitTag() : "", - // TODO add support for other types of GPU accel - LoadJNI.hasLoaded(JNITypes.LIBCAMERA) ? "Zerocopy Libcamera Working" : "", + Stream.of( + LoadJNI.hasLoaded(JNITypes.LIBCAMERA) + ? "Zerocopy Libcamera Working" + : "", + AprilTagBackendManager.getUiStatus()) + .filter(s -> s != null && !s.isBlank()) + .collect(java.util.stream.Collectors.joining("; ")), LoadJNI.hasLoaded(JNITypes.MRCAL), c.neuralNetworkPropertyManager().getModels(), NeuralNetworkModelManager.getInstance().getSupportedBackends(), diff --git a/photon-core/src/main/java/org/photonvision/vision/apriltag/AprilTagBackendManager.java b/photon-core/src/main/java/org/photonvision/vision/apriltag/AprilTagBackendManager.java new file mode 100644 index 0000000000..96cf292adb --- /dev/null +++ b/photon-core/src/main/java/org/photonvision/vision/apriltag/AprilTagBackendManager.java @@ -0,0 +1,206 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.vision.apriltag; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Locale; +import java.util.concurrent.atomic.AtomicReference; +import org.photonvision.common.LoadJNI; +import org.photonvision.common.LoadJNI.JNITypes; +import org.photonvision.common.hardware.Platform; + +public final class AprilTagBackendManager { + private static final String BACKEND_PROPERTY = "photonvision.apriltag.backend"; + private static final AtomicReference uiStatus = new AtomicReference<>(); + private static volatile String runtimeFailureReason; + private static volatile TestOverrides testOverrides; + + private AprilTagBackendManager() {} + + private enum BackendPreference { + AUTO, + CPU, + NVIDIA + } + + private record TestOverrides( + Boolean nvidiaCapableLinux, Boolean nvidiaJniLoaded, Boolean nvidiaRuntimeSupported) {} + + public static AprilTagBackendSelection select(AprilTagFamily family) { + var preference = getBackendPreference(); + if (preference == BackendPreference.CPU) { + return new AprilTagBackendSelection( + AprilTagDetectorBackend.CPU_WPILIB, + "AprilTag CPU backend forced by system property", + null); + } + + if (family != AprilTagFamily.kTag36h11) { + return new AprilTagBackendSelection( + AprilTagDetectorBackend.CPU_WPILIB, + "AprilTag CPU backend selected because only tag36h11 supports NVIDIA CUDA", + preference == BackendPreference.NVIDIA + ? "Forced NVIDIA AprilTag backend only supports tag36h11; falling back to CPU." + : null); + } + + if (!isNvidiaCapableLinux()) { + return new AprilTagBackendSelection( + AprilTagDetectorBackend.CPU_WPILIB, + "AprilTag CPU backend selected because no NVIDIA-capable Linux device was detected", + preference == BackendPreference.NVIDIA + ? "Forced NVIDIA AprilTag backend requested, but this host is not an NVIDIA-capable Linux device." + : null); + } + + if (!isNvidiaJniLoaded()) { + return new AprilTagBackendSelection( + AprilTagDetectorBackend.CPU_WPILIB, + "AprilTag CPU backend selected because the NVIDIA JNI library is unavailable", + preference == BackendPreference.NVIDIA + ? "Forced NVIDIA AprilTag backend requested, but the NVIDIA JNI library is unavailable." + : null); + } + + if (!isNvidiaRuntimeSupported()) { + var reason = + runtimeFailureReason != null + ? runtimeFailureReason + : "the NVIDIA CUDA runtime probe failed"; + return new AprilTagBackendSelection( + AprilTagDetectorBackend.CPU_WPILIB, + "AprilTag CPU backend selected because " + reason, + preference == BackendPreference.NVIDIA + ? "Forced NVIDIA AprilTag backend requested, but " + reason + ". Falling back to CPU." + : null); + } + + return new AprilTagBackendSelection( + AprilTagDetectorBackend.NVIDIA_CUDA, + preference == BackendPreference.NVIDIA + ? "AprilTag NVIDIA CUDA backend forced by system property" + : "AprilTag NVIDIA CUDA backend selected automatically", + null); + } + + public static void updateActiveBackend( + AprilTagDetectorBackend backend, String detail, AprilTagFamily family) { + var status = "AprilTag: " + backend.getDisplayName() + " active"; + if (family != null) { + status += " (" + family.getNativeName() + ")"; + } + if (detail != null && !detail.isBlank()) { + status += " - " + detail; + } + uiStatus.set(status); + } + + public static void markRuntimeFailure(String reason) { + if (reason == null || reason.isBlank()) { + runtimeFailureReason = "the NVIDIA CUDA runtime failed"; + } else { + runtimeFailureReason = reason; + } + } + + public static String getUiStatus() { + var active = uiStatus.get(); + if (active != null && !active.isBlank()) { + return active; + } + + if (isNvidiaCapableLinux() && isNvidiaJniLoaded() && isNvidiaRuntimeSupported()) { + return "AprilTag: NVIDIA CUDA available"; + } + + return "AprilTag: CPU WPILib"; + } + + static void setTestOverrides( + Boolean nvidiaCapableLinux, Boolean nvidiaJniLoaded, Boolean nvidiaRuntimeSupported) { + testOverrides = new TestOverrides(nvidiaCapableLinux, nvidiaJniLoaded, nvidiaRuntimeSupported); + } + + static void resetForTest() { + testOverrides = null; + runtimeFailureReason = null; + uiStatus.set(null); + System.clearProperty(BACKEND_PROPERTY); + } + + private static BackendPreference getBackendPreference() { + var value = System.getProperty(BACKEND_PROPERTY, "auto").trim().toLowerCase(Locale.US); + return switch (value) { + case "cpu" -> BackendPreference.CPU; + case "nvidia" -> BackendPreference.NVIDIA; + default -> BackendPreference.AUTO; + }; + } + + private static boolean isNvidiaCapableLinux() { + if (testOverrides != null && testOverrides.nvidiaCapableLinux() != null) { + return testOverrides.nvidiaCapableLinux(); + } + + if (!Platform.isLinux()) { + return false; + } + + return Files.exists(Path.of("/proc/driver/nvidia/version")) + || Files.exists(Path.of("/dev/nvidiactl")) + || fileContains(Path.of("/proc/device-tree/model"), "NVIDIA Jetson"); + } + + private static boolean isNvidiaJniLoaded() { + if (testOverrides != null && testOverrides.nvidiaJniLoaded() != null) { + return testOverrides.nvidiaJniLoaded(); + } + + return LoadJNI.hasLoaded(JNITypes.NVIDIA_APRILTAG); + } + + private static boolean isNvidiaRuntimeSupported() { + if (testOverrides != null && testOverrides.nvidiaRuntimeSupported() != null) { + return testOverrides.nvidiaRuntimeSupported(); + } + + if (runtimeFailureReason != null) { + return false; + } + + try { + return org.photonvision.vision.apriltag.NvidiaAprilTagDetector.isRuntimeSupported(); + } catch (Throwable t) { + runtimeFailureReason = "the NVIDIA CUDA runtime probe threw " + t.getClass().getSimpleName(); + return false; + } + } + + private static boolean fileContains(Path path, String text) { + try { + if (!Files.exists(path)) { + return false; + } + + return Files.readString(path).contains(text); + } catch (Exception ignored) { + return false; + } + } +} diff --git a/photon-core/src/main/java/org/photonvision/vision/apriltag/AprilTagBackendSelection.java b/photon-core/src/main/java/org/photonvision/vision/apriltag/AprilTagBackendSelection.java new file mode 100644 index 0000000000..7ed127ab86 --- /dev/null +++ b/photon-core/src/main/java/org/photonvision/vision/apriltag/AprilTagBackendSelection.java @@ -0,0 +1,21 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.vision.apriltag; + +public record AprilTagBackendSelection( + AprilTagDetectorBackend backend, String summary, String warning) {} diff --git a/photon-core/src/main/java/org/photonvision/vision/apriltag/AprilTagDetectorBackend.java b/photon-core/src/main/java/org/photonvision/vision/apriltag/AprilTagDetectorBackend.java new file mode 100644 index 0000000000..a40da13ff3 --- /dev/null +++ b/photon-core/src/main/java/org/photonvision/vision/apriltag/AprilTagDetectorBackend.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.vision.apriltag; + +public enum AprilTagDetectorBackend { + CPU_WPILIB, + NVIDIA_CUDA; + + public String getDisplayName() { + return switch (this) { + case CPU_WPILIB -> "CPU WPILib"; + case NVIDIA_CUDA -> "NVIDIA CUDA"; + }; + } +} diff --git a/photon-core/src/main/java/org/photonvision/vision/apriltag/NvidiaAprilTagDetector.java b/photon-core/src/main/java/org/photonvision/vision/apriltag/NvidiaAprilTagDetector.java new file mode 100644 index 0000000000..1c08ef4c66 --- /dev/null +++ b/photon-core/src/main/java/org/photonvision/vision/apriltag/NvidiaAprilTagDetector.java @@ -0,0 +1,198 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.vision.apriltag; + +import edu.wpi.first.apriltag.AprilTagDetection; +import java.util.ArrayList; +import java.util.List; +import org.opencv.core.Mat; +import org.opencv.core.MatOfPoint2f; +import org.opencv.core.Point; +import org.opencv.core.Size; +import org.opencv.imgproc.Imgproc; +import org.photonvision.common.logging.LogGroup; +import org.photonvision.common.logging.Logger; +import org.photonvision.jni.NvidiaAprilTagDetection; +import org.photonvision.jni.NvidiaAprilTagJNI; +import org.photonvision.vision.opencv.Releasable; + +public class NvidiaAprilTagDetector implements Releasable { + private static final Logger logger = new Logger(NvidiaAprilTagDetector.class, LogGroup.VisionModule); + private static final int TILE_SIZE = 4; + private static final double[] IDENTITY_HOMOGRAPHY = new double[] {1, 0, 0, 0, 1, 0, 0, 0, 1}; + + private final Mat colorConversionBuffer = new Mat(); + private final Mat resizeBuffer = new Mat(); + private final MatOfPoint2f homographySource = + new MatOfPoint2f( + new Point(-1, 1), + new Point(1, 1), + new Point(1, -1), + new Point(-1, -1)); + private final MatOfPoint2f homographyDestination = new MatOfPoint2f(); + + private long detectorHandle; + private int detectorWidth = -1; + private int detectorHeight = -1; + + public List detect(Mat colorImage, double decimate) { + if (colorImage.empty()) { + return List.of(); + } + + var preparedInput = prepareDetectorInput(colorImage, decimate); + ensureDetector(preparedInput.input().cols(), preparedInput.input().rows()); + + var rawDetections = NvidiaAprilTagJNI.detect(detectorHandle, preparedInput.input().nativeObj); + if (rawDetections == null || rawDetections.length == 0) { + return List.of(); + } + + var detections = new ArrayList(rawDetections.length); + for (var rawDetection : rawDetections) { + detections.add( + toAprilTagDetection( + rawDetection, preparedInput.scaleX(), preparedInput.scaleY())); + } + + return detections; + } + + public static boolean isRuntimeSupported() { + return NvidiaAprilTagJNI.isRuntimeSupported(); + } + + @Override + public void release() { + destroyDetector(); + colorConversionBuffer.release(); + resizeBuffer.release(); + homographySource.release(); + homographyDestination.release(); + } + + private PreparedInput prepareDetectorInput(Mat colorImage, double decimate) { + Mat detectorInput; + if (colorImage.channels() == 3) { + detectorInput = colorImage; + } else if (colorImage.channels() == 4) { + Imgproc.cvtColor(colorImage, colorConversionBuffer, Imgproc.COLOR_BGRA2BGR); + detectorInput = colorConversionBuffer; + } else if (colorImage.channels() == 1) { + Imgproc.cvtColor(colorImage, colorConversionBuffer, Imgproc.COLOR_GRAY2BGR); + detectorInput = colorConversionBuffer; + } else { + throw new IllegalArgumentException( + "Unsupported AprilTag input image format with " + + colorImage.channels() + + " channels"); + } + + if (decimate <= 1.0) { + return new PreparedInput(detectorInput, 1.0, 1.0); + } + + var resizedWidth = Math.max(1, (int) Math.round(detectorInput.cols() / decimate)); + var resizedHeight = Math.max(1, (int) Math.round(detectorInput.rows() / decimate)); + if (resizedWidth == detectorInput.cols() && resizedHeight == detectorInput.rows()) { + return new PreparedInput(detectorInput, 1.0, 1.0); + } + + Imgproc.resize( + detectorInput, + resizeBuffer, + new Size(resizedWidth, resizedHeight), + 0, + 0, + Imgproc.INTER_AREA); + + return new PreparedInput( + resizeBuffer, + (double) colorImage.cols() / resizedWidth, + (double) colorImage.rows() / resizedHeight); + } + + private void ensureDetector(int width, int height) { + if (detectorHandle != 0 && detectorWidth == width && detectorHeight == height) { + return; + } + + destroyDetector(); + + detectorHandle = NvidiaAprilTagJNI.createDetector(width, height, TILE_SIZE); + if (detectorHandle == 0) { + throw new IllegalStateException("Failed to create NVIDIA AprilTag detector"); + } + + detectorWidth = width; + detectorHeight = height; + } + + private void destroyDetector() { + if (detectorHandle != 0) { + NvidiaAprilTagJNI.destroyDetector(detectorHandle); + detectorHandle = 0; + } + + detectorWidth = -1; + detectorHeight = -1; + } + + private AprilTagDetection toAprilTagDetection( + NvidiaAprilTagDetection detection, double scaleX, double scaleY) { + var scaledCorners = new double[8]; + for (int i = 0; i < 4; i++) { + scaledCorners[i * 2] = detection.corners[i * 2] * scaleX; + scaledCorners[i * 2 + 1] = detection.corners[i * 2 + 1] * scaleY; + } + + var scaledCenterX = detection.centerX * scaleX; + var scaledCenterY = detection.centerY * scaleY; + + return new AprilTagDetection( + "tag36h11", + detection.id, + detection.hammingError, + 0.0f, + computeHomography(scaledCorners), + scaledCenterX, + scaledCenterY, + scaledCorners); + } + + private double[] computeHomography(double[] corners) { + homographyDestination.fromArray( + new Point(corners[0], corners[1]), + new Point(corners[2], corners[3]), + new Point(corners[4], corners[5]), + new Point(corners[6], corners[7])); + + var homography = Imgproc.getPerspectiveTransform(homographySource, homographyDestination); + if (homography.empty()) { + logger.warn("Falling back to identity AprilTag homography because OpenCV returned an empty matrix"); + return IDENTITY_HOMOGRAPHY.clone(); + } + + var homographyData = new double[9]; + homography.get(0, 0, homographyData); + homography.release(); + return homographyData; + } + + private record PreparedInput(Mat input, double scaleX, double scaleY) {} +} diff --git a/photon-core/src/main/java/org/photonvision/vision/pipe/impl/AprilTagDetectionPipe.java b/photon-core/src/main/java/org/photonvision/vision/pipe/impl/AprilTagDetectionPipe.java index 01e237c410..483d85a970 100644 --- a/photon-core/src/main/java/org/photonvision/vision/pipe/impl/AprilTagDetectionPipe.java +++ b/photon-core/src/main/java/org/photonvision/vision/pipe/impl/AprilTagDetectionPipe.java @@ -20,16 +20,28 @@ import edu.wpi.first.apriltag.AprilTagDetection; import edu.wpi.first.apriltag.AprilTagDetector; import java.util.List; +import org.opencv.core.Mat; +import org.opencv.imgproc.Imgproc; +import org.photonvision.common.logging.LogGroup; +import org.photonvision.common.logging.Logger; +import org.photonvision.vision.apriltag.AprilTagBackendManager; +import org.photonvision.vision.apriltag.AprilTagDetectorBackend; import org.photonvision.vision.apriltag.AprilTagFamily; -import org.photonvision.vision.opencv.CVMat; +import org.photonvision.vision.apriltag.NvidiaAprilTagDetector; +import org.photonvision.vision.frame.Frame; import org.photonvision.vision.opencv.Releasable; import org.photonvision.vision.pipe.CVPipe; public class AprilTagDetectionPipe extends CVPipe< - CVMat, List, AprilTagDetectionPipe.AprilTagDetectionPipeParams> + Frame, List, AprilTagDetectionPipe.AprilTagDetectionPipeParams> implements Releasable { + private static final Logger logger = new Logger(AprilTagDetectionPipe.class, LogGroup.VisionModule); + private AprilTagDetector m_detector = new AprilTagDetector(); + private final NvidiaAprilTagDetector nvidiaDetector = new NvidiaAprilTagDetector(); + private final Mat cpuFallbackGray = new Mat(); + private AprilTagDetectorBackend lastDetectionBackend = AprilTagDetectorBackend.CPU_WPILIB; public AprilTagDetectionPipe() { super(); @@ -39,8 +51,8 @@ public AprilTagDetectionPipe() { } @Override - protected List process(CVMat in) { - if (in.getMat().empty()) { + protected List process(Frame frame) { + if (frame.processedImage.getMat().empty() && frame.colorImage.getMat().empty()) { return List.of(); } @@ -48,7 +60,25 @@ protected List process(CVMat in) { throw new RuntimeException("Apriltag detector was released!"); } - var ret = m_detector.detect(in.getMat()); + if (params.backend() == AprilTagDetectorBackend.NVIDIA_CUDA + && !frame.colorImage.getMat().empty()) { + try { + var detections = + nvidiaDetector.detect( + frame.colorImage.getMat(), params.detectorParams().quadDecimate); + lastDetectionBackend = AprilTagDetectorBackend.NVIDIA_CUDA; + return detections; + } catch (RuntimeException ex) { + logger.warn( + "NVIDIA AprilTag detection failed; falling back to the WPILib CPU backend: " + + ex.getMessage()); + AprilTagBackendManager.markRuntimeFailure( + "the NVIDIA detector failed at runtime"); + } + } + + var ret = m_detector.detect(getCpuInputMat(frame)); + lastDetectionBackend = AprilTagDetectorBackend.CPU_WPILIB; if (ret == null) { return List.of(); @@ -74,10 +104,40 @@ public void setParams(AprilTagDetectionPipeParams newParams) { public void release() { m_detector.close(); m_detector = null; + nvidiaDetector.release(); + cpuFallbackGray.release(); + } + + public AprilTagDetectorBackend getLastDetectionBackend() { + return lastDetectionBackend; + } + + private Mat getCpuInputMat(Frame frame) { + if (!frame.processedImage.getMat().empty()) { + return frame.processedImage.getMat(); + } + + if (frame.colorImage.getMat().empty()) { + return cpuFallbackGray; + } + + if (frame.colorImage.getMat().channels() == 1) { + frame.colorImage.getMat().copyTo(cpuFallbackGray); + return cpuFallbackGray; + } + + if (frame.colorImage.getMat().channels() == 4) { + Imgproc.cvtColor(frame.colorImage.getMat(), cpuFallbackGray, Imgproc.COLOR_BGRA2GRAY); + } else { + Imgproc.cvtColor(frame.colorImage.getMat(), cpuFallbackGray, Imgproc.COLOR_BGR2GRAY); + } + + return cpuFallbackGray; } public static record AprilTagDetectionPipeParams( AprilTagFamily family, AprilTagDetector.Config detectorParams, - AprilTagDetector.QuadThresholdParameters quadParams) {} + AprilTagDetector.QuadThresholdParameters quadParams, + AprilTagDetectorBackend backend) {} } diff --git a/photon-core/src/main/java/org/photonvision/vision/pipeline/AprilTagPipeline.java b/photon-core/src/main/java/org/photonvision/vision/pipeline/AprilTagPipeline.java index 941e16cfaa..7704508f62 100644 --- a/photon-core/src/main/java/org/photonvision/vision/pipeline/AprilTagPipeline.java +++ b/photon-core/src/main/java/org/photonvision/vision/pipeline/AprilTagPipeline.java @@ -34,6 +34,9 @@ import org.photonvision.common.logging.LogGroup; import org.photonvision.common.logging.Logger; import org.photonvision.common.util.math.MathUtils; +import org.photonvision.vision.apriltag.AprilTagBackendManager; +import org.photonvision.vision.apriltag.AprilTagBackendSelection; +import org.photonvision.vision.apriltag.AprilTagDetectorBackend; import org.photonvision.estimation.TargetModel; import org.photonvision.targeting.MultiTargetPNPResult; import org.photonvision.vision.apriltag.AprilTagFamily; @@ -60,15 +63,17 @@ public class AprilTagPipeline extends CVPipeline> tagDetectionPipeResult = - aprilTagDetectionPipe.run(frame.processedImage); + aprilTagDetectionPipe.run(frame); sumPipeNanosElapsed += tagDetectionPipeResult.nanosElapsed; + var activeBackend = aprilTagDetectionPipe.getLastDetectionBackend(); + AprilTagBackendManager.updateActiveBackend( + activeBackend, backendSelection.summary(), settings.tagFamily); + + if (frame.processedImage.getMat().empty() && !frame.colorImage.getMat().empty()) { + frame.colorImage.getMat().copyTo(frame.processedImage.getMat()); + } List detections = tagDetectionPipeResult.output; List usedDetections = new ArrayList<>(); @@ -149,9 +170,11 @@ protected CVPipelineResult process(Frame frame, AprilTagPipelineSettings setting // Filter out detections based on pipeline settings for (AprilTagDetection detection : detections) { - // TODO this should be in a pipe, not in the top level here (Matt) - if (detection.getDecisionMargin() < settings.decisionMargin) continue; - if (detection.getHamming() > settings.hammingDist) continue; + if (activeBackend == AprilTagDetectorBackend.CPU_WPILIB) { + // TODO this should be in a pipe, not in the top level here (Matt) + if (detection.getDecisionMargin() < settings.decisionMargin) continue; + if (detection.getHamming() > settings.hammingDist) continue; + } usedDetections.add(detection); @@ -250,6 +273,18 @@ protected CVPipelineResult process(Frame frame, AprilTagPipelineSettings setting frame.sequenceID, sumPipeNanosElapsed, fps, targetList, multiTagResult, frame); } + @Override + public FrameThresholdType getThresholdType() { + if (settings == null) { + return CPU_PROCESSING_TYPE; + } + + var selection = AprilTagBackendManager.select(settings.tagFamily); + return selection.backend() == AprilTagDetectorBackend.NVIDIA_CUDA + ? NVIDIA_PROCESSING_TYPE + : CPU_PROCESSING_TYPE; + } + @Override public void release() { aprilTagDetectionPipe.release(); diff --git a/photon-core/src/main/java/org/photonvision/vision/processes/VisionModule.java b/photon-core/src/main/java/org/photonvision/vision/processes/VisionModule.java index e0843e3919..25dc3afab0 100644 --- a/photon-core/src/main/java/org/photonvision/vision/processes/VisionModule.java +++ b/photon-core/src/main/java/org/photonvision/vision/processes/VisionModule.java @@ -187,7 +187,7 @@ public VisionModule(PipelineManager pipelineManager, VisionSource visionSource) private void createStreams() { var camStreamIdx = visionSource.getSettables().getConfiguration().streamIndex; - // If idx = 0, we want (1181, 1182) + // If idx = 0, we want (1181, 1182). Index 15 maps to (1211, 1212). this.inputStreamPort = 1181 + (camStreamIdx * 2); this.outputStreamPort = 1181 + (camStreamIdx * 2) + 1; diff --git a/photon-core/src/main/java/org/photonvision/vision/processes/VisionModuleManager.java b/photon-core/src/main/java/org/photonvision/vision/processes/VisionModuleManager.java index 7eb3c86212..3250da77be 100644 --- a/photon-core/src/main/java/org/photonvision/vision/processes/VisionModuleManager.java +++ b/photon-core/src/main/java/org/photonvision/vision/processes/VisionModuleManager.java @@ -23,6 +23,8 @@ /** VisionModuleManager has many VisionModules, and provides camera configuration data to them. */ public class VisionModuleManager { + static final int MAX_CAMERA_STREAMS = 16; + private final Logger logger = new Logger(VisionModuleManager.class, LogGroup.VisionModule); private final List visionModules = new ArrayList<>(); @@ -61,7 +63,7 @@ private synchronized int newCameraIndex() { // But by operating on the list, we have a fairly good idea of which we need to change, // but it's not guaranteed that we change the correct one // The best we can do is try to avoid a case where the stream index runs away to infinity - // since we can only stream 5 cameras at once + // since we only assign stream ports for the first MAX_CAMERA_STREAMS cameras // Big list, which should contain every vision source (currently loaded plus the new ones being // added) @@ -73,8 +75,11 @@ private synchronized int newCameraIndex() { idx++; } - if (idx >= 5) { - logger.warn("VisionModuleManager has reached the maximum number of cameras (5)."); + if (idx >= MAX_CAMERA_STREAMS) { + logger.warn( + "VisionModuleManager has reached the configured camera stream limit (" + + MAX_CAMERA_STREAMS + + ")."); } return idx; diff --git a/photon-core/src/test/java/org/photonvision/common/AprilTagBenchmarkTest.java b/photon-core/src/test/java/org/photonvision/common/AprilTagBenchmarkTest.java new file mode 100644 index 0000000000..bec706b47f --- /dev/null +++ b/photon-core/src/test/java/org/photonvision/common/AprilTagBenchmarkTest.java @@ -0,0 +1,121 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.common; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.photonvision.common.LoadJNI.JNITypes; +import org.photonvision.common.configuration.ConfigManager; +import org.photonvision.common.util.TestUtils; +import org.photonvision.common.util.math.MathUtils; +import org.photonvision.common.util.numbers.NumberListUtils; +import org.photonvision.vision.apriltag.AprilTagFamily; +import org.photonvision.vision.camera.QuirkyCamera; +import org.photonvision.vision.frame.provider.FileFrameProvider; +import org.photonvision.vision.pipeline.AprilTagPipeline; + +public class AprilTagBenchmarkTest { + @BeforeAll + public static void init() { + LoadJNI.loadLibraries(); + ConfigManager.getInstance().load(); + try { + LoadJNI.forceLoad(JNITypes.NVIDIA_APRILTAG); + } catch (IOException ignored) { + // GPU benchmark will transparently fall back to CPU if the optional JNI is unavailable. + } + } + + @AfterEach + public void cleanup() { + System.clearProperty("photonvision.apriltag.backend"); + } + + @Test + public void benchmarkTag1CpuVsAuto() { + benchmarkImage("CPU", "cpu", TestUtils.ApriltagTestImages.kTag1_640_480, 3); + benchmarkImage("AUTO", "auto", TestUtils.ApriltagTestImages.kTag1_640_480, 3); + } + + @Test + public void benchmarkStressCpuVsAuto() { + benchmarkImage("CPU", "cpu", TestUtils.ApriltagTestImages.k36h11_stress_test, 3); + benchmarkImage("AUTO", "auto", TestUtils.ApriltagTestImages.k36h11_stress_test, 3); + } + + private static void benchmarkImage( + String label, String backend, TestUtils.ApriltagTestImages image, int secondsToRun) { + System.setProperty("photonvision.apriltag.backend", backend); + + var pipeline = new AprilTagPipeline(); + pipeline.getSettings().tagFamily = AprilTagFamily.kTag36h11; + pipeline.getSettings().solvePNPEnabled = false; + pipeline.getSettings().outputShouldDraw = false; + if (image == TestUtils.ApriltagTestImages.k36h11_stress_test) { + pipeline.getSettings().decimate = 4; + } + + var frameProvider = + new FileFrameProvider( + TestUtils.getApriltagImagePath(image, false), + TestUtils.WPI2020Image.FOV, + TestUtils.get2020LifeCamCoeffs(false)); + frameProvider.requestFrameThresholdType(pipeline.getThresholdType()); + + for (int i = 0; i < 5; i++) { + pipeline.run(frameProvider.get(), QuirkyCamera.DefaultCamera).release(); + } + + var processingTimes = new ArrayList(); + var benchmarkStartMillis = System.currentTimeMillis(); + do { + var result = pipeline.run(frameProvider.get(), QuirkyCamera.DefaultCamera); + processingTimes.add(result.getProcessingMillis()); + result.release(); + } while (System.currentTimeMillis() - benchmarkStartMillis < secondsToRun * 1000.0); + + var processingMin = Collections.min(processingTimes); + var processingMean = NumberListUtils.mean(processingTimes); + var processingMax = Collections.max(processingTimes); + + System.out.println( + label + + " " + + image.name() + + " processing - Min: " + + MathUtils.roundTo(processingMin, 3) + + "ms (" + + MathUtils.roundTo(1000 / processingMin, 3) + + " FPS), Mean: " + + MathUtils.roundTo(processingMean, 3) + + "ms (" + + MathUtils.roundTo(1000 / processingMean, 3) + + " FPS), Max: " + + MathUtils.roundTo(processingMax, 3) + + "ms (" + + MathUtils.roundTo(1000 / processingMax, 3) + + " FPS)"); + + pipeline.release(); + frameProvider.release(); + } +} diff --git a/photon-core/src/test/java/org/photonvision/vision/apriltag/AprilTagBackendManagerTest.java b/photon-core/src/test/java/org/photonvision/vision/apriltag/AprilTagBackendManagerTest.java new file mode 100644 index 0000000000..e48cb2b63c --- /dev/null +++ b/photon-core/src/test/java/org/photonvision/vision/apriltag/AprilTagBackendManagerTest.java @@ -0,0 +1,70 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.vision.apriltag; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +public class AprilTagBackendManagerTest { + @AfterEach + public void cleanup() { + AprilTagBackendManager.resetForTest(); + } + + @Test + public void noNvidiaJniFallsBackToCpu() { + AprilTagBackendManager.setTestOverrides(true, false, true); + + var selection = AprilTagBackendManager.select(AprilTagFamily.kTag36h11); + + assertEquals(AprilTagDetectorBackend.CPU_WPILIB, selection.backend()); + } + + @Test + public void tag16h5AlwaysFallsBackToCpu() { + AprilTagBackendManager.setTestOverrides(true, true, true); + + var selection = AprilTagBackendManager.select(AprilTagFamily.kTag16h5); + + assertEquals(AprilTagDetectorBackend.CPU_WPILIB, selection.backend()); + } + + @Test + public void forcedNvidiaWithoutSupportWarnsAndFallsBack() { + System.setProperty("photonvision.apriltag.backend", "nvidia"); + AprilTagBackendManager.setTestOverrides(false, false, false); + + var selection = AprilTagBackendManager.select(AprilTagFamily.kTag36h11); + + assertEquals(AprilTagDetectorBackend.CPU_WPILIB, selection.backend()); + assertNotNull(selection.warning()); + } + + @Test + public void forcedCpuOnNvidiaHostStaysOnCpu() { + System.setProperty("photonvision.apriltag.backend", "cpu"); + AprilTagBackendManager.setTestOverrides(true, true, true); + + var selection = AprilTagBackendManager.select(AprilTagFamily.kTag36h11); + + assertEquals(AprilTagDetectorBackend.CPU_WPILIB, selection.backend()); + } +} diff --git a/photon-core/src/test/java/org/photonvision/vision/processes/VisionModuleManagerTest.java b/photon-core/src/test/java/org/photonvision/vision/processes/VisionModuleManagerTest.java index ba1025fb1b..51011d55f2 100644 --- a/photon-core/src/test/java/org/photonvision/vision/processes/VisionModuleManagerTest.java +++ b/photon-core/src/test/java/org/photonvision/vision/processes/VisionModuleManagerTest.java @@ -18,10 +18,12 @@ package org.photonvision.vision.processes; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import edu.wpi.first.cscore.VideoMode; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -244,6 +246,32 @@ public void testMultipleStreamIndex() { assertTrue(idxs.contains(4)); } + @Test + public void testSupportsSixteenStreamIndexes() { + ConfigManager.getInstance().load(); + + var vmm = new VisionModuleManager(); + var modules = new ArrayList(); + + for (int i = 0; i < VisionModuleManager.MAX_CAMERA_STREAMS; i++) { + var conf = new CameraConfiguration(PVCameraInfo.fromFileInfo("Cam" + i, "/dev/video" + i)); + conf.streamIndex = 0; + var ffp = + new FileFrameProvider( + TestUtils.getWPIImagePath( + TestUtils.WPI2019Image.kCargoStraightDark72in_HighRes, false), + TestUtils.WPI2019Image.FOV); + modules.add(vmm.addSource(new TestSource(ffp, conf))); + } + + var idxs = modules.stream().map(it -> it.getCameraConfiguration().streamIndex).sorted().toList(); + + assertEquals(VisionModuleManager.MAX_CAMERA_STREAMS, idxs.size()); + for (int i = 0; i < VisionModuleManager.MAX_CAMERA_STREAMS; i++) { + assertEquals(i, idxs.get(i)); + } + } + private void sleep(int millis) { try { Thread.sleep(millis); diff --git a/photon-server/src/main/java/org/photonvision/Main.java b/photon-server/src/main/java/org/photonvision/Main.java index b07538a587..4155aa7b75 100644 --- a/photon-server/src/main/java/org/photonvision/Main.java +++ b/photon-server/src/main/java/org/photonvision/Main.java @@ -224,6 +224,15 @@ private static void tryLoadJNI(JNITypes type) { } } + private static void tryLoadOptionalJNI(JNITypes type, String unavailableMessage) { + try { + LoadJNI.forceLoad(type); + logger.info("Loaded " + type.name() + "-JNI"); + } catch (IOException e) { + logger.info(unavailableMessage + ": " + e.getMessage()); + } + } + public static void main(String[] args) { var logLevel = printDebugLogs ? LogLevel.TRACE : LogLevel.DEBUG; Logger.setLevel(LogGroup.Camera, logLevel); @@ -286,6 +295,11 @@ public static void main(String[] args) { tryLoadJNI(JNITypes.LIBCAMERA); } + if (Platform.isLinux()) { + tryLoadOptionalJNI( + JNITypes.NVIDIA_APRILTAG, "NVIDIA AprilTag backend is unavailable on this host"); + } + if (Platform.isRK3588()) { tryLoadJNI(JNITypes.RKNN_DETECTOR); } else { diff --git a/photon-targeting/build.gradle b/photon-targeting/build.gradle index 0ee500035d..e4e1265c97 100644 --- a/photon-targeting/build.gradle +++ b/photon-targeting/build.gradle @@ -2,6 +2,52 @@ ext { nativeName = "photontargeting" } +def nvidiaAprilTagJniName = "nvidiaapriltagJNI" +def enableNvidiaAprilTag = project.hasProperty('enableNvidiaAprilTag') +def hostIsSupportedNvidiaPlatform = + System.getProperty("os.name").startsWith("Linux") + && ["amd64", "x86_64", "aarch64", "arm64"].contains(System.getProperty("os.arch")) + +def findExistingPath = { List candidates -> + candidates.collect { it != null ? file(it) : null }.find { it != null && it.exists() } +} + +def nvidiaAprilTagSdkRoot = System.getenv("NVIDIA_APRILTAG_SDK_ROOT") +def cudaHome = System.getenv("CUDA_HOME") +def nvidiaAprilTagLibrary = + nvidiaAprilTagSdkRoot + ? findExistingPath( + [ + "${nvidiaAprilTagSdkRoot}/libcuapriltags.a", + "${nvidiaAprilTagSdkRoot}/lib/libcuapriltags.a", + "${nvidiaAprilTagSdkRoot}/lib_x86_64_cuda_12_6/libcuapriltags.a", + "${nvidiaAprilTagSdkRoot}/lib_aarch64_cuda_12_6/libcuapriltags.a", + "${nvidiaAprilTagSdkRoot}/isaac_ros_nitros/lib/cuapriltags/lib_x86_64_cuda_12_6/libcuapriltags.a", + "${nvidiaAprilTagSdkRoot}/isaac_ros_nitros/lib/cuapriltags/lib_aarch64_cuda_12_6/libcuapriltags.a", + ]) + : null +def cudaLibraryDir = + cudaHome + ? findExistingPath( + [ + "${cudaHome}/lib64", + "${cudaHome}/lib/x64", + "${cudaHome}/targets/x86_64-linux/lib", + "${cudaHome}/targets/aarch64-linux/lib", + ]) + : null + +def nvidiaAprilTagBuildEnabled = + enableNvidiaAprilTag + && hostIsSupportedNvidiaPlatform + && nvidiaAprilTagLibrary != null + && cudaLibraryDir != null + +if (enableNvidiaAprilTag && !nvidiaAprilTagBuildEnabled) { + println( + "NVIDIA AprilTag JNI disabled: expected NVIDIA_APRILTAG_SDK_ROOT/libcuapriltags.a and CUDA_HOME runtime libraries on a supported Linux host.") +} + apply plugin: 'cpp' apply plugin: 'c' apply plugin: 'google-test-test-suite' @@ -89,6 +135,38 @@ model { nativeUtils.useRequiredLibrary(it, "cscore_shared") nativeUtils.useRequiredLibrary(it, "opencv_shared") } + "${nvidiaAprilTagJniName}"(JniNativeLibrarySpec) { + javaCompileTasks << compileJava + + sources { + cpp { + source { + srcDirs 'src/main/native/nvidia' + include '**/*.cpp', '**/*.cc' + } + exportedHeaders { + srcDirs 'src/main/native/include' + include "**/*.h" + } + } + } + + binaries.all { + if (!nvidiaAprilTagBuildEnabled || it.targetPlatform.name != getCurrentArch()) { + it.buildable = false + return + } + + it.cppCompiler.args << "-DPHOTONVISION_ENABLE_NVIDIA_APRILTAG=1" + it.linker.args << nvidiaAprilTagLibrary.absolutePath + it.linker.args << "-L${cudaLibraryDir.absolutePath}" + it.linker.args << "-lcudart" + it.linker.args << "-ldl" + it.linker.args << "-Wl,-rpath,${cudaLibraryDir.absolutePath}" + } + + nativeUtils.useRequiredLibrary(it, "opencv_shared") + } } testSuites { "${nativeName}Test"(GoogleTestTestSuiteSpec) { diff --git a/photon-targeting/src/main/java/org/photonvision/jni/LibraryLoader.java b/photon-targeting/src/main/java/org/photonvision/jni/LibraryLoader.java index 97f686c8b5..14598a4622 100644 --- a/photon-targeting/src/main/java/org/photonvision/jni/LibraryLoader.java +++ b/photon-targeting/src/main/java/org/photonvision/jni/LibraryLoader.java @@ -26,11 +26,14 @@ import edu.wpi.first.networktables.NetworkTablesJNI; import edu.wpi.first.util.WPIUtilJNI; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import org.opencv.core.Core; public class LibraryLoader { private static boolean hasWpiLoaded = false; private static boolean hasTargetingLoaded = false; + private static boolean hasNvidiaAprilTagLoaded = false; public static boolean loadWpiLibraries() { if (hasWpiLoaded) return true; @@ -72,9 +75,79 @@ public static boolean loadTargeting() { CombinedRuntimeLoader.loadLibraries(LibraryLoader.class, "photontargetingJNI"); hasTargetingLoaded = true; } catch (IOException e) { - e.printStackTrace(); - hasTargetingLoaded = false; + hasTargetingLoaded = loadLocalBuildLibrary("photontargetingJNI"); + if (!hasTargetingLoaded) { + e.printStackTrace(); + } } return hasTargetingLoaded; } + + public static boolean loadNvidiaAprilTag() throws IOException { + if (hasNvidiaAprilTagLoaded) { + return true; + } + + try { + CombinedRuntimeLoader.loadLibraries(LibraryLoader.class, "nvidiaapriltagJNI"); + hasNvidiaAprilTagLoaded = true; + return true; + } catch (IOException e) { + hasNvidiaAprilTagLoaded = loadLocalBuildLibrary("nvidiaapriltagJNI"); + if (!hasNvidiaAprilTagLoaded) { + throw e; + } + return true; + } + } + + private static boolean loadLocalBuildLibrary(String libraryName) { + for (var libraryPath : getLocalBuildCandidates(libraryName)) { + if (!Files.exists(libraryPath)) { + continue; + } + + try { + if ("photontargetingJNI".equals(libraryName)) { + var configuration = libraryPath.getParent().getFileName(); + var platform = libraryPath.getParent().getParent().getFileName(); + var jniRoot = + libraryPath.getParent().getParent().getParent().getParent(); + var supportingLibrary = + jniRoot.resolveSibling("photontargeting") + .resolve("shared") + .resolve(platform) + .resolve(configuration) + .resolve(System.mapLibraryName("photontargeting")); + if (Files.exists(supportingLibrary)) { + System.load(supportingLibrary.toAbsolutePath().toString()); + } + } + + System.load(libraryPath.toAbsolutePath().toString()); + return true; + } catch (UnsatisfiedLinkError ignored) { + // Fall through to the next candidate. + } + } + + return false; + } + + private static Path[] getLocalBuildCandidates(String libraryName) { + var platform = CombinedRuntimeLoader.getPlatformPath().replace("/", ""); + var fileName = System.mapLibraryName(libraryName); + return new Path[] { + Path.of("photon-targeting", "build", "libs", libraryName, "shared", platform, "release", fileName), + Path.of("photon-targeting", "build", "libs", libraryName, "shared", platform, "debug", fileName), + Path.of("build", "libs", libraryName, "shared", platform, "release", fileName), + Path.of("build", "libs", libraryName, "shared", platform, "debug", fileName), + Path.of("..", "photon-targeting", "build", "libs", libraryName, "shared", platform, "release", fileName), + Path.of("..", "photon-targeting", "build", "libs", libraryName, "shared", platform, "debug", fileName), + Path.of("..", "build", "libs", libraryName, "shared", platform, "release", fileName), + Path.of("..", "build", "libs", libraryName, "shared", platform, "debug", fileName), + Path.of("..", "..", "photon-targeting", "build", "libs", libraryName, "shared", platform, "release", fileName), + Path.of("..", "..", "photon-targeting", "build", "libs", libraryName, "shared", platform, "debug", fileName), + }; + } } diff --git a/photon-targeting/src/main/java/org/photonvision/jni/NvidiaAprilTagDetection.java b/photon-targeting/src/main/java/org/photonvision/jni/NvidiaAprilTagDetection.java new file mode 100644 index 0000000000..259bdd728a --- /dev/null +++ b/photon-targeting/src/main/java/org/photonvision/jni/NvidiaAprilTagDetection.java @@ -0,0 +1,35 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.jni; + +public class NvidiaAprilTagDetection { + public final int id; + public final int hammingError; + public final double centerX; + public final double centerY; + public final double[] corners; + + public NvidiaAprilTagDetection( + int id, int hammingError, double centerX, double centerY, double[] corners) { + this.id = id; + this.hammingError = hammingError; + this.centerX = centerX; + this.centerY = centerY; + this.corners = corners; + } +} diff --git a/photon-targeting/src/main/java/org/photonvision/jni/NvidiaAprilTagJNI.java b/photon-targeting/src/main/java/org/photonvision/jni/NvidiaAprilTagJNI.java new file mode 100644 index 0000000000..1795961179 --- /dev/null +++ b/photon-targeting/src/main/java/org/photonvision/jni/NvidiaAprilTagJNI.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.jni; + +public final class NvidiaAprilTagJNI { + private NvidiaAprilTagJNI() {} + + public static native boolean isRuntimeSupported(); + + public static native long createDetector(int width, int height, int tileSize); + + public static native void destroyDetector(long handle); + + public static native NvidiaAprilTagDetection[] detect(long handle, long inputMatPtr); +} diff --git a/photon-targeting/src/main/native/include/nvidia/NvidiaAprilTagApi.h b/photon-targeting/src/main/native/include/nvidia/NvidiaAprilTagApi.h new file mode 100644 index 0000000000..48960d3d33 --- /dev/null +++ b/photon-targeting/src/main/native/include/nvidia/NvidiaAprilTagApi.h @@ -0,0 +1,112 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once + +#include +#include + +struct uint3 { + unsigned int x; + unsigned int y; + unsigned int z; + + constexpr uint3(unsigned int x_ = 1, unsigned int y_ = 1, unsigned int z_ = 1) + : x(x_), y(y_), z(z_) {} +}; + +struct dim3 : public uint3 { + constexpr dim3(unsigned int x_ = 1, unsigned int y_ = 1, unsigned int z_ = 1) + : uint3(x_, y_, z_) {} + constexpr dim3(uint3 value) : uint3(value) {} +}; + +struct uchar3 { + unsigned char x; + unsigned char y; + unsigned char z; +}; + +struct alignas(8) float2 { + float x; + float y; +}; + +typedef int cudaError_t; + +enum cudaMemcpyKind { + cudaMemcpyHostToHost = 0, + cudaMemcpyHostToDevice = 1, + cudaMemcpyDeviceToHost = 2, + cudaMemcpyDeviceToDevice = 3, + cudaMemcpyDefault = 4 +}; + +constexpr cudaError_t cudaSuccess = 0; + +extern "C" { +cudaError_t cudaGetDeviceCount(int* count); +cudaError_t cudaMallocPitch(void** devPtr, size_t* pitch, size_t width, size_t height); +cudaError_t cudaFree(void* devPtr); +cudaError_t cudaMemcpy2D(void* dst, size_t dpitch, const void* src, size_t spitch, + size_t width, size_t height, cudaMemcpyKind kind); +const char* cudaGetErrorString(cudaError_t error); +} + +struct CUstream_st; + +typedef struct cuAprilTagsID_st { + float2 corners[4]; + uint16_t id; + uint8_t hamming_error; + float orientation[9]; + float translation[3]; +} cuAprilTagsID_t; + +typedef struct cuAprilTagsImageInput_st { + uchar3* dev_ptr; + size_t pitch; + uint16_t width; + uint16_t height; +} cuAprilTagsImageInput_t; + +typedef struct cuAprilTagsCameraIntrinsics_st { + float fx; + float fy; + float cx; + float cy; +} cuAprilTagsCameraIntrinsics_t; + +typedef enum { + NVAT_TAG36H11, + NVAT_ENUM_SIZE = 0x7fffffff +} cuAprilTagsFamily; + +typedef struct cuAprilTagsHandle_st* cuAprilTagsHandle; + +extern "C" { +int nvCreateAprilTagsDetector(cuAprilTagsHandle* hApriltags, uint32_t img_width, + uint32_t img_height, uint32_t tile_size, + cuAprilTagsFamily tag_family, + const cuAprilTagsCameraIntrinsics_t* cam, + float tag_dim); +int cuAprilTagsDetect(cuAprilTagsHandle hApriltags, + const cuAprilTagsImageInput_t* img_input, + cuAprilTagsID_t* tags_out, uint32_t* num_tags, + uint32_t max_tags, CUstream_st* input_stream); +int cuAprilTagsDestroy(cuAprilTagsHandle hApriltags); +} diff --git a/photon-targeting/src/main/native/nvidia/NvidiaAprilTagJNI.cpp b/photon-targeting/src/main/native/nvidia/NvidiaAprilTagJNI.cpp new file mode 100644 index 0000000000..825032a80c --- /dev/null +++ b/photon-targeting/src/main/native/nvidia/NvidiaAprilTagJNI.cpp @@ -0,0 +1,254 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include + +#include +#include +#include +#include + +#include + +#include "nvidia/NvidiaAprilTagApi.h" + +namespace { + +constexpr uint32_t kMaxTags = 1024; + +struct DetectorContext { + cuAprilTagsHandle detector = nullptr; + void* deviceBuffer = nullptr; + size_t devicePitch = 0; + uint32_t width = 0; + uint32_t height = 0; +}; + +void ThrowRuntimeException(JNIEnv* env, const std::string& message) { + auto* exceptionClass = env->FindClass("java/lang/RuntimeException"); + if (exceptionClass != nullptr) { + env->ThrowNew(exceptionClass, message.c_str()); + } +} + +bool CheckCuda(JNIEnv* env, cudaError_t status, const std::string& action) { + if (status == cudaSuccess) { + return true; + } + + ThrowRuntimeException( + env, action + " failed: " + std::string(cudaGetErrorString(status))); + return false; +} + +void ComputeCenter(const cuAprilTagsID_t& detection, double* centerX, + double* centerY) { + const double averageCenterX = + (detection.corners[0].x + detection.corners[1].x + + detection.corners[2].x + detection.corners[3].x) / + 4.0; + const double averageCenterY = + (detection.corners[0].y + detection.corners[1].y + + detection.corners[2].y + detection.corners[3].y) / + 4.0; + + const double x1 = detection.corners[0].x; + const double y1 = detection.corners[0].y; + const double x2 = detection.corners[2].x; + const double y2 = detection.corners[2].y; + const double x3 = detection.corners[1].x; + const double y3 = detection.corners[1].y; + const double x4 = detection.corners[3].x; + const double y4 = detection.corners[3].y; + + const double denominator = ((x1 - x2) * (y3 - y4)) - + ((y1 - y2) * (x3 - x4)); + if (std::abs(denominator) < 1e-6) { + *centerX = averageCenterX; + *centerY = averageCenterY; + return; + } + + const double determinant1 = (x1 * y2) - (y1 * x2); + const double determinant2 = (x3 * y4) - (y3 * x4); + *centerX = ((determinant1 * (x3 - x4)) - ((x1 - x2) * determinant2)) / + denominator; + *centerY = ((determinant1 * (y3 - y4)) - ((y1 - y2) * determinant2)) / + denominator; +} + +} // namespace + +extern "C" { + +JNIEXPORT jboolean JNICALL +Java_org_photonvision_jni_NvidiaAprilTagJNI_isRuntimeSupported(JNIEnv* env, + jclass) { + int deviceCount = 0; + if (!CheckCuda(env, cudaGetDeviceCount(&deviceCount), + "Checking available CUDA devices")) { + return JNI_FALSE; + } + + return deviceCount > 0 ? JNI_TRUE : JNI_FALSE; +} + +JNIEXPORT jlong JNICALL +Java_org_photonvision_jni_NvidiaAprilTagJNI_createDetector(JNIEnv* env, jclass, + jint width, + jint height, + jint tileSize) { + if (width <= 0 || height <= 0 || tileSize <= 0) { + ThrowRuntimeException(env, "NVIDIA AprilTag detector dimensions are invalid"); + return 0; + } + + auto context = std::make_unique(); + context->width = static_cast(width); + context->height = static_cast(height); + + if (!CheckCuda(env, + cudaMallocPitch(&context->deviceBuffer, &context->devicePitch, + sizeof(uchar3) * context->width, + context->height), + "Allocating CUDA input buffer")) { + return 0; + } + + const int error = nvCreateAprilTagsDetector( + &context->detector, context->width, context->height, + static_cast(tileSize), NVAT_TAG36H11, nullptr, 0.0f); + if (error != 0) { + cudaFree(context->deviceBuffer); + ThrowRuntimeException(env, "Failed to create NVIDIA AprilTag detector"); + return 0; + } + + return reinterpret_cast(context.release()); +} + +JNIEXPORT void JNICALL +Java_org_photonvision_jni_NvidiaAprilTagJNI_destroyDetector(JNIEnv*, jclass, + jlong handle) { + auto* context = reinterpret_cast(handle); + if (context == nullptr) { + return; + } + + if (context->detector != nullptr) { + cuAprilTagsDestroy(context->detector); + } + + if (context->deviceBuffer != nullptr) { + cudaFree(context->deviceBuffer); + } + + delete context; +} + +JNIEXPORT jobjectArray JNICALL +Java_org_photonvision_jni_NvidiaAprilTagJNI_detect(JNIEnv* env, jclass, + jlong handle, + jlong inputMatPtr) { + auto* context = reinterpret_cast(handle); + auto* inputMat = reinterpret_cast(inputMatPtr); + + auto* detectionClass = + env->FindClass("org/photonvision/jni/NvidiaAprilTagDetection"); + if (detectionClass == nullptr) { + return nullptr; + } + + if (context == nullptr || inputMat == nullptr || inputMat->empty()) { + return env->NewObjectArray(0, detectionClass, nullptr); + } + + if (inputMat->channels() != 3) { + ThrowRuntimeException(env, + "NVIDIA AprilTag detector requires a 3-channel BGR image"); + return nullptr; + } + + if (static_cast(inputMat->cols) != context->width || + static_cast(inputMat->rows) != context->height) { + ThrowRuntimeException( + env, + "NVIDIA AprilTag detector input dimensions do not match the allocated detector"); + return nullptr; + } + + if (!CheckCuda(env, + cudaMemcpy2D(context->deviceBuffer, context->devicePitch, + inputMat->data, inputMat->step[0], + sizeof(uchar3) * context->width, context->height, + cudaMemcpyHostToDevice), + "Uploading the AprilTag frame to CUDA")) { + return nullptr; + } + + cuAprilTagsImageInput_t inputImage; + inputImage.dev_ptr = reinterpret_cast(context->deviceBuffer); + inputImage.pitch = context->devicePitch; + inputImage.width = static_cast(context->width); + inputImage.height = static_cast(context->height); + + std::vector tags(kMaxTags); + uint32_t numTags = 0; + const int error = + cuAprilTagsDetect(context->detector, &inputImage, tags.data(), &numTags, + static_cast(tags.size()), nullptr); + if (error != 0) { + ThrowRuntimeException(env, "NVIDIA AprilTag detection failed"); + return nullptr; + } + + auto detectionConstructor = + env->GetMethodID(detectionClass, "", "(IIDD[D)V"); + if (detectionConstructor == nullptr) { + return nullptr; + } + + auto output = env->NewObjectArray(static_cast(numTags), detectionClass, + nullptr); + for (uint32_t i = 0; i < numTags; i++) { + const auto& detection = tags[i]; + double centerX = 0.0; + double centerY = 0.0; + ComputeCenter(detection, ¢erX, ¢erY); + + jdoubleArray corners = env->NewDoubleArray(8); + double cornersData[8]; + for (int cornerIdx = 0; cornerIdx < 4; cornerIdx++) { + cornersData[cornerIdx * 2] = detection.corners[cornerIdx].x; + cornersData[cornerIdx * 2 + 1] = detection.corners[cornerIdx].y; + } + env->SetDoubleArrayRegion(corners, 0, 8, cornersData); + + auto detectionObject = + env->NewObject(detectionClass, detectionConstructor, + static_cast(detection.id), + static_cast(detection.hamming_error), centerX, + centerY, corners); + env->SetObjectArrayElement(output, static_cast(i), detectionObject); + env->DeleteLocalRef(corners); + env->DeleteLocalRef(detectionObject); + } + + return output; +} + +} // extern "C" diff --git a/scripts/generatePiImage.sh b/scripts/generatePiImage.sh index 50cea0acb0..0a05639cb1 100755 --- a/scripts/generatePiImage.sh +++ b/scripts/generatePiImage.sh @@ -1,11 +1,19 @@ if [ "$#" -ne 2 ]; then echo "Illegal number of parameters -- expected (Image release URL) (image suffix)" + echo "Optional environment overrides:" + echo " PV_ROOT_PARTITION Root filesystem partition number (default: 2)" + echo " PV_PHOTON_DIR Photon install directory inside the image (default: /opt/photonvision)" + echo " PV_SYSTEMD_UNIT_DIR systemd wants directory inside the image (default: /etc/systemd/system/multi-user.target.wants)" exit 1 fi # 1st arg should be the release to download the image template from. The release ought to only have one # artifact for a "xz" image. +ROOT_PARTITION="${PV_ROOT_PARTITION:-2}" +PHOTON_DIR="${PV_PHOTON_DIR:-/opt/photonvision}" +SYSTEMD_UNIT_DIR="${PV_SYSTEMD_UNIT_DIR:-/etc/systemd/system/multi-user.target.wants}" + NEW_JAR=$(realpath $(find . -name photonvision\*-linuxarm64.jar)) echo "Using jar: " $NEW_JAR echo "Downloading image from" $1 @@ -35,21 +43,21 @@ echo "Unzipped image: " $IMAGE_FILE " -- mounting" TMP=$(mktemp -d) LOOP=$(sudo losetup --show -fP "${IMAGE_FILE}") echo "Image mounted! Copying jar..." -sudo mount ${LOOP}p2 $TMP +sudo mount ${LOOP}p${ROOT_PARTITION} $TMP pushd . -cd $TMP/opt/photonvision +cd "$TMP$PHOTON_DIR" sudo cp $NEW_JAR photonvision.jar echo "Jar updated! Creating service..." -cd $TMP/etc/systemd/system/multi-user.target.wants +cd "$TMP$SYSTEMD_UNIT_DIR" sudo bash -c "printf \ \"[Unit] Description=Service that runs PhotonVision [Service] -WorkingDirectory=/opt/photonvision -ExecStart=/usr/bin/java -Xmx512m -jar /opt/photonvision/photonvision.jar +WorkingDirectory=${PHOTON_DIR} +ExecStart=/usr/bin/java -Xmx512m -jar ${PHOTON_DIR}/photonvision.jar ExecStop=/bin/systemctl kill photonvision Type=simple Restart=on-failure