ignore

.vscode/settings.json

@@ -2,5 +2,18 @@
     "python.testing.unittestEnabled": false,
     "python.testing.pytestEnabled": true,
     "python.testing.cwd": "photon-lib/py",
-    "java.configuration.updateBuildConfiguration": "automatic"
+    "java.configuration.updateBuildConfiguration": "automatic",
+    "ROS2.distro": "humble",
+    "python.autoComplete.extraPaths": [
+        "/opt/ros/humble/lib/python3.10/site-packages",
+        "/opt/ros/humble/local/lib/python3.10/dist-packages",
+        "/home/kevin/Robot/AdaptiGraph/PyFleX/bindings/build",
+        ""
+    ],
+    "python.analysis.extraPaths": [
+        "/opt/ros/humble/lib/python3.10/site-packages",
+        "/opt/ros/humble/local/lib/python3.10/dist-packages",
+        "/home/kevin/Robot/AdaptiGraph/PyFleX/bindings/build",
+        ""
+    ]
 }

README.md

@@ -25,8 +25,71 @@ If you are interested in contributing code or documentation to the project, plea
 
 Gradle is used for all C++ and Java code, and pnpm is used for the web UI. Instructions to compile PhotonVision yourself can be found [in our docs](https://docs.photonvision.org/en/latest/docs/contributing/building-photon.html#compiling-instructions).
 
+This repo currently requires JDK 17 and Node 22+ for local builds.
+
 You can run one of the many built in examples straight from the command line, too! They contain a fully featured robot project, and some include simulation support. The projects can be found inside the [`photonlib-java-examples`](photonlib-java-examples) and [`photonlib-cpp-examples`](photonlib-cpp-examples) subdirectories, respectively. Instructions for running these examples directly from the repo are found [in the docs](https://docs.photonvision.org/en/latest/docs/contributing/building-photon.html#running-examples).
 
+### Optional NVIDIA AprilTag Backend
+
+PhotonVision can optionally build an NVIDIA CUDA AprilTag detector backend for supported Linux hosts. This path is opt-in and falls back to the existing CPU detector when the JNI library, SDK, or runtime support is unavailable.
+
+Build and run with the optional backend enabled:
+
+```bash
+export JAVA_HOME=/path/to/jdk-17
+export PATH="$JAVA_HOME/bin:/path/to/node-v22/bin:$PATH"
+export NVIDIA_APRILTAG_SDK_ROOT=/path/to/cuapriltags-root
+export CUDA_HOME=/path/to/cuda
+
+./gradlew :photon-targeting:nvidiaapriltagJNILinuxx86-64ReleaseSharedLibrary \
+  :photon-server:run \
+  -PenableNvidiaAprilTag \
+  -Dphotonvision.apriltag.backend=auto
+```
+
+Backend selection controls:
+
+- `-PenableNvidiaAprilTag` enables the optional CUDA JNI build.
+- `NVIDIA_APRILTAG_SDK_ROOT` must contain `libcuapriltags.a`.
+- `CUDA_HOME` must contain a compatible CUDA runtime, usually under `lib64/libcudart.so`.
+- `-Dphotonvision.apriltag.backend=auto|cpu|nvidia` controls runtime backend selection for testing.
+
+The current implementation only enables the NVIDIA path for `tag36h11`. Other families continue to use the existing CPU detector.
+
+### Generating Device Images
+
+PhotonVision release artifacts for coprocessors are board images such as `.img.xz` or `.tar.xz`. This repo does not generate a bootable desktop ISO.
+
+To build a Linux ARM64 jar and inject it into a base PhotonVision device image locally:
+
+```bash
+./gradlew :photon-server:shadowJar -PArchOverride=linuxarm64
+./scripts/generatePiImage.sh <base-image-url> <image-suffix>
+```
+
+Example for a Raspberry Pi image:
+
+```bash
+./gradlew :photon-server:shadowJar -PArchOverride=linuxarm64
+./scripts/generatePiImage.sh \
+  https://github.com/PhotonVision/photon-image-modifier/releases/download/<image-version>/photonvision_raspi.img.xz \
+  RaspberryPi
+```
+
+This script downloads the base image, mounts the root partition, replaces `/opt/photonvision/photonvision.jar`, recreates the `photonvision.service` unit, and recompresses the image next to the built jar.
+
+Example for a custom Jetson-style image:
+
+```bash
+./gradlew :photon-server:shadowJar -PArchOverride=linuxarm64
+PV_ROOT_PARTITION=1 \
+./scripts/generatePiImage.sh \
+  https://example.com/your-jetson-photon-base.img.xz \
+  JetsonOrinNano
+```
+
+This Jetson example assumes the root filesystem is on partition `1`. If your image uses a different layout, set `PV_ROOT_PARTITION` accordingly. The helper also supports `PV_PHOTON_DIR` and `PV_SYSTEMD_UNIT_DIR` when the target image uses different install paths.
+
 ## Gradle Arguments
 
 Note that these are case sensitive!
@@ -44,6 +107,7 @@ Note that these are case sensitive!
 - `-PtgtPw`: Specifies custom password for `./gradlew deploy` to SSH into
 - `-Pprofile`: enables JVM profiling
 - `-PwithSanitizers`: On Linux, enables `-fsanitize=address,undefined,leak`
+- `-PenableNvidiaAprilTag`: builds the optional NVIDIA CUDA AprilTag JNI when the SDK is available
 
 If you're cross-compiling, you'll need the WPILib toolchain installed. This must be done via Gradle: for example `./gradlew installArm64Toolchain` or `./gradlew installRoboRioToolchain`
 

docs/source/docs/contributing/building-photon.md

@@ -101,6 +101,40 @@ Running the following command under the root directory will build the jar under
       ``gradlew shadowJar``
 ```
 
+### Build and Run PhotonVision with the Optional NVIDIA AprilTag Backend
+
+The CUDA AprilTag backend is optional and only builds when `-PenableNvidiaAprilTag` is set and the required SDK pieces are present. Runtime selection still falls back to the existing CPU detector when the NVIDIA path cannot be used.
+
+Current requirements for the optional backend:
+
+- Linux host with a supported NVIDIA GPU and driver
+- JDK 17
+- Node 22+
+- `NVIDIA_APRILTAG_SDK_ROOT` pointing at a directory that contains `libcuapriltags.a`
+- `CUDA_HOME` pointing at a CUDA install that contains `lib64/libcudart.so`
+
+Example build and run flow on Linux x86_64:
+
+```bash
+export JAVA_HOME=/path/to/jdk-17
+export PATH="$JAVA_HOME/bin:/path/to/node-v22/bin:$PATH"
+export NVIDIA_APRILTAG_SDK_ROOT=/path/to/cuapriltags-root
+export CUDA_HOME=/path/to/cuda
+
+./gradlew :photon-targeting:nvidiaapriltagJNILinuxx86-64ReleaseSharedLibrary \
+  :photon-server:run \
+  -PenableNvidiaAprilTag \
+  -Dphotonvision.apriltag.backend=auto
+```
+
+Useful backend selection modes:
+
+- `-Dphotonvision.apriltag.backend=auto`: prefer NVIDIA when supported, otherwise fall back to CPU
+- `-Dphotonvision.apriltag.backend=cpu`: force the existing CPU path
+- `-Dphotonvision.apriltag.backend=nvidia`: request the CUDA path and log a fallback reason if it cannot be used
+
+The current NVIDIA implementation only applies to `tag36h11`. Other tag families continue to run on the CPU detector.
+
 ### Build and Run PhotonVision on a Raspberry Pi Coprocessor
 
 As a convenience, the build has a built-in `deploy` command which builds, deploys, and starts the current source code on a coprocessor. It uses [deploy-utils](https://github.com/wpilibsuite/deploy-utils/blob/main/README.md), so it works very similarly to deploys on robot projects.
@@ -134,6 +168,67 @@ An architecture override is required to specify the deploy target's architecture
 
 The `deploy` command is tested against Raspberry Pi coprocessors. Other similar coprocessors may work too.
 
+### Generate a Device Image Locally
+
+PhotonVision coprocessor releases are delivered as board images such as `.img.xz` or `.tar.xz`. This repo does not produce a desktop installer ISO.
+
+If you want to inject a locally built Linux ARM64 jar into an existing PhotonVision base image, use the helper script in `scripts/generatePiImage.sh`.
+
+1. Build a Linux ARM64 jar:
+
+```bash
+./gradlew :photon-server:shadowJar -PArchOverride=linuxarm64
+```
+
+2. Download and repack a board image by passing the base image URL and the desired artifact suffix:
+
+```bash
+./scripts/generatePiImage.sh <base-image-url> <image-suffix>
+```
+
+Example for Raspberry Pi:
+
+```bash
+./scripts/generatePiImage.sh \
+  https://github.com/PhotonVision/photon-image-modifier/releases/download/<image-version>/photonvision_raspi.img.xz \
+  RaspberryPi
+```
+
+What the script does:
+
+- downloads the base `.img.xz`
+- decompresses it
+- mounts the root filesystem with `losetup`
+- replaces `/opt/photonvision/photonvision.jar`
+- rewrites the `photonvision.service` systemd unit
+- recompresses the result as `photonvision*-image_<suffix>.xz`
+
+This flow requires a Linux host with `sudo`, loop-device support, `wget`, and `xz-utils`.
+
+Example for a custom Jetson-style image:
+
+```bash
+./gradlew :photon-server:shadowJar -PArchOverride=linuxarm64
+PV_ROOT_PARTITION=1 \
+./scripts/generatePiImage.sh \
+  https://example.com/your-jetson-photon-base.img.xz \
+  JetsonOrinNano
+```
+
+The Jetson example above assumes:
+
+- the base image is already a PhotonVision-ready Linux ARM64 image
+- the root filesystem is on partition `1`
+- PhotonVision should live at `/opt/photonvision`
+
+If your Jetson image uses different paths, the helper also supports:
+
+- `PV_ROOT_PARTITION`
+- `PV_PHOTON_DIR`
+- `PV_SYSTEMD_UNIT_DIR`
+
+This repo does not currently publish an official Jetson base image in CI, so you must provide the base `.img.xz` yourself.
+
 ### Using PhotonLib Builds
 
 The build process automatically generates a vendordep JSON of your local build at `photon-lib/build/generated/vendordeps/photonlib.json`.

docs/source/docs/quick-start/networking.md

@@ -94,7 +94,7 @@ The address in the code above (`photonvision.local`) is the hostname of the copr
 
 ## Camera Stream Ports
 
-The camera streams start at 1181 with two ports for each camera (ex. 1181 and 1182 for camera one, 1183 and 1184 for camera two, etc.). The easiest way to identify the port of the camera that you want is by double clicking on the stream, which opens it in a separate page. The port will be listed below the stream.
+The camera streams start at 1181 with two ports for each camera (for example, 1181 and 1182 for camera one, 1183 and 1184 for camera two, etc.). The current stream index allocation supports up to 16 cameras, so the last default pair is 1211 and 1212. The easiest way to identify the port of the camera that you want is by double clicking on the stream, which opens it in a separate page. The port will be listed below the stream.
 
 :::{warning}
 If your camera stream isn't sent to the same port as it's originally found on, its stream will not be visible in the UI.

photon-core/src/main/java/org/photonvision/common/LoadJNI.java

@@ -28,6 +28,7 @@ public class LoadJNI {
     public enum JNITypes {
         RUBIK_DETECTOR("tensorflowlite", "tensorflowlite_c", "external_delegate", "rubik_jni"),
         RKNN_DETECTOR("rga", "rknnrt", "rknn_jni"),
+        NVIDIA_APRILTAG("nvidiaapriltagJNI"),
         MRCAL("mrcal_jni"),
         LIBCAMERA("photonlibcamera");
 
@@ -45,7 +46,11 @@ public static synchronized void forceLoad(JNITypes type) throws IOException {
             return;
         }
 
-        CombinedRuntimeLoader.loadLibraries(LoadJNI.class, type.libraries);
+        if (type == JNITypes.NVIDIA_APRILTAG) {
+            LibraryLoader.loadNvidiaAprilTag();
+        } else {
+            CombinedRuntimeLoader.loadLibraries(LoadJNI.class, type.libraries);
+        }
         loadedMap.put(type, true);
     }
 

photon-core/src/main/java/org/photonvision/common/configuration/CameraConfiguration.java

@@ -60,7 +60,7 @@ public class CameraConfiguration {
     public List<CameraCalibrationCoefficients> calibrations = new ArrayList<>();
     public int currentPipelineIndex = 0;
 
-    public int streamIndex = 0; // 0 index means ports [1181, 1182], 1 means [1183, 1184], etc...
+    public int streamIndex = 0; // 0 index means ports [1181, 1182], 1 means [1183, 1184], etc.
 
     // Ignore the pipes, as we serialize them to their own column to hack around
     // polymorphic lists

photon-core/src/main/java/org/photonvision/common/dataflow/websocket/UIPhotonConfiguration.java

@@ -18,6 +18,7 @@
 package org.photonvision.common.dataflow.websocket;
 
 import java.util.List;
+import java.util.stream.Stream;
 import org.photonvision.PhotonVersion;
 import org.photonvision.common.LoadJNI;
 import org.photonvision.common.LoadJNI.JNITypes;
@@ -28,6 +29,7 @@
 import org.photonvision.common.hardware.Platform;
 import org.photonvision.common.networking.NetworkManager;
 import org.photonvision.common.networking.NetworkUtils;
+import org.photonvision.vision.apriltag.AprilTagBackendManager;
 import org.photonvision.vision.processes.VisionModule;
 import org.photonvision.vision.processes.VisionSourceManager;
 
@@ -56,8 +58,13 @@ public static UIPhotonConfiguration programStateToUi(PhotonConfiguration c) {
                                 OsImageData.IMAGE_METADATA.isPresent()
                                         ? OsImageData.IMAGE_METADATA.get().commitTag()
                                         : "",
-                                // TODO add support for other types of GPU accel
-                                LoadJNI.hasLoaded(JNITypes.LIBCAMERA) ? "Zerocopy Libcamera Working" : "",
+                                Stream.of(
+                                                LoadJNI.hasLoaded(JNITypes.LIBCAMERA)
+                                                        ? "Zerocopy Libcamera Working"
+                                                        : "",
+                                                AprilTagBackendManager.getUiStatus())
+                                        .filter(s -> s != null && !s.isBlank())
+                                        .collect(java.util.stream.Collectors.joining("; ")),
                                 LoadJNI.hasLoaded(JNITypes.MRCAL),
                                 c.neuralNetworkPropertyManager().getModels(),
                                 NeuralNetworkModelManager.getInstance().getSupportedBackends(),