From a2c9586752e9ea731424f8636762a1833365c7ee Mon Sep 17 00:00:00 2001
From: Wild Me <jason@wildme.org>
Date: Fri, 27 Mar 2026 15:01:42 -0700
Subject: [PATCH 1/9] Switch to temp file model to avoid timeouts

---
 .../export/EncounterCOCOExportFile.java       | 121 ++++++++++--------
 .../export/EncounterSearchExportCOCO.java     |  52 +++++---
 .../export/EncounterCOCOExportFileTest.java   |  45 ++-----
 3 files changed, 112 insertions(+), 106 deletions(-)

diff --git a/src/main/java/org/ecocean/export/EncounterCOCOExportFile.java b/src/main/java/org/ecocean/export/EncounterCOCOExportFile.java
index d931e079ac..c83f81714e 100644
--- a/src/main/java/org/ecocean/export/EncounterCOCOExportFile.java
+++ b/src/main/java/org/ecocean/export/EncounterCOCOExportFile.java
@@ -52,50 +52,13 @@ public void writeTo(OutputStream outputStream) throws IOException {
             mediaAssetToImageId.put(uuid, imageId++);
         }
 
-        // Build JSON arrays
-        JSONArray imagesArray = new JSONArray();
-        for (Map.Entry<String, MediaAsset> entry : mediaAssetMap.entrySet()) {
-            MediaAsset ma = entry.getValue();
-            int imgId = mediaAssetToImageId.get(entry.getKey());
-            imagesArray.put(buildImageObject(ma, imgId));
-        }
-
-        JSONArray annotationsArray = new JSONArray();
-        int annotationId = 1;
-        for (Encounter enc : encounters) {
-            if (enc.getAnnotations() == null) continue;
-            for (Annotation ann : enc.getAnnotations()) {
-                if (!isValidAnnotation(ann)) continue;
-                MediaAsset ma = ann.getMediaAsset();
-                int imgId = mediaAssetToImageId.get(ma.getUUID());
-                annotationsArray.put(buildAnnotationObject(ann, annotationId++, imgId,
-                    categoryMap, individualIdMap, enc));
-            }
-        }
-
-        // Build complete COCO JSON
-        JSONObject coco = new JSONObject();
-        coco.put("info", buildInfo(individualIdMap));
-        coco.put("licenses", buildLicenses());
-        coco.put("categories", buildCategories(categoryMap));
-        coco.put("images", imagesArray);
-        coco.put("annotations", annotationsArray);
-
-        // Write ZIP
+        // Write ZIP: images first, then JSON manifest so it only references
+        // images that were actually written successfully.
         try (ZipOutputStream zipOut = new ZipOutputStream(outputStream)) {
-            // Write annotations.json
-            log.info("COCO Export: Writing annotations JSON...");
-            byte[] jsonBytes = coco.toString(2).getBytes(StandardCharsets.UTF_8);
-            ZipEntry jsonEntry = new ZipEntry("coco/annotations/instances.json");
-            zipOut.putNextEntry(jsonEntry);
-            zipOut.write(jsonBytes);
-            zipOut.closeEntry();
-            zipOut.flush();
-
-            // Write images with streaming to minimize memory usage
             int totalImages = mediaAssetMap.size();
             int processedImages = 0;
             int failedImages = 0;
+            Set<String> exportedUuids = new LinkedHashSet<>();
             log.info("COCO Export: Starting export of " + totalImages + " images...");
 
             for (Map.Entry<String, MediaAsset> entry : mediaAssetMap.entrySet()) {
@@ -103,22 +66,61 @@ public void writeTo(OutputStream outputStream) throws IOException {
                 processedImages++;
                 try {
                     boolean success = writeImageToZip(ma, zipOut);
-                    if (!success) {
+                    if (success) {
+                        exportedUuids.add(entry.getKey());
+                    } else {
                         failedImages++;
                     }
-                    // Progress logging every 100 images
                     if (processedImages % 100 == 0) {
                         log.info("COCO Export: Processed " + processedImages + "/" + totalImages +
                                  " images (" + failedImages + " failed)");
                     }
                 } catch (Exception e) {
                     failedImages++;
-                    log.warning("COCO Export: Failed to export image " + ma.getUUID() + ": " + e.getMessage());
+                    log.warning("COCO Export: Failed to export image " + ma.getUUID() +
+                        ": " + e.getMessage());
+                }
+            }
+
+            // Build JSON arrays using only successfully exported images
+            JSONArray imagesArray = new JSONArray();
+            for (String uuid : exportedUuids) {
+                MediaAsset ma = mediaAssetMap.get(uuid);
+                int imgId = mediaAssetToImageId.get(uuid);
+                imagesArray.put(buildImageObject(ma, imgId));
+            }
+
+            JSONArray annotationsArray = new JSONArray();
+            int annotationId = 1;
+            for (Encounter enc : encounters) {
+                if (enc.getAnnotations() == null) continue;
+                for (Annotation ann : enc.getAnnotations()) {
+                    if (!isValidAnnotation(ann)) continue;
+                    MediaAsset ma = ann.getMediaAsset();
+                    if (!exportedUuids.contains(ma.getUUID())) continue;
+                    int imgId = mediaAssetToImageId.get(ma.getUUID());
+                    annotationsArray.put(buildAnnotationObject(ann, annotationId++, imgId,
+                        categoryMap, individualIdMap, enc));
                 }
             }
 
+            JSONObject coco = new JSONObject();
+            coco.put("info", buildInfo(individualIdMap));
+            coco.put("licenses", buildLicenses());
+            coco.put("categories", buildCategories(categoryMap));
+            coco.put("images", imagesArray);
+            coco.put("annotations", annotationsArray);
+
+            // Write annotations.json as the last entry
+            log.info("COCO Export: Writing annotations JSON...");
+            byte[] jsonBytes = coco.toString(2).getBytes(StandardCharsets.UTF_8);
+            ZipEntry jsonEntry = new ZipEntry("coco/annotations/instances.json");
+            zipOut.putNextEntry(jsonEntry);
+            zipOut.write(jsonBytes);
+            zipOut.closeEntry();
+
             zipOut.finish();
-            log.info("COCO Export: Completed. " + (processedImages - failedImages) + "/" + totalImages +
+            log.info("COCO Export: Completed. " + exportedUuids.size() + "/" + totalImages +
                      " images exported successfully, " + failedImages + " failed.");
         }
     }
@@ -162,23 +164,19 @@ private boolean writeImageToZip(MediaAsset ma, ZipOutputStream zipOut) throws IO
      * Detects the content type of a URL, handling redirects.
      */
     private String detectContentType(URL url) {
+        HttpURLConnection httpConn = null;
         try {
             URLConnection conn = url.openConnection();
             conn.setConnectTimeout(CONNECTION_TIMEOUT_MS);
             conn.setReadTimeout(READ_TIMEOUT_MS);
 
             if (conn instanceof HttpURLConnection) {
-                ((HttpURLConnection) conn).setRequestMethod("HEAD");
-                ((HttpURLConnection) conn).setInstanceFollowRedirects(true);
-            }
-
-            String contentType = conn.getContentType();
-
-            if (conn instanceof HttpURLConnection) {
-                ((HttpURLConnection) conn).disconnect();
+                httpConn = (HttpURLConnection) conn;
+                httpConn.setRequestMethod("HEAD");
+                httpConn.setInstanceFollowRedirects(true);
             }
 
-            return contentType;
+            return conn.getContentType();
         } catch (Exception e) {
             // Fall back to checking file extension
             String path = url.getPath().toLowerCase();
@@ -186,6 +184,10 @@ private String detectContentType(URL url) {
                 return "image/jpeg";
             }
             return null;
+        } finally {
+            if (httpConn != null) {
+                httpConn.disconnect();
+            }
         }
     }
 
@@ -247,12 +249,19 @@ private void convertAndWriteImage(URL url, ZipOutputStream zipOut) throws IOExce
             if (image.getColorModel().hasAlpha()) {
                 BufferedImage rgbImage = new BufferedImage(
                     image.getWidth(), image.getHeight(), BufferedImage.TYPE_INT_RGB);
-                rgbImage.createGraphics().drawImage(image, 0, 0, java.awt.Color.WHITE, null);
+                java.awt.Graphics2D g2d = rgbImage.createGraphics();
+                try {
+                    g2d.drawImage(image, 0, 0, java.awt.Color.WHITE, null);
+                } finally {
+                    g2d.dispose();
+                }
                 image = rgbImage;
             }
 
-            // Write directly to zip stream
-            ImageIO.write(image, "jpg", zipOut);
+            // Buffer the JPEG output to avoid ImageIO closing the ZipOutputStream
+            java.io.ByteArrayOutputStream imgBuf = new java.io.ByteArrayOutputStream();
+            ImageIO.write(image, "jpg", imgBuf);
+            zipOut.write(imgBuf.toByteArray());
         } finally {
             if (conn instanceof HttpURLConnection) {
                 ((HttpURLConnection) conn).disconnect();
diff --git a/src/main/java/org/ecocean/servlet/export/EncounterSearchExportCOCO.java b/src/main/java/org/ecocean/servlet/export/EncounterSearchExportCOCO.java
index 913a3055c5..ca3d09f19e 100644
--- a/src/main/java/org/ecocean/servlet/export/EncounterSearchExportCOCO.java
+++ b/src/main/java/org/ecocean/servlet/export/EncounterSearchExportCOCO.java
@@ -14,9 +14,12 @@
 import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
+import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.io.PrintWriter;
+import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Vector;
@@ -41,16 +44,14 @@ public void doPost(HttpServletRequest request, HttpServletResponse response)
 
         myShepherd.beginDBTransaction();
 
+        File tempFile = null;
         try {
-            // Process query
+            // Query and filter encounters (requires open DB transaction)
             EncounterQueryResult queryResult = EncounterQueryProcessor.processQuery(
                 myShepherd, request, "year descending, month descending, day descending");
             Vector<?> rEncounters = queryResult.getResult();
 
-            // Filter hidden encounters
             HiddenEncReporter hiddenData = new HiddenEncReporter(rEncounters, request, myShepherd);
-
-            // Convert to list, excluding hidden
             List<Encounter> encounters = new ArrayList<>();
             for (Object obj : rEncounters) {
                 Encounter enc = (Encounter) obj;
@@ -59,29 +60,44 @@ public void doPost(HttpServletRequest request, HttpServletResponse response)
                 }
             }
 
-            // Set response headers
-            response.setContentType("application/zip");
-            response.setHeader("Content-Disposition", "attachment; filename=\"wildbook-coco-export.zip\"");
+            // Build ZIP to temp file so we can detect errors before committing
+            // the HTTP response and set an accurate Content-Length.
+            File tmpDir = new File(CommonConfiguration.getUploadTmpDir(context));
+            if (!tmpDir.exists()) tmpDir.mkdirs();
+            tempFile = File.createTempFile("wildbook-coco-export-", ".zip", tmpDir);
+            tempFile.deleteOnExit();
+            EncounterCOCOExportFile exportFile = new EncounterCOCOExportFile(encounters, myShepherd);
+            try (FileOutputStream fos = new FileOutputStream(tempFile)) {
+                exportFile.writeTo(fos);
+            }
 
-            // Write export
+            // Stream complete file to client
+            response.setContentType("application/zip");
+            response.setHeader("Content-Disposition",
+                "attachment; filename=\"wildbook-coco-export.zip\"");
+            response.setContentLengthLong(tempFile.length());
             OutputStream out = response.getOutputStream();
-            EncounterCOCOExportFile exportFile = new EncounterCOCOExportFile(encounters, myShepherd);
-            exportFile.writeTo(out);
+            Files.copy(tempFile.toPath(), out);
             out.flush();
 
         } catch (Exception e) {
             e.printStackTrace();
-            response.setContentType("text/html");
-            PrintWriter out = response.getWriter();
-            out.println(ServletUtilities.getHeader(request));
-            out.println("<html><body><p><strong>Error encountered</strong></p>");
-            out.println("<p>Error: " + e.getMessage() + "</p>");
-            out.println("<p>Please let the webmaster know you encountered an error at: EncounterSearchExportCOCO servlet</p></body></html>");
-            out.println(ServletUtilities.getFooter(context));
-            out.close();
+            if (!response.isCommitted()) {
+                response.setContentType("text/html");
+                PrintWriter out = response.getWriter();
+                out.println(ServletUtilities.getHeader(request));
+                out.println("<html><body><p><strong>Error encountered</strong></p>");
+                out.println("<p>Error: " + e.getMessage() + "</p>");
+                out.println("<p>Please let the webmaster know you encountered an error at: EncounterSearchExportCOCO servlet</p></body></html>");
+                out.println(ServletUtilities.getFooter(context));
+                out.close();
+            }
         } finally {
             myShepherd.rollbackDBTransaction();
             myShepherd.closeDBTransaction();
+            if (tempFile != null) {
+                tempFile.delete();
+            }
         }
     }
 }
diff --git a/src/test/java/org/ecocean/export/EncounterCOCOExportFileTest.java b/src/test/java/org/ecocean/export/EncounterCOCOExportFileTest.java
index 97ec570347..b8b119b7a2 100644
--- a/src/test/java/org/ecocean/export/EncounterCOCOExportFileTest.java
+++ b/src/test/java/org/ecocean/export/EncounterCOCOExportFileTest.java
@@ -24,28 +24,17 @@
 class EncounterCOCOExportFileTest {
 
     @Test
-    void testBuildsCOCOStructure() throws Exception {
-        // Create mock objects
+    void testManifestExcludesFailedImages() throws Exception {
+        // When an image URL is null (download fails), the manifest should NOT
+        // reference that image or its annotations — ensuring consistency.
         Shepherd shepherd = mock(Shepherd.class);
 
-        // Create a mock MediaAsset
         MediaAsset ma = mock(MediaAsset.class);
         when(ma.getUUID()).thenReturn("test-media-uuid");
         when(ma.getWidth()).thenReturn(800.0);
         when(ma.getHeight()).thenReturn(600.0);
-        when(ma.webURL()).thenReturn(null); // Skip actual image download in test
-
-        // Create a mock Feature with bbox
-        Feature feature = mock(Feature.class);
-        JSONObject params = new JSONObject();
-        params.put("x", 100);
-        params.put("y", 200);
-        params.put("width", 300);
-        params.put("height", 400);
-        params.put("theta", 0.5);
-        when(feature.getParameters()).thenReturn(params);
-
-        // Create a mock Annotation
+        when(ma.webURL()).thenReturn(null); // image cannot be fetched
+
         Annotation ann = mock(Annotation.class);
         when(ann.getId()).thenReturn("test-ann-uuid");
         when(ann.getIAClass()).thenReturn("whale_shark");
@@ -54,12 +43,10 @@ void testBuildsCOCOStructure() throws Exception {
         when(ann.getViewpoint()).thenReturn("left");
         when(ann.getTheta()).thenReturn(0.5);
 
-        // Create a mock MarkedIndividual
         MarkedIndividual ind = mock(MarkedIndividual.class);
         when(ind.getId()).thenReturn("test-individual-uuid");
         when(ind.getDisplayName()).thenReturn("Stumpy");
 
-        // Create a mock Encounter
         Encounter enc = mock(Encounter.class);
         ArrayList<Annotation> annotations = new ArrayList<>();
         annotations.add(ann);
@@ -69,19 +56,16 @@ void testBuildsCOCOStructure() throws Exception {
         List<Encounter> encounters = new ArrayList<>();
         encounters.add(enc);
 
-        // Run export
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
         EncounterCOCOExportFile exportFile = new EncounterCOCOExportFile(encounters, shepherd);
         exportFile.writeTo(baos);
 
-        // Parse ZIP and extract annotations.json
         byte[] zipBytes = baos.toByteArray();
         assertTrue(zipBytes.length > 0, "Export should produce output");
 
         String jsonContent = extractJsonFromZip(zipBytes);
         assertNotNull(jsonContent, "Should contain annotations.json");
 
-        // Verify JSON structure
         JSONObject coco = new JSONObject(jsonContent);
         assertTrue(coco.has("info"));
         assertTrue(coco.has("licenses"));
@@ -89,21 +73,18 @@ void testBuildsCOCOStructure() throws Exception {
         assertTrue(coco.has("images"));
         assertTrue(coco.has("annotations"));
 
-        // Verify categories
+        // Image failed to export, so both images and annotations arrays should be empty
+        assertEquals(0, coco.getJSONArray("images").length(),
+            "Failed images should be excluded from manifest");
+        assertEquals(0, coco.getJSONArray("annotations").length(),
+            "Annotations for failed images should be excluded from manifest");
+
+        // Categories are built from encounter data, independent of image success
         JSONArray categories = coco.getJSONArray("categories");
         assertEquals(1, categories.length());
         assertEquals("whale_shark", categories.getJSONObject(0).getString("name"));
 
-        // Verify annotations
-        JSONArray anns = coco.getJSONArray("annotations");
-        assertEquals(1, anns.length());
-        JSONObject annJson = anns.getJSONObject(0);
-        assertEquals("left", annJson.getString("viewpoint"));
-        assertEquals(0.5, annJson.getDouble("theta"), 0.001);
-        assertEquals("test-individual-uuid", annJson.getString("individual_uuid"));
-        assertEquals("Stumpy", annJson.getString("name"));
-
-        // Verify individual_id_mapping in info
+        // Individual mapping is also independent of image success
         JSONObject info = coco.getJSONObject("info");
         assertTrue(info.has("individual_id_mapping"));
         JSONObject mapping = info.getJSONObject("individual_id_mapping");

From dd644c80e2b3af81cb501da24b1faf8f2aa030f1 Mon Sep 17 00:00:00 2001
From: Wild Me <jason@wildme.org>
Date: Mon, 6 Apr 2026 12:03:29 -0700
Subject: [PATCH 2/9] Update with Codex code review and changed architecture

---
 .claude/settings.local.json                   |  10 +
 .../SearchPages/components/ExportModal.jsx    | 100 +++++++-
 .../export/EncounterCOCOExportFile.java       |  25 +-
 .../export/EncounterSearchExportCOCO.java     | 213 ++++++++++++++++--
 4 files changed, 316 insertions(+), 32 deletions(-)
 create mode 100644 .claude/settings.local.json

diff --git a/.claude/settings.local.json b/.claude/settings.local.json
new file mode 100644
index 0000000000..d55bcc6706
--- /dev/null
+++ b/.claude/settings.local.json
@@ -0,0 +1,10 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(git log:*)",
+      "Bash(gh pr:*)",
+      "Bash(mvn compile:*)",
+      "Bash(mvn test:*)"
+    ]
+  }
+}
diff --git a/frontend/src/pages/SearchPages/components/ExportModal.jsx b/frontend/src/pages/SearchPages/components/ExportModal.jsx
index 6acc928e49..edfb5d0501 100644
--- a/frontend/src/pages/SearchPages/components/ExportModal.jsx
+++ b/frontend/src/pages/SearchPages/components/ExportModal.jsx
@@ -11,7 +11,7 @@ import {
   Spinner,
   Alert,
 } from "react-bootstrap";
-import { useState } from "react";
+import { useState, useRef, useCallback, useEffect } from "react";
 import { FormattedMessage } from "react-intl";
 
 const downloadFunction = async (url, setLoading) => {
@@ -75,6 +75,18 @@ export default function ExportDialog({ open, setOpen, searchQueryId }) {
   });
 
   const [error, setError] = useState(null);
+  const [cocoProgress, setCocoProgress] = useState(null);
+  const cocoPollingRef = useRef(null);
+
+  // Clean up polling interval on unmount (e.g., modal close)
+  useEffect(() => {
+    return () => {
+      if (cocoPollingRef.current) {
+        clearInterval(cocoPollingRef.current);
+        cocoPollingRef.current = null;
+      }
+    };
+  }, []);
 
   const setLoading = (key, value) => {
     setLoadingStates((prev) => ({ ...prev, [key]: value }));
@@ -90,6 +102,78 @@ export default function ExportDialog({ open, setOpen, searchQueryId }) {
     }
   };
 
+  const handleCocoExport = useCallback(async () => {
+    setError(null);
+    setLoading("cocoFormat", true);
+    setCocoProgress(null);
+
+    try {
+      // Start the async export job
+      const startUrl = `/EncounterSearchExportCOCO?action=start&searchQueryId=${searchQueryId}&regularQuery=true`;
+      const startResp = await fetch(startUrl);
+      const startData = await startResp.json();
+      if (!startResp.ok || !startData.jobId) {
+        throw new Error(startData.error || "Failed to start export");
+      }
+      const { jobId } = startData;
+
+      // Poll for progress
+      const result = await new Promise((resolve, reject) => {
+        cocoPollingRef.current = setInterval(async () => {
+          try {
+            const statusResp = await fetch(
+              `/EncounterSearchExportCOCO?action=status&jobId=${jobId}`,
+            );
+            const status = await statusResp.json();
+
+            if (status.totalImages > 0) {
+              setCocoProgress(status);
+            }
+
+            if (status.status === "complete") {
+              clearInterval(cocoPollingRef.current);
+              cocoPollingRef.current = null;
+              resolve(jobId);
+            } else if (status.status === "error") {
+              clearInterval(cocoPollingRef.current);
+              cocoPollingRef.current = null;
+              reject(new Error(status.error || "Export failed"));
+            }
+          } catch (e) {
+            clearInterval(cocoPollingRef.current);
+            cocoPollingRef.current = null;
+            reject(e);
+          }
+        }, 3000);
+      });
+
+      // Download the completed file
+      const downloadResp = await fetch(
+        `/EncounterSearchExportCOCO?action=download&jobId=${result}`,
+      );
+      if (!downloadResp.ok) throw new Error("Download failed");
+      const blob = await downloadResp.blob();
+      const downloadUrl = window.URL.createObjectURL(blob);
+      const a = document.createElement("a");
+      a.href = downloadUrl;
+      a.download = "wildbook-coco-export.zip";
+      document.body.appendChild(a);
+      a.click();
+      a.remove();
+      window.URL.revokeObjectURL(downloadUrl);
+    } catch (err) {
+      console.error("COCO export error:", err);
+      setError(`Failed to export: ${err.message}`);
+    } finally {
+      if (cocoPollingRef.current) {
+        clearInterval(cocoPollingRef.current);
+        cocoPollingRef.current = null;
+      }
+      setLoading("cocoFormat", false);
+      setCocoProgress(null);
+    }
+  }, [searchQueryId]);
+
   const scrollToSection = (sectionId) => {
     setActiveSection(sectionId);
     const element = document.getElementById(sectionId);
@@ -249,17 +333,12 @@ export default function ExportDialog({ open, setOpen, searchQueryId }) {
                       <Card.Text className="text-muted small">
                         <FormattedMessage id="COCO_FORMAT_DESCRIPTION" />
                       </Card.Text>
-                      <div className="d-flex gap-2">
+                      <div className="d-flex gap-2 align-items-center">
                         <Button
                           className="my-3"
                           variant="outline-primary"
                           size="sm"
-                          onClick={() =>
-                            handleDownload(
-                              `/EncounterSearchExportCOCO?searchQueryId=${searchQueryId}&regularQuery=true`,
-                              "cocoFormat",
-                            )
-                          }
+                          onClick={handleCocoExport}
                           disabled={loadingStates.cocoFormat}
                         >
                           {loadingStates.cocoFormat ? (
@@ -272,7 +351,10 @@ export default function ExportDialog({ open, setOpen, searchQueryId }) {
                                 aria-hidden="true"
                                 className="me-2"
                               />
-                              <FormattedMessage id="EXPORTING" />
+                              {cocoProgress && cocoProgress.totalImages > 0
+                                ? `${cocoProgress.processedImages} / ${cocoProgress.totalImages}`
+                                : <FormattedMessage id="EXPORTING" />
+                              }
                             </>
                           ) : (
                             <FormattedMessage id="EXPORT_ZIP_FILE" />
diff --git a/src/main/java/org/ecocean/export/EncounterCOCOExportFile.java b/src/main/java/org/ecocean/export/EncounterCOCOExportFile.java
index c83f81714e..44b2f8b5e9 100644
--- a/src/main/java/org/ecocean/export/EncounterCOCOExportFile.java
+++ b/src/main/java/org/ecocean/export/EncounterCOCOExportFile.java
@@ -34,11 +34,20 @@ public class EncounterCOCOExportFile {
     private final List<Encounter> encounters;
     private final Shepherd shepherd;
 
+    // Progress tracking for async exports
+    private volatile int totalImages;
+    private volatile int processedImages;
+    private volatile int failedImages;
+
     public EncounterCOCOExportFile(List<Encounter> encounters, Shepherd shepherd) {
         this.encounters = encounters;
         this.shepherd = shepherd;
     }
 
+    public int getTotalImages() { return totalImages; }
+    public int getProcessedImages() { return processedImages; }
+    public int getFailedImages() { return failedImages; }
+
     public void writeTo(OutputStream outputStream) throws IOException {
         // Collect data
         Map<String, MediaAsset> mediaAssetMap = collectUniqueMediaAssets();
@@ -55,15 +64,14 @@ public void writeTo(OutputStream outputStream) throws IOException {
         // Write ZIP: images first, then JSON manifest so it only references
         // images that were actually written successfully.
         try (ZipOutputStream zipOut = new ZipOutputStream(outputStream)) {
-            int totalImages = mediaAssetMap.size();
-            int processedImages = 0;
-            int failedImages = 0;
+            totalImages = mediaAssetMap.size();
+            processedImages = 0;
+            failedImages = 0;
             Set<String> exportedUuids = new LinkedHashSet<>();
             log.info("COCO Export: Starting export of " + totalImages + " images...");
 
             for (Map.Entry<String, MediaAsset> entry : mediaAssetMap.entrySet()) {
                 MediaAsset ma = entry.getValue();
-                processedImages++;
                 try {
                     boolean success = writeImageToZip(ma, zipOut);
                     if (success) {
@@ -71,15 +79,16 @@ public void writeTo(OutputStream outputStream) throws IOException {
                     } else {
                         failedImages++;
                     }
-                    if (processedImages % 100 == 0) {
-                        log.info("COCO Export: Processed " + processedImages + "/" + totalImages +
-                                 " images (" + failedImages + " failed)");
-                    }
                 } catch (Exception e) {
                     failedImages++;
                     log.warning("COCO Export: Failed to export image " + ma.getUUID() +
                         ": " + e.getMessage());
                 }
+                processedImages++;
+                if (processedImages % 100 == 0) {
+                    log.info("COCO Export: Processed " + processedImages + "/" + totalImages +
+                             " images (" + failedImages + " failed)");
+                }
             }
 
             // Build JSON arrays using only successfully exported images
diff --git a/src/main/java/org/ecocean/servlet/export/EncounterSearchExportCOCO.java b/src/main/java/org/ecocean/servlet/export/EncounterSearchExportCOCO.java
index ca3d09f19e..b090c6f73b 100644
--- a/src/main/java/org/ecocean/servlet/export/EncounterSearchExportCOCO.java
+++ b/src/main/java/org/ecocean/servlet/export/EncounterSearchExportCOCO.java
@@ -18,14 +18,40 @@
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
-import java.io.PrintWriter;
 import java.nio.file.Files;
 import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
+import java.util.UUID;
 import java.util.Vector;
+import java.util.concurrent.ConcurrentHashMap;
 
+/**
+ * Async COCO export servlet. Three actions:
+ *   ?action=start  — kicks off background export, returns JSON {"jobId":"..."}
+ *   ?action=status&jobId=...  — returns JSON with progress/completion
+ *   ?action=download&jobId=...  — streams the completed ZIP file
+ *
+ * Legacy (no action param) falls back to synchronous export for backwards compatibility.
+ */
 public class EncounterSearchExportCOCO extends HttpServlet {
 
+    private static final long MAX_JOB_AGE_MS = 60 * 60 * 1000; // 1 hour
+
+    private static final Map<String, ExportJob> jobs = new ConcurrentHashMap<>();
+
+    static class ExportJob {
+        final String jobId;
+        final long createdAt = System.currentTimeMillis();
+        volatile String status = "running"; // running | complete | error
+        volatile String errorMessage;
+        volatile File tempFile;
+        volatile EncounterCOCOExportFile exportFile;
+
+        ExportJob(String jobId) { this.jobId = jobId; }
+    }
+
     public void init(ServletConfig config) throws ServletException {
         super.init(config);
     }
@@ -38,19 +64,158 @@ public void doGet(HttpServletRequest request, HttpServletResponse response)
     public void doPost(HttpServletRequest request, HttpServletResponse response)
             throws ServletException, IOException {
 
+        String action = request.getParameter("action");
+        if ("start".equals(action)) {
+            handleStart(request, response);
+        } else if ("status".equals(action)) {
+            handleStatus(request, response);
+        } else if ("download".equals(action)) {
+            handleDownload(request, response);
+        } else {
+            handleSynchronous(request, response);
+        }
+    }
+
+    private void handleStart(HttpServletRequest request, HttpServletResponse response)
+            throws IOException {
+        purgeStaleJobs();
+
         String context = ServletUtilities.getContext(request);
         Shepherd myShepherd = new Shepherd(context);
-        myShepherd.setAction("EncounterSearchExportCOCO.class");
+        myShepherd.setAction("EncounterSearchExportCOCO.start");
+        myShepherd.beginDBTransaction();
+
+        // Collect encounter IDs on the request thread (JDO objects stay local)
+        List<String> encounterIds;
+        try {
+            EncounterQueryResult queryResult = EncounterQueryProcessor.processQuery(
+                myShepherd, request, "year descending, month descending, day descending");
+            Vector<?> rEncounters = queryResult.getResult();
+            HiddenEncReporter hiddenData = new HiddenEncReporter(rEncounters, request, myShepherd);
+
+            encounterIds = new ArrayList<>();
+            for (Object obj : rEncounters) {
+                Encounter enc = (Encounter) obj;
+                if (!hiddenData.contains(enc)) {
+                    encounterIds.add(enc.getCatalogNumber());
+                }
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+            sendJson(response, 500, "{\"error\":\"Query failed: " +
+                escapeJson(e.getMessage()) + "\"}");
+            return;
+        } finally {
+            myShepherd.rollbackDBTransaction();
+            myShepherd.closeDBTransaction();
+        }
+
+        String jobId = UUID.randomUUID().toString();
+        ExportJob job = new ExportJob(jobId);
+        jobs.put(jobId, job);
+
+        // Background thread opens its own Shepherd and re-fetches encounters
+        List<String> encIds = encounterIds;
+        Thread exportThread = new Thread(() -> {
+            Shepherd bgShepherd = new Shepherd(context);
+            bgShepherd.setAction("EncounterSearchExportCOCO.export-" + jobId);
+            bgShepherd.beginDBTransaction();
+            try {
+                List<Encounter> encounters = new ArrayList<>();
+                for (String id : encIds) {
+                    Encounter enc = bgShepherd.getEncounter(id);
+                    if (enc != null) encounters.add(enc);
+                }
+                job.exportFile = new EncounterCOCOExportFile(encounters, bgShepherd);
+
+                File tmpDir = new File(CommonConfiguration.getUploadTmpDir(context));
+                if (!tmpDir.exists()) tmpDir.mkdirs();
+                File tempFile = File.createTempFile("wildbook-coco-export-", ".zip", tmpDir);
+                tempFile.deleteOnExit();
+                try (FileOutputStream fos = new FileOutputStream(tempFile)) {
+                    job.exportFile.writeTo(fos);
+                }
+                job.tempFile = tempFile;
+                job.status = "complete";
+            } catch (Exception e) {
+                e.printStackTrace();
+                job.status = "error";
+                job.errorMessage = e.getMessage();
+            } finally {
+                bgShepherd.rollbackDBTransaction();
+                bgShepherd.closeDBTransaction();
+            }
+        }, "coco-export-" + jobId);
+        exportThread.setDaemon(true);
+        exportThread.start();
 
+        sendJson(response, 200, "{\"jobId\":\"" + jobId + "\"}");
+    }
+
+    private void handleStatus(HttpServletRequest request, HttpServletResponse response)
+            throws IOException {
+        String jobId = request.getParameter("jobId");
+        ExportJob job = (jobId != null) ? jobs.get(jobId) : null;
+        if (job == null) {
+            sendJson(response, 404, "{\"error\":\"Job not found\"}");
+            return;
+        }
+        StringBuilder json = new StringBuilder();
+        json.append("{\"jobId\":\"").append(job.jobId).append("\"");
+        json.append(",\"status\":\"").append(job.status).append("\"");
+        if (job.exportFile != null) {
+            json.append(",\"totalImages\":").append(job.exportFile.getTotalImages());
+            json.append(",\"processedImages\":").append(job.exportFile.getProcessedImages());
+            json.append(",\"failedImages\":").append(job.exportFile.getFailedImages());
+        }
+        if (job.errorMessage != null) {
+            json.append(",\"error\":\"").append(escapeJson(job.errorMessage)).append("\"");
+        }
+        json.append("}");
+        sendJson(response, 200, json.toString());
+    }
+
+    private void handleDownload(HttpServletRequest request, HttpServletResponse response)
+            throws IOException {
+        String jobId = request.getParameter("jobId");
+        // Atomic remove prevents concurrent downloads of the same job
+        ExportJob job = (jobId != null) ? jobs.remove(jobId) : null;
+        if (job == null) {
+            sendJson(response, 404, "{\"error\":\"Job not found\"}");
+            return;
+        }
+        if (!"complete".equals(job.status) || job.tempFile == null || !job.tempFile.exists()) {
+            sendJson(response, 400, "{\"error\":\"Export not ready\"}");
+            return;
+        }
+
+        try {
+            response.setContentType("application/zip");
+            response.setHeader("Content-Disposition",
+                "attachment; filename=\"wildbook-coco-export.zip\"");
+            response.setContentLengthLong(job.tempFile.length());
+            OutputStream out = response.getOutputStream();
+            Files.copy(job.tempFile.toPath(), out);
+            out.flush();
+        } finally {
+            job.tempFile.delete();
+        }
+    }
+
+    /** Synchronous fallback for legacy/non-JS callers. */
+    private void handleSynchronous(HttpServletRequest request, HttpServletResponse response)
+            throws IOException {
+
+        String context = ServletUtilities.getContext(request);
+        Shepherd myShepherd = new Shepherd(context);
+        myShepherd.setAction("EncounterSearchExportCOCO.class");
         myShepherd.beginDBTransaction();
 
         File tempFile = null;
         try {
-            // Query and filter encounters (requires open DB transaction)
             EncounterQueryResult queryResult = EncounterQueryProcessor.processQuery(
                 myShepherd, request, "year descending, month descending, day descending");
             Vector<?> rEncounters = queryResult.getResult();
-
             HiddenEncReporter hiddenData = new HiddenEncReporter(rEncounters, request, myShepherd);
             List<Encounter> encounters = new ArrayList<>();
             for (Object obj : rEncounters) {
@@ -60,8 +225,6 @@ public void doPost(HttpServletRequest request, HttpServletResponse response)
                 }
             }
 
-            // Build ZIP to temp file so we can detect errors before committing
-            // the HTTP response and set an accurate Content-Length.
             File tmpDir = new File(CommonConfiguration.getUploadTmpDir(context));
             if (!tmpDir.exists()) tmpDir.mkdirs();
             tempFile = File.createTempFile("wildbook-coco-export-", ".zip", tmpDir);
@@ -71,7 +234,6 @@ public void doPost(HttpServletRequest request, HttpServletResponse response)
                 exportFile.writeTo(fos);
             }
 
-            // Stream complete file to client
             response.setContentType("application/zip");
             response.setHeader("Content-Disposition",
                 "attachment; filename=\"wildbook-coco-export.zip\"");
@@ -83,14 +245,7 @@ public void doPost(HttpServletRequest request, HttpServletResponse response)
         } catch (Exception e) {
             e.printStackTrace();
             if (!response.isCommitted()) {
-                response.setContentType("text/html");
-                PrintWriter out = response.getWriter();
-                out.println(ServletUtilities.getHeader(request));
-                out.println("<html><body><p><strong>Error encountered</strong></p>");
-                out.println("<p>Error: " + e.getMessage() + "</p>");
-                out.println("<p>Please let the webmaster know you encountered an error at: EncounterSearchExportCOCO servlet</p></body></html>");
-                out.println(ServletUtilities.getFooter(context));
-                out.close();
+                sendJson(response, 500, "{\"error\":\"" + escapeJson(e.getMessage()) + "\"}");
             }
         } finally {
             myShepherd.rollbackDBTransaction();
@@ -100,4 +255,32 @@ public void doPost(HttpServletRequest request, HttpServletResponse response)
             }
         }
     }
+
+    private void sendJson(HttpServletResponse response, int status, String json)
+            throws IOException {
+        response.setStatus(status);
+        response.setContentType("application/json");
+        response.setCharacterEncoding("UTF-8");
+        response.getWriter().write(json);
+    }
+
+    private static String escapeJson(String s) {
+        if (s == null) return "";
+        return s.replace("\\", "\\\\").replace("\"", "\\\"")
+                .replace("\n", "\\n").replace("\r", "\\r");
+    }
+
+    private void purgeStaleJobs() {
+        long now = System.currentTimeMillis();
+        Iterator<Map.Entry<String, ExportJob>> it = jobs.entrySet().iterator();
+        while (it.hasNext()) {
+            ExportJob job = it.next().getValue();
+            if ((now - job.createdAt) > MAX_JOB_AGE_MS) {
+                if (job.tempFile != null && job.tempFile.exists()) {
+                    job.tempFile.delete();
+                }
+                it.remove();
+            }
+        }
+    }
 }

From a0fa1fe73881ff81039862f208b8c633a0663716 Mon Sep 17 00:00:00 2001
From: Wild Me <jason@wildme.org>
Date: Thu, 9 Apr 2026 12:06:21 -0700
Subject: [PATCH 3/9] Expand modal states

---
 frontend/src/locale/de.json                       |  4 ++++
 frontend/src/locale/en.json                       |  4 ++++
 frontend/src/locale/es.json                       |  4 ++++
 frontend/src/locale/fr.json                       |  4 ++++
 frontend/src/locale/it.json                       |  4 ++++
 .../pages/SearchPages/components/ExportModal.jsx  | 15 ++++++++++++---
 .../ecocean/export/EncounterCOCOExportFile.java   |  5 +++++
 .../servlet/export/EncounterSearchExportCOCO.java |  1 +
 8 files changed, 38 insertions(+), 3 deletions(-)

diff --git a/frontend/src/locale/de.json b/frontend/src/locale/de.json
index 779b61b988..6ade186041 100644
--- a/frontend/src/locale/de.json
+++ b/frontend/src/locale/de.json
@@ -647,6 +647,10 @@
   "ENCOUNTER_ANNOTATION": "Begegnungsanmerkung",
   "COCO_FORMAT": "COCO-Format",
   "COCO_FORMAT_DESCRIPTION": "Exportieren Sie Annotationen im COCO-Format für KI/ML-Training. Enthält Bilder und Begrenzungsrahmen.",
+  "COCO_PHASE_PREPARING": "Preparing...",
+  "COCO_PHASE_IMAGES": "Downloading images",
+  "COCO_PHASE_MANIFEST": "Building manifest...",
+  "COCO_PHASE_PACKAGING": "Packaging ZIP...",
   "EXPORT_ZIP_FILE": "ZIP-Datei exportieren",
   "THIS_ENCOUNTER": "Diese Begegnung",
   "CLICK_ANNOTATION_TO_SEE_MATCH_RESULTS": "Klicken Sie auf eine Anmerkung, um die Übereinstimmungsergebnisse zu sehen.",
diff --git a/frontend/src/locale/en.json b/frontend/src/locale/en.json
index c942603f8f..8dadb2e428 100644
--- a/frontend/src/locale/en.json
+++ b/frontend/src/locale/en.json
@@ -647,6 +647,10 @@
     "ENCOUNTER_ANNOTATION": "Encounter Annotation",
     "COCO_FORMAT": "COCO Format",
     "COCO_FORMAT_DESCRIPTION": "Export annotations in COCO format for AI/ML training. Includes images and bounding boxes.",
+    "COCO_PHASE_PREPARING": "Preparing...",
+    "COCO_PHASE_IMAGES": "Downloading images",
+    "COCO_PHASE_MANIFEST": "Building manifest...",
+    "COCO_PHASE_PACKAGING": "Packaging ZIP...",
     "EXPORT_ZIP_FILE": "Export ZIP File",
     "THIS_ENCOUNTER": "This Encounter",
     "CLICK_ANNOTATION_TO_SEE_MATCH_RESULTS": "Click on an annotation to view match results",
diff --git a/frontend/src/locale/es.json b/frontend/src/locale/es.json
index bfd47a3b52..35065326c0 100644
--- a/frontend/src/locale/es.json
+++ b/frontend/src/locale/es.json
@@ -646,6 +646,10 @@
   "ENCOUNTER_ANNOTATION": "Anotación de encuentro",
   "COCO_FORMAT": "Formato COCO",
   "COCO_FORMAT_DESCRIPTION": "Exportar anotaciones en formato COCO para entrenamiento de IA/ML. Incluye imágenes y cuadros delimitadores.",
+  "COCO_PHASE_PREPARING": "Preparing...",
+  "COCO_PHASE_IMAGES": "Downloading images",
+  "COCO_PHASE_MANIFEST": "Building manifest...",
+  "COCO_PHASE_PACKAGING": "Packaging ZIP...",
   "EXPORT_ZIP_FILE": "Exportar archivo ZIP",
   "THIS_ENCOUNTER": "Este encuentro",
   "CLICK_ANNOTATION_TO_SEE_MATCH_RESULTS": "Haz clic en la anotación para ver los resultados de coincidencia",
diff --git a/frontend/src/locale/fr.json b/frontend/src/locale/fr.json
index 46ccf1e5bf..13947216d9 100644
--- a/frontend/src/locale/fr.json
+++ b/frontend/src/locale/fr.json
@@ -646,6 +646,10 @@
   "ENCOUNTER_ANNOTATION": "Annotation de rencontre",
   "COCO_FORMAT": "Format COCO",
   "COCO_FORMAT_DESCRIPTION": "Exporter les annotations au format COCO pour l'entraînement IA/ML. Inclut les images et les boîtes englobantes.",
+  "COCO_PHASE_PREPARING": "Preparing...",
+  "COCO_PHASE_IMAGES": "Downloading images",
+  "COCO_PHASE_MANIFEST": "Building manifest...",
+  "COCO_PHASE_PACKAGING": "Packaging ZIP...",
   "EXPORT_ZIP_FILE": "Exporter le fichier ZIP",
   "THIS_ENCOUNTER": "Cette rencontre",
   "CLICK_ANNOTATION_TO_SEE_MATCH_RESULTS": "Cliquez sur l’annotation pour voir les résultats de correspondance",
diff --git a/frontend/src/locale/it.json b/frontend/src/locale/it.json
index 8cb1997a62..a8e77289de 100644
--- a/frontend/src/locale/it.json
+++ b/frontend/src/locale/it.json
@@ -646,6 +646,10 @@
   "ENCOUNTER_ANNOTATION": "Annotazione dell'incontro",
   "COCO_FORMAT": "Formato COCO",
   "COCO_FORMAT_DESCRIPTION": "Esporta le annotazioni in formato COCO per l'addestramento AI/ML. Include immagini e bounding box.",
+  "COCO_PHASE_PREPARING": "Preparing...",
+  "COCO_PHASE_IMAGES": "Downloading images",
+  "COCO_PHASE_MANIFEST": "Building manifest...",
+  "COCO_PHASE_PACKAGING": "Packaging ZIP...",
   "EXPORT_ZIP_FILE": "Esporta file ZIP",
   "THIS_ENCOUNTER": "Questo incontro",
   "CLICK_ANNOTATION_TO_SEE_MATCH_RESULTS": "Clicca sull'annotazione per vedere i risultati della corrispondenza",
diff --git a/frontend/src/pages/SearchPages/components/ExportModal.jsx b/frontend/src/pages/SearchPages/components/ExportModal.jsx
index edfb5d0501..f1578c7103 100644
--- a/frontend/src/pages/SearchPages/components/ExportModal.jsx
+++ b/frontend/src/pages/SearchPages/components/ExportModal.jsx
@@ -126,7 +126,7 @@ export default function ExportDialog({ open, setOpen, searchQueryId }) {
             );
             const status = await statusResp.json();
 
-            if (status.totalImages > 0) {
+            if (status.totalImages > 0 || status.phase) {
               setCocoProgress(status);
             }
 
@@ -351,8 +351,17 @@ export default function ExportDialog({ open, setOpen, searchQueryId }) {
                                 aria-hidden="true"
                                 className="me-2"
                               />
-                              {cocoProgress && cocoProgress.totalImages > 0
-                                ? `${cocoProgress.processedImages} / ${cocoProgress.totalImages}`
+                              {cocoProgress?.phase === "images" && cocoProgress.totalImages > 0
+                                ? <>
+                                    <FormattedMessage id="COCO_PHASE_IMAGES" />
+                                    {` ${cocoProgress.processedImages} / ${cocoProgress.totalImages}`}
+                                  </>
+                                : cocoProgress?.phase === "manifest"
+                                ? <FormattedMessage id="COCO_PHASE_MANIFEST" />
+                                : cocoProgress?.phase === "packaging"
+                                ? <FormattedMessage id="COCO_PHASE_PACKAGING" />
+                                : cocoProgress?.phase === "preparing"
+                                ? <FormattedMessage id="COCO_PHASE_PREPARING" />
                                 : <FormattedMessage id="EXPORTING" />
                               }
                             </>
diff --git a/src/main/java/org/ecocean/export/EncounterCOCOExportFile.java b/src/main/java/org/ecocean/export/EncounterCOCOExportFile.java
index 44b2f8b5e9..6493207392 100644
--- a/src/main/java/org/ecocean/export/EncounterCOCOExportFile.java
+++ b/src/main/java/org/ecocean/export/EncounterCOCOExportFile.java
@@ -38,6 +38,7 @@ public class EncounterCOCOExportFile {
     private volatile int totalImages;
     private volatile int processedImages;
     private volatile int failedImages;
+    private volatile String phase = "preparing";
 
     public EncounterCOCOExportFile(List<Encounter> encounters, Shepherd shepherd) {
         this.encounters = encounters;
@@ -47,6 +48,7 @@ public EncounterCOCOExportFile(List<Encounter> encounters, Shepherd shepherd) {
     public int getTotalImages() { return totalImages; }
     public int getProcessedImages() { return processedImages; }
     public int getFailedImages() { return failedImages; }
+    public String getPhase() { return phase; }
 
     public void writeTo(OutputStream outputStream) throws IOException {
         // Collect data
@@ -67,6 +69,7 @@ public void writeTo(OutputStream outputStream) throws IOException {
             totalImages = mediaAssetMap.size();
             processedImages = 0;
             failedImages = 0;
+            phase = "images";
             Set<String> exportedUuids = new LinkedHashSet<>();
             log.info("COCO Export: Starting export of " + totalImages + " images...");
 
@@ -92,6 +95,7 @@ public void writeTo(OutputStream outputStream) throws IOException {
             }
 
             // Build JSON arrays using only successfully exported images
+            phase = "manifest";
             JSONArray imagesArray = new JSONArray();
             for (String uuid : exportedUuids) {
                 MediaAsset ma = mediaAssetMap.get(uuid);
@@ -121,6 +125,7 @@ public void writeTo(OutputStream outputStream) throws IOException {
             coco.put("annotations", annotationsArray);
 
             // Write annotations.json as the last entry
+            phase = "packaging";
             log.info("COCO Export: Writing annotations JSON...");
             byte[] jsonBytes = coco.toString(2).getBytes(StandardCharsets.UTF_8);
             ZipEntry jsonEntry = new ZipEntry("coco/annotations/instances.json");
diff --git a/src/main/java/org/ecocean/servlet/export/EncounterSearchExportCOCO.java b/src/main/java/org/ecocean/servlet/export/EncounterSearchExportCOCO.java
index b090c6f73b..716ad77aed 100644
--- a/src/main/java/org/ecocean/servlet/export/EncounterSearchExportCOCO.java
+++ b/src/main/java/org/ecocean/servlet/export/EncounterSearchExportCOCO.java
@@ -167,6 +167,7 @@ private void handleStatus(HttpServletRequest request, HttpServletResponse respon
             json.append(",\"totalImages\":").append(job.exportFile.getTotalImages());
             json.append(",\"processedImages\":").append(job.exportFile.getProcessedImages());
             json.append(",\"failedImages\":").append(job.exportFile.getFailedImages());
+            json.append(",\"phase\":\"").append(job.exportFile.getPhase()).append("\"");
         }
         if (job.errorMessage != null) {
             json.append(",\"error\":\"").append(escapeJson(job.errorMessage)).append("\"");

From 1a9c5ab1c83bfff949ef6bfe31791d8091b9397d Mon Sep 17 00:00:00 2001
From: Wild Me <jason@wildme.org>
Date: Fri, 10 Apr 2026 22:58:16 -0700
Subject: [PATCH 4/9] Add more names

---
 .../export/EncounterCOCOExportFile.java       | 32 ++++++++++++++++---
 1 file changed, 27 insertions(+), 5 deletions(-)

diff --git a/src/main/java/org/ecocean/export/EncounterCOCOExportFile.java b/src/main/java/org/ecocean/export/EncounterCOCOExportFile.java
index 6493207392..bd3e2157a0 100644
--- a/src/main/java/org/ecocean/export/EncounterCOCOExportFile.java
+++ b/src/main/java/org/ecocean/export/EncounterCOCOExportFile.java
@@ -3,6 +3,7 @@
 import org.ecocean.Annotation;
 import org.ecocean.Encounter;
 import org.ecocean.MarkedIndividual;
+import org.ecocean.MultiValue;
 import org.ecocean.media.MediaAsset;
 import org.ecocean.shepherd.core.Shepherd;
 import org.json.JSONArray;
@@ -478,14 +479,12 @@ private JSONObject buildAnnotationObject(Annotation ann, int annotationId, int i
             individualIds.put(individualIdMap.get(ind.getId()));
             annJson.put("individual_uuid", ind.getId());
             String displayName = ind.getDisplayName();
-            if (displayName != null) {
-                annJson.put("name", displayName);
-            } else {
-                annJson.put("name", JSONObject.NULL);
-            }
+            annJson.put("name", displayName != null ? displayName : JSONObject.NULL);
+            annJson.put("all_names", buildAllNames(ind));
         } else {
             annJson.put("individual_uuid", JSONObject.NULL);
             annJson.put("name", JSONObject.NULL);
+            annJson.put("all_names", new JSONArray());
         }
         annJson.put("individual_ids", individualIds);
 
@@ -496,4 +495,27 @@ private JSONObject buildAnnotationObject(Annotation ann, int annotationId, int i
 
         return annJson;
     }
+
+    /**
+     * Builds a JSONArray of all name key:value pairs from a MarkedIndividual's names MultiValue.
+     * Each entry is {"key": "...", "value": "..."}.
+     */
+    private JSONArray buildAllNames(MarkedIndividual ind) {
+        JSONArray allNames = new JSONArray();
+        MultiValue names = ind.getNames();
+        if (names == null) return allNames;
+        Set<String> keys = names.getKeys();
+        if (keys == null) return allNames;
+        for (String key : keys) {
+            List<String> vals = names.getValuesByKey(key);
+            if (vals == null) continue;
+            for (String val : vals) {
+                JSONObject entry = new JSONObject();
+                entry.put("key", key);
+                entry.put("value", val);
+                allNames.put(entry);
+            }
+        }
+        return allNames;
+    }
 }

From 0bc16099195d4ef7424106b95eecb67f0ef74e8c Mon Sep 17 00:00:00 2001
From: Wild Me <jason@wildme.org>
Date: Sat, 11 Apr 2026 10:30:22 -0700
Subject: [PATCH 5/9] Optimizing for larger datasets

---
 frontend/src/locale/de.json                   |   1 +
 frontend/src/locale/en.json                   |   1 +
 frontend/src/locale/es.json                   |   1 +
 frontend/src/locale/fr.json                   |   1 +
 frontend/src/locale/it.json                   |   1 +
 .../SearchPages/components/ExportModal.jsx    |  78 ++-
 .../export/EncounterCOCOExportFile.java       | 539 +++++++++---------
 .../export/EncounterSearchExportCOCO.java     |  68 ++-
 8 files changed, 378 insertions(+), 312 deletions(-)

diff --git a/frontend/src/locale/de.json b/frontend/src/locale/de.json
index 6ade186041..d856b32087 100644
--- a/frontend/src/locale/de.json
+++ b/frontend/src/locale/de.json
@@ -651,6 +651,7 @@
   "COCO_PHASE_IMAGES": "Downloading images",
   "COCO_PHASE_MANIFEST": "Building manifest...",
   "COCO_PHASE_PACKAGING": "Packaging ZIP...",
+  "COCO_RETRY_DOWNLOAD": "Download wiederholen",
   "EXPORT_ZIP_FILE": "ZIP-Datei exportieren",
   "THIS_ENCOUNTER": "Diese Begegnung",
   "CLICK_ANNOTATION_TO_SEE_MATCH_RESULTS": "Klicken Sie auf eine Anmerkung, um die Übereinstimmungsergebnisse zu sehen.",
diff --git a/frontend/src/locale/en.json b/frontend/src/locale/en.json
index 8dadb2e428..352552ee76 100644
--- a/frontend/src/locale/en.json
+++ b/frontend/src/locale/en.json
@@ -651,6 +651,7 @@
     "COCO_PHASE_IMAGES": "Downloading images",
     "COCO_PHASE_MANIFEST": "Building manifest...",
     "COCO_PHASE_PACKAGING": "Packaging ZIP...",
+    "COCO_RETRY_DOWNLOAD": "Retry Download",
     "EXPORT_ZIP_FILE": "Export ZIP File",
     "THIS_ENCOUNTER": "This Encounter",
     "CLICK_ANNOTATION_TO_SEE_MATCH_RESULTS": "Click on an annotation to view match results",
diff --git a/frontend/src/locale/es.json b/frontend/src/locale/es.json
index 35065326c0..787521f82f 100644
--- a/frontend/src/locale/es.json
+++ b/frontend/src/locale/es.json
@@ -650,6 +650,7 @@
   "COCO_PHASE_IMAGES": "Downloading images",
   "COCO_PHASE_MANIFEST": "Building manifest...",
   "COCO_PHASE_PACKAGING": "Packaging ZIP...",
+  "COCO_RETRY_DOWNLOAD": "Reintentar descarga",
   "EXPORT_ZIP_FILE": "Exportar archivo ZIP",
   "THIS_ENCOUNTER": "Este encuentro",
   "CLICK_ANNOTATION_TO_SEE_MATCH_RESULTS": "Haz clic en la anotación para ver los resultados de coincidencia",
diff --git a/frontend/src/locale/fr.json b/frontend/src/locale/fr.json
index 13947216d9..36d858acbc 100644
--- a/frontend/src/locale/fr.json
+++ b/frontend/src/locale/fr.json
@@ -650,6 +650,7 @@
   "COCO_PHASE_IMAGES": "Downloading images",
   "COCO_PHASE_MANIFEST": "Building manifest...",
   "COCO_PHASE_PACKAGING": "Packaging ZIP...",
+  "COCO_RETRY_DOWNLOAD": "Réessayer le téléchargement",
   "EXPORT_ZIP_FILE": "Exporter le fichier ZIP",
   "THIS_ENCOUNTER": "Cette rencontre",
   "CLICK_ANNOTATION_TO_SEE_MATCH_RESULTS": "Cliquez sur l’annotation pour voir les résultats de correspondance",
diff --git a/frontend/src/locale/it.json b/frontend/src/locale/it.json
index a8e77289de..679b3995b5 100644
--- a/frontend/src/locale/it.json
+++ b/frontend/src/locale/it.json
@@ -650,6 +650,7 @@
   "COCO_PHASE_IMAGES": "Downloading images",
   "COCO_PHASE_MANIFEST": "Building manifest...",
   "COCO_PHASE_PACKAGING": "Packaging ZIP...",
+  "COCO_RETRY_DOWNLOAD": "Riprova download",
   "EXPORT_ZIP_FILE": "Esporta file ZIP",
   "THIS_ENCOUNTER": "Questo incontro",
   "CLICK_ANNOTATION_TO_SEE_MATCH_RESULTS": "Clicca sull'annotazione per vedere i risultati della corrispondenza",
diff --git a/frontend/src/pages/SearchPages/components/ExportModal.jsx b/frontend/src/pages/SearchPages/components/ExportModal.jsx
index f1578c7103..4a833ebdef 100644
--- a/frontend/src/pages/SearchPages/components/ExportModal.jsx
+++ b/frontend/src/pages/SearchPages/components/ExportModal.jsx
@@ -76,6 +76,7 @@ export default function ExportDialog({ open, setOpen, searchQueryId }) {
 
   const [error, setError] = useState(null);
   const [cocoProgress, setCocoProgress] = useState(null);
+  const [cocoJobId, setCocoJobId] = useState(null);
   const cocoPollingRef = useRef(null);
 
   // Clean up polling interval on unmount (e.g., modal close)
@@ -117,14 +118,28 @@ export default function ExportDialog({ open, setOpen, searchQueryId }) {
       }
       const { jobId } = startData;
 
-      // Poll for progress
+      // Poll for progress (timeout after 2 hours for very large exports)
+      const MAX_POLL_MS = 2 * 60 * 60 * 1000;
+      const pollStart = Date.now();
+      let consecutiveErrors = 0;
       const result = await new Promise((resolve, reject) => {
         cocoPollingRef.current = setInterval(async () => {
           try {
+            if (Date.now() - pollStart > MAX_POLL_MS) {
+              clearInterval(cocoPollingRef.current);
+              cocoPollingRef.current = null;
+              reject(new Error("Export timed out after 2 hours"));
+              return;
+            }
+
             const statusResp = await fetch(
               `/EncounterSearchExportCOCO?action=status&jobId=${jobId}`,
             );
+            if (!statusResp.ok) {
+              throw new Error(`Status check failed (HTTP ${statusResp.status})`);
+            }
             const status = await statusResp.json();
+            consecutiveErrors = 0;
 
             if (status.totalImages > 0 || status.phase) {
               setCocoProgress(status);
@@ -140,27 +155,21 @@ export default function ExportDialog({ open, setOpen, searchQueryId }) {
               reject(new Error(status.error || "Export failed"));
             }
           } catch (e) {
-            clearInterval(cocoPollingRef.current);
-            cocoPollingRef.current = null;
-            reject(e);
+            consecutiveErrors++;
+            // Tolerate up to 3 transient network errors before giving up
+            if (consecutiveErrors >= 3) {
+              clearInterval(cocoPollingRef.current);
+              cocoPollingRef.current = null;
+              reject(e);
+            }
           }
         }, 3000);
       });
 
       // Download the completed file
-      const downloadResp = await fetch(
-        `/EncounterSearchExportCOCO?action=download&jobId=${result}`,
-      );
-      if (!downloadResp.ok) throw new Error("Download failed");
-      const blob = await downloadResp.blob();
-      const downloadUrl = window.URL.createObjectURL(blob);
-      const a = document.createElement("a");
-      a.href = downloadUrl;
-      a.download = "wildbook-coco-export.zip";
-      document.body.appendChild(a);
-      a.click();
-      a.remove();
-      window.URL.revokeObjectURL(downloadUrl);
+      setCocoJobId(result);
+      await downloadCocoFile(result);
+      setCocoJobId(null);
     } catch (err) {
       console.error("COCO export error:", err);
       setError(`Failed to export: ${err.message}`);
@@ -174,6 +183,37 @@ export default function ExportDialog({ open, setOpen, searchQueryId }) {
     }
   }, [searchQueryId]);
 
+  const downloadCocoFile = useCallback(async (jobId) => {
+    const downloadResp = await fetch(
+      `/EncounterSearchExportCOCO?action=download&jobId=${jobId}`,
+    );
+    if (!downloadResp.ok) throw new Error("Download failed");
+    const blob = await downloadResp.blob();
+    const downloadUrl = window.URL.createObjectURL(blob);
+    const a = document.createElement("a");
+    a.href = downloadUrl;
+    a.download = "wildbook-coco-export.zip";
+    document.body.appendChild(a);
+    a.click();
+    a.remove();
+    window.URL.revokeObjectURL(downloadUrl);
+  }, []);
+
+  const handleCocoRetry = useCallback(async () => {
+    if (!cocoJobId) return;
+    setError(null);
+    setLoading("cocoFormat", true);
+    try {
+      await downloadCocoFile(cocoJobId);
+      setCocoJobId(null);
+    } catch (err) {
+      console.error("COCO retry download error:", err);
+      setError(`Download failed: ${err.message}. You can retry.`);
+    } finally {
+      setLoading("cocoFormat", false);
+    }
+  }, [cocoJobId, downloadCocoFile]);
+
   const scrollToSection = (sectionId) => {
     setActiveSection(sectionId);
     const element = document.getElementById(sectionId);
@@ -338,7 +378,7 @@ export default function ExportDialog({ open, setOpen, searchQueryId }) {
                           className="my-3"
                           variant="outline-primary"
                           size="sm"
-                          onClick={handleCocoExport}
+                          onClick={cocoJobId ? handleCocoRetry : handleCocoExport}
                           disabled={loadingStates.cocoFormat}
                         >
                           {loadingStates.cocoFormat ? (
@@ -365,6 +405,8 @@ export default function ExportDialog({ open, setOpen, searchQueryId }) {
                                 : <FormattedMessage id="EXPORTING" />
                               }
                             </>
+                          ) : cocoJobId ? (
+                            <FormattedMessage id="COCO_RETRY_DOWNLOAD" />
                           ) : (
                             <FormattedMessage id="EXPORT_ZIP_FILE" />
                           )}
diff --git a/src/main/java/org/ecocean/export/EncounterCOCOExportFile.java b/src/main/java/org/ecocean/export/EncounterCOCOExportFile.java
index bd3e2157a0..0e44fbe906 100644
--- a/src/main/java/org/ecocean/export/EncounterCOCOExportFile.java
+++ b/src/main/java/org/ecocean/export/EncounterCOCOExportFile.java
@@ -26,24 +26,114 @@
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
 
+/**
+ * Generates a COCO-format ZIP export from a list of encounters.
+ *
+ * Design: all JDO/database data is eagerly extracted into plain data structures
+ * during {@link #extractData()}, so the caller can close the DB transaction
+ * before calling {@link #writeTo(OutputStream)} which does long-running HTTP
+ * image downloads.
+ */
 public class EncounterCOCOExportFile {
     private static final Logger log = Logger.getLogger(EncounterCOCOExportFile.class.getName());
     private static final int BUFFER_SIZE = 8192;
     private static final int CONNECTION_TIMEOUT_MS = 30000;
     private static final int READ_TIMEOUT_MS = 60000;
 
-    private final List<Encounter> encounters;
-    private final Shepherd shepherd;
-
     // Progress tracking for async exports
     private volatile int totalImages;
     private volatile int processedImages;
     private volatile int failedImages;
     private volatile String phase = "preparing";
 
+    // --- Plain data extracted from JDO objects ---
+    // Populated by extractData(), consumed by writeTo()
+    private Map<String, URL> mediaAssetUrls;           // uuid -> webURL
+    private Map<String, Boolean> mediaAssetIsJpeg;      // uuid -> true if JPEG
+    private Map<String, ImageMeta> mediaAssetMeta;      // uuid -> metadata for JSON
+    private Map<String, Integer> mediaAssetToImageId;   // uuid -> sequential image ID
+    private Map<String, Integer> categoryMap;            // iaClass -> category ID
+    private Map<String, Integer> individualIdMap;        // individual ID -> sequential int
+    private List<AnnotationData> annotationDataList;     // flattened annotation records
+
+    /** Immutable snapshot of image metadata needed for the COCO JSON. */
+    static class ImageMeta {
+        final String uuid;
+        final int width;
+        final int height;
+        final String dateTime; // nullable
+        final Double latitude; // nullable
+        final Double longitude; // nullable
+
+        ImageMeta(MediaAsset ma) {
+            this.uuid = ma.getUUID();
+            this.width = (int) ma.getWidth();
+            this.height = (int) ma.getHeight();
+            this.dateTime = ma.getDateTime() != null ? ma.getDateTime().toString() : null;
+            this.latitude = ma.getLatitude();
+            this.longitude = ma.getLongitude();
+        }
+    }
+
+    /** Immutable snapshot of one annotation's data needed for the COCO JSON. */
+    static class AnnotationData {
+        final String annotationId;
+        final String mediaAssetUuid;
+        final String iaClass;
+        final int[] bbox;
+        final String viewpoint; // nullable
+        final double theta;
+        final String individualId; // nullable
+        final String individualDisplayName; // nullable
+        final List<String[]> allNames; // list of [key, value] pairs
+        final String locationId; // nullable
+        final String encounterId;
+
+        AnnotationData(Annotation ann, Encounter enc) {
+            this.annotationId = ann.getId();
+            this.mediaAssetUuid = ann.getMediaAsset().getUUID();
+            this.iaClass = ann.getIAClass().trim();
+            this.bbox = ann.getBbox();
+            this.viewpoint = ann.getViewpoint();
+            this.theta = ann.getTheta();
+            this.locationId = enc.getLocationID();
+            this.encounterId = enc.getCatalogNumber();
+
+            MarkedIndividual ind = enc.getIndividual();
+            if (ind != null && ind.getId() != null) {
+                this.individualId = ind.getId();
+                this.individualDisplayName = ind.getDisplayName();
+                this.allNames = extractNames(ind);
+            } else {
+                this.individualId = null;
+                this.individualDisplayName = null;
+                this.allNames = Collections.emptyList();
+            }
+        }
+
+        private static List<String[]> extractNames(MarkedIndividual ind) {
+            List<String[]> result = new ArrayList<>();
+            MultiValue names = ind.getNames();
+            if (names == null) return result;
+            Set<String> keys = names.getKeys();
+            if (keys == null) return result;
+            for (String key : keys) {
+                List<String> vals = names.getValuesByKey(key);
+                if (vals == null) continue;
+                for (String val : vals) {
+                    result.add(new String[]{key, val});
+                }
+            }
+            return result;
+        }
+    }
+
+    /**
+     * Constructs the export file and eagerly extracts all data from JDO objects.
+     * After this constructor returns, the caller may close the DB transaction.
+     */
     public EncounterCOCOExportFile(List<Encounter> encounters, Shepherd shepherd) {
-        this.encounters = encounters;
-        this.shepherd = shepherd;
+        extractData(encounters);
     }
 
     public int getTotalImages() { return totalImages; }
@@ -51,41 +141,107 @@ public EncounterCOCOExportFile(List<Encounter> encounters, Shepherd shepherd) {
     public int getFailedImages() { return failedImages; }
     public String getPhase() { return phase; }
 
-    public void writeTo(OutputStream outputStream) throws IOException {
-        // Collect data
-        Map<String, MediaAsset> mediaAssetMap = collectUniqueMediaAssets();
-        Map<String, Integer> categoryMap = buildCategoryMap();
-        Map<String, Integer> individualIdMap = buildIndividualIdMap();
+    /**
+     * Eagerly extracts all needed data from JDO-managed objects into plain
+     * data structures. Must be called while the DB transaction is still active.
+     */
+    private void extractData(List<Encounter> encounters) {
+        // Collect unique media assets and their metadata/URLs/content types
+        mediaAssetUrls = new LinkedHashMap<>();
+        mediaAssetIsJpeg = new LinkedHashMap<>();
+        mediaAssetMeta = new LinkedHashMap<>();
+        for (Encounter enc : encounters) {
+            if (enc.getAnnotations() == null) continue;
+            for (Annotation ann : enc.getAnnotations()) {
+                if (!isValidAnnotation(ann)) continue;
+                MediaAsset ma = ann.getMediaAsset();
+                String uuid = ma.getUUID();
+                if (!mediaAssetUrls.containsKey(uuid)) {
+                    URL url = ma.webURL();
+                    mediaAssetUrls.put(uuid, url);
+                    mediaAssetMeta.put(uuid, new ImageMeta(ma));
+                    mediaAssetIsJpeg.put(uuid, detectIsJpeg(ma, url));
+                }
+            }
+        }
 
-        // Assign image IDs
-        Map<String, Integer> mediaAssetToImageId = new LinkedHashMap<>();
+        // Assign sequential image IDs
+        mediaAssetToImageId = new LinkedHashMap<>();
         int imageId = 1;
-        for (String uuid : mediaAssetMap.keySet()) {
+        for (String uuid : mediaAssetUrls.keySet()) {
             mediaAssetToImageId.put(uuid, imageId++);
         }
 
+        // Build category map
+        Set<String> iaClasses = new LinkedHashSet<>();
+        for (Encounter enc : encounters) {
+            if (enc.getAnnotations() == null) continue;
+            for (Annotation ann : enc.getAnnotations()) {
+                String iaClass = ann.getIAClass();
+                if (iaClass != null && !iaClass.trim().isEmpty()) {
+                    iaClasses.add(iaClass.trim());
+                }
+            }
+        }
+        categoryMap = new LinkedHashMap<>();
+        int catId = 1;
+        for (String iaClass : iaClasses) {
+            categoryMap.put(iaClass, catId++);
+        }
+
+        // Build individual ID map
+        Set<String> individualIds = new TreeSet<>();
+        for (Encounter enc : encounters) {
+            MarkedIndividual ind = enc.getIndividual();
+            if (ind != null && ind.getId() != null) {
+                individualIds.add(ind.getId());
+            }
+        }
+        individualIdMap = new LinkedHashMap<>();
+        int idx = 0;
+        for (String indId : individualIds) {
+            individualIdMap.put(indId, idx++);
+        }
+
+        // Extract all annotation data
+        annotationDataList = new ArrayList<>();
+        for (Encounter enc : encounters) {
+            if (enc.getAnnotations() == null) continue;
+            for (Annotation ann : enc.getAnnotations()) {
+                if (!isValidAnnotation(ann)) continue;
+                annotationDataList.add(new AnnotationData(ann, enc));
+            }
+        }
+    }
+
+    /**
+     * Writes the COCO ZIP to the given output stream. This method performs
+     * long-running HTTP image downloads and does NOT require a DB transaction.
+     */
+    public void writeTo(OutputStream outputStream) throws IOException {
         // Write ZIP: images first, then JSON manifest so it only references
         // images that were actually written successfully.
         try (ZipOutputStream zipOut = new ZipOutputStream(outputStream)) {
-            totalImages = mediaAssetMap.size();
+            totalImages = mediaAssetUrls.size();
             processedImages = 0;
             failedImages = 0;
             phase = "images";
             Set<String> exportedUuids = new LinkedHashSet<>();
             log.info("COCO Export: Starting export of " + totalImages + " images...");
 
-            for (Map.Entry<String, MediaAsset> entry : mediaAssetMap.entrySet()) {
-                MediaAsset ma = entry.getValue();
+            for (Map.Entry<String, URL> entry : mediaAssetUrls.entrySet()) {
+                String uuid = entry.getKey();
+                URL url = entry.getValue();
                 try {
-                    boolean success = writeImageToZip(ma, zipOut);
+                    boolean success = writeImageToZip(uuid, url, zipOut);
                     if (success) {
-                        exportedUuids.add(entry.getKey());
+                        exportedUuids.add(uuid);
                     } else {
                         failedImages++;
                     }
                 } catch (Exception e) {
                     failedImages++;
-                    log.warning("COCO Export: Failed to export image " + ma.getUUID() +
+                    log.warning("COCO Export: Failed to export image " + uuid +
                         ": " + e.getMessage());
                 }
                 processedImages++;
@@ -99,29 +255,23 @@ public void writeTo(OutputStream outputStream) throws IOException {
             phase = "manifest";
             JSONArray imagesArray = new JSONArray();
             for (String uuid : exportedUuids) {
-                MediaAsset ma = mediaAssetMap.get(uuid);
+                ImageMeta meta = mediaAssetMeta.get(uuid);
                 int imgId = mediaAssetToImageId.get(uuid);
-                imagesArray.put(buildImageObject(ma, imgId));
+                imagesArray.put(buildImageObject(meta, imgId));
             }
 
             JSONArray annotationsArray = new JSONArray();
             int annotationId = 1;
-            for (Encounter enc : encounters) {
-                if (enc.getAnnotations() == null) continue;
-                for (Annotation ann : enc.getAnnotations()) {
-                    if (!isValidAnnotation(ann)) continue;
-                    MediaAsset ma = ann.getMediaAsset();
-                    if (!exportedUuids.contains(ma.getUUID())) continue;
-                    int imgId = mediaAssetToImageId.get(ma.getUUID());
-                    annotationsArray.put(buildAnnotationObject(ann, annotationId++, imgId,
-                        categoryMap, individualIdMap, enc));
-                }
+            for (AnnotationData ad : annotationDataList) {
+                if (!exportedUuids.contains(ad.mediaAssetUuid)) continue;
+                int imgId = mediaAssetToImageId.get(ad.mediaAssetUuid);
+                annotationsArray.put(buildAnnotationObject(ad, annotationId++, imgId));
             }
 
             JSONObject coco = new JSONObject();
-            coco.put("info", buildInfo(individualIdMap));
+            coco.put("info", buildInfo());
             coco.put("licenses", buildLicenses());
-            coco.put("categories", buildCategories(categoryMap));
+            coco.put("categories", buildCategories());
             coco.put("images", imagesArray);
             coco.put("annotations", annotationsArray);
 
@@ -142,83 +292,52 @@ public void writeTo(OutputStream outputStream) throws IOException {
 
     /**
      * Writes an image directly to the ZipOutputStream using streaming.
-     * This avoids loading entire images into memory, critical for large exports.
      *
      * @return true if image was written successfully, false if skipped
      */
-    private boolean writeImageToZip(MediaAsset ma, ZipOutputStream zipOut) throws IOException {
-        URL url = ma.webURL();
+    private boolean writeImageToZip(String uuid, URL url, ZipOutputStream zipOut) throws IOException {
         if (url == null) {
-            log.fine("COCO Export: Skipping image " + ma.getUUID() + " - no URL available");
+            log.fine("COCO Export: Skipping image " + uuid + " - no URL available");
             return false;
         }
 
-        String contentType = detectContentType(url);
-        boolean isJpeg = isJpegContentType(contentType);
+        boolean isJpeg = Boolean.TRUE.equals(mediaAssetIsJpeg.get(uuid));
 
-        ZipEntry imgEntry = new ZipEntry("coco/images/" + ma.getUUID() + ".jpg");
+        ZipEntry imgEntry = new ZipEntry("coco/images/" + uuid + ".jpg");
         zipOut.putNextEntry(imgEntry);
 
         try {
             if (isJpeg) {
-                // Stream JPEG directly - most memory efficient
                 streamImageDirectly(url, zipOut);
             } else {
-                // Non-JPEG: must decode and re-encode (uses more memory but unavoidable)
                 convertAndWriteImage(url, zipOut);
             }
         } finally {
             zipOut.closeEntry();
-            zipOut.flush(); // Keep data flowing to client
+            zipOut.flush();
         }
 
         return true;
     }
 
     /**
-     * Detects the content type of a URL, handling redirects.
+     * Determines if a MediaAsset is JPEG using DB metadata (no HTTP request).
+     * Falls back to URL path extension if metadata is unavailable.
      */
-    private String detectContentType(URL url) {
-        HttpURLConnection httpConn = null;
-        try {
-            URLConnection conn = url.openConnection();
-            conn.setConnectTimeout(CONNECTION_TIMEOUT_MS);
-            conn.setReadTimeout(READ_TIMEOUT_MS);
-
-            if (conn instanceof HttpURLConnection) {
-                httpConn = (HttpURLConnection) conn;
-                httpConn.setRequestMethod("HEAD");
-                httpConn.setInstanceFollowRedirects(true);
-            }
-
-            return conn.getContentType();
-        } catch (Exception e) {
-            // Fall back to checking file extension
+    private static boolean detectIsJpeg(MediaAsset ma, URL url) {
+        String[] mimeType = ma.getMimeType();
+        if (mimeType != null && mimeType.length >= 2) {
+            String minor = mimeType[1].split(";")[0].trim();
+            return "jpeg".equals(minor) || "jpg".equals(minor);
+        }
+        // Fall back to URL path extension
+        if (url != null) {
             String path = url.getPath().toLowerCase();
-            if (path.endsWith(".jpg") || path.endsWith(".jpeg")) {
-                return "image/jpeg";
-            }
-            return null;
-        } finally {
-            if (httpConn != null) {
-                httpConn.disconnect();
-            }
+            return path.endsWith(".jpg") || path.endsWith(".jpeg");
         }
+        return false;
     }
 
-    /**
-     * Checks if content type indicates JPEG.
-     */
-    private boolean isJpegContentType(String contentType) {
-        if (contentType == null) return false;
-        contentType = contentType.toLowerCase();
-        return contentType.contains("image/jpeg") || contentType.contains("image/jpg");
-    }
-
-    /**
-     * Streams image bytes directly from URL to ZipOutputStream without decoding.
-     * This is the most memory-efficient method for JPEG images.
-     */
     private void streamImageDirectly(URL url, ZipOutputStream zipOut) throws IOException {
         URLConnection conn = url.openConnection();
         conn.setConnectTimeout(CONNECTION_TIMEOUT_MS);
@@ -241,10 +360,6 @@ private void streamImageDirectly(URL url, ZipOutputStream zipOut) throws IOExcep
         }
     }
 
-    /**
-     * Converts a non-JPEG image to JPEG and writes to ZipOutputStream.
-     * This requires loading the image into memory but handles PNG, GIF, etc.
-     */
     private void convertAndWriteImage(URL url, ZipOutputStream zipOut) throws IOException {
         URLConnection conn = url.openConnection();
         conn.setConnectTimeout(CONNECTION_TIMEOUT_MS);
@@ -260,7 +375,6 @@ private void convertAndWriteImage(URL url, ZipOutputStream zipOut) throws IOExce
                 throw new IOException("Failed to decode image");
             }
 
-            // For images with alpha channel (PNG), convert to RGB for JPEG
             if (image.getColorModel().hasAlpha()) {
                 BufferedImage rgbImage = new BufferedImage(
                     image.getWidth(), image.getHeight(), BufferedImage.TYPE_INT_RGB);
@@ -273,7 +387,6 @@ private void convertAndWriteImage(URL url, ZipOutputStream zipOut) throws IOExce
                 image = rgbImage;
             }
 
-            // Buffer the JPEG output to avoid ImageIO closing the ZipOutputStream
             java.io.ByteArrayOutputStream imgBuf = new java.io.ByteArrayOutputStream();
             ImageIO.write(image, "jpg", imgBuf);
             zipOut.write(imgBuf.toByteArray());
@@ -284,174 +397,49 @@ private void convertAndWriteImage(URL url, ZipOutputStream zipOut) throws IOExce
         }
     }
 
-    /**
-     * Collects all unique MediaAssets from valid annotations.
-     * An annotation is valid if it has a non-null/non-empty iaClass and a valid bbox.
-     */
-    private Map<String, MediaAsset> collectUniqueMediaAssets() {
-        Map<String, MediaAsset> mediaAssetMap = new LinkedHashMap<>();
-        for (Encounter enc : encounters) {
-            if (enc.getAnnotations() == null) continue;
-            for (Annotation ann : enc.getAnnotations()) {
-                if (!isValidAnnotation(ann)) continue;
-                MediaAsset ma = ann.getMediaAsset();
-                if (ma != null && ma.getUUID() != null) {
-                    mediaAssetMap.putIfAbsent(ma.getUUID(), ma);
-                }
-            }
-        }
-        return mediaAssetMap;
-    }
-
-    /**
-     * Builds a map from iaClass to sequential category ID.
-     */
-    private Map<String, Integer> buildCategoryMap() {
-        Set<String> iaClasses = new LinkedHashSet<>();
-        for (Encounter enc : encounters) {
-            if (enc.getAnnotations() == null) continue;
-            for (Annotation ann : enc.getAnnotations()) {
-                String iaClass = ann.getIAClass();
-                if (iaClass != null && !iaClass.trim().isEmpty()) {
-                    iaClasses.add(iaClass.trim());
-                }
-            }
-        }
-        Map<String, Integer> categoryMap = new LinkedHashMap<>();
-        int id = 1;
-        for (String iaClass : iaClasses) {
-            categoryMap.put(iaClass, id++);
-        }
-        return categoryMap;
-    }
-
-    /**
-     * Builds a map from MarkedIndividual UUID to sequential integer ID.
-     */
-    private Map<String, Integer> buildIndividualIdMap() {
-        Set<String> individualIds = new TreeSet<>(); // TreeSet for sorted, deterministic order
-        for (Encounter enc : encounters) {
-            MarkedIndividual ind = enc.getIndividual();
-            if (ind != null && ind.getId() != null) {
-                individualIds.add(ind.getId());
-            }
-        }
-        Map<String, Integer> idMap = new LinkedHashMap<>();
-        int idx = 0;
-        for (String indId : individualIds) {
-            idMap.put(indId, idx++);
-        }
-        return idMap;
-    }
-
-    /**
-     * Checks if an annotation is valid for COCO export.
-     */
     private boolean isValidAnnotation(Annotation ann) {
         if (ann == null) return false;
         String iaClass = ann.getIAClass();
         if (iaClass == null || iaClass.trim().isEmpty()) return false;
         int[] bbox = ann.getBbox();
         if (bbox == null || bbox.length < 4) return false;
-        if (bbox[2] <= 0 || bbox[3] <= 0) return false; // width/height must be positive
+        if (bbox[2] <= 0 || bbox[3] <= 0) return false;
         MediaAsset ma = ann.getMediaAsset();
         if (ma == null || ma.getUUID() == null) return false;
         return true;
     }
 
-    /**
-     * Builds the "info" section of COCO JSON.
-     */
-    private JSONObject buildInfo(Map<String, Integer> individualIdMap) {
-        JSONObject info = new JSONObject();
-        info.put("description", "Wildbook COCO Export");
-        info.put("version", "1.0");
-        info.put("date_created", ZonedDateTime.now().format(DateTimeFormatter.ISO_INSTANT));
-        info.put("contributor", "Wildbook");
-
-        JSONObject idMapping = new JSONObject();
-        for (Map.Entry<String, Integer> entry : individualIdMap.entrySet()) {
-            idMapping.put(entry.getKey(), entry.getValue());
-        }
-        info.put("individual_id_mapping", idMapping);
-        return info;
-    }
-
-    /**
-     * Builds the "licenses" section of COCO JSON.
-     */
-    private JSONArray buildLicenses() {
-        JSONArray licenses = new JSONArray();
-        JSONObject license = new JSONObject();
-        license.put("id", 1);
-        license.put("name", "See Wildbook Terms");
-        license.put("url", "");
-        licenses.put(license);
-        return licenses;
-    }
-
-    /**
-     * Builds the "categories" section of COCO JSON.
-     */
-    private JSONArray buildCategories(Map<String, Integer> categoryMap) {
-        JSONArray categories = new JSONArray();
-        for (Map.Entry<String, Integer> entry : categoryMap.entrySet()) {
-            JSONObject cat = new JSONObject();
-            cat.put("id", entry.getValue());
-            cat.put("name", entry.getKey());
-            cat.put("supercategory", "animal");
-            categories.put(cat);
-        }
-        return categories;
-    }
+    // --- JSON builders using plain data (no JDO access) ---
 
-    /**
-     * Builds an image object for COCO JSON.
-     */
-    private JSONObject buildImageObject(MediaAsset ma, int imageId) {
+    private JSONObject buildImageObject(ImageMeta meta, int imageId) {
         JSONObject img = new JSONObject();
         img.put("id", imageId);
-        img.put("file_name", ma.getUUID() + ".jpg");
-        img.put("width", (int) ma.getWidth());
-        img.put("height", (int) ma.getHeight());
-        img.put("uuid", ma.getUUID());
-
-        // Always include all fields - use null for missing values so every record
-        // has the same schema (enables NaN counting in EDA instead of probing for missing keys)
-        img.put("date_captured", ma.getDateTime() != null ? ma.getDateTime().toString() : JSONObject.NULL);
-        Double lat = ma.getLatitude();
-        Double lon = ma.getLongitude();
-        img.put("gps_lat_captured", lat != null ? lat : JSONObject.NULL);
-        img.put("gps_lon_captured", lon != null ? lon : JSONObject.NULL);
+        img.put("file_name", meta.uuid + ".jpg");
+        img.put("width", meta.width);
+        img.put("height", meta.height);
+        img.put("uuid", meta.uuid);
+        img.put("date_captured", meta.dateTime != null ? meta.dateTime : JSONObject.NULL);
+        img.put("gps_lat_captured", meta.latitude != null ? meta.latitude : JSONObject.NULL);
+        img.put("gps_lon_captured", meta.longitude != null ? meta.longitude : JSONObject.NULL);
         return img;
     }
 
-    /**
-     * Builds an annotation object for COCO JSON.
-     */
-    private JSONObject buildAnnotationObject(Annotation ann, int annotationId, int imageId,
-                                              Map<String, Integer> categoryMap,
-                                              Map<String, Integer> individualIdMap,
-                                              Encounter enc) {
+    private JSONObject buildAnnotationObject(AnnotationData ad, int annotationId, int imageId) {
         JSONObject annJson = new JSONObject();
         annJson.put("id", annotationId);
         annJson.put("image_id", imageId);
+        annJson.put("category_id", categoryMap.get(ad.iaClass));
 
-        String iaClass = ann.getIAClass().trim();
-        annJson.put("category_id", categoryMap.get(iaClass));
-
-        int[] bbox = ann.getBbox();
+        int[] bbox = ad.bbox;
         JSONArray bboxArray = new JSONArray();
-        bboxArray.put(bbox[0]); // x
-        bboxArray.put(bbox[1]); // y
-        bboxArray.put(bbox[2]); // width
-        bboxArray.put(bbox[3]); // height
+        bboxArray.put(bbox[0]);
+        bboxArray.put(bbox[1]);
+        bboxArray.put(bbox[2]);
+        bboxArray.put(bbox[3]);
         annJson.put("bbox", bboxArray);
 
-        int area = bbox[2] * bbox[3];
-        annJson.put("area", area);
+        annJson.put("area", bbox[2] * bbox[3]);
 
-        // Segmentation: rectangle polygon from bbox
         JSONArray segmentation = new JSONArray();
         JSONArray polygon = new JSONArray();
         int x = bbox[0], y = bbox[1], w = bbox[2], h = bbox[3];
@@ -463,24 +451,23 @@ private JSONObject buildAnnotationObject(Annotation ann, int annotationId, int i
         annJson.put("segmentation", segmentation);
 
         annJson.put("iscrowd", 0);
+        annJson.put("uuid", ad.annotationId);
+        annJson.put("viewpoint", ad.viewpoint != null ? ad.viewpoint : JSONObject.NULL);
+        annJson.put("theta", ad.theta);
 
-        // Custom fields
-        annJson.put("uuid", ann.getId());
-
-        String viewpoint = ann.getViewpoint();
-        annJson.put("viewpoint", viewpoint != null ? viewpoint : JSONObject.NULL);
-
-        annJson.put("theta", ann.getTheta());
-
-        // Individual info
-        MarkedIndividual ind = enc.getIndividual();
         JSONArray individualIds = new JSONArray();
-        if (ind != null && ind.getId() != null && individualIdMap.containsKey(ind.getId())) {
-            individualIds.put(individualIdMap.get(ind.getId()));
-            annJson.put("individual_uuid", ind.getId());
-            String displayName = ind.getDisplayName();
-            annJson.put("name", displayName != null ? displayName : JSONObject.NULL);
-            annJson.put("all_names", buildAllNames(ind));
+        if (ad.individualId != null && individualIdMap.containsKey(ad.individualId)) {
+            individualIds.put(individualIdMap.get(ad.individualId));
+            annJson.put("individual_uuid", ad.individualId);
+            annJson.put("name", ad.individualDisplayName != null ? ad.individualDisplayName : JSONObject.NULL);
+            JSONArray allNames = new JSONArray();
+            for (String[] kv : ad.allNames) {
+                JSONObject entry = new JSONObject();
+                entry.put("key", kv[0]);
+                entry.put("value", kv[1]);
+                allNames.put(entry);
+            }
+            annJson.put("all_names", allNames);
         } else {
             annJson.put("individual_uuid", JSONObject.NULL);
             annJson.put("name", JSONObject.NULL);
@@ -488,34 +475,46 @@ private JSONObject buildAnnotationObject(Annotation ann, int annotationId, int i
         }
         annJson.put("individual_ids", individualIds);
 
-        // Encounter info
-        String locationId = enc.getLocationID();
-        annJson.put("location_id", locationId != null ? locationId : JSONObject.NULL);
-        annJson.put("encounter_id", enc.getCatalogNumber());
+        annJson.put("location_id", ad.locationId != null ? ad.locationId : JSONObject.NULL);
+        annJson.put("encounter_id", ad.encounterId);
 
         return annJson;
     }
 
-    /**
-     * Builds a JSONArray of all name key:value pairs from a MarkedIndividual's names MultiValue.
-     * Each entry is {"key": "...", "value": "..."}.
-     */
-    private JSONArray buildAllNames(MarkedIndividual ind) {
-        JSONArray allNames = new JSONArray();
-        MultiValue names = ind.getNames();
-        if (names == null) return allNames;
-        Set<String> keys = names.getKeys();
-        if (keys == null) return allNames;
-        for (String key : keys) {
-            List<String> vals = names.getValuesByKey(key);
-            if (vals == null) continue;
-            for (String val : vals) {
-                JSONObject entry = new JSONObject();
-                entry.put("key", key);
-                entry.put("value", val);
-                allNames.put(entry);
-            }
+    private JSONObject buildInfo() {
+        JSONObject info = new JSONObject();
+        info.put("description", "Wildbook COCO Export");
+        info.put("version", "1.0");
+        info.put("date_created", ZonedDateTime.now().format(DateTimeFormatter.ISO_INSTANT));
+        info.put("contributor", "Wildbook");
+
+        JSONObject idMapping = new JSONObject();
+        for (Map.Entry<String, Integer> entry : individualIdMap.entrySet()) {
+            idMapping.put(entry.getKey(), entry.getValue());
+        }
+        info.put("individual_id_mapping", idMapping);
+        return info;
+    }
+
+    private JSONArray buildLicenses() {
+        JSONArray licenses = new JSONArray();
+        JSONObject license = new JSONObject();
+        license.put("id", 1);
+        license.put("name", "See Wildbook Terms");
+        license.put("url", "");
+        licenses.put(license);
+        return licenses;
+    }
+
+    private JSONArray buildCategories() {
+        JSONArray categories = new JSONArray();
+        for (Map.Entry<String, Integer> entry : categoryMap.entrySet()) {
+            JSONObject cat = new JSONObject();
+            cat.put("id", entry.getValue());
+            cat.put("name", entry.getKey());
+            cat.put("supercategory", "animal");
+            categories.put(cat);
         }
-        return allNames;
+        return categories;
     }
 }
diff --git a/src/main/java/org/ecocean/servlet/export/EncounterSearchExportCOCO.java b/src/main/java/org/ecocean/servlet/export/EncounterSearchExportCOCO.java
index 716ad77aed..46ea08b32c 100644
--- a/src/main/java/org/ecocean/servlet/export/EncounterSearchExportCOCO.java
+++ b/src/main/java/org/ecocean/servlet/export/EncounterSearchExportCOCO.java
@@ -114,9 +114,11 @@ private void handleStart(HttpServletRequest request, HttpServletResponse respons
         ExportJob job = new ExportJob(jobId);
         jobs.put(jobId, job);
 
-        // Background thread opens its own Shepherd and re-fetches encounters
+        // Background thread opens its own Shepherd, extracts data, closes the
+        // DB transaction, then does the long-running image I/O without a DB connection.
         List<String> encIds = encounterIds;
         Thread exportThread = new Thread(() -> {
+            // Phase 1: extract all JDO data while transaction is alive
             Shepherd bgShepherd = new Shepherd(context);
             bgShepherd.setAction("EncounterSearchExportCOCO.export-" + jobId);
             bgShepherd.beginDBTransaction();
@@ -126,8 +128,20 @@ private void handleStart(HttpServletRequest request, HttpServletResponse respons
                     Encounter enc = bgShepherd.getEncounter(id);
                     if (enc != null) encounters.add(enc);
                 }
+                // Constructor eagerly extracts all JDO data into plain objects
                 job.exportFile = new EncounterCOCOExportFile(encounters, bgShepherd);
+            } catch (Throwable t) {
+                t.printStackTrace();
+                job.status = "error";
+                job.errorMessage = t.getMessage() != null ? t.getMessage() : t.getClass().getName();
+                return;
+            } finally {
+                bgShepherd.rollbackDBTransaction();
+                bgShepherd.closeDBTransaction();
+            }
 
+            // Phase 2: write ZIP (image downloads + JSON manifest) — no DB needed
+            try {
                 File tmpDir = new File(CommonConfiguration.getUploadTmpDir(context));
                 if (!tmpDir.exists()) tmpDir.mkdirs();
                 File tempFile = File.createTempFile("wildbook-coco-export-", ".zip", tmpDir);
@@ -137,13 +151,10 @@ private void handleStart(HttpServletRequest request, HttpServletResponse respons
                 }
                 job.tempFile = tempFile;
                 job.status = "complete";
-            } catch (Exception e) {
-                e.printStackTrace();
+            } catch (Throwable t) {
+                t.printStackTrace();
                 job.status = "error";
-                job.errorMessage = e.getMessage();
-            } finally {
-                bgShepherd.rollbackDBTransaction();
-                bgShepherd.closeDBTransaction();
+                job.errorMessage = t.getMessage() != null ? t.getMessage() : t.getClass().getName();
             }
         }, "coco-export-" + jobId);
         exportThread.setDaemon(true);
@@ -179,8 +190,7 @@ private void handleStatus(HttpServletRequest request, HttpServletResponse respon
     private void handleDownload(HttpServletRequest request, HttpServletResponse response)
             throws IOException {
         String jobId = request.getParameter("jobId");
-        // Atomic remove prevents concurrent downloads of the same job
-        ExportJob job = (jobId != null) ? jobs.remove(jobId) : null;
+        ExportJob job = (jobId != null) ? jobs.get(jobId) : null;
         if (job == null) {
             sendJson(response, 404, "{\"error\":\"Job not found\"}");
             return;
@@ -190,17 +200,15 @@ private void handleDownload(HttpServletRequest request, HttpServletResponse resp
             return;
         }
 
-        try {
-            response.setContentType("application/zip");
-            response.setHeader("Content-Disposition",
-                "attachment; filename=\"wildbook-coco-export.zip\"");
-            response.setContentLengthLong(job.tempFile.length());
-            OutputStream out = response.getOutputStream();
-            Files.copy(job.tempFile.toPath(), out);
-            out.flush();
-        } finally {
-            job.tempFile.delete();
-        }
+        // Job stays in the map — retryable until the 1-hour purge removes it.
+        // This accommodates slow connections and failed downloads.
+        response.setContentType("application/zip");
+        response.setHeader("Content-Disposition",
+            "attachment; filename=\"wildbook-coco-export.zip\"");
+        response.setContentLengthLong(job.tempFile.length());
+        OutputStream out = response.getOutputStream();
+        Files.copy(job.tempFile.toPath(), out);
+        out.flush();
     }
 
     /** Synchronous fallback for legacy/non-JS callers. */
@@ -212,7 +220,8 @@ private void handleSynchronous(HttpServletRequest request, HttpServletResponse r
         myShepherd.setAction("EncounterSearchExportCOCO.class");
         myShepherd.beginDBTransaction();
 
-        File tempFile = null;
+        // Phase 1: extract data while transaction is alive
+        EncounterCOCOExportFile exportFile;
         try {
             EncounterQueryResult queryResult = EncounterQueryProcessor.processQuery(
                 myShepherd, request, "year descending, month descending, day descending");
@@ -225,12 +234,25 @@ private void handleSynchronous(HttpServletRequest request, HttpServletResponse r
                     encounters.add(enc);
                 }
             }
+            exportFile = new EncounterCOCOExportFile(encounters, myShepherd);
+        } catch (Exception e) {
+            e.printStackTrace();
+            if (!response.isCommitted()) {
+                sendJson(response, 500, "{\"error\":\"" + escapeJson(e.getMessage()) + "\"}");
+            }
+            return;
+        } finally {
+            myShepherd.rollbackDBTransaction();
+            myShepherd.closeDBTransaction();
+        }
 
+        // Phase 2: write ZIP (no DB needed)
+        File tempFile = null;
+        try {
             File tmpDir = new File(CommonConfiguration.getUploadTmpDir(context));
             if (!tmpDir.exists()) tmpDir.mkdirs();
             tempFile = File.createTempFile("wildbook-coco-export-", ".zip", tmpDir);
             tempFile.deleteOnExit();
-            EncounterCOCOExportFile exportFile = new EncounterCOCOExportFile(encounters, myShepherd);
             try (FileOutputStream fos = new FileOutputStream(tempFile)) {
                 exportFile.writeTo(fos);
             }
@@ -249,8 +271,6 @@ private void handleSynchronous(HttpServletRequest request, HttpServletResponse r
                 sendJson(response, 500, "{\"error\":\"" + escapeJson(e.getMessage()) + "\"}");
             }
         } finally {
-            myShepherd.rollbackDBTransaction();
-            myShepherd.closeDBTransaction();
             if (tempFile != null) {
                 tempFile.delete();
             }

From 1c92642d1487cf96766befb36d3611de91bbd58c Mon Sep 17 00:00:00 2001
From: Wild Me <jason@wildme.org>
Date: Sat, 11 Apr 2026 15:53:11 -0700
Subject: [PATCH 6/9] More large dataset timeout fixing

---
 .../SearchPages/components/ExportModal.jsx    | 75 +++++++++++--------
 1 file changed, 45 insertions(+), 30 deletions(-)

diff --git a/frontend/src/pages/SearchPages/components/ExportModal.jsx b/frontend/src/pages/SearchPages/components/ExportModal.jsx
index 4a833ebdef..bfc6ec2121 100644
--- a/frontend/src/pages/SearchPages/components/ExportModal.jsx
+++ b/frontend/src/pages/SearchPages/components/ExportModal.jsx
@@ -89,6 +89,13 @@ export default function ExportDialog({ open, setOpen, searchQueryId }) {
     };
   }, []);
 
+  // Auto-expire the retry button after 1 hour (matches server-side job TTL)
+  useEffect(() => {
+    if (!cocoJobId) return;
+    const timer = setTimeout(() => setCocoJobId(null), 60 * 60 * 1000);
+    return () => clearTimeout(timer);
+  }, [cocoJobId]);
+
   const setLoading = (key, value) => {
     setLoadingStates((prev) => ({ ...prev, [key]: value }));
   };
@@ -166,10 +173,11 @@ export default function ExportDialog({ open, setOpen, searchQueryId }) {
         }, 3000);
       });
 
-      // Download the completed file
+      // Trigger native browser download — no buffering in JS memory.
+      // The Content-Disposition: attachment header tells the browser to
+      // save the file without navigating away from the page.
       setCocoJobId(result);
-      await downloadCocoFile(result);
-      setCocoJobId(null);
+      triggerCocoDownload(result);
     } catch (err) {
       console.error("COCO export error:", err);
       setError(`Failed to export: ${err.message}`);
@@ -183,37 +191,36 @@ export default function ExportDialog({ open, setOpen, searchQueryId }) {
     }
   }, [searchQueryId]);
 
-  const downloadCocoFile = useCallback(async (jobId) => {
-    const downloadResp = await fetch(
-      `/EncounterSearchExportCOCO?action=download&jobId=${jobId}`,
-    );
-    if (!downloadResp.ok) throw new Error("Download failed");
-    const blob = await downloadResp.blob();
-    const downloadUrl = window.URL.createObjectURL(blob);
+  const triggerCocoDownload = useCallback(async (jobId) => {
+    // Pre-flight check: verify the job is still available before triggering
+    // the native download (which can't surface HTTP errors in-app).
+    try {
+      const checkResp = await fetch(
+        `/EncounterSearchExportCOCO?action=status&jobId=${jobId}`,
+      );
+      if (!checkResp.ok) {
+        setCocoJobId(null);
+        setError("Export is no longer available. Please run a new export.");
+        return;
+      }
+      const checkData = await checkResp.json();
+      if (checkData.status !== "complete") {
+        setCocoJobId(null);
+        setError("Export is no longer available. Please run a new export.");
+        return;
+      }
+    } catch {
+      // Network error on pre-flight — try the download anyway
+    }
+
     const a = document.createElement("a");
-    a.href = downloadUrl;
+    a.href = `/EncounterSearchExportCOCO?action=download&jobId=${jobId}`;
     a.download = "wildbook-coco-export.zip";
     document.body.appendChild(a);
     a.click();
     a.remove();
-    window.URL.revokeObjectURL(downloadUrl);
   }, []);
 
-  const handleCocoRetry = useCallback(async () => {
-    if (!cocoJobId) return;
-    setError(null);
-    setLoading("cocoFormat", true);
-    try {
-      await downloadCocoFile(cocoJobId);
-      setCocoJobId(null);
-    } catch (err) {
-      console.error("COCO retry download error:", err);
-      setError(`Download failed: ${err.message}. You can retry.`);
-    } finally {
-      setLoading("cocoFormat", false);
-    }
-  }, [cocoJobId, downloadCocoFile]);
-
   const scrollToSection = (sectionId) => {
     setActiveSection(sectionId);
     const element = document.getElementById(sectionId);
@@ -378,7 +385,7 @@ export default function ExportDialog({ open, setOpen, searchQueryId }) {
                           className="my-3"
                           variant="outline-primary"
                           size="sm"
-                          onClick={cocoJobId ? handleCocoRetry : handleCocoExport}
+                          onClick={handleCocoExport}
                           disabled={loadingStates.cocoFormat}
                         >
                           {loadingStates.cocoFormat ? (
@@ -405,12 +412,20 @@ export default function ExportDialog({ open, setOpen, searchQueryId }) {
                                 : <FormattedMessage id="EXPORTING" />
                               }
                             </>
-                          ) : cocoJobId ? (
-                            <FormattedMessage id="COCO_RETRY_DOWNLOAD" />
                           ) : (
                             <FormattedMessage id="EXPORT_ZIP_FILE" />
                           )}
                         </Button>
+                        {cocoJobId && !loadingStates.cocoFormat && (
+                          <Button
+                            className="my-3"
+                            variant="primary"
+                            size="sm"
+                            onClick={() => triggerCocoDownload(cocoJobId)}
+                          >
+                            <FormattedMessage id="COCO_RETRY_DOWNLOAD" />
+                          </Button>
+                        )}
                       </div>
                     </Card.Body>
                   </Card>

From 70b301a684d7bfbb733b708f05b497b4bf57190d Mon Sep 17 00:00:00 2001
From: Wild Me <jason@wildme.org>
Date: Sat, 11 Apr 2026 22:10:45 -0700
Subject: [PATCH 7/9] Allow browser resume

---
 .../export/EncounterSearchExportCOCO.java     | 56 ++++++++++++++++++-
 1 file changed, 53 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/ecocean/servlet/export/EncounterSearchExportCOCO.java b/src/main/java/org/ecocean/servlet/export/EncounterSearchExportCOCO.java
index 46ea08b32c..62ba990e6c 100644
--- a/src/main/java/org/ecocean/servlet/export/EncounterSearchExportCOCO.java
+++ b/src/main/java/org/ecocean/servlet/export/EncounterSearchExportCOCO.java
@@ -18,6 +18,7 @@
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.io.RandomAccessFile;
 import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.Iterator;
@@ -201,13 +202,62 @@ private void handleDownload(HttpServletRequest request, HttpServletResponse resp
         }
 
         // Job stays in the map — retryable until the 1-hour purge removes it.
-        // This accommodates slow connections and failed downloads.
+        long fileLength = job.tempFile.length();
+        long start = 0;
+        long end = fileLength - 1;
+
+        // Support Range requests so browsers can resume interrupted downloads
+        // instead of restarting from the beginning.
+        String rangeHeader = request.getHeader("Range");
+        if (rangeHeader != null && rangeHeader.startsWith("bytes=")) {
+            String rangeSpec = rangeHeader.substring(6).trim();
+            String[] parts = rangeSpec.split("-", 2);
+            try {
+                if (!parts[0].isEmpty()) {
+                    start = Long.parseLong(parts[0]);
+                }
+                if (parts.length > 1 && !parts[1].isEmpty()) {
+                    end = Long.parseLong(parts[1]);
+                }
+            } catch (NumberFormatException e) {
+                response.setStatus(HttpServletResponse.SC_REQUESTED_RANGE_NOT_SATISFIABLE);
+                response.setHeader("Content-Range", "bytes */" + fileLength);
+                return;
+            }
+            if (start < 0 || start > end || start >= fileLength) {
+                response.setStatus(HttpServletResponse.SC_REQUESTED_RANGE_NOT_SATISFIABLE);
+                response.setHeader("Content-Range", "bytes */" + fileLength);
+                return;
+            }
+            if (end >= fileLength) {
+                end = fileLength - 1;
+            }
+            long contentLength = end - start + 1;
+            response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
+            response.setHeader("Content-Range",
+                "bytes " + start + "-" + end + "/" + fileLength);
+            response.setContentLengthLong(contentLength);
+        } else {
+            response.setContentLengthLong(fileLength);
+        }
+
         response.setContentType("application/zip");
         response.setHeader("Content-Disposition",
             "attachment; filename=\"wildbook-coco-export.zip\"");
-        response.setContentLengthLong(job.tempFile.length());
+        response.setHeader("Accept-Ranges", "bytes");
+
         OutputStream out = response.getOutputStream();
-        Files.copy(job.tempFile.toPath(), out);
+        try (RandomAccessFile raf = new RandomAccessFile(job.tempFile, "r")) {
+            raf.seek(start);
+            byte[] buffer = new byte[65536];
+            long remaining = end - start + 1;
+            int read;
+            while (remaining > 0 && (read = raf.read(buffer, 0,
+                    (int) Math.min(buffer.length, remaining))) != -1) {
+                out.write(buffer, 0, read);
+                remaining -= read;
+            }
+        }
         out.flush();
     }
 

From 1b767b0ad516146803d40459aca7a9ad7c7f7b40 Mon Sep 17 00:00:00 2001
From: Wild Me <jason@wildme.org>
Date: Sat, 11 Apr 2026 22:40:01 -0700
Subject: [PATCH 8/9] adapt for nginx reverse proxy

---
 .../export/EncounterSearchExportCOCO.java     | 28 +++++++++++++++++--
 1 file changed, 25 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/ecocean/servlet/export/EncounterSearchExportCOCO.java b/src/main/java/org/ecocean/servlet/export/EncounterSearchExportCOCO.java
index 62ba990e6c..30debce003 100644
--- a/src/main/java/org/ecocean/servlet/export/EncounterSearchExportCOCO.java
+++ b/src/main/java/org/ecocean/servlet/export/EncounterSearchExportCOCO.java
@@ -188,27 +188,34 @@ private void handleStatus(HttpServletRequest request, HttpServletResponse respon
         sendJson(response, 200, json.toString());
     }
 
+    private static final java.util.logging.Logger log =
+        java.util.logging.Logger.getLogger(EncounterSearchExportCOCO.class.getName());
+
     private void handleDownload(HttpServletRequest request, HttpServletResponse response)
             throws IOException {
         String jobId = request.getParameter("jobId");
         ExportJob job = (jobId != null) ? jobs.get(jobId) : null;
         if (job == null) {
+            log.warning("COCO Download: job not found, jobId=" + jobId);
             sendJson(response, 404, "{\"error\":\"Job not found\"}");
             return;
         }
         if (!"complete".equals(job.status) || job.tempFile == null || !job.tempFile.exists()) {
+            log.warning("COCO Download: job not ready, jobId=" + jobId +
+                " status=" + job.status + " tempFile=" + job.tempFile +
+                " exists=" + (job.tempFile != null && job.tempFile.exists()));
             sendJson(response, 400, "{\"error\":\"Export not ready\"}");
             return;
         }
 
-        // Job stays in the map — retryable until the 1-hour purge removes it.
         long fileLength = job.tempFile.length();
         long start = 0;
         long end = fileLength - 1;
 
-        // Support Range requests so browsers can resume interrupted downloads
-        // instead of restarting from the beginning.
         String rangeHeader = request.getHeader("Range");
+        log.info("COCO Download: jobId=" + jobId + " fileLength=" + fileLength +
+            " Range=" + rangeHeader + " method=" + request.getMethod());
+
         if (rangeHeader != null && rangeHeader.startsWith("bytes=")) {
             String rangeSpec = rangeHeader.substring(6).trim();
             String[] parts = rangeSpec.split("-", 2);
@@ -237,15 +244,24 @@ private void handleDownload(HttpServletRequest request, HttpServletResponse resp
             response.setHeader("Content-Range",
                 "bytes " + start + "-" + end + "/" + fileLength);
             response.setContentLengthLong(contentLength);
+            log.info("COCO Download: sending 206, start=" + start + " end=" + end +
+                " contentLength=" + contentLength);
         } else {
             response.setContentLengthLong(fileLength);
+            log.info("COCO Download: sending 200, full file, length=" + fileLength);
         }
 
         response.setContentType("application/zip");
         response.setHeader("Content-Disposition",
             "attachment; filename=\"wildbook-coco-export.zip\"");
         response.setHeader("Accept-Ranges", "bytes");
+        // Tell nginx to stream directly to the client instead of buffering.
+        // Without this, nginx buffers up to proxy_max_temp_file_size (default 1GB),
+        // then stalls Tomcat's writes, eventually timing out and cutting the connection
+        // — causing browsers to restart the download in an infinite loop.
+        response.setHeader("X-Accel-Buffering", "no");
 
+        long bytesSent = 0;
         OutputStream out = response.getOutputStream();
         try (RandomAccessFile raf = new RandomAccessFile(job.tempFile, "r")) {
             raf.seek(start);
@@ -256,9 +272,15 @@ private void handleDownload(HttpServletRequest request, HttpServletResponse resp
                     (int) Math.min(buffer.length, remaining))) != -1) {
                 out.write(buffer, 0, read);
                 remaining -= read;
+                bytesSent += read;
             }
+        } catch (IOException e) {
+            log.warning("COCO Download: connection broken after " + bytesSent +
+                " bytes (of " + (end - start + 1) + " expected). " + e.getMessage());
+            throw e;
         }
         out.flush();
+        log.info("COCO Download: completed, sent " + bytesSent + " bytes for jobId=" + jobId);
     }
 
     /** Synchronous fallback for legacy/non-JS callers. */

From 8cd4f3713839fdb4097215950321ebb104d82af3 Mon Sep 17 00:00:00 2001
From: Wild Me <jason@wildme.org>
Date: Sun, 12 Apr 2026 21:32:32 -0700
Subject: [PATCH 9/9] refine empty target annotations check

---
 src/main/java/org/ecocean/ia/Task.java | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/ecocean/ia/Task.java b/src/main/java/org/ecocean/ia/Task.java
index d840c5a561..b38d4b7a78 100644
--- a/src/main/java/org/ecocean/ia/Task.java
+++ b/src/main/java/org/ecocean/ia/Task.java
@@ -502,6 +502,14 @@ public String getStatus(Shepherd myShepherd) {
                 status = "completed";
             } else if (logs.toString().indexOf("score") > -1) {
                 status = "completed";
+            } else if (islObj.optJSONObject("status") != null &&
+                islObj.optJSONObject("status").optJSONObject("error") != null &&
+                islObj.optJSONObject("status").optJSONObject("error").optBoolean(
+                    "emptyTargetAnnotations", false)) {
+                // No target annotations to match against is a terminal state, not a failure.
+                // Treating it as completed lets import progress reach 100%.
+                System.out.println("[Task.getStatus] emptyTargetAnnotations detected for task " + getId() + ", marking completed");
+                status = "completed";
             } else if (islObj.toString().indexOf("HTTP error code") > -1) {
                 status = "error";
             } else if (!islObj.optString("queueStatus").equals("")) {
@@ -511,7 +519,7 @@ public String getStatus(Shepherd myShepherd) {
                 status = "queuing";
             }
             // if(islObj.optString("queueStatus").equals("queued")){sendIdentify=false;}
-            // if(status.equals("waiting to queue"))System.out.println("islObj: "+islObj.toString());
+            if(status.equals("waiting to queue"))System.out.println("[Task.getStatus DEBUG] waiting to queue, islObj: "+islObj.toString());
         }
         return status;
     }