From b5457e114eaec848a06f59af955cc61a53c8d8e2 Mon Sep 17 00:00:00 2001 From: Wild Me Date: Mon, 13 Apr 2026 16:41:58 -0700 Subject: [PATCH 1/3] Find missed Encounter/individual reindexing opportunities Find missed Encounter/individual reindexing opportunities --- src/main/java/org/ecocean/Encounter.java | 3 +++ .../java/org/ecocean/MarkedIndividual.java | 8 +++++++ .../servlet/IndividualAddEncounter.java | 3 +++ .../servlet/IndividualCreateForProject.java | 3 +++ .../servlet/IndividualRemoveEncounter.java | 13 ++++++++++- src/main/webapp/iaResultsSetID.jsp | 23 +++++++++++++++++++ 6 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/ecocean/Encounter.java b/src/main/java/org/ecocean/Encounter.java index b7c972c26b..58f7ca7699 100644 --- a/src/main/java/org/ecocean/Encounter.java +++ b/src/main/java/org/ecocean/Encounter.java @@ -5050,6 +5050,9 @@ public org.json.JSONObject processPatch(org.json.JSONArray patchArr, User user, this.setDWCDateLastModified(); this._log(resArr); this.setSkipAutoIndexing(false); + // Explicitly reindex since postStore fired while skipAutoIndexing was true + IndexingManager im = IndexingManagerFactory.getIndexingManager(); + if (im != null) im.addIndexingQueueEntry(this, false); return rtn; } diff --git a/src/main/java/org/ecocean/MarkedIndividual.java b/src/main/java/org/ecocean/MarkedIndividual.java index 601473162e..725901d535 100644 --- a/src/main/java/org/ecocean/MarkedIndividual.java +++ b/src/main/java/org/ecocean/MarkedIndividual.java @@ -2606,9 +2606,12 @@ public void removeFromNamesCache() { // Need request to record which user did it public void mergeIndividual(MarkedIndividual other, String username, Shepherd myShepherd) { + IndexingManager im = IndexingManagerFactory.getIndexingManager(); for (Encounter enc : other.getEncounters()) { other.removeEncounter(enc); enc.setIndividual(this); + // Reindex each transferred encounter + if (im != null) im.addIndexingQueueEntry(enc, false); } this.names.merge(other.getNames()); this.setComments(getMergedComments(other, username)); @@ -2653,6 +2656,11 @@ public void mergeIndividual(MarkedIndividual other, String username, Shepherd my myShepherd.updateDBTransaction(); } refreshDependentProperties(); + // Reindex both individuals after merge + if (im != null) { + im.addIndexingQueueEntry(this, false); + im.addIndexingQueueEntry(other, false); + } } public String getMergedComments(MarkedIndividual other, HttpServletRequest request, diff --git a/src/main/java/org/ecocean/servlet/IndividualAddEncounter.java b/src/main/java/org/ecocean/servlet/IndividualAddEncounter.java index 7fef424d75..768b8902da 100644 --- a/src/main/java/org/ecocean/servlet/IndividualAddEncounter.java +++ b/src/main/java/org/ecocean/servlet/IndividualAddEncounter.java @@ -109,6 +109,9 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) addToMe.refreshDependentProperties(); myShepherd.updateDBTransaction(); System.out.println("Now adding the Encounter to the individual"); + // Reindex encounter after individual assignment + IndexingManager im = IndexingManagerFactory.getIndexingManager(); + if (im != null) im.addIndexingQueueEntry(enc2add, false); } enc2add.setMatchedBy(request.getParameter("matchType")); enc2add.addComments("

" + request.getRemoteUser() + " on " + diff --git a/src/main/java/org/ecocean/servlet/IndividualCreateForProject.java b/src/main/java/org/ecocean/servlet/IndividualCreateForProject.java index a7484f7c41..85427e9eac 100644 --- a/src/main/java/org/ecocean/servlet/IndividualCreateForProject.java +++ b/src/main/java/org/ecocean/servlet/IndividualCreateForProject.java @@ -79,6 +79,9 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) myShepherd.updateDBTransaction(); enc.setIndividual(individual); myShepherd.updateDBTransaction(); + // Reindex encounter after individual assignment + IndexingManager im = IndexingManagerFactory.getIndexingManager(); + if (im != null) im.addIndexingQueueEntry(enc, false); res.put("newIndividualId", individual.getId()); res.put("newIndividualName", individual.getName(projectId)); diff --git a/src/main/java/org/ecocean/servlet/IndividualRemoveEncounter.java b/src/main/java/org/ecocean/servlet/IndividualRemoveEncounter.java index 141bd82796..49e1d9317f 100644 --- a/src/main/java/org/ecocean/servlet/IndividualRemoveEncounter.java +++ b/src/main/java/org/ecocean/servlet/IndividualRemoveEncounter.java @@ -2,6 +2,8 @@ import org.ecocean.CommonConfiguration; import org.ecocean.Encounter; +import org.ecocean.IndexingManager; +import org.ecocean.IndexingManagerFactory; import org.ecocean.MarkedIndividual; import org.ecocean.social.SocialUnit; import org.ecocean.shepherd.core.Shepherd; @@ -107,7 +109,16 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) } if (!locked) { myShepherd.commitDBTransaction(); - //if (enc2remove != null) enc2remove.opensearchIndexDeep(); + // Reindex encounter and old individual after assignment change + IndexingManager im = IndexingManagerFactory.getIndexingManager(); + if (im != null) { + if (enc2remove != null) { + im.addIndexingQueueEntry(enc2remove, false); + } + if (!wasRemoved && removeFromMe != null) { + im.addIndexingQueueEntry(removeFromMe, false); + } + } out.println(ServletUtilities.getHeader(request)); response.setStatus(HttpServletResponse.SC_OK); out.println("Success: Encounter #" + diff --git a/src/main/webapp/iaResultsSetID.jsp b/src/main/webapp/iaResultsSetID.jsp index 4ae9dc5d16..01ee4ceae4 100644 --- a/src/main/webapp/iaResultsSetID.jsp +++ b/src/main/webapp/iaResultsSetID.jsp @@ -242,6 +242,15 @@ if ((request.getParameter("taskId") != null) && (request.getParameter("number") setImportTaskComplete(myShepherd, oenc); } indiv.refreshNamesCache(); + // Reindex encounters and individual after ID assignment + IndexingManager im = IndexingManagerFactory.getIndexingManager(); + if (im != null) { + im.addIndexingQueueEntry(enc, false); + for (Encounter oenc : otherEncs) { + im.addIndexingQueueEntry(oenc, false); + } + im.addIndexingQueueEntry(indiv, false); + } if ((indiv != null) && (enc != null)) IndividualAddEncounter.executeEmails(myShepherd, request, indiv, isNewIndiv, enc, context, langCode); @@ -270,6 +279,14 @@ if ((request.getParameter("taskId") != null) && (request.getParameter("number") IndividualAddEncounter.executeEmails(myShepherd, request, indiv, false, oenc, context, langCode); setImportTaskComplete(myShepherd, oenc); } + // Reindex encounters and individual after ID assignment + IndexingManager im2 = IndexingManagerFactory.getIndexingManager(); + if (im2 != null) { + for (Encounter oenc : otherEncs) { + im2.addIndexingQueueEntry(oenc, false); + } + im2.addIndexingQueueEntry(indiv, false); + } } // target enc has indy @@ -291,6 +308,12 @@ if ((request.getParameter("taskId") != null) && (request.getParameter("number") IndividualAddEncounter.executeEmails(myShepherd, request, oindiv, false, enc, context, langCode); setImportTaskComplete(myShepherd, enc); + // Reindex encounter and individual after ID assignment + IndexingManager im3 = IndexingManagerFactory.getIndexingManager(); + if (im3 != null) { + im3.addIndexingQueueEntry(enc, false); + im3.addIndexingQueueEntry(oindiv, false); + } } From 5c958142543a8a09ebe3564236cfc4ca097a197a Mon Sep 17 00:00:00 2001 From: JasonWildMe Date: Fri, 17 Apr 2026 10:13:35 -0700 Subject: [PATCH 2/3] Fix stale individualNumberEncounters on sibling encounters (#1514) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a MarkedIndividual's encounter list changes at a mutation site that was not already queueing the individual for a deep reindex, sibling encounters retained stale denormalized fields in OpenSearch, including individualNumberEncounters, individualFirstEncounterDate, etc. Per Codex design review: queue the MarkedIndividual itself — not each encounter — because IndexingManager.addIndexingQueueEntry on an individual triggers opensearchIndexDeep, which cascades to every encounter with built-in per-id dedup. Adds a shared helper IndexingManager.queueIndividualsByIdForDeepReindex(Shepherd, Collection) and calls it post-commit at the missed mutation sites: - AnnotationEdit — annotation-level swap/assign of individuals - EncounterForm — manualID path on new encounter submission - IBEISIA.assignFromIAAPI — IA-driven individual assignment - EncounterVMData — Visual Matcher "matchID" path - ImportIA — per-name individual creation/reuse - merge.jsp — individual merge followed by encounter reassignment - BulkImporter / api.BulkImport — both fg and bg paths via getTouchedIndividualIds() on the importer - StandardImport — end-of-import sweep of individualCache values - ImportSRGD — rows that create/attach individuals - ImportExcelMetadata — first and second flows (latter is dead but kept defensive) - ImportTask.deleteWithRelated — returns surviving individual ids to the BulkImport caller for post-commit queueing - DeleteImportTask — same, inline Deliberately not addressed here (covered by #1548): pre-existing pre-commit queueing in Encounter.processPatch, MarkedIndividual.mergeIndividual, and iaResultsSetID.jsp. Those race against the enclosing commit and survive rollback; moving them requires a cross-cutting refactor that would balloon this PR. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../java/org/ecocean/IndexingManager.java | 18 ++++++++++ src/main/java/org/ecocean/api/BulkImport.java | 34 ++++++++++++++++++- .../org/ecocean/api/bulk/BulkImporter.java | 14 ++++++++ .../java/org/ecocean/identity/IBEISIA.java | 4 +++ .../org/ecocean/servlet/AnnotationEdit.java | 23 +++++++++++-- .../org/ecocean/servlet/EncounterForm.java | 12 +++++++ .../org/ecocean/servlet/EncounterVMData.java | 4 +++ .../servlet/importer/DeleteImportTask.java | 16 +++++++++ .../servlet/importer/ImportExcelMetadata.java | 20 +++++++++++ .../ecocean/servlet/importer/ImportIA.java | 4 +++ .../ecocean/servlet/importer/ImportSRGD.java | 14 ++++++++ .../ecocean/servlet/importer/ImportTask.java | 14 +++++++- .../servlet/importer/StandardImport.java | 6 ++++ src/main/webapp/merge.jsp | 8 ++++- 14 files changed, 185 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/ecocean/IndexingManager.java b/src/main/java/org/ecocean/IndexingManager.java index 8edb2ccdc3..358ac99fda 100644 --- a/src/main/java/org/ecocean/IndexingManager.java +++ b/src/main/java/org/ecocean/IndexingManager.java @@ -83,6 +83,24 @@ public void run() { } + // GH-1514: queue deep reindex for each MarkedIndividual identified by id, + // so sibling encounters pick up refreshed individualNumberEncounters (and + // the other individual-derived denormalized fields on the encounter index). + // Safe to call with an empty or null set. Callers should invoke this AFTER + // the caller's DB transaction has committed, since IndexingManager spins + // a background Shepherd that reads the individual by id. + public static void queueIndividualsByIdForDeepReindex(Shepherd myShepherd, + java.util.Collection individualIds) { + if ((myShepherd == null) || (individualIds == null) || individualIds.isEmpty()) return; + IndexingManager im = IndexingManagerFactory.getIndexingManager(); + if (im == null) return; + for (String id : individualIds) { + if (id == null) continue; + MarkedIndividual indiv = myShepherd.getMarkedIndividualQuiet(id); + if (indiv != null) im.addIndexingQueueEntry(indiv, false); + } + } + //Removes an oject's UUID from the queue public void removeIndexingQueueEntry(String objectID) { if (indexingQueue.contains(objectID)) { diff --git a/src/main/java/org/ecocean/api/BulkImport.java b/src/main/java/org/ecocean/api/BulkImport.java index c371fd4df4..4663aa8706 100644 --- a/src/main/java/org/ecocean/api/BulkImport.java +++ b/src/main/java/org/ecocean/api/BulkImport.java @@ -143,6 +143,9 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response) JSONObject matchingSetFilter = new JSONObject(); JSONObject encAssets = null; String dupId = null; // gets set as bulkImporId to be used in finally block + // GH-1514: hoisted so the finally block can queue a post-commit deep + // reindex of individuals the foreground importer touched. + BulkImporter fgImporter = null; long startProcess = System.currentTimeMillis(); Shepherd myShepherd = new Shepherd(context); @@ -425,6 +428,9 @@ public void run() { JSONObject bgEncAssets = null; boolean success = false; + // GH-1514: hoisted so the finally block can queue + // post-commit deep reindex of touched individuals. + BulkImporter bgImporter = null; try { User bgUser = bgShepherd.getUser(currentUsername); initializeImportTask(bulkImportId, bgUser, payload, @@ -448,6 +454,7 @@ public void run() { rtn.put("numberMediaAssetsCreated", maMap.size()); BulkImporter importer = new BulkImporter(bulkImportId, validatedRows, maMap, bgUser, bgShepherd); + bgImporter = importer; JSONObject results = null; if (!blockedByMAErrors) { try { @@ -518,6 +525,15 @@ public void run() { bgShepherd.rollbackDBTransaction(); } bgShepherd.closeDBTransaction(); + // GH-1514: post-commit, queue deep reindex of touched + // individuals so sibling encounters pick up refreshed + // individualNumberEncounters. The bulkOpensearchIndex + // pass only does shallow individual doc indexing. + if (success && bgImporter != null) { + org.ecocean.IndexingManager + .queueIndividualsByIdForDeepReindex(bgShepherd, + bgImporter.getTouchedIndividualIds()); + } if (success && !bgSkipDetection) initiateIA(bulkImportId, bgSkipIdentification, bgEncAssets, matchingSetFilter); @@ -562,6 +578,7 @@ public void run() { } else { BulkImporter importer = new BulkImporter(bulkImportId, validatedRows, maMap, currentUser, myShepherd); + fgImporter = importer; BulkImporter.logProgress(bulkImportId, "doPost: fg pre-createImport()", startTime); JSONObject results = importer.createImport(); @@ -624,6 +641,12 @@ public void run() { myShepherd.rollbackDBTransaction(); } myShepherd.closeDBTransaction(); + // GH-1514: post-commit, queue deep reindex of individuals touched by + // the foreground import so sibling encounters refresh individualNumberEncounters. + if ((statusCode == 200) && !validateOnly && fgImporter != null) { + org.ecocean.IndexingManager.queueIndividualsByIdForDeepReindex(myShepherd, + fgImporter.getTouchedIndividualIds()); + } if ((statusCode == 200) && !skipDetection) initiateIA(dupId, skipIdentification, encAssets, matchingSetFilter); } @@ -647,6 +670,8 @@ protected void doDelete(HttpServletRequest request, HttpServletResponse response Shepherd myShepherd = new Shepherd(context); myShepherd.setAction("api.Bulk.doDelete"); + java.util.Set touchedSurvivingIndividualIds = + new java.util.LinkedHashSet(); myShepherd.beginDBTransaction(); try { User currentUser = myShepherd.getUser(request); @@ -655,7 +680,8 @@ protected void doDelete(HttpServletRequest request, HttpServletResponse response if (args.length < 2) throw new ServletException("bad api path"); String bulkImportId = args[1]; // this may throw IOException (and will if currentUser is null or cannot delete) - ImportTask.deleteWithRelated(bulkImportId, currentUser, myShepherd); + touchedSurvivingIndividualIds = ImportTask.deleteWithRelated(bulkImportId, + currentUser, myShepherd); statusCode = 204; rtn.put("success", true); } catch (IOException ex) { @@ -676,6 +702,12 @@ protected void doDelete(HttpServletRequest request, HttpServletResponse response } myShepherd.closeDBTransaction(); } + // GH-1514: post-commit, queue deep reindex of individuals that survived + // the bulk-import delete so their remaining encounters refresh in OpenSearch. + if (statusCode == 204) { + org.ecocean.IndexingManager.queueIndividualsByIdForDeepReindex(myShepherd, + touchedSurvivingIndividualIds); + } response.setStatus(statusCode); response.setCharacterEncoding("UTF-8"); response.setHeader("Content-Type", "application/json"); diff --git a/src/main/java/org/ecocean/api/bulk/BulkImporter.java b/src/main/java/org/ecocean/api/bulk/BulkImporter.java index 2388d7b60f..8cf2e02ffa 100644 --- a/src/main/java/org/ecocean/api/bulk/BulkImporter.java +++ b/src/main/java/org/ecocean/api/bulk/BulkImporter.java @@ -49,6 +49,20 @@ public class BulkImporter { private Map encounterCache = new HashMap(); private Map occurrenceCache = new HashMap(); private Map individualCache = new HashMap(); + + // GH-1514: individual ids that were either created OR had encounters added + // during this import. Caller should queue these for a deep reindex AFTER + // committing, so sibling encounters on pre-existing individuals pick up + // refreshed individualNumberEncounters. + public java.util.Set getTouchedIndividualIds() { + java.util.Set ids = new java.util.LinkedHashSet(); + for (MarkedIndividual indiv : individualCache.values()) { + if (indiv != null && indiv.getIndividualID() != null) { + ids.add(indiv.getIndividualID()); + } + } + return ids; + } private Map userCache = new HashMap(); private Map keywordCache = new HashMap(); private Map projectCache = new HashMap(); diff --git a/src/main/java/org/ecocean/identity/IBEISIA.java b/src/main/java/org/ecocean/identity/IBEISIA.java index 43ae4650d8..5f5184c3c6 100644 --- a/src/main/java/org/ecocean/identity/IBEISIA.java +++ b/src/main/java/org/ecocean/identity/IBEISIA.java @@ -2559,6 +2559,10 @@ public static JSONObject assignFromIAAPI(JSONObject arg, Shepherd myShepherd, bo rtn.put("annotations", ja); } myShepherd.commitDBTransaction(); + // GH-1514: post-commit, queue deep reindex of the target individual so + // sibling encounters pick up refreshed individualNumberEncounters etc. + org.ecocean.IndexingManager.queueIndividualsByIdForDeepReindex(myShepherd, + java.util.Collections.singleton(indivId)); return rtn; } diff --git a/src/main/java/org/ecocean/servlet/AnnotationEdit.java b/src/main/java/org/ecocean/servlet/AnnotationEdit.java index 24e327ca8e..0fa4c5cc50 100644 --- a/src/main/java/org/ecocean/servlet/AnnotationEdit.java +++ b/src/main/java/org/ecocean/servlet/AnnotationEdit.java @@ -2,6 +2,8 @@ import java.io.IOException; import java.io.PrintWriter; +import java.util.LinkedHashSet; +import java.util.Set; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -11,6 +13,7 @@ import org.ecocean.AccessControl; import org.ecocean.Annotation; import org.ecocean.Encounter; +import org.ecocean.IndexingManager; import org.ecocean.media.Feature; import org.ecocean.media.MediaAsset; import org.ecocean.MarkedIndividual; @@ -53,6 +56,10 @@ public class AnnotationEdit extends HttpServlet { out.println("access denied"); } JSONObject rtn = new JSONObject("{\"success\": false}"); + // GH-1514: collect individual ids touched during this request so we can + // queue a post-commit deep reindex, refreshing individualNumberEncounters + // (and related denormalized fields) on every sibling encounter. + Set touchedIndividualIds = new LinkedHashSet<>(); String annId = jsonIn.optString("id", null); Annotation annot = myShepherd.getAnnotation(annId); if (annot == null) { @@ -83,6 +90,7 @@ public class AnnotationEdit extends HttpServlet { if (indiv != null) { indiv.removeEncounter(enc1); indiv.addEncounter(enc2); + touchedIndividualIds.add(indivId1); } } if (indivId2 != null) { @@ -90,6 +98,7 @@ public class AnnotationEdit extends HttpServlet { if (indiv != null) { indiv.removeEncounter(enc2); indiv.addEncounter(enc1); + touchedIndividualIds.add(indivId2); } } // enc2.setIndividualID(indivId1); @@ -179,9 +188,12 @@ public class AnnotationEdit extends HttpServlet { } else if (Util.stringExists(assignIndivId)) { Encounter enc = annot.findEncounter(myShepherd); if (enc.hasMarkedIndividual()) { - MarkedIndividual oldIndiv = myShepherd.getMarkedIndividualQuiet( - enc.getIndividualID()); - oldIndiv.removeEncounter(enc); + String oldIndivId = enc.getIndividualID(); + MarkedIndividual oldIndiv = myShepherd.getMarkedIndividualQuiet(oldIndivId); + if (oldIndiv != null) { + oldIndiv.removeEncounter(enc); + touchedIndividualIds.add(oldIndivId); + } } boolean newIndiv = false; MarkedIndividual indiv = myShepherd.getMarkedIndividualQuiet(assignIndivId); @@ -191,6 +203,7 @@ public class AnnotationEdit extends HttpServlet { } else { indiv.addEncounter(enc); } + touchedIndividualIds.add(assignIndivId); // enc.setIndividualID(assignIndivId); System.out.println("INFO: AnnotationEdit assigned " + indiv + " on " + enc + " via " + annot); @@ -202,6 +215,10 @@ public class AnnotationEdit extends HttpServlet { } if (rtn.optBoolean("success", false)) { myShepherd.commitDBTransaction(); + // GH-1514: post-commit, queue deep reindex of affected individuals so + // sibling encounters pick up refreshed individualNumberEncounters etc. + IndexingManager.queueIndividualsByIdForDeepReindex(myShepherd, + touchedIndividualIds); } else { myShepherd.rollbackDBTransaction(); } diff --git a/src/main/java/org/ecocean/servlet/EncounterForm.java b/src/main/java/org/ecocean/servlet/EncounterForm.java index 4586404df5..92f8a3dc19 100644 --- a/src/main/java/org/ecocean/servlet/EncounterForm.java +++ b/src/main/java/org/ecocean/servlet/EncounterForm.java @@ -27,6 +27,7 @@ import org.apache.commons.fileupload.servlet.ServletFileUpload; import org.ecocean.CommonConfiguration; import org.ecocean.Encounter; +import org.ecocean.IndexingManager; import org.ecocean.ia.Task; import org.ecocean.identity.IBEISIA; import org.ecocean.IAJsonProperties; @@ -620,6 +621,10 @@ else if (formValues.get("location") != null) { System.out.println(" ENCOUNTERFORM:"); System.out.println(" ENCOUNTERFORM:"); } + // GH-1514: track individual id so we can queue a post-commit deep + // reindex; new encounter on existing individual changes sibling + // individualNumberEncounters. + String manualIndID = null; if (formValues.get("manualID") != null && formValues.get("manualID").toString().length() > 0) { String indID = formValues.get("manualID").toString(); @@ -637,6 +642,7 @@ else if (formValues.get("location") != null) { } if (ind != null) enc.setIndividual(ind); enc.setFieldID(indID); + manualIndID = ind != null ? ind.getIndividualID() : indID; } if (formValues.get("occurrenceID") != null && formValues.get("occurrenceID").toString().length() > 0) { @@ -877,6 +883,12 @@ else if (formValues.get("location") != null) { if (!spamBot) { newnum = myShepherd.storeNewEncounter(enc, encID); enc.refreshAssetFormats(myShepherd); + // GH-1514: post-commit deep reindex so individualNumberEncounters + // updates on sibling encounters of the manually-assigned individual. + if (!"fail".equals(newnum) && manualIndID != null) { + IndexingManager.queueIndividualsByIdForDeepReindex(myShepherd, + java.util.Collections.singleton(manualIndID)); + } // *after* persisting this madness, then lets kick MediaAssets to IA for whatever fate awaits them // note: we dont send Annotations here, as they are always(forever?) trivial annotations, so pretty disposable diff --git a/src/main/java/org/ecocean/servlet/EncounterVMData.java b/src/main/java/org/ecocean/servlet/EncounterVMData.java index 242deb15a1..2d7bf153f6 100644 --- a/src/main/java/org/ecocean/servlet/EncounterVMData.java +++ b/src/main/java/org/ecocean/servlet/EncounterVMData.java @@ -92,6 +92,10 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) matchID + ".

"); enc.setMatchedBy("Visual Matcher"); myShepherd.commitDBTransaction(); + // GH-1514: post-commit, queue deep reindex of the individual + // so sibling encounters pick up refreshed denormalized fields. + org.ecocean.IndexingManager.queueIndividualsByIdForDeepReindex( + myShepherd, java.util.Collections.singleton(matchID)); redirUrl = "encounters/encounter.jsp?number=" + enc.getCatalogNumber(); } else { rtn.put("error", "unauthorized"); diff --git a/src/main/java/org/ecocean/servlet/importer/DeleteImportTask.java b/src/main/java/org/ecocean/servlet/importer/DeleteImportTask.java index a8e35c819e..29bdba54bd 100644 --- a/src/main/java/org/ecocean/servlet/importer/DeleteImportTask.java +++ b/src/main/java/org/ecocean/servlet/importer/DeleteImportTask.java @@ -45,6 +45,13 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) PrintWriter out = response.getWriter(); boolean locked = false; + // GH-1514: collect surviving individuals touched by this delete so we can + // queue deep reindex post-commit. Only individuals that survive (i.e. had + // other encounters besides those being removed) need reindex; dead-empty + // individuals are thrown away. + java.util.Set touchedSurvivingIndividualIds = + new java.util.LinkedHashSet(); + myShepherd.beginDBTransaction(); if (request.getParameter("taskID") != null && myShepherd.getImportTask(request.getParameter("taskID")) != null && @@ -99,6 +106,10 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) } myShepherd.throwAwayMarkedIndividual(mark); myShepherd.updateDBTransaction(); + } else { + // GH-1514: individual survives; its remaining encounters + // need a fresh individualNumberEncounters after commit. + touchedSurvivingIndividualIds.add(mark.getIndividualID()); } } // handle projects @@ -132,6 +143,11 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) myShepherd.closeDBTransaction(); } if (!locked) { + // GH-1514: post-commit, queue deep reindex for individuals that + // survived the delete so their remaining encounters get fresh + // individualNumberEncounters in OpenSearch. + org.ecocean.IndexingManager.queueIndividualsByIdForDeepReindex( + myShepherd, touchedSurvivingIndividualIds); out.println(ServletUtilities.getHeader(request)); out.println("Success! I have successfully removed ImportTask " + request.getParameter("taskID") + "."); diff --git a/src/main/java/org/ecocean/servlet/importer/ImportExcelMetadata.java b/src/main/java/org/ecocean/servlet/importer/ImportExcelMetadata.java index 4685f10b97..eb849d813f 100644 --- a/src/main/java/org/ecocean/servlet/importer/ImportExcelMetadata.java +++ b/src/main/java/org/ecocean/servlet/importer/ImportExcelMetadata.java @@ -81,6 +81,11 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) StringBuffer messages = new StringBuffer(); boolean successfullyWroteFile = false; File finalFile = new File(tempSubdir, "temp.csv"); + // GH-1514: collect individual ids touched by either import flow so we + // can queue a post-commit deep reindex and refresh individualNumberEncounters + // on sibling encounters. + java.util.Set touchedIndividualIds = + new java.util.LinkedHashSet(); try { MultipartParser mp = new MultipartParser(request, @@ -475,6 +480,10 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) myShepherd.commitDBTransaction(); if (newShark) { myShepherd.storeNewMarkedIndividual(indie); } + // GH-1514: remember the touched individual id. + if (indie != null && indie.getIndividualID() != null) { + touchedIndividualIds.add(indie.getIndividualID()); + } } } else { myShepherd.rollbackDBTransaction(); } // out.println("Imported row: "+line); @@ -493,6 +502,9 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) if (!locked) { myShepherd.commitDBTransaction(); myShepherd.closeDBTransaction(); + // GH-1514: post-commit, queue deep reindex of touched individuals. + org.ecocean.IndexingManager.queueIndividualsByIdForDeepReindex(myShepherd, + touchedIndividualIds); out.println(ServletUtilities.getHeader(request)); out.println( "

Success! I have successfully uploaded and imported your SRGD CSV file.

"); @@ -629,6 +641,14 @@ public void processExcel(File dataFile, HttpServletResponse response, boolean co myShepherd.beginDBTransaction(); if (ind != null) ind.addEncounter(enc); myShepherd.commitDBTransaction(); + // GH-1514: deep reindex the touched individual so sibling + // encounters refresh individualNumberEncounters. (processExcel + // currently has no in-tree callers but is kept on the safe side.) + if (ind != null && ind.getIndividualID() != null) { + org.ecocean.IndexingManager.queueIndividualsByIdForDeepReindex( + myShepherd, + java.util.Collections.singleton(ind.getIndividualID())); + } // New Close it. if (i % printPeriod == 0) { out.println("Parsed row (" + i + "), containing Enc " + diff --git a/src/main/java/org/ecocean/servlet/importer/ImportIA.java b/src/main/java/org/ecocean/servlet/importer/ImportIA.java index 3ef09ca20f..a92b28fa45 100644 --- a/src/main/java/org/ecocean/servlet/importer/ImportIA.java +++ b/src/main/java/org/ecocean/servlet/importer/ImportIA.java @@ -300,6 +300,10 @@ note also that adding encounters to individuals should(!) gracefully adjust the myShepherd.storeNewAnnotation(ann); } myShepherd.commitDBTransaction(); + // GH-1514: post-commit deep reindex so sibling encounters on + // the named individual pick up refreshed individualNumberEncounters. + org.ecocean.IndexingManager.queueIndividualsByIdForDeepReindex( + myShepherd, java.util.Collections.singleton(name)); String annLog = ""; String annWeb = ""; diff --git a/src/main/java/org/ecocean/servlet/importer/ImportSRGD.java b/src/main/java/org/ecocean/servlet/importer/ImportSRGD.java index 33867f71fa..c1d5ecea7b 100644 --- a/src/main/java/org/ecocean/servlet/importer/ImportSRGD.java +++ b/src/main/java/org/ecocean/servlet/importer/ImportSRGD.java @@ -71,6 +71,11 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) StringBuffer messages = new StringBuffer(); boolean successfullyWroteFile = false; File finalFile = new File(tempSubdir, "temp.csv"); + // GH-1514: individual ids touched across all rows; used at end-of-import + // to queue a post-commit deep reindex so sibling encounters refresh + // individualNumberEncounters etc. + java.util.Set touchedIndividualIds = + new java.util.LinkedHashSet(); try { MultipartParser mp = new MultipartParser(request, @@ -463,6 +468,11 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) myShepherd.commitDBTransaction(); if (newShark) { myShepherd.storeNewMarkedIndividual(indie); } + // GH-1514: remember the touched individual id so we + // can queue deep reindex at end-of-import, post-commit. + if (indie != null && indie.getIndividualID() != null) { + touchedIndividualIds.add(indie.getIndividualID()); + } } } else { myShepherd.rollbackDBTransaction(); } // out.println("Imported row: "+line); @@ -481,6 +491,10 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) if (!locked) { myShepherd.commitDBTransaction(); myShepherd.closeDBTransaction(); + // GH-1514: post-commit, queue deep reindex of touched individuals + // so their sibling encounters refresh individualNumberEncounters. + org.ecocean.IndexingManager.queueIndividualsByIdForDeepReindex(myShepherd, + touchedIndividualIds); out.println(ServletUtilities.getHeader(request)); out.println( "

Success! I have successfully uploaded and imported your SRGD CSV file.

"); diff --git a/src/main/java/org/ecocean/servlet/importer/ImportTask.java b/src/main/java/org/ecocean/servlet/importer/ImportTask.java index e84d2d8191..dac2d8a994 100644 --- a/src/main/java/org/ecocean/servlet/importer/ImportTask.java +++ b/src/main/java/org/ecocean/servlet/importer/ImportTask.java @@ -489,8 +489,15 @@ with the idea that a single commit outside (in the caller) should do the job. no an exception does a rollback, but very likely many of the steps up until that point has been commited, so not sure what state that leaves things in the actual db */ - public static void deleteWithRelated(String id, User user, Shepherd myShepherd) + // GH-1514: returns the set of MarkedIndividual ids whose encounter lists were + // reduced but not fully emptied by this delete. Callers should queue these + // individuals for deep reindex AFTER committing the enclosing transaction so + // their surviving encounters pick up refreshed individualNumberEncounters. + public static java.util.Set deleteWithRelated(String id, User user, + Shepherd myShepherd) throws IOException { + java.util.Set touchedSurvivingIndividualIds = + new java.util.LinkedHashSet(); if ((id == null) || (user == null)) throw new IOException("must provide id and user"); ImportTask itask = myShepherd.getImportTask(id); if (itask == null) throw new IOException("invalid ImportTask id=" + id); @@ -556,6 +563,10 @@ public static void deleteWithRelated(String id, User user, Shepherd myShepherd) } myShepherd.throwAwayMarkedIndividual(mark); // myShepherd.updateDBTransaction(); + } else { + // GH-1514: individual survives; its remaining encounters + // need a fresh individualNumberEncounters after commit. + touchedSurvivingIndividualIds.add(mark.getIndividualID()); } } // handle projects @@ -583,6 +594,7 @@ public static void deleteWithRelated(String id, User user, Shepherd myShepherd) throw new IOException("general exception on ImportTask delete: " + ex); } Util.mark("ImportTask.deleteWithRelated(" + id + ") completed"); + return touchedSurvivingIndividualIds; } // this is hobbled together from some complex code in import.jsp diff --git a/src/main/java/org/ecocean/servlet/importer/StandardImport.java b/src/main/java/org/ecocean/servlet/importer/StandardImport.java index 4503b26274..03927b211a 100644 --- a/src/main/java/org/ecocean/servlet/importer/StandardImport.java +++ b/src/main/java/org/ecocean/servlet/importer/StandardImport.java @@ -446,6 +446,12 @@ public void doImport(String filename, File dataFile, HttpServletRequest request, OpenSearch.setPermissionsNeeded(myShepherd, true); myShepherd.commitDBTransaction(); myShepherd.closeDBTransaction(); + // GH-1514: post-commit, queue deep reindex of every individual + // touched by this import so sibling encounters refresh + // individualNumberEncounters. individualCache values are the + // resolved MarkedIndividual UUIDs. + org.ecocean.IndexingManager.queueIndividualsByIdForDeepReindex(myShepherd, + new java.util.LinkedHashSet(individualCache.values())); if (itask != null) out.println("
  • ImportTask id = " + itask.getId() + "
    • "); diff --git a/src/main/webapp/merge.jsp b/src/main/webapp/merge.jsp index a6d12bb9f3..99e27c89a7 100644 --- a/src/main/webapp/merge.jsp +++ b/src/main/webapp/merge.jsp @@ -584,13 +584,19 @@ table.compareZone tr th { enc.setIndividual(markA); } - } + } catch (Exception e) { System.out.println("Exception on merge.jsp! indIdA="+indIdA+" indIdB="+indIdB); myShepherd.rollbackDBTransaction(); } finally { myShepherd.commitDBTransaction(); myShepherd.closeDBTransaction(); + // GH-1514: post-commit, queue a deep reindex of the retained individual + // so sibling encounters pick up refreshed individualNumberEncounters. + if (indIdA != null) { + org.ecocean.IndexingManager.queueIndividualsByIdForDeepReindex(myShepherd, + java.util.Collections.singleton(indIdA)); + } } %> From 4f6cb01b34175f33e2fa5fd396af8f1ff9fc497d Mon Sep 17 00:00:00 2001 From: JasonWildMe Date: Fri, 17 Apr 2026 10:24:28 -0700 Subject: [PATCH 3/3] Address Codex blockers on #1514 fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. IndexingManager.queueIndividualsByIdForDeepReindex was dereferencing the caller's Shepherd, which in servlets is closed by the finally block before the queue call runs (closeDBTransaction). The underlying PM is closed, so getMarkedIndividualQuiet returned null and every affected site silently queued nothing. Open a fresh short-lived read-only Shepherd for the id->object resolution instead. 2. StandardImport.loadIndividual only cached newly-created individuals in individualCache. Pre-existing individuals that had encounters added by the import were never queued for deep reindex — exactly the stale-sibling case #1514 is about. Also cache existing individuals when they are touched via addEncounter. 3. merge.jsp queued inside the finally block unconditionally, so the rollback path still reached the queue. Added a mergeSuccess flag so only the happy path triggers reindex queueing. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../java/org/ecocean/IndexingManager.java | 25 +++++++++++++++---- .../servlet/importer/StandardImport.java | 6 +++++ src/main/webapp/merge.jsp | 11 +++++--- 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/ecocean/IndexingManager.java b/src/main/java/org/ecocean/IndexingManager.java index 358ac99fda..7e4f1bd5ed 100644 --- a/src/main/java/org/ecocean/IndexingManager.java +++ b/src/main/java/org/ecocean/IndexingManager.java @@ -89,15 +89,30 @@ public void run() { // Safe to call with an empty or null set. Callers should invoke this AFTER // the caller's DB transaction has committed, since IndexingManager spins // a background Shepherd that reads the individual by id. + // + // Opens its own short-lived read-only Shepherd for the id->object resolution + // rather than reusing the caller's. Callers in servlets typically close their + // Shepherd in a finally block before (or alongside) queueing; reusing it here + // would silently no-op because getMarkedIndividualQuiet uses the underlying + // closed PersistenceManager. The passed-in Shepherd is used only for its + // context string. public static void queueIndividualsByIdForDeepReindex(Shepherd myShepherd, java.util.Collection individualIds) { - if ((myShepherd == null) || (individualIds == null) || individualIds.isEmpty()) return; + if ((individualIds == null) || individualIds.isEmpty()) return; IndexingManager im = IndexingManagerFactory.getIndexingManager(); if (im == null) return; - for (String id : individualIds) { - if (id == null) continue; - MarkedIndividual indiv = myShepherd.getMarkedIndividualQuiet(id); - if (indiv != null) im.addIndexingQueueEntry(indiv, false); + String context = (myShepherd != null) ? myShepherd.getContext() : "context0"; + Shepherd shep = new Shepherd(context); + shep.setAction("IndexingManager.queueIndividualsByIdForDeepReindex"); + shep.beginDBTransaction(); + try { + for (String id : individualIds) { + if (id == null) continue; + MarkedIndividual indiv = shep.getMarkedIndividualQuiet(id); + if (indiv != null) im.addIndexingQueueEntry(indiv, false); + } + } finally { + shep.rollbackAndClose(); } } diff --git a/src/main/java/org/ecocean/servlet/importer/StandardImport.java b/src/main/java/org/ecocean/servlet/importer/StandardImport.java index 03927b211a..474c762f24 100644 --- a/src/main/java/org/ecocean/servlet/importer/StandardImport.java +++ b/src/main/java/org/ecocean/servlet/importer/StandardImport.java @@ -1875,6 +1875,12 @@ public MarkedIndividual loadIndividual(Row row, Encounter enc, Shepherd myShephe if (!newIndividual) { mark.addEncounter(enc); enc.setIndividual(mark); + // GH-1514: pre-existing individual had an encounter added; cache + // its UUID so the end-of-import sweep queues it for deep reindex + // and refreshes sibling encounters' individualNumberEncounters. + if (mark.getIndividualID() != null) { + individualCache.put(individualID, mark.getIndividualID()); + } // System.out.println("loadIndividual notnew individual: "+mark.getDisplayName(request, myShepherd)); } else { enc.setIndividual(mark); diff --git a/src/main/webapp/merge.jsp b/src/main/webapp/merge.jsp index 99e27c89a7..18bd35f508 100644 --- a/src/main/webapp/merge.jsp +++ b/src/main/webapp/merge.jsp @@ -21,6 +21,8 @@ String langCode=ServletUtilities.getLanguageCode(request); String indIdA = request.getParameter("individualA"); String indIdB = request.getParameter("individualB"); String[] encIds = request.getParameterValues("encounterId"); +// GH-1514: tracks whether the merge succeeded; only then queue deep reindex. +boolean mergeSuccess = false; props = ShepherdProperties.getProperties("merge.properties", langCode,context); myShepherd.setAction("merge.jsp"); myShepherd.beginDBTransaction(); @@ -583,6 +585,8 @@ table.compareZone tr th { if (enc == null) throw new RuntimeException("Bad Encounter id=" + encId); enc.setIndividual(markA); } + // GH-1514: flag success so we queue deep reindex only on the happy path. + mergeSuccess = true; } catch (Exception e) { @@ -591,9 +595,10 @@ table.compareZone tr th { } finally { myShepherd.commitDBTransaction(); myShepherd.closeDBTransaction(); - // GH-1514: post-commit, queue a deep reindex of the retained individual - // so sibling encounters pick up refreshed individualNumberEncounters. - if (indIdA != null) { + // GH-1514: post-commit (success path only) queue a deep reindex of the + // retained individual so sibling encounters pick up refreshed + // individualNumberEncounters. + if (mergeSuccess && indIdA != null) { org.ecocean.IndexingManager.queueIndividualsByIdForDeepReindex(myShepherd, java.util.Collections.singleton(indIdA)); }