diff --git a/src/main/java/org/ecocean/Encounter.java b/src/main/java/org/ecocean/Encounter.java index b7c972c26b..58f7ca7699 100644 --- a/src/main/java/org/ecocean/Encounter.java +++ b/src/main/java/org/ecocean/Encounter.java @@ -5050,6 +5050,9 @@ public org.json.JSONObject processPatch(org.json.JSONArray patchArr, User user, this.setDWCDateLastModified(); this._log(resArr); this.setSkipAutoIndexing(false); + // Explicitly reindex since postStore fired while skipAutoIndexing was true + IndexingManager im = IndexingManagerFactory.getIndexingManager(); + if (im != null) im.addIndexingQueueEntry(this, false); return rtn; } diff --git a/src/main/java/org/ecocean/IndexingManager.java b/src/main/java/org/ecocean/IndexingManager.java index 8edb2ccdc3..7e4f1bd5ed 100644 --- a/src/main/java/org/ecocean/IndexingManager.java +++ b/src/main/java/org/ecocean/IndexingManager.java @@ -83,6 +83,39 @@ public void run() { } + // GH-1514: queue deep reindex for each MarkedIndividual identified by id, + // so sibling encounters pick up refreshed individualNumberEncounters (and + // the other individual-derived denormalized fields on the encounter index). + // Safe to call with an empty or null set. Callers should invoke this AFTER + // the caller's DB transaction has committed, since IndexingManager spins + // a background Shepherd that reads the individual by id. + // + // Opens its own short-lived read-only Shepherd for the id->object resolution + // rather than reusing the caller's. Callers in servlets typically close their + // Shepherd in a finally block before (or alongside) queueing; reusing it here + // would silently no-op because getMarkedIndividualQuiet uses the underlying + // closed PersistenceManager. The passed-in Shepherd is used only for its + // context string. + public static void queueIndividualsByIdForDeepReindex(Shepherd myShepherd, + java.util.Collection individualIds) { + if ((individualIds == null) || individualIds.isEmpty()) return; + IndexingManager im = IndexingManagerFactory.getIndexingManager(); + if (im == null) return; + String context = (myShepherd != null) ? myShepherd.getContext() : "context0"; + Shepherd shep = new Shepherd(context); + shep.setAction("IndexingManager.queueIndividualsByIdForDeepReindex"); + shep.beginDBTransaction(); + try { + for (String id : individualIds) { + if (id == null) continue; + MarkedIndividual indiv = shep.getMarkedIndividualQuiet(id); + if (indiv != null) im.addIndexingQueueEntry(indiv, false); + } + } finally { + shep.rollbackAndClose(); + } + } + //Removes an oject's UUID from the queue public void removeIndexingQueueEntry(String objectID) { if (indexingQueue.contains(objectID)) { diff --git a/src/main/java/org/ecocean/MarkedIndividual.java b/src/main/java/org/ecocean/MarkedIndividual.java index 601473162e..725901d535 100644 --- a/src/main/java/org/ecocean/MarkedIndividual.java +++ b/src/main/java/org/ecocean/MarkedIndividual.java @@ -2606,9 +2606,12 @@ public void removeFromNamesCache() { // Need request to record which user did it public void mergeIndividual(MarkedIndividual other, String username, Shepherd myShepherd) { + IndexingManager im = IndexingManagerFactory.getIndexingManager(); for (Encounter enc : other.getEncounters()) { other.removeEncounter(enc); enc.setIndividual(this); + // Reindex each transferred encounter + if (im != null) im.addIndexingQueueEntry(enc, false); } this.names.merge(other.getNames()); this.setComments(getMergedComments(other, username)); @@ -2653,6 +2656,11 @@ public void mergeIndividual(MarkedIndividual other, String username, Shepherd my myShepherd.updateDBTransaction(); } refreshDependentProperties(); + // Reindex both individuals after merge + if (im != null) { + im.addIndexingQueueEntry(this, false); + im.addIndexingQueueEntry(other, false); + } } public String getMergedComments(MarkedIndividual other, HttpServletRequest request, diff --git a/src/main/java/org/ecocean/api/BulkImport.java b/src/main/java/org/ecocean/api/BulkImport.java index c371fd4df4..4663aa8706 100644 --- a/src/main/java/org/ecocean/api/BulkImport.java +++ b/src/main/java/org/ecocean/api/BulkImport.java @@ -143,6 +143,9 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response) JSONObject matchingSetFilter = new JSONObject(); JSONObject encAssets = null; String dupId = null; // gets set as bulkImporId to be used in finally block + // GH-1514: hoisted so the finally block can queue a post-commit deep + // reindex of individuals the foreground importer touched. + BulkImporter fgImporter = null; long startProcess = System.currentTimeMillis(); Shepherd myShepherd = new Shepherd(context); @@ -425,6 +428,9 @@ public void run() { JSONObject bgEncAssets = null; boolean success = false; + // GH-1514: hoisted so the finally block can queue + // post-commit deep reindex of touched individuals. + BulkImporter bgImporter = null; try { User bgUser = bgShepherd.getUser(currentUsername); initializeImportTask(bulkImportId, bgUser, payload, @@ -448,6 +454,7 @@ public void run() { rtn.put("numberMediaAssetsCreated", maMap.size()); BulkImporter importer = new BulkImporter(bulkImportId, validatedRows, maMap, bgUser, bgShepherd); + bgImporter = importer; JSONObject results = null; if (!blockedByMAErrors) { try { @@ -518,6 +525,15 @@ public void run() { bgShepherd.rollbackDBTransaction(); } bgShepherd.closeDBTransaction(); + // GH-1514: post-commit, queue deep reindex of touched + // individuals so sibling encounters pick up refreshed + // individualNumberEncounters. The bulkOpensearchIndex + // pass only does shallow individual doc indexing. + if (success && bgImporter != null) { + org.ecocean.IndexingManager + .queueIndividualsByIdForDeepReindex(bgShepherd, + bgImporter.getTouchedIndividualIds()); + } if (success && !bgSkipDetection) initiateIA(bulkImportId, bgSkipIdentification, bgEncAssets, matchingSetFilter); @@ -562,6 +578,7 @@ public void run() { } else { BulkImporter importer = new BulkImporter(bulkImportId, validatedRows, maMap, currentUser, myShepherd); + fgImporter = importer; BulkImporter.logProgress(bulkImportId, "doPost: fg pre-createImport()", startTime); JSONObject results = importer.createImport(); @@ -624,6 +641,12 @@ public void run() { myShepherd.rollbackDBTransaction(); } myShepherd.closeDBTransaction(); + // GH-1514: post-commit, queue deep reindex of individuals touched by + // the foreground import so sibling encounters refresh individualNumberEncounters. + if ((statusCode == 200) && !validateOnly && fgImporter != null) { + org.ecocean.IndexingManager.queueIndividualsByIdForDeepReindex(myShepherd, + fgImporter.getTouchedIndividualIds()); + } if ((statusCode == 200) && !skipDetection) initiateIA(dupId, skipIdentification, encAssets, matchingSetFilter); } @@ -647,6 +670,8 @@ protected void doDelete(HttpServletRequest request, HttpServletResponse response Shepherd myShepherd = new Shepherd(context); myShepherd.setAction("api.Bulk.doDelete"); + java.util.Set touchedSurvivingIndividualIds = + new java.util.LinkedHashSet(); myShepherd.beginDBTransaction(); try { User currentUser = myShepherd.getUser(request); @@ -655,7 +680,8 @@ protected void doDelete(HttpServletRequest request, HttpServletResponse response if (args.length < 2) throw new ServletException("bad api path"); String bulkImportId = args[1]; // this may throw IOException (and will if currentUser is null or cannot delete) - ImportTask.deleteWithRelated(bulkImportId, currentUser, myShepherd); + touchedSurvivingIndividualIds = ImportTask.deleteWithRelated(bulkImportId, + currentUser, myShepherd); statusCode = 204; rtn.put("success", true); } catch (IOException ex) { @@ -676,6 +702,12 @@ protected void doDelete(HttpServletRequest request, HttpServletResponse response } myShepherd.closeDBTransaction(); } + // GH-1514: post-commit, queue deep reindex of individuals that survived + // the bulk-import delete so their remaining encounters refresh in OpenSearch. + if (statusCode == 204) { + org.ecocean.IndexingManager.queueIndividualsByIdForDeepReindex(myShepherd, + touchedSurvivingIndividualIds); + } response.setStatus(statusCode); response.setCharacterEncoding("UTF-8"); response.setHeader("Content-Type", "application/json"); diff --git a/src/main/java/org/ecocean/api/bulk/BulkImporter.java b/src/main/java/org/ecocean/api/bulk/BulkImporter.java index 2388d7b60f..8cf2e02ffa 100644 --- a/src/main/java/org/ecocean/api/bulk/BulkImporter.java +++ b/src/main/java/org/ecocean/api/bulk/BulkImporter.java @@ -49,6 +49,20 @@ public class BulkImporter { private Map encounterCache = new HashMap(); private Map occurrenceCache = new HashMap(); private Map individualCache = new HashMap(); + + // GH-1514: individual ids that were either created OR had encounters added + // during this import. Caller should queue these for a deep reindex AFTER + // committing, so sibling encounters on pre-existing individuals pick up + // refreshed individualNumberEncounters. + public java.util.Set getTouchedIndividualIds() { + java.util.Set ids = new java.util.LinkedHashSet(); + for (MarkedIndividual indiv : individualCache.values()) { + if (indiv != null && indiv.getIndividualID() != null) { + ids.add(indiv.getIndividualID()); + } + } + return ids; + } private Map userCache = new HashMap(); private Map keywordCache = new HashMap(); private Map projectCache = new HashMap(); diff --git a/src/main/java/org/ecocean/identity/IBEISIA.java b/src/main/java/org/ecocean/identity/IBEISIA.java index 43ae4650d8..5f5184c3c6 100644 --- a/src/main/java/org/ecocean/identity/IBEISIA.java +++ b/src/main/java/org/ecocean/identity/IBEISIA.java @@ -2559,6 +2559,10 @@ public static JSONObject assignFromIAAPI(JSONObject arg, Shepherd myShepherd, bo rtn.put("annotations", ja); } myShepherd.commitDBTransaction(); + // GH-1514: post-commit, queue deep reindex of the target individual so + // sibling encounters pick up refreshed individualNumberEncounters etc. + org.ecocean.IndexingManager.queueIndividualsByIdForDeepReindex(myShepherd, + java.util.Collections.singleton(indivId)); return rtn; } diff --git a/src/main/java/org/ecocean/servlet/AnnotationEdit.java b/src/main/java/org/ecocean/servlet/AnnotationEdit.java index 24e327ca8e..0fa4c5cc50 100644 --- a/src/main/java/org/ecocean/servlet/AnnotationEdit.java +++ b/src/main/java/org/ecocean/servlet/AnnotationEdit.java @@ -2,6 +2,8 @@ import java.io.IOException; import java.io.PrintWriter; +import java.util.LinkedHashSet; +import java.util.Set; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -11,6 +13,7 @@ import org.ecocean.AccessControl; import org.ecocean.Annotation; import org.ecocean.Encounter; +import org.ecocean.IndexingManager; import org.ecocean.media.Feature; import org.ecocean.media.MediaAsset; import org.ecocean.MarkedIndividual; @@ -53,6 +56,10 @@ public class AnnotationEdit extends HttpServlet { out.println("access denied"); } JSONObject rtn = new JSONObject("{\"success\": false}"); + // GH-1514: collect individual ids touched during this request so we can + // queue a post-commit deep reindex, refreshing individualNumberEncounters + // (and related denormalized fields) on every sibling encounter. + Set touchedIndividualIds = new LinkedHashSet<>(); String annId = jsonIn.optString("id", null); Annotation annot = myShepherd.getAnnotation(annId); if (annot == null) { @@ -83,6 +90,7 @@ public class AnnotationEdit extends HttpServlet { if (indiv != null) { indiv.removeEncounter(enc1); indiv.addEncounter(enc2); + touchedIndividualIds.add(indivId1); } } if (indivId2 != null) { @@ -90,6 +98,7 @@ public class AnnotationEdit extends HttpServlet { if (indiv != null) { indiv.removeEncounter(enc2); indiv.addEncounter(enc1); + touchedIndividualIds.add(indivId2); } } // enc2.setIndividualID(indivId1); @@ -179,9 +188,12 @@ public class AnnotationEdit extends HttpServlet { } else if (Util.stringExists(assignIndivId)) { Encounter enc = annot.findEncounter(myShepherd); if (enc.hasMarkedIndividual()) { - MarkedIndividual oldIndiv = myShepherd.getMarkedIndividualQuiet( - enc.getIndividualID()); - oldIndiv.removeEncounter(enc); + String oldIndivId = enc.getIndividualID(); + MarkedIndividual oldIndiv = myShepherd.getMarkedIndividualQuiet(oldIndivId); + if (oldIndiv != null) { + oldIndiv.removeEncounter(enc); + touchedIndividualIds.add(oldIndivId); + } } boolean newIndiv = false; MarkedIndividual indiv = myShepherd.getMarkedIndividualQuiet(assignIndivId); @@ -191,6 +203,7 @@ public class AnnotationEdit extends HttpServlet { } else { indiv.addEncounter(enc); } + touchedIndividualIds.add(assignIndivId); // enc.setIndividualID(assignIndivId); System.out.println("INFO: AnnotationEdit assigned " + indiv + " on " + enc + " via " + annot); @@ -202,6 +215,10 @@ public class AnnotationEdit extends HttpServlet { } if (rtn.optBoolean("success", false)) { myShepherd.commitDBTransaction(); + // GH-1514: post-commit, queue deep reindex of affected individuals so + // sibling encounters pick up refreshed individualNumberEncounters etc. + IndexingManager.queueIndividualsByIdForDeepReindex(myShepherd, + touchedIndividualIds); } else { myShepherd.rollbackDBTransaction(); } diff --git a/src/main/java/org/ecocean/servlet/EncounterForm.java b/src/main/java/org/ecocean/servlet/EncounterForm.java index 4586404df5..92f8a3dc19 100644 --- a/src/main/java/org/ecocean/servlet/EncounterForm.java +++ b/src/main/java/org/ecocean/servlet/EncounterForm.java @@ -27,6 +27,7 @@ import org.apache.commons.fileupload.servlet.ServletFileUpload; import org.ecocean.CommonConfiguration; import org.ecocean.Encounter; +import org.ecocean.IndexingManager; import org.ecocean.ia.Task; import org.ecocean.identity.IBEISIA; import org.ecocean.IAJsonProperties; @@ -620,6 +621,10 @@ else if (formValues.get("location") != null) { System.out.println(" ENCOUNTERFORM:"); System.out.println(" ENCOUNTERFORM:"); } + // GH-1514: track individual id so we can queue a post-commit deep + // reindex; new encounter on existing individual changes sibling + // individualNumberEncounters. + String manualIndID = null; if (formValues.get("manualID") != null && formValues.get("manualID").toString().length() > 0) { String indID = formValues.get("manualID").toString(); @@ -637,6 +642,7 @@ else if (formValues.get("location") != null) { } if (ind != null) enc.setIndividual(ind); enc.setFieldID(indID); + manualIndID = ind != null ? ind.getIndividualID() : indID; } if (formValues.get("occurrenceID") != null && formValues.get("occurrenceID").toString().length() > 0) { @@ -877,6 +883,12 @@ else if (formValues.get("location") != null) { if (!spamBot) { newnum = myShepherd.storeNewEncounter(enc, encID); enc.refreshAssetFormats(myShepherd); + // GH-1514: post-commit deep reindex so individualNumberEncounters + // updates on sibling encounters of the manually-assigned individual. + if (!"fail".equals(newnum) && manualIndID != null) { + IndexingManager.queueIndividualsByIdForDeepReindex(myShepherd, + java.util.Collections.singleton(manualIndID)); + } // *after* persisting this madness, then lets kick MediaAssets to IA for whatever fate awaits them // note: we dont send Annotations here, as they are always(forever?) trivial annotations, so pretty disposable diff --git a/src/main/java/org/ecocean/servlet/EncounterVMData.java b/src/main/java/org/ecocean/servlet/EncounterVMData.java index 242deb15a1..2d7bf153f6 100644 --- a/src/main/java/org/ecocean/servlet/EncounterVMData.java +++ b/src/main/java/org/ecocean/servlet/EncounterVMData.java @@ -92,6 +92,10 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) matchID + ".

"); enc.setMatchedBy("Visual Matcher"); myShepherd.commitDBTransaction(); + // GH-1514: post-commit, queue deep reindex of the individual + // so sibling encounters pick up refreshed denormalized fields. + org.ecocean.IndexingManager.queueIndividualsByIdForDeepReindex( + myShepherd, java.util.Collections.singleton(matchID)); redirUrl = "encounters/encounter.jsp?number=" + enc.getCatalogNumber(); } else { rtn.put("error", "unauthorized"); diff --git a/src/main/java/org/ecocean/servlet/IndividualAddEncounter.java b/src/main/java/org/ecocean/servlet/IndividualAddEncounter.java index 7fef424d75..768b8902da 100644 --- a/src/main/java/org/ecocean/servlet/IndividualAddEncounter.java +++ b/src/main/java/org/ecocean/servlet/IndividualAddEncounter.java @@ -109,6 +109,9 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) addToMe.refreshDependentProperties(); myShepherd.updateDBTransaction(); System.out.println("Now adding the Encounter to the individual"); + // Reindex encounter after individual assignment + IndexingManager im = IndexingManagerFactory.getIndexingManager(); + if (im != null) im.addIndexingQueueEntry(enc2add, false); } enc2add.setMatchedBy(request.getParameter("matchType")); enc2add.addComments("

" + request.getRemoteUser() + " on " + diff --git a/src/main/java/org/ecocean/servlet/IndividualCreateForProject.java b/src/main/java/org/ecocean/servlet/IndividualCreateForProject.java index a7484f7c41..85427e9eac 100644 --- a/src/main/java/org/ecocean/servlet/IndividualCreateForProject.java +++ b/src/main/java/org/ecocean/servlet/IndividualCreateForProject.java @@ -79,6 +79,9 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) myShepherd.updateDBTransaction(); enc.setIndividual(individual); myShepherd.updateDBTransaction(); + // Reindex encounter after individual assignment + IndexingManager im = IndexingManagerFactory.getIndexingManager(); + if (im != null) im.addIndexingQueueEntry(enc, false); res.put("newIndividualId", individual.getId()); res.put("newIndividualName", individual.getName(projectId)); diff --git a/src/main/java/org/ecocean/servlet/IndividualRemoveEncounter.java b/src/main/java/org/ecocean/servlet/IndividualRemoveEncounter.java index 141bd82796..49e1d9317f 100644 --- a/src/main/java/org/ecocean/servlet/IndividualRemoveEncounter.java +++ b/src/main/java/org/ecocean/servlet/IndividualRemoveEncounter.java @@ -2,6 +2,8 @@ import org.ecocean.CommonConfiguration; import org.ecocean.Encounter; +import org.ecocean.IndexingManager; +import org.ecocean.IndexingManagerFactory; import org.ecocean.MarkedIndividual; import org.ecocean.social.SocialUnit; import org.ecocean.shepherd.core.Shepherd; @@ -107,7 +109,16 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) } if (!locked) { myShepherd.commitDBTransaction(); - //if (enc2remove != null) enc2remove.opensearchIndexDeep(); + // Reindex encounter and old individual after assignment change + IndexingManager im = IndexingManagerFactory.getIndexingManager(); + if (im != null) { + if (enc2remove != null) { + im.addIndexingQueueEntry(enc2remove, false); + } + if (!wasRemoved && removeFromMe != null) { + im.addIndexingQueueEntry(removeFromMe, false); + } + } out.println(ServletUtilities.getHeader(request)); response.setStatus(HttpServletResponse.SC_OK); out.println("Success: Encounter #" + diff --git a/src/main/java/org/ecocean/servlet/importer/DeleteImportTask.java b/src/main/java/org/ecocean/servlet/importer/DeleteImportTask.java index a8e35c819e..29bdba54bd 100644 --- a/src/main/java/org/ecocean/servlet/importer/DeleteImportTask.java +++ b/src/main/java/org/ecocean/servlet/importer/DeleteImportTask.java @@ -45,6 +45,13 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) PrintWriter out = response.getWriter(); boolean locked = false; + // GH-1514: collect surviving individuals touched by this delete so we can + // queue deep reindex post-commit. Only individuals that survive (i.e. had + // other encounters besides those being removed) need reindex; dead-empty + // individuals are thrown away. + java.util.Set touchedSurvivingIndividualIds = + new java.util.LinkedHashSet(); + myShepherd.beginDBTransaction(); if (request.getParameter("taskID") != null && myShepherd.getImportTask(request.getParameter("taskID")) != null && @@ -99,6 +106,10 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) } myShepherd.throwAwayMarkedIndividual(mark); myShepherd.updateDBTransaction(); + } else { + // GH-1514: individual survives; its remaining encounters + // need a fresh individualNumberEncounters after commit. + touchedSurvivingIndividualIds.add(mark.getIndividualID()); } } // handle projects @@ -132,6 +143,11 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) myShepherd.closeDBTransaction(); } if (!locked) { + // GH-1514: post-commit, queue deep reindex for individuals that + // survived the delete so their remaining encounters get fresh + // individualNumberEncounters in OpenSearch. + org.ecocean.IndexingManager.queueIndividualsByIdForDeepReindex( + myShepherd, touchedSurvivingIndividualIds); out.println(ServletUtilities.getHeader(request)); out.println("Success! I have successfully removed ImportTask " + request.getParameter("taskID") + "."); diff --git a/src/main/java/org/ecocean/servlet/importer/ImportExcelMetadata.java b/src/main/java/org/ecocean/servlet/importer/ImportExcelMetadata.java index 4685f10b97..eb849d813f 100644 --- a/src/main/java/org/ecocean/servlet/importer/ImportExcelMetadata.java +++ b/src/main/java/org/ecocean/servlet/importer/ImportExcelMetadata.java @@ -81,6 +81,11 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) StringBuffer messages = new StringBuffer(); boolean successfullyWroteFile = false; File finalFile = new File(tempSubdir, "temp.csv"); + // GH-1514: collect individual ids touched by either import flow so we + // can queue a post-commit deep reindex and refresh individualNumberEncounters + // on sibling encounters. + java.util.Set touchedIndividualIds = + new java.util.LinkedHashSet(); try { MultipartParser mp = new MultipartParser(request, @@ -475,6 +480,10 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) myShepherd.commitDBTransaction(); if (newShark) { myShepherd.storeNewMarkedIndividual(indie); } + // GH-1514: remember the touched individual id. + if (indie != null && indie.getIndividualID() != null) { + touchedIndividualIds.add(indie.getIndividualID()); + } } } else { myShepherd.rollbackDBTransaction(); } // out.println("Imported row: "+line); @@ -493,6 +502,9 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) if (!locked) { myShepherd.commitDBTransaction(); myShepherd.closeDBTransaction(); + // GH-1514: post-commit, queue deep reindex of touched individuals. + org.ecocean.IndexingManager.queueIndividualsByIdForDeepReindex(myShepherd, + touchedIndividualIds); out.println(ServletUtilities.getHeader(request)); out.println( "

Success! I have successfully uploaded and imported your SRGD CSV file.

"); @@ -629,6 +641,14 @@ public void processExcel(File dataFile, HttpServletResponse response, boolean co myShepherd.beginDBTransaction(); if (ind != null) ind.addEncounter(enc); myShepherd.commitDBTransaction(); + // GH-1514: deep reindex the touched individual so sibling + // encounters refresh individualNumberEncounters. (processExcel + // currently has no in-tree callers but is kept on the safe side.) + if (ind != null && ind.getIndividualID() != null) { + org.ecocean.IndexingManager.queueIndividualsByIdForDeepReindex( + myShepherd, + java.util.Collections.singleton(ind.getIndividualID())); + } // New Close it. if (i % printPeriod == 0) { out.println("Parsed row (" + i + "), containing Enc " + diff --git a/src/main/java/org/ecocean/servlet/importer/ImportIA.java b/src/main/java/org/ecocean/servlet/importer/ImportIA.java index 3ef09ca20f..a92b28fa45 100644 --- a/src/main/java/org/ecocean/servlet/importer/ImportIA.java +++ b/src/main/java/org/ecocean/servlet/importer/ImportIA.java @@ -300,6 +300,10 @@ note also that adding encounters to individuals should(!) gracefully adjust the myShepherd.storeNewAnnotation(ann); } myShepherd.commitDBTransaction(); + // GH-1514: post-commit deep reindex so sibling encounters on + // the named individual pick up refreshed individualNumberEncounters. + org.ecocean.IndexingManager.queueIndividualsByIdForDeepReindex( + myShepherd, java.util.Collections.singleton(name)); String annLog = ""; String annWeb = ""; diff --git a/src/main/java/org/ecocean/servlet/importer/ImportSRGD.java b/src/main/java/org/ecocean/servlet/importer/ImportSRGD.java index 33867f71fa..c1d5ecea7b 100644 --- a/src/main/java/org/ecocean/servlet/importer/ImportSRGD.java +++ b/src/main/java/org/ecocean/servlet/importer/ImportSRGD.java @@ -71,6 +71,11 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) StringBuffer messages = new StringBuffer(); boolean successfullyWroteFile = false; File finalFile = new File(tempSubdir, "temp.csv"); + // GH-1514: individual ids touched across all rows; used at end-of-import + // to queue a post-commit deep reindex so sibling encounters refresh + // individualNumberEncounters etc. + java.util.Set touchedIndividualIds = + new java.util.LinkedHashSet(); try { MultipartParser mp = new MultipartParser(request, @@ -463,6 +468,11 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) myShepherd.commitDBTransaction(); if (newShark) { myShepherd.storeNewMarkedIndividual(indie); } + // GH-1514: remember the touched individual id so we + // can queue deep reindex at end-of-import, post-commit. + if (indie != null && indie.getIndividualID() != null) { + touchedIndividualIds.add(indie.getIndividualID()); + } } } else { myShepherd.rollbackDBTransaction(); } // out.println("Imported row: "+line); @@ -481,6 +491,10 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) if (!locked) { myShepherd.commitDBTransaction(); myShepherd.closeDBTransaction(); + // GH-1514: post-commit, queue deep reindex of touched individuals + // so their sibling encounters refresh individualNumberEncounters. + org.ecocean.IndexingManager.queueIndividualsByIdForDeepReindex(myShepherd, + touchedIndividualIds); out.println(ServletUtilities.getHeader(request)); out.println( "

Success! I have successfully uploaded and imported your SRGD CSV file.

"); diff --git a/src/main/java/org/ecocean/servlet/importer/ImportTask.java b/src/main/java/org/ecocean/servlet/importer/ImportTask.java index e84d2d8191..dac2d8a994 100644 --- a/src/main/java/org/ecocean/servlet/importer/ImportTask.java +++ b/src/main/java/org/ecocean/servlet/importer/ImportTask.java @@ -489,8 +489,15 @@ with the idea that a single commit outside (in the caller) should do the job. no an exception does a rollback, but very likely many of the steps up until that point has been commited, so not sure what state that leaves things in the actual db */ - public static void deleteWithRelated(String id, User user, Shepherd myShepherd) + // GH-1514: returns the set of MarkedIndividual ids whose encounter lists were + // reduced but not fully emptied by this delete. Callers should queue these + // individuals for deep reindex AFTER committing the enclosing transaction so + // their surviving encounters pick up refreshed individualNumberEncounters. + public static java.util.Set deleteWithRelated(String id, User user, + Shepherd myShepherd) throws IOException { + java.util.Set touchedSurvivingIndividualIds = + new java.util.LinkedHashSet(); if ((id == null) || (user == null)) throw new IOException("must provide id and user"); ImportTask itask = myShepherd.getImportTask(id); if (itask == null) throw new IOException("invalid ImportTask id=" + id); @@ -556,6 +563,10 @@ public static void deleteWithRelated(String id, User user, Shepherd myShepherd) } myShepherd.throwAwayMarkedIndividual(mark); // myShepherd.updateDBTransaction(); + } else { + // GH-1514: individual survives; its remaining encounters + // need a fresh individualNumberEncounters after commit. + touchedSurvivingIndividualIds.add(mark.getIndividualID()); } } // handle projects @@ -583,6 +594,7 @@ public static void deleteWithRelated(String id, User user, Shepherd myShepherd) throw new IOException("general exception on ImportTask delete: " + ex); } Util.mark("ImportTask.deleteWithRelated(" + id + ") completed"); + return touchedSurvivingIndividualIds; } // this is hobbled together from some complex code in import.jsp diff --git a/src/main/java/org/ecocean/servlet/importer/StandardImport.java b/src/main/java/org/ecocean/servlet/importer/StandardImport.java index 4503b26274..474c762f24 100644 --- a/src/main/java/org/ecocean/servlet/importer/StandardImport.java +++ b/src/main/java/org/ecocean/servlet/importer/StandardImport.java @@ -446,6 +446,12 @@ public void doImport(String filename, File dataFile, HttpServletRequest request, OpenSearch.setPermissionsNeeded(myShepherd, true); myShepherd.commitDBTransaction(); myShepherd.closeDBTransaction(); + // GH-1514: post-commit, queue deep reindex of every individual + // touched by this import so sibling encounters refresh + // individualNumberEncounters. individualCache values are the + // resolved MarkedIndividual UUIDs. + org.ecocean.IndexingManager.queueIndividualsByIdForDeepReindex(myShepherd, + new java.util.LinkedHashSet(individualCache.values())); if (itask != null) out.println("
  • ImportTask id = " + itask.getId() + "
  • "); @@ -1869,6 +1875,12 @@ public MarkedIndividual loadIndividual(Row row, Encounter enc, Shepherd myShephe if (!newIndividual) { mark.addEncounter(enc); enc.setIndividual(mark); + // GH-1514: pre-existing individual had an encounter added; cache + // its UUID so the end-of-import sweep queues it for deep reindex + // and refreshes sibling encounters' individualNumberEncounters. + if (mark.getIndividualID() != null) { + individualCache.put(individualID, mark.getIndividualID()); + } // System.out.println("loadIndividual notnew individual: "+mark.getDisplayName(request, myShepherd)); } else { enc.setIndividual(mark); diff --git a/src/main/webapp/iaResultsSetID.jsp b/src/main/webapp/iaResultsSetID.jsp index 4ae9dc5d16..01ee4ceae4 100644 --- a/src/main/webapp/iaResultsSetID.jsp +++ b/src/main/webapp/iaResultsSetID.jsp @@ -242,6 +242,15 @@ if ((request.getParameter("taskId") != null) && (request.getParameter("number") setImportTaskComplete(myShepherd, oenc); } indiv.refreshNamesCache(); + // Reindex encounters and individual after ID assignment + IndexingManager im = IndexingManagerFactory.getIndexingManager(); + if (im != null) { + im.addIndexingQueueEntry(enc, false); + for (Encounter oenc : otherEncs) { + im.addIndexingQueueEntry(oenc, false); + } + im.addIndexingQueueEntry(indiv, false); + } if ((indiv != null) && (enc != null)) IndividualAddEncounter.executeEmails(myShepherd, request, indiv, isNewIndiv, enc, context, langCode); @@ -270,6 +279,14 @@ if ((request.getParameter("taskId") != null) && (request.getParameter("number") IndividualAddEncounter.executeEmails(myShepherd, request, indiv, false, oenc, context, langCode); setImportTaskComplete(myShepherd, oenc); } + // Reindex encounters and individual after ID assignment + IndexingManager im2 = IndexingManagerFactory.getIndexingManager(); + if (im2 != null) { + for (Encounter oenc : otherEncs) { + im2.addIndexingQueueEntry(oenc, false); + } + im2.addIndexingQueueEntry(indiv, false); + } } // target enc has indy @@ -291,6 +308,12 @@ if ((request.getParameter("taskId") != null) && (request.getParameter("number") IndividualAddEncounter.executeEmails(myShepherd, request, oindiv, false, enc, context, langCode); setImportTaskComplete(myShepherd, enc); + // Reindex encounter and individual after ID assignment + IndexingManager im3 = IndexingManagerFactory.getIndexingManager(); + if (im3 != null) { + im3.addIndexingQueueEntry(enc, false); + im3.addIndexingQueueEntry(oindiv, false); + } } diff --git a/src/main/webapp/merge.jsp b/src/main/webapp/merge.jsp index a6d12bb9f3..18bd35f508 100644 --- a/src/main/webapp/merge.jsp +++ b/src/main/webapp/merge.jsp @@ -21,6 +21,8 @@ String langCode=ServletUtilities.getLanguageCode(request); String indIdA = request.getParameter("individualA"); String indIdB = request.getParameter("individualB"); String[] encIds = request.getParameterValues("encounterId"); +// GH-1514: tracks whether the merge succeeded; only then queue deep reindex. +boolean mergeSuccess = false; props = ShepherdProperties.getProperties("merge.properties", langCode,context); myShepherd.setAction("merge.jsp"); myShepherd.beginDBTransaction(); @@ -583,14 +585,23 @@ table.compareZone tr th { if (enc == null) throw new RuntimeException("Bad Encounter id=" + encId); enc.setIndividual(markA); } + // GH-1514: flag success so we queue deep reindex only on the happy path. + mergeSuccess = true; - } + } catch (Exception e) { System.out.println("Exception on merge.jsp! indIdA="+indIdA+" indIdB="+indIdB); myShepherd.rollbackDBTransaction(); } finally { myShepherd.commitDBTransaction(); myShepherd.closeDBTransaction(); + // GH-1514: post-commit (success path only) queue a deep reindex of the + // retained individual so sibling encounters pick up refreshed + // individualNumberEncounters. + if (mergeSuccess && indIdA != null) { + org.ecocean.IndexingManager.queueIndividualsByIdForDeepReindex(myShepherd, + java.util.Collections.singleton(indIdA)); + } } %>