From 023342fbcdfdc3d098f1a03ed7874510a94e26be Mon Sep 17 00:00:00 2001 From: DerekFurstPitt Date: Thu, 26 Mar 2026 16:27:18 -0400 Subject: [PATCH 1/2] WIP: changed how enqueing is handled all at the same time in redis --- src/hubmap_translator.py | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/src/hubmap_translator.py b/src/hubmap_translator.py index 8e99f208..526ed958 100644 --- a/src/hubmap_translator.py +++ b/src/hubmap_translator.py @@ -798,24 +798,31 @@ def enqueue_reindex(self, entity_id, reindex_queue, priority): url = f"{self.entity_api_url}/entities/batch-ids" associated_metadata = {} - try: - response = requests.post(url, headers=self.request_headers, json=list(target_ids)) - if response.status_code == 200: - associated_metadata = response.json() - else: - self.logger.error(f"Failed to fetch batch metadata: {response.status_code}") - associated_metadata = {} - except Exception as e: - logger.error(f"Unable to retrieve uuid and hubmap_id from entity-api. Proceed with enqueuing but this info will be missing from logging and status. {e}") + # try: + # response = requests.post(url, headers=self.request_headers, json=list(target_ids)) + # if response.status_code == 200: + # associated_metadata = response.json() + # else: + # self.logger.error(f"Failed to fetch batch metadata: {response.status_code}") + # associated_metadata = {} + # except Exception as e: + # logger.error(f"Unable to retrieve uuid and hubmap_id from entity-api. Proceed with enqueuing but this info will be missing from logging and status. {e}") + jobs = [] for related_entity_id in target_ids: - reindex_queue.enqueue( - job_metadata = associated_metadata.get(related_entity_id), - task_func=reindex_entity_queued_wrapper, - entity_id=related_entity_id, - args=[related_entity_id, self.token], + meta = associated_metadata.get(related_entity_id) or {} + jobs.append({ + "entity_id": related_entity_id, + "args": [related_entity_id, self.token], + "kwargs": {}, + "metadata": meta, + }) + if jobs: + reindex_queue.bulk_enqueue( + task_func = reindex_entity_queued_wrapper, + jobs=jobs, priority=subsequent_priority ) - logger.info(f"Finished executing translate() on {entity['entity_type']} of uuid: {entity_id}") + logger.info(f"Bulk-enqueued {len(jobs)} related entities for {entity_id}") return reference_id except ValueError as e: raise ValueError(e) From b16a4604ba5c5ad6b069efb5cb2a7da8b52cddb8 Mon Sep 17 00:00:00 2001 From: DerekFurstPitt Date: Fri, 27 Mar 2026 15:57:46 -0400 Subject: [PATCH 2/2] readded batch transfer call to hubmap_translator enqueue reindex --- src/hubmap_translator.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/hubmap_translator.py b/src/hubmap_translator.py index 526ed958..d4727daf 100644 --- a/src/hubmap_translator.py +++ b/src/hubmap_translator.py @@ -798,15 +798,15 @@ def enqueue_reindex(self, entity_id, reindex_queue, priority): url = f"{self.entity_api_url}/entities/batch-ids" associated_metadata = {} - # try: - # response = requests.post(url, headers=self.request_headers, json=list(target_ids)) - # if response.status_code == 200: - # associated_metadata = response.json() - # else: - # self.logger.error(f"Failed to fetch batch metadata: {response.status_code}") - # associated_metadata = {} - # except Exception as e: - # logger.error(f"Unable to retrieve uuid and hubmap_id from entity-api. Proceed with enqueuing but this info will be missing from logging and status. {e}") + try: + response = requests.post(url, headers=self.request_headers, json=list(target_ids)) + if response.status_code == 200: + associated_metadata = response.json() + else: + self.logger.error(f"Failed to fetch batch metadata: {response.status_code}") + associated_metadata = {} + except Exception as e: + logger.error(f"Unable to retrieve uuid and hubmap_id from entity-api. Proceed with enqueuing but this info will be missing from logging and status. {e}") jobs = [] for related_entity_id in target_ids: meta = associated_metadata.get(related_entity_id) or {}