handling duplicates in csv in attach URLs, disabling fuse generation in exports for projects with attached URLs

Author: ero1311

superannotate/db/exports.py

@@ -12,7 +12,7 @@
 from tqdm import tqdm
 
 from ..api import API
-from ..common import annotation_status_str_to_int
+from ..common import annotation_status_str_to_int, upload_state_int_to_str
 from ..exceptions import (
     SABaseException, SAExistingExportNameException,
     SANonExistingExportNameException
@@ -123,6 +123,11 @@ def prepare_export(
     """
     if not isinstance(project, dict):
         project = get_project_metadata_bare(project)
+    upload_state = upload_state_int_to_str(project.get("upload_state"))
+    if upload_state == "External" and include_fuse == True:
+        logger.info(
+            "Include fuse functionality is not supported for  projects containing images attached with URLs"
+        )
     team_id, project_id = project["team_id"], project["id"]
     if annotation_statuses is None:
         annotation_statuses = [2, 3, 4, 5]

superannotate/db/projects.py

@@ -973,7 +973,11 @@ def attach_image_urls_to_project(
     annotation_status = common.annotation_status_str_to_int(annotation_status)
     team_id, project_id = project["team_id"], project["id"]
     image_data = pd.read_csv(attachments)
+    image_data = image_data[~image_data["url"].isnull()]
     existing_names = image_data[~image_data["name"].isnull()]
+    duplicate_idx_csv = existing_names.duplicated(subset="name", keep="first")
+    duplicate_images = existing_names[duplicate_idx_csv]["name"].tolist()
+    existing_names = existing_names[~duplicate_idx_csv]
     existing_images = search_images((project, project_folder))
     duplicate_idx = []
     for ind, _ in image_data[image_data["name"].isnull()].iterrows():
@@ -983,10 +987,11 @@ def attach_image_urls_to_project(
                 image_data.at[ind, "name"] = name_try
                 existing_images.append(name_try)
                 break
+    image_data.drop_duplicates(subset="name", keep="first", inplace=True)
     for ind, row in existing_names.iterrows():
         if row["name"] in existing_images:
             duplicate_idx.append(ind)
-    duplicate_images = image_data.loc[duplicate_idx]["name"].tolist()
+    duplicate_images.extend(image_data.loc[duplicate_idx]["name"].tolist())
     image_data.drop(labels=duplicate_idx, inplace=True)
     if len(duplicate_images) != 0:
         logger.warning(

tests/attach_urls.csv

@@ -6,4 +6,5 @@ https://drive.google.com/uc?export=download&id=1SfGcn9hdkVM35ZP0S93eStsE7Ti4GtHU
 https://drive.google.com/uc?export=download&id=1SfGcn9hdkVM35ZP0S93eStsE7Ti4GtHU,6022a74b5384c50017c366cv
 https://drive.google.com/uc?export=download&id=1geS2YtQiTYuiduEirKVYxBujHJaIWA3V,
 https://drive.google.com/uc?export=download&id=1geS2YtQiTYuiduEirKVYxBujHJaIWA3V,6022a74b26aec4002575b9rm
-
+https://drive.google.com/uc?export=download&id=1SfGcn9hdkVM35ZP0S93eStsE7Ti4GtHU,6022a74b5384c50017c366cv
+,fjvnvjsvnsdjcndsjcndjcs

tests/test_attach_image_urls.py

@@ -21,12 +21,12 @@ def test_attach_image_urls():
 
     assert len(uploaded) == 7
     assert len(could_not_upload) == 0
-    assert len(existing_images) == 0
+    assert len(existing_images) == 1
 
     uploaded, could_not_upload, existing_images = sa.attach_image_urls_to_project(
         project, PATH_TO_URLS
     )
 
     assert len(uploaded) == 2
     assert len(could_not_upload) == 0
-    assert len(existing_images) == 5
+    assert len(existing_images) == 6