Merge pull request #226 from superannotateai/f-193

F 193

Author: dshabin

src/superannotate/lib/app/interface/sdk_interface.py

@@ -2249,6 +2249,8 @@ def attach_image_urls_to_project(
         logger.warning(
             constances.ALREADY_EXISTING_FILES_WARNING.format(len(duplicate_images))
         )
+    logger.info(constances.ATTACHING_FILES_MESSAGE.format(len(images_to_upload)))
+
     use_case = controller.interactive_attach_urls(
         project_name=project_name,
         folder_name=folder_name,
@@ -2392,11 +2394,14 @@ def upload_annotations_from_folder_to_project(
         annotation_paths=annotation_paths,  # noqa: E203
         client_s3_bucket=from_s3_bucket,
     )
-    with tqdm(
-        total=len(annotation_paths), desc="Uploading annotations"
-    ) as progress_bar:
-        for _ in use_case.execute():
-            progress_bar.update(1)
+    if use_case.is_valid():
+        with tqdm(
+            total=len(use_case.annotations_to_upload), desc="Uploading annotations"
+        ) as progress_bar:
+            for _ in use_case.execute():
+                progress_bar.update(1)
+    else:
+        raise AppException(use_case.response.errors)
     return use_case.data
 
 
@@ -3513,6 +3518,7 @@ def attach_document_urls_to_project(
         logger.warning(
             constances.ALREADY_EXISTING_FILES_WARNING.format(len(duplicate_images))
         )
+    logger.info(constances.ATTACHING_FILES_MESSAGE.format(len(images_to_upload)))
     use_case = controller.interactive_attach_urls(
         project_name=project_name,
         folder_name=folder_name,

src/superannotate/lib/core/__init__.py

@@ -83,12 +83,18 @@
 ATTACH_USER_LIMIT_ERROR_MESSAGE = "The number of items you want to attach  exceeds the limit of your subscription plan."
 
 
-COPY_FOLDER_LIMIT_ERROR_MESSAGE = "The number of items you want to copy exceeds the limit of 50 000 items per folder."
+COPY_FOLDER_LIMIT_ERROR_MESSAGE = (
+    "The number of items you want to copy exceeds the limit of 50 000 items per folder."
+)
 COPY_PROJECT_LIMIT_ERROR_MESSAGE = "The number of items you want to copy exceeds the limit of 500 000 items per project."
-COPY_SUPER_LIMIT_ERROR_MESSAGE = "The number of items you want to copy exceeds the limit of your subscription plan."
+COPY_SUPER_LIMIT_ERROR_MESSAGE = (
+    "The number of items you want to copy exceeds the limit of your subscription plan."
+)
 
 
-MOVE_FOLDER_LIMIT_ERROR_MESSAGE = "The number of items you want to move exceeds the limit of 50 000 items per folder."
+MOVE_FOLDER_LIMIT_ERROR_MESSAGE = (
+    "The number of items you want to move exceeds the limit of 50 000 items per folder."
+)
 MOVE_PROJECT_LIMIT_ERROR_MESSAGE = "The number of items you want to move exceeds the limit of 500 000 items per project."
 
 __alL__ = (

src/superannotate/lib/core/usecases/images.py

@@ -835,14 +835,17 @@ def execute(self):
 
                 outline_color = 4 * (255,)
                 for instance in self.annotations["instances"]:
-                    if (not instance.get("className")) or (not class_color_map.get(instance["className"])):
+                    if (not instance.get("className")) or (
+                        not class_color_map.get(instance["className"])
+                    ):
                         continue
                     color = class_color_map.get(instance["className"])
                     if not color:
                         class_color_map[instance["className"]] = self.generate_color()
                     for image in images:
                         fill_color = (
-                            *class_color_map[instance["className"]], 255 if image.type == "fuse" else self.TRANSPARENCY
+                            *class_color_map[instance["className"]],
+                            255 if image.type == "fuse" else self.TRANSPARENCY,
                         )
                         if instance["type"] == "bbox":
                             image.content.draw_bbox(
@@ -911,7 +914,9 @@ def execute(self):
                 weight, height = image.get_size()
                 empty_image_arr = np.full((height, weight, 4), [0, 0, 0, 255], np.uint8)
                 for annotation in self.annotations["instances"]:
-                    if (not annotation.get("className")) or (not class_color_map.get(annotation["className"])):
+                    if (not annotation.get("className")) or (
+                        not class_color_map.get(annotation["className"])
+                    ):
                         continue
                     fill_color = *class_color_map[annotation["className"]], 255
                     for part in annotation["parts"]:
@@ -2002,17 +2007,28 @@ def validate_limitations(self):
         if self._move:
             if self._from_project.uuid == self._to_project.uuid:
                 if self._from_folder.uuid == self._to_folder.uuid:
-                    raise AppValidationException("Cannot move image if source_project == destination_project.")
+                    raise AppValidationException(
+                        "Cannot move image if source_project == destination_project."
+                    )
                 elif response.data.folder_limit.remaining_image_count < 1:
-                    raise AppValidationException(constances.MOVE_FOLDER_LIMIT_ERROR_MESSAGE)
+                    raise AppValidationException(
+                        constances.MOVE_FOLDER_LIMIT_ERROR_MESSAGE
+                    )
             elif response.data.project_limit.remaining_image_count < 1:
-                raise AppValidationException(constances.MOVE_PROJECT_LIMIT_ERROR_MESSAGE)
+                raise AppValidationException(
+                    constances.MOVE_PROJECT_LIMIT_ERROR_MESSAGE
+                )
         else:
             if response.data.folder_limit.remaining_image_count < 1:
                 raise AppValidationException(constances.COPY_FOLDER_LIMIT_ERROR_MESSAGE)
             if response.data.project_limit.remaining_image_count < 1:
-                raise AppValidationException(constances.COPY_PROJECT_LIMIT_ERROR_MESSAGE)
-            if response.data.super_user_limit and response.data.super_user_limit.remaining_image_count < 1:
+                raise AppValidationException(
+                    constances.COPY_PROJECT_LIMIT_ERROR_MESSAGE
+                )
+            if (
+                response.data.super_user_limit
+                and response.data.super_user_limit.remaining_image_count < 1
+            ):
                 raise AppValidationException(constances.COPY_SUPER_LIMIT_ERROR_MESSAGE)
 
     @property
@@ -2689,6 +2705,8 @@ def execute(self):
                 else:
                     from_s3 = None
 
+                for _ in range(len(annotations_to_upload) - len(response.data.images)):
+                    yield
                 with concurrent.futures.ThreadPoolExecutor(
                     max_workers=self.MAX_WORKERS
                 ) as executor:
@@ -2720,7 +2738,6 @@ def execute(self):
                 failed_annotations = [
                     annotation.path for annotation in failed_annotations
                 ]
-
             self._response.data = (
                 uploaded_annotations,
                 failed_annotations,
@@ -2732,42 +2749,41 @@ def execute(self):
     def upload_to_s3(
         self, image_id: int, image_info, bucket, from_s3, image_id_name_map
     ):
-        if from_s3:
-            file = io.BytesIO()
-            s3_object = from_s3.Object(
-                self._client_s3_bucket, image_id_name_map[image_id].path
-            )
-            s3_object.download_fileobj(file)
-            file.seek(0)
-            annotation_json = json.load(file)
-        else:
-            annotation_json = json.load(open(image_id_name_map[image_id].path))
-
-        self.fill_classes_data(annotation_json)
-
-        if not self._is_valid_json(annotation_json):
-            logger.warning(f"Invalid json {image_id_name_map[image_id].path}")
-            return image_id_name_map[image_id], False
-        bucket.put_object(
-            Key=image_info["annotation_json_path"], Body=json.dumps(annotation_json),
-        )
-        if self._project.project_type == constances.ProjectType.PIXEL.value:
-            mask_filename = (
-                image_id_name_map[image_id].name + constances.ANNOTATION_MASK_POSTFIX
-            )
+        try:
             if from_s3:
                 file = io.BytesIO()
                 s3_object = from_s3.Object(
-                    self._client_s3_bucket, f"{self._folder_path}/{mask_filename}"
+                    self._client_s3_bucket, image_id_name_map[image_id].path
                 )
                 s3_object.download_fileobj(file)
                 file.seek(0)
+                annotation_json = json.load(file)
             else:
-                with open(f"{self._folder_path}/{mask_filename}", "rb") as mask_file:
-                    file = io.BytesIO(mask_file.read())
-
-            bucket.put_object(Key=image_info["annotation_bluemap_path"], Body=file)
-        return image_id_name_map[image_id], True
+                annotation_json = json.load(open(image_id_name_map[image_id].path))
+            self.fill_classes_data(annotation_json)
+            if not self._is_valid_json(annotation_json):
+                logger.warning(f"Invalid json {image_id_name_map[image_id].path}")
+                return image_id_name_map[image_id], False
+            bucket.put_object(
+                Key=image_info["annotation_json_path"],
+                Body=json.dumps(annotation_json),
+            )
+            if self._project.project_type == constances.ProjectType.PIXEL.value:
+                mask_path = image_id_name_map[image_id].path.replace(
+                    "___pixel.json", constances.ANNOTATION_MASK_POSTFIX
+                )
+                if from_s3:
+                    file = io.BytesIO()
+                    s3_object = from_s3.Object(self._client_s3_bucket, mask_path)
+                    s3_object.download_fileobj(file)
+                    file.seek(0)
+                else:
+                    with open(mask_path, "rb") as mask_file:
+                        file = io.BytesIO(mask_file.read())
+                bucket.put_object(Key=image_info["annotation_bluemap_path"], Body=file)
+            return image_id_name_map[image_id], True
+        except Exception as _:
+            return image_id_name_map[image_id], False
 
     def report_missing_data(self):
         if self.missing_classes:

tests/integration/test_recursive_folder_pixel.py

@@ -0,0 +1,32 @@
+import os
+from os.path import dirname
+
+import src.superannotate as sa
+from tests.integration.base import BaseTestCase
+
+
+class TestRecursiveFolderPixel(BaseTestCase):
+    PROJECT_NAME = "pixel_recursive_test"
+    PROJECT_DESCRIPTION = "Desc"
+    PROJECT_TYPE = "Pixel"
+    S3_FOLDER_PATH = "pixel_all_fuse"
+    JSON_POSTFIX = "*.json"
+
+
+    def test_recursive_upload_pixel(self):
+        uploaded, _, duplicated = sa.upload_images_from_folder_to_project(self.PROJECT_NAME,
+                                                                          self.S3_FOLDER_PATH,
+                                                                          from_s3_bucket="test-openseadragon-1212",
+                                                                          recursive_subfolders=True
+                                                                          )
+
+        uploaded, failed, missing = sa.upload_annotations_from_folder_to_project(self.PROJECT_NAME,
+                                                                          self.S3_FOLDER_PATH,
+                                                                          from_s3_bucket="test-openseadragon-1212",
+                                                                          recursive_subfolders=True
+                                                                          )
+        self.assertEqual(115, len(uploaded))
+        self.assertEqual(0, len(failed))
+        self.assertEqual(0, len(missing))
+
+