Merge pull request #182 from superannotateai/re-design-sdk

Re design sdk

Author: VaghinakDev

pytest.ini

@@ -2,4 +2,4 @@
 minversion = 3.0
 log_cli=true
 python_files = test_*.py
-;addopts = -n32 --dist=loadscope
+addopts = -n32 --dist=loadscope

requirements.txt

@@ -2,7 +2,7 @@ pydicom>=2.0.0
 boto3>=1.14.53
 requests==2.26.0
 requests-toolbelt>=0.9.1
-tqdm>=4.48.2
+tqdm=4.48.2
 pillow>=7.2.0
 numpy>=1.19.0
 matplotlib>=3.3.1

src/superannotate/lib/app/interface/sdk_interface.py

@@ -609,9 +609,7 @@ def upload_images_from_public_urls_to_project(
 
     project_name, folder_name = extract_project_folder(project)
     existing_images = controller.get_duplicated_images(
-        project_name=project_name,
-        folder_name=folder_name,
-        images=img_names,
+        project_name=project_name, folder_name=folder_name, images=img_names,
     )
 
     image_name_url_map = {}
@@ -1548,6 +1546,28 @@ def upload_images_from_folder_to_project(
     """
 
     project_name, folder_name = extract_project_folder(project)
+    if recursive_subfolders:
+        logger.info(
+            "When using recursive subfolder parsing same name images in different subfolders will overwrite each other."
+        )
+
+    if not isinstance(extensions, (list, tuple)):
+        raise AppException(
+            "extensions should be a list or a tuple in upload_images_from_folder_to_project"
+        )
+
+    project_folder_name = project_name + (
+        f"/{folder_name}" if folder_name != "root" else ""
+    )
+
+    logger.info(
+        "Uploading all images with extensions %s from %s to project %s. Excluded file patterns are: %s.",
+        extensions,
+        folder_path,
+        project_folder_name,
+        exclude_file_patterns,
+    )
+
     use_case = controller.upload_images_from_folder_to_project(
         project_name=project_name,
         folder_name=folder_name,
@@ -1559,7 +1579,14 @@ def upload_images_from_folder_to_project(
         recursive_sub_folders=recursive_subfolders,
         image_quality_in_editor=image_quality_in_editor,
     )
-    images_to_upload, _ = use_case.images_to_upload
+    images_to_upload, duplicates = use_case.images_to_upload
+    if len(duplicates):
+        logger.warning(
+            "%s already existing images found that won't be uploaded.", len(duplicates)
+        )
+    logger.info(
+        "Uploading %s images to project %s.", len(images_to_upload), project_folder_name
+    )
     if use_case.is_valid():
         with tqdm(total=len(images_to_upload), desc="Uploading images") as progress_bar:
             for _ in use_case.execute():
@@ -2061,6 +2088,9 @@ def create_annotation_classes_from_classes_json(
     :rtype: list of dicts
     """
     if not isinstance(classes_json, list):
+        logger.info(
+            "Creating annotation classes in project %s from %s.", project, classes_json,
+        )
         if from_s3_bucket:
             from_session = boto3.Session()
             from_s3 = from_session.resource("s3")
@@ -2073,6 +2103,7 @@ def create_annotation_classes_from_classes_json(
             annotation_classes = json.load(open(classes_json))
     else:
         annotation_classes = classes_json
+
     response = controller.create_annotation_classes(
         project_name=project, annotation_classes=annotation_classes,
     )
@@ -2491,9 +2522,26 @@ def upload_annotations_from_folder_to_project(
             "The function does not support projects containing videos attached with URLs"
         )
 
+    if recursive_subfolders:
+        logger.info(
+            "When using recursive subfolder parsing same name annotations in different subfolders will overwrite each other.",
+        )
+
+    logger.info(
+        "The JSON files should follow specific naming convention. For Vector projects they should be named '<image_name>___objects.json', for Pixel projects JSON file should be names '<image_name>___pixel.json' and also second mask image file should be present with the name '<image_name>___save.png'. In both cases image with <image_name> should be already present on the platform."
+    )
+    logger.info("Existing annotations will be overwritten.",)
+    logger.info(
+        "Uploading all annotations from %s to project %s.", folder_path, project_name
+    )
+
     annotation_paths = get_annotation_paths(
         folder_path, from_s3_bucket, recursive_subfolders
     )
+    logger.info(
+        "Uploading %s annotations to project %s.", len(annotation_paths), project_name
+    )
+
     uploaded_annotations = []
     failed_annotations = []
     missing_annotations = []
@@ -2509,8 +2557,6 @@ def upload_annotations_from_folder_to_project(
                 annotation_paths=annotation_paths[i : i + chunk_size],  # noqa: E203
                 client_s3_bucket=from_s3_bucket,
             )
-            if response.errors:
-                logger.warning(response.errors)
             if response.data:
                 uploaded_annotations.extend(response.data[0])
                 missing_annotations.extend(response.data[1])
@@ -3617,7 +3663,7 @@ def delete_annotations(project: str, image_names: List[str] = None):
     project_name, folder_name = extract_project_folder(project)
 
     response = controller.delete_annotations(
-        project_name=project, folder_name=folder_name, image_names=image_names
+        project_name=project_name, folder_name=folder_name, image_names=image_names
     )
     if response.errors:
         raise AppException(response.errors)

src/superannotate/lib/core/enums.py

@@ -23,6 +23,10 @@ def get_value(cls, name):
             if enum.name.lower() == name.lower():
                 return enum.value
 
+    @classmethod
+    def values(cls):
+        return [enum.name.lower() for enum in list(cls)]
+
 
 class ProjectType(BaseTitledEnum):
     VECTOR = "Vector", 1

src/superannotate/lib/core/usecases.py

@@ -682,6 +682,8 @@ def execute(self):
             folder_id=self._folder.uuid,
             images=[image.name for image in self._attachments],
         )
+        if isinstance(response, dict) and "error" in response:
+            raise AppException(response["error"])
         duplications = [image["name"] for image in response]
         meta = {}
         to_upload = []
@@ -2445,7 +2447,7 @@ def execute(self):
         )
 
         if not response.ok:
-            self._response.errors = AppException(response.json()['error'])
+            self._response.errors = AppException(response.json()["error"])
 
         in_progress = response.ok
         if in_progress:
@@ -2725,11 +2727,7 @@ def execute(self):
                 unique_annotation_classes.append(annotation_class)
 
         created = []
-        logger.info(
-            "Creating annotation classes in project %s from %s.",
-            self._project.name,
-            self._annotation_classes,
-        )
+
         for i in range(0, len(unique_annotation_classes), self.CHUNK_SIZE):
             created += self._service.set_annotation_classes(
                 project_id=self._project.uuid,
@@ -3423,9 +3421,10 @@ def execute(self):
             filter(lambda detail: detail.id is not None, images_detail)
         )
         if missing_annotations:
-            self._response.errors = AppException(
-                f"Couldn't find image {','.join(map(lambda x: x.path, missing_annotations))} for annotation upload."
-            )
+            for missing in missing_annotations:
+                logger.warning(
+                    f"Couldn't find image {missing.path} for annotation upload."
+                )
 
         if self._pre_annotation:
             auth_data = self._backend_service.get_pre_annotation_upload_data(
@@ -4302,6 +4301,16 @@ def exclude_file_patterns(self):
             return constances.DEFAULT_FILE_EXCLUDE_PATTERNS
         return self._exclude_file_patterns
 
+    def validate_annotation_status(self):
+        if self._annotation_status and self._annotation_status not in constances.AnnotationStatus.values():
+            raise AppValidationException("Invalid annotations status")
+
+    def validate_extensions(self):
+        if self._extensions and not all(
+                [extension in constances.DEFAULT_IMAGE_EXTENSIONS  for extension in self._extensions]
+        ):
+            raise AppValidationException("")
+
     def validate_project_type(self):
         if self._project.project_type == constances.ProjectType.VIDEO.value:
             raise AppValidationException(
@@ -4510,7 +4519,7 @@ def execute(self) -> Response:
 
         if self._folder.name == "root" and not self._image_names:
             response = self._backend_service.delete_image_annotations(
-                project_id=self._project.uuid, team_id=self._project.team_id,
+                project_id=self._project.uuid, team_id=self._project.team_id, image_names=self._image_names
             )
         else:
             response = self._backend_service.delete_image_annotations(
@@ -4536,21 +4545,22 @@ def execute(self) -> Response:
                     self._response.errors = "Annotations delete fails."
                     break
                 else:
-                    logger.info(f"Annotations deleted")
+                    logger.info("Annotations deleted")
                     break
         else:
-            self._response.errors = AppException("Invalid image names.")
+            self._response.errors = AppException("Invalid image names or empty folder.")
         return self._response
 
 
 class GetDuplicateImages(BaseUseCase):
-    def __init__(self,
-                 service: SuerannotateServiceProvider,
-                 project_id :int,
-                 team_id: int,
-                 folder_id: int,
-                 images: List[str]
-                 ):
+    def __init__(
+        self,
+        service: SuerannotateServiceProvider,
+        project_id: int,
+        team_id: int,
+        folder_id: int,
+        images: List[str],
+    ):
         super().__init__()
         self._service = service
         self._project_id = project_id
@@ -4566,8 +4576,7 @@ def execute(self):
                 project_id=self._project_id,
                 team_id=self._team_id,
                 folder_id=self._folder_id,
-                images=self._images[i: i + self._chunk_size],
+                images=self._images[i : i + self._chunk_size],
             )
             duplicates += [image["name"] for image in duplications]
         return duplicates
-

src/superannotate/lib/infrastructure/controller.py

@@ -708,11 +708,12 @@ def get_project_workflow(self, project_name: str):
 
     def search_annotation_classes(self, project_name: str, name_prefix: str = None):
         project_entity = self._get_project(project_name)
+        condition = Condition("name", name_prefix, EQ) if name_prefix else None
         use_case = usecases.GetAnnotationClassesUseCase(
             classes=AnnotationClassRepository(
                 service=self._backend_client, project=project_entity
             ),
-            condition=Condition("name", name_prefix, EQ),
+            condition=condition
         )
         return use_case.execute()
 
@@ -1471,15 +1472,16 @@ def delete_annotations(
         )
         return use_case.execute()
 
-
-    def get_duplicated_images(self,project_name: str ,folder_name :str ,images: List[str]):
+    def get_duplicated_images(
+        self, project_name: str, folder_name: str, images: List[str]
+    ):
         project = self._get_project(project_name)
         folder = self._get_folder(project, folder_name)
         use_case = usecases.GetDuplicateImages(
             service=self._backend_client,
             project_id=project.uuid,
-            team_id= project.team_id,
-            folder_id= folder.uuid,
-            images=images
+            team_id=project.team_id,
+            folder_id=folder.uuid,
+            images=images,
         )
         return use_case.execute()

tests/integration/test_annotation_delete.py

@@ -10,6 +10,7 @@ class TestAnnotationDelete(BaseTestCase):
     PROJECT_NAME = "TestAnnotationDelete"
     PROJECT_DESCRIPTION = "desc"
     PROJECT_TYPE = "Vector"
+    TEST_FOLDER_NAME = "folder"
     TEST_FOLDER_PATH = "data_set/sample_project_vector_single_image"
     EXAMPLE_IMAGE_1 = "example_image_1.jpg"
     EXAMPLE_IMAGE_2 = "example_image_2.jpg"
@@ -48,7 +49,7 @@ def test_delete_annotations(self):
     @pytest.mark.skip(
         "waiting for deployment to dev",
     )
-    def test_delete_annotations_by_not_exsisting_name(self):
+    def test_delete_annotations_by_not_existing_name(self):
         sa.upload_images_from_folder_to_project(
             self.PROJECT_NAME, self.folder_path, annotation_status="InProgress"
         )
@@ -59,3 +60,4 @@ def test_delete_annotations_by_not_exsisting_name(self):
             self.PROJECT_NAME, f"{self.folder_path}"
         )
         self.assertRaises(Exception, sa.delete_annotations, self.PROJECT_NAME, [self.EXAMPLE_IMAGE_2])
+

tests/profiling/profiling.py

@@ -32,16 +32,19 @@
 # print(results)
 # print(sum(results)/10)
 
-import time
-results = []
-projects = "11", "22", "33", "44", "55", "66", "77", "88", "99"
-
-for project in projects:
-    start = time.time()
-    sa.search_annotation_classes(project)
-    end = time.time()
-    results.append(end-start)
+# import time
+# results = []
+# projects = "11", "22", "33", "44", "55", "66", "77", "88", "99"
+#
+# for project in projects:
+#     start = time.time()
+#     sa.search_annotation_classes(project)
+#     end = time.time()
+#     results.append(end-start)
+#
+# print(results)
+# print(sum(results)/9)
 
-print(results)
-print(sum(results)/9)
+from src.superannotate.lib.core.enums import AnnotationStatus
 
+pass