merge

Author: dshabin

pytest.ini

@@ -2,4 +2,4 @@
 minversion = 3.0
 log_cli=true
 python_files = test_*.py
-addopts = -n32 --dist=loadscope
+;addopts = -n32 --dist=loadscope

src/superannotate/__init__.py

@@ -60,6 +60,7 @@
 from superannotate.lib.app.interface.sdk_interface import create_project
 from superannotate.lib.app.interface.sdk_interface import create_project_from_metadata
 from superannotate.lib.app.interface.sdk_interface import delete_annotation_class
+from superannotate.lib.app.interface.sdk_interface import delete_annotations
 from superannotate.lib.app.interface.sdk_interface import (
     delete_contributor_to_team_invitation,
 )
@@ -235,6 +236,7 @@
     "assign_images",
     "unassign_images",
     "download_image_annotations",
+    "delete_annotations",
     "upload_image_to_project",
     "upload_image_annotations",
     "upload_images_from_public_urls_to_project",

src/superannotate/lib/app/interface/sdk_interface.py

@@ -1503,7 +1503,7 @@ def get_image_annotations(project: Union[str, dict], image_name: str):
         project_name=project_name, folder_name=folder_name, image_name=image_name
     )
     if res.errors:
-        raise AppValidationException(res)
+        raise AppException(res)
     return res.data
 
 
@@ -1561,7 +1561,7 @@ def upload_images_from_folder_to_project(
     )
     images_to_upload, _ = use_case.images_to_upload
     if use_case.is_valid():
-        with tqdm(total=len(images_to_upload)) as progress_bar:
+        with tqdm(total=len(images_to_upload), desc="Uploading images") as progress_bar:
             for _ in use_case.execute():
                 progress_bar.update(1)
         return use_case.data
@@ -1866,7 +1866,7 @@ def upload_videos_from_folder_to_project(
     )
 
     uploaded_images, failed_images = [], []
-    for path in tqdm(video_paths):
+    for path in tqdm(video_paths, desc="Uploading videos"):
         with tempfile.TemporaryDirectory() as temp_path:
             res = controller.extract_video_frames(
                 project_name=project_name,
@@ -1960,7 +1960,9 @@ def upload_video_to_project(
         )
         images_to_upload, _ = use_case.images_to_upload
         if use_case.is_valid():
-            with tqdm(total=len(images_to_upload)) as progress_bar:
+            with tqdm(
+                total=len(images_to_upload), desc="Uploading frames."
+            ) as progress_bar:
                 for _ in use_case.execute():
                     progress_bar.update(1)
             return use_case.data[0]
@@ -2495,8 +2497,10 @@ def upload_annotations_from_folder_to_project(
     uploaded_annotations = []
     failed_annotations = []
     missing_annotations = []
-    chunk_size = 10
-    with tqdm(total=len(annotation_paths)) as progress_bar:
+    chunk_size = 50
+    with tqdm(
+        total=len(annotation_paths), desc="Uploading annotations"
+    ) as progress_bar:
         for i in range(0, len(annotation_paths), chunk_size):
             response = controller.upload_annotations_from_folder(
                 project_name=project_name,
@@ -2556,7 +2560,9 @@ def upload_preannotations_from_folder_to_project(
     failed_annotations = []
     missing_annotations = []
     chunk_size = 10
-    with tqdm(total=len(annotation_paths)) as progress_bar:
+    with tqdm(
+        total=len(annotation_paths), desc="Uploading pre annotations"
+    ) as progress_bar:
         for i in range(0, len(annotation_paths), chunk_size):
             response = controller.upload_annotations_from_folder(
                 project_name=project_name,
@@ -3518,7 +3524,7 @@ def _upload_s3_image(image_path: str):
             executor.submit(upload_method, image_path)
             for image_path in images_to_upload
         ]
-        with tqdm(total=len(images_to_upload)) as progress_bar:
+        with tqdm(total=len(images_to_upload), desc="Uploading images") as progress_bar:
             for future in concurrent.futures.as_completed(results):
                 processed_image = future.result()
                 if processed_image.uploaded and processed_image.entity:
@@ -3594,3 +3600,24 @@ def aggregate_annotations_as_df(
         verbose,
         folder_names,
     )
+
+
+@Trackable
+@validate_input
+def delete_annotations(project: str, image_names: List[str] = None):
+    """
+    Delete image annotations from a given list of images.
+
+    :param project: project name or folder path (e.g., "project1/folder1")
+    :type project: str
+    :param image_names:  image names. If None, all image annotations from a given project/folder will be deleted.
+    :type image_names: list of strs
+    """
+
+    project_name, folder_name = extract_project_folder(project)
+
+    response = controller.delete_annotations(
+        project_name=project, folder_name=folder_name, image_names=image_names
+    )
+    if response.errors:
+        raise AppException(response.errors)

src/superannotate/lib/core/serviceproviders.py

@@ -296,3 +296,17 @@ def run_prediction(
         self, team_id: int, project_id: int, ml_model_id: int, image_ids: list
     ):
         raise NotImplementedError
+
+    def delete_image_annotations(
+        self,
+        team_id: int,
+        project_id: int,
+        folder_id: int = None,
+        image_names: List[str] = None,
+    ) -> int:
+        raise NotImplementedError
+
+    def get_annotations_delete_progress(
+        self, team_id: int, project_id: int, poll_id: int
+    ):
+        raise NotImplementedError

src/superannotate/lib/core/usecases.py

@@ -15,6 +15,7 @@
 from pathlib import Path
 from typing import Iterable
 from typing import List
+from typing import Optional
 
 import boto3
 import cv2
@@ -363,6 +364,9 @@ def execute(self):
                     ] = new_project_annotation_classes.insert(annotation_class_copy)
 
             if self._include_contributors:
+                self._project = self._projects.get_one(
+                    uuid=self._project.uuid, team_id=self._project.team_id
+                )
                 for user in self._project.users:
                     self._backend_service.share_project(
                         project.uuid,
@@ -504,7 +508,7 @@ class UploadImageS3UseCase(BaseUseCase):
     def __init__(
         self,
         project: ProjectEntity,
-        project_settings: BaseReadOnlyRepository,
+        project_settings: List[ProjectSettingEntity],
         image_path: str,
         image: io.BytesIO,
         s3_repo: BaseManageableRepository,
@@ -535,7 +539,7 @@ def execute(self):
             huge_image, huge_width, huge_height = image_processor.generate_huge()
             quality = 60
             if not self._image_quality_in_editor:
-                for setting in self._project_settings.get_all():
+                for setting in self._project_settings:
                     if setting.attribute == "ImageQuality":
                         quality = setting.value
             else:
@@ -679,7 +683,11 @@ def execute(self):
             folder_id=self._folder.uuid,
             images=[image.name for image in self._attachments],
         )
-        duplications = [image["name"] for image in duplications]
+        try:
+            duplications = [image["name"] for image in duplications]
+        except Exception:
+            print(duplications)
+            raise
         meta = {}
         to_upload = []
         for image in self._attachments:
@@ -1062,7 +1070,7 @@ def execute(self):
             headers=annotations["annotation_json_path"]["headers"],
         )
         if not response.ok:
-            raise AppException(f"Couldn't load annotations.")
+            raise AppException("Couldn't load annotations.")
 
         image_annotations = response.json()
         from_project_annotation_classes = (
@@ -1153,7 +1161,7 @@ def execute(self):
                 headers=annotations["annotation_bluemap_path"]["headers"],
             )
             if not response.ok:
-                raise AppException(f"Couldn't load annotations.")
+                raise AppException("Couldn't load annotations.")
             self.to_project_s3_repo.insert(
                 S3FileEntity(
                     auth_data["annotation_bluemap_path"]["filePath"], response.content
@@ -2077,12 +2085,15 @@ def execute(self):
                 file_postfix = "___objects.json"
             else:
                 file_postfix = "___pixel.json"
+                data["annotation_mask_filename"] = f"{self._image_name}___save.png"
+            data["annotation_json_filename"] = f"{self._image_name}{file_postfix}"
+
             response = requests.get(
                 url=credentials["annotation_json_path"]["url"],
                 headers=credentials["annotation_json_path"]["headers"],
             )
             if not response.ok:
-                logger.warning(f"Couldn't load annotations.")
+                logger.warning("Couldn't load annotations.")
                 self._response.data = data
                 return self._response
             data["annotation_json"] = response.json()
@@ -2094,7 +2105,6 @@ def execute(self):
                     headers=annotation_blue_map_creds["headers"],
                 )
                 data["annotation_mask"] = io.BytesIO(response.content)
-                data["annotation_mask_filename"] = f"{self._image_name}___save.png"
 
             self._response.data = data
 
@@ -2177,7 +2187,7 @@ def execute(self):
             url=annotation_json_creds["url"], headers=annotation_json_creds["headers"],
         )
         if not response.ok:
-            raise AppException(f"Couldn't load annotations.")
+            raise AppException("Couldn't load annotations.")
         data["preannotation_json"] = response.json()
         data["preannotation_json_filename"] = f"{self._image_name}{file_postfix}"
         if self._project.project_type == constances.ProjectType.PIXEL.value:
@@ -2254,7 +2264,7 @@ def execute(self):
                 headers=annotation_json_creds["headers"],
             )
             if not response.ok:
-                logger.warning(f"Couldn't load annotations.")
+                logger.warning("Couldn't load annotations.")
                 self._response.data = (None, None)
                 return self._response
             data["annotation_json"] = response.json()
@@ -2333,7 +2343,7 @@ def execute(self):
             url=annotation_json_creds["url"], headers=annotation_json_creds["headers"],
         )
         if not response.ok:
-            raise AppException(f"Couldn't load annotations.")
+            raise AppException("Couldn't load annotations.")
         data["preannotation_json"] = response.json()
         data["preannotation_json_filename"] = f"{self._image_name}{file_postfix}"
         mask_path = None
@@ -3277,10 +3287,11 @@ def __init__(
         self,
         project: ProjectEntity,
         folder: FolderEntity,
-        annotation_classes: BaseReadOnlyRepository,
+        annotation_classes: List[AnnotationClassEntity],
         folder_path: str,
         annotation_paths: List[str],
         backend_service_provider: SuerannotateServiceProvider,
+        templates: List[dict],
         pre_annotation: bool = False,
         client_s3_bucket=None,
     ):
@@ -3293,6 +3304,7 @@ def __init__(
         self._annotation_paths = annotation_paths
         self._client_s3_bucket = client_s3_bucket
         self._pre_annotation = pre_annotation
+        self._templates = templates
 
     @property
     def s3_client(self):
@@ -3301,7 +3313,7 @@ def s3_client(self):
     @property
     def annotation_classes_name_map(self) -> dict:
         classes_data = defaultdict(dict)
-        annotation_classes = self._annotation_classes.get_all()
+        annotation_classes = self._annotation_classes
         for annotation_class in annotation_classes:
             class_info = {"id": annotation_class.uuid}
             if annotation_class.attribute_groups:
@@ -3327,11 +3339,8 @@ def annotation_postfix(self):
         )
 
     def get_templates_mapping(self):
-        templates = self._backend_service.get_templates(
-            team_id=self._project.team_id
-        ).get("data", [])
         templates_map = {}
-        for template in templates:
+        for template in self._templates:
             templates_map[template["name"]] = template["id"]
         return templates_map
 
@@ -4276,7 +4285,7 @@ def __init__(
         self._project = project
         self._folder = folder
         self._folder_path = folder_path
-        self._settings = settings
+        self._settings = settings.get_all()
         self._s3_repo = s3_repo
         self._backend_client = backend_client
         self._image_quality_in_editor = image_quality_in_editor
@@ -4486,6 +4495,59 @@ def execute(self):
         return self._response
 
 
+class DeleteAnnotations(BaseUseCase):
+    POLL_AWAIT_TIME = 2
+
+    def __init__(
+        self,
+        project: ProjectEntity,
+        folder: FolderEntity,
+        backend_service: SuerannotateServiceProvider,
+        image_names: Optional[List[str]] = None,
+    ):
+        super().__init__()
+        self._project = project
+        self._folder = folder
+        self._image_names = image_names
+        self._backend_service = backend_service
+
+    def execute(self) -> Response:
+
+        if self._folder.name == "root" and not self._image_names:
+            poll_id = self._backend_service.delete_image_annotations(
+                project_id=self._project.uuid, team_id=self._project.team_id,
+            )
+        else:
+            poll_id = self._backend_service.delete_image_annotations(
+                project_id=self._project.uuid,
+                team_id=self._project.team_id,
+                folder_id=self._folder.uuid,
+                image_names=self._image_names,
+            )
+
+        if poll_id:
+            timeout_start = time.time()
+            while time.time() < timeout_start + self.POLL_AWAIT_TIME:
+                progress = int(
+                    self._backend_service.get_annotations_delete_progress(
+                        project_id=self._project.uuid,
+                        team_id=self._project.team_id,
+                        poll_id=poll_id,
+                    ).get("process", -1)
+                )
+                if 0 < progress < 100:
+                    logger.info(f"Delete annotations in progress {progress}/100")
+                elif 0 > progress:
+                    self._response.errors = "Annotations delete fails."
+                    break
+                else:
+                    logger.info(f"Annotations deleted")
+                    break
+        else:
+            self._response.errors = AppException("Invalid image names.")
+        return self._response
+
+
 
 class GetDuplicateImages(BaseUseCase):
     def __init__(self,