Merge pull request #227 from superannotateai/sdk_backend_error_fix

Fixed backend error

Author: VaghinakDev

pytest.ini

@@ -2,4 +2,4 @@
 minversion = 3.0
 log_cli=true
 python_files = test_*.py
-addopts = -n32 --dist=loadscope
+addopts = -n auto --dist=loadscope

src/superannotate/lib/core/usecases/images.py

@@ -281,20 +281,22 @@ def execute(self):
         except AppValidationException as e:
             self._response.errors = e
             return self._response
-
-        backend_response = self._backend_service.attach_files(
-            project_id=self._project.uuid,
-            folder_id=self._folder.uuid,
-            team_id=self._project.team_id,
-            files=to_upload,
-            annotation_status_code=self.annotation_status_code,
-            upload_state_code=self.upload_state_code,
-            meta=meta,
-        )
-        if isinstance(backend_response, dict) and "error" in backend_response:
-            self._response.errors = AppException(backend_response["error"])
+        if to_upload:
+            backend_response = self._backend_service.attach_files(
+                project_id=self._project.uuid,
+                folder_id=self._folder.uuid,
+                team_id=self._project.team_id,
+                files=to_upload,
+                annotation_status_code=self.annotation_status_code,
+                upload_state_code=self.upload_state_code,
+                meta=meta,
+            )
+            if isinstance(backend_response, dict) and "error" in backend_response:
+                self._response.errors = AppException(backend_response["error"])
+            else:
+                self._response.data = backend_response, duplications
         else:
-            self._response.data = backend_response, duplications
+            self._response.data = [], duplications
         return self._response
 
 
@@ -1088,7 +1090,7 @@ def __init__(
         self,
         project: ProjectEntity,
         folder: FolderEntity,
-        s3_repo: BaseManageableRepository,
+        s3_repo,
         settings: BaseManageableRepository,
         backend_client: SuerannotateServiceProvider,
         annotation_status: str,
@@ -1112,15 +1114,19 @@ def __init__(
         self._annotation_status = annotation_status
         self._auth_data = None
 
-    def validate_auth_data(self):
-        response = self._backend_client.get_s3_upload_auth_token(
-            team_id=self._project.team_id,
-            folder_id=self._folder.uuid,
-            project_id=self._project.uuid,
+    @property
+    def s3_repo(self):
+        self._auth_data = self._backend_client.get_s3_upload_auth_token(
+            self._project.team_id, self._folder.uuid, self._project.uuid
+        )
+        if "error" in self._auth_data:
+            raise AppException(self._auth_data.get("error"))
+        return self._s3_repo(
+            self._auth_data["accessKeyId"],
+            self._auth_data["secretAccessKey"],
+            self._auth_data["sessionToken"],
+            self._auth_data["bucket"],
         )
-        if "error" in response:
-            raise AppException(response.get("error"))
-        self._auth_data = response
 
     def validate_project_type(self):
         if self._project.project_type in [
@@ -1194,7 +1200,7 @@ def execute(self) -> Response:
                 else Path(self._image_path).name,
                 project_settings=self._settings.get_all(),
                 image=image_bytes,
-                s3_repo=self._s3_repo,
+                s3_repo=self.s3_repo,
                 upload_path=self.auth_data["filePath"],
                 image_quality_in_editor=self._image_quality_in_editor,
             ).execute()
@@ -1458,7 +1464,7 @@ def execute(self):
                     folder=self._folder,
                     backend_service_provider=self._backend_client,
                     attachments=[
-                        image.entity for image in uploaded_images[i : i + 100]
+                        image.entity for image in uploaded_images[i : i + 100]  # noqa: E203
                     ],
                     annotation_status=self._annotation_status,
                     upload_state_code=constances.UploadState.BASIC.value,
@@ -1580,6 +1586,27 @@ def __init__(
         self._annotation_status = annotation_status
         self._image_quality_in_editor = image_quality_in_editor
         self._settings = settings
+        self._auth_data = None
+
+    @property
+    def auth_data(self):
+        if not self._auth_data:
+            self._auth_data = self._backend_service.get_s3_upload_auth_token(
+            self._project.team_id, self._folder.uuid, self._project.uuid
+        )
+        return self._auth_data
+
+    @property
+    def s3_repo(self):
+
+        if "error" in self.auth_data:
+            raise AppException(self._auth_data.get("error"))
+        return self._s3_repo(
+            self.auth_data["accessKeyId"],
+            self.auth_data["secretAccessKey"],
+            self.auth_data["sessionToken"],
+            self.auth_data["bucket"],
+        )
 
     def validate_limitations(self):
         response = self._backend_service.get_limitations(
@@ -1624,18 +1651,6 @@ def validate_annotation_status(self):
         else:
             self._annotation_status = constances.AnnotationStatus.NOT_STARTED
 
-    def get_auth_data(self, project_id: int, team_id: int, folder_id: int):
-        response = self._backend_service.get_s3_upload_auth_token(
-            team_id, folder_id, project_id
-        )
-        if "error" in response:
-            raise AppException(response.get("error"))
-        return response
-
-    @property
-    def s3_repo(self):
-        return self._s3_repo
-
     def upload_image(self, image_url, image_name=None):
         download_response = DownloadImageFromPublicUrlUseCase(
             project=self._project, image_url=image_url, image_name=image_name
@@ -1661,10 +1676,8 @@ def upload_image(self, image_url, image_name=None):
                     project_settings=self._settings,
                     image_path=image_name,
                     image=content,
-                    s3_repo=self._s3_repo,
-                    upload_path=self.get_auth_data(
-                        self._project.uuid, self._project.team_id, self._folder.uuid
-                    )["filePath"],
+                    s3_repo=self.s3_repo,
+                    upload_path=self.auth_data["filePath"],
                     image_quality_in_editor=self._image_quality_in_editor,
                 ).execute()
 
@@ -1911,7 +1924,7 @@ def execute(self):
                 response = AttachFileUrlsUseCase(
                     project=self._project,
                     folder=self._folder,
-                    attachments=self._attachments[i : i + 500],  # noqa: E203
+                    attachments=self._attachments[i : i + self.CHUNK_SIZE],  # noqa: E203
                     backend_service_provider=self._backend_service,
                     annotation_status=self._annotation_status,
                     upload_state_code=self._upload_state_code,
@@ -1937,7 +1950,7 @@ def __init__(
         to_folder: FolderEntity,
         backend_service: SuerannotateServiceProvider,
         images: BaseManageableRepository,
-        to_upload_s3_repo: BaseManageableRepository,
+        s3_repo,
         project_settings: List[ProjectSettingEntity],
         include_annotations: Optional[bool] = True,
         copy_annotation_status: Optional[bool] = True,
@@ -1950,7 +1963,7 @@ def __init__(
         self._image_name = image_name
         self._to_project = to_project
         self._to_folder = to_folder
-        self._to_upload_s3_repo = to_upload_s3_repo
+        self._s3_repo = s3_repo
         self._project_settings = project_settings
         self._include_annotations = include_annotations
         self._copy_annotation_status = copy_annotation_status
@@ -2002,6 +2015,20 @@ def validate_limitations(self):
             if response.data.super_user_limit and response.data.super_user_limit.remaining_image_count < 1:
                 raise AppValidationException(constances.COPY_SUPER_LIMIT_ERROR_MESSAGE)
 
+    @property
+    def s3_repo(self):
+        self._auth_data = self._backend_service.get_s3_upload_auth_token(
+            self._to_project.team_id, self._to_folder.uuid, self._to_project.uuid
+        )
+        if "error" in self._auth_data:
+            raise AppException(self._auth_data.get("error"))
+        return self._s3_repo(
+            self._auth_data["accessKeyId"],
+            self._auth_data["secretAccessKey"],
+            self._auth_data["sessionToken"],
+            self._auth_data["bucket"],
+        )
+
     def execute(self) -> Response:
         if self.is_valid():
             image = (
@@ -2038,7 +2065,7 @@ def execute(self) -> Response:
                 image=image_bytes,
                 project_settings=self._project_settings,
                 upload_path=auth_data["filePath"],
-                s3_repo=self._to_upload_s3_repo,
+                s3_repo=self.s3_repo,
             ).execute()
             if s3_response.errors:
                 raise AppException(s3_response.errors)

src/superannotate/lib/infrastructure/controller.py

@@ -298,50 +298,6 @@ def update_project(self, name: str, project_data: dict) -> Response:
         use_case = usecases.UpdateProjectUseCase(project, project_data, self.projects)
         return use_case.execute()
 
-    def upload_images(
-        self,
-        project_name: str,
-        folder_name: str,
-        images: List[ImageEntity],
-        annotation_status: str = None,
-    ):
-        project = self._get_project(project_name)
-        folder = self._get_folder(project, folder_name)
-        use_case = usecases.AttachFileUrlsUseCase(
-            project=project,
-            folder=folder,
-            backend_service_provider=self._backend_client,
-            attachments=images,
-            annotation_status=annotation_status,
-            upload_state_code=constances.UploadState.BASIC.value,
-        )
-        return use_case.execute()
-
-    def upload_image_to_s3(
-        self,
-        project_name: str,
-        image_path: str,  # image path to upload
-        image_bytes: io.BytesIO,
-        folder_name: str = None,  # project folder path
-        image_quality_in_editor: str = None,
-    ):
-        project = self._get_project(project_name)
-        folder = self._get_folder(project, folder_name)
-        s3_repo = self.get_s3_repository(self.team_id, project.uuid, folder.uuid)
-        auth_data = self.get_auth_data(project.uuid, self.team_id, folder.uuid)
-        use_case = usecases.UploadImageS3UseCase(
-            project=project,
-            project_settings=ProjectSettingsRepository(
-                self._backend_client, project
-            ).get_all(),
-            image_path=image_path,
-            image=image_bytes,
-            s3_repo=s3_repo,
-            upload_path=auth_data["filePath"],
-            image_quality_in_editor=image_quality_in_editor,
-        )
-        return use_case.execute()
-
     def upload_image_to_project(
         self,
         project_name: str,
@@ -367,7 +323,7 @@ def upload_image_to_project(
             settings=ProjectSettingsRepository(
                 service=self._backend_client, project=project
             ),
-            s3_repo=self.get_s3_repository(self.team_id, project.uuid, folder.uuid),
+            s3_repo=self.s3_repo,
             backend_client=self._backend_client,
             image_path=image_path,
             image_bytes=image_bytes,
@@ -455,7 +411,7 @@ def upload_images_from_public_urls_to_project(
             settings=ProjectSettingsRepository(
                 service=self._backend_client, project=project
             ),
-            s3_repo=self.get_s3_repository(self.team_id, project.uuid, folder.uuid),
+            s3_repo=self.s3_repo,
             image_quality_in_editor=image_quality_in_editor,
             annotation_status=annotation_status,
         )
@@ -492,27 +448,6 @@ def clone_project(
         )
         return use_case.execute()
 
-    def attach_urls(
-        self,
-        project_name: str,
-        files: List[ImageEntity],
-        folder_name: str = None,
-        annotation_status: str = None,
-        upload_state_code: int = None,
-    ):
-        project = self._get_project(project_name)
-        folder = self._get_folder(project, folder_name)
-
-        use_case = usecases.AttachFileUrlsUseCase(
-            project=project,
-            folder=folder,
-            attachments=files,
-            backend_service_provider=self._backend_client,
-            annotation_status=annotation_status,
-            upload_state_code=upload_state_code,
-        )
-        return use_case.execute()
-
     def interactive_attach_urls(
         self,
         project_name: str,
@@ -723,9 +658,7 @@ def copy_image(
             project_settings=ProjectSettingsRepository(
                 self._backend_client, to_project
             ).get_all(),
-            to_upload_s3_repo=self.get_s3_repository(
-                self.team_id, to_project.uuid, to_folder.uuid
-            ),
+            s3_repo=self.s3_repo,
             copy_annotation_status=copy_annotation_status,
             move=move,
         )
@@ -1593,13 +1526,6 @@ def list_images(
         )
         return use_case.execute()
 
-    @staticmethod
-    def upload_file_to_s3(to_s3_bucket, path, s3_key: str):
-        use_case = usecases.UploadFileToS3UseCase(
-            to_s3_bucket=to_s3_bucket, path=path, s3_key=s3_key
-        )
-        return use_case.execute()
-
     def search_models(
         self,
         name: str,
@@ -1643,28 +1569,3 @@ def delete_annotations(
             image_names=image_names,
         )
         return use_case.execute()
-
-    def get_duplicated_images(
-        self, project_name: str, folder_name: str, images: List[str]
-    ):
-        project = self._get_project(project_name)
-        folder = self._get_folder(project, folder_name)
-        use_case = usecases.GetBulkImages(
-            service=self._backend_client,
-            project_id=project.uuid,
-            team_id=project.team_id,
-            folder_id=folder.uuid,
-            images=images,
-        )
-        return use_case.execute().data
-
-    def get_project_limitations(self, project_name: str, folder_name: str):
-        project = self._get_project(project_name)
-        folder = self._get_folder(project, folder_name)
-
-        return usecases.GetUserLimitsUseCase(
-            service=self._backend_client,
-            project_id=project.uuid,
-            team_id=project.team_id,
-            folder_id=folder.uuid,
-        ).execute()

tests/integration/test_annotation_delete.py

@@ -1,5 +1,6 @@
 import os
 from os.path import dirname
+import pytest
 
 import src.superannotate as sa
 from tests.integration.base import BaseTestCase
@@ -70,6 +71,7 @@ def test_delete_annotations_by_not_existing_name(self):
         )
         self.assertRaises(Exception, sa.delete_annotations, self.PROJECT_NAME, [self.EXAMPLE_IMAGE_2])
 
+    @pytest.mark.flaky(reruns=2)
     def test_delete_annotations_wrong_path(self):
         sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME)
         sa.upload_images_from_folder_to_project(