Added limitation validation

Author: Vaghinak Basentsyan

.pre-commit-config.yaml

@@ -36,9 +36,9 @@ repos:
     hooks:
       - id: black
         name: Uncompromising Code Formatter (black)
-#  - repo: 'https://github.com/asottile/dead'
-#    rev: v1.3.0
-#    hooks:
-#      - id: dead
+  - repo: 'https://github.com/asottile/dead'
+    rev: v1.3.0
+    hooks:
+      - id: dead
 files: src/
 exclude: src/lib/app/analytics | src/lib/app/converters | src/lib/app/input_convertors

src/superannotate/lib/app/common.py

@@ -180,12 +180,6 @@ def hex_to_rgb(hex_string):
     return tuple(int(h[i : i + 2], 16) for i in (0, 2, 4))
 
 
-def rgb_to_hex(rgb_tuple):
-    """Converts RGB values to HEX values
-    """
-    return "#%02x%02x%02x" % rgb_tuple
-
-
 def blue_color_generator(n, hex_values=True):
     """ Blue colors generator for SuperAnnotate blue mask.
     """
@@ -292,13 +286,6 @@ def save_web_format(output_dir, classes, files_dict):
         json.dump(classes, fw)
 
 
-def dump_output(output_dir, platform, classes, files_dict):
-    if platform == "Web":
-        save_web_format(output_dir, classes, files_dict)
-    else:
-        save_desktop_format(output_dir, classes, files_dict)
-
-
 def write_to_json(output_path, json_data):
     with open(output_path, "w") as fw:
         json.dump(json_data, fw, indent=2)

src/superannotate/lib/app/interface/sdk_interface.py

@@ -6,9 +6,6 @@
 import tempfile
 import time
 import uuid
-from collections import Counter
-from collections import namedtuple
-from io import BytesIO
 from pathlib import Path
 from typing import Iterable
 from typing import List
@@ -44,7 +41,6 @@
 from lib.app.serializers import TeamSerializer
 from lib.core.enums import ImageQuality
 from lib.core.exceptions import AppException
-from lib.core.exceptions import AppValidationException
 from lib.infrastructure.controller import Controller
 from plotly.subplots import make_subplots
 from pydantic import EmailStr
@@ -595,6 +591,7 @@ def copy_image(
     )
 
 
+# TODO refactor
 @Trackable
 @validate_arguments
 def upload_images_from_public_urls_to_project(
@@ -625,100 +622,22 @@ def upload_images_from_public_urls_to_project(
     :rtype: tuple of list of strs
     """
 
-    if img_names is not None and len(img_names) != len(img_urls):
-        raise AppException("Not all image URLs have corresponding names.")
-
     project_name, folder_name = extract_project_folder(project)
 
-    images_to_upload = []
-    ProcessedImage = namedtuple("ProcessedImage", ["url", "uploaded", "path", "entity"])
-
-    def _upload_image(image_url, image_name=None) -> ProcessedImage:
-        download_response = controller.download_image_from_public_url(
-            project_name=project_name, image_url=image_url
-        )
-        if not download_response.errors:
-            content, content_name = download_response.data
-            image_name = image_name if image_name else content_name
-            duplicated_images = [
-                image.name
-                for image in controller.get_duplicated_images(
-                    project_name=project_name,
-                    folder_name=folder_name,
-                    images=[image_name],
-                )
-            ]
-            if image_name not in duplicated_images:
-                upload_response = controller.upload_image_to_s3(
-                    project_name=project_name,
-                    image_path=image_name,
-                    image_bytes=content,
-                    folder_name=folder_name,
-                    image_quality_in_editor=image_quality_in_editor,
-                )
-                if upload_response.errors:
-                    logger.warning(upload_response.errors)
-                else:
-                    return ProcessedImage(
-                        url=image_url,
-                        uploaded=True,
-                        path=image_url,
-                        entity=upload_response.data,
-                    )
-        logger.warning(download_response.errors)
-        return ProcessedImage(
-            url=image_url, uploaded=False, path=image_name, entity=None
-        )
-
-    logger.info("Downloading %s images", len(img_urls))
-    with tqdm(total=len(img_urls), desc="Downloading") as progress_bar:
-        with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
-            failed_images = []
-            if img_names:
-                results = [
-                    executor.submit(_upload_image, url, img_urls[idx])
-                    for idx, url in enumerate(img_urls)
-                ]
-            else:
-                results = [executor.submit(_upload_image, url) for url in img_urls]
-            for future in concurrent.futures.as_completed(results):
-                processed_image = future.result()
-                if processed_image.uploaded and processed_image.entity:
-                    images_to_upload.append(processed_image)
-                else:
-                    failed_images.append(processed_image)
-                progress_bar.update(1)
-
-    uploaded = []
-    duplicates = []
-    for i in range(0, len(images_to_upload), 500):
-        response = controller.upload_images(
-            project_name=project_name,
-            folder_name=folder_name,
-            images=[
-                image.entity for image in images_to_upload[i : i + 500]  # noqa: E203
-            ],
-            annotation_status=annotation_status,
-        )
-
-        attachments, duplications = response.data
-        uploaded.extend([attachment["name"] for attachment in attachments])
-        duplicates.extend([duplication["name"] for duplication in duplications])
-    uploaded_image_urls = list(
-        {
-            image.entity.name
-            for image in images_to_upload
-            if image.entity.name in uploaded
-        }
-    )
-    failed_image_urls = [image.url for image in failed_images]
-
-    return (
-        uploaded_image_urls,
-        uploaded,
-        duplicates,
-        failed_image_urls,
+    use_case = controller.upload_images_from_public_urls_to_project(
+        project_name=project_name,
+        folder_name=folder_name,
+        image_urls=img_urls,
+        image_names=img_names,
+        annotation_status=annotation_status,
+        image_quality_in_editor=image_quality_in_editor,
     )
+    if use_case.is_valid():
+        with tqdm(total=len(img_urls), desc="Uploading images") as progress_bar:
+            for _ in use_case.execute():
+                progress_bar.update(1)
+        return use_case.data
+    raise AppException(use_case.response.errors)
 
 
 @Trackable
@@ -3320,33 +3239,17 @@ def upload_image_to_project(
     :type image_quality_in_editor: str
     """
     project_name, folder_name = extract_project_folder(project)
-
-    project = controller.get_project_metadata(project_name).data
-    if project["project"].project_type == constances.ProjectType.VIDEO.value:
-        raise AppException(
-            "The function does not support projects containing videos attached with URLs"
-        )
-
-    if not isinstance(img, io.BytesIO):
-        if from_s3_bucket:
-            image_bytes = controller.get_image_from_s3(from_s3_bucket, image_name)
-        else:
-            image_bytes = io.BytesIO(open(img, "rb").read())
-    else:
-        image_bytes = img
-    upload_response = controller.upload_image_to_s3(
+    response = controller.upload_image_to_project(
         project_name=project_name,
-        image_path=image_name if image_name else Path(img).name,
-        image_bytes=image_bytes,
         folder_name=folder_name,
-        image_quality_in_editor=image_quality_in_editor,
-    )
-    controller.upload_images(
-        project_name=project_name,
-        folder_name=folder_name,
-        images=[upload_response.data],  # noqa: E203
+        image_name=image_name,
+        image=img,
         annotation_status=annotation_status,
+        image_quality_in_editor=image_quality_in_editor,
+        from_s3_bucket=from_s3_bucket,
     )
+    if response.errors:
+        raise AppException(response.errors)
 
 
 def search_models(
@@ -3408,103 +3311,31 @@ def upload_images_to_project(
     :return: uploaded, could-not-upload, existing-images filepaths
     :rtype: tuple (3 members) of list of strs
     """
-    uploaded_image_entities = []
-    failed_images = []
     project_name, folder_name = extract_project_folder(project)
-    project = controller.get_project_metadata(project_name).data
-    if project["project"].project_type == constances.ProjectType.VIDEO.value:
-        raise AppException(
-            "The function does not support projects containing videos attached with URLs"
-        )
-
-    ProcessedImage = namedtuple("ProcessedImage", ["uploaded", "path", "entity"])
-
-    def _upload_local_image(image_path: str):
-        try:
-            with open(image_path, "rb") as image:
-                image_bytes = BytesIO(image.read())
-                upload_response = controller.upload_image_to_s3(
-                    project_name=project_name,
-                    image_path=image_path,
-                    image_bytes=image_bytes,
-                    folder_name=folder_name,
-                    image_quality_in_editor=image_quality_in_editor,
-                )
 
-                if not upload_response.errors and upload_response.data:
-                    entity = upload_response.data
-                    return ProcessedImage(
-                        uploaded=True, path=entity.path, entity=entity
-                    )
-                else:
-                    return ProcessedImage(uploaded=False, path=image_path, entity=None)
-        except FileNotFoundError:
-            return ProcessedImage(uploaded=False, path=image_path, entity=None)
-
-    def _upload_s3_image(image_path: str):
-        try:
-            image_bytes = controller.get_image_from_s3(
-                s3_bucket=from_s3_bucket, image_path=image_path
-            ).data
-        except AppValidationException as e:
-            logger.warning(e)
-            return image_path
-        upload_response = controller.upload_image_to_s3(
-            project_name=project_name,
-            image_path=image_path,
-            image_bytes=image_bytes,
-            folder_name=folder_name,
-            image_quality_in_editor=image_quality_in_editor,
-        )
-        if not upload_response.errors and upload_response.data:
-            entity = upload_response.data
-            return ProcessedImage(uploaded=True, path=entity.path, entity=entity)
-        else:
-            return ProcessedImage(uploaded=False, path=image_path, entity=None)
-
-    filtered_paths = img_paths
-    duplication_counter = Counter(filtered_paths)
-    images_to_upload, duplicated_images = (
-        set(filtered_paths),
-        [item for item in duplication_counter if duplication_counter[item] > 1],
+    project_folder_name = project_name + (f"/{folder_name}" if folder_name else "")
+    use_case = controller.upload_images_to_project(
+        project_name=project_name,
+        folder_name=folder_name,
+        annotation_status=annotation_status,
+        image_quality_in_editor=image_quality_in_editor,
+        paths=img_paths,
+        from_s3_bucket=from_s3_bucket,
     )
-    upload_method = _upload_s3_image if from_s3_bucket else _upload_local_image
-
-    with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
-        results = [
-            executor.submit(upload_method, image_path)
-            for image_path in images_to_upload
-        ]
-        with tqdm(total=len(images_to_upload), desc="Uploading images") as progress_bar:
-            for future in concurrent.futures.as_completed(results):
-                processed_image = future.result()
-                if processed_image.uploaded and processed_image.entity:
-                    uploaded_image_entities.append(processed_image.entity)
-                else:
-                    failed_images.append(processed_image.path)
-                progress_bar.update(1)
-    uploaded = []
-    duplicates = []
-
-    logger.info("Uploading %s images to project.", len(images_to_upload))
-
-    for i in range(0, len(uploaded_image_entities), 500):
-        response = controller.upload_images(
-            project_name=project_name,
-            folder_name=folder_name,
-            images=uploaded_image_entities[i : i + 500],  # noqa: E203
-            annotation_status=annotation_status,
-        )
-        attachments, duplications = response.data
-        uploaded.extend(attachments)
-        duplicates.extend(duplications)
-
+    images_to_upload, duplicates = use_case.images_to_upload
     if len(duplicates):
         logger.warning(
             "%s already existing images found that won't be uploaded.", len(duplicates)
         )
-
-    return uploaded, failed_images, duplicates
+    logger.info(
+        "Uploading %s images to project %s.", len(images_to_upload), project_folder_name
+    )
+    if use_case.is_valid():
+        with tqdm(total=len(images_to_upload), desc="Uploading images") as progress_bar:
+            for _ in use_case.execute():
+                progress_bar.update(1)
+        return use_case.data
+    raise AppException(use_case.response.errors)
 
 
 @Trackable

src/superannotate/lib/core/__init__.py

@@ -55,7 +55,21 @@
 TOKEN_UUID = "token"
 
 
-DEPRECATED_VIDEO_PROJECTS_MESSAGE = "The function does not support projects containing videos attached with URLs"
+DEPRECATED_VIDEO_PROJECTS_MESSAGE = (
+    "The function does not support projects containing videos attached with URLs"
+)
+
+UPLOAD_FOLDER_LIMIT_ERROR_MESSAGE = "The number of items you want to upload exceeds the limit of 50 000 items per folder."
+UPLOAD_PROJECT_LIMIT_ERROR_MESSAGE = "The number of items you want to upload exceeds the limit of 500 000 items per project."
+UPLOAD_USER_LIMIT_ERROR_MESSAGE = "The number of items you want to upload  exceeds the limit of your subscription plan."
+
+ATTACH_FOLDER_LIMIT_ERROR_MESSAGE = "The number of items you want to attach exceeds the limit of 50 000 items per folder."
+ATTACH_PROJECT_LIMIT_ERROR_MESSAGE = "The number of items you want to attach exceeds the limit of 500 000 items per project."
+ATTACH_USER_LIMIT_ERROR_MESSAGE = "The number of items you want to attach  exceeds the limit of your subscription plan."
+
+COPY_FOLDER_LIMIT_ERROR_MESSAGE = (
+    "The number of items you want to copy exceeds the limit of 50 000 items per folder."
+)
 
 __version__ = "?"