Merge pull request #187 from superannotateai/fix_cli

Fix cli

Author: dshabin

src/superannotate/lib/app/interface/cli_interface.py

@@ -1,14 +1,9 @@
-import concurrent.futures
 import json
 import logging
 import os
 import sys
 import tempfile
 import uuid
-from collections import Counter
-from collections import namedtuple
-from io import BytesIO
-from pathlib import Path
 from typing import Any
 from typing import Optional
 
@@ -19,12 +14,21 @@
 from lib.app.helpers import split_project_path
 from lib.app.input_converters.conversion import import_annotation
 from lib.app.interface.base_interface import BaseInterfaceFacade
+from lib.app.interface.sdk_interface import attach_image_urls_to_project
+from lib.app.interface.sdk_interface import attach_video_urls_to_project
+from lib.app.interface.sdk_interface import create_folder
+from lib.app.interface.sdk_interface import create_project
+from lib.app.interface.sdk_interface import upload_images_from_folder_to_project
+from lib.app.interface.sdk_interface import upload_videos_from_folder_to_project
 from lib.app.serializers import ImageSerializer
 from lib.core.entities import ConfigEntity
+from lib.infrastructure.controller import Controller
 from lib.infrastructure.repositories import ConfigRepository
 from tqdm import tqdm
 
+
 logger = logging.getLogger()
+controller = Controller(logger)
 
 
 class CLIFacade(BaseInterfaceFacade):
@@ -72,18 +76,14 @@ def create_project(self, name: str, description: str, type: str):
         """
         To create a new project
         """
-        response = self.controller.create_project(name, description, type)
-        if response.errors:
-            return response.errors
-        return response.data
+        create_project(name, description, type)
+        sys.exit(0)
 
     def create_folder(self, project: str, name: str):
         """
         To create a new folder
         """
-        response = self.controller.create_folder(project=project, folder_name=name)
-        if response.errors:
-            logger.critical(response.errors)
+        create_folder(project, name)
         sys.exit(0)
 
     def upload_images(
@@ -104,79 +104,19 @@ def upload_images(
         Optional argument extensions accepts comma separated list of image extensions to look for.
         If the argument is not given then value jpg,jpeg,png,tif,tiff,webp,bmp is assumed.
         """
-        uploaded_image_entities = []
-        failed_images = []
-        project_name, folder_name = split_project_path(project)
-        ProcessedImage = namedtuple("ProcessedImage", ["uploaded", "path", "entity"])
-
-        def upload_image(image_path: str):
-            with open(image_path, "rb") as image:
-                image_bytes = BytesIO(image.read())
-                upload_response = self.controller.upload_image_to_s3(
-                    project_name=project_name,
-                    image_path=image_path,
-                    image_bytes=image_bytes,
-                    folder_name=folder_name,
-                    image_quality_in_editor=image_quality_in_editor,
-                )
-
-                if not upload_response.errors and upload_response.data:
-                    entity = upload_response.data
-                    return ProcessedImage(
-                        uploaded=True, path=entity.path, entity=entity
-                    )
-                else:
-                    return ProcessedImage(uploaded=False, path=image_path, entity=None)
-
-        paths = []
-
-        if isinstance(extensions, str):
-            extensions = extensions.strip().split(",")
+        if not isinstance(extensions, list):
+            extensions = extensions.split(",")
 
-        for extension in extensions:
-            if recursive_subfolders:
-                paths += list(Path(folder).rglob(f"*.{extension.lower()}"))
-                if os.name != "nt":
-                    paths += list(Path(folder).rglob(f"*.{extension.upper()}"))
-            else:
-                paths += list(Path(folder).glob(f"*.{extension.lower()}"))
-                if os.name != "nt":
-                    paths += list(Path(folder).glob(f"*.{extension.upper()}"))
-
-        filtered_paths = []
-        for path in paths:
-            not_in_exclude_list = [
-                x not in Path(path).name for x in exclude_file_patterns
-            ]
-            if all(not_in_exclude_list):
-                filtered_paths.append(path)
-
-        duplication_counter = Counter(filtered_paths)
-        images_to_upload, duplicated_images = (
-            set(filtered_paths),
-            [item for item in duplication_counter if duplication_counter[item] > 1],
+        upload_images_from_folder_to_project(
+            project,
+            folder_path=folder,
+            extensions=extensions,
+            annotation_status=set_annotation_status,
+            from_s3_bucket=None,
+            exclude_file_patterns=exclude_file_patterns,
+            recursive_subfolders=recursive_subfolders,
+            image_quality_in_editor=image_quality_in_editor,
         )
-        with tqdm(total=len(images_to_upload)) as progress_bar:
-            with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
-                results = [
-                    executor.submit(upload_image, image_path)
-                    for image_path in images_to_upload
-                ]
-                for future in concurrent.futures.as_completed(results):
-                    processed_image = future.result()
-                    if processed_image.uploaded and processed_image.entity:
-                        uploaded_image_entities.append(processed_image.entity)
-                    else:
-                        failed_images.append(processed_image.path)
-                    progress_bar.update(1)
-
-        for i in range(0, len(uploaded_image_entities), 500):
-            self.controller.upload_images(
-                project_name=project_name,
-                folder_name=folder_name,
-                images=uploaded_image_entities[i : i + 500],  # noqa: E203
-                annotation_status=set_annotation_status,
-            )
         sys.exit(0)
 
     def export_project(
@@ -305,13 +245,22 @@ def attach_image_urls(
         """
         To attach image URLs to project use:
         """
-        self._attach_urls(project, attachments, annotation_status)
+
+        attach_image_urls_to_project(
+            project=project,
+            attachments=attachments,
+            annotation_status=annotation_status,
+        )
         sys.exit(0)
 
     def attach_video_urls(
         self, project: str, attachments: str, annotation_status: Optional[Any] = None
     ):
-        self._attach_urls(project, attachments, annotation_status)
+        attach_video_urls_to_project(
+            project=project,
+            attachments=attachments,
+            annotation_status=annotation_status,
+        )
         sys.exit(0)
 
     def _attach_urls(
@@ -372,58 +321,17 @@ def upload_videos(
         start-time specifies time (in seconds) from which to start extracting frames, default is 0.0.
         end-time specifies time (in seconds) up to which to extract frames. If it is not specified, then up to end is assumed.
         """
-        project_name, folder_name = split_project_path(project)
-
-        uploaded_image_entities = []
-        failed_images = []
-
-        def _upload_image(image_path: str) -> str:
-            with open(image_path, "rb") as image:
-                image_bytes = BytesIO(image.read())
-                upload_response = self.controller.upload_image_to_s3(
-                    project_name=project_name,
-                    image_path=image_path,
-                    image_bytes=image_bytes,
-                    folder_name=folder_name,
-                )
-                if not upload_response.errors:
-                    uploaded_image_entities.append(upload_response.data)
-                else:
-                    return image_path
 
-        video_paths = []
-        for extension in extensions:
-            if not recursive:
-                video_paths += list(Path(folder).glob(f"*.{extension.lower()}"))
-                if os.name != "nt":
-                    video_paths += list(Path(folder).glob(f"*.{extension.upper()}"))
-            else:
-                video_paths += list(Path(folder).rglob(f"*.{extension.lower()}"))
-                if os.name != "nt":
-                    video_paths += list(Path(folder).rglob(f"*.{extension.upper()}"))
-        video_paths = [str(path) for path in video_paths]
-
-        for path in video_paths:
-            with tempfile.TemporaryDirectory() as temp_path:
-                res = self.controller.extract_video_frames(
-                    project_name=project_name,
-                    folder_name=folder_name,
-                    video_path=path,
-                    extract_path=temp_path,
-                    target_fps=int(target_fps),
-                    start_time=float(start_time),
-                    end_time=end_time if not end_time else float(end_time),
-                    annotation_status=set_annotation_status,
-                )
-                if not res.errors:
-                    extracted_frame_paths = res.data
-                    for image_path in extracted_frame_paths:
-                        failed_images.append(_upload_image(image_path))
-        for i in range(0, len(uploaded_image_entities), 500):
-            self.controller.upload_images(
-                project_name=project_name,
-                folder_name=folder_name,
-                images=uploaded_image_entities[i : i + 500],  # noqa: E203
-                annotation_status=set_annotation_status,
-            )
+        upload_videos_from_folder_to_project(
+            project=project,
+            folder_path=folder,
+            extensions=extensions,
+            exclude_file_patterns=(),
+            recursive_subfolders=recursive,
+            target_fps=target_fps,
+            start_time=start_time,
+            end_time=end_time,
+            annotation_status=set_annotation_status,
+            image_quality_in_editor=None,
+        )
         sys.exit(0)

src/superannotate/lib/app/interface/sdk_interface.py

@@ -1150,7 +1150,9 @@ def assign_images(project: Union[str, dict], image_names: List[str], user: str):
         )
         return
 
-    controller.assign_images(project_name, folder_name, image_names, user)
+    response = controller.assign_images(project_name, folder_name, image_names, user)
+    if not response.errors:
+        logger.info(f"Assign images to user {user}")
 
 
 @Trackable
@@ -1558,9 +1560,7 @@ def upload_images_from_folder_to_project(
             "extensions should be a list or a tuple in upload_images_from_folder_to_project"
         )
 
-    project_folder_name = project_name + (
-        f"/{folder_name}" if folder_name != "root" else ""
-    )
+    project_folder_name = project_name + (f"/{folder_name}" if folder_name else "")
 
     logger.info(
         "Uploading all images with extensions %s from %s to project %s. Excluded file patterns are: %s.",
@@ -1886,6 +1886,8 @@ def upload_videos_from_folder_to_project(
         if all(not_in_exclude_list):
             filtered_paths.append(path)
 
+    project_folder_name = project_name + (f"/{folder_name}" if folder_name else "")
+
     logger.info(
         "Uploading all videos with extensions %s from %s to project %s. Excluded file patterns are: %s.",
         extensions,
@@ -1894,8 +1896,7 @@ def upload_videos_from_folder_to_project(
         exclude_file_patterns,
     )
 
-    uploaded_images, failed_images = [], []
-    for path in tqdm(video_paths, desc="Uploading videos"):
+    for path in video_paths:
         with tempfile.TemporaryDirectory() as temp_path:
             res = controller.extract_video_frames(
                 project_name=project_name,
@@ -1917,17 +1918,31 @@ def upload_videos_from_folder_to_project(
                 annotation_status=annotation_status,
                 image_quality_in_editor=image_quality_in_editor,
             )
-            images_to_upload, _ = use_case.images_to_upload
+            images_to_upload, duplicates = use_case.images_to_upload
+            logger.info(
+                "Extracted %s frames from video. Now uploading to platform.",
+                len(res.data),
+            )
+            logger.info(
+                "Uploading %s images to project %s.",
+                len(images_to_upload),
+                str(project_folder_name),
+            )
+            if len(duplicates):
+                logger.warning(
+                    "%s already existing images found that won't be uploaded.",
+                    len(duplicates),
+                )
             if use_case.is_valid():
-                for _ in use_case.execute():
-                    pass
-                uploaded, failed_images, _ = use_case.data
-                uploaded_images.append(uploaded)
-                failed_images.append(failed_images)
+                with tqdm(
+                    total=len(images_to_upload), desc="Uploading images"
+                ) as progress_bar:
+                    for _ in use_case.execute():
+                        progress_bar.update(1)
             else:
                 raise AppValidationException(use_case.response.errors)
 
-    return uploaded_images, failed_images
+    return
 
 
 @Trackable

src/superannotate/lib/core/plugin.py

@@ -1,5 +1,6 @@
 import io
 import logging
+import os
 from pathlib import Path
 from typing import List
 from typing import Tuple
@@ -234,17 +235,17 @@ def extract_frames(
             )
 
         ratio = fps / target_fps
-        extracted_frames_paths = []
         zero_fill_count = len(str(frames_count))
 
         rotate_code = VideoPlugin.get_video_rotate_code(video_path)
 
         frame_number = 0
         extracted_frame_number = 0
         extracted_frame_ratio = ratio
-        logger.info("Extracting frames from video to %s.", extracted_frames_paths)
+        logger.info("Extracting frames from video to %s.", extract_path)
+        extracted_frames_paths = []
 
-        while len(extracted_frames_paths) < limit:
+        while len(os.listdir(extract_path)) < limit:
             success, frame = video.read()
             if success:
                 frame_time = video.get(cv2.CAP_PROP_POS_MSEC) / 1000.0

src/superannotate/lib/core/usecases.py

@@ -1705,8 +1705,6 @@ def execute(self):
                         f"Cant assign {', '.join(self._image_names[i: i + self.CHUNK_SIZE])}"
                     )
                     continue
-            logger.info(f"Assign images to user {self._user}")
-
         return self._response