Merge develop

Author: dshabin

.pylintrc

@@ -1,5 +1,5 @@
 [BASIC]
-good-names=df,f,e,i,j,k,im,pt,pr
+good-names=df,f,e,i,j,k,im,pt,pr,m
 
 bad-functions=
   apply,
@@ -23,7 +23,8 @@ disable=
     no-else-return,
     too-many-lines,
     too-many-arguments,
-    too-many-instance-attributes
+    too-many-instance-attributes,
+    broad-except
 
 [SPELLING]
 

docs/source/cli.rst

@@ -79,6 +79,19 @@ to look for. If the argument is not given then value *jpg,jpeg,png,tif,tiff,webp
 
 ----------
 
+.. _ref_attach_image_urls:
+
+Attaching image URLs
+~~~~~~~~~~~~~~~~~~~~
+
+To attach image URLs to project use:
+
+.. code-block:: bash
+
+   superannotatecli attach-image-urls --project <project_name/folder_name> --attachments <csv_path> [--annotation_status <annotation_status>]
+
+----------
+
 .. _ref_upload_videos:
 
 Uploading videos

superannotate/db/debug_interview.py

@@ -6,6 +6,7 @@
 import zipfile
 from datetime import datetime
 from pathlib import Path
+import shutil
 
 import boto3
 import requests
@@ -128,6 +129,7 @@ def prepare_export(
         logger.info(
             "Include fuse functionality is not supported for  projects containing images attached with URLs"
         )
+        include_fuse = False
     team_id, project_id = project["team_id"], project["id"]
     if annotation_statuses is None:
         annotation_statuses = [2, 3, 4, 5]
@@ -208,6 +210,15 @@ def __upload_files_to_aws_thread(
             already_uploaded[i] = True
 
 
+def _download_file(url, local_filename):
+    with requests.get(url, stream=True) as r:
+        r.raise_for_status()
+        with open(local_filename, 'wb') as f:
+            for chunk in r.iter_content(chunk_size=8192):
+                f.write(chunk)
+    return local_filename
+
+
 def download_export(
     project, export, folder_path, extract_zip_contents=True, to_s3_bucket=None
 ):
@@ -242,25 +253,26 @@ def download_export(
         break
 
     filename = Path(res['path']).name
-    r = requests.get(res['download'], allow_redirects=True)
-    if to_s3_bucket is None:
-        filepath = Path(folder_path) / filename
-        open(filepath, 'wb').write(r.content)
-        if extract_zip_contents:
-            with zipfile.ZipFile(filepath, 'r') as f:
-                f.extractall(folder_path)
-            Path.unlink(filepath)
-            logger.info("Extracted %s to folder %s", filepath, folder_path)
-        else:
-            logger.info("Downloaded export ID %s to %s", res['id'], filepath)
-    else:
-        with tempfile.TemporaryDirectory() as tmpdirname:
-            filepath = Path(tmpdirname) / filename
-            open(filepath, 'wb').write(r.content)
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        temp_filepath = Path(tmpdirname) / filename
+        _download_file(res['download'], temp_filepath)
+        if to_s3_bucket is None:
+            filepath = Path(folder_path) / filename
+            shutil.copyfile(temp_filepath, filepath)
             if extract_zip_contents:
                 with zipfile.ZipFile(filepath, 'r') as f:
-                    f.extractall(tmpdirname)
+                    f.extractall(folder_path)
                 Path.unlink(filepath)
+                logger.info("Extracted %s to folder %s", filepath, folder_path)
+            else:
+                logger.info(
+                    "Downloaded export ID %s to %s", res['id'], filepath
+                )
+        else:
+            if extract_zip_contents:
+                with zipfile.ZipFile(temp_filepath, 'r') as f:
+                    f.extractall(tmpdirname)
+                Path.unlink(temp_filepath)
             files_to_upload = []
             for file in Path(tmpdirname).rglob("*.*"):
                 if not file.is_file():
@@ -295,4 +307,4 @@ def download_export(
                     t.join()
             finish_event.set()
             tqdm_thread.join()
-        logger.info("Exported to AWS %s/%s", to_s3_bucket, folder_path)
+            logger.info("Exported to AWS %s/%s", to_s3_bucket, folder_path)

superannotate/db/exports.py

@@ -722,7 +722,7 @@ def get_image_bytes(project, image_name, variant='original'):
         raise SABaseException(
             0, "Image download variant should be either original or lores"
         )
-    image = get_image_metadata(project, image_name)
+    image = get_image_metadata((project, project_folder), image_name)
     team_id, project_id, image_id, folder_id = image["team_id"], image[
         "project_id"], image["id"], image['folder_id']
     params = {
@@ -1223,7 +1223,7 @@ def set_images_annotation_statuses(project, image_names, annotation_status):
         data["image_names"] = image_names[start_index:start_index + NUM_TO_SEND]
         response = _api.send_request(
             req_type='PUT',
-            path=f'/image/updateAnnotationStatusBulk',
+            path='/image/updateAnnotationStatusBulk',
             params=params,
             json_req=data
         )

superannotate/db/images.py

@@ -105,7 +105,6 @@ def get_project_and_folder_metadata(project):
     :rtype: dict
     """
     if isinstance(project, dict):
-        project = project
         folder = None
     elif isinstance(project, tuple):
         if len(project) != 2:

superannotate/db/project_api.py

@@ -118,7 +118,7 @@ def upload_image_to_project(
         )
         key = upload_image_array_to_s3(bucket, *images_info_and_array, prefix)
     except Exception as e:
-        raise SABaseException(0, "Couldn't upload to data server. " + str(e))
+        raise SABaseException(0, "Couldn't upload to data server.") from e
 
     __create_image(
         [img_name], [key],
@@ -375,18 +375,15 @@ def copy_image(
         except SABaseException:
             break
         else:
-            found_copied = False
             for m in p.finditer(new_name):
                 if m.start() + len(m.group()
                                   ) + len(extension) - 1 == len(new_name):
                     num = int(m.group()[2:-2])
-                    found_copied = True
+                    new_name = new_name[:m.start() +
+                                        2] + str(num + 1) + ")" + extension
                     break
-            if not found_copied:
-                new_name = Path(new_name).stem + "_(1)" + extension
             else:
-                new_name = new_name[:m.start() +
-                                    2] + str(num + 1) + ")" + extension
+                new_name = Path(new_name).stem + "_(1)" + extension
 
     upload_image_to_project(
         (destination_project, destination_project_folder), img_b, new_name

superannotate/db/project_images.py

@@ -266,7 +266,8 @@ def upload_video_to_project(
         if "ffprobe" in str(e):
             warning_str = "This could be because ffmpeg package is not installed. To install it, run: sudo apt install ffmpeg"
         logger.warning(
-            "Couldn't read video metadata to determine rotation. " + warning_str
+            "Couldn't read video metadata to determine rotation. %s",
+            warning_str
         )
 
     video = cv2.VideoCapture(str(video_path), cv2.CAP_FFMPEG)
@@ -340,7 +341,7 @@ def upload_video_to_project(
     )
 
     filenames = upload_images_from_folder_to_project(
-        project,
+        (project, project_folder),
         tempdir.name,
         extensions=["jpg"],
         annotation_status=annotation_status,
@@ -767,9 +768,6 @@ def __create_image(
         "team_id": str(team_id),
         "images": [],
         "annotation_status": annotation_status,
-        "team_id": str(team_id),
-        "images": [],
-        "annotation_status": annotation_status,
         "meta": {},
         "upload_state": upload_state_code
     }
@@ -1086,7 +1084,7 @@ def __attach_image_urls_to_project_thread(
     for i in range(start_index, end_index):
         if i >= len_img_paths:
             break
-        name, url = img_names_urls[i]
+        name, _ = img_names_urls[i]
         tried_upload[thread_id].append(name)
         img_name_hash = str(uuid.uuid4()) + Path(name).suffix
         key = prefix + img_name_hash
@@ -1190,7 +1188,7 @@ def upload_images_from_public_urls_to_project(
         ),
         daemon=True
     )
-    logger.info('Downloading %s images' % len(img_urls))
+    logger.info('Downloading %s images', len(img_urls))
     tqdm_thread.start()
     with tempfile.TemporaryDirectory() as save_dir_name:
         save_dir = Path(save_dir_name)
@@ -1227,7 +1225,7 @@ def upload_images_from_public_urls_to_project(
         finish_event.set()
         tqdm_thread.join()
         images_uploaded_paths, images_not_uploaded_paths, duplicate_images_paths = upload_images_to_project(
-            project,
+            (project, project_folder),
             images_to_upload,
             annotation_status=annotation_status,
             image_quality_in_editor=image_quality_in_editor
@@ -1313,7 +1311,7 @@ def upload_images_from_google_cloud_to_project(
                 path_to_url[str(image_save_pth)] = image_blob.name
                 images_to_upload.append(image_save_pth)
         images_uploaded_paths, images_not_uploaded_paths, duplicate_images_paths = upload_images_to_project(
-            project,
+            (project, project_folder),
             images_to_upload,
             annotation_status=annotation_status,
             image_quality_in_editor=image_quality_in_editor
@@ -1409,7 +1407,7 @@ def upload_images_from_azure_blob_to_project(
                 path_to_url[str(image_save_pth)] = image_blob.name
                 images_to_upload.append(image_save_pth)
         images_uploaded_paths, images_not_uploaded_paths, duplicate_images_paths = upload_images_to_project(
-            project,
+            (project, project_folder),
             images_to_upload,
             annotation_status=annotation_status,
             image_quality_in_editor=image_quality_in_editor
@@ -2229,11 +2227,10 @@ def get_project_default_image_quality_in_editor(project):
     for setting in get_project_settings(project):
         if "attribute" in setting and setting["attribute"] == "ImageQuality":
             return setting["value"]
-    else:
-        raise SABaseException(
-            0,
-            "Image quality in editor should be 'compressed', 'original' or None for project settings value"
-        )
+    raise SABaseException(
+        0,
+        "Image quality in editor should be 'compressed', 'original' or None for project settings value"
+    )
 
 
 def get_project_metadata(

superannotate/db/projects.py

@@ -228,12 +228,15 @@ def run_training(
             raise SABaseException(0, "Invalid project types")
         project_type = types.pop()
 
-    upload_state = upload_state_int_to_str(project.get("upload_state"))
-    if upload_state == "External":
-        raise SABaseException(
-            0,
-            "The function does not support projects containing images attached with URLs"
+    for single_project in project:
+        upload_state = upload_state_int_to_str(
+            single_project.get("upload_state")
         )
+        if upload_state == "External":
+            raise SABaseException(
+                0,
+                "The function does not support projects containing images attached with URLs"
+            )
 
     base_model = base_model.get(project_type, None)
     if not base_model:

superannotate/ml/ml_funcs.py

@@ -1 +1 @@
-__version__ = "4.1.1b4"
+__version__ = "4.1.2"