Merge pull request #29 from superannotateai/develop-coco-fix

Develop coco fix - mixp

Author: dshabin

superannotate/api.py

@@ -1,6 +1,7 @@
 import os
 import json
 import logging
+import sys
 from pathlib import Path
 
 import requests
@@ -88,8 +89,8 @@ def init(self, config_location=None):
                 path=f'/team/{self.team_id}',
             )
 
-            self.user_id = response.json()['creator_id']
-            self.team_name = response.json()['name']
+            self.user_id = response.json().get('creator_id', None)
+            self.team_name = response.json().get('name', None)
 
             if not self._verify:
                 urllib3.disable_warnings(
@@ -106,10 +107,11 @@ def init(self, config_location=None):
                 raise SABaseException(
                     0, "Couldn't reach superannotate " + response.text
                 )
-            mp.track(
-                self.user_id, "SDK init",
-                get_default(self.team_name, self.user_id)
-            )
+            if "pytest" not in sys.modules:
+                mp.track(
+                    self.user_id, "SDK init",
+                    get_default(self.team_name, self.user_id)
+                )
         except SABaseException:
             self._authenticated = False
             self._session = None

superannotate/db/annotation_classes.py

@@ -330,6 +330,21 @@ def fill_class_and_attribute_names(annotations_json, annotation_classes_dict):
 def fill_class_and_attribute_ids(annotation_json, annotation_classes_dict):
     if "instances" not in annotation_json:
         return
+
+    unknown_classes = {}
+    for ann in [i for i in annotation_json["instances"] if "className" in i]:
+        if "className" not in ann:
+            continue
+        annotation_class_name = ann["className"]
+        if not annotation_class_name in annotation_classes_dict:
+            if annotation_class_name not in unknown_classes:
+                class_num = -(len(unknown_classes) + 1)
+                unknown_classes[annotation_class_name] = {
+                    'id': class_num,
+                    'attribute_groups': {}
+                }
+    annotation_classes_dict = {**annotation_classes_dict, **unknown_classes}
+
     for ann in annotation_json["instances"]:
         if "className" not in ann:
             logger.warning("No className in annotation instance")

superannotate/input_converters/converters/coco_converters/coco_api.py

@@ -103,6 +103,8 @@ def _area(bitmask):
 
 def _toBbox(bitmask):
     y, x = np.where(bitmask)
+    if x.tolist() == [] and y.tolist() == []:
+        return [0, 0, 0, 0]
     xmin = int(min(x))
     xmax = int(max(x))
     ymin = int(min(y))

superannotate/input_converters/converters/coco_converters/coco_converter.py

@@ -169,11 +169,10 @@ def _parse_json_into_common_format(self, sa_annotation_json, fpath):
 
         if 'name' not in sa_annotation_json[
             'metadata'] or sa_annotation_json['metadata']['name'] is None:
-            fname = fpath.split('/')[-1]
+            fname = fpath.name
             fname = fname[:-len(
-                self.project_type_to_json_ending[self.project_type]
+                self.project_type_to_json_ending[self.project_type.lower()]
             )]
-
             sa_annotation_json['metadata']['name'] = fname
         sa_annotation_json['metadata']['image_path'] = str(
             Path(fpath).parent / sa_annotation_json['metadata']['name']

superannotate/mixp/decorators.py

@@ -34,7 +34,8 @@ def __call__(self, *args, **kwargs):
                 )
                 properties.pop("project_name", None)
                 properties = {**default, **properties}
-                mp.track(user_id, event_name, properties)
+                if "pytest" not in sys.modules:
+                    mp.track(user_id, event_name, properties)
         except:
-            print('--- mix panel exception ---')
+            pass
         return self.function(*args, **kwargs)

superannotate/mixp/utils/parsers.py

@@ -1047,15 +1047,63 @@ def upload_images_from_folder_to_project(*args, **kwargs):
     folder_path = kwargs.get("folder_path", None)
     if not folder_path:
         folder_path = args[1]
-    from ...common import DEFAULT_IMAGE_EXTENSIONS as extension
+
+    recursive_subfolders = kwargs.get("recursive_subfolders", None)
+    if not recursive_subfolders:
+        recursive_subfolders = args[6:7]
+        if recursive_subfolders:
+            recursive_subfolders = recursive_subfolders[0]
+        else:
+            recursive_subfolders = False
+
+    from ...common import DEFAULT_IMAGE_EXTENSIONS
+    extensions = DEFAULT_IMAGE_EXTENSIONS
+    extensions = kwargs.get("extensions", None)
+    if not extensions:
+        extensions = args[2:3]
+        if extensions:
+            extensions = extensions[0]
+        else:
+            extensions = DEFAULT_IMAGE_EXTENSIONS
+
+    from ...common import DEFAULT_FILE_EXCLUDE_PATTERNS
+    exclude_file_patterns = DEFAULT_FILE_EXCLUDE_PATTERNS
+    exclude_file_patterns = kwargs.get("exclude_file_patterns", None)
+    if not exclude_file_patterns:
+        exclude_file_patterns = args[5:6]
+        if exclude_file_patterns:
+            exclude_file_patterns = exclude_file_patterns[0]
+        else:
+            exclude_file_patterns = DEFAULT_FILE_EXCLUDE_PATTERNS
+
     from pathlib import Path
-    glob_iterator = Path(folder_path).glob(f'*.{extension}')
+    import os
+
+    paths = []
+    for extension in extensions:
+        if not recursive_subfolders:
+            paths += list(Path(folder_path).glob(f'*.{extension.lower()}'))
+            if os.name != "nt":
+                paths += list(Path(folder_path).glob(f'*.{extension.upper()}'))
+        else:
+            paths += list(Path(folder_path).rglob(f'*.{extension.lower()}'))
+            if os.name != "nt":
+                paths += list(Path(folder_path).rglob(f'*.{extension.upper()}'))
+
+    filtered_paths = []
+    for path in paths:
+        not_in_exclude_list = [
+            x not in Path(path).name for x in exclude_file_patterns
+        ]
+        if all(not_in_exclude_list):
+            filtered_paths.append(path)
+
     return {
         "event_name": "upload_images_from_folder_to_project",
         "properties":
             {
                 "Image Count":
-                    sum(1 for _ in glob_iterator),
+                    len(filtered_paths),
                 "Custom Extentions":
                     bool(args[2:3] or kwargs.get("extensions", None)),
                 "Annotation Status":

tests/converter_test/COCO/input/fromSuperAnnotate/cats_dogs_vector_instance_segm_empty_array/classes/classes.json

@@ -0,0 +1,12 @@
+[
+    {
+        "id": 1,
+        "project_id": 1,
+        "name": "class1",
+        "color": "#626bef",
+        "count": 0,
+        "createdAt": "2021-03-31T21:13:10.000Z",
+        "updatedAt": "2021-03-31T21:13:10.000Z",
+        "attribute_groups": []
+    }
+]

tests/converter_test/COCO/input/fromSuperAnnotate/cats_dogs_vector_instance_segm_empty_array/some_image.JPG___objects.json

@@ -0,0 +1,53 @@
+{
+    "metadata": {
+        "lastAction": {
+            "email": "",
+            "timestamp": 1619092786567
+        },
+        "width": 2448,
+        "height": 3264,
+        "name": "q.JPG",
+        "projectId": 1,
+        "isPredicted": false,
+        "status": "Completed",
+        "pinned": false,
+        "annotatorEmail": "",
+        "qaEmail": ""
+    },
+    "comments": [],
+    "tags": [],
+    "instances": [
+        {
+            "type": "polygon",
+            "classId": 1,
+            "probability": 100,
+            "points": [
+                2448,
+                2612.07,
+                2448,
+                2651.45,
+                2448,
+                988.83
+            ],
+            "groupId": 0,
+            "pointLabels": {},
+            "locked": false,
+            "visible": true,
+            "attributes": [],
+            "trackingId": null,
+            "error": null,
+            "createdAt": "2021-04-21T06:56:10.242Z",
+            "createdBy": {
+                "email": "",
+                "role": ""
+            },
+            "creationType": "Manual",
+            "updatedAt": "2021-04-21T06:56:10.242Z",
+            "updatedBy": {
+                "email": "",
+                "role": ""
+            },
+            "className": "class1"
+        }
+    ]
+}

tests/converter_test/COCO/input/fromSuperAnnotate/cats_dogs_vector_instance_segm_empty_array/truth.json

@@ -0,0 +1 @@
+{"info": {"description": "This is instance_test_vector dataset.", "url": "https://superannotate.ai", "version": "1.0", "year": 2021, "contributor": "Superannotate AI", "date_created": "17/05/2021"}, "licenses": [{"url": "https://superannotate.ai", "id": 1, "name": "Superannotate AI"}], "images": [{"id": 1, "file_name": "q.JPG", "height": 3264, "width": 2448, "license": 1}], "annotations": [{"id": 1, "image_id": 1, "segmentation": [[2448, 2612.07, 2448, 2651.45, 2448, 988.83]], "iscrowd": 0, "bbox": [0, 0, 0, 0], "area": 0, "category_id": 1}], "categories": [{"id": 1, "name": "class1", "supercategory": "class1", "isthing": 1, "color": [1, 0, 0]}]}

tests/converter_test/COCO/input/fromSuperAnnotate/vector_no_name/classes/classes.json

@@ -0,0 +1 @@
+[{"id":599064,"project_id":76203,"name":"some","color":"#c444de","count":0,"createdAt":"2021-05-04T07:26:32.000Z","updatedAt":"2021-05-04T07:26:32.000Z","attribute_groups":[]}]