Merge branch 'develop' of https://github.com/superannotateai/superannotate-python-sdk into develop

Author: Vaghinak Basentsyan

src/superannotate/lib/app/interface/sdk_interface.py

@@ -2445,23 +2445,24 @@ def upload_annotations_from_folder_to_project(
     """
 
     project_name, folder_name = extract_project_folder(project)
+    project_folder_name = project_name + (f"/{folder_name}" if folder_name else "")
 
     if recursive_subfolders:
         logger.info(
             "When using recursive subfolder parsing same name annotations in different "
             "subfolders will overwrite each other.",
         )
-    logger.info("The JSON files should follow a specific naming convention, matching file names already present "
-                "on the platform. Existing annotations will be overwritten")
+    logger.info(
+        "The JSON files should follow a specific naming convention, matching file names already present "
+        "on the platform. Existing annotations will be overwritten"
+    )
 
     annotation_paths = get_annotation_paths(
         folder_path, from_s3_bucket, recursive_subfolders
     )
-    if not annotation_paths:
-        raise AppException("Could not find annotations matching existing items on the platform.")
 
     logger.info(
-        "Uploading %s annotations to project %s.", len(annotation_paths), project_name
+        f"Uploading {len(annotation_paths)} annotations from {folder_path} to the project {project_folder_name}."
     )
     response = controller.upload_annotations_from_folder(
         project_name=project_name,
@@ -2506,31 +2507,28 @@ def upload_preannotations_from_folder_to_project(
     :rtype: tuple of list of strs
     """
     project_name, folder_name = extract_project_folder(project)
+    project_folder_name = project_name + (f"/{folder_name}" if folder_name else "")
     project = controller.get_project_metadata(project_name).data
     if project["project"].project_type in [
         constances.ProjectType.VIDEO.value,
         constances.ProjectType.DOCUMENT.value,
     ]:
         raise AppException(LIMITED_FUNCTIONS[project["project"].project_type])
-
     if recursive_subfolders:
         logger.info(
-            "When using recursive subfolder parsing same name annotations in different subfolders will overwrite each other.",
+            "When using recursive subfolder parsing same name annotations in different "
+            "subfolders will overwrite each other.",
         )
-
     logger.info(
-        "The JSON files should follow specific naming convention. For Vector projects they should be named '<image_name>___objects.json', for Pixel projects JSON file should be names '<image_name>___pixel.json' and also second mask image file should be present with the name '<image_name>___save.png'. In both cases image with <image_name> should be already present on the platform."
+        "The JSON files should follow a specific naming convention, matching file names already present "
+        "on the platform. Existing annotations will be overwritten"
     )
     logger.info("Existing annotations will be overwritten.",)
-    logger.info(
-        "Uploading all annotations from %s to project %s.", folder_path, project_name
-    )
-
     annotation_paths = get_annotation_paths(
         folder_path, from_s3_bucket, recursive_subfolders
     )
     logger.info(
-        "Uploading %s annotations to project %s.", len(annotation_paths), project_name
+        f"Uploading {len(annotation_paths)} annotations from {folder_path} to the project {project_folder_name}."
     )
     response = controller.upload_annotations_from_folder(
         project_name=project_name,

src/superannotate/lib/core/helpers.py

@@ -8,7 +8,7 @@
 def map_annotation_classes_name(annotation_classes, reporter: Reporter) -> dict:
     classes_data = defaultdict(dict)
     for annotation_class in annotation_classes:
-        class_info = {"id": annotation_class.uuid}
+        class_info = {"id": annotation_class.uuid, "attribute_groups": {}}
         if annotation_class.attribute_groups:
             for attribute_group in annotation_class.attribute_groups:
                 attribute_group_data = defaultdict(dict)
@@ -27,12 +27,11 @@ def map_annotation_classes_name(annotation_classes, reporter: Reporter) -> dict:
                         " Only one of the annotation class attribute groups will be used."
                         " This will result in errors in annotation upload."
                     )
-                class_info["attribute_groups"] = {
-                    attribute_group["name"]: {
-                        "id": attribute_group["id"],
-                        "attributes": attribute_group_data,
-                    }
+                class_info["attribute_groups"][attribute_group["name"]] = {
+                    "id": attribute_group["id"],
+                    "attributes": attribute_group_data,
                 }
+
         if annotation_class.name in classes_data.keys():
             reporter.log_warning(
                 f"Duplicate annotation class name {annotation_class.name}."
@@ -43,6 +42,16 @@ def map_annotation_classes_name(annotation_classes, reporter: Reporter) -> dict:
     return classes_data
 
 
+def fill_document_tags(
+    annotations: dict, annotation_classes: dict,
+):
+    new_tags = []
+    for tag in annotations["tags"]:
+        if annotation_classes.get(tag):
+            new_tags.append(annotation_classes[tag]["id"])
+    annotations["tags"] = new_tags
+
+
 def fill_annotation_ids(
     annotations: dict,
     annotation_classes_name_maps: dict,
@@ -95,7 +104,10 @@ def fill_annotation_ids(
                 reporter.log_warning(
                     f"Couldn't find annotation group {attribute['groupName']}."
                 )
-                reporter.store_message("missing_attribute_groups", f"{annotation['className']}.{attribute['groupName']}")
+                reporter.store_message(
+                    "missing_attribute_groups",
+                    f"{annotation['className']}.{attribute['groupName']}",
+                )
                 continue
             attribute["groupId"] = annotation_classes_name_maps[annotation_class_name][
                 "attribute_groups"
@@ -118,12 +130,16 @@ def fill_annotation_ids(
             ][attribute["groupName"]]["attributes"][attribute["name"]]
 
 
-def convert_to_video_editor_json(data: dict, class_name_mapper: dict, reporter: Reporter):
+def convert_to_video_editor_json(
+    data: dict, class_name_mapper: dict, reporter: Reporter
+):
+    id_generator = ClassIdGenerator()
+
     def safe_time(timestamp):
         return "0" if str(timestamp) == "0.0" else timestamp
 
     def convert_timestamp(timestamp):
-        return timestamp / 10 ** 6
+        return timestamp / 10 ** 6 if timestamp else "0"
 
     editor_data = {
         "instances": [],
@@ -132,20 +148,26 @@ def convert_timestamp(timestamp):
         "metadata": {
             "duration": convert_timestamp(data["metadata"]["duration"]),
             "name": data["metadata"]["name"],
-            "width": data["metadata"]["width"],
-            "height": data["metadata"]["height"],
+            "width": data["metadata"].get("width"),
+            "height": data["metadata"].get("height"),
         },
     }
     for instance in data["instances"]:
         meta = instance["meta"]
-        class_name = meta["className"]
+        class_name = meta.get("className")
         editor_instance = {
             "attributes": [],
             "timeline": {},
             "type": meta["type"],
-            "classId": class_name_mapper.get(class_name, {}).get("id", -1),
-            "locked": True,
+            # TODO check
+            "locked": False,
         }
+        if class_name:
+            editor_instance["classId"] = class_name_mapper.get(class_name, {}).get(
+                "id", id_generator.send(class_name)
+            )
+        else:
+            editor_instance["classId"] = id_generator.send("unknown_class")
         if meta.get("pointLabels", None):
             editor_instance["pointLabels"] = meta["pointLabels"]
         active_attributes = set()
@@ -168,20 +190,41 @@ def convert_timestamp(timestamp):
                     editor_instance["timeline"][timestamp]["points"] = timestamp_data[
                         "points"
                     ]
-
-                if not class_name_mapper.get(meta["className"], None):
+                if not class_name:
+                    continue
+                elif not class_name_mapper.get(class_name):
                     reporter.store_message("missing_classes", meta["className"])
                     continue
 
                 existing_attributes_in_current_instance = set()
                 for attribute in timestamp_data["attributes"]:
-                    group_name, attr_name = attribute.get("groupName"), attribute.get("name")
-                    if not class_name_mapper[class_name].get("attribute_groups", {}).get(group_name):
-                        reporter.store_message("missing_attribute_groups", f"{class_name}.{group_name}")
-                    elif not class_name_mapper[class_name]["attribute_groups"][group_name].get("attributes", {}).get(attr_name):
-                        reporter.store_message("missing_attributes", f"{class_name}.{group_name}.{attr_name}")
+                    group_name, attr_name = (
+                        attribute.get("groupName"),
+                        attribute.get("name"),
+                    )
+                    if (
+                        not class_name_mapper[class_name]
+                        .get("attribute_groups", {})
+                        .get(group_name)
+                    ):
+                        reporter.store_message(
+                            "missing_attribute_groups", f"{class_name}.{group_name}"
+                        )
+                    elif (
+                        not class_name_mapper[class_name]["attribute_groups"][
+                            group_name
+                        ]
+                        .get("attributes", {})
+                        .get(attr_name)
+                    ):
+                        reporter.store_message(
+                            "missing_attributes",
+                            f"{class_name}.{group_name}.{attr_name}",
+                        )
                     else:
-                        existing_attributes_in_current_instance.add((group_name, attr_name))
+                        existing_attributes_in_current_instance.add(
+                            (group_name, attr_name)
+                        )
                 attributes_to_add = (
                     existing_attributes_in_current_instance - active_attributes
                 )
@@ -227,3 +270,15 @@ def default(self, obj):
         if isinstance(obj, set):
             return list(obj)
         return json.JSONEncoder.default(self, obj)
+
+
+class ClassIdGenerator:
+    def __init__(self):
+        self.classes = defaultdict(int)
+        self.idx = -1
+
+    def send(self, class_name: str):
+        if class_name not in self.classes:
+            self.classes[class_name] = self.idx
+            self.idx -= 1
+        return self.classes[class_name]

src/superannotate/lib/core/plugin.py

@@ -259,7 +259,7 @@ def get_extractable_frames(
         total_with_fps = sum(
             1
             for _ in VideoPlugin.frames_generator(
-                video_path,start_time, end_time, target_fps, log=False
+                video_path, start_time, end_time, target_fps, log=False
             )
         )
         zero_fill_count = len(str(total))
@@ -288,7 +288,7 @@ def extract_frames(
         for frame in VideoPlugin.frames_generator(
             video_path, start_time, end_time, target_fps
         ):
-            if len(extracted_frames_paths) > limit:
+            if len(extracted_frames_paths) >= limit:
                 break
             path = str(
                 Path(extract_path)

src/superannotate/lib/core/types.py

@@ -48,7 +48,7 @@ class Metadata(BaseModel):
 
 
 class PointLabels(BaseModel):
-    __root__: Dict[constr(regex=r'^[0-9]*$'), str]
+    __root__: Dict[constr(regex=r"^[0-9]*$"), str]
 
 
 class BaseInstance(BaseModel):
@@ -227,10 +227,10 @@ class VideoInstance(BaseModel):
 
 class VideoAnnotation(BaseModel):
     metadata: VideoMetaData
-    instances: List[VideoInstance]
-    tags: List[str]
+    instances: Optional[List[VideoInstance]]
+    tags: Optional[List[str]]
 
 
 class DocumentAnnotation(BaseModel):
     instances: list
-    tags: List[str]
+    tags: Optional[List[str]]