Merge pull request #400 from superannotateai/friday

Friday

Author: VaghinakDev

src/superannotate/__init__.py

@@ -51,6 +51,7 @@
 from superannotate.lib.app.interface.sdk_interface import download_image_annotations
 from superannotate.lib.app.interface.sdk_interface import download_model
 from superannotate.lib.app.interface.sdk_interface import get_annotations
+from superannotate.lib.app.interface.sdk_interface import get_annotations_per_frame
 from superannotate.lib.app.interface.sdk_interface import get_exports
 from superannotate.lib.app.interface.sdk_interface import get_folder_metadata
 from superannotate.lib.app.interface.sdk_interface import get_image_annotations
@@ -130,6 +131,7 @@
     "get_exports",
     # annotations
     "get_annotations",
+    "get_annotations_per_frame",
     # converters
     "convert_json_version",
     "import_annotation",

src/superannotate/lib/app/interface/sdk_interface.py

@@ -1561,7 +1561,7 @@ def create_annotation_class(
         attribute_groups=attribute_groups,
         class_type=type,
     )
-    return response.data.dict()
+    return BaseSerializers(response.data).serialize()
 
 
 @Trackable

src/superannotate/lib/core/data_handlers.py

@@ -8,13 +8,14 @@
 from typing import Dict
 from typing import List
 
-import lib.core as constances
-from lib.core.enums import ClassTypeEnum
-from lib.core.reporter import Reporter
 from superannotate_schemas.schemas.classes import AnnotationClass
 from superannotate_schemas.schemas.classes import Attribute
 from superannotate_schemas.schemas.classes import AttributeGroup
 
+import lib.core as constances
+from lib.core.enums import ClassTypeEnum
+from lib.core.reporter import Reporter
+
 
 class BaseDataHandler(metaclass=ABCMeta):
     @abstractmethod
@@ -40,17 +41,15 @@ def __init__(self, annotation_classes: List[AnnotationClass]):
 
     @lru_cache()
     def get_annotation_class(
-        self, name: str, class_type: ClassTypeEnum
+            self, name: str
     ) -> AnnotationClass:
         for annotation_class in self._annotation_classes:
-            if annotation_class.name == name and class_type.equals(
-                annotation_class.type
-            ):
+            if annotation_class.name == name:
                 return annotation_class
 
     @lru_cache()
     def get_attribute_group(
-        self, annotation_class: AnnotationClass, attr_group_name: str
+            self, annotation_class: AnnotationClass, attr_group_name: str
     ) -> AttributeGroup:
         for attr_group in annotation_class.attribute_groups:
             if attr_group.name == attr_group_name:
@@ -108,9 +107,7 @@ class DocumentTagHandler(BaseAnnotationDateHandler):
     def handle(self, annotation: dict):
         new_tags = []
         for tag in annotation["tags"]:
-            annotation_class = self.get_annotation_class(
-                tag, class_type=ClassTypeEnum.OBJECT
-            )
+            annotation_class = self.get_annotation_class(tag)
             if annotation_class:
                 new_tags.append(annotation_class.id)
         annotation["tags"] = new_tags
@@ -119,10 +116,10 @@ def handle(self, annotation: dict):
 
 class MissingIDsHandler(BaseAnnotationDateHandler):
     def __init__(
-        self,
-        annotation_classes: List[AnnotationClass],
-        templates: List[dict],
-        reporter: Reporter,
+            self,
+            annotation_classes: List[AnnotationClass],
+            templates: List[dict],
+            reporter: Reporter,
     ):
         super().__init__(annotation_classes)
         self.validate_existing_classes(annotation_classes)
@@ -175,9 +172,7 @@ def handle(self, annotation: dict):
                 annotation_instance["classId"] = -1
             else:
                 class_name = annotation_instance["className"]
-                annotation_type = annotation_instance.get("type", ClassTypeEnum.OBJECT)
-                class_type = self._get_class_type(annotation_type)
-                annotation_class = self.get_annotation_class(class_name, class_type)
+                annotation_class = self.get_annotation_class(class_name)
                 if not annotation_class:
                     self.reporter.log_warning(f"Couldn't find class {class_name}")
                     self.reporter.store_message("missing_classes", class_name)
@@ -199,7 +194,7 @@ def handle(self, annotation: dict):
             template["name"]: template["id"] for template in self._templates
         }
         for annotation_instance in (
-            i for i in annotation["instances"] if i.get("type", None) == "template"
+                i for i in annotation["instances"] if i.get("type", None) == "template"
         ):
             annotation_instance["templateId"] = template_name_id_map.get(
                 annotation_instance.get("templateName", ""), -1
@@ -209,13 +204,8 @@ def handle(self, annotation: dict):
             i for i in annotation["instances"] if "className" in i and i["classId"] > 0
         ]:
             annotation_class_name = annotation_instance["className"]
-            annotation_class_type = self._get_class_type(
-                annotation_instance.get("type", ClassTypeEnum.OBJECT)
-            )
+            annotation_class = self.get_annotation_class(annotation_class_name)
 
-            annotation_class = self.get_annotation_class(
-                annotation_class_name, annotation_class_type
-            )
             if not annotation_class:
                 self.reporter.log_warning(
                     f"Couldn't find annotation class {annotation_class_name}"
@@ -291,9 +281,7 @@ def convert_timestamp(timestamp):
                 "locked": False,
             }
             if class_name:
-                annotation_class = self.get_annotation_class(
-                    class_name, ClassTypeEnum.OBJECT
-                )
+                annotation_class = self.get_annotation_class(class_name)
                 if annotation_class:
                     editor_instance["classId"] = annotation_class.id
                 else:
@@ -328,9 +316,7 @@ def convert_timestamp(timestamp):
                         ] = timestamp_data["points"]
                     if not class_name:
                         continue
-                    annotation_class = self.get_annotation_class(
-                        class_name, ClassTypeEnum.OBJECT
-                    )
+                    annotation_class = self.get_annotation_class(class_name)
                     if not annotation_class:
                         self.reporter.store_message(
                             "missing_classes", meta["className"]
@@ -365,10 +351,10 @@ def convert_timestamp(timestamp):
                                 (group_name, attr_name)
                             )
                     attributes_to_add = (
-                        existing_attributes_in_current_instance - active_attributes
+                            existing_attributes_in_current_instance - active_attributes
                     )
                     attributes_to_delete = (
-                        active_attributes - existing_attributes_in_current_instance
+                            active_attributes - existing_attributes_in_current_instance
                     )
                     if attributes_to_add or attributes_to_delete:
                         editor_instance["timeline"][timestamp][

src/superannotate/lib/core/usecases/annotations.py

@@ -554,6 +554,9 @@ def execute(self):
             return self._response
         if not response.data:
             self._response.errors = AppException(f"Video {self._video_name} not found.")
-
-        self._response.data = VideoFrameGenerator(response.data[1], fps=self._fps)
+        annotations = response.data
+        if annotations:
+            self._response.data = list(VideoFrameGenerator(response.data[0], fps=self._fps))
+        else:
+            self._response.data = []
         return self._response

src/superannotate/lib/core/usecases/images.py

@@ -2710,12 +2710,12 @@ def __init__(
         self,
         annotation_classes: BaseManageableRepository,
         annotation_class: AnnotationClassEntity,
-        project_name: str,
+        project: ProjectEntity,
     ):
         super().__init__()
         self._annotation_classes = annotation_classes
         self._annotation_class = annotation_class
-        self._project_name = project_name
+        self._project = project
 
     def validate_uniqueness(self):
         annotation_classes = self._annotation_classes.get_all(
@@ -2730,11 +2730,20 @@ def validate_uniqueness(self):
         ):
             raise AppValidationException("Annotation class already exits.")
 
+    def validate_project_type(self):
+        if (
+            self._project.project_type != ProjectType.VECTOR.value
+            and self._annotation_class.type == "tag"
+        ):
+            raise AppException(
+                f"Predefined tagging functionality is not supported for projects of type {ProjectType.get_name(self._project.project_type)}."
+            )
+
     def execute(self):
         if self.is_valid():
             logger.info(
                 "Creating annotation class in project %s with name %s",
-                self._project_name,
+                self._project.name,
                 self._annotation_class.name,
             )
             created = self._annotation_classes.insert(entity=self._annotation_class)
@@ -2813,7 +2822,7 @@ def execute(self):
             str(self._download_path),
         )
         classes = self._annotation_classes_repo.get_all()
-        classes = [entity.dict() for entity in classes]
+        classes = [entity.dict(by_alias=True) for entity in classes]
         json_path = f"{self._download_path}/classes.json"
         json.dump(classes, open(json_path, "w"), indent=4)
         self._response.data = json_path
@@ -2836,31 +2845,34 @@ def __init__(
         self._annotation_classes = annotation_classes
         self._project = project
 
-    def validate_annotation_classes(self):
-        if "attribute_groups" not in self._annotation_classes:
-            raise AppValidationException("Field attribute_groups is required.")
+    def validate_project_type(self):
+        if self._project.project_type != ProjectType.VECTOR.value and "tag" in [
+            i.type for i in self._annotation_classes
+        ]:
+            raise AppException(
+                f"Predefined tagging functionality is not supported for projects of type {ProjectType.get_name(self._project.project_type)}."
+            )
 
     def execute(self):
-        existing_annotation_classes = self._annotation_classes_repo.get_all()
-        existing_classes_name = [i.name for i in existing_annotation_classes]
-        unique_annotation_classes = []
-        for annotation_class in self._annotation_classes:
-            if annotation_class.name in existing_classes_name:
-                logger.warning(
-                    "Annotation class %s already in project. Skipping.",
-                    annotation_class.name,
+        if self.is_valid():
+            existing_annotation_classes = self._annotation_classes_repo.get_all()
+            existing_classes_name = [i.name for i in existing_annotation_classes]
+            unique_annotation_classes = []
+            for annotation_class in self._annotation_classes:
+                if annotation_class.name in existing_classes_name:
+                    logger.warning(
+                        "Annotation class %s already in project. Skipping.",
+                        annotation_class.name,
+                    )
+                    continue
+                else:
+                    unique_annotation_classes.append(annotation_class)
+            created = []
+            for i in range(len(unique_annotation_classes) - self.CHUNK_SIZE, 0, self.CHUNK_SIZE):
+                created += self._annotation_classes_repo.bulk_insert(
+                    entities=unique_annotation_classes[i : i + self.CHUNK_SIZE],  # noqa: E203
                 )
-                continue
-            else:
-                unique_annotation_classes.append(annotation_class)
-
-        created = []
-
-        for i in range(0, len(unique_annotation_classes), self.CHUNK_SIZE):
-            created += self._annotation_classes_repo.bulk_insert(
-                entities=unique_annotation_classes[i : i + self.CHUNK_SIZE],
-            )
-        self._response.data = created
+            self._response.data = created
         return self._response
 
 

src/superannotate/lib/core/video_convertor.py

@@ -141,5 +141,5 @@ def _process(self):
                         )
 
     def __iter__(self):
-        for frame_no in range(1, int(self._frames_count)):
+        for frame_no in range(1, int(self._frames_count) + 1):
             yield self.get_frame(frame_no).dict()

src/superannotate/lib/infrastructure/controller.py

@@ -55,8 +55,8 @@ def __init__(self, config_path: str = None, token: str = None):
         self._user_id = None
         self._team_name = None
         self._reporter = None
-        self._ssl_verify = not (os.environ.get("SA_TESTING", "False").lower() == "false")
-
+        self._testing = os.getenv("SA_TESTING", 'False').lower() in ('true', '1', 't')
+        self._ssl_verify = not self._testing
         self._backend_url = os.environ.get("SA_URL", constances.BACKEND_URL)
 
         if token:
@@ -114,6 +114,7 @@ def initialize_backend_client(self):
             auth_token=self._token,
             logger=self._logger,
             verify_ssl=self._ssl_verify,
+            testing=self._testing
         )
         self._backend_client.get_session.cache_clear()
         return self._backend_client
@@ -1142,7 +1143,7 @@ def create_annotation_class(
         use_case = usecases.CreateAnnotationClassUseCase(
             annotation_classes=annotation_classes,
             annotation_class=annotation_class,
-            project_name=project_name,
+            project=project,
         )
         use_case.execute()
         return use_case.execute()

src/superannotate/lib/infrastructure/services.py

@@ -43,16 +43,23 @@ class BaseBackendService(SuerannotateServiceProvider):
     """
 
     def __init__(
-        self, api_url: str, auth_token: str, logger, paginate_by=None, verify_ssl=False
+        self, api_url: str, auth_token: str, logger, paginate_by=None, verify_ssl=False, testing: bool = False
     ):
         self.api_url = api_url
         self._auth_token = auth_token
         self.logger = logger
         self._paginate_by = paginate_by
-        self._verify_ssl = False  # TODO fix False
+        self._verify_ssl = verify_ssl
         self.team_id = auth_token.split("=")[-1]
+        self._testing = testing
         self.get_session()
 
+    @property
+    def assets_provider_url(self):
+        if self._testing:
+            return "https://assets-provider.devsuperannotate.com/api/v1/"
+        return "https://assets-provider.superannotate.com/api/v1/"
+
     @timed_lru_cache(seconds=360)
     def get_session(self):
         session = requests.Session()
@@ -166,7 +173,6 @@ class SuperannotateBackendService(BaseBackendService):
     Manage projects, images and team in the Superannotate
     """
     DEFAULT_CHUNK_SIZE = 1000
-    STREAMED_DATA_PROVIDER_URL = "https://assets-provider.devsuperannotate.com"
 
     URL_USERS = "users"
     URL_LIST_PROJECTS = "projects"
@@ -215,7 +221,7 @@ class SuperannotateBackendService(BaseBackendService):
     URL_DELETE_ANNOTATIONS = "annotations/remove"
     URL_DELETE_ANNOTATIONS_PROGRESS = "annotations/getRemoveStatus"
     URL_GET_LIMITS = "project/{}/limitationDetails"
-    URL_GET_ANNOTATIONS = "api/v1/images/annotations/stream"
+    URL_GET_ANNOTATIONS = "images/annotations/stream"
 
     def get_project(self, uuid: int, team_id: int):
         get_project_url = urljoin(self.api_url, self.URL_GET_PROJECT.format(uuid))
@@ -1011,7 +1017,7 @@ def get_limitations(
         )
 
     def get_annotations(self, project_id: int, team_id: int, folder_id: int, items: List[str]) -> List[dict]:
-        get_limits_url = urljoin(self.STREAMED_DATA_PROVIDER_URL, self.URL_GET_ANNOTATIONS)
+        get_limits_url = urljoin(self.assets_provider_url, self.URL_GET_ANNOTATIONS)
         query_params = {
             "team_id": team_id,
             "project_id": project_id,

src/superannotate/lib/infrastructure/stream_data_handler.py

@@ -13,18 +13,22 @@ def map_image_names_to_fetch_streamed_data(data: List[str]):
 
 
 class StreamedAnnotations:
-    DELIMITER = b";)"
+    DELIMITER = b'\n:)\n'
 
     def __init__(self, headers: dict):
         self._headers = headers
         self._annotations = []
 
-    async def fetch(self, method: str, session: aiohttp.ClientSession, url: str, data: dict = None, params: dict = None):
+    async def fetch(self, method: str, session: aiohttp.ClientSession, url: str, data: dict = None,
+                    params: dict = None):
         response = await session._request(method, url, json=data, params=params)
         buffer = b""
         async for line in response.content:
             slices = line.split(self.DELIMITER)
-            if slices[0]:
+            if len(slices) == 1:
+                buffer += slices[0]
+                continue
+            elif slices[0]:
                 self._annotations.append(json.loads(buffer + slices[0]))
             for data in slices[1:-1]:
                 self._annotations.append(json.loads(data))
@@ -43,7 +47,6 @@ async def get_data(
             map_function: Callable = lambda x: x,
             verify_ssl: bool = False,
     ):
-
         async with aiohttp.ClientSession(raise_for_status=True, headers=self._headers,
                                          connector=aiohttp.TCPConnector(ssl=verify_ssl)) as session:
 

src/superannotate/version.py

@@ -1 +1,2 @@
 __version__ = "4.3.0b5"
+