Merge pull request #398 from superannotateai/friday-fix-annotations

Adjust convertor and tessts

Author: VaghinakDev

src/superannotate/__init__.py

@@ -51,6 +51,7 @@
 from superannotate.lib.app.interface.sdk_interface import download_image_annotations
 from superannotate.lib.app.interface.sdk_interface import download_model
 from superannotate.lib.app.interface.sdk_interface import get_annotations
+from superannotate.lib.app.interface.sdk_interface import get_annotations_per_frame
 from superannotate.lib.app.interface.sdk_interface import get_exports
 from superannotate.lib.app.interface.sdk_interface import get_folder_metadata
 from superannotate.lib.app.interface.sdk_interface import get_image_annotations
@@ -130,6 +131,7 @@
     "get_exports",
     # annotations
     "get_annotations",
+    "get_annotations_per_frame",
     # converters
     "convert_json_version",
     "import_annotation",

src/superannotate/lib/app/interface/sdk_interface.py

@@ -1561,7 +1561,6 @@ def create_annotation_class(
         attribute_groups=attribute_groups,
         class_type=type,
     )
-    return response.data.dict()
     return BaseSerializers(response.data).serialize()
 
 

src/superannotate/lib/core/usecases/annotations.py

@@ -554,6 +554,9 @@ def execute(self):
             return self._response
         if not response.data:
             self._response.errors = AppException(f"Video {self._video_name} not found.")
-
-        self._response.data = VideoFrameGenerator(response.data[1], fps=self._fps)
+        annotations = response.data
+        if annotations:
+            self._response.data = list(VideoFrameGenerator(response.data[0], fps=self._fps))
+        else:
+            self._response.data = []
         return self._response

src/superannotate/lib/core/usecases/images.py

@@ -2822,7 +2822,7 @@ def execute(self):
             str(self._download_path),
         )
         classes = self._annotation_classes_repo.get_all()
-        classes = [entity.dict() for entity in classes]
+        classes = [entity.dict(by_alias=True) for entity in classes]
         json_path = f"{self._download_path}/classes.json"
         json.dump(classes, open(json_path, "w"), indent=4)
         self._response.data = json_path

src/superannotate/lib/core/video_convertor.py

@@ -141,5 +141,5 @@ def _process(self):
                         )
 
     def __iter__(self):
-        for frame_no in range(1, int(self._frames_count)):
+        for frame_no in range(1, int(self._frames_count) + 1):
             yield self.get_frame(frame_no).dict()

src/superannotate/lib/infrastructure/controller.py

@@ -55,8 +55,8 @@ def __init__(self, config_path: str = None, token: str = None):
         self._user_id = None
         self._team_name = None
         self._reporter = None
-        self._ssl_verify = not (os.environ.get("SA_TESTING", "False").lower() == "false")
-
+        self._testing = os.getenv("SA_TESTING", 'False').lower() in ('true', '1', 't')
+        self._ssl_verify = not self._testing
         self._backend_url = os.environ.get("SA_URL", constances.BACKEND_URL)
 
         if token:
@@ -114,6 +114,7 @@ def initialize_backend_client(self):
             auth_token=self._token,
             logger=self._logger,
             verify_ssl=self._ssl_verify,
+            testing=self._testing
         )
         self._backend_client.get_session.cache_clear()
         return self._backend_client

src/superannotate/lib/infrastructure/services.py

@@ -43,16 +43,23 @@ class BaseBackendService(SuerannotateServiceProvider):
     """
 
     def __init__(
-        self, api_url: str, auth_token: str, logger, paginate_by=None, verify_ssl=False
+        self, api_url: str, auth_token: str, logger, paginate_by=None, verify_ssl=False, testing: bool = False
     ):
         self.api_url = api_url
         self._auth_token = auth_token
         self.logger = logger
         self._paginate_by = paginate_by
-        self._verify_ssl = False  # TODO fix False
+        self._verify_ssl = verify_ssl
         self.team_id = auth_token.split("=")[-1]
+        self._testing = testing
         self.get_session()
 
+    @property
+    def assets_provider_url(self):
+        if self._testing:
+            return "https://assets-provider.devsuperannotate.com/api/v1/"
+        return "https://assets-provider.superannotate.com/api/v1/"
+
     @timed_lru_cache(seconds=360)
     def get_session(self):
         session = requests.Session()
@@ -166,7 +173,6 @@ class SuperannotateBackendService(BaseBackendService):
     Manage projects, images and team in the Superannotate
     """
     DEFAULT_CHUNK_SIZE = 1000
-    STREAMED_DATA_PROVIDER_URL = "https://assets-provider.devsuperannotate.com"
 
     URL_USERS = "users"
     URL_LIST_PROJECTS = "projects"
@@ -215,7 +221,7 @@ class SuperannotateBackendService(BaseBackendService):
     URL_DELETE_ANNOTATIONS = "annotations/remove"
     URL_DELETE_ANNOTATIONS_PROGRESS = "annotations/getRemoveStatus"
     URL_GET_LIMITS = "project/{}/limitationDetails"
-    URL_GET_ANNOTATIONS = "api/v1/images/annotations/stream"
+    URL_GET_ANNOTATIONS = "images/annotations/stream"
 
     def get_project(self, uuid: int, team_id: int):
         get_project_url = urljoin(self.api_url, self.URL_GET_PROJECT.format(uuid))
@@ -1011,7 +1017,7 @@ def get_limitations(
         )
 
     def get_annotations(self, project_id: int, team_id: int, folder_id: int, items: List[str]) -> List[dict]:
-        get_limits_url = urljoin(self.STREAMED_DATA_PROVIDER_URL, self.URL_GET_ANNOTATIONS)
+        get_limits_url = urljoin(self.assets_provider_url, self.URL_GET_ANNOTATIONS)
         query_params = {
             "team_id": team_id,
             "project_id": project_id,

src/superannotate/lib/infrastructure/stream_data_handler.py

@@ -13,18 +13,22 @@ def map_image_names_to_fetch_streamed_data(data: List[str]):
 
 
 class StreamedAnnotations:
-    DELIMITER = b";)"
+    DELIMITER = b';)'
 
     def __init__(self, headers: dict):
         self._headers = headers
         self._annotations = []
 
-    async def fetch(self, method: str, session: aiohttp.ClientSession, url: str, data: dict = None, params: dict = None):
+    async def fetch(self, method: str, session: aiohttp.ClientSession, url: str, data: dict = None,
+                    params: dict = None):
         response = await session._request(method, url, json=data, params=params)
         buffer = b""
         async for line in response.content:
             slices = line.split(self.DELIMITER)
-            if slices[0]:
+            if len(slices) == 1:
+                buffer += slices[0]
+                continue
+            elif slices[0]:
                 self._annotations.append(json.loads(buffer + slices[0]))
             for data in slices[1:-1]:
                 self._annotations.append(json.loads(data))
@@ -43,7 +47,6 @@ async def get_data(
             map_function: Callable = lambda x: x,
             verify_ssl: bool = False,
     ):
-
         async with aiohttp.ClientSession(raise_for_status=True, headers=self._headers,
                                          connector=aiohttp.TCPConnector(ssl=verify_ssl)) as session:
 

tests/integration/annotations/test_get_annotations_per_frame.py

@@ -1,21 +1,52 @@
+import json
 import os
 from os.path import dirname
+from pathlib import Path
+
+import src.superannotate as sa
 from tests.integration.base import BaseTestCase
 
 
 class TestGetAnnotations(BaseTestCase):
     PROJECT_NAME = "test attach video urls"
-    PATH_TO_URLS = "data_set/attach_urls.csv"
+    PATH_TO_URLS = "data_set/attach_video_for_annotation.csv"
     PATH_TO_URLS_WITHOUT_NAMES = "data_set/attach_urls_with_no_name.csv"
     PATH_TO_50K_URLS = "data_set/501_urls.csv"
     PROJECT_DESCRIPTION = "desc"
+    ANNOTATIONS_PATH = "data_set/video_annotation"
+    VIDEO_NAME = "video.mp4"
+    CLASSES_PATH = "data_set/video_annotation/classes/classes.json"
     PROJECT_TYPE = "Video"
 
     @property
     def csv_path(self):
-        return os.path.join(dirname(dirname(__file__)), self.PATH_TO_URLS)
+        return os.path.join(dirname(dirname(dirname(__file__))), self.PATH_TO_URLS)
+
+    @property
+    def classes_path(self):
+        return os.path.join(self.folder_path, self.CLASSES_PATH)
 
     @property
-    def csv_path_without_name_column(self):
-        return os.path.join(dirname(dirname(__file__)), self.PATH_TO_URLS_WITHOUT_NAMES)
+    def folder_path(self):
+        return Path(__file__).parent.parent.parent
+
+    @property
+    def annotations_path(self):
+        return os.path.join(self.folder_path, self.ANNOTATIONS_PATH)
+
+    def test_video_annotation_upload(self):
+        sa.create_annotation_classes_from_classes_json(self.PROJECT_NAME, self.classes_path)
 
+        _, _, _ = sa.attach_video_urls_to_project(
+            self.PROJECT_NAME,
+            self.csv_path,
+        )
+        sa.upload_annotations_from_folder_to_project(self.PROJECT_NAME, self.annotations_path)
+        annotations = sa.get_annotations_per_frame(self.PROJECT_NAME, self.VIDEO_NAME, 1)
+        self.assertEqual(
+            len(annotations),
+            int(
+                json.load(open(f"{self.annotations_path}/{self.VIDEO_NAME}.json"))["metadata"]["duration"] / (
+                            1000 * 1000)
+            )
+        )