Merge pull request #420 from superannotateai/firday-integrations

Added integrations

Author: VaghinakDev

docs/source/superannotate.sdk.rst

@@ -38,6 +38,7 @@ ________
 .. autofunction:: superannotate.attach_image_urls_to_project
 .. autofunction:: superannotate.upload_images_from_public_urls_to_project
 .. autofunction:: superannotate.attach_document_urls_to_project
+.. autofunction:: superannotate.attach_items_from_integrated_storage
 .. autofunction:: superannotate.upload_image_to_project
 .. autofunction:: superannotate.delete_annotations
 .. _ref_upload_images_from_folder_to_project:
@@ -108,10 +109,11 @@ __________________
 
 ----------
 
-Team contributors
+Team
 _________________
 
 .. autofunction:: superannotate.get_team_metadata
+.. autofunction:: superannotate.get_integrations
 .. autofunction:: superannotate.invite_contributors_to_team
 .. autofunction:: superannotate.search_team_contributors
 

src/superannotate/__init__.py

@@ -25,6 +25,7 @@
     attach_document_urls_to_project,
 )
 from superannotate.lib.app.interface.sdk_interface import attach_image_urls_to_project
+from superannotate.lib.app.interface.sdk_interface import attach_items_from_integrated_storage
 from superannotate.lib.app.interface.sdk_interface import attach_video_urls_to_project
 from superannotate.lib.app.interface.sdk_interface import benchmark
 from superannotate.lib.app.interface.sdk_interface import clone_project
@@ -56,6 +57,7 @@
 from superannotate.lib.app.interface.sdk_interface import get_folder_metadata
 from superannotate.lib.app.interface.sdk_interface import get_image_annotations
 from superannotate.lib.app.interface.sdk_interface import get_image_metadata
+from superannotate.lib.app.interface.sdk_interface import get_integrations
 from superannotate.lib.app.interface.sdk_interface import (
     get_project_and_folder_metadata,
 )
@@ -135,6 +137,9 @@
     # annotations
     "get_annotations",
     "get_annotations_per_frame",
+    # integrations
+    "get_integrations",
+    "attach_items_from_integrated_storage",
     # converters
     "convert_json_version",
     "import_annotation",

src/superannotate/lib/app/interface/sdk_interface.py

@@ -34,6 +34,7 @@
 from lib.app.serializers import SettingsSerializer
 from lib.app.serializers import TeamSerializer
 from lib.core import LIMITED_FUNCTIONS
+from lib.core.entities.integrations import IntegrationEntity
 from lib.core.entities.project_entities import AnnotationClassEntity
 from lib.core.enums import ImageQuality
 from lib.core.exceptions import AppException
@@ -2923,3 +2924,49 @@ def upload_priority_scores(project: NotEmptyStr, scores: List[PriorityScore]):
     if response.errors:
         raise AppException(response.errors)
     return response.data
+
+
+@Trackable
+@validate_arguments
+def get_integrations():
+    """Get all integrations per team
+
+    :return: metadata objects of all integrations of the team.
+    :rtype: list of dicts
+    """
+    response = Controller.get_default().get_integrations()
+    if response.errors:
+        raise AppException(response.errors)
+    integrations = response.data
+    return BaseSerializers.serialize_iterable(integrations, ("name", "type", "root"))
+
+
+@Trackable
+@validate_arguments
+def attach_items_from_integrated_storage(
+        project: NotEmptyStr,
+        integration: Union[NotEmptyStr, IntegrationEntity],
+        folder_path: Optional[NotEmptyStr] = None
+):
+    """Link images from integrated external storage to SuperAnnotate.
+
+    :param project: project name or folder path where items should be attached (e.g., “project1/folder1”).
+    :type project: str
+
+    :param project: project name or folder path where items should be attached (e.g., “project1/folder1”).
+    :type project: str
+
+    :param integration:  existing integration name or metadata dict to pull items from.
+     Mandatory keys in integration metadata’s dict is “name”.
+    :type integration: str or dict
+
+    :param folder_path: Points to an exact folder/directory within given storage.
+    If None, items are fetched from the root directory.
+    :type folder_path: str
+    """
+    project_name, folder_name = extract_project_folder(project)
+    if isinstance(integration, str):
+        integration = IntegrationEntity(name=integration)
+    response = Controller.get_default().attach_integrations(project_name, folder_name, integration, folder_path)
+    if response.errors:
+        raise AppException(response.errors)

src/superannotate/lib/app/mixp/utils/parsers.py

@@ -1,5 +1,6 @@
 import lib.core as constances
 from lib.app.helpers import extract_project_folder
+from lib.core.entities import IntegrationEntity
 from lib.core.enums import ProjectType
 from lib.infrastructure.controller import Controller
 
@@ -1238,3 +1239,29 @@ def upload_priority_scores(*args, **kwargs):
         "event_name": "upload_priority_scores",
         "properties": {"Score Count": len(scores)},
     }
+
+
+def get_integrations(*args, **kwargs):
+    return {
+        "event_name": "get_integrations",
+        "properties": {},
+    }
+
+
+def attach_items_from_integrated_storage(*args, **kwargs):
+    project = kwargs.get("project", args[0])
+    integration = kwargs.get("integration", args[1])
+    folder_path = kwargs.get("folder_path", args[2])
+
+    project_name, _ = extract_project_folder(project)
+    if isinstance(integration, str):
+        integration = IntegrationEntity(name=integration)
+    project = Controller.get_default().get_project_metadata(project_name)
+    return {
+        "event_name": "attach_items_from_integrated_storage",
+        "properties": {
+            "project_type": ProjectType.get_name(project.project_type),
+            "integration_name": integration.name,
+            "folder_path": bool(folder_path)
+        },
+    }

src/superannotate/lib/app/serializers.py

@@ -1,4 +1,8 @@
 from abc import ABC
+from typing import Any
+from typing import List
+from typing import Set
+from typing import Union
 
 import superannotate.lib.core as constance
 from pydantic import BaseModel
@@ -11,12 +15,36 @@ class BaseSerializers(ABC):
     def __init__(self, entity: BaseEntity):
         self._entity = entity
 
-    def serialize(self):
-        if isinstance(self._entity, dict):
-            return self._entity
-        if isinstance(self._entity, BaseModel):
-            return self._entity.dict(by_alias=True)
-        return self._entity.to_dict()
+    def serialize(self, fields: List[str] = None, by_alias: bool = True, flat: bool = False):
+        return self._serialize(self._entity, fields, by_alias, flat)
+
+    @staticmethod
+    def _serialize(entity: Any, fields: List[str] = None, by_alias: bool = False, flat: bool = False):
+        if isinstance(entity, dict):
+            return entity
+        if isinstance(entity, BaseModel):
+            if fields:
+                fields = set(fields)
+                if len(fields) == 1:
+                    if flat:
+                        return entity.dict(include=fields, by_alias=by_alias)[next(iter(fields))]
+                    return entity.dict(include=fields, by_alias=by_alias)
+                return entity.dict(include=fields, by_alias=by_alias)
+            return entity.dict(by_alias=by_alias)
+        return entity.to_dict()
+
+    @classmethod
+    def serialize_iterable(
+            cls,
+            data: List[Any],
+            fields: Union[List[str], Set[str]] = None,
+            by_alias: bool = False,
+            flat: bool = False
+    ) -> List[Any]:
+        serialized_data = []
+        for i in data:
+            serialized_data.append(cls._serialize(i, fields, by_alias, flat))
+        return serialized_data
 
 
 class UserSerializer(BaseSerializers):

src/superannotate/lib/core/entities/__init__.py

@@ -1,3 +1,4 @@
+from lib.core.entities.integrations import IntegrationEntity
 from lib.core.entities.project_entities import AnnotationClassEntity
 from lib.core.entities.project_entities import BaseEntity
 from lib.core.entities.project_entities import ConfigEntity
@@ -33,6 +34,7 @@
     "UserEntity",
     "TeamEntity",
     "MLModelEntity",
+    "IntegrationEntity",
     # annotations
     "DocumentAnnotation",
     "VideoAnnotation",

src/superannotate/lib/core/entities/base.py

@@ -0,0 +1,9 @@
+from datetime import datetime
+
+from pydantic import BaseModel
+from pydantic import Field
+
+
+class TimedBaseModel(BaseModel):
+    created_at: datetime = Field(None, alias="createdAt")
+    updated_at: datetime = Field(None, alias="updatedAt")

src/superannotate/lib/core/entities/integrations.py

@@ -0,0 +1,14 @@
+from lib.core.entities.base import TimedBaseModel
+from pydantic import Field
+
+
+class IntegrationEntity(TimedBaseModel):
+    id: int = None
+    user_id: str = None
+    name: str
+    type: str = "aws"
+    root: str = Field(None, alias="bucket_name")
+    source: int = None
+
+    class Config:
+        arbitrary_types_allowed = True

src/superannotate/lib/core/repositories.py

@@ -8,7 +8,7 @@
 from lib.core.conditions import Condition
 from lib.core.entities import BaseEntity
 from lib.core.entities import ProjectEntity
-from lib.core.serviceproviders import SuerannotateServiceProvider
+from lib.core.serviceproviders import SuperannotateServiceProvider
 
 
 class BaseReadOnlyRepository(ABC):
@@ -50,7 +50,7 @@ def _drop_nones(data: dict):
 
 
 class BaseProjectRelatedManageableRepository(BaseManageableRepository):
-    def __init__(self, service: SuerannotateServiceProvider, project: ProjectEntity):
+    def __init__(self, service: SuperannotateServiceProvider, project: ProjectEntity):
         self._service = service
         self._project = project