Merge branch 'friday' into df_aggregation

# Conflicts:
#	src/superannotate/lib/app/mixp/utils/parsers.py
#	src/superannotate/lib/core/entities/utils.py

Author: Vaghinak Basentsyan

src/superannotate/lib/app/interface/sdk_interface.py

@@ -3636,6 +3636,7 @@ def attach_document_urls_to_project(
     raise AppException(use_case.response.errors)
 
 
+@Trackable
 @validate_arguments
 def validate_annotations(
     project_type: ProjectTypes, annotations_json: Union[NotEmptyStr, Path]
@@ -3652,7 +3653,9 @@ def validate_annotations(
         """
     with open(annotations_json) as file:
         annotation_data = json.loads(file.read())
-        response = controller.validate_annotations(project_type, annotation_data, allow_extra=False)
+        response = controller.validate_annotations(
+            project_type, annotation_data, allow_extra=False
+        )
         if response.errors:
             raise AppException(response.errors)
         is_valid, _ = response.data

src/superannotate/lib/app/mixp/utils/parsers.py

@@ -1481,3 +1481,13 @@ def attach_document_urls_to_project(*args, **kwargs):
 
 def delete_annotations(*args, **kwargs):
     return {"event_name": "delete_annotations", "properties": {}}
+
+
+def validate_annotations(*args, **kwargs):
+    project_type = kwargs.get("project_type", None)
+    if not project_type:
+        project_type = args[0]
+    return {
+        "event_name": "validate_annotations",
+        "properties": {"Project Type": project_type},
+    }

src/superannotate/lib/core/entities/pixel.py

@@ -3,6 +3,7 @@
 
 from lib.core.entities.utils import BaseImageInstance
 from lib.core.entities.utils import BaseModel
+from lib.core.entities.utils import Comment
 from lib.core.entities.utils import Metadata
 from lib.core.entities.utils import Tag
 from pydantic import Field
@@ -32,3 +33,4 @@ class PixelAnnotation(BaseModel):
     metadata: PixelMetaData
     instances: List[PixelAnnotationInstance]
     tags: Optional[List[Tag]] = Field(list())
+    comments: Optional[List[Comment]] = Field(list())

src/superannotate/lib/core/entities/utils.py

@@ -9,6 +9,7 @@
 from pydantic import EmailStr
 from pydantic import Extra
 from pydantic import Field
+from pydantic import StrRegexError
 from pydantic import validator
 from pydantic.errors import EnumMemberError
 
@@ -26,9 +27,12 @@ def enum_error_handling(self) -> str:
 
 DATE_REGEX = r"\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d(?:\.\d{3})Z"
 
+DATE_TIME_FORMAT_ERROR_MESSAGE = (
+    "does not match expected format YYYY-MM-DDTHH:MM:SS.fffZ"
+)
 
-class BaseModel(PyDanticBaseModel):
 
+class BaseModel(PyDanticBaseModel):
     class Config:
         extra = Extra.allow
         use_enum_values = True
@@ -86,7 +90,7 @@ class Attribute(BaseModel):
 
 
 class Tag(BaseModel):
-    __root__: str
+    __root__: NotEmptyStr
 
 
 class AttributeGroup(BaseModel):
@@ -106,6 +110,15 @@ class TimedBaseModel(BaseModel):
     created_at: constr(regex=DATE_REGEX) = Field(None, alias="createdAt")
     updated_at: constr(regex=DATE_REGEX) = Field(None, alias="updatedAt")
 
+    @validator("created_at", "updated_at" , pre=True)
+    def validate_created_at(cls, value):
+        try:
+            if value is not None:
+                constr(regex=DATE_REGEX).validate(value)
+        except (TypeError, StrRegexError):
+            raise TypeError(DATE_TIME_FORMAT_ERROR_MESSAGE)
+        return value
+
 
 class UserAction(BaseModel):
     email: EmailStr
@@ -126,15 +139,16 @@ def clean_creation_type(cls, _):
 
 class LastUserAction(BaseModel):
     email: EmailStr
-    timestamp: float
+    timestamp: int
 
 
 class BaseInstance(TrackableModel, TimedBaseModel):
-    class_id: Optional[str] = Field(None, alias="classId")
-    class_name: Optional[str] = Field(None, alias="className")
+    class_id: Optional[int] = Field(None, alias="classId")
+    class_name: Optional[NotEmptyStr] = Field(None, alias="className")
 
 
 class MetadataBase(BaseModel):
+    url: Optional[str]
     name: NotEmptyStr
     last_action: Optional[LastUserAction] = Field(None, alias="lastAction")
     width: Optional[int]
@@ -146,7 +160,7 @@ class MetadataBase(BaseModel):
 
 
 class PointLabels(BaseModel):
-    __root__: Dict[constr(regex=r"^[0-9]*$"), str]  # noqa: F722 E261
+    __root__: Dict[constr(regex=r"^[0-9]*$"), NotEmptyStr]  # noqa: F722 E261
 
 
 class Correspondence(BaseModel):
@@ -163,7 +177,7 @@ class Comment(TimedBaseModel, TrackableModel):
 
 class BaseImageInstance(BaseInstance):
     class_id: Optional[int] = Field(None, alias="classId")
-    class_name: str = Field(alias="className")
+    class_name: NotEmptyStr = Field(alias="className")
     visible: Optional[bool]
     locked: Optional[bool]
     probability: Optional[int] = Field(100)

src/superannotate/lib/core/entities/vector.py

@@ -7,6 +7,7 @@
 from lib.core.entities.utils import BboxPoints
 from lib.core.entities.utils import Comment
 from lib.core.entities.utils import Metadata
+from lib.core.entities.utils import NotEmptyStr
 from lib.core.entities.utils import Tag
 from lib.core.entities.utils import VectorAnnotationTypeEnum
 from pydantic import conlist
@@ -73,7 +74,7 @@ class Template(BaseVectorInstance):
     points: conlist(TemplatePoint, min_items=1)
     connections: List[TemplateConnection]
     template_id: Optional[int] = Field(None, alias="templateId")
-    template_name: str = Field(alias="templateName")
+    template_name: NotEmptyStr = Field(alias="templateName")
 
 
 class CuboidPoint(BaseModel):
@@ -113,7 +114,7 @@ def return_action(cls, values):
         try:
             instance_type = values["type"]
         except KeyError:
-            raise ValueError("meta.type required")
+            raise ValueError("metadata.type required")
         try:
             return ANNOTATION_TYPES[instance_type](**values)
         except KeyError:

src/superannotate/lib/core/entities/video_export.py

@@ -1,5 +1,4 @@
 from enum import Enum
-from typing import Dict
 from typing import List
 from typing import Optional
 from typing import Union
@@ -9,6 +8,7 @@
 from lib.core.entities.utils import BaseModel
 from lib.core.entities.utils import BboxPoints
 from lib.core.entities.utils import MetadataBase
+from lib.core.entities.utils import NotEmptyStr
 from lib.core.entities.utils import PointLabels
 from lib.core.entities.utils import Tag
 from pydantic import conlist
@@ -21,7 +21,6 @@ class VideoType(str, Enum):
 
 
 class MetaData(MetadataBase):
-    url: Optional[str]
     duration: Optional[int]
     error: Optional[bool]
 
@@ -41,7 +40,7 @@ class EventTimeStamp(BaseTimeStamp):
 
 class InstanceMetadata(BaseInstance):
     type: VideoType
-    class_name: Optional[str] = Field(alias="className")
+    class_name: Optional[NotEmptyStr] = Field(alias="className")
     point_labels: Optional[PointLabels] = Field(None, alias="pointLabels")
     start: int
     end: int
@@ -58,14 +57,6 @@ class EventInstanceMetadata(InstanceMetadata):
     type: VideoType = Field(VideoType.EVENT.value, const=True)
 
 
-class BaseVideoInstance(BaseModel):
-    metadata: InstanceMetadata
-    id: Optional[str]
-    type: VideoType
-    locked: Optional[bool]
-    timeline: Dict[float, BaseTimeStamp]
-
-
 class BaseParameter(BaseModel):
     start: int
     end: int

src/superannotate/lib/core/usecases/images.py

@@ -3154,11 +3154,11 @@ def execute(self):
 
 class ValidateAnnotationUseCase(BaseUseCase):
     def __init__(
-            self,
-            project_type: str,
-            annotation: dict,
-            validators: BaseAnnotationValidator,
-            allow_extra: bool = True
+        self,
+        project_type: str,
+        annotation: dict,
+        validators: BaseAnnotationValidator,
+        allow_extra: bool = True,
     ):
         super().__init__()
         self._project_type = project_type

src/superannotate/lib/infrastructure/controller.py

@@ -1642,10 +1642,13 @@ def delete_annotations(
         )
         return use_case.execute()
 
-    def validate_annotations(self, project_type: str, annotation: dict, allow_extra: bool = False):
+    def validate_annotations(
+        self, project_type: str, annotation: dict, allow_extra: bool = False
+    ):
         use_case = usecases.ValidateAnnotationUseCase(
-            project_type, annotation,
+            project_type,
+            annotation,
             validators=self.annotation_validators,
-            allow_extra=allow_extra
+            allow_extra=allow_extra,
         )
         return use_case.execute()

src/superannotate/lib/infrastructure/validators.py

@@ -22,8 +22,9 @@ def wrap_error(e: ValidationError) -> str:
     error_messages = defaultdict(list)
     for error in e.errors():
         errors_list = list(error["loc"])
-        errors_list[1::2] = [f"[{i}]" for i in errors_list[1::2]]
-        errors_list[2::2] = [f".{i}" for i in errors_list[2::2]]
+        errors_list[1::] = [
+            f"[{i}]" if isinstance(i, int) else f".{i}" for i in errors_list[1::]
+        ]
         error_messages["".join(errors_list)].append(error["msg"])
     texts = ["\n"]
     for field, text in error_messages.items():

tests/data_set/document_annotation/text_file_example_1.json

@@ -1 +1,38 @@
-{"metadata":{"name":"text_file_example_1","status":"Completed","url":"https://sa-public-files.s3.us-west-2.amazonaws.com/Text+project/text_file_example_1.txt","projectId":160158,"annotatorEmail":null,"qaEmail":null,"lastAction":{"email":"shab.prog@gmail.com","timestamp":1634899229953}},"instances":[{"start":253,"end":593,"classId":873208,"createdAt":"2021-10-22T10:40:26.151Z","createdBy":{"email":"shab.prog@gmail.com","role":"Admin"},"updatedAt":"2021-10-22T10:40:29.953Z","updatedBy":{"email":"shab.prog@gmail.com","role":"Admin"},"attributes":[],"creationType":"Manual","className":"vid"}],"tags":["vid"],"freeText":""}
+{
+  "metadata": {
+    "name": "text_file_example_1",
+    "status": "Completed",
+    "url": "https://sa-public-files.s3.us-west-2.amazonaws.com/Text+project/text_file_example_1.txt",
+    "projectId": 160158,
+    "annotatorEmail": null,
+    "qaEmail": null,
+    "lastAction": {
+      "email": "shab.prog@gmail.com",
+      "timestamp": 1634899229953
+    }
+  },
+  "instances": [
+    {
+      "start": 253,
+      "end": 593,
+      "classId": 873208,
+      "createdAt": "2021-10-22T10:40:26.151Z",
+      "createdBy": {
+        "email": "shab.prog@gmail.com",
+        "role": "Admin"
+      },
+      "updatedAt": "2021-10-22T10:40:29.953Z",
+      "updatedBy": {
+        "email": "shab.prog@gmail.com",
+        "role": "Admin"
+      },
+      "attributes": [],
+      "creationType": "Manual",
+      "className": "vid"
+    }
+  ],
+  "tags": [
+    "vid"
+  ],
+  "freeText": ""
+}