Merge pull request #201 from superannotateai/FRIDAY-274

Friday 274

Author: VaghinakDev

src/superannotate/lib/app/interface/sdk_interface.py

@@ -33,7 +33,6 @@
 from lib.app.helpers import get_annotation_paths
 from lib.app.helpers import reformat_metrics_json
 from lib.app.interface.types import AnnotationType
-from lib.app.interface.types import ClassesJson
 from lib.app.interface.types import NotEmptyStr
 from lib.app.interface.types import Status
 from lib.app.interface.types import validate_arguments
@@ -46,9 +45,11 @@
 from lib.core.enums import ImageQuality
 from lib.core.exceptions import AppException
 from lib.core.exceptions import AppValidationException
+from lib.core.types import ClassesJson
 from lib.infrastructure.controller import Controller
 from plotly.subplots import make_subplots
 from pydantic import EmailStr
+from pydantic import parse_obj_as
 from pydantic import StrictBool
 from tqdm import tqdm
 
@@ -1969,12 +1970,18 @@ def create_annotation_classes_from_classes_json(
             from_s3_object = from_s3.Object(from_s3_bucket, classes_json)
             from_s3_object.download_fileobj(file)
             file.seek(0)
-            annotation_classes = json.load(file)
+            annotation_classes = parse_obj_as(List[ClassesJson], json.load(file))
         else:
-            annotation_classes = json.load(open(classes_json))
+            annotation_classes = parse_obj_as(
+                List[ClassesJson], json.load(open(classes_json))
+            )
+
     else:
         annotation_classes = classes_json
 
+    annotation_classes = [
+        annotation_class.dict() for annotation_class in annotation_classes
+    ]
     response = controller.create_annotation_classes(
         project_name=project, annotation_classes=annotation_classes,
     )

src/superannotate/lib/app/interface/types.py

@@ -1,16 +1,12 @@
 from functools import wraps
-from typing import List
-from typing import Optional
 from typing import Union
 
 from lib.core.enums import AnnotationStatus
-from pydantic import BaseModel
 from pydantic import constr
 from pydantic import StrictStr
 from pydantic import validate_arguments as pydantic_validate_arguments
 from pydantic import ValidationError
 
-
 NotEmptyStr = constr(strict=True, min_length=1)
 
 
@@ -36,17 +32,6 @@ def validate(cls, value: Union[str]) -> Union[str]:
         return value
 
 
-class AttributeGroup(BaseModel):
-    name: StrictStr
-    is_multiselect: Optional[bool]
-
-
-class ClassesJson(BaseModel):
-    name: StrictStr
-    color: StrictStr
-    attribute_groups: List[AttributeGroup]
-
-
 def validate_arguments(func):
     @wraps(func)
     def wrapped(*args, **kwargs):

src/superannotate/lib/core/types.py

@@ -0,0 +1,121 @@
+from typing import List
+from typing import Optional
+from typing import Union
+
+from pydantic import BaseModel
+from pydantic import constr
+from pydantic import StrictStr
+
+
+NotEmptyStr = constr(strict=True, min_length=1)
+
+
+class Attribute(BaseModel):
+    name: NotEmptyStr
+
+
+class AttributeGroup(BaseModel):
+    name: StrictStr
+    attributes: List[Attribute]
+
+
+class ClassesJson(BaseModel):
+    name: StrictStr
+    color: StrictStr
+    attribute_groups: List[AttributeGroup]
+
+
+class Metadata(BaseModel):
+    name: Optional[NotEmptyStr]
+
+
+class BaseInstance(BaseModel):
+    type: NotEmptyStr
+    classId: int
+    groupId: Optional[int]
+    attributes: List[Attribute]
+
+
+class Point(BaseInstance):
+    x: float
+    y: float
+
+
+class PolyLine(BaseInstance):
+    points: List[float]
+
+
+class Polygon(BaseInstance):
+    points: List[float]
+
+
+class BboxPoints(BaseModel):
+    x1: float
+    x2: float
+    y1: float
+    y2: float
+
+
+class Bbox(BaseInstance):
+    points: BboxPoints
+
+
+class Ellipse(BaseInstance):
+    cx: float
+    cy: float
+    rx: float
+    ry: float
+
+
+class TemplatePoint(BaseModel):
+    id: int
+    x: float
+    y: float
+
+
+class TemplateConnection(BaseModel):
+    id: int
+    to: int
+
+
+class Template(BaseInstance):
+    points: List[TemplatePoint]
+    connections: List[Optional[TemplateConnection]]
+    templateId: int
+
+
+class EmptyPoint(BaseModel):
+    x: float
+    y: float
+
+
+class CuboidPoint(BaseModel):
+    f1: EmptyPoint
+    f2: EmptyPoint
+    r1: EmptyPoint
+    r2: EmptyPoint
+
+
+class Cuboid(BaseInstance):
+    points: CuboidPoint
+
+
+class VectorAnnotation(BaseModel):
+    metadata: Metadata
+    instances: List[Union[Template, Cuboid, Point, PolyLine, Polygon, Bbox, Ellipse]]
+
+
+class PixelAnnotationPart(BaseModel):
+    color: NotEmptyStr
+
+
+class PixelAnnotationInstance(BaseModel):
+    classId: int
+    groupId: int
+    parts: List[PixelAnnotationPart]
+    attributes: List[Attribute]
+
+
+class PixelAnnotation(BaseModel):
+    metadata: Metadata
+    instances: List[PixelAnnotationInstance]

src/superannotate/lib/core/usecases.py

@@ -51,7 +51,10 @@
 from lib.core.repositories import BaseReadOnlyRepository
 from lib.core.response import Response
 from lib.core.serviceproviders import SuerannotateServiceProvider
+from lib.core.types import PixelAnnotation
+from lib.core.types import VectorAnnotation
 from PIL import UnidentifiedImageError
+from pydantic import ValidationError
 
 logger = logging.getLogger("root")
 
@@ -3448,13 +3451,23 @@ def annotations_to_upload(self):
             self._annotations_to_upload = annotations_to_upload
         return self._annotations_to_upload
 
+    def _is_valid_json(self, json_data: dict):
+        try:
+            if self._project.project_type == constances.ProjectType.PIXEL.value:
+                PixelAnnotation(**json_data)
+            else:
+                VectorAnnotation(**json_data)
+            return True
+        except ValidationError as _:
+            return False
+
     def execute(self):
         uploaded_annotations = []
         missing_annotations = []
         failed_annotations = []
         for _ in range(0, len(self.annotations_to_upload), self.AUTH_DATA_CHUNK_SIZE):
             annotations_to_upload = self.annotations_to_upload[
-                _ : _ + self.CHUNK_SIZE  # noqa: E203
+                _ : _ + self.AUTH_DATA_CHUNK_SIZE  # noqa: E203
             ]
 
             if self._pre_annotation:
@@ -3504,22 +3517,21 @@ def execute(self):
                         for image_id, image_info in response.data.images.items()
                     ]
                     for future in concurrent.futures.as_completed(results):
-                        future.result()
+                        annotation, uploaded = future.result()
+                        if uploaded:
+                            uploaded_annotations.append(annotation)
+                        else:
+                            failed_annotations.append(annotation)
                         yield
 
-                uploaded_annotations.extend(
-                    [annotation.path for annotation in self.annotations_to_upload]
-                )
+                uploaded_annotations = [annotation.path for annotation in uploaded_annotations]
                 missing_annotations.extend(
                     [annotation.path for annotation in self._missing_annotations]
                 )
-                failed_annotations.extend(
-                    [
-                        annotation
-                        for annotation in self._annotation_paths
-                        if annotation not in uploaded_annotations + missing_annotations
-                    ]
-                )
+                failed_annotations = [
+                    annotation.path for annotation in failed_annotations
+                ]
+
             self._response.data = (
                 uploaded_annotations,
                 failed_annotations,
@@ -3542,6 +3554,9 @@ def upload_to_s3(
             annotation_json = json.load(open(image_id_name_map[image_id].path))
 
         self.fill_classes_data(annotation_json)
+
+        if not self._is_valid_json(annotation_json):
+            return image_id_name_map[image_id], False
         bucket.put_object(
             Key=image_info["annotation_json_path"], Body=json.dumps(annotation_json),
         )
@@ -3561,6 +3576,7 @@ def upload_to_s3(
                     file = io.BytesIO(mask_file.read())
 
             bucket.put_object(Key=image_info["annotation_bluemap_path"], Body=file)
+        return image_id_name_map[image_id], True
 
 
 class CreateModelUseCase(BaseUseCase):