Merge pull request #664 from superannotateai/FRIDAY_2178

Removed jsonschema

Author: VaghinakDev

requirements.txt

@@ -9,8 +9,8 @@ pandas~=2.0
 ffmpeg-python~=0.2
 pillow>=9.5,~=10.0
 tqdm~=4.66.1
-requests~=2.31.0
-aiofiles==23.1.0
+requests==2.*
+aiofiles==23.*
 fire==0.4.0
 mixpanel==4.8.3
-jsonschema==3.2.0
+superannotate-schemas==1.0.46

src/superannotate/__init__.py

@@ -3,7 +3,7 @@
 import sys
 
 
-__version__ = "4.4.17"
+__version__ = "4.4.18b1"
 
 sys.path.append(os.path.split(os.path.realpath(__file__))[0])
 

src/superannotate/lib/app/interface/sdk_interface.py

@@ -79,7 +79,7 @@
     "Tiled",
     "Other",
     "PointCloud",
-    "CustomEditor",
+    "GenAI",
 ]
 
 ANNOTATION_STATUS = Literal[

src/superannotate/lib/core/entities/classes.py

@@ -73,6 +73,7 @@ class AttributeGroup(TimedBaseModel):
     group_type: Optional[GroupTypeEnum]
     class_id: Optional[StrictInt]
     name: Optional[StrictStr]
+    isRequired: bool = Field(default=False)
     attributes: Optional[List[Attribute]]
     default_value: Any
 

src/superannotate/lib/core/enums.py

@@ -93,7 +93,7 @@ class ProjectType(BaseTitledEnum):
     TILED = "Tiled", 5
     OTHER = "Other", 6
     POINT_CLOUD = "PointCloud", 7
-    CUSTOM_EDITOR = "CustomEditor", 8
+    GEN_AI = "GenAI", 8
     UNSUPPORTED_TYPE_1 = "UnsupportedType", 9
     UNSUPPORTED_TYPE_2 = "UnsupportedType", 10
 

src/superannotate/lib/core/usecases/annotations.py

@@ -25,10 +25,8 @@
 
 import aiofiles
 import boto3
-import jsonschema.validators
 import lib.core as constants
-from jsonschema import Draft7Validator
-from jsonschema import ValidationError
+import superannotate_schemas
 from lib.core.conditions import Condition
 from lib.core.conditions import CONDITION_EQ as EQ
 from lib.core.entities import BaseItemEntity
@@ -309,7 +307,7 @@ def __init__(
 
     def validate_project_type(self):
         if self._project.type == constants.ProjectType.PIXEL.value:
-            raise ValidationError("Unsupported project type.")
+            raise AppException("Unsupported project type.")
 
     def _validate_json(self, json_data: dict) -> list:
         if self._project.type >= constants.ProjectType.PIXEL.value:
@@ -1227,7 +1225,7 @@ def execute(self):
 
 class ValidateAnnotationUseCase(BaseReportableUseCase):
     DEFAULT_VERSION = "V1.00"
-    SCHEMAS: Dict[str, Draft7Validator] = {}
+    SCHEMAS: Dict[str, superannotate_schemas.Draft7Validator] = {}
     PATTERN_MAP = {
         "\\d{4}-[01]\\d-[0-3]\\dT[0-2]\\d:[0-5]\\d:[0-5]\\d(?:\\.\\d{3})Z": "does not match YYYY-MM-DDTHH:MM:SS.fffZ",
         "^(?=.{1,254}$)(?=.{1,64}@)[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+(?:\\.[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$": "invalid email",
@@ -1278,15 +1276,17 @@ def oneOf(validator, oneOf, instance, schema):  # noqa
                     const_key, instance
                 )
                 if not instance_type:
-                    yield ValidationError("type required")
+                    yield superannotate_schemas.ValidationError("type required")
                     return
                 if const_key and instance_type == _type:
                     errs = list(
                         validator.descend(instance, sub_schema, schema_path=index)
                     )
                     if not errs:
                         return
-                    yield ValidationError("invalid instance", context=errs)
+                    yield superannotate_schemas.ValidationError(
+                        "invalid instance", context=errs
+                    )
                     return
             else:
                 subschemas = enumerate(oneOf)
@@ -1299,24 +1299,25 @@ def oneOf(validator, oneOf, instance, schema):  # noqa
                         break
                     all_errors.extend(errs)
                 else:
-                    yield ValidationError(
+                    yield superannotate_schemas.ValidationError(
                         f"{instance!r} is not valid under any of the given schemas",
                         context=all_errors[:1],
                     )
-                # yield from jsonschema._validators.oneOf(  # noqa
-                #     validator, oneOf, instance, schema
-                # )
         if const_key:
-            yield ValidationError(f"invalid {'.'.join(const_key)}")
+            yield superannotate_schemas.ValidationError(
+                f"invalid {'.'.join(const_key)}"
+            )
 
     @staticmethod
     def _pattern(validator, patrn, instance, schema):
         if validator.is_type(instance, "string") and not re.search(patrn, instance):
             _patrn = ValidateAnnotationUseCase.PATTERN_MAP.get(patrn)
             if _patrn:
-                yield ValidationError(f"{instance}  {_patrn}")
+                yield superannotate_schemas.ValidationError(f"{instance}  {_patrn}")
             else:
-                yield ValidationError(f"{instance} does not match {patrn}")
+                yield superannotate_schemas.ValidationError(
+                    f"{instance} does not match {patrn}"
+                )
 
     @staticmethod
     def iter_errors(self, instance, _schema=None):
@@ -1325,7 +1326,7 @@ def iter_errors(self, instance, _schema=None):
         if _schema is True:
             return
         elif _schema is False:
-            yield jsonschema.exceptions.ValidationError(
+            yield superannotate_schemas.ValidationError(
                 f"False schema does not allow {instance!r}",
                 validator=None,
                 validator_value=None,
@@ -1334,7 +1335,7 @@ def iter_errors(self, instance, _schema=None):
             )
             return
 
-        scope = jsonschema.validators._id_of(_schema)  # noqa
+        scope = superannotate_schemas.validators._id_of(_schema)  # noqa
         _schema = copy.copy(_schema)
         if scope:
             self.resolver.push_scope(scope)
@@ -1344,7 +1345,7 @@ def iter_errors(self, instance, _schema=None):
                 ref = _schema.pop("$ref")
                 validators.append(("$ref", ref))
 
-            validators.extend(jsonschema.validators.iteritems(_schema))
+            validators.extend(superannotate_schemas.validators.iteritems(_schema))
 
             for k, v in validators:
                 validator = self.VALIDATORS.get(k)
@@ -1381,7 +1382,7 @@ def extract_path(path):
                     real_path.append(item)
         return real_path
 
-    def _get_validator(self, version: str) -> Draft7Validator:
+    def _get_validator(self, version: str) -> superannotate_schemas.Draft7Validator:
         key = f"{self._project_type}__{version}"
         validator = ValidateAnnotationUseCase.SCHEMAS.get(key)
         if not validator:
@@ -1393,7 +1394,7 @@ def _get_validator(self, version: str) -> Draft7Validator:
             if not schema_response.data:
                 ValidateAnnotationUseCase.SCHEMAS[key] = lambda x: x
                 return ValidateAnnotationUseCase.SCHEMAS[key]
-            validator = jsonschema.Draft7Validator(schema_response.data)
+            validator = superannotate_schemas.Draft7Validator(schema_response.data)
             from functools import partial
 
             iter_errors = partial(self.iter_errors, validator)

src/superannotate/lib/infrastructure/services/http_client.py

@@ -1,4 +1,5 @@
 import asyncio
+import io
 import json
 import logging
 import platform
@@ -13,7 +14,6 @@
 
 import aiohttp
 import requests
-from aiohttp.client_exceptions import ClientError
 from lib.core.exceptions import AppException
 from lib.core.service_types import ServiceResponse
 from lib.core.serviceproviders import BaseClient
@@ -229,19 +229,39 @@ class AIOHttpSession(aiohttp.ClientSession):
     RETRY_LIMIT = 3
     BACKOFF_FACTOR = 0.3
 
-    async def request(self, *args, **kwargs) -> aiohttp.ClientResponse:
-        attempts = self.RETRY_LIMIT
-        delay = 0
-        for _ in range(attempts):
-            delay += self.BACKOFF_FACTOR
-            attempts -= 1
-            try:
-                response = await super()._request(*args, **kwargs)
-            except ClientError:
-                if not attempts:
-                    raise
+    class AIOHttpSession(aiohttp.ClientSession):
+        RETRY_STATUS_CODES = [401, 403, 502, 503, 504]
+        RETRY_LIMIT = 3
+        BACKOFF_FACTOR = 0.3
+
+        @staticmethod
+        def _copy_form_data(data: aiohttp.FormData) -> aiohttp.FormData:
+            form_data = aiohttp.FormData(quote_fields=False)
+            for field in data._fields:  # noqa
+                if isinstance(field[2], io.IOBase):
+                    field[2].seek(0)
+                form_data.add_field(
+                    value=field[2],
+                    content_type=field[1].get("Content-Type", ""),
+                    **field[0],
+                )
+            return form_data
+
+        async def request(self, *args, **kwargs) -> aiohttp.ClientResponse:
+            attempts = self.RETRY_LIMIT
+            delay = 0
+            for _ in range(attempts):
+                delay += self.BACKOFF_FACTOR
+                try:
+                    response = await super()._request(*args, **kwargs)
+                    if attempts <= 1 or response.status not in self.RETRY_STATUS_CODES:
+                        return response
+                except (aiohttp.ClientError, RuntimeError) as e:
+                    if attempts <= 1:
+                        raise
+                    if isinstance(e, RuntimeError):
+                        data = kwargs["data"]
+                        if isinstance(data, aiohttp.FormData):
+                            kwargs["data"] = self._copy_form_data(data)
+                attempts -= 1
                 await asyncio.sleep(delay)
-                continue
-            if response.status not in self.RETRY_STATUS_CODES or not attempts:
-                return response
-            await asyncio.sleep(delay)

tests/integration/classes/test_create_annotation_class.py

@@ -44,6 +44,7 @@ def test_create_annotation_class_with_attr_and_default_value(self):
             attribute_groups=[
                 {
                     "name": "test",
+                    "isRequired:": False,
                     "attributes": [{"name": "Car"}, {"name": "Track"}, {"name": "Bus"}],
                     "default_value": "Bus",
                 }

tests/integration/export/test_export.py

@@ -106,4 +106,4 @@ def test_upload_s3(self):
             Bucket=self.TEST_S3_BUCKET, Prefix=self.TMP_DIR
         ).get("Contents", []):
             files.append(object_data["Key"])
-        self.assertEqual(33, len(files))
+        self.assertEqual(25, len(files))

tests/integration/projects/test_basic_project.py

@@ -11,6 +11,16 @@
 sa = SAClient()
 
 
+class TestGenAIProjectBasic(BaseTestCase):
+    PROJECT_NAME = "TestGenAICreate"
+    PROJECT_TYPE = "GenAI"
+    PROJECT_DESCRIPTION = "DESCRIPTION"
+
+    def test_search(self):
+        projects = sa.search_projects(self.PROJECT_NAME, return_metadata=True)
+        assert projects
+
+
 class TestProjectBasic(BaseTestCase):
     PROJECT_NAME = "TestWorkflowGet"
     PROJECT_TYPE = "Vector"