Updated convert_project_type

Author: VaghinakDev

requirements.txt

@@ -14,7 +14,7 @@ ffmpeg-python>=0.2.0
 fire==0.4.0
 mixpanel==4.8.3
 pydantic>=1.10.2
-setuptools~=57.4.0
+setuptools>=57.4.0
 aiohttp==3.8.1
 email-validator>=1.0.3
 nest-asyncio==1.5.4

src/superannotate/__init__.py

@@ -1,6 +1,6 @@
 import os
 import sys
-
+import typing
 
 __version__ = "4.4.8"
 
@@ -19,6 +19,7 @@
 from superannotate.lib.app.input_converters import import_annotation  # noqa
 from superannotate.lib.app.interface.sdk_interface import SAClient  # noqa
 from superannotate.lib.app.server import SAServer  # noqa
+from superannotate.lib.app.server.utils import setup_app  # noqa
 from superannotate.lib.core import PACKAGE_VERSION_INFO_MESSAGE  # noqa
 from superannotate.lib.core import PACKAGE_VERSION_MAJOR_UPGRADE  # noqa
 from superannotate.lib.core import PACKAGE_VERSION_UPGRADE  # noqa
@@ -27,10 +28,18 @@
 
 SESSIONS = {}
 
+
+def create_app(apps: typing.List[str]) -> SAServer:
+    setup_app(apps)
+    server = SAServer()
+    return server
+
+
 __all__ = [
     "__version__",
     "SAClient",
     "SAServer",
+    "create_app",
     # Utils
     "enums",
     "AppException",

src/superannotate/lib/app/input_converters/sa_conversion.py

@@ -5,6 +5,7 @@
 import numpy as np
 from lib.app.exceptions import AppException
 from lib.core import DEPRICATED_DOCUMENT_VIDEO_MESSAGE
+from shapely.geometry import Polygon
 from superannotate.logger import get_default_logger
 
 from ..common import blue_color_generator
@@ -26,59 +27,57 @@ def from_pixel_to_vector(json_paths, output_dir):
         mask_name = str(json_path).replace("___pixel.json", "___save.png")
         img = cv2.imread(mask_name)
         H, W, _ = img.shape
-
         sa_json = json.load(open(json_path))
         instances = sa_json["instances"]
+        new_instances = []
         idx = 0
         sa_instances = []
+
         for instance in instances:
             if "parts" not in instance.keys():
                 if "type" in instance.keys() and instance["type"] == "meta":
                     sa_instances.append(instance)
                 continue
-
             parts = instance["parts"]
+            if len(parts) > 1:
+                idx += 1
+                group_id = idx
+            else:
+                group_id = 0
+            from collections import defaultdict
 
-            polygons = []
             for part in parts:
                 color = list(hex_to_rgb(part["color"]))
                 mask = np.zeros((H, W), dtype=np.uint8)
                 mask[np.all((img == color[::-1]), axis=2)] = 255
-                contours, _ = cv2.findContours(
-                    mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE
-                )
-                part_polygons = []
-                for contour in contours:
-                    segment = contour.flatten().tolist()
-                    if len(segment) > 6:
-                        part_polygons.append(segment)
-                polygons.append(part_polygons)
-
-            for part_polygons in polygons:
-                if len(part_polygons) > 1:
-                    idx += 1
-                    group_id = idx
-                else:
-                    group_id = 0
 
-                for polygon in part_polygons:
+                #  child contour index hierarchy[0][[i][3]
+                contours, hierarchy = cv2.findContours(
+                    mask, cv2.RETR_CCOMP, cv2.CHAIN_APPROX_SIMPLE
+                )
+                parent_child_map = defaultdict(list)
+                for idx, _hierarchy in enumerate(hierarchy[0]):
+
+                    if len(contours[idx].flatten().tolist()) <= 6:
+                        continue
+                    if _hierarchy[3] < 0:
+                        parent_child_map[idx] = []
+                    else:
+                        parent_child_map[_hierarchy[3]].append(idx)
+
+                for outer, inners in parent_child_map.items():
+                    outer_points = contours[outer].flatten().tolist()
+                    exclude_points = [contours[i].flatten().tolist() for i in inners]
                     temp = instance.copy()
-                    del temp["parts"]
+                    del temp['parts']
                     temp["pointLabels"] = {}
                     temp["groupId"] = group_id
                     temp["type"] = "polygon"
-                    temp["points"] = polygon
-                    sa_instances.append(temp.copy())
-                    temp["type"] = "bbox"
-                    temp["points"] = {
-                        "x1": min(polygon[::2]),
-                        "x2": max(polygon[::2]),
-                        "y1": min(polygon[1::2]),
-                        "y2": max(polygon[1::2]),
-                    }
-                    sa_instances.append(temp.copy())
+                    temp["points"] = outer_points
+                    temp["exclude"] = exclude_points
+                    new_instances.append(temp)
 
-        sa_json["instances"] = sa_instances
+        sa_json['instances'] = new_instances
         write_to_json(output_dir / file_name, sa_json)
         img_names.append(file_name.replace("___objects.json", ""))
     return img_names

src/superannotate/lib/app/interface/cli_interface.py

@@ -264,7 +264,6 @@ def create_server(self, name: str, path: str = None):
         This will create a directory by the given name in your current or provided directory.
         """
         path = Path(os.path.expanduser(path if path else ".")) / name
-
         if path.exists():
             raise Exception(f"Directory already exists {str(path.absolute())}")
         path.mkdir(parents=True)

src/superannotate/lib/app/server/__init__.py

@@ -1,4 +1,5 @@
+from lib.app.server.core import Response
 from lib.app.server.core import SAServer
 
 
-__all__ = ["SAServer"]
+__all__ = ["SAServer", "Response"]

src/superannotate/lib/app/server/__wsgi.py

@@ -1,16 +1,5 @@
-from importlib import import_module
-
-from superannotate import SAServer
-
+from superannotate import create_app
 
 APPS = ["app"]
 
-
-def create_app():
-    server = SAServer()
-    for path in APPS:
-        import_module(path)
-    return server
-
-
-app = create_app()
+app = create_app(APPS)

src/superannotate/lib/app/server/core.py

@@ -1,12 +1,23 @@
 import json
+import pathlib
 import typing
+from datetime import datetime
 
+from jinja2 import Environment
+from jinja2 import FileSystemLoader
+from superannotate.logger import get_server_logger
 from werkzeug.exceptions import HTTPException
 from werkzeug.routing import Map
 from werkzeug.routing import Rule
 from werkzeug.serving import run_simple
 from werkzeug.wrappers import Request
-from werkzeug.wrappers import Response
+from werkzeug.wrappers import Response as BaseResponse
+
+logger = get_server_logger()
+
+
+class Response(BaseResponse):
+    ...
 
 
 class SingletonMeta(type):
@@ -23,8 +34,14 @@ class SAServer(metaclass=SingletonMeta):
     def __init__(self):
         self._url_map: Map = Map([])
         self._view_function_map: typing.Dict[str, typing.Callable] = {}
+        self.jinja_env = Environment(
+            loader=FileSystemLoader(str(pathlib.Path(__file__).parent / "templates")),
+            autoescape=True,
+        )
 
-    def route(self, rule: str, methods: typing.List[str] = None, **options: typing.Any):
+    def route(
+        self, rule: str, methods: typing.List[str] = None, **options: typing.Any
+    ) -> typing.Any:
         """Decorate a view function to register it with the given URL
         rule and options. Calls :meth:`add_url_rule`, which has more
         details about the implementation.
@@ -135,17 +152,47 @@ def index():
     def _dispatch_request(self, request):
         """Dispatches the request."""
         adapter = self._url_map.bind_to_environ(request.environ)
+        response = None
+        content = None
         try:
             endpoint, values = adapter.match()
             view_func = self._view_function_map.get(endpoint)
             if not view_func:
                 return Response(status=404)
-            response = view_func(request, **values)
-            if isinstance(response, dict):
-                return Response(json.dumps(response), content_type="application/json")
-            return Response(response)
+            content = view_func(request, **values)
+            if isinstance(content, Response):
+                response = content
+            elif isinstance(content, (list, dict)):
+                response = Response(
+                    json.dumps(content), content_type="application/json"
+                )
+            else:
+                response = Response(content)
+            return response
         except HTTPException as e:
             return e
+        finally:
+            if "monitor" not in request.full_path and "log" not in request.full_path:
+                data = {
+                    "date": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+                    "request": {
+                        "method": request.full_path,
+                        "path": request.url,
+                        "headers": dict(request.headers.items()),
+                        "data": request.data.decode("utf-8"),
+                    },
+                    "response": {
+                        "headers": dict(request.headers.items()) if response else None,
+                        # 'data': response.data.decode('utf-8') if response else None,
+                        "data": content.data.decode("utf-8")
+                        if isinstance(content, Response)
+                        else content,
+                        "status_code": response.status_code if response else None,
+                    },
+                }
+                # import ndjson
+                print(11111, request.full_path)
+                logger.info(json.dumps(data))
 
     def wsgi_app(self, environ, start_response):
         """WSGI application that processes requests and returns responses."""
@@ -197,3 +244,7 @@ def run(
             ssl_context=ssl_context,
             **kwargs
         )
+
+    def render_template(self, template_name, **context):
+        t = self.jinja_env.get_template(template_name)
+        return Response(t.render(context), mimetype="text/html")

src/superannotate/lib/app/server/default_app.py

@@ -0,0 +1,54 @@
+import json
+
+from lib.app.server import SAServer
+from lib.core import LOG_FILE_LOCATION
+
+app = SAServer()
+
+LOG_FILE = "/var/log/orchestra/consumer.log"
+
+
+@app.route("/monitor", methods=["GET"])
+def monitor_view(request):
+    return app.render_template("monitor.html", **{})
+
+
+@app.route("/logs", methods=["GET"])
+def logs(request):
+    data = []
+    limit = 20
+    items = []
+    cursor = None
+    get_cursor = lambda x: max(x - 2048, 0)
+
+    with open(
+        f"{LOG_FILE_LOCATION}/sa_server.log",
+    ) as log_file:
+        log_file.seek(0, 2)
+        file_size = log_file.tell()
+        cursor = get_cursor(file_size)
+        while True:
+            log_file.seek(cursor, 0)
+            lines = log_file.read().splitlines()[-limit:]
+            # if cursor == 0 and len(lines) >= limit:
+            #     continue
+            for line in lines:
+                try:
+                    items.append(json.loads(line))
+                except Exception:
+                    ...
+            if len(lines) >= limit or cursor == 0:
+                return items
+            cursor = get_cursor(cursor)
+            items = []
+
+
+#
+# @app.route("/_log_stream", methods=["GET"])
+# def log_stream(request):
+#     def generate():
+#         for line in Pygtail(LOG_FILE, every_n=1):
+#             yield "data:" + str(line) + "\n\n"
+#             time.sleep(0.5)
+#
+#     return Response(generate(), mimetype="text/event-stream")

src/superannotate/lib/app/server/templates/monitor.html

@@ -0,0 +1,42 @@
+<!DOCTYPE html>
+<html lang="en" xmlns="http://www.w3.org/1999/xhtml" dir="ltr" class="uk-height-1-1">
+<head>
+    <meta charset="utf-8"/>
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <title>flasktest</title>
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/bulma/0.7.1/css/bulma.min.css"/>
+    <script defer src="https://use.fontawesome.com/releases/v5.3.1/js/all.js"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
+</head>
+<body>
+<section class="section">
+    <div class="container">
+        <div class="columns is-centered is-mobile">
+            <div class="column is-muted notification is-four-fifths">
+                <h1 class="title">SuperServer</h1>
+                <h2 class="subtitle">Progress example</h2>
+                <div id="progress"></div>
+            </div>
+        </div>
+    </div>
+    <div class="container">
+        <div class="columns is-centered is-mobile">
+            <div class="column is-dark notification is-four-fifths">
+                <div class="is-size-7 has-text-warning" id="display">
+                    <ul id="display_list">
+                    </ul>
+                </div>
+            </div>
+        </div>
+    </div>
+</section>
+
+<script type="text/javascript">
+    $.get("http://127.0.0.1:5002/logs", function (data) {
+        $("body").append("<p>" + JSON.stringify(data) + "</p>");
+    });
+
+
+</script>
+</body>
+</html>

src/superannotate/lib/app/server/utils.py

@@ -0,0 +1,8 @@
+import typing
+from importlib import import_module
+
+
+def setup_app(apps: typing.List[str]):
+    apps.extend(["superannotate.lib.app.server.default_app"])
+    for path in apps:
+        import_module(path)