refactor: Drop Beautiful Soup and use html.parser

.vscode/launch.json

@@ -4,6 +4,12 @@
   // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
   "version": "0.2.0",
   "configurations": [
+    {
+      "name": "Unit Tests",
+      "type": "python",
+      "request": "launch",
+      "module": "tests.unit"
+    },
     {
       "name": "Simple",
       "type": "python",

Simple/__init__.py

@@ -18,6 +18,7 @@
 from .document import Document
 from .exceptions import ProcessException
 from .logs import create_logger
+from .html import html
 
 
 @dataclass
@@ -94,10 +95,12 @@ def main(args: List[str]) -> int:
         doc = Document(opts.input)
         with opts.output.open("wt") as f:
             logger.info(f"Writing output to '{opts.output}'")
-            f.write(doc.render(data).prettify())
+            f.write(html(doc.render(data)))
     except ProcessException as ex:
         ex.doc.adapter.critical(
-            str(ex.exn), exc_info=None if opts.log_level > logging.DEBUG else ex
+            str(ex.exn),
+            exc_info=None if opts.log_level > logging.DEBUG else ex,
+            **ex.extra,
         )
         return 1
     except Exception as ex:

Simple/document.py

@@ -5,23 +5,26 @@
  https://opensource.org/licenses/MIT
 """
 
+from Simple.html.exceptions import DocumentException
 from .exceptions import ProcessException
 from .logs import DocumentLogAdapter
 import itertools
 import os
 import logging
 import copy
 from pathlib import Path
-from typing import Any, Dict, Iterable, Iterator, List, Optional, TypeVar
+from typing import *
 
-from bs4 import BeautifulSoup, Tag  # type: ignore
+from .html.tags import Node, Tag, TextNode
+from .html.document import Document as HTMLDocument, read
 
 logger = logging.getLogger(__name__)
 
 
 class Document:
     path: Path
     cwd: Path
+    html: HTMLDocument
     adapter: DocumentLogAdapter
     parent: Optional["Document"]
     is_component: bool
@@ -39,19 +42,22 @@ def __init__(
 
         try:
             with self.path.open("rt") as f:
-                logger.debug(f"Parsing file '{path}'")
-                root = BeautifulSoup(f, "html.parser")
+                logger.info(f"Parsing file '{path}'")
+                self.html = read(f)
         except IOError as ex:
-            self.adapter.critical(f"Cannot parse document: {ex}")
             raise ProcessException(self, Exception(f"Cannot parse document: {ex}"))
-
-        self.html = next(tags(itertools.chain(root.children)))
-        if self.html is not None and self.html.name == "def":
-            self.adapter.debug("Input is defining a component")
+        except DocumentException as ex:
+            raise ProcessException(self, ex)
+
+        root = self.html.root
+        if root is None:
+            raise ProcessException(self, Exception("HTML is empty"))
+        if root.name == "def":
+            self.adapter.debug("Input is defining a component", extra=dict(node=root))
             self.is_component = True
-            self.name = str(self.html.attrs["name"])
+            self.name = root.attrs["name"].lower()
             try:
-                self.inputs = [s.strip() for s in self.html.attrs["props"].split(",")]
+                self.inputs = [s.strip() for s in root.attrs["props"].split(",")]
             except KeyError:
                 self.inputs = []
         else:
@@ -60,61 +66,73 @@ def __init__(
             self.name = "__root__"
 
         self.components = {}
-        for tag in self.html.find_all("include"):  # type: Tag
-            tag.extract()
+        for tag in self.html.find_all("include"):
+            # tag.remove()
             if "src" not in tag.attrs:
                 raise ProcessException(
-                    self, Exception("Component include does not have a source link")
+                    self,
+                    Exception("Component include does not have a source link"),
+                    extra=dict(node=tag),
                 )
             src = tag.attrs["src"]
             component = Document(self.cwd / src, parent=self)
             if not component.is_component:
                 raise ProcessException(
-                    component, Exception("Does not define a component")
+                    component,
+                    Exception("Does not define a component"),
+                    extra=dict(node=tag),
                 )
             self.components[component.name] = component
 
-    def render(self, context: Dict[str, str]) -> Tag:
-        # Getting a working copy of the structure
-        html = copy.deepcopy(self.html)
-
-        self._replace_props(html, context)
-        self._replace_components(html, context)
-
-        return html
-
-    def _replace_components(self, html: BeautifulSoup, context: Dict[str, str]):
-        for name, component in self.components.items():
-            tags = html.find_all(name.lower())
-            if len(tags) == 0:
-                self.adapter.warn(f"<{name} /> is unused")
-            else:
-                for tag in tags:
-                    props = dict(tag.attrs)
-                    for s in props.keys():
-                        if s not in component.inputs:
-                            self.adapter.warn(f"Unknown attribute '{s}' in <{name} />")
-                    child_context = {**context, **props}
-                    chtml = component.render(child_context)
-                    tag.replace_with(chtml)
-                    chtml.replace_with_children()
-
-    def _replace_props(self, html: BeautifulSoup, context: Dict[str, str]):
-        for c in html.find_all("content"):
-            if "prop" not in c.attrs:
-                self.adapter.warn(f"Content tag should have a 'prop' attribute")
-                if c.remove is not None:
-                    c.remove()
-            elif c.attrs["prop"] not in context:
-                self.adapter.warn(
-                    f"Variable '{c.attrs['prop']}' not defined in context"
+    def render(self, context: Dict[str, str]) -> HTMLDocument:
+        return HTMLDocument(
+            [
+                cnode
+                for node in (
+                    self.html.root.children if self.is_component else self.html.children
                 )
-                if c.remove is not None:
-                    c.remove()
-            else:
-                self.adapter.debug(f"Replacing reference to {c.attrs['prop']}")
-                c.replace_with(context[c.attrs["prop"]])
-
-
-def tags(it: Iterator[Any]) -> Iterator[Tag]:
-    return filter(lambda v: isinstance(v, Tag), it)
+                for cnode in self.inflate(node, context)
+            ]
+        )
+
+    def inflate(self, node: Node, context: Dict[str, str]) -> List[Node]:
+        if isinstance(node, Tag):
+            if node.name in self.components:
+                component = self.components[node.name]
+                for s in node.attrs.keys():
+                    if s not in component.inputs:
+                        self.adapter.warn(
+                            f"Unknown attribute {s!r} in <{node.name}/>",
+                            extra=dict(node=node),
+                        )
+                child_context = {**context, **node.attrs}
+                return component.render(child_context).children
+
+            elif node.name == "include":
+                return []
+            elif node.name == "content":
+                if "prop" not in node.attrs:
+                    self.adapter.warn(
+                        f"Content tag should have a prop attribute",
+                        node=node,
+                    )
+                elif node.attrs["prop"] not in context:
+                    self.adapter.warn(
+                        f"Variable {node.attrs['prop']!r} not defined in context",
+                        extra=dict(
+                            node=node,
+                        ),
+                    )
+                self.adapter.debug(
+                    f"Replacing reference to {node.attrs['prop']!r}",
+                    extra=dict(
+                        node=node,
+                    ),
+                )
+                text = context[node.attrs["prop"]]
+                return [TextNode(range=node.range.start.range(text), text=text)]
+            children = [n for c in node.children for n in self.inflate(c, context)]
+            node = copy.deepcopy(node)
+            node.children = children
+            return [node]
+        return [node]

Simple/exceptions.py

@@ -4,13 +4,20 @@
  This software is released under the MIT License.
  https://opensource.org/licenses/MIT
 """
+from typing import Optional
 import Simple
 
 
 class ProcessException(Exception):
-    def __init__(self, doc: "Simple.document.Document", exn: Exception) -> None:
+    def __init__(
+        self,
+        doc: "Simple.document.Document",
+        exn: Exception,
+        extra: Optional[dict] = None,
+    ) -> None:
         self.doc = doc
         self.exn = exn
+        self.extra = extra or {}
 
     def __str__(self) -> str:
         return f"{self.doc.path}: {self.exn}"

Simple/html/__init__.py

@@ -0,0 +1,45 @@
+"""
+ Copyright (c) 2021 SolarLiner, jdrprod, Arxaqapi
+
+ This software is released under the MIT License.
+ https://opensource.org/licenses/MIT
+"""
+
+from dataclasses import dataclass
+import os
+from typing import *
+
+
+class HTML:
+    def __html__(self) -> str:
+        raise NotImplementedError()
+
+
+def html(node: HTML) -> str:
+    return node.__html__()
+
+
+@dataclass()
+class Position:
+    line: int
+    col: int
+
+    def range(self, text: str) -> "Range":
+        if "\n" in text:
+            lines = 1 + sum(1 for _ in filter(lambda c: c == "\n", text))
+            new_col = text[::-1].index("\n")
+            return Range(start=self, end=Position(self.line + lines, new_col))
+        else:
+            return Range(start=self, end=Position(self.line, self.col + len(text)))
+
+    def __str__(self) -> str:
+        return f"{self.line}:{self.col}"
+
+
+@dataclass()
+class Range:
+    start: Position
+    end: Position
+
+    def __str__(self) -> str:
+        return f"{self.start}-{self.end}"