feat: export tokenizer classes from package __init__

complex_tokenization/__init__.py

@@ -0,0 +1,15 @@
+from complex_tokenization.tokenizer import (
+    BNETokenizer,
+    BoundlessBPETokenizer,
+    BPETokenizer,
+    SuperBPETokenizer,
+    Tokenizer,
+)
+
+__all__ = [
+    "Tokenizer",
+    "BPETokenizer",
+    "BNETokenizer",
+    "BoundlessBPETokenizer",
+    "SuperBPETokenizer",
+]

tests/test_imports.py

@@ -0,0 +1,27 @@
+"""Test that key classes are importable from the top-level package."""
+
+
+class TestImports:
+    def test_import_tokenizer(self):
+        from complex_tokenization import Tokenizer
+        assert Tokenizer is not None
+
+    def test_import_bpe(self):
+        from complex_tokenization import BPETokenizer
+        assert BPETokenizer is not None
+
+    def test_import_bne(self):
+        from complex_tokenization import BNETokenizer
+        assert BNETokenizer is not None
+
+    def test_import_boundless(self):
+        from complex_tokenization import BoundlessBPETokenizer
+        assert BoundlessBPETokenizer is not None
+
+    def test_import_super(self):
+        from complex_tokenization import SuperBPETokenizer
+        assert SuperBPETokenizer is not None
+
+    def test_all_exports(self):
+        import complex_tokenization
+        assert len(complex_tokenization.__all__) == 5