diff --git a/complex_tokenization/__init__.py b/complex_tokenization/__init__.py index e69de29..588166e 100644 --- a/complex_tokenization/__init__.py +++ b/complex_tokenization/__init__.py @@ -0,0 +1,15 @@ +from complex_tokenization.tokenizer import ( + BNETokenizer, + BoundlessBPETokenizer, + BPETokenizer, + SuperBPETokenizer, + Tokenizer, +) + +__all__ = [ + "Tokenizer", + "BPETokenizer", + "BNETokenizer", + "BoundlessBPETokenizer", + "SuperBPETokenizer", +] diff --git a/tests/test_imports.py b/tests/test_imports.py new file mode 100644 index 0000000..fa9d2eb --- /dev/null +++ b/tests/test_imports.py @@ -0,0 +1,27 @@ +"""Test that key classes are importable from the top-level package.""" + + +class TestImports: + def test_import_tokenizer(self): + from complex_tokenization import Tokenizer + assert Tokenizer is not None + + def test_import_bpe(self): + from complex_tokenization import BPETokenizer + assert BPETokenizer is not None + + def test_import_bne(self): + from complex_tokenization import BNETokenizer + assert BNETokenizer is not None + + def test_import_boundless(self): + from complex_tokenization import BoundlessBPETokenizer + assert BoundlessBPETokenizer is not None + + def test_import_super(self): + from complex_tokenization import SuperBPETokenizer + assert SuperBPETokenizer is not None + + def test_all_exports(self): + import complex_tokenization + assert len(complex_tokenization.__all__) == 5