Feature/datasets

cspell/library-words.txt

@@ -0,0 +1,3 @@
+huggingface
+resp
+passthrough

cspell/project-words.txt

@@ -6,3 +6,4 @@ legomena
 MATTR
 TTR
 ngram
+prov

pyproject.toml

@@ -4,7 +4,11 @@ version = "0.1.0"
 description = "Fine-tuning for stylistic fidelity"
 readme = "README.md"
 requires-python = ">=3.12"
-dependencies = []
+dependencies = [
+    "datasets>=4.5.0",
+    "dotenv>=0.9.9",
+    "huggingface-hub>=1.4.1",
+]
 
 [build-system]
 requires = ["setuptools>=68", "wheel"]
@@ -20,6 +24,7 @@ dev = [
     "flake8-cognitive-complexity>=0.1.0",
     "isort>=7.0.0",
     "mypy>=1.19.1",
+    "notebook>=7.5.3",
     "pre-commit>=4.5.1",
     "pydoclint>=0.8.3",
     "pytest>=9.0.2",
@@ -31,6 +36,7 @@ dev = [
 line-length = 79
 target-version = "py312"
 force-exclude = true
+extend-ignore = ["D107"]
 
 [tool.ruff.lint]
 select = [
@@ -78,6 +84,10 @@ no_implicit_optional = true
 module = ["tests.*"]
 ignore_errors = true
 
+[[tool.mypy.overrides]]
+module = ["datasets", "datasets.*", "huggingface_hub", "huggingface_hub.*"]
+ignore_missing_imports = true
+
 [tool.isort]
 profile = "black"
 line_length = 79

src/voice/__init__.py

@@ -4,3 +4,8 @@
 This package provides tools for fine-tuning and evaluating
 LLMs for stylistic fidelity.
 """
+
+from voice.datasets import DatasetSpec, get_dataset
+from voice.stylometry import get_metrics
+
+__all__: list[str] = ["DatasetSpec", "get_metrics", "get_dataset"]

src/voice/datasets/__init__.py

@@ -0,0 +1,15 @@
+"""
+Datasets package for VOICE.
+
+This package contains user-facing abstractions for dataset management,
+in particular we expose:
+
+- `DatasetSpec` - Lightweight, declarative specification of a dataset.
+- `VoiceDataset` - Split-aware wrapper around a Hugging Face dataset.
+- `get_dataset` - Entrypoint for materialising a dataset from a specification.
+"""
+
+from voice.datasets.dataset import DatasetSpec, VoiceDataset
+from voice.datasets.get_dataset import get_dataset
+
+__all__: list[str] = ["DatasetSpec", "VoiceDataset", "get_dataset"]

src/voice/datasets/_schema.py

@@ -0,0 +1,67 @@
+"""
+Dataset schema primitives.
+
+This module defines lightweight schema-level abstractions shared across
+the dataset loading pipeline.
+
+This module is private by convention and not part of the public API.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+
+
+class Split(str, Enum):
+    """
+    Enumeration of supported dataset splits.
+
+    .. attribute :: TRAIN
+
+        Name of the training split
+
+    .. attribute :: VALIDATION
+
+        Name of the validation split
+
+    .. attribute :: TEST
+
+        Name of the test split
+    """
+
+    TRAIN = "train"
+    VALIDATION = "validation"
+    TEST = "test"
+
+    def __repr__(self) -> str:
+        """
+        Return the split as a string.
+
+        :return: Readable string of the split
+        """
+        return f"'{self.value}'"
+
+    @classmethod
+    def parse(cls, value: Split | str) -> Split:
+        """
+        Parse a split value from a string or Split instance.
+
+        :param value: Split name or Split enum value
+        :return: Parsed Split enum value
+        :raises TypeError: If the value is not a Split or string
+        :raises ValueError: If the value does not correspond to a valid split
+        """
+        if isinstance(value, cls):
+            return value
+        if not isinstance(value, str):
+            raise TypeError(
+                f"`split` must be a Split or str, got {type(value).__name__}"
+            )
+        v = value.strip().lower()
+        try:
+            return cls(v)
+        except ValueError as e:
+            allowed = ", ".join(s.value for s in cls)
+            raise ValueError(
+                f"Invalid split {value!r}. Allowed: {allowed}"
+            ) from e

src/voice/datasets/_specs.py

@@ -0,0 +1,16 @@
+"""
+Pre-defined dataset specifications for testing.
+
+All specifications are of type `voice.datasets.DatasetSpec`.
+
+This module is private by convention and not part of the public API.
+"""
+
+from voice.datasets import DatasetSpec
+from voice.datasets._schema import Split
+
+BUSH_LATEST = DatasetSpec(
+    repo_id="AccelerateScience/bush-dataset",
+    revision="main",
+    splits=(Split.TRAIN, Split.VALIDATION, Split.TEST),
+)