diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0eab24b..74210e4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -74,16 +74,3 @@ repos: - id: pydoclint args: ["--config=pyproject.toml"] exclude: ^tests/ - - # Cognitive complexity limit via flake8-cognitive-complexity (keep for src; exclude tests) - - repo: https://github.com/pycqa/flake8 - rev: 7.3.0 - hooks: - - id: flake8 - additional_dependencies: - - flake8-cognitive-complexity - args: - - --max-line-length=79 - - --extend-ignore=E203 - - --max-cognitive-complexity=10 - exclude: ^tests/ diff --git a/README.md b/README.md index 9c1ae2b..f6a441d 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,14 @@ -# VOICE - Fine-tuning for Stylistic Fidelity +# 🗣️ VOICE + +... --- + +## Documentation + +``` +docs/ +├── 00_data.md — Included datasets +├── 01_evals.md — Evaluation suite: scoring and interpretation +└── 02_stylometry.md — Stylometric metrics: definition and catalogue +``` diff --git a/cspell/library-words.txt b/cspell/library-words.txt index 46d0eca..db6cebd 100644 --- a/cspell/library-words.txt +++ b/cspell/library-words.txt @@ -9,6 +9,9 @@ ylabel frameon allclose linalg +mathcal +mathbb +mathbf dotenv wandb jinja diff --git a/cspell/project-words.txt b/cspell/project-words.txt index 7fc1aa7..ecd0a7f 100644 --- a/cspell/project-words.txt +++ b/cspell/project-words.txt @@ -17,3 +17,6 @@ resumability resumable hyperparams redef +evals +submatrix +vocab diff --git a/docs/00_data.md b/docs/00_data.md new file mode 100644 index 0000000..e504061 --- /dev/null +++ b/docs/00_data.md @@ -0,0 +1,22 @@ +# Datasets + +--- + +Two example datasets are included, each containing press conference Q&A transcripts split into train, validation and test sets: + +| President | HuggingFace | +|---|---| +| Barack Obama | [`AccelerateScience/bo-press-conference-qa`](https://huggingface.co/datasets/AccelerateScience/bo-press-conference-qa) | +| George W. Bush | [`AccelerateScience/gwb-press-conference-qa`](https://huggingface.co/datasets/AccelerateScience/gwb-press-conference-qa) | + +Each example is a single JSONL record in chat format: + +```json +{ + "messages": [ + {"role": "system", "content": "You are president Barack Obama ..."}, + {"role": "user", "content": ""}, + {"role": "assistant", "content": ""} + ] +} +``` diff --git a/docs/01_stylometry.md b/docs/01_stylometry.md new file mode 100644 index 0000000..ee053e4 --- /dev/null +++ b/docs/01_stylometry.md @@ -0,0 +1,52 @@ +# Stylometric Metrics + +--- + +A **stylometric metric** is any function + +$$f : \mathcal{T} \rightarrow \mathbb{R}$$ + +that maps a text string to a real scalar capturing some surface property of writing style. VOICE treats each metric as a distribution over a corpus: given a set of texts, $f$ is applied to each one to produce a sample from the author's stylometric distribution for that feature. + +## Implemented Metrics + +### Word Length Distribution +Moments of the per-word character length distribution. + +| Metric | Description | +|---|---| +| `avg_word_length` | Mean word length | +| `std_word_length` | Standard deviation of word length | +| `skew_word_length` | Skewness of word length | +| `kurtosis_word_length` | Kurtosis of word length | + +### Vocabulary Richness +Type–token and word statistics measuring lexical diversity. + +| Metric | Description | +|---|---| +| `type_token_ratio` | Unique tokens / total tokens | +| `moving_avg_type_token_ratio` | TTR averaged over a sliding window (MATTR) | +| `hapax_legomena_ratio` | Fraction of words appearing exactly once | +| `dis_legomena_ratio` | Fraction of words appearing exactly twice | +| `tri_legomena_ratio` | Fraction of words appearing exactly three times | + +### Function Words +| Metric | Description | +|---|---| +| `function_word_ratio` | Proportion of tokens drawn from a closed function-word list | + +### Character N-gram Diversity +Type–token ratio and MATTR computed over character $n$-grams for $n \in \{3, 4, 5\}$. + +| Metric | Description | +|---|---| +| `char_{n}gram_type_token_ratio` | Character $n$-gram TTR | +| `char_{n}gram_moving_avg_type_token_ratio` | Character $n$-gram MATTR | + +### Text Length +| Metric | Description | +|---|---| +| `num_words` | Total word count | + +> **Note:** `num_words` may be deprecated in a future release. It does not tend to be used as a signature for authorship attribution in the broader stylometry literature. diff --git a/docs/02_evals.md b/docs/02_evals.md new file mode 100644 index 0000000..564c1a2 --- /dev/null +++ b/docs/02_evals.md @@ -0,0 +1,115 @@ +# Evaluation Suite + +--- + +## How Scoring Works + +For a set of model completions $\mathcal{C} = \{c_1, \dots, c_n\}$ and a set of reference answers $\mathcal{D} = \{d_1, \dots, d_n\}$ drawn from the same underlying question corpus, VOICE computes an **alignment score** in $[0, 1]$ measuring how stylometrically similar the completions are to the reference author. Both $\mathcal{C}$ and $\mathcal{D}$ are responses to the same set of prompts, so any observed distributional difference reflects style rather than content. + +### Step 1: Wasserstein Distance + +For each metric $f$, apply it to every text to get empirical distributions: + +$$X = \{f(c_i)\}_{i=1}^n \quad \text{and} \quad Y = \{f(d_i)\}_{i=1}^n$$ + +and compute their Wasserstein-1 distance $W(X, Y)$. A smaller distance means the two distributions are closer in shape. + +### Step 2: Calibration + +To interpret $W(X, Y)$ we need to know what distance is *expected even between two samples of the same size from the same author*. VOICE estimates this from the **training split** of $\mathcal{D}$ by bootstrap resampling: repeatedly drawing two non-overlapping subsets of size $n$ and computing their mutual Wasserstein distance. This yields a reference distribution $\mathcal{W}_0$ of self-distances that characterises natural within-author variation under the same sample size as the completions being evaluated. + +### Step 3: Tail Value + +The observed distance is located within $\mathcal{W}_0$ via its empirical CDF: + +$$p = \hat{F}_{\mathcal{W}_0}(W(X, Y)), \qquad t = 1 - p$$ + +$t$ is the **tail value**: the fraction of self-distances that *exceed* the observed distance. A high $t$ means the model's completions are no further from the reference than typical same-author samples, i.e. they are stylistically consistent with the reference author. + +### Step 4: Group and Overall Score + +Metrics are organised into groups (see [Stylometric Metrics](01_stylometry.md)). Each group captures a distinct linguistic object (word length, vocabulary richness, .etc) and the metrics within it represent different ways of characterising that same object. Because they study the same underlying property, within-group metrics are highly correlated; averaging within groups before aggregating across them prevents any single linguistic dimension from dominating the score simply by having more metrics defined for it. + +Within each group the tail values are averaged: + +$$\bar{t}_g = \frac{1}{|g|} \sum_{f \in g} t_f$$ + +The overall **alignment score** $\mathcal{S}$ is then the mean over groups, weighting each group equally: + +$$\mathcal{S} = \frac{1}{G} \sum_{g=1}^{G} \bar{t}_g \in [0, 1]$$ + +A score of **1** would mean the completions are stylistically indistinguishable from the reference even relative to same-author variation; in practice this is unrealistic but the directionality is still informative. + +--- + +## This Is Not a Hypothesis Test + +The tail value $t$ resembles a p-value but is not one and should not be interpreted as such. + +A classical hypothesis test asks whether there is sufficient evidence to reject a null hypothesis. That framing entails a binary decision and is sensitive to sample size: with enough data, even negligible stylistic differences will cross any fixed significance threshold. + +VOICE is not concerned with whether two distributions are statistically indistinguishable. It asks how similar they are, on a continuous scale, relative to the natural within-author variation present in the training corpus. The calibration distribution $\mathcal{W}_0$ serves as a normalisation baseline, not a null to be rejected. There is no significance threshold and no p-value, only a score. + +--- + +## Uncertainty Quantification + +The uncertainty in $\mathcal{S}$ arises from the finite number of completions $n$. It is estimated using jackknife resampling over the completion set, operated directly at the level of tail values so that no analytical propagation through the calibration CDF is required. Operating at the tail value level naturally produces asymmetric confidence intervals that respect the $[0, 1]$ boundary of the score. + +### Jackknife Replicates + +For each $i = 1, \dots, n$, remove $c_i$ and its paired reference $d_i$ and run the full pipeline (recomputing the Wasserstein distance and locating it within $\mathcal{W}_0$) for every metric $f$. This yields a matrix of leave-one-out tail values $\{t_{f,-i}\}$, or equivalently $n$ replicate vectors $\mathbf{t}_{-i} \in \mathbb{R}^M$ where $M$ is the total number of metrics across all groups. + +### Confidence Intervals + +Per-metric, group, and overall confidence intervals are all read directly from the empirical distribution of the corresponding jackknife replicates. + +For an individual metric $f$, the replicates $\{t_{f,-i}\}$ give a distribution of leave-one-out tail values from which percentile intervals are taken directly. + +For a group $g$, the replicate group scores are computed as: + +$$\bar{t}_{g,-i} = \frac{1}{|g|} \sum_{f \in g} t_{f,-i}$$ + +and percentile intervals are read from $\{\bar{t}_{g,-i}\}$. + +For the overall score, the replicate scores are: + +$$\mathcal{S}_{-i} = \frac{1}{G} \sum_{g=1}^G \bar{t}_{g,-i}$$ + +and percentile intervals are read from $\{\mathcal{S}_{-i}\}$. + +--- + +## Example + +Suppose a model achieves an alignment score of **0.2**. A natural reading is: + +> On average across stylometric groups, the model's completions sit at the 80th percentile of same-author self-distances, meaning the completions are stylistically closer to the reference than 20% of within-author sample pairs drawn from the training corpus. + +--- + +## Diagnostics + +The API exposes intermediate results for deeper inspection: + +```python +results = make_comparison(completions, true_ds) # uncertainty=True by default + +# Per-metric tail values +results.metric_tails # dict[str, float] + +# Per-group average tail values +results.group_tails # dict[MetricGroup, float] + +# Overall score +results.score # float + +# Jackknife percentile confidence intervals (default 90%) +results.metric_tail_cis() # dict[str, tuple[float, float]] | None +results.group_tail_cis() # dict[MetricGroup, tuple[float, float]] | None +results.score_ci() # tuple[float, float] | None +``` + +Per-metric tails are useful for identifying which stylometric dimensions are misaligned; per-group tails aggregate correlated metrics and correspond directly to the terms summed in the overall score. + +The confidence interval methods accept a `confidence` keyword (default 0.90, see `UNCERTAINTY_DEFAULTS`). Passing `uncertainty=False` to `make_comparison` skips the jackknife pass, in which case the interval methods return `None`. diff --git a/pyproject.toml b/pyproject.toml index 2ec89c9..95ecd18 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,8 +42,6 @@ where = ["src"] [dependency-groups] dev = [ "detect-secrets>=1.5.0", - "flake8>=7.3.0", - "flake8-cognitive-complexity>=0.1.0", "isort>=7.0.0", "mypy>=1.19.1", "notebook>=7.5.3", diff --git a/src/voice/_defaults.py b/src/voice/_defaults.py index 7118cc0..d7e9d38 100644 --- a/src/voice/_defaults.py +++ b/src/voice/_defaults.py @@ -31,17 +31,12 @@ class MetricGroup(str, Enum): .. attribute :: CHAR_NGRAM_DIVERSITY Character n-gram TTR and MATTR across n = 3, 4, 5. - - .. attribute :: TEXT_LENGTH - - Raw token count. """ WORD_LENGTH_DISTRIBUTION = "word_length_distribution" VOCABULARY_RICHNESS = "vocabulary_richness" FUNCTION_WORDS = "function_words" CHAR_NGRAM_DIVERSITY = "char_ngram_diversity" - TEXT_LENGTH = "text_length" @dataclass(frozen=True) @@ -111,6 +106,26 @@ class CalibrationDefaults: CALIBRATION_DEFAULTS: CalibrationDefaults = CalibrationDefaults() +@dataclass(frozen=True) +class UncertaintyDefaults: + """ + Default parameters for uncertainty quantification. + + These defaults control the confidence intervals reported alongside + comparison results. + + .. attribute :: confidence + + Confidence level for percentile intervals read from the + jackknife replicate distributions. + """ + + confidence: float = 0.90 # Safe default for validation/test size = 150 + + +UNCERTAINTY_DEFAULTS: UncertaintyDefaults = UncertaintyDefaults() + + @dataclass(frozen=True) class PlottingDefaults: """ diff --git a/src/voice/comparison/_utils.py b/src/voice/comparison/_utils.py index a684a69..e1756ea 100644 --- a/src/voice/comparison/_utils.py +++ b/src/voice/comparison/_utils.py @@ -35,6 +35,38 @@ def calibrated_percentile(value: float, dist: np.ndarray) -> float: return float(np.mean(dist <= value)) +# ----------------------------------------------------------------------------- +# Percentile interval +# ----------------------------------------------------------------------------- + + +def percentile_interval( + values: np.ndarray, confidence: float +) -> tuple[float, float]: + """ + Compute a central percentile interval from an empirical distribution. + + For a confidence level c, this returns the (alpha/2, 1 - alpha/2) + empirical percentiles of `values` with alpha = 1 - c. The interval is + asymmetric in general and bounded by the range of `values`. + + :param values: Array of samples from an empirical distribution + :param confidence: Confidence level, strictly between 0 and 1 + :return: (lower, upper) percentile interval + :raises ValueError: If `values` is empty or `confidence` is not in [0, 1] + """ + values = np.asarray(values, dtype=float).ravel() + if values.size == 0: + raise ValueError("values must be non-empty.") + if not 0.0 < confidence < 1.0: + raise ValueError("confidence must be strictly between 0 and 1.") + + alpha = 1.0 - confidence + lower = float(np.percentile(values, 100.0 * alpha / 2.0)) + upper = float(np.percentile(values, 100.0 * (1.0 - alpha / 2.0))) + return lower, upper + + # ----------------------------------------------------------------------------- # Bootstrap null distribution # ----------------------------------------------------------------------------- diff --git a/src/voice/comparison/comparison.py b/src/voice/comparison/comparison.py index 7173b68..de11e45 100644 --- a/src/voice/comparison/comparison.py +++ b/src/voice/comparison/comparison.py @@ -20,11 +20,13 @@ from voice._defaults import ( CALIBRATION_DEFAULTS, + UNCERTAINTY_DEFAULTS, MetricGroup, ) from voice.comparison._utils import ( bootstrap_null_distribution, calibrated_percentile, + percentile_interval, ) from voice.datasets import VoiceDataset from voice.datasets.dataset import Example @@ -185,6 +187,11 @@ class ComparisonResults: A score of 1 indicates perfect stylistic indistinguishability; 0 indicates maximum divergence. + When uncertainty quantification is enabled, the matrix of jackknife + leave-one-out tail-value replicates is attached and per-metric, + per-group and overall-score percentile confidence intervals are read + directly from its empirical distributions. + .. attribute :: metrics Tuple of stylometric metric names to run comparison for. @@ -192,6 +199,7 @@ class ComparisonResults: metrics: tuple[str, ...] _entries: dict[str, ComparisonEntry] = field(default_factory=dict) + _replicates: np.ndarray | None = field(default=None, repr=False) def __post_init__(self) -> None: """ @@ -222,9 +230,10 @@ def __repr__(self) -> str: ) + "}" ) - return ( - f"ComparisonResults(groups={group_part}, score={self.score:.4f})" - ) + score_part = f"{self.score:.4f}" + if (ci := self.score_ci()) is not None: + score_part += f" [{ci[0]:.4f}, {ci[1]:.4f}]" + return f"ComparisonResults(groups={group_part}, score={score_part})" @property def num_metrics(self) -> int: @@ -267,6 +276,95 @@ def score(self) -> float: return 0.0 return sum(e.avg_tail for e in entries.values()) / len(entries) + @property + def _group_indices(self) -> dict[MetricGroup, list[int]]: + """ + Indices into `metrics` (and replicate matrix columns) per group. + + :return: Mapping of MetricGroup to column indices + """ + registry = get_metrics() + indices: dict[MetricGroup, list[int]] = {} + for j, metric in enumerate(self.metrics): + indices.setdefault(registry[metric].group, []).append(j) + return indices + + @property + def has_uncertainty(self) -> bool: + """Whether jackknife uncertainty estimates are attached.""" + return self._replicates is not None + + def metric_tail_cis( + self, + confidence: float = UNCERTAINTY_DEFAULTS.confidence, + ) -> dict[str, tuple[float, float]] | None: + """ + Jackknife confidence interval for each per-metric tail value. + + Percentile intervals are read directly from the empirical + distribution of leave-one-out tail values for each metric. + + :param confidence: Confidence level, strictly between 0 and 1 + :return: Mapping of metric name to (lower, upper) interval or None if + uncertainty quantification was not run + """ + if self._replicates is None: + return None + return { + m: percentile_interval(self._replicates[:, j], confidence) + for j, m in enumerate(self.metrics) + } + + def group_tail_cis( + self, + confidence: float = UNCERTAINTY_DEFAULTS.confidence, + ) -> dict[MetricGroup, tuple[float, float]] | None: + """ + Jackknife confidence interval for each group's average tail value. + + Each leave-one-out replicate's tail values are averaged within the + group, and the percentile interval is read from the resulting + empirical distribution of replicate group scores. + + :param confidence: Confidence level, strictly between 0 and 1 + :return: Mapping of MetricGroup to (lower, upper) interval or None if + uncertainty quantification was not run + """ + if self._replicates is None: + return None + return { + group: percentile_interval( + self._replicates[:, idx].mean(axis=1), confidence + ) + for group, idx in self._group_indices.items() + } + + def score_ci( + self, + confidence: float = UNCERTAINTY_DEFAULTS.confidence, + ) -> tuple[float, float] | None: + """ + Jackknife confidence interval for the overall alignment score. + + Each leave-one-out replicate yields a replicate score (the mean of + its per-group average tail values, groups weighted equally); the + percentile interval is read from the empirical distribution of these + replicate scores. + + :param confidence: Confidence level, strictly between 0 and 1 + :return: (lower, upper) interval for the score, or None if + uncertainty quantification was not run + """ + if self._replicates is None: + return None + group_scores = np.stack( + [ + self._replicates[:, idx].mean(axis=1) + for idx in self._group_indices.values() + ] + ) + return percentile_interval(group_scores.mean(axis=0), confidence) + @property def metric_tails(self) -> dict[str, float]: """ @@ -356,6 +454,7 @@ def make_comparison( true_ds: VoiceDataset, *, metrics: Sequence[str] | None = None, + uncertainty: bool = True, ) -> ComparisonResults: """ Compare stylometric distributions of `completions` to the true corpus. @@ -371,14 +470,25 @@ def make_comparison( values, and an overall alignment score equal to the mean of group-level average tail values (groups weighted equally). + When `uncertainty` is True (the default), jackknife resampling over the + completion set is used: each completion and its paired reference are + left out in turn, the Wasserstein distance is recomputed, and the + leave-one-out distance is located within the same + self-Wasserstein distribution. Per-metric, per-group and overall score + percentile confidence intervals are then available on the returned + results, read directly from the replicate distributions. + :param completions: Sequence of examples to analyse :param true_ds: VoiceDataset providing the true corpus :param metrics: Optional subset of metric names; defaults to all metrics + :param uncertainty: Whether to estimate uncertainty :return: ComparisonResults populated for the selected metrics :raises ValueError: - If completions is empty, - If sample sizes are invalid - If metrics are not registered + - If `uncertainty` is True and there are fewer than two completions + or the observed split does not match the completions in length """ if not completions: raise ValueError("completions must be non-empty.") @@ -392,20 +502,52 @@ def make_comparison( observed_split = completions[0].split ds_observed_split = true_ds[observed_split] + n = len(completions) + if uncertainty: + if n < 2: + raise ValueError( + "uncertainty quantification requires at least 2 completions." + ) + if len(ds_observed_split) != n: + raise ValueError( + "uncertainty quantification requires completions to be " + "paired one-to-one with the observed split " + f"({n} completions vs {len(ds_observed_split)} references)." + ) + + distributions: dict[str, tuple[np.ndarray, np.ndarray, np.ndarray]] = {} + for metric in metric_list: x = stylometric_distribution(completions, metric) y = stylometric_distribution(ds_observed_split, metric) d = float(wasserstein_distance(x, y)) + # Cached per metric: the jackknife pass below reuses this + # distribution rather than re-running the bootstrap per replicate. self_dist = self_wasserstein_distribution( true_ds, metric=metric, - sample_size=len(completions), + sample_size=n, ) p = calibrated_percentile(d, self_dist) results.add(metric=metric, wasserstein=d, percentile=p) + if uncertainty: + distributions[metric] = (x, y, self_dist) + + if uncertainty: + replicates = np.empty((n, len(metric_list)), dtype=float) + for i in range(n): + mask = np.ones(n, dtype=bool) + mask[i] = False + for j, metric in enumerate(metric_list): + x, y, self_dist = distributions[metric] + d_i = float(wasserstein_distance(x[mask], y[mask])) + replicates[i, j] = 1.0 - calibrated_percentile(d_i, self_dist) + + results._replicates = replicates + return results diff --git a/src/voice/finetune/_orchestrator.py b/src/voice/finetune/_orchestrator.py index f06727c..40a74e9 100644 --- a/src/voice/finetune/_orchestrator.py +++ b/src/voice/finetune/_orchestrator.py @@ -177,8 +177,11 @@ def _score_split( Alignment results are written to ``alignment/epoch_{n}/{split}.json`` with keys ``score``, - ``metric_tails``, and ``group_tails``. Only the overall score is - returned; the full tails are available on disk only. + ``score_ci``, ``metric_tails``, ``metric_tail_cis``, ``group_tails``, + and ``group_tail_cis``. Confidence intervals are jackknife percentile + intervals at the default confidence level and are serialised as + ``[lower, upper]`` pairs. Only the overall score is returned; the + full tails and intervals are available on disk only. :param run_dir: Run directory. :param split: Split name (``"validation"`` or ``"test"``). @@ -191,16 +194,25 @@ def _score_split( completions = _load_completions(run_dir, split, epoch) results = make_comparison(completions, reference_ds) + group_cis = results.group_tail_cis() + out_path = run_dir.alignment_path(split, epoch) out_path.parent.mkdir(parents=True, exist_ok=True) out_path.write_text( json.dumps( { "score": results.score, + "score_ci": results.score_ci(), "metric_tails": results.metric_tails, + "metric_tail_cis": results.metric_tail_cis(), "group_tails": { g.value: v for g, v in results.group_tails.items() }, + "group_tail_cis": ( + None + if group_cis is None + else {g.value: ci for g, ci in group_cis.items()} + ), }, indent=2, ) diff --git a/src/voice/stylometry/__init__.py b/src/voice/stylometry/__init__.py index e4da059..1e52bc6 100644 --- a/src/voice/stylometry/__init__.py +++ b/src/voice/stylometry/__init__.py @@ -19,7 +19,6 @@ calculate_hapax_legomena_ratio, calculate_kurtosis_word_length, calculate_moving_avg_type_token_ratio, - calculate_num_words, calculate_skew_word_length, calculate_std_word_length, calculate_tri_legomena_ratio, @@ -38,7 +37,6 @@ "calculate_std_word_length", "calculate_skew_word_length", "calculate_kurtosis_word_length", - "calculate_num_words", "calculate_hapax_legomena_ratio", "calculate_dis_legomena_ratio", "calculate_tri_legomena_ratio", diff --git a/src/voice/stylometry/metrics.py b/src/voice/stylometry/metrics.py index 3f4666c..634c963 100644 --- a/src/voice/stylometry/metrics.py +++ b/src/voice/stylometry/metrics.py @@ -9,7 +9,6 @@ - Standard deviation of word length (`std_word_length`) - Skewness of word length (`skew_word_length`) - Standard kurtosis of word length (`kurtosis_word_length`) - - Total number of words (`num_words`) - Hapax legomena ratio (`hapax_legomena_ratio`) - Dis legomena ratio (`dis_legomena_ratio`) - Tri legomena ratio (`tri_legomena_ratio`) @@ -235,23 +234,6 @@ def calculate_kurtosis_word_length(text: str) -> float: # ----------------------------------------------------------------------------- -@metric( - "num_words", - group=MetricGroup.TEXT_LENGTH, - description="Number of words in text", -) -def calculate_num_words(text: str) -> float: - """ - Calculate the number of words in a string. - - Number of words in an empty string is 0.0. - - :param text: Provided text - :return: Number of words as a float - """ - return float(len(word_tokenize(text))) - - @metric( "hapax_legomena_ratio", group=MetricGroup.VOCABULARY_RICHNESS, diff --git a/tests/voice/comparison/test_comparison.py b/tests/voice/comparison/test_comparison.py index 82666d3..84f1973 100644 --- a/tests/voice/comparison/test_comparison.py +++ b/tests/voice/comparison/test_comparison.py @@ -20,7 +20,10 @@ from datasets import Dataset from voice._defaults import MetricGroup -from voice.comparison._utils import calibrated_percentile +from voice.comparison._utils import ( + calibrated_percentile, + percentile_interval, +) from voice.comparison.comparison import ( ComparisonEntry, ComparisonResults, @@ -85,7 +88,7 @@ def metric_registry(): return { "len": _Metric( fn=lambda s: float(len(s)), - group=MetricGroup.TEXT_LENGTH, + group=MetricGroup.WORD_LENGTH_DISTRIBUTION, ), "vowels": _Metric( fn=lambda s: float(sum(c in "aeiou" for c in s.lower())), @@ -154,6 +157,42 @@ def test_calibrated_percentile_flattens_non_1d_inputs(): assert calibrated_percentile(2.0, dist) == 0.5 +# ----------------------------------------------------------------------------- +# percentile_interval +# ----------------------------------------------------------------------------- + + +def test_percentile_interval_rejects_empty_values(): + with pytest.raises(ValueError, match=r"values must be non-empty"): + percentile_interval(np.array([]), 0.9) + + +@pytest.mark.parametrize("confidence", [0.0, 1.0, -0.1, 1.5]) +def test_percentile_interval_rejects_invalid_confidence(confidence): + with pytest.raises(ValueError, match=r"strictly between 0 and 1"): + percentile_interval(np.array([0.1, 0.2]), confidence) + + +def test_percentile_interval_hand_computed_example(): + # 25th/75th percentiles of 5 evenly spaced points (linear interpolation) + values = np.array([0.1, 0.2, 0.3, 0.4, 0.5]) + lo, hi = percentile_interval(values, 0.5) + assert lo == pytest.approx(0.2) + assert hi == pytest.approx(0.4) + + +def test_percentile_interval_collapses_for_constant_values(): + lo, hi = percentile_interval(np.full(10, 0.7), 0.9) + assert lo == pytest.approx(0.7) + assert hi == pytest.approx(0.7) + + +def test_percentile_interval_is_bounded_by_value_range(): + values = np.array([0.0, 0.0, 0.0, 0.9]) + lo, hi = percentile_interval(values, 0.99) + assert 0.0 <= lo <= hi <= 0.9 + + # ----------------------------------------------------------------------------- # self_wasserstein_distribution # ----------------------------------------------------------------------------- @@ -253,7 +292,9 @@ def raising_metric(s: str) -> float: comparison, "get_metrics", lambda: { - "len": _Metric(fn=raising_metric, group=MetricGroup.TEXT_LENGTH) + "len": _Metric( + fn=raising_metric, group=MetricGroup.WORD_LENGTH_DISTRIBUTION + ) }, ) @@ -281,7 +322,9 @@ def test_self_wasserstein_distribution_is_zero_for_constant_metric( comparison, "get_metrics", lambda: { - "const": _Metric(fn=lambda _s: 1.0, group=MetricGroup.TEXT_LENGTH) + "const": _Metric( + fn=lambda _s: 1.0, group=MetricGroup.WORD_LENGTH_DISTRIBUTION + ) }, ) @@ -305,7 +348,7 @@ def test_self_wasserstein_distribution_is_zero_for_constant_metric( def test_comparison_entry_is_frozen_and_slots(): e = ComparisonEntry( metric="m", - group=MetricGroup.TEXT_LENGTH, + group=MetricGroup.WORD_LENGTH_DISTRIBUTION, wasserstein=1.0, percentile=0.9, tail=0.1, @@ -317,7 +360,7 @@ def test_comparison_entry_is_frozen_and_slots(): def test_comparison_group_entry_is_frozen_and_slots(): e = GroupEntry( - group=MetricGroup.TEXT_LENGTH, + group=MetricGroup.WORD_LENGTH_DISTRIBUTION, avg_percentile=0.9, avg_tail=0.1, ) @@ -341,7 +384,7 @@ def test_comparison_results_properties_and_add_logic(patch_metrics): r.add(metric="len", wasserstein=np.float64(1.25), percentile=0.9) e = r.get("len") assert e.metric == "len" - assert e.group == MetricGroup.TEXT_LENGTH + assert e.group == MetricGroup.WORD_LENGTH_DISTRIBUTION assert isinstance(e.wasserstein, float) assert isinstance(e.percentile, float) assert e.tail == pytest.approx(0.1) @@ -351,7 +394,7 @@ def test_comparison_results_properties_and_add_logic(patch_metrics): assert e2.group == MetricGroup.VOCABULARY_RICHNESS # Group-level checks - ge_len = r.get_group(MetricGroup.TEXT_LENGTH) + ge_len = r.get_group(MetricGroup.WORD_LENGTH_DISTRIBUTION) assert ge_len.avg_percentile == pytest.approx(0.9) assert ge_len.avg_tail == pytest.approx(0.1) @@ -375,10 +418,12 @@ def test_comparison_results_score_weights_groups_equally(patch_metrics): # Add a second TEXT_LENGTH metric to verify equal group weighting. registry = { "len": _Metric( - fn=lambda s: float(len(s)), group=MetricGroup.TEXT_LENGTH + fn=lambda s: float(len(s)), + group=MetricGroup.WORD_LENGTH_DISTRIBUTION, ), "len2": _Metric( - fn=lambda s: float(len(s)), group=MetricGroup.TEXT_LENGTH + fn=lambda s: float(len(s)), + group=MetricGroup.WORD_LENGTH_DISTRIBUTION, ), "vowels": _Metric( fn=lambda s: float(sum(c in "aeiou" for c in s.lower())), @@ -418,10 +463,96 @@ def test_comparison_results_group_tails(patch_metrics): r.add(metric="vowels", wasserstein=0.5, percentile=0.3) tails = r.group_tails - assert tails[MetricGroup.TEXT_LENGTH] == pytest.approx(0.2) + assert tails[MetricGroup.WORD_LENGTH_DISTRIBUTION] == pytest.approx(0.2) assert tails[MetricGroup.VOCABULARY_RICHNESS] == pytest.approx(0.7) +def test_comparison_results_uncertainty_defaults_to_none(patch_metrics): + r = ComparisonResults(metrics=("len", "vowels")) + assert r.has_uncertainty is False + assert r.metric_tail_cis() is None + assert r.group_tail_cis() is None + assert r.score_ci() is None + + +def test_comparison_results_confidence_intervals(patch_metrics): + r = ComparisonResults(metrics=("len", "vowels")) + r.add(metric="len", wasserstein=1.0, percentile=0.8) + r.add(metric="vowels", wasserstein=0.5, percentile=0.4) + r._replicates = np.array( + [ + [0.1, 0.5], + [0.2, 0.6], + [0.3, 0.7], + [0.4, 0.8], + [0.5, 0.9], + ] + ) + + assert r.has_uncertainty is True + + cis = r.metric_tail_cis(confidence=0.5) + assert cis["len"] == pytest.approx((0.2, 0.4)) + assert cis["vowels"] == pytest.approx((0.6, 0.8)) + + # One metric per group, so group CI equals metric CI + g = r.group_tail_cis(confidence=0.5) + assert g[MetricGroup.WORD_LENGTH_DISTRIBUTION] == pytest.approx((0.2, 0.4)) + assert g[MetricGroup.VOCABULARY_RICHNESS] == pytest.approx((0.6, 0.8)) + + # Replicate scores are row means over groups: [0.3, 0.4, 0.5, 0.6, 0.7] + assert r.score_ci(confidence=0.5) == pytest.approx((0.4, 0.6)) + + +def test_comparison_results_cis_average_replicates_within_groups( + patch_metrics, monkeypatch +): + registry = { + "len": _Metric( + fn=lambda s: float(len(s)), + group=MetricGroup.WORD_LENGTH_DISTRIBUTION, + ), + "len2": _Metric( + fn=lambda s: float(len(s)), + group=MetricGroup.WORD_LENGTH_DISTRIBUTION, + ), + "vowels": _Metric( + fn=lambda s: float(sum(c in "aeiou" for c in s.lower())), + group=MetricGroup.VOCABULARY_RICHNESS, + ), + } + import voice.comparison.comparison as comparison + + monkeypatch.setattr(comparison, "get_metrics", lambda: dict(registry)) + + r = ComparisonResults(metrics=("len", "len2", "vowels")) + r._replicates = np.array( + [ + [0.0, 0.2, 0.9], + [0.2, 0.4, 0.9], + [0.4, 0.6, 0.9], + ] + ) + + # Group replicates: word_length = [0.1, 0.3, 0.5], vocab = [0.9] * 3 + g = r.group_tail_cis(confidence=0.5) + assert g[MetricGroup.WORD_LENGTH_DISTRIBUTION] == pytest.approx((0.2, 0.4)) + assert g[MetricGroup.VOCABULARY_RICHNESS] == pytest.approx((0.9, 0.9)) + + # Score replicates: mean over groups = [0.5, 0.6, 0.7] + assert r.score_ci(confidence=0.5) == pytest.approx((0.55, 0.65)) + + +def test_comparison_results_repr_includes_ci_when_available(patch_metrics): + r = ComparisonResults(metrics=("len",)) + r.add(metric="len", wasserstein=1.0, percentile=0.8) + assert "[" not in repr(r) + + # 5th/95th percentiles of [0.1, 0.2, 0.3] (default confidence 0.90) + r._replicates = np.array([[0.1], [0.2], [0.3]]) + assert "[0.1100, 0.2900]" in repr(r) + + # ----------------------------------------------------------------------------- # make_comparison # ----------------------------------------------------------------------------- @@ -458,7 +589,7 @@ def fake_self_dist(*_args, **_kwargs): comparison.self_wasserstein_distribution = fake_self_dist # type: ignore[assignment] - out = make_comparison(completions, true_ds) + out = make_comparison(completions, true_ds, uncertainty=False) assert out.metrics == ("len", "vowels") for m in out.metrics: @@ -486,7 +617,9 @@ def test_make_comparison_respects_subset_metrics(patch_metrics, monkeypatch): lambda *_args, **_kwargs: np.array([0.0, 1.0], dtype=float), ) - out = make_comparison(completions, true_ds, metrics=["len"]) + out = make_comparison( + completions, true_ds, metrics=["len"], uncertainty=False + ) assert out.metrics == ("len",) assert "len" in out.as_dict() with pytest.raises(KeyError): @@ -523,7 +656,9 @@ def recording_getitem(self, key): monkeypatch.setattr(VoiceDataset, "__getitem__", recording_getitem) - _ = make_comparison(completions, true_ds, metrics=["len"]) + _ = make_comparison( + completions, true_ds, metrics=["len"], uncertainty=False + ) assert Split.TEST in calls @@ -557,6 +692,167 @@ def spy_self_dist(*_args, sample_size: int = 0, **_kwargs): comparison, "self_wasserstein_distribution", spy_self_dist ) - make_comparison(completions, true_ds, metrics=["len"]) + make_comparison(completions, true_ds, metrics=["len"], uncertainty=False) assert all(s == len(completions) for s in captured) + + +# ----------------------------------------------------------------------------- +# make_comparison: uncertainty quantification +# ----------------------------------------------------------------------------- + + +def _paired_setup(n: int = 5) -> tuple[list[Example], VoiceDataset]: + """Completions paired one-to-one with the TEST split.""" + p = _pinned(splits=(Split.TRAIN, Split.TEST)) + true_ds = VoiceDataset( + datasets={ + Split.TRAIN: _canonical_hf_ds([f"a{i}" for i in range(40)]), + Split.TEST: _canonical_hf_ds( + ["ref text " * (i + 1) for i in range(n)] + ), + }, + spec=p, + ) + completions = [ + _ex(answer="hello there " * (i + 2), split=Split.TEST) + for i in range(n) + ] + return completions, true_ds + + +def test_make_comparison_uncertainty_populates_cis_and_caches_self_dist( + patch_metrics, monkeypatch +): + completions, true_ds = _paired_setup(n=5) + + import voice.comparison.comparison as comparison + + calls: list[str] = [] + + def spy_self_dist(*_args, metric: str = "", **_kwargs): + calls.append(metric) + return np.linspace(0.0, 50.0, 100) + + monkeypatch.setattr( + comparison, "self_wasserstein_distribution", spy_self_dist + ) + + out = make_comparison(completions, true_ds) + + assert out.has_uncertainty is True + + cis = out.metric_tail_cis() + assert set(cis) == {"len", "vowels"} + for lo, hi in cis.values(): + assert 0.0 <= lo <= hi <= 1.0 + + assert set(out.group_tail_cis()) == { + MetricGroup.WORD_LENGTH_DISTRIBUTION, + MetricGroup.VOCABULARY_RICHNESS, + } + + lo, hi = out.score_ci() + assert 0.0 <= lo <= hi <= 1.0 + + # A wider confidence level produces a containing interval + lo99, hi99 = out.score_ci(confidence=0.99) + assert lo99 <= lo + assert hi <= hi99 + + # Self-Wasserstein bootstrap is cached: computed exactly once per + # metric, never per jackknife replicate. + assert sorted(calls) == ["len", "vowels"] + + +def test_make_comparison_uncertainty_false_skips_jackknife( + patch_metrics, monkeypatch +): + completions, true_ds = _paired_setup(n=5) + + import voice.comparison.comparison as comparison + + monkeypatch.setattr( + comparison, + "self_wasserstein_distribution", + lambda *_args, **_kwargs: np.array([0.0, 1.0], dtype=float), + ) + + out = make_comparison(completions, true_ds, uncertainty=False) + + assert out.has_uncertainty is False + assert out.metric_tail_cis() is None + assert out.group_tail_cis() is None + assert out.score_ci() is None + + +def test_make_comparison_uncertainty_requires_two_completions( + patch_metrics, monkeypatch +): + completions, true_ds = _paired_setup(n=1) + + with pytest.raises(ValueError, match=r"at least 2 completions"): + make_comparison(completions, true_ds) + + +def test_make_comparison_uncertainty_requires_paired_split( + patch_metrics, monkeypatch +): + completions, true_ds = _paired_setup(n=5) + + with pytest.raises(ValueError, match=r"paired one-to-one"): + make_comparison(completions[:3], true_ds) + + +def test_make_comparison_score_ci_matches_metric_ci_for_single_metric( + patch_metrics, monkeypatch +): + completions, true_ds = _paired_setup(n=5) + + import voice.comparison.comparison as comparison + + monkeypatch.setattr( + comparison, + "self_wasserstein_distribution", + lambda *_args, **_kwargs: np.linspace(0.0, 50.0, 100), + ) + + out = make_comparison(completions, true_ds, metrics=["len"]) + + assert out.score_ci() == pytest.approx(out.metric_tail_cis()["len"]) + assert out.score_ci() == pytest.approx( + out.group_tail_cis()[MetricGroup.WORD_LENGTH_DISTRIBUTION] + ) + + +def test_make_comparison_ci_collapses_for_identical_paired_data( + patch_metrics, monkeypatch +): + n = 5 + answers = [f"some answer {i} " * (i + 1) for i in range(n)] + p = _pinned(splits=(Split.TRAIN, Split.TEST)) + true_ds = VoiceDataset( + datasets={ + Split.TRAIN: _canonical_hf_ds([f"a{i}" for i in range(40)]), + Split.TEST: _canonical_hf_ds(answers), + }, + spec=p, + ) + completions = [_ex(answer=a, split=Split.TEST) for a in answers] + + import voice.comparison.comparison as comparison + + monkeypatch.setattr( + comparison, + "self_wasserstein_distribution", + lambda *_args, **_kwargs: np.linspace(0.0, 50.0, 100), + ) + + # x == y for every metric, so every leave-one-out distance is 0 and + # all replicates are identical: each interval collapses to a point. + out = make_comparison(completions, true_ds) + + lo, hi = out.score_ci() + assert lo == pytest.approx(hi) + for m_lo, m_hi in out.metric_tail_cis().values(): + assert m_lo == pytest.approx(m_hi) diff --git a/tests/voice/stylometry/test_metrics.py b/tests/voice/stylometry/test_metrics.py index 69fc947..8adc336 100644 --- a/tests/voice/stylometry/test_metrics.py +++ b/tests/voice/stylometry/test_metrics.py @@ -80,7 +80,6 @@ def test_get_metrics_contains_expected_metric_names(): "std_word_length", "skew_word_length", "kurtosis_word_length", - "num_words", "hapax_legomena_ratio", "dis_legomena_ratio", "tri_legomena_ratio", @@ -100,15 +99,19 @@ def test_get_metrics_contains_expected_metric_names(): def test_metric_decorator_rejects_duplicate_names(monkeypatch): # Isolate decorator behaviour by swapping the module registry. monkeypatch.setattr(metrics, "_REGISTRY", {}) - metrics.metric("x", group=MetricGroup.TEXT_LENGTH)(lambda t: 0.0) + metrics.metric("x", group=MetricGroup.WORD_LENGTH_DISTRIBUTION)( + lambda t: 0.0 + ) with pytest.raises(ValueError, match=r"Duplicate metric name 'x'"): - metrics.metric("x", group=MetricGroup.TEXT_LENGTH)(lambda t: 1.0) + metrics.metric("x", group=MetricGroup.WORD_LENGTH_DISTRIBUTION)( + lambda t: 1.0 + ) def test_metric_spec_is_frozen(): spec = metrics.MetricSpec( fn=lambda t: 0.0, - group=MetricGroup.TEXT_LENGTH, + group=MetricGroup.WORD_LENGTH_DISTRIBUTION, description="x", ) assert type(spec).__dataclass_params__.frozen is True @@ -152,7 +155,6 @@ def test_metric_groups_cover_all_enum_values(): "std_word_length": MetricGroup.WORD_LENGTH_DISTRIBUTION, "skew_word_length": MetricGroup.WORD_LENGTH_DISTRIBUTION, "kurtosis_word_length": MetricGroup.WORD_LENGTH_DISTRIBUTION, - "num_words": MetricGroup.TEXT_LENGTH, "hapax_legomena_ratio": MetricGroup.VOCABULARY_RICHNESS, "dis_legomena_ratio": MetricGroup.VOCABULARY_RICHNESS, "tri_legomena_ratio": MetricGroup.VOCABULARY_RICHNESS, @@ -256,13 +258,6 @@ def fake_moment(units, order: int) -> float: # ----------------------------------------------------------------------------- -def test_num_words_counts_tokens(monkeypatch): - monkeypatch.setattr( - metrics, "word_tokenize", lambda t: ["a", "b", "c", "d"] - ) - assert metrics.calculate_num_words("ignored") == 4.0 - - def test_hapax_dis_tri_legomena_ratios_exact_on_handmade_tokens(monkeypatch): tokens = ["a", "b", "b", "c", "c", "c"] monkeypatch.setattr(metrics, "word_tokenize", lambda t: tokens) @@ -457,6 +452,5 @@ def test_all_metrics_run_on_simple_sentence_smoke(): def test_selected_metrics_on_tiny_text_regression(): text = "a bb ccc" - assert metrics.calculate_num_words(text) == 3.0 assert metrics.calculate_avg_word_length(text) == pytest.approx(2.0) assert metrics.calculate_hapax_legomena_ratio(text) == pytest.approx(1.0)