Skip to content

Group correlated stylometric metrics #5

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
fingriffin merged 2 commits into from
Apr 9, 2026
Merged

Group correlated stylometric metrics #5

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
dac5ce6
feat(stylometry): group correlated stylometric metrics
fin-griffin Apr 9, 2026
8ea2628
Merge branch 'feature/new-evals' into feature/group-metrics
fin-griffin Apr 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 41 additions & 0 deletions src/voice/stylometry/_defaults.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,47 @@
"""

from dataclasses import dataclass
from enum import Enum


class MetricGroup(str, Enum):
"""
Enumeration of stylometric metric groups.

Metrics within the same group are expected to be correlated;
the group is the unit of analysis for Bonferroni correction.

.. attribute :: WORD_LENGTH_DISTRIBUTION

Moments of the word length distribution (mean, std, skew, kurtosis).

.. attribute :: LEXICAL_RICHNESS

Type-token ratio and its moving-average variant.

.. attribute :: LEGOMENA

Hapax, dis and tri legomena ratios.

.. attribute :: FUNCTION_WORDS

Function word usage ratio.

.. attribute :: CHAR_NGRAM_DIVERSITY

Character n-gram TTR and MATTR across n = 3, 4, 5.

.. attribute :: TEXT_LENGTH

Raw token count.
"""

WORD_LENGTH_DISTRIBUTION = "word_length_distribution"
LEXICAL_RICHNESS = "lexical_richness"
LEGOMENA = "legomena"
FUNCTION_WORDS = "function_words"
CHAR_NGRAM_DIVERSITY = "char_ngram_diversity"
TEXT_LENGTH = "text_length"


@dataclass(frozen=True)
Expand Down
90 changes: 74 additions & 16 deletions src/voice/stylometry/metrics.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
from collections.abc import Callable
from dataclasses import dataclass

from voice.stylometry._defaults import PARAMETER_DEFAULTS
from voice.stylometry._defaults import PARAMETER_DEFAULTS, MetricGroup
from voice.stylometry._lexicons import FUNCTION_WORDS
from voice.stylometry._sequence_stats import (
length_central_moment_from_units,
Expand All @@ -55,12 +55,17 @@ class MetricSpec:

Function mapping a string to a float.

.. attribute :: group

The metric group this metric belongs to.

.. attribute :: description

Description of the metric.
"""

fn: MetricFn
group: MetricGroup
description: str | None = None

def __repr__(self) -> str:
Expand All @@ -70,21 +75,25 @@ def __repr__(self) -> str:
:return: Readable string representation of the MetricSpec
"""
fn_name = getattr(self.fn, "__name__", repr(self.fn))
return f"MetricSpec(fn={fn_name}, description={self.description!r})"
return (
f"MetricSpec("
f"fn={fn_name}, "
f"group={self.group.name}, "
f"description={self.description!r})"
)


_REGISTRY: dict[str, MetricSpec] = {}


def metric(
name: str, *, description: str | None = None
name: str, *, group: MetricGroup, description: str | None = None
) -> Callable[[MetricFn], MetricFn]:
"""
Register a metric function in the global registry.

Description can also be optionally added.

:param name: Name of the metric (snake case)
:param group: The metric group this metric belongs to
:param description: Description of the metric
:return: Decorator that registers the metric function
"""
Expand All @@ -101,7 +110,9 @@ def decorator(fn: MetricFn) -> MetricFn:
"""
if name in _REGISTRY:
raise ValueError(f"Duplicate metric name '{name}'")
_REGISTRY[name] = MetricSpec(fn=fn, description=description)
_REGISTRY[name] = MetricSpec(
fn=fn, group=group, description=description
)
return fn

return decorator
Expand All @@ -121,7 +132,11 @@ def get_metrics() -> dict[str, MetricSpec]:
# -----------------------------------------------------------------------------


@metric("avg_word_length", description="Average word length")
@metric(
"avg_word_length",
group=MetricGroup.WORD_LENGTH_DISTRIBUTION,
description="Average word length",
)
def calculate_avg_word_length(text: str) -> float:
"""
Calculate the average word length of a string.
Expand All @@ -137,7 +152,11 @@ def calculate_avg_word_length(text: str) -> float:
return sum(len(word) for word in words) / len(words)


@metric("std_word_length", description="Standard deviation of word length")
@metric(
"std_word_length",
group=MetricGroup.WORD_LENGTH_DISTRIBUTION,
description="Standard deviation of word length",
)
def calculate_std_word_length(text: str) -> float:
"""
Calculate the standard deviation of word length in a string.
Expand All @@ -151,7 +170,11 @@ def calculate_std_word_length(text: str) -> float:
return math.sqrt(length_central_moment_from_units(words, order=2))


@metric("skew_word_length", description="Skewness of word length")
@metric(
"skew_word_length",
group=MetricGroup.WORD_LENGTH_DISTRIBUTION,
description="Skewness of word length",
)
def calculate_skew_word_length(text: str) -> float:
"""
Calculate the skewness of word length in a string.
Expand All @@ -171,7 +194,11 @@ def calculate_skew_word_length(text: str) -> float:
return float(m3 / (m2**1.5))


@metric("kurtosis_word_length", description="Kurtosis of word length")
@metric(
"kurtosis_word_length",
group=MetricGroup.WORD_LENGTH_DISTRIBUTION,
description="Kurtosis of word length",
)
def calculate_kurtosis_word_length(text: str) -> float:
"""
Calculate the kurtosis of word length in a string.
Expand Down Expand Up @@ -199,7 +226,11 @@ def calculate_kurtosis_word_length(text: str) -> float:
# -----------------------------------------------------------------------------


@metric("num_words", description="Number of words in text")
@metric(
"num_words",
group=MetricGroup.TEXT_LENGTH,
description="Number of words in text",
)
def calculate_num_words(text: str) -> float:
"""
Calculate the number of words in a string.
Expand All @@ -212,7 +243,11 @@ def calculate_num_words(text: str) -> float:
return float(len(word_tokenize(text)))


@metric("hapax_legomena_ratio", description="Ratio of hapax legomena")
@metric(
"hapax_legomena_ratio",
group=MetricGroup.LEGOMENA,
description="Ratio of hapax legomena",
)
def calculate_hapax_legomena_ratio(text: str) -> float:
"""
Calculate the ratio of hapax legomena in a string.
Expand All @@ -236,7 +271,11 @@ def calculate_hapax_legomena_ratio(text: str) -> float:
return hapax / len(words)


@metric("dis_legomena_ratio", description="Ratio of dis legomena")
@metric(
"dis_legomena_ratio",
group=MetricGroup.LEGOMENA,
description="Ratio of dis legomena",
)
def calculate_dis_legomena_ratio(text: str) -> float:
"""
Calculate the ratio of dis legomena in a string.
Expand All @@ -260,7 +299,11 @@ def calculate_dis_legomena_ratio(text: str) -> float:
return dis / len(words)


@metric("tri_legomena_ratio", description="Ratio of tri legomena")
@metric(
"tri_legomena_ratio",
group=MetricGroup.LEGOMENA,
description="Ratio of tri legomena",
)
def calculate_tri_legomena_ratio(text: str) -> float:
"""
Calculate the ratio of tri legomena in a string.
Expand All @@ -284,7 +327,11 @@ def calculate_tri_legomena_ratio(text: str) -> float:
return tri / len(words)


@metric("function_word_ratio", description="Ratio of function words")
@metric(
"function_word_ratio",
group=MetricGroup.FUNCTION_WORDS,
description="Ratio of function words",
)
def calculate_function_word_ratio(text: str) -> float:
"""
Calculate the ratio of function words in a string.
Expand All @@ -309,7 +356,11 @@ def calculate_function_word_ratio(text: str) -> float:
# -----------------------------------------------------------------------------


@metric("type_token_ratio", description="Type-token ratio")
@metric(
"type_token_ratio",
group=MetricGroup.LEXICAL_RICHNESS,
description="Type-token ratio",
)
def calculate_type_token_ratio(text: str) -> float:
"""
Calculate the type-token ratio (TTR) of a string.
Expand All @@ -327,6 +378,7 @@ def calculate_type_token_ratio(text: str) -> float:

@metric(
"moving_avg_type_token_ratio",
group=MetricGroup.LEXICAL_RICHNESS,
description="Moving average type-token ratio",
)
def calculate_moving_avg_type_token_ratio(text: str) -> float:
Expand Down Expand Up @@ -360,6 +412,7 @@ def calculate_moving_avg_type_token_ratio(text: str) -> float:

@metric(
"char_3gram_type_token_ratio",
group=MetricGroup.CHAR_NGRAM_DIVERSITY,
description="Type-token ratio of character 3-grams",
)
def calculate_char_3gram_type_token_ratio(text: str) -> float:
Expand All @@ -379,6 +432,7 @@ def calculate_char_3gram_type_token_ratio(text: str) -> float:

@metric(
"char_4gram_type_token_ratio",
group=MetricGroup.CHAR_NGRAM_DIVERSITY,
description="Type-token ratio of character 4-grams",
)
def calculate_char_4gram_type_token_ratio(text: str) -> float:
Expand All @@ -398,6 +452,7 @@ def calculate_char_4gram_type_token_ratio(text: str) -> float:

@metric(
"char_5gram_type_token_ratio",
group=MetricGroup.CHAR_NGRAM_DIVERSITY,
description="Type-token ratio of character 5-grams",
)
def calculate_char_5gram_type_token_ratio(text: str) -> float:
Expand All @@ -422,6 +477,7 @@ def calculate_char_5gram_type_token_ratio(text: str) -> float:

@metric(
"char_3gram_moving_avg_type_token_ratio",
group=MetricGroup.CHAR_NGRAM_DIVERSITY,
description="Moving average type-token ratio of character 3-grams",
)
def calculate_char_3gram_moving_avg_type_token_ratio(text: str) -> float:
Expand Down Expand Up @@ -450,6 +506,7 @@ def calculate_char_3gram_moving_avg_type_token_ratio(text: str) -> float:

@metric(
"char_4gram_moving_avg_type_token_ratio",
group=MetricGroup.CHAR_NGRAM_DIVERSITY,
description="Moving average type-token ratio of character 4-grams",
)
def calculate_char_4gram_moving_avg_type_token_ratio(text: str) -> float:
Expand Down Expand Up @@ -478,6 +535,7 @@ def calculate_char_4gram_moving_avg_type_token_ratio(text: str) -> float:

@metric(
"char_5gram_moving_avg_type_token_ratio",
group=MetricGroup.CHAR_NGRAM_DIVERSITY,
description="Moving average type-token ratio of character 5-grams",
)
def calculate_char_5gram_moving_avg_type_token_ratio(text: str) -> float:
Expand Down
66 changes: 61 additions & 5 deletions tests/voice/stylometry/test_metrics.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import pytest

import voice.stylometry.metrics as metrics
from voice.stylometry._defaults import MetricGroup

# -----------------------------------------------------------------------------
# Helpers
Expand Down Expand Up @@ -92,29 +93,84 @@ def test_get_metrics_contains_expected_metric_names():
def test_metric_decorator_rejects_duplicate_names(monkeypatch):
# Isolate decorator behaviour by swapping the module registry.
monkeypatch.setattr(metrics, "_REGISTRY", {})
metrics.metric("x")(lambda t: 0.0)
metrics.metric("x", group=MetricGroup.TEXT_LENGTH)(lambda t: 0.0)
with pytest.raises(ValueError, match=r"Duplicate metric name 'x'"):
metrics.metric("x")(lambda t: 1.0)
metrics.metric("x", group=MetricGroup.TEXT_LENGTH)(lambda t: 1.0)


def test_metric_spec_is_frozen():
spec = metrics.MetricSpec(fn=lambda t: 0.0, description="x")
spec = metrics.MetricSpec(
fn=lambda t: 0.0,
group=MetricGroup.TEXT_LENGTH,
description="x",
)
assert type(spec).__dataclass_params__.frozen is True

with pytest.raises(dataclasses.FrozenInstanceError):
setattr(spec, "description", "y") # noqa: B010


def test_metric_spec_repr_contains_function_name_and_description():
def test_metric_spec_repr_contains_function_name_group_and_description():
spec = metrics.MetricSpec(
fn=metrics.calculate_avg_word_length, description="Average word length"
fn=metrics.calculate_avg_word_length,
group=MetricGroup.WORD_LENGTH_DISTRIBUTION,
description="Average word length",
)
r = repr(spec)
assert "MetricSpec(" in r
assert "calculate_avg_word_length" in r
assert "WORD_LENGTH_DISTRIBUTION" in r
assert "Average word length" in r


# -----------------------------------------------------------------------------
# MetricGroup
# -----------------------------------------------------------------------------


def test_all_metrics_have_a_valid_group():
for name, spec in _all_metric_items():
assert isinstance(spec.group, MetricGroup), (
f"Metric '{name}' has group {spec.group!r}, expected a MetricGroup"
)


def test_metric_groups_cover_all_enum_values():
assigned = {spec.group for _, spec in _all_metric_items()}
assert assigned == set(MetricGroup)


_EXPECTED_GROUPS: dict[str, MetricGroup] = {
"avg_word_length": MetricGroup.WORD_LENGTH_DISTRIBUTION,
"std_word_length": MetricGroup.WORD_LENGTH_DISTRIBUTION,
"skew_word_length": MetricGroup.WORD_LENGTH_DISTRIBUTION,
"kurtosis_word_length": MetricGroup.WORD_LENGTH_DISTRIBUTION,
"num_words": MetricGroup.TEXT_LENGTH,
"hapax_legomena_ratio": MetricGroup.LEGOMENA,
"dis_legomena_ratio": MetricGroup.LEGOMENA,
"tri_legomena_ratio": MetricGroup.LEGOMENA,
"function_word_ratio": MetricGroup.FUNCTION_WORDS,
"type_token_ratio": MetricGroup.LEXICAL_RICHNESS,
"moving_avg_type_token_ratio": MetricGroup.LEXICAL_RICHNESS,
"char_3gram_type_token_ratio": MetricGroup.CHAR_NGRAM_DIVERSITY,
"char_4gram_type_token_ratio": MetricGroup.CHAR_NGRAM_DIVERSITY,
"char_5gram_type_token_ratio": MetricGroup.CHAR_NGRAM_DIVERSITY,
"char_3gram_moving_avg_type_token_ratio": MetricGroup.CHAR_NGRAM_DIVERSITY,
"char_4gram_moving_avg_type_token_ratio": MetricGroup.CHAR_NGRAM_DIVERSITY,
"char_5gram_moving_avg_type_token_ratio": MetricGroup.CHAR_NGRAM_DIVERSITY,
}


@pytest.mark.parametrize(
"metric_name,expected_group", _EXPECTED_GROUPS.items()
)
def test_metric_group_assignments(
metric_name: str, expected_group: MetricGroup
):
spec = metrics.get_metrics()[metric_name]
assert spec.group == expected_group


# -----------------------------------------------------------------------------
# Global invariants (over all metrics)
# -----------------------------------------------------------------------------
Expand Down
Loading