Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
9569b2e
Adds an overridable dataset-loading hook to UniMorphAdapter and a Ukr…
pkuchmiichuk Jul 10, 2026
8e911dd
Adds the Ukrainian verb-lexicon generator built on the VESUM adapter.
pkuchmiichuk Jul 11, 2026
a1a4175
Adds the bleached noun lexicon builder and parses VESUM once for both…
pkuchmiichuk Jul 14, 2026
d4a9213
Adds the Ukrainian sentence-frame generator and a capitalizing templa…
pkuchmiichuk Jul 14, 2026
84ba9e2
Adds the config-driven exhaustive fill stage for the Ukrainian frames.
pkuchmiichuk Jul 14, 2026
68e0599
Adds the 2AFC stage with case-contrast pairing and a masked-LM scorer.
pkuchmiichuk Jul 21, 2026
be3e227
Selects unambiguous case forms and broadens the bleached noun set.
pkuchmiichuk Jul 21, 2026
934f3d5
Adds frequency-based verb selection with a committed wordfreq-derived…
pkuchmiichuk Jul 22, 2026
72c1efe
Defaults each list constraint's discriminator to its own type.
pkuchmiichuk Jul 22, 2026
cb62b9f
Warns when a batch constraint property expression cannot be evaluated.
pkuchmiichuk Jul 22, 2026
563c970
Excludes configured verbs from the verb slot via a set-membership con…
pkuchmiichuk Jul 22, 2026
8296397
Adds anchor verb comparisons to the Ukrainian pair stage.
pkuchmiichuk Jul 22, 2026
279d04a
Warns when a list constraint type is not enforced during assignment.
pkuchmiichuk Jul 22, 2026
685b78f
Adds the list partitioning stage for the Ukrainian pipeline.
pkuchmiichuk Jul 22, 2026
9347f45
Loads causal language models in the checkpoint's own dtype.
pkuchmiichuk Jul 22, 2026
136f701
Passes the requested dtype from the scorer to the language model.
pkuchmiichuk Jul 22, 2026
70808bb
Scores Ukrainian pairs with the causal model on the local GPU.
pkuchmiichuk Jul 22, 2026
dd03698
Adds the jsPsych and JATOS deployment stage for the Ukrainian pipeline.
pkuchmiichuk Jul 22, 2026
843074b
Adds the Ukrainian pipeline Makefile and marks the deferred model imp…
pkuchmiichuk Jul 22, 2026
b2f8514
Adds the README for the Ukrainian argument structure gallery.
pkuchmiichuk Jul 22, 2026
cb638d8
Formats the Ukrainian pipeline scripts with ruff.
pkuchmiichuk Jul 22, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion bead/items/adapters/huggingface.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,10 @@ class HuggingFaceLanguageModel(HuggingFaceAdapterMixin, ModelAdapter):
Device to run model on. Falls back to CPU if device unavailable.
model_version : str
Version string for cache tracking.
dtype : str
Torch dtype to load the weights in, such as ``"bfloat16"``. Defaults to
``"auto"``, which keeps the dtype the checkpoint was saved in; loading a
half-precision checkpoint as float32 would double its memory.

Examples
--------
Expand All @@ -77,17 +81,21 @@ def __init__(
cache: ModelOutputCache,
device: DeviceType = "cpu",
model_version: str = "unknown",
dtype: str = "auto",
) -> None:
super().__init__(model_name, cache, model_version)
self.device = self._validate_device(device)
self.dtype = dtype
self._model: PreTrainedModel | None = None
self._tokenizer: PreTrainedTokenizerBase | None = None

def _load_model(self) -> None:
"""Load model and tokenizer lazily on first use."""
if self._model is None:
logger.info(f"Loading causal LM: {self.model_name}")
self._model = AutoModelForCausalLM.from_pretrained(self.model_name)
self._model = AutoModelForCausalLM.from_pretrained(
self.model_name, dtype=self.dtype
)
self._model.to(self.device)
self._model.eval()

Expand Down
6 changes: 6 additions & 0 deletions bead/items/scoring.py
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,9 @@ class LanguageModelScorer(ItemScorer):
Key in item.rendered_elements to use as text (default: "text").
model_version : str
Version string for cache tracking.
dtype : str
Torch dtype to load the weights in, such as ``"bfloat16"``. Defaults to
``"auto"``, keeping the dtype the checkpoint was saved in.

Examples
--------
Expand All @@ -155,12 +158,14 @@ def __init__(
device: str = "cpu",
text_key: str = "text",
model_version: str = "unknown",
dtype: str = "auto",
) -> None:
self.model_name = model_name
self.cache_dir = Path(cache_dir) if cache_dir else None
self.device = device
self.text_key = text_key
self.model_version = model_version
self.dtype = dtype

# lazy loading of model and cache
self._model: HuggingFaceLanguageModel | None = None
Expand Down Expand Up @@ -193,6 +198,7 @@ def model(self) -> HuggingFaceLanguageModel:
cache=self._cache,
device=self.device, # type: ignore[arg-type]
model_version=self.model_version,
dtype=self.dtype,
)

return self._model
Expand Down
24 changes: 12 additions & 12 deletions bead/lists/constraints.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ class UniquenessConstraint(ListConstraint):
Higher values are weighted more heavily during partitioning.
"""

constraint_type: typing.Literal["uniqueness"]
constraint_type: typing.Literal["uniqueness"] = "uniqueness"
property_expression: str
context: dict[str, ContextValue] = dx.field(default_factory=dict)
allow_null: bool = False
Expand Down Expand Up @@ -111,7 +111,7 @@ class ConditionalUniquenessConstraint(ListConstraint):
Constraint priority.
"""

constraint_type: typing.Literal["conditional_uniqueness"]
constraint_type: typing.Literal["conditional_uniqueness"] = "conditional_uniqueness"
property_expression: str
condition_expression: str
context: dict[str, ContextValue] = dx.field(default_factory=dict)
Expand Down Expand Up @@ -140,7 +140,7 @@ class BalanceConstraint(ListConstraint):
Constraint priority.
"""

constraint_type: typing.Literal["balance"]
constraint_type: typing.Literal["balance"] = "balance"
property_expression: str
context: dict[str, ContextValue] = dx.field(default_factory=dict)
target_counts: dict[str, int] | None = None
Expand Down Expand Up @@ -191,7 +191,7 @@ class QuantileConstraint(ListConstraint):
Constraint priority.
"""

constraint_type: typing.Literal["quantile"]
constraint_type: typing.Literal["quantile"] = "quantile"
property_expression: str
context: dict[str, ContextValue] = dx.field(default_factory=dict)
n_quantiles: int = 5
Expand Down Expand Up @@ -231,7 +231,7 @@ class GroupedQuantileConstraint(ListConstraint):
Constraint priority.
"""

constraint_type: typing.Literal["grouped_quantile"]
constraint_type: typing.Literal["grouped_quantile"] = "grouped_quantile"
property_expression: str
group_by_expression: str
context: dict[str, ContextValue] = dx.field(default_factory=dict)
Expand Down Expand Up @@ -468,7 +468,7 @@ class DiversityConstraint(ListConstraint):
Constraint priority.
"""

constraint_type: typing.Literal["diversity"]
constraint_type: typing.Literal["diversity"] = "diversity"
property_expression: str
min_unique_values: int
context: dict[str, ContextValue] = dx.field(default_factory=dict)
Expand Down Expand Up @@ -505,7 +505,7 @@ class SizeConstraint(ListConstraint):
Constraint priority.
"""

constraint_type: typing.Literal["size"]
constraint_type: typing.Literal["size"] = "size"
min_size: int | None = None
max_size: int | None = None
exact_size: int | None = None
Expand Down Expand Up @@ -580,7 +580,7 @@ class OrderingConstraint(ListConstraint):
Constraint priority (unused for static partitioning).
"""

constraint_type: typing.Literal["ordering"]
constraint_type: typing.Literal["ordering"] = "ordering"
precedence_pairs: tuple[dx.Embed[OrderingPair], ...] = ()
no_adjacent_property: str | None = None
block_by_property: str | None = None
Expand Down Expand Up @@ -637,7 +637,7 @@ class BatchCoverageConstraint(BatchConstraint):
Constraint priority.
"""

constraint_type: typing.Literal["coverage"]
constraint_type: typing.Literal["coverage"] = "coverage"
property_expression: str
context: dict[str, ContextValue] = dx.field(default_factory=dict)
target_values: tuple[str | int | float, ...] | None = None
Expand Down Expand Up @@ -673,7 +673,7 @@ class BatchBalanceConstraint(BatchConstraint):
Constraint priority.
"""

constraint_type: typing.Literal["balance"]
constraint_type: typing.Literal["balance"] = "balance"
property_expression: str
target_distribution: dict[str, float]
context: dict[str, ContextValue] = dx.field(default_factory=dict)
Expand Down Expand Up @@ -724,7 +724,7 @@ class BatchDiversityConstraint(BatchConstraint):
Constraint priority.
"""

constraint_type: typing.Literal["diversity"]
constraint_type: typing.Literal["diversity"] = "diversity"
property_expression: str
max_lists_per_value: int
context: dict[str, ContextValue] = dx.field(default_factory=dict)
Expand Down Expand Up @@ -757,7 +757,7 @@ class BatchMinOccurrenceConstraint(BatchConstraint):
Constraint priority.
"""

constraint_type: typing.Literal["min_occurrence"]
constraint_type: typing.Literal["min_occurrence"] = "min_occurrence"
property_expression: str
min_occurrences: int
context: dict[str, ContextValue] = dx.field(default_factory=dict)
Expand Down
63 changes: 59 additions & 4 deletions bead/lists/partitioner.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

from __future__ import annotations

import logging
from collections import Counter, defaultdict
from collections.abc import Callable, Hashable
from typing import Any
Expand Down Expand Up @@ -43,6 +44,8 @@
# without dict-invariance noise.
type BalanceMetrics = dict[str, "MetadataValue"]

logger = logging.getLogger(__name__)


class ListPartitioner:
"""Partitions items into balanced experimental lists.
Expand Down Expand Up @@ -80,6 +83,8 @@ def __init__(self, random_seed: int | None = None) -> None:
self.random_seed = random_seed
self._rng = np.random.default_rng(random_seed)
self.dsl_evaluator = DSLEvaluator()
self._unresolved_expressions: set[str] = set()
self._unenforced_types: set[str] = set()

def partition(
self,
Expand Down Expand Up @@ -448,6 +453,8 @@ def _count_violations(
elif isinstance(constraint, SizeConstraint):
if not self._check_size(exp_list, constraint):
is_violated = True
else:
self._warn_unenforced(str(constraint.constraint_type))

if is_violated:
priority = constraint.priority
Expand Down Expand Up @@ -574,6 +581,50 @@ def _check_size(self, exp_list: ExperimentList, constraint: SizeConstraint) -> b

return True

def _warn_unresolved(self, property_expression: str, error: Exception) -> None:
"""Report a property expression that cannot be evaluated.

Scoring skips items whose property cannot be read. Without a warning a
misconfigured constraint scores zero on every iteration and silently has
no effect, so each expression is reported once.

Parameters
----------
property_expression : str
The expression that failed to evaluate.
error : Exception
The failure raised while evaluating it.
"""
if property_expression in self._unresolved_expressions:
return
self._unresolved_expressions.add(property_expression)
logger.warning(
"Constraint property expression %r could not be evaluated (%s). "
"The constraint cannot be satisfied and is being ignored.",
property_expression,
error,
)

def _warn_unenforced(self, constraint_type: str) -> None:
"""Report a constraint type that assignment cannot enforce.

Unhandled types are stored on the list and serialised, so without a
warning a configured constraint appears active while having no effect.

Parameters
----------
constraint_type : str
Discriminator of the constraint that is not enforced.
"""
if constraint_type in self._unenforced_types:
return
self._unenforced_types.add(constraint_type)
logger.warning(
"List constraint type %r is not enforced during assignment. It is "
"recorded on the list but does not influence partitioning.",
constraint_type,
)

def _extract_property_value(
self,
item_id: UUID,
Expand Down Expand Up @@ -1034,7 +1085,8 @@ def _compute_batch_coverage_score(
metadata,
)
observed_values.add(value)
except Exception:
except Exception as error:
self._warn_unresolved(constraint.property_expression, error)
continue

# Compute coverage
Expand Down Expand Up @@ -1085,7 +1137,8 @@ def _compute_batch_balance_score(
)
counts[value] += 1
total += 1
except Exception:
except Exception as error:
self._warn_unresolved(constraint.property_expression, error)
continue

if total == 0:
Expand Down Expand Up @@ -1140,7 +1193,8 @@ def _compute_batch_diversity_score(
metadata,
)
value_to_lists[value].add(list_idx)
except Exception:
except Exception as error:
self._warn_unresolved(constraint.property_expression, error)
continue

if not value_to_lists:
Expand Down Expand Up @@ -1193,7 +1247,8 @@ def _compute_batch_min_occurrence_score(
metadata,
)
counts[value] += 1
except Exception:
except Exception as error:
self._warn_unresolved(constraint.property_expression, error)
continue

if not counts:
Expand Down
20 changes: 19 additions & 1 deletion bead/resources/adapters/unimorph.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,24 @@ def __init__(self, cache: AdapterCache | None = None) -> None:
self.cache = cache
self._datasets: dict[str, pd.DataFrame] = {} # Cache datasets by language

def _load_dataset(self, lang_code: str) -> pd.DataFrame:
"""Load the raw ``(lemma, form, features)`` DataFrame for a language.

Override this in a subclass to read a non-standard UniMorph file layout
(e.g. a language whose data ships extra columns).

Parameters
----------
lang_code : str
ISO 639-3 language code.

Returns
-------
pd.DataFrame
DataFrame with ``lemma``, ``form``, and ``features`` columns.
"""
return load_dataset(lang_code)

def fetch_items(
self,
query: str | None = None,
Expand Down Expand Up @@ -113,7 +131,7 @@ def fetch_items(
try:
# Load dataset for language (cached at instance level)
if lang_code not in self._datasets:
self._datasets[lang_code] = load_dataset(lang_code)
self._datasets[lang_code] = self._load_dataset(lang_code)

dataset = self._datasets[lang_code]

Expand Down
Loading