Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 20 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

## [0.9.0] - 2026-07-24

### Added

- `HuggingFaceLanguageModel` and `LanguageModelScorer` accept a `dtype` parameter
(default `"auto"`) selecting the precision causal language-model weights load in;
`LanguageModelScorer` threads it through to the adapter.
- `ListPartitioner` warns, once per offending expression or type, when a constraint
property expression cannot be evaluated or a constraint type is not enforced during
assignment, so a misconfigured constraint no longer fails silently.

### Changed

- `ListConstraint` subclasses now default `constraint_type` to their own discriminator
value, so it no longer has to be restated when constructing a constraint.
- Causal language models load in the checkpoint's own dtype by default rather than being
forced to float32, halving the memory used by a half-precision checkpoint.

## [0.8.0] - 2026-07-13

### Added
Expand Down Expand Up @@ -647,7 +665,8 @@ guards as type-checkers.
- CI/CD: GitHub Actions for testing, docs, PyPI publishing
- Read the Docs integration

[Unreleased]: https://github.com/FACTSlab/bead/compare/v0.8.0...HEAD
[Unreleased]: https://github.com/FACTSlab/bead/compare/v0.9.0...HEAD
[0.9.0]: https://github.com/FACTSlab/bead/compare/v0.8.0...v0.9.0
[0.8.0]: https://github.com/FACTSlab/bead/compare/v0.7.0...v0.8.0
[0.7.0]: https://github.com/FACTSlab/bead/compare/v0.6.0...v0.7.0
[0.6.0]: https://github.com/FACTSlab/bead/compare/v0.5.0...v0.6.0
Expand Down
2 changes: 1 addition & 1 deletion bead/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,6 @@

from __future__ import annotations

__version__ = "0.7.0"
__version__ = "0.9.0"
__author__ = "Aaron Steven White"
__email__ = "aaron.white@rochester.edu"
10 changes: 9 additions & 1 deletion bead/items/adapters/huggingface.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,10 @@ class HuggingFaceLanguageModel(HuggingFaceAdapterMixin, ModelAdapter):
Device to run model on. Falls back to CPU if device unavailable.
model_version : str
Version string for cache tracking.
dtype : str
Torch dtype to load the weights in, such as ``"bfloat16"``. Defaults to
``"auto"``, which keeps the dtype the checkpoint was saved in; loading a
half-precision checkpoint as float32 would double its memory.

Examples
--------
Expand All @@ -77,17 +81,21 @@ def __init__(
cache: ModelOutputCache,
device: DeviceType = "cpu",
model_version: str = "unknown",
dtype: str = "auto",
) -> None:
super().__init__(model_name, cache, model_version)
self.device = self._validate_device(device)
self.dtype = dtype
self._model: PreTrainedModel | None = None
self._tokenizer: PreTrainedTokenizerBase | None = None

def _load_model(self) -> None:
"""Load model and tokenizer lazily on first use."""
if self._model is None:
logger.info(f"Loading causal LM: {self.model_name}")
self._model = AutoModelForCausalLM.from_pretrained(self.model_name)
self._model = AutoModelForCausalLM.from_pretrained(
self.model_name, dtype=self.dtype
)
self._model.to(self.device)
self._model.eval()

Expand Down
6 changes: 6 additions & 0 deletions bead/items/scoring.py
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,9 @@ class LanguageModelScorer(ItemScorer):
Key in item.rendered_elements to use as text (default: "text").
model_version : str
Version string for cache tracking.
dtype : str
Torch dtype to load the weights in, such as ``"bfloat16"``. Defaults to
``"auto"``, keeping the dtype the checkpoint was saved in.

Examples
--------
Expand All @@ -155,12 +158,14 @@ def __init__(
device: str = "cpu",
text_key: str = "text",
model_version: str = "unknown",
dtype: str = "auto",
) -> None:
self.model_name = model_name
self.cache_dir = Path(cache_dir) if cache_dir else None
self.device = device
self.text_key = text_key
self.model_version = model_version
self.dtype = dtype

# lazy loading of model and cache
self._model: HuggingFaceLanguageModel | None = None
Expand Down Expand Up @@ -193,6 +198,7 @@ def model(self) -> HuggingFaceLanguageModel:
cache=self._cache,
device=self.device, # type: ignore[arg-type]
model_version=self.model_version,
dtype=self.dtype,
)

return self._model
Expand Down
24 changes: 12 additions & 12 deletions bead/lists/constraints.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ class UniquenessConstraint(ListConstraint):
Higher values are weighted more heavily during partitioning.
"""

constraint_type: typing.Literal["uniqueness"]
constraint_type: typing.Literal["uniqueness"] = "uniqueness"
property_expression: str
context: dict[str, ContextValue] = dx.field(default_factory=dict)
allow_null: bool = False
Expand Down Expand Up @@ -111,7 +111,7 @@ class ConditionalUniquenessConstraint(ListConstraint):
Constraint priority.
"""

constraint_type: typing.Literal["conditional_uniqueness"]
constraint_type: typing.Literal["conditional_uniqueness"] = "conditional_uniqueness"
property_expression: str
condition_expression: str
context: dict[str, ContextValue] = dx.field(default_factory=dict)
Expand Down Expand Up @@ -140,7 +140,7 @@ class BalanceConstraint(ListConstraint):
Constraint priority.
"""

constraint_type: typing.Literal["balance"]
constraint_type: typing.Literal["balance"] = "balance"
property_expression: str
context: dict[str, ContextValue] = dx.field(default_factory=dict)
target_counts: dict[str, int] | None = None
Expand Down Expand Up @@ -191,7 +191,7 @@ class QuantileConstraint(ListConstraint):
Constraint priority.
"""

constraint_type: typing.Literal["quantile"]
constraint_type: typing.Literal["quantile"] = "quantile"
property_expression: str
context: dict[str, ContextValue] = dx.field(default_factory=dict)
n_quantiles: int = 5
Expand Down Expand Up @@ -231,7 +231,7 @@ class GroupedQuantileConstraint(ListConstraint):
Constraint priority.
"""

constraint_type: typing.Literal["grouped_quantile"]
constraint_type: typing.Literal["grouped_quantile"] = "grouped_quantile"
property_expression: str
group_by_expression: str
context: dict[str, ContextValue] = dx.field(default_factory=dict)
Expand Down Expand Up @@ -468,7 +468,7 @@ class DiversityConstraint(ListConstraint):
Constraint priority.
"""

constraint_type: typing.Literal["diversity"]
constraint_type: typing.Literal["diversity"] = "diversity"
property_expression: str
min_unique_values: int
context: dict[str, ContextValue] = dx.field(default_factory=dict)
Expand Down Expand Up @@ -505,7 +505,7 @@ class SizeConstraint(ListConstraint):
Constraint priority.
"""

constraint_type: typing.Literal["size"]
constraint_type: typing.Literal["size"] = "size"
min_size: int | None = None
max_size: int | None = None
exact_size: int | None = None
Expand Down Expand Up @@ -580,7 +580,7 @@ class OrderingConstraint(ListConstraint):
Constraint priority (unused for static partitioning).
"""

constraint_type: typing.Literal["ordering"]
constraint_type: typing.Literal["ordering"] = "ordering"
precedence_pairs: tuple[dx.Embed[OrderingPair], ...] = ()
no_adjacent_property: str | None = None
block_by_property: str | None = None
Expand Down Expand Up @@ -637,7 +637,7 @@ class BatchCoverageConstraint(BatchConstraint):
Constraint priority.
"""

constraint_type: typing.Literal["coverage"]
constraint_type: typing.Literal["coverage"] = "coverage"
property_expression: str
context: dict[str, ContextValue] = dx.field(default_factory=dict)
target_values: tuple[str | int | float, ...] | None = None
Expand Down Expand Up @@ -673,7 +673,7 @@ class BatchBalanceConstraint(BatchConstraint):
Constraint priority.
"""

constraint_type: typing.Literal["balance"]
constraint_type: typing.Literal["balance"] = "balance"
property_expression: str
target_distribution: dict[str, float]
context: dict[str, ContextValue] = dx.field(default_factory=dict)
Expand Down Expand Up @@ -724,7 +724,7 @@ class BatchDiversityConstraint(BatchConstraint):
Constraint priority.
"""

constraint_type: typing.Literal["diversity"]
constraint_type: typing.Literal["diversity"] = "diversity"
property_expression: str
max_lists_per_value: int
context: dict[str, ContextValue] = dx.field(default_factory=dict)
Expand Down Expand Up @@ -757,7 +757,7 @@ class BatchMinOccurrenceConstraint(BatchConstraint):
Constraint priority.
"""

constraint_type: typing.Literal["min_occurrence"]
constraint_type: typing.Literal["min_occurrence"] = "min_occurrence"
property_expression: str
min_occurrences: int
context: dict[str, ContextValue] = dx.field(default_factory=dict)
Expand Down
63 changes: 59 additions & 4 deletions bead/lists/partitioner.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

from __future__ import annotations

import logging
from collections import Counter, defaultdict
from collections.abc import Callable, Hashable
from typing import Any
Expand Down Expand Up @@ -43,6 +44,8 @@
# without dict-invariance noise.
type BalanceMetrics = dict[str, "MetadataValue"]

logger = logging.getLogger(__name__)


class ListPartitioner:
"""Partitions items into balanced experimental lists.
Expand Down Expand Up @@ -80,6 +83,8 @@ def __init__(self, random_seed: int | None = None) -> None:
self.random_seed = random_seed
self._rng = np.random.default_rng(random_seed)
self.dsl_evaluator = DSLEvaluator()
self._unresolved_expressions: set[str] = set()
self._unenforced_types: set[str] = set()

def partition(
self,
Expand Down Expand Up @@ -448,6 +453,8 @@ def _count_violations(
elif isinstance(constraint, SizeConstraint):
if not self._check_size(exp_list, constraint):
is_violated = True
else:
self._warn_unenforced(str(constraint.constraint_type))

if is_violated:
priority = constraint.priority
Expand Down Expand Up @@ -574,6 +581,50 @@ def _check_size(self, exp_list: ExperimentList, constraint: SizeConstraint) -> b

return True

def _warn_unresolved(self, property_expression: str, error: Exception) -> None:
"""Report a property expression that cannot be evaluated.

Scoring skips items whose property cannot be read. Without a warning a
misconfigured constraint scores zero on every iteration and silently has
no effect, so each expression is reported once.

Parameters
----------
property_expression : str
The expression that failed to evaluate.
error : Exception
The failure raised while evaluating it.
"""
if property_expression in self._unresolved_expressions:
return
self._unresolved_expressions.add(property_expression)
logger.warning(
"Constraint property expression %r could not be evaluated (%s). "
"The constraint cannot be satisfied and is being ignored.",
property_expression,
error,
)

def _warn_unenforced(self, constraint_type: str) -> None:
"""Report a constraint type that assignment cannot enforce.

Unhandled types are stored on the list and serialised, so without a
warning a configured constraint appears active while having no effect.

Parameters
----------
constraint_type : str
Discriminator of the constraint that is not enforced.
"""
if constraint_type in self._unenforced_types:
return
self._unenforced_types.add(constraint_type)
logger.warning(
"List constraint type %r is not enforced during assignment. It is "
"recorded on the list but does not influence partitioning.",
constraint_type,
)

def _extract_property_value(
self,
item_id: UUID,
Expand Down Expand Up @@ -1034,7 +1085,8 @@ def _compute_batch_coverage_score(
metadata,
)
observed_values.add(value)
except Exception:
except Exception as error:
self._warn_unresolved(constraint.property_expression, error)
continue

# Compute coverage
Expand Down Expand Up @@ -1085,7 +1137,8 @@ def _compute_batch_balance_score(
)
counts[value] += 1
total += 1
except Exception:
except Exception as error:
self._warn_unresolved(constraint.property_expression, error)
continue

if total == 0:
Expand Down Expand Up @@ -1140,7 +1193,8 @@ def _compute_batch_diversity_score(
metadata,
)
value_to_lists[value].add(list_idx)
except Exception:
except Exception as error:
self._warn_unresolved(constraint.property_expression, error)
continue

if not value_to_lists:
Expand Down Expand Up @@ -1193,7 +1247,8 @@ def _compute_batch_min_occurrence_score(
metadata,
)
counts[value] += 1
except Exception:
except Exception as error:
self._warn_unresolved(constraint.property_expression, error)
continue

if not counts:
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "hatchling.build"

[project]
name = "bead"
version = "0.8.0"
version = "0.9.0"
description = "Lexicon and Template Collection Construction Pipeline for Acceptability and Inference Judgment Data"
authors = [{name = "Aaron Steven White", email = "aaron.white@rochester.edu"}]
readme = "README.md"
Expand Down
Loading
Loading