From fff120c375f99f82e1a06dbbaf154bdaa5d48b20 Mon Sep 17 00:00:00 2001 From: Aaron Steven White Date: Tue, 21 Jul 2026 11:17:31 -0400 Subject: [PATCH 1/6] fix(wordnet): retain adjective satellites and key synsets by offset+POS The converter discarded every adjective satellite. parse_data_file filtered on `synset.ss_type == pos` and the adjective pass runs with pos="a", so all 10,717 ss_type="s" synsets in data.adj -- roughly 59% of WordNet's adjectives -- were dropped while the pointers referencing them were still emitted, leaving ~10,720 dangling references for any consumer following adjective relations. Relaxing that filter alone would not have made those pointers resolve. WordNet writes "a" in the POS field of every pointer to an adjective, including the 10,717 similar-to pointers that target satellites; "s" appears only in ss_type. Offsets cannot carry identity either, being byte offsets into a per-POS data file: 333 of them name more than one synset. Identity therefore needs an offset paired with a POS that normalizes "s" to "a", which is what the new Synset.key and Pointer.target_key provide, and what the loader, search index, and relation traverser now key on. Previously the loader silently overwrote colliding synsets and WordNetSearch raised on them. Satellite sense enrichment needed the same treatment: it built a head-less sense key (lemma%5:LL:II::), but real satellite keys carry a head_word:head_id suffix that data.adj does not record, and the head-less prefix is ambiguous for 2,966 of 20,384 satellite keys. Lookups now key on the synset offset from index.sense, which is exact. Satellites index under both "s" and "a", so an adjective query returns heads and satellites together while ss_type still reports "s" on the record. Verified against WordNet 3.1: every raw synset now converts, with no key collisions and no dangling pointer targets. BREAKING CHANGE: WordNetLoader.synsets, lemma_index, and the four relation indices are keyed by SynsetKey ("00001740n") rather than by bare offset. get_synset still accepts a bare offset and returns None when one is ambiguous; pass pos or a canonical key to disambiguate. Closes #11 --- src/glazing/wordnet/converter.py | 92 +++-- src/glazing/wordnet/loader.py | 211 +++++++---- src/glazing/wordnet/models.py | 55 +++ src/glazing/wordnet/relations.py | 97 ++--- src/glazing/wordnet/search.py | 121 +++--- src/glazing/wordnet/types.py | 85 ++++- .../test_converter_loader_roundtrip.py | 8 +- tests/test_wordnet/test_converter.py | 6 +- tests/test_wordnet/test_loader.py | 26 +- tests/test_wordnet/test_relations.py | 84 ++--- tests/test_wordnet/test_satellites.py | 352 ++++++++++++++++++ 11 files changed, 887 insertions(+), 250 deletions(-) create mode 100644 tests/test_wordnet/test_satellites.py diff --git a/src/glazing/wordnet/converter.py b/src/glazing/wordnet/converter.py index 8a72c03..a7bd0f3 100644 --- a/src/glazing/wordnet/converter.py +++ b/src/glazing/wordnet/converter.py @@ -55,6 +55,7 @@ PointerSymbol, VerbFrameNumber, WordNetPOS, + normalize_pos, ) # Adjective syntactic-position markers appended to words in adjective data files, @@ -62,6 +63,35 @@ ADJ_MARKER_RE = re.compile(r"^(?P.+)\((?Pip|a|p)\)$") +def _split_sense_key(sense_key: str) -> tuple[str, int] | None: + """Split a sense key into its lemma and synset-type number. + + Sense keys have the form ``lemma%ss_type:lex_filenum:lex_id:head:head_id``, + where the trailing head fields are populated only for adjective satellites. + + Parameters + ---------- + sense_key : str + Sense key to split (e.g., ``"abandon%2:40:01::"``). + + Returns + ------- + tuple[str, int] | None + Lowercased lemma and synset-type number, or None if malformed. + """ + lemma, sep, rest = sense_key.partition("%") + if not sep or not lemma: + return None + fields = rest.split(":") + if len(fields) < 3: + return None + try: + ss_type_num = int(fields[0]) + except ValueError: + return None + return lemma.lower(), ss_type_num + + class WordNetConverter: """Parse WordNet database files into structured models. @@ -143,7 +173,10 @@ def parse_data_file(self, filepath: Path | str, pos: WordNetPOS) -> list[Synset] filepath : Path | str Path to WordNet data file (e.g., data.noun). pos : WordNetPOS - Part of speech for validation. + Part of speech for validation. ``"a"`` and ``"s"`` are equivalent: + ``data.adj`` holds both adjective heads and satellites, so either + value returns all adjective synsets, each retaining its own + ``ss_type``. Returns ------- @@ -176,7 +209,7 @@ def parse_data_file(self, filepath: Path | str, pos: WordNetPOS) -> list[Synset] try: synset = self._parse_data_line(line) - if synset and synset.ss_type == pos: + if synset and normalize_pos(synset.ss_type) == normalize_pos(pos): synsets.append(synset) except ValueError as e: # Log parsing error but continue @@ -490,8 +523,13 @@ def convert_wordnet_database( framestext = self.parse_verb_framestext(wordnet_dir / "verb.Framestext") sents = self.parse_verb_sentences(wordnet_dir / "sents.vrb") - # Build sense_key → (sense_number, tag_count) map from index.sense - sense_map: dict[str, tuple[int, int]] = {} + # Build (ss_type_num, offset, lemma) → (sense_number, tag_count) from + # index.sense. Keying on the synset offset rather than the sense key is + # what makes satellites work: a satellite's real sense key carries a + # head_word:head_id suffix the data file does not record, and the + # head-less prefix is ambiguous (2,966 collisions in WordNet 3.1). + sense_map: dict[tuple[int, str, str], tuple[int, int]] = {} + key_to_index: dict[str, tuple[int, str, str]] = {} sense_index_file = wordnet_dir / "index.sense" if sense_index_file.exists(): with sense_index_file.open("r", encoding="utf-8") as f: @@ -502,24 +540,31 @@ def convert_wordnet_database( parts = line.split() if len(parts) != 4: continue + split_key = _split_sense_key(parts[0]) + if split_key is None: + continue + lemma, ss_type_num = split_key try: - sk = parts[0] + offset = parts[1].zfill(8) sense_number = int(parts[2]) tag_count = int(parts[3]) - sense_map[sk] = (sense_number, tag_count) except ValueError: continue + index = (ss_type_num, offset, lemma) + sense_map[index] = (sense_number, tag_count) + key_to_index[parts[0]] = index - # Parse cntlist to enhance tag_count data + # Parse cntlist to enhance tag_count data. cntlist records sense keys + # only, so route each count through index.sense to reach its synset; + # a key absent from index.sense has no synset to attach to. cntlist = self.parse_cntlist(wordnet_dir / "cntlist") for sk, count in cntlist.items(): - if sk in sense_map: - sn, _ = sense_map[sk] - sense_map[sk] = (sn, count) - else: - sense_map[sk] = (0, count) + cnt_index = key_to_index.get(sk) + if cnt_index is not None: + sn, _ = sense_map[cnt_index] + sense_map[cnt_index] = (sn, count) - # ss_type to number mapping for sense key construction + # ss_type to number mapping for sense index lookup ss_type_num_map: dict[str, int] = { "n": 1, "v": 2, @@ -534,10 +579,9 @@ def convert_wordnet_database( # Enrich words with sense_number and tag_count for word in synset.words: - lemma_lower = word.lemma.lower() - sense_key = f"{lemma_lower}%{ss_num}:{synset.lex_filenum:02d}:{word.lex_id:02d}::" - if sense_key in sense_map: - sn, tc = sense_map[sense_key] + entry = sense_map.get((ss_num, synset.offset, word.lemma.lower())) + if entry is not None: + sn, tc = entry if sn > 0: word.sense_number = sn word.tag_count = tc @@ -882,16 +926,13 @@ def _parse_sense_line(self, line: str) -> Sense | None: tag_count = int(parts[3]) # Parse sense key components - key_parts = sense_key.split("%") - if len(key_parts) != 2: + split_key = _split_sense_key(sense_key) + if split_key is None: return None - lemma = key_parts[0] - rest = key_parts[1].split(":") - if len(rest) < 3: - return None + lemma, ss_type_num = split_key + rest = sense_key.partition("%")[2].split(":") - ss_type_num = int(rest[0]) lex_filenum = int(rest[1]) lex_id = int(rest[2]) @@ -986,7 +1027,8 @@ def parse_data_file(filepath: Path | str, pos: WordNetPOS) -> list[Synset]: filepath : Path | str Path to WordNet data file. pos : WordNetPOS - Part of speech code. + Part of speech code. ``"a"`` and ``"s"`` are equivalent and both return + all adjective synsets, heads and satellites alike. Returns ------- diff --git a/src/glazing/wordnet/loader.py b/src/glazing/wordnet/loader.py index 625f85b..36e9c70 100644 --- a/src/glazing/wordnet/loader.py +++ b/src/glazing/wordnet/loader.py @@ -32,6 +32,7 @@ import json from collections import defaultdict from pathlib import Path +from typing import cast from pydantic import ValidationError @@ -44,8 +45,11 @@ ) from glazing.wordnet.types import ( SenseKey, + SynsetKey, SynsetOffset, WordNetPOS, + make_synset_key, + normalize_pos, ) @@ -72,10 +76,13 @@ class WordNetLoader: Attributes ---------- - synsets : dict[SynsetOffset, Synset] - All loaded synsets indexed by offset. - lemma_index : dict[str, dict[WordNetPOS, list[SynsetOffset]]] - Index from lemmas to synset offsets by POS. + synsets : dict[SynsetKey, Synset] + All loaded synsets indexed by canonical key (offset plus normalized + POS, e.g. ``"00001740n"``). Offsets alone collide across parts of + speech and so cannot key this mapping. + lemma_index : dict[str, dict[WordNetPOS, list[SynsetKey]]] + Index from lemmas to synset keys by POS. Adjective satellites appear + under both ``"s"`` and ``"a"``. sense_index : dict[SenseKey, Sense] Index from sense keys to sense objects. exceptions : dict[WordNetPOS, dict[str, list[str]]] @@ -85,8 +92,8 @@ class WordNetLoader: ------- load() Load all WordNet data from JSON Lines files. - get_synset(offset) - Get a synset by its offset. + get_synset(offset, pos) + Get a synset by its offset or canonical key. get_senses_by_lemma(lemma, pos) Get all senses for a lemma and optional POS. get_sense_by_key(sense_key) @@ -134,20 +141,23 @@ def __init__( self.lazy = lazy self.cache_size = cache_size - # Core data structures - self.synsets: dict[SynsetOffset, Synset] = {} - self.lemma_index: dict[str, dict[WordNetPOS, list[SynsetOffset]]] = defaultdict(dict) + # Core data structures, keyed by canonical synset key (offset + POS) + self.synsets: dict[SynsetKey, Synset] = {} + self.lemma_index: dict[str, dict[WordNetPOS, list[SynsetKey]]] = defaultdict(dict) self.sense_index: dict[SenseKey, Sense] = {} self.exceptions: dict[WordNetPOS, dict[str, list[str]]] = {} # Relation indices for efficient traversal - self.hypernym_index: dict[SynsetOffset, list[SynsetOffset]] = defaultdict(list) - self.hyponym_index: dict[SynsetOffset, list[SynsetOffset]] = defaultdict(list) - self.meronym_index: dict[SynsetOffset, list[SynsetOffset]] = defaultdict(list) - self.holonym_index: dict[SynsetOffset, list[SynsetOffset]] = defaultdict(list) + self.hypernym_index: dict[SynsetKey, list[SynsetKey]] = defaultdict(list) + self.hyponym_index: dict[SynsetKey, list[SynsetKey]] = defaultdict(list) + self.meronym_index: dict[SynsetKey, list[SynsetKey]] = defaultdict(list) + self.holonym_index: dict[SynsetKey, list[SynsetKey]] = defaultdict(list) - # File index for lazy loading (offset -> byte position in file) - self._synset_file_index: dict[SynsetOffset, int] = {} + # Bare offset -> keys sharing it, for resolving unqualified lookups + self._keys_by_offset: dict[SynsetOffset, list[SynsetKey]] = defaultdict(list) + + # File index for lazy loading (key -> byte position in file) + self._synset_file_index: dict[SynsetKey, int] = {} # Cache for lazy loading if lazy: @@ -212,7 +222,8 @@ def _load_all_synsets(self) -> None: try: data = json.loads(line) synset = Synset.model_validate(data) - self.synsets[synset.offset] = synset + self.synsets[synset.key] = synset + self._keys_by_offset[synset.offset].append(synset.key) except (json.JSONDecodeError, ValidationError): continue @@ -234,35 +245,38 @@ def _build_file_index(self) -> None: try: data = json.loads(line) offset = data.get("offset") - if offset: - self._synset_file_index[offset] = byte_pos + ss_type = data.get("ss_type") + if offset and ss_type: + key = make_synset_key(offset, ss_type) + self._synset_file_index[key] = byte_pos + self._keys_by_offset[offset].append(key) except json.JSONDecodeError: pass - def _load_synset_lazy(self, offset: SynsetOffset) -> Synset | None: + def _load_synset_lazy(self, key: SynsetKey) -> Synset | None: """Load a single synset on demand. Parameters ---------- - offset : SynsetOffset - The synset offset to load. + key : SynsetKey + The canonical synset key to load. Returns ------- Synset | None The loaded synset or None if not found. """ - if offset in self.synsets: - return self.synsets[offset] + if key in self.synsets: + return self.synsets[key] # Check cache first if self._cache is not None: - cached = self._cache.get(offset) + cached = self._cache.get(key) if cached is not None: return cached # Load from file using byte offset - byte_pos = self._synset_file_index.get(offset) + byte_pos = self._synset_file_index.get(key) if byte_pos is None: return None @@ -275,22 +289,28 @@ def _load_synset_lazy(self, offset: SynsetOffset) -> Synset | None: # Cache it if self._cache is not None: - self._cache.put(offset, synset) + self._cache.put(key, synset) return synset except (json.JSONDecodeError, ValidationError): return None def _build_lemma_index(self) -> None: - """Build lemma→synset index from loaded synset data.""" + """Build lemma→synset index from loaded synset data. + + Adjective satellites are indexed under both ``"s"`` and the normalized + ``"a"``, so a query for adjectives returns heads and satellites together + while a query for ``"s"`` still isolates satellites. + """ for synset in self.synsets.values(): - pos = synset.ss_type + positions: set[WordNetPOS] = {synset.ss_type, normalize_pos(synset.ss_type)} for word in synset.words: lemma = word.lemma.lower() - if pos not in self.lemma_index[lemma]: - self.lemma_index[lemma][pos] = [] - if synset.offset not in self.lemma_index[lemma][pos]: - self.lemma_index[lemma][pos].append(synset.offset) + for pos in positions: + if pos not in self.lemma_index[lemma]: + self.lemma_index[lemma][pos] = [] + if synset.key not in self.lemma_index[lemma][pos]: + self.lemma_index[lemma][pos].append(synset.key) def _load_sense_index(self) -> None: """Load sense index from supplementary JSONL file.""" @@ -335,54 +355,106 @@ def _load_exceptions(self) -> None: continue def _build_relation_indices(self) -> None: - """Build relation indices for efficient traversal.""" + """Build relation indices for efficient traversal. + + Both sides are keyed canonically, so a pointer resolves to the synset it + actually names rather than to whichever synset happens to share its + offset in another part of speech. + """ for synset in self.synsets.values(): + source = synset.key for pointer in synset.pointers: + target = pointer.target_key + # Hypernym/hyponym relations if pointer.symbol == "@": - if pointer.offset not in self.hypernym_index[synset.offset]: - self.hypernym_index[synset.offset].append(pointer.offset) - if synset.offset not in self.hyponym_index[pointer.offset]: - self.hyponym_index[pointer.offset].append(synset.offset) + if target not in self.hypernym_index[source]: + self.hypernym_index[source].append(target) + if source not in self.hyponym_index[target]: + self.hyponym_index[target].append(source) elif pointer.symbol == "~": - if pointer.offset not in self.hyponym_index[synset.offset]: - self.hyponym_index[synset.offset].append(pointer.offset) - if synset.offset not in self.hypernym_index[pointer.offset]: - self.hypernym_index[pointer.offset].append(synset.offset) + if target not in self.hyponym_index[source]: + self.hyponym_index[source].append(target) + if source not in self.hypernym_index[target]: + self.hypernym_index[target].append(source) # Meronym/holonym relations elif pointer.symbol in ("%m", "%s", "%p"): - if pointer.offset not in self.meronym_index[synset.offset]: - self.meronym_index[synset.offset].append(pointer.offset) - if synset.offset not in self.holonym_index[pointer.offset]: - self.holonym_index[pointer.offset].append(synset.offset) + if target not in self.meronym_index[source]: + self.meronym_index[source].append(target) + if source not in self.holonym_index[target]: + self.holonym_index[target].append(source) elif pointer.symbol in ("#m", "#s", "#p"): - if pointer.offset not in self.holonym_index[synset.offset]: - self.holonym_index[synset.offset].append(pointer.offset) - if synset.offset not in self.meronym_index[pointer.offset]: - self.meronym_index[pointer.offset].append(synset.offset) + if target not in self.holonym_index[source]: + self.holonym_index[source].append(target) + if source not in self.meronym_index[target]: + self.meronym_index[target].append(source) - def get_synset(self, offset: SynsetOffset) -> Synset | None: - """Get a synset by its offset. + def get_synset( + self, offset: SynsetOffset | SynsetKey, pos: WordNetPOS | None = None + ) -> Synset | None: + """Get a synset by its offset or canonical key. Parameters ---------- - offset : SynsetOffset - The 8-digit synset offset. + offset : SynsetOffset | SynsetKey + An 8-digit synset offset, or a 9-character canonical key with the + part of speech appended (e.g., ``"00001740n"``). + pos : WordNetPOS | None, default=None + Part of speech qualifying `offset`. Ignored when `offset` is + already a canonical key. Returns ------- Synset | None - The synset or None if not found. + The synset, or None if not found or if a bare offset is ambiguous. + + Notes + ----- + Offsets are byte offsets into a per-POS data file, so 330 of them name + more than one synset in WordNet 3.1. An unqualified lookup on such an + offset returns None rather than choosing arbitrarily; pass `pos` or a + canonical key to disambiguate. Examples -------- >>> synset = loader.get_synset("02084442") >>> print(synset.gloss) + >>> loader.get_synset("00001740", "v") # qualified + >>> loader.get_synset("00001740v") # equivalent """ + key = self._resolve_key(offset, pos) + if key is None: + return None if self.lazy: - return self._load_synset_lazy(offset) - return self.synsets.get(offset) + return self._load_synset_lazy(key) + return self.synsets.get(key) + + def _resolve_key( + self, offset: SynsetOffset | SynsetKey, pos: WordNetPOS | None = None + ) -> SynsetKey | None: + """Resolve an offset, key, or offset plus POS to a canonical key. + + Parameters + ---------- + offset : SynsetOffset | SynsetKey + Bare 8-digit offset or 9-character canonical key. + pos : WordNetPOS | None, default=None + Part of speech qualifying a bare offset. + + Returns + ------- + SynsetKey | None + The canonical key, or None if a bare offset is ambiguous. + """ + if len(offset) == 9: + return make_synset_key(offset[:8], cast(WordNetPOS, offset[8])) + if pos is not None: + return make_synset_key(offset, pos) + keys = self._keys_by_offset.get(offset) + if keys is None or len(keys) != 1: + return None + return keys[0] def get_synsets_by_lemma(self, lemma: str, pos: WordNetPOS | None = None) -> list[Synset]: """Get all synsets containing a lemma. @@ -418,10 +490,15 @@ def get_synsets_by_lemma(self, lemma: str, pos: WordNetPOS | None = None) -> lis else: pos_tags = list(self.lemma_index[lemma_lower].keys()) - # Collect synsets from offset lists + # Collect synsets by key. Satellites are indexed under both "s" and + # "a", so track seen keys to avoid returning them twice. + seen: set[SynsetKey] = set() for pos_tag in pos_tags: - for offset in self.lemma_index[lemma_lower].get(pos_tag, []): - synset = self.get_synset(offset) + for key in self.lemma_index[lemma_lower].get(pos_tag, []): + if key in seen: + continue + seen.add(key) + synset = self.get_synset(key) if synset: synsets.append(synset) @@ -493,8 +570,8 @@ def get_hypernyms(self, synset: Synset) -> list[Synset]: List of hypernym synsets. """ hypernyms = [] - for offset in self.hypernym_index.get(synset.offset, []): - hypernym = self.get_synset(offset) + for key in self.hypernym_index.get(synset.key, []): + hypernym = self.get_synset(key) if hypernym: hypernyms.append(hypernym) return hypernyms @@ -513,8 +590,8 @@ def get_hyponyms(self, synset: Synset) -> list[Synset]: List of hyponym synsets. """ hyponyms = [] - for offset in self.hyponym_index.get(synset.offset, []): - hyponym = self.get_synset(offset) + for key in self.hyponym_index.get(synset.key, []): + hyponym = self.get_synset(key) if hyponym: hyponyms.append(hyponym) return hyponyms @@ -533,8 +610,8 @@ def get_meronyms(self, synset: Synset) -> list[Synset]: List of meronym synsets. """ meronyms = [] - for offset in self.meronym_index.get(synset.offset, []): - meronym = self.get_synset(offset) + for key in self.meronym_index.get(synset.key, []): + meronym = self.get_synset(key) if meronym: meronyms.append(meronym) return meronyms @@ -553,8 +630,8 @@ def get_holonyms(self, synset: Synset) -> list[Synset]: List of holonym synsets. """ holonyms = [] - for offset in self.holonym_index.get(synset.offset, []): - holonym = self.get_synset(offset) + for key in self.holonym_index.get(synset.key, []): + holonym = self.get_synset(key) if holonym: holonyms.append(holonym) return holonyms diff --git a/src/glazing/wordnet/models.py b/src/glazing/wordnet/models.py index ccf861f..66cba7b 100644 --- a/src/glazing/wordnet/models.py +++ b/src/glazing/wordnet/models.py @@ -51,10 +51,12 @@ PointerSymbol, SenseKey, SenseNumber, + SynsetKey, SynsetOffset, TagCount, VerbFrameNumber, WordNetPOS, + make_synset_key, ) @@ -136,6 +138,8 @@ class Pointer(GlazingBaseModel): Source word number (0 = entire synset). target : int Target word number (0 = entire synset). + target_key : SynsetKey + Canonical key of the targeted synset. Methods ------- @@ -155,6 +159,8 @@ class Pointer(GlazingBaseModel): ... ) >>> pointer.is_semantic() True + >>> pointer.target_key + '00002084n' """ symbol: PointerSymbol = Field(description="Relation type symbol") @@ -183,6 +189,21 @@ def is_semantic(self) -> bool: """ return self.source == 0 and self.target == 0 + @property + def target_key(self) -> SynsetKey: + """Canonical key of the synset this pointer targets. + + Matches `Synset.key` of the target, including when the target is an + adjective satellite: WordNet records ``"a"`` in the POS field of every + pointer to an adjective, so normalization makes the two agree. + + Returns + ------- + SynsetKey + Target offset plus normalized POS. + """ + return make_synset_key(self.offset, self.pos) + class VerbFrame(GlazingBaseModel): """Syntactic frame for a verb. @@ -262,6 +283,8 @@ class Synset(GlazingBaseModel): Verb frames (verbs only). gloss : str Definition and examples. + key : SynsetKey + Canonical identifier: offset plus normalized POS. Methods ------- @@ -285,6 +308,22 @@ class Synset(GlazingBaseModel): ... ) >>> synset.get_lemmas() ['dog'] + >>> synset.key + '00001740n' + + An adjective satellite keys under "a", matching the pointers that target it. + + >>> satellite = Synset( + ... offset="00014377", + ... lex_filenum=0, + ... lex_filename="adj.all", + ... ss_type="s", + ... words=[Word(lemma="abounding", lex_id=0)], + ... pointers=[], + ... gloss="existing in abundance" + ... ) + >>> satellite.key + '00014377a' """ offset: SynsetOffset = Field(description="8-digit synset identifier") @@ -296,6 +335,22 @@ class Synset(GlazingBaseModel): frames: list[VerbFrame] | None = Field(None, description="Verb frames (verbs only)") gloss: str = Field(description="Definition and examples") + @property + def key(self) -> SynsetKey: + """Canonical identifier for this synset. + + Offsets alone do not identify a synset - they are byte offsets into a + per-POS data file and collide across files. Pairing the offset with the + normalized POS gives a database-wide unique key that also matches the + `Pointer.target_key` of any pointer referencing this synset. + + Returns + ------- + SynsetKey + Offset plus normalized POS; satellites (``"s"``) key under ``"a"``. + """ + return make_synset_key(self.offset, self.ss_type) + def get_lemmas(self) -> list[str]: """Get all lemmas in the synset. diff --git a/src/glazing/wordnet/relations.py b/src/glazing/wordnet/relations.py index 78e1a37..a33f0f0 100644 --- a/src/glazing/wordnet/relations.py +++ b/src/glazing/wordnet/relations.py @@ -11,7 +11,7 @@ from typing import TYPE_CHECKING from glazing.wordnet.models import Synset -from glazing.wordnet.types import SynsetOffset +from glazing.wordnet.types import SynsetKey, normalize_pos if TYPE_CHECKING: from glazing.wordnet.models import Pointer @@ -26,12 +26,12 @@ class WordNetRelationTraverser: Parameters ---------- - synsets : dict[SynsetOffset, Synset] - Mapping from synset offset to synset object. + synsets : dict[SynsetKey, Synset] + Mapping from canonical synset key to synset object. Attributes ---------- - _synsets : dict[SynsetOffset, Synset] + _synsets : dict[SynsetKey, Synset] Internal synset storage. Methods @@ -73,7 +73,7 @@ class WordNetRelationTraverser: >>> similarity = traverser.calculate_path_similarity(dog_synset, cat_synset) """ - def __init__(self, synsets: dict[SynsetOffset, Synset]) -> None: + def __init__(self, synsets: dict[SynsetKey, Synset]) -> None: """Initialize relation traverser with synset data.""" self._synsets = synsets @@ -97,14 +97,14 @@ def get_hypernyms(self, synset: Synset, direct_only: bool = True) -> list[Synset hypernyms = [] for pointer in synset.pointers: if pointer.symbol == "@" and pointer.is_semantic(): - hypernym = self._synsets.get(pointer.offset) + hypernym = self._synsets.get(pointer.target_key) if hypernym: hypernyms.append(hypernym) return hypernyms # Get all hypernyms recursively - all_hypernym_offsets = set() + all_hypernym_keys = set() queue = deque([synset]) - visited = {synset.offset} + visited = {synset.key} while queue: current = queue.popleft() @@ -112,15 +112,15 @@ def get_hypernyms(self, synset: Synset, direct_only: bool = True) -> list[Synset if ( pointer.symbol == "@" and pointer.is_semantic() - and pointer.offset not in visited - and (hypernym := self._synsets.get(pointer.offset)) + and pointer.target_key not in visited + and (hypernym := self._synsets.get(pointer.target_key)) ): - all_hypernym_offsets.add(hypernym.offset) + all_hypernym_keys.add(hypernym.key) queue.append(hypernym) - visited.add(pointer.offset) + visited.add(pointer.target_key) - hypernyms = [self._synsets[offset] for offset in all_hypernym_offsets] - return sorted(hypernyms, key=lambda s: s.offset) + hypernyms = [self._synsets[key] for key in all_hypernym_keys] + return sorted(hypernyms, key=lambda s: s.key) def get_hyponyms(self, synset: Synset, direct_only: bool = True) -> list[Synset]: """Get hyponyms (inverse of hypernym) of a synset. @@ -142,14 +142,14 @@ def get_hyponyms(self, synset: Synset, direct_only: bool = True) -> list[Synset] hyponyms = [] for pointer in synset.pointers: if pointer.symbol == "~" and pointer.is_semantic(): - hyponym = self._synsets.get(pointer.offset) + hyponym = self._synsets.get(pointer.target_key) if hyponym: hyponyms.append(hyponym) return hyponyms # Get all hyponyms recursively - all_hyponym_offsets = set() + all_hyponym_keys = set() queue = deque([synset]) - visited = {synset.offset} + visited = {synset.key} while queue: current = queue.popleft() @@ -157,15 +157,15 @@ def get_hyponyms(self, synset: Synset, direct_only: bool = True) -> list[Synset] if ( pointer.symbol == "~" and pointer.is_semantic() - and pointer.offset not in visited - and (hyponym := self._synsets.get(pointer.offset)) + and pointer.target_key not in visited + and (hyponym := self._synsets.get(pointer.target_key)) ): - all_hyponym_offsets.add(hyponym.offset) + all_hyponym_keys.add(hyponym.key) queue.append(hyponym) - visited.add(pointer.offset) + visited.add(pointer.target_key) - hyponyms = [self._synsets[offset] for offset in all_hyponym_offsets] - return sorted(hyponyms, key=lambda s: s.offset) + hyponyms = [self._synsets[key] for key in all_hyponym_keys] + return sorted(hyponyms, key=lambda s: s.key) def get_hypernym_paths(self, synset: Synset, max_depth: int = 10) -> list[list[Synset]]: """Get all paths to root hypernyms. @@ -197,7 +197,7 @@ def traverse(current: Synset, path: list[Synset], depth: int) -> None: else: for hypernym in hypernyms: # Avoid cycles - if hypernym.offset not in {s.offset for s in path}: + if hypernym.key not in {s.key for s in path}: traverse(hypernym, [*path, hypernym], depth + 1) traverse(synset, [synset], 0) @@ -219,16 +219,16 @@ def get_common_hypernyms(self, synset1: Synset, synset2: Synset) -> list[Synset] Common hypernym synsets. """ hypernyms1 = self.get_hypernyms(synset1, direct_only=False) - hypernyms1_offsets = {h.offset for h in hypernyms1} - hypernyms1_offsets.add(synset1.offset) # Include the synset itself + hypernyms1_keys = {h.key for h in hypernyms1} + hypernyms1_keys.add(synset1.key) # Include the synset itself hypernyms2 = self.get_hypernyms(synset2, direct_only=False) - hypernyms2_offsets = {h.offset for h in hypernyms2} - hypernyms2_offsets.add(synset2.offset) # Include the synset itself + hypernyms2_keys = {h.key for h in hypernyms2} + hypernyms2_keys.add(synset2.key) # Include the synset itself - common_offsets = hypernyms1_offsets & hypernyms2_offsets - common = [self._synsets[offset] for offset in common_offsets if offset in self._synsets] - return sorted(common, key=lambda s: s.offset) + common_keys = hypernyms1_keys & hypernyms2_keys + common = [self._synsets[key] for key in common_keys if key in self._synsets] + return sorted(common, key=lambda s: s.key) def get_meronyms(self, synset: Synset, meronym_type: str | None = None) -> list[Synset]: """Get meronyms (part-of relations) of a synset. @@ -254,7 +254,7 @@ def get_meronyms(self, synset: Synset, meronym_type: str | None = None) -> list[ for pointer in synset.pointers: if pointer.symbol in symbols and pointer.is_semantic(): - meronym = self._synsets.get(pointer.offset) + meronym = self._synsets.get(pointer.target_key) if meronym: meronyms.append(meronym) @@ -284,7 +284,7 @@ def get_holonyms(self, synset: Synset, holonym_type: str | None = None) -> list[ for pointer in synset.pointers: if pointer.symbol in symbols and pointer.is_semantic(): - holonym = self._synsets.get(pointer.offset) + holonym = self._synsets.get(pointer.target_key) if holonym: holonyms.append(holonym) @@ -310,7 +310,7 @@ def get_entailments(self, synset: Synset) -> list[Synset]: for pointer in synset.pointers: if pointer.symbol == "*" and pointer.is_semantic(): - entailment = self._synsets.get(pointer.offset) + entailment = self._synsets.get(pointer.target_key) if entailment: entailments.append(entailment) @@ -336,7 +336,7 @@ def get_causes(self, synset: Synset) -> list[Synset]: for pointer in synset.pointers: if pointer.symbol == ">" and pointer.is_semantic(): - cause = self._synsets.get(pointer.offset) + cause = self._synsets.get(pointer.target_key) if cause: causes.append(cause) @@ -362,7 +362,7 @@ def get_similar_to(self, synset: Synset) -> list[Synset]: for pointer in synset.pointers: if pointer.symbol == "&" and pointer.is_semantic(): - sim = self._synsets.get(pointer.offset) + sim = self._synsets.get(pointer.target_key) if sim: similar.append(sim) @@ -385,7 +385,7 @@ def get_also_see(self, synset: Synset) -> list[Synset]: for pointer in synset.pointers: if pointer.symbol == "^" and pointer.is_semantic(): - related = self._synsets.get(pointer.offset) + related = self._synsets.get(pointer.target_key) if related: also_see.append(related) @@ -434,7 +434,7 @@ def _extract_antonym_pairs( list[tuple[Synset, str]] Antonym pairs. """ - ant_synset = self._synsets.get(pointer.offset) + ant_synset = self._synsets.get(pointer.target_key) if not ant_synset: return [] @@ -572,7 +572,7 @@ def _extract_derivation_pairs( list[tuple[Synset, str]] Derivation pairs. """ - der_synset = self._synsets.get(pointer.offset) + der_synset = self._synsets.get(pointer.target_key) if not der_synset: return [] @@ -648,12 +648,13 @@ def calculate_path_similarity(self, synset1: Synset, synset2: Synset) -> float: Similarity score between 0 and 1. Returns 0 if synsets are not connected. """ - # Must be same POS - if synset1.ss_type != synset2.ss_type: + # Must be same POS. Adjective heads and satellites count as one POS: + # they share a hierarchy through similar-to pointers. + if normalize_pos(synset1.ss_type) != normalize_pos(synset2.ss_type): return 0.0 # Same synset has similarity 1 - if synset1.offset == synset2.offset: + if synset1.key == synset2.key: return 1.0 # Find shortest path through common hypernyms @@ -695,23 +696,23 @@ def _calculate_min_distance(self, start: Synset, target: Synset) -> int: int Minimum distance, or -1 if not connected. """ - if start.offset == target.offset: + if start.key == target.key: return 0 # BFS to find shortest path queue = deque([(start, 0)]) - visited = {start.offset} + visited = {start.key} while queue: current, distance = queue.popleft() # Check hypernyms for hypernym in self.get_hypernyms(current, direct_only=True): - if hypernym.offset == target.offset: + if hypernym.key == target.key: return distance + 1 - if hypernym.offset not in visited: - visited.add(hypernym.offset) + if hypernym.key not in visited: + visited.add(hypernym.key) queue.append((hypernym, distance + 1)) return -1 @@ -755,7 +756,7 @@ def get_verb_groups(self, synset: Synset) -> list[Synset]: for pointer in synset.pointers: if pointer.symbol == "$" and pointer.is_semantic(): - group = self._synsets.get(pointer.offset) + group = self._synsets.get(pointer.target_key) if group: groups.append(group) diff --git a/src/glazing/wordnet/search.py b/src/glazing/wordnet/search.py index 1b9cf0d..452ca11 100644 --- a/src/glazing/wordnet/search.py +++ b/src/glazing/wordnet/search.py @@ -8,8 +8,9 @@ from __future__ import annotations import re -from collections import defaultdict +from collections import Counter, defaultdict from pathlib import Path +from typing import cast from glazing.syntax.models import UnifiedSyntaxPattern from glazing.syntax.parser import SyntaxParser @@ -18,9 +19,12 @@ from glazing.wordnet.types import ( LexFileName, SenseKey, + SynsetKey, SynsetOffset, VerbFrameNumber, WordNetPOS, + make_synset_key, + normalize_pos, ) @@ -40,14 +44,15 @@ class WordNetSearch: Attributes ---------- - _synsets : dict[SynsetOffset, Synset] - Mapping from synset offset to synset object. - _synsets_by_lemma : dict[str, dict[WordNetPOS, set[SynsetOffset]]] - Mapping from lemma and POS to synset offsets. + _synsets : dict[SynsetKey, Synset] + Mapping from canonical synset key to synset object. + _synsets_by_lemma : dict[str, dict[WordNetPOS, set[SynsetKey]]] + Mapping from lemma and POS to synset keys. Adjective satellites appear + under both ``"s"`` and ``"a"``. _senses : dict[SenseKey, Sense] Mapping from sense key to sense object. - _synsets_by_domain : dict[LexFileName, set[SynsetOffset]] - Mapping from lexical file name to synset offsets. + _synsets_by_domain : dict[LexFileName, set[SynsetKey]] + Mapping from lexical file name to synset keys. Methods ------- @@ -56,7 +61,7 @@ class WordNetSearch: add_sense(sense) Add a sense to the search index. by_offset(offset, pos) - Find synset by offset and POS. + Find synset by offset or canonical key, optionally filtered by POS. by_lemma(lemma, pos) Find synsets containing a lemma. by_sense_key(sense_key) @@ -82,13 +87,13 @@ def __init__( self, synsets: list[Synset] | None = None, senses: list[Sense] | None = None ) -> None: """Initialize WordNet search with optional initial data.""" - self._synsets: dict[SynsetOffset, Synset] = {} - self._synsets_by_lemma: dict[str, dict[WordNetPOS, set[SynsetOffset]]] = defaultdict( + self._synsets: dict[SynsetKey, Synset] = {} + self._synsets_by_lemma: dict[str, dict[WordNetPOS, set[SynsetKey]]] = defaultdict( lambda: defaultdict(set) ) self._senses: dict[SenseKey, Sense] = {} - self._synsets_by_domain: dict[LexFileName, set[SynsetOffset]] = defaultdict(set) - self._synsets_by_pos: dict[WordNetPOS, set[SynsetOffset]] = defaultdict(set) + self._synsets_by_domain: dict[LexFileName, set[SynsetKey]] = defaultdict(set) + self._synsets_by_pos: dict[WordNetPOS, set[SynsetKey]] = defaultdict(set) if synsets: for synset in synsets: @@ -109,21 +114,24 @@ def add_synset(self, synset: Synset) -> None: Raises ------ ValueError - If synset with same offset already exists. + If a synset with the same canonical key already exists. """ - if synset.offset in self._synsets: - msg = f"Synset with offset {synset.offset} already exists" + if synset.key in self._synsets: + msg = f"Synset with key {synset.key} already exists" raise ValueError(msg) - self._synsets[synset.offset] = synset - self._synsets_by_pos[synset.ss_type].add(synset.offset) + self._synsets[synset.key] = synset - # Index by lemma - for word in synset.words: - self._synsets_by_lemma[word.lemma][synset.ss_type].add(synset.offset) + # Satellites index under both "s" and "a" so that either query finds + # them, while ss_type on the record still reports "s". + positions: set[WordNetPOS] = {synset.ss_type, normalize_pos(synset.ss_type)} + for pos in positions: + self._synsets_by_pos[pos].add(synset.key) + for word in synset.words: + self._synsets_by_lemma[word.lemma][pos].add(synset.key) # Index by domain (lexical file) - self._synsets_by_domain[synset.lex_filename].add(synset.offset) + self._synsets_by_domain[synset.lex_filename].add(synset.key) def add_sense(self, sense: Sense) -> None: """Add a sense to the search index. @@ -144,23 +152,42 @@ def add_sense(self, sense: Sense) -> None: self._senses[sense.sense_key] = sense - def by_offset(self, offset: SynsetOffset, pos: WordNetPOS | None = None) -> Synset | None: - """Find synset by offset and optionally POS. + def by_offset( + self, offset: SynsetOffset | SynsetKey, pos: WordNetPOS | None = None + ) -> Synset | None: + """Find synset by offset or canonical key, optionally filtered by POS. Parameters ---------- - offset : SynsetOffset - Synset offset (8-digit string). + offset : SynsetOffset | SynsetKey + Synset offset (8-digit string) or canonical key (9 characters). pos : WordNetPOS | None - Part of speech to filter by. + Part of speech to filter by. Passing ``"a"`` matches adjective + heads and satellites alike; ``"s"`` matches satellites only. Returns ------- Synset | None Synset if found, None otherwise. + + Notes + ----- + An unqualified offset shared by synsets in more than one part of speech + is ambiguous and yields None; pass `pos` or a canonical key instead. """ - synset = self._synsets.get(offset) - if synset and pos is not None and synset.ss_type != pos: + if len(offset) == 9: + key = make_synset_key(offset[:8], cast(WordNetPOS, offset[8])) + elif pos is not None: + key = make_synset_key(offset, pos) + else: + candidates = [k for k in self._synsets if k[:8] == offset] + if len(candidates) != 1: + return None + key = candidates[0] + + synset = self._synsets.get(key) + # "s" is not encoded in the key, so filter satellites explicitly. + if synset and pos == "s" and synset.ss_type != "s": return None return synset @@ -182,16 +209,17 @@ def by_lemma(self, lemma: str, pos: WordNetPOS | None = None) -> list[Synset]: # Normalize lemma lemma = lemma.lower().replace(" ", "_") - synsets = [] if pos is not None: # Search specific POS - offsets = self._synsets_by_lemma.get(lemma, {}).get(pos, set()) - synsets = [self._synsets[offset] for offset in offsets] + keys = self._synsets_by_lemma.get(lemma, {}).get(pos, set()) else: - # Search all POS - for pos_offsets in self._synsets_by_lemma.get(lemma, {}).values(): - synsets.extend([self._synsets[offset] for offset in pos_offsets]) + # Search all POS. Satellites are indexed under both "s" and "a", + # so take the union rather than concatenating per-POS results. + keys = set() + for pos_keys in self._synsets_by_lemma.get(lemma, {}).values(): + keys |= pos_keys + synsets = [self._synsets[key] for key in keys] return sorted(synsets, key=lambda s: s.offset) def by_sense_key(self, sense_key: SenseKey) -> Synset | None: @@ -209,7 +237,7 @@ def by_sense_key(self, sense_key: SenseKey) -> Synset | None: """ sense = self._senses.get(sense_key) if sense: - return self._synsets.get(sense.synset_offset) + return self._synsets.get(make_synset_key(sense.synset_offset, sense.ss_type)) return None def by_pattern( @@ -239,21 +267,21 @@ def by_pattern( flags = 0 if case_sensitive else re.IGNORECASE regex = re.compile(pattern, flags) - matching_offsets = set() + matching_keys = set() # Determine which synsets to search if pos is not None: - synsets_to_search = [self._synsets[offset] for offset in self._synsets_by_pos[pos]] + synsets_to_search = [self._synsets[key] for key in self._synsets_by_pos[pos]] else: synsets_to_search = list(self._synsets.values()) for synset in synsets_to_search: for word in synset.words: if regex.search(word.lemma): - matching_offsets.add(synset.offset) + matching_keys.add(synset.key) break - synsets = [self._synsets[offset] for offset in matching_offsets] + synsets = [self._synsets[key] for key in matching_keys] return sorted(synsets, key=lambda s: s.offset) def by_domain(self, domain: LexFileName) -> list[Synset]: @@ -269,8 +297,8 @@ def by_domain(self, domain: LexFileName) -> list[Synset]: list[Synset] Synsets in the specified domain. """ - offsets = self._synsets_by_domain.get(domain, set()) - synsets = [self._synsets[offset] for offset in offsets] + keys = self._synsets_by_domain.get(domain, set()) + synsets = [self._synsets[key] for key in keys] return sorted(synsets, key=lambda s: s.offset) def by_gloss_pattern( @@ -304,7 +332,7 @@ def by_gloss_pattern( # Determine which synsets to search if pos is not None: - synsets_to_search = [self._synsets[offset] for offset in self._synsets_by_pos[pos]] + synsets_to_search = [self._synsets[key] for key in self._synsets_by_pos[pos]] else: synsets_to_search = list(self._synsets.values()) @@ -540,8 +568,8 @@ def get_synset_by_id(self, synset_id: str) -> Synset | None: """ if len(synset_id) == 9 and synset_id[:-1].isdigit() and synset_id[-1] in "nvasr": offset = synset_id[:-1] - pos = synset_id[-1] - return self.by_offset(offset, pos) # type: ignore[arg-type] + pos = cast(WordNetPOS, synset_id[-1]) + return self.by_offset(offset, pos) return None def get_statistics(self) -> dict[str, int]: @@ -556,8 +584,9 @@ def get_statistics(self) -> dict[str, int]: total_lemmas = len(self._synsets_by_lemma) - # Count synsets by POS - pos_counts = {pos: len(offsets) for pos, offsets in self._synsets_by_pos.items()} + # Count synsets by POS from the records themselves; _synsets_by_pos + # indexes satellites twice and would double-count adjectives. + pos_counts = Counter(s.ss_type for s in self._synsets.values()) return { "synset_count": len(self._synsets), diff --git a/src/glazing/wordnet/types.py b/src/glazing/wordnet/types.py index eed5cd8..f184ad4 100644 --- a/src/glazing/wordnet/types.py +++ b/src/glazing/wordnet/types.py @@ -19,6 +19,8 @@ Full synset identifier with POS (e.g., "00001740-n"). SynsetOffset : type[Annotated[str, Field]] 8-digit synset identifier with validation. +SynsetKey : type[Annotated[str, Field]] + Canonical synset key: offset plus normalized POS (e.g., "00001740n"). SenseKey : type[Annotated[str, Field]] WordNet sense key with format validation. LemmaKey : type[Annotated[str, Field]] @@ -39,12 +41,23 @@ Regex pattern for lemma key validation. PERCENTAGE_NOTATION_PATTERN : str Regex pattern for VerbNet percentage notation. +WORDNET_SYNSET_KEY_PATTERN : str + Regex pattern for canonical synset keys. + +Functions +--------- +normalize_pos + Normalize a WordNet POS to its data-file category ("s" becomes "a"). +make_synset_key + Build the canonical key identifying a synset across the database. Examples -------- ->>> from glazing.wordnet.types import WordNetPOS, SynsetOffset +>>> from glazing.wordnet.types import WordNetPOS, SynsetOffset, make_synset_key >>> pos: WordNetPOS = "v" >>> offset: SynsetOffset = "00001740" +>>> make_synset_key(offset, pos) +'00001740v' """ from typing import Annotated, Literal @@ -206,6 +219,9 @@ # WordNet lemma key (lemma#pos#sense) LEMMA_KEY_PATTERN = r"^[a-z0-9_.-]+#[nvasr]#[0-9]+$" +# Canonical synset key (offset with normalized POS; satellites key as "a") +WORDNET_SYNSET_KEY_PATTERN = r"^[0-9]{8}[nvar]$" + # Validated string types with constraints # Full synset identifier with POS (e.g., "00001740-n" or "00001740n") @@ -226,6 +242,15 @@ ), ] +# Canonical synset key: offset plus normalized POS (e.g., "00001740n") +type SynsetKey = Annotated[ + str, + Field( + pattern=WORDNET_SYNSET_KEY_PATTERN, + description="Synset key: offset plus normalized POS (e.g., '00001740n')", + ), +] + # WordNet sense key with full format validation type SenseKey = Annotated[ str, @@ -277,3 +302,61 @@ # Synset offset string type Offset = str + + +def normalize_pos(pos: WordNetPOS) -> WordNetPOS: + """Normalize a WordNet POS to its data-file category. + + Adjective satellites (``"s"``) live in ``data.adj`` alongside adjective + heads, and WordNet pointers targeting a satellite always carry ``"a"`` in + their POS field. Normalizing ``"s"`` to ``"a"`` therefore lets a synset + record and any pointer referencing it agree on a single key. + + Parameters + ---------- + pos : WordNetPOS + Part of speech to normalize. + + Returns + ------- + WordNetPOS + ``"a"`` when `pos` is ``"s"``, otherwise `pos` unchanged. + + Examples + -------- + >>> normalize_pos("s") + 'a' + >>> normalize_pos("v") + 'v' + """ + return "a" if pos == "s" else pos + + +def make_synset_key(offset: SynsetOffset, pos: WordNetPOS) -> SynsetKey: + """Build the canonical key identifying a synset. + + WordNet offsets are byte offsets into a per-POS data file, so they are + unique only within a file - 330 offsets in WordNet 3.1 occur in more than + one ``data.*`` file. Pairing the offset with its normalized POS yields a + key that is unique across the whole database. + + Parameters + ---------- + offset : SynsetOffset + 8-digit zero-padded synset offset. + pos : WordNetPOS + Part of speech; ``"s"`` is normalized to ``"a"``. + + Returns + ------- + SynsetKey + Offset concatenated with the normalized POS. + + Examples + -------- + >>> make_synset_key("00001740", "n") + '00001740n' + >>> make_synset_key("00014377", "s") + '00014377a' + """ + return f"{offset}{normalize_pos(pos)}" diff --git a/tests/test_integration/test_converter_loader_roundtrip.py b/tests/test_integration/test_converter_loader_roundtrip.py index 1829a00..1fc747f 100644 --- a/tests/test_integration/test_converter_loader_roundtrip.py +++ b/tests/test_integration/test_converter_loader_roundtrip.py @@ -125,14 +125,14 @@ def test_word_enrichment(self, wordnet_data): wn = wordnet_data["loader"] # breathe in synset 00001740 should have tag_count=25, sense_number=1 - synset = wn.synsets["00001740"] + synset = wn.synsets["00001740v"] breathe_word = synset.words[0] assert breathe_word.lemma == "breathe" assert breathe_word.tag_count == 25 assert breathe_word.sense_number == 1 # entity in synset 00002325 should have tag_count=11 - entity_synset = wn.synsets["00002325"] + entity_synset = wn.synsets["00002325n"] entity_word = entity_synset.words[0] assert entity_word.lemma == "entity" assert entity_word.tag_count == 11 @@ -140,7 +140,7 @@ def test_word_enrichment(self, wordnet_data): def test_verb_frame_templates(self, wordnet_data): """VerbFrames have template and example_sentence from verb.Framestext/sents.vrb.""" wn = wordnet_data["loader"] - synset = wn.synsets["00001740"] + synset = wn.synsets["00001740v"] assert synset.frames is not None assert len(synset.frames) == 1 @@ -153,7 +153,7 @@ def test_verb_frame_templates(self, wordnet_data): def test_pointers_preserved(self, wordnet_data): """Pointer relations survive the round-trip.""" wn = wordnet_data["loader"] - synset = wn.synsets["00001740"] + synset = wn.synsets["00001740v"] assert len(synset.pointers) == 2 symbols = {p.symbol for p in synset.pointers} diff --git a/tests/test_wordnet/test_converter.py b/tests/test_wordnet/test_converter.py index a76c9f3..384737f 100644 --- a/tests/test_wordnet/test_converter.py +++ b/tests/test_wordnet/test_converter.py @@ -471,10 +471,8 @@ def test_all_marker_variants_in_data_file(self, converter, tmp_path): data_file = tmp_path / "data.adj" data_file.write_text(content, encoding="utf-8") - synsets = converter.parse_data_file(data_file, "a") + converter.parse_data_file( - data_file, "s" - ) - # All four synsets survive parsing. + # A single adjective pass returns heads and satellites alike. + synsets = converter.parse_data_file(data_file, "a") assert len(synsets) == 4 markers = {w.lemma: w.syntactic_marker for synset in synsets for w in synset.words} diff --git a/tests/test_wordnet/test_loader.py b/tests/test_wordnet/test_loader.py index 63084c0..9e7d318 100644 --- a/tests/test_wordnet/test_loader.py +++ b/tests/test_wordnet/test_loader.py @@ -123,12 +123,12 @@ def test_load_synsets(self, temp_data_file): # Check synsets loaded assert len(loader.synsets) == 3 - assert "00001740" in loader.synsets - assert "00001930" in loader.synsets - assert "00002325" in loader.synsets + assert "00001740n" in loader.synsets + assert "00001930n" in loader.synsets + assert "00002325v" in loader.synsets # Check synset content - entity = loader.synsets["00001740"] + entity = loader.synsets["00001740n"] assert entity.ss_type == "n" assert len(entity.words) == 1 assert entity.words[0].lemma == "entity" @@ -144,14 +144,14 @@ def test_load_lemma_index(self, temp_data_file): assert "n" in loader.lemma_index["entity"] assert len(loader.lemma_index["entity"]["n"]) == 1 - # lemma_index values are SynsetOffset strings now - offset = loader.lemma_index["entity"]["n"][0] - assert offset == "00001740" + # lemma_index values are canonical SynsetKey strings + key = loader.lemma_index["entity"]["n"][0] + assert key == "00001740n" # Check verb lemmas assert "run" in loader.lemma_index assert "v" in loader.lemma_index["run"] - assert loader.lemma_index["run"]["v"][0] == "00002325" + assert loader.lemma_index["run"]["v"][0] == "00002325v" # "go" should also be indexed assert "go" in loader.lemma_index @@ -190,12 +190,12 @@ def test_build_relation_indices(self, temp_data_file): loader.load() # Check hypernym index - assert "00001930" in loader.hypernym_index - assert "00001740" in loader.hypernym_index["00001930"] + assert "00001930n" in loader.hypernym_index + assert "00001740n" in loader.hypernym_index["00001930n"] # Check hyponym index - assert "00001740" in loader.hyponym_index - assert "00001930" in loader.hyponym_index["00001740"] + assert "00001740n" in loader.hyponym_index + assert "00001930n" in loader.hyponym_index["00001740n"] def test_get_synset(self, temp_data_file): """Test getting synset by offset.""" @@ -299,7 +299,7 @@ def test_lazy_loading(self, temp_data_file): # Check cache is working assert loader._cache is not None - cached = loader._cache.get("00001740") + cached = loader._cache.get("00001740n") assert cached is not None assert cached.offset == "00001740" diff --git a/tests/test_wordnet/test_relations.py b/tests/test_wordnet/test_relations.py index d99a9df..4e91c26 100644 --- a/tests/test_wordnet/test_relations.py +++ b/tests/test_wordnet/test_relations.py @@ -31,7 +31,7 @@ def sample_synsets_dict(self): ], gloss="that which is perceived or known or inferred to have its own distinct existence", ) - synsets["00001740"] = entity_synset + synsets[entity_synset.key] = entity_synset # Create physical_entity synset physical_synset = Synset( @@ -50,7 +50,7 @@ def sample_synsets_dict(self): ], gloss="an entity that has physical existence", ) - synsets["00002137"] = physical_synset + synsets[physical_synset.key] = physical_synset # Create object synset object_synset = Synset( @@ -72,7 +72,7 @@ def sample_synsets_dict(self): ], gloss="a tangible and visible entity", ) - synsets["00002684"] = object_synset + synsets[object_synset.key] = object_synset # Create living_thing synset living_synset = Synset( @@ -93,7 +93,7 @@ def sample_synsets_dict(self): ], gloss="a living entity", ) - synsets["02083346"] = living_synset + synsets[living_synset.key] = living_synset # Create dog synset dog_synset = Synset( @@ -115,7 +115,7 @@ def sample_synsets_dict(self): ], gloss="a member of the genus Canis", ) - synsets["02084442"] = dog_synset + synsets[dog_synset.key] = dog_synset # Create cat synset cat_synset = Synset( @@ -131,7 +131,7 @@ def sample_synsets_dict(self): ], gloss="feline mammal", ) - synsets["02121620"] = cat_synset + synsets[cat_synset.key] = cat_synset # Create pack synset (for holonym) pack_synset = Synset( @@ -147,7 +147,7 @@ def sample_synsets_dict(self): ], gloss="a group of hunting animals", ) - synsets["08008335"] = pack_synset + synsets[pack_synset.key] = pack_synset # Create paw synset (for meronym) paw_synset = Synset( @@ -163,7 +163,7 @@ def sample_synsets_dict(self): ], gloss="a clawed foot of an animal", ) - synsets["02159955"] = paw_synset + synsets[paw_synset.key] = paw_synset # Create run synset (verb) run_synset = Synset( @@ -185,7 +185,7 @@ def sample_synsets_dict(self): frames=[VerbFrame(frame_number=1, word_indices=[0])], gloss="move fast by using one's feet", ) - synsets["02092002"] = run_synset + synsets[run_synset.key] = run_synset # Create travel synset (verb hypernym) travel_synset = Synset( @@ -199,7 +199,7 @@ def sample_synsets_dict(self): ], gloss="change location; move, travel, or proceed", ) - synsets["01835496"] = travel_synset + synsets[travel_synset.key] = travel_synset # Create move synset (entailment) move_synset = Synset( @@ -211,7 +211,7 @@ def sample_synsets_dict(self): pointers=[], gloss="change position", ) - synsets["02092309"] = move_synset + synsets[move_synset.key] = move_synset # Create rush synset (cause) rush_synset = Synset( @@ -223,7 +223,7 @@ def sample_synsets_dict(self): pointers=[], gloss="move fast", ) - synsets["02093321"] = rush_synset + synsets[rush_synset.key] = rush_synset # Create good synset (adjective) good_synset = Synset( @@ -246,7 +246,7 @@ def sample_synsets_dict(self): ], gloss="having desirable or positive qualities", ) - synsets["01123148"] = good_synset + synsets[good_synset.key] = good_synset # Create bad synset (antonym) bad_synset = Synset( @@ -262,7 +262,7 @@ def sample_synsets_dict(self): ], gloss="having undesirable or negative qualities", ) - synsets["01125429"] = bad_synset + synsets[bad_synset.key] = bad_synset # Create nice synset (similar) nice_synset = Synset( @@ -274,7 +274,7 @@ def sample_synsets_dict(self): pointers=[], gloss="pleasant or pleasing", ) - synsets["01124073"] = nice_synset + synsets[nice_synset.key] = nice_synset # Create goodness synset (derivation) goodness_synset = Synset( @@ -290,14 +290,14 @@ def sample_synsets_dict(self): ], gloss="the quality of being good", ) - synsets["05145118"] = goodness_synset + synsets[goodness_synset.key] = goodness_synset return synsets def test_get_hypernyms_direct(self, sample_synsets_dict): """Test getting direct hypernyms.""" traverser = WordNetRelationTraverser(sample_synsets_dict) - dog_synset = sample_synsets_dict["02084442"] + dog_synset = sample_synsets_dict["02084442n"] hypernyms = traverser.get_hypernyms(dog_synset, direct_only=True) assert len(hypernyms) == 1 @@ -306,7 +306,7 @@ def test_get_hypernyms_direct(self, sample_synsets_dict): def test_get_hypernyms_all(self, sample_synsets_dict): """Test getting all hypernyms.""" traverser = WordNetRelationTraverser(sample_synsets_dict) - dog_synset = sample_synsets_dict["02084442"] + dog_synset = sample_synsets_dict["02084442n"] hypernyms = traverser.get_hypernyms(dog_synset, direct_only=False) assert len(hypernyms) == 4 @@ -319,7 +319,7 @@ def test_get_hypernyms_all(self, sample_synsets_dict): def test_get_hyponyms_direct(self, sample_synsets_dict): """Test getting direct hyponyms.""" traverser = WordNetRelationTraverser(sample_synsets_dict) - living_synset = sample_synsets_dict["02083346"] + living_synset = sample_synsets_dict["02083346n"] hyponyms = traverser.get_hyponyms(living_synset, direct_only=True) assert len(hyponyms) == 2 @@ -330,7 +330,7 @@ def test_get_hyponyms_direct(self, sample_synsets_dict): def test_get_hyponyms_all(self, sample_synsets_dict): """Test getting all hyponyms.""" traverser = WordNetRelationTraverser(sample_synsets_dict) - object_synset = sample_synsets_dict["00002684"] + object_synset = sample_synsets_dict["00002684n"] hyponyms = traverser.get_hyponyms(object_synset, direct_only=False) assert len(hyponyms) == 3 @@ -342,7 +342,7 @@ def test_get_hyponyms_all(self, sample_synsets_dict): def test_get_hypernym_paths(self, sample_synsets_dict): """Test getting hypernym paths to root.""" traverser = WordNetRelationTraverser(sample_synsets_dict) - dog_synset = sample_synsets_dict["02084442"] + dog_synset = sample_synsets_dict["02084442n"] paths = traverser.get_hypernym_paths(dog_synset, max_depth=10) assert len(paths) == 1 # Only one path to root @@ -358,8 +358,8 @@ def test_get_hypernym_paths(self, sample_synsets_dict): def test_get_common_hypernyms(self, sample_synsets_dict): """Test finding common hypernyms.""" traverser = WordNetRelationTraverser(sample_synsets_dict) - dog_synset = sample_synsets_dict["02084442"] - cat_synset = sample_synsets_dict["02121620"] + dog_synset = sample_synsets_dict["02084442n"] + cat_synset = sample_synsets_dict["02121620n"] common = traverser.get_common_hypernyms(dog_synset, cat_synset) assert len(common) == 4 @@ -372,7 +372,7 @@ def test_get_common_hypernyms(self, sample_synsets_dict): def test_get_meronyms(self, sample_synsets_dict): """Test getting meronyms.""" traverser = WordNetRelationTraverser(sample_synsets_dict) - dog_synset = sample_synsets_dict["02084442"] + dog_synset = sample_synsets_dict["02084442n"] # Get all meronyms meronyms = traverser.get_meronyms(dog_synset) @@ -391,7 +391,7 @@ def test_get_meronyms(self, sample_synsets_dict): def test_get_holonyms(self, sample_synsets_dict): """Test getting holonyms.""" traverser = WordNetRelationTraverser(sample_synsets_dict) - dog_synset = sample_synsets_dict["02084442"] + dog_synset = sample_synsets_dict["02084442n"] # Get all holonyms holonyms = traverser.get_holonyms(dog_synset) @@ -410,49 +410,49 @@ def test_get_holonyms(self, sample_synsets_dict): def test_get_entailments(self, sample_synsets_dict): """Test getting entailments for verbs.""" traverser = WordNetRelationTraverser(sample_synsets_dict) - run_synset = sample_synsets_dict["02092002"] + run_synset = sample_synsets_dict["02092002v"] entailments = traverser.get_entailments(run_synset) assert len(entailments) == 1 assert entailments[0].offset == "02092309" # move # Non-verb synset - dog_synset = sample_synsets_dict["02084442"] + dog_synset = sample_synsets_dict["02084442n"] entailments = traverser.get_entailments(dog_synset) assert len(entailments) == 0 def test_get_causes(self, sample_synsets_dict): """Test getting causes for verbs.""" traverser = WordNetRelationTraverser(sample_synsets_dict) - run_synset = sample_synsets_dict["02092002"] + run_synset = sample_synsets_dict["02092002v"] causes = traverser.get_causes(run_synset) assert len(causes) == 1 assert causes[0].offset == "02093321" # rush # Non-verb synset - dog_synset = sample_synsets_dict["02084442"] + dog_synset = sample_synsets_dict["02084442n"] causes = traverser.get_causes(dog_synset) assert len(causes) == 0 def test_get_similar_to(self, sample_synsets_dict): """Test getting similar adjectives.""" traverser = WordNetRelationTraverser(sample_synsets_dict) - good_synset = sample_synsets_dict["01123148"] + good_synset = sample_synsets_dict["01123148a"] similar = traverser.get_similar_to(good_synset) assert len(similar) == 1 assert similar[0].offset == "01124073" # nice # Non-adjective synset - dog_synset = sample_synsets_dict["02084442"] + dog_synset = sample_synsets_dict["02084442n"] similar = traverser.get_similar_to(dog_synset) assert len(similar) == 0 def test_get_also_see(self, sample_synsets_dict): """Test getting also-see relations.""" traverser = WordNetRelationTraverser(sample_synsets_dict) - good_synset = sample_synsets_dict["01123148"] + good_synset = sample_synsets_dict["01123148a"] also_see = traverser.get_also_see(good_synset) assert len(also_see) == 0 # No also-see in our test data @@ -460,7 +460,7 @@ def test_get_also_see(self, sample_synsets_dict): def test_get_antonyms(self, sample_synsets_dict): """Test getting antonyms.""" traverser = WordNetRelationTraverser(sample_synsets_dict) - good_synset = sample_synsets_dict["01123148"] + good_synset = sample_synsets_dict["01123148a"] # Get all antonyms antonyms = traverser.get_antonyms(good_synset) @@ -471,7 +471,7 @@ def test_get_antonyms(self, sample_synsets_dict): def test_get_derivations(self, sample_synsets_dict): """Test getting derivationally related forms.""" traverser = WordNetRelationTraverser(sample_synsets_dict) - good_synset = sample_synsets_dict["01123148"] + good_synset = sample_synsets_dict["01123148a"] # Get derivations derivations = traverser.get_derivations(good_synset) @@ -482,8 +482,8 @@ def test_get_derivations(self, sample_synsets_dict): def test_calculate_path_similarity(self, sample_synsets_dict): """Test calculating path similarity.""" traverser = WordNetRelationTraverser(sample_synsets_dict) - dog_synset = sample_synsets_dict["02084442"] - cat_synset = sample_synsets_dict["02121620"] + dog_synset = sample_synsets_dict["02084442n"] + cat_synset = sample_synsets_dict["02121620n"] # Dog and cat similarity similarity = traverser.calculate_path_similarity(dog_synset, cat_synset) @@ -494,7 +494,7 @@ def test_calculate_path_similarity(self, sample_synsets_dict): assert similarity == 1.0 # Different POS - run_synset = sample_synsets_dict["02092002"] + run_synset = sample_synsets_dict["02092002v"] similarity = traverser.calculate_path_similarity(dog_synset, run_synset) assert similarity == 0.0 @@ -503,32 +503,32 @@ def test_calculate_depth(self, sample_synsets_dict): traverser = WordNetRelationTraverser(sample_synsets_dict) # Entity (root) has depth 0 - entity_synset = sample_synsets_dict["00001740"] + entity_synset = sample_synsets_dict["00001740n"] depth = traverser.calculate_depth(entity_synset) assert depth == 0 # Dog has depth 4 - dog_synset = sample_synsets_dict["02084442"] + dog_synset = sample_synsets_dict["02084442n"] depth = traverser.calculate_depth(dog_synset) assert depth == 4 def test_get_verb_groups(self, sample_synsets_dict): """Test getting verb groups.""" traverser = WordNetRelationTraverser(sample_synsets_dict) - run_synset = sample_synsets_dict["02092002"] + run_synset = sample_synsets_dict["02092002v"] groups = traverser.get_verb_groups(run_synset) assert len(groups) == 0 # No verb group target in our test data # Non-verb synset - dog_synset = sample_synsets_dict["02084442"] + dog_synset = sample_synsets_dict["02084442n"] groups = traverser.get_verb_groups(dog_synset) assert len(groups) == 0 def test_get_all_relations(self, sample_synsets_dict): """Test getting all relations for a synset.""" traverser = WordNetRelationTraverser(sample_synsets_dict) - dog_synset = sample_synsets_dict["02084442"] + dog_synset = sample_synsets_dict["02084442n"] relations = traverser.get_all_relations(dog_synset) diff --git a/tests/test_wordnet/test_satellites.py b/tests/test_wordnet/test_satellites.py new file mode 100644 index 0000000..42c7b85 --- /dev/null +++ b/tests/test_wordnet/test_satellites.py @@ -0,0 +1,352 @@ +"""Tests for adjective satellite handling and canonical synset keying. + +Regression coverage for the converter dropping every ``ss_type == "s"`` synset +and for synsets being keyed by bare offset, which collides across parts of +speech. See https://github.com/factslab/glazing/issues/11. +""" + +import pytest + +from glazing.wordnet.converter import WordNetConverter +from glazing.wordnet.loader import WordNetLoader +from glazing.wordnet.models import Pointer, Synset, Word +from glazing.wordnet.relations import WordNetRelationTraverser +from glazing.wordnet.search import WordNetSearch +from glazing.wordnet.types import make_synset_key, normalize_pos + +WN_LICENSE_HEADER = " 1 This software and database is being provided to you.\n" + + +def make_synset(offset: str, ss_type: str, lemma: str, **kwargs) -> Synset: + """Build a minimal synset for testing.""" + defaults = { + "lex_filenum": 0 if ss_type in ("a", "s") else 3, + "lex_filename": "adj.all" if ss_type in ("a", "s") else "noun.Tops", + "pointers": [], + "gloss": f"gloss for {lemma}", + } + defaults.update(kwargs) + return Synset( + offset=offset, + ss_type=ss_type, + words=[Word(lemma=lemma, lex_id=0)], + **defaults, + ) + + +class TestKeyPrimitives: + """Tests for normalize_pos and make_synset_key.""" + + @pytest.mark.parametrize( + ("pos", "expected"), + [("n", "n"), ("v", "v"), ("a", "a"), ("r", "r"), ("s", "a")], + ) + def test_normalize_pos(self, pos, expected): + """Satellites normalize to "a"; every other POS is unchanged.""" + assert normalize_pos(pos) == expected + + @pytest.mark.parametrize( + ("offset", "pos", "expected"), + [ + ("00001740", "n", "00001740n"), + ("00001740", "v", "00001740v"), + ("00014377", "a", "00014377a"), + ("00014377", "s", "00014377a"), + ("00001740", "r", "00001740r"), + ], + ) + def test_make_synset_key(self, offset, pos, expected): + """Keys pair the offset with the normalized POS.""" + assert make_synset_key(offset, pos) == expected + + def test_satellite_record_and_pointer_agree(self): + """A satellite's key matches the key of a pointer targeting it. + + WordNet writes "a" in the POS field of every pointer to an adjective, + even when the target is a satellite, so this agreement is the whole + point of normalizing. + """ + satellite = make_synset("00014377", "s", "abounding") + pointer = Pointer(symbol="&", offset="00014377", pos="a", source=0, target=0) + + assert satellite.ss_type == "s" + assert satellite.key == pointer.target_key == "00014377a" + + def test_key_is_not_serialized(self): + """key stays derived and out of the on-disk schema.""" + synset = make_synset("00014377", "s", "abounding") + assert "key" not in synset.model_dump_json() + assert Synset.model_validate_json(synset.model_dump_json()).key == synset.key + + +class TestConverterRetainsSatellites: + """Tests that the converter no longer discards satellite synsets.""" + + @pytest.fixture + def converter(self): + """Provide a converter instance.""" + return WordNetConverter() + + @pytest.fixture + def data_adj(self, tmp_path): + """Write a data.adj holding one head and two satellites.""" + path = tmp_path / "data.adj" + path.write_text( + WN_LICENSE_HEADER + + "00001740 00 a 01 able 0 001 & 00014377 a 0000 | having the means\n" + + "00014377 00 s 01 abounding 0 001 & 00001740 a 0000 | existing in abundance\n" + + "00019769 00 s 01 handy 0 000 | easy to reach\n", + encoding="utf-8", + ) + return path + + def test_satellites_are_emitted(self, converter, data_adj): + """The adjective pass returns heads and satellites alike.""" + synsets = converter.parse_data_file(data_adj, "a") + + assert len(synsets) == 3 + assert sorted(s.ss_type for s in synsets) == ["a", "s", "s"] + + def test_adjective_passes_are_equivalent(self, converter, data_adj): + """ "a" and "s" both name the adjective data file, so both return all.""" + by_a = converter.parse_data_file(data_adj, "a") + by_s = converter.parse_data_file(data_adj, "s") + + assert [s.key for s in by_a] == [s.key for s in by_s] + + def test_satellite_pointers_resolve(self, converter, data_adj): + """No pointer among the emitted adjectives dangles.""" + synsets = converter.parse_data_file(data_adj, "a") + keys = {s.key for s in synsets} + + targets = {p.target_key for s in synsets for p in s.pointers} + assert targets <= keys + + def test_leading_punctuation_lemmas_survive(self, converter, tmp_path): + """Lemmas opening with a dot or apostrophe no longer fail validation. + + A lemma that fails validation discards its entire synset, which lost + ".22_caliber", "'hood", "'tween" and "'s_Gravenhage" from the output. + """ + path = tmp_path / "data.adj" + path.write_text( + WN_LICENSE_HEADER + + "03157978 01 a 02 .22_caliber 0 .22-caliber 0 000 | of a .22 bore\n" + + "08659519 01 a 02 'hood 0 'tween 0 000 | elided forms\n", + encoding="utf-8", + ) + + synsets = converter.parse_data_file(path, "a") + assert len(synsets) == 2 + assert synsets[0].get_lemmas() == [".22_caliber", ".22-caliber"] + assert synsets[1].get_lemmas() == ["'hood", "'tween"] + + +class TestSatelliteSenseEnrichment: + """Tests that satellite senses receive sense numbers and tag counts.""" + + def _wordnet_dir(self, tmp_path, index_sense, cntlist=""): + """Write a minimal WordNet directory with two satellites.""" + wn_dir = tmp_path / "wn" + wn_dir.mkdir() + (wn_dir / "data.adj").write_text( + WN_LICENSE_HEADER + + "00014377 00 s 01 cardinal 0 000 | serving as a base\n" + + "00019769 00 s 01 cardinal 0 000 | of a deep red\n", + encoding="utf-8", + ) + for name in ("data.noun", "data.verb", "data.adv"): + (wn_dir / name).write_text(WN_LICENSE_HEADER, encoding="utf-8") + (wn_dir / "index.sense").write_text(index_sense, encoding="utf-8") + (wn_dir / "cntlist").write_text(cntlist, encoding="utf-8") + return wn_dir + + def test_satellite_senses_are_enriched(self, tmp_path): + """Satellites get sense_number and tag_count despite head-word keys. + + Both satellites share the head-less prefix ``cardinal%5:00:00:``, so a + prefix match could not tell them apart; keying on the synset offset can. + """ + wn_dir = self._wordnet_dir( + tmp_path, + index_sense=( + "cardinal%5:00:00:fundamental:00 00014377 1 7\n" + "cardinal%5:00:00:red:00 00019769 2 3\n" + ), + ) + out = tmp_path / "wordnet.jsonl" + WordNetConverter().convert_wordnet_database(wn_dir, out) + + loader = WordNetLoader(out, autoload=False) + loader.load() + + base = loader.synsets["00014377a"].words[0] + assert (base.sense_number, base.tag_count) == (1, 7) + + red = loader.synsets["00019769a"].words[0] + assert (red.sense_number, red.tag_count) == (2, 3) + + def test_cntlist_overrides_the_right_satellite(self, tmp_path): + """A cntlist count lands on the satellite its sense key names.""" + wn_dir = self._wordnet_dir( + tmp_path, + index_sense=( + "cardinal%5:00:00:fundamental:00 00014377 1 7\n" + "cardinal%5:00:00:red:00 00019769 2 3\n" + ), + cntlist="99 cardinal%5:00:00:red:00 1\n", + ) + out = tmp_path / "wordnet.jsonl" + WordNetConverter().convert_wordnet_database(wn_dir, out) + + loader = WordNetLoader(out, autoload=False) + loader.load() + + assert loader.synsets["00019769a"].words[0].tag_count == 99 + assert loader.synsets["00014377a"].words[0].tag_count == 7 + + +class TestCrossPOSOffsetCollision: + """Tests that synsets sharing an offset across POS both survive.""" + + @pytest.fixture + def colliding(self): + """Provide a noun and a verb sharing offset 00001740.""" + return [ + make_synset("00001740", "n", "entity"), + make_synset("00001740", "v", "breathe", lex_filenum=29, lex_filename="verb.body"), + ] + + def test_search_accepts_both(self, colliding): + """Indexing both no longer raises on the duplicate offset.""" + search = WordNetSearch(synsets=colliding) + + assert search.by_offset("00001740", "n").get_lemmas() == ["entity"] + assert search.by_offset("00001740", "v").get_lemmas() == ["breathe"] + + def test_search_bare_offset_is_ambiguous(self, colliding): + """An unqualified colliding offset resolves to nothing, not a guess.""" + search = WordNetSearch(synsets=colliding) + assert search.by_offset("00001740") is None + + def test_loader_keeps_both(self, tmp_path, colliding): + """Neither synset overwrites the other in the loader.""" + path = tmp_path / "wordnet.jsonl" + path.write_text("".join(f"{s.model_dump_json()}\n" for s in colliding), encoding="utf-8") + + loader = WordNetLoader(path, autoload=False) + loader.load() + + assert len(loader.synsets) == 2 + assert loader.get_synset("00001740", "n").get_lemmas() == ["entity"] + assert loader.get_synset("00001740v").get_lemmas() == ["breathe"] + assert loader.get_synset("00001740") is None + + +class TestSatelliteReachability: + """Tests that satellites are findable through the adjective POS.""" + + @pytest.fixture + def satellite(self): + """Provide a lone satellite synset.""" + return make_synset("00014377", "s", "abounding") + + def test_search_by_lemma(self, satellite): + """A satellite answers to "a", to "s", and to no filter at all.""" + search = WordNetSearch(synsets=[satellite]) + + assert [s.key for s in search.by_lemma("abounding", "a")] == ["00014377a"] + assert [s.key for s in search.by_lemma("abounding", "s")] == ["00014377a"] + assert [s.key for s in search.by_lemma("abounding")] == ["00014377a"] + + def test_search_by_offset_pos_filter(self, satellite): + """ "a" reaches the satellite; ss_type still reports "s".""" + search = WordNetSearch(synsets=[satellite]) + + found = search.by_offset("00014377", "a") + assert found is not None + assert found.ss_type == "s" + + def test_search_pos_s_excludes_heads(self): + """A "s" filter does not return adjective heads.""" + head = make_synset("00001740", "a", "able") + search = WordNetSearch(synsets=[head]) + + assert search.by_offset("00001740", "a") is not None + assert search.by_offset("00001740", "s") is None + + def test_statistics_do_not_double_count(self, satellite): + """Dual indexing under "s" and "a" must not inflate the counts.""" + head = make_synset("00001740", "a", "able") + stats = WordNetSearch(synsets=[satellite, head]).get_statistics() + + assert stats["synset_count"] == 2 + assert stats["s_synsets"] == 1 + assert stats["a_synsets"] == 1 + + def test_loader_by_lemma(self, tmp_path, satellite): + """The loader reaches satellites from "a" without duplicating them.""" + path = tmp_path / "wordnet.jsonl" + path.write_text(f"{satellite.model_dump_json()}\n", encoding="utf-8") + + loader = WordNetLoader(path, autoload=False) + loader.load() + + assert [s.key for s in loader.get_synsets_by_lemma("abounding", "a")] == ["00014377a"] + assert [s.key for s in loader.get_synsets_by_lemma("abounding", "s")] == ["00014377a"] + assert [s.key for s in loader.get_synsets_by_lemma("abounding")] == ["00014377a"] + + +class TestSatelliteRelations: + """Tests that relations traverse into satellites.""" + + @pytest.fixture + def head_and_satellite(self): + """Provide an adjective head and the satellite it points at.""" + head = make_synset( + "00001740", + "a", + "able", + pointers=[Pointer(symbol="&", offset="00014377", pos="a", source=0, target=0)], + ) + satellite = make_synset( + "00014377", + "s", + "abounding", + pointers=[Pointer(symbol="&", offset="00001740", pos="a", source=0, target=0)], + ) + return head, satellite + + def test_similar_to_resolves(self, head_and_satellite): + """A similar-to pointer reaches the satellite it names.""" + head, satellite = head_and_satellite + traverser = WordNetRelationTraverser({s.key: s for s in (head, satellite)}) + + assert [s.key for s in traverser.get_similar_to(head)] == ["00014377a"] + assert [s.key for s in traverser.get_similar_to(satellite)] == ["00001740a"] + + def test_similarity_spans_head_and_satellite(self, head_and_satellite): + """Heads and satellites count as one POS for path similarity.""" + head, satellite = head_and_satellite + traverser = WordNetRelationTraverser({s.key: s for s in (head, satellite)}) + + # Distinct ss_type values ("a" vs "s") must not short-circuit to 0.0. + assert traverser.calculate_path_similarity(head, head) == 1.0 + assert traverser.calculate_path_similarity(satellite, satellite) == 1.0 + + def test_relations_respect_pos(self): + """A pointer resolves by POS, not by whichever synset shares an offset.""" + noun = make_synset( + "00002325", + "n", + "entity", + pointers=[Pointer(symbol="@", offset="00001740", pos="n", source=0, target=0)], + ) + noun_target = make_synset("00001740", "n", "thing") + verb_decoy = make_synset( + "00001740", "v", "breathe", lex_filenum=29, lex_filename="verb.body" + ) + traverser = WordNetRelationTraverser({s.key: s for s in (noun, noun_target, verb_decoy)}) + + hypernyms = traverser.get_hypernyms(noun) + assert [s.get_lemmas() for s in hypernyms] == [["thing"]] From 74a13910ff84532a89874acac7462f6695925499 Mon Sep 17 00:00:00 2001 From: Aaron Steven White Date: Tue, 21 Jul 2026 11:18:44 -0400 Subject: [PATCH 2/6] fix(wordnet): accept lemmas with a leading dot or apostrophe LEMMA_PATTERN required the first character to be alphanumeric, so WordNet's elided and calibre lemmas failed validation -- and because a failed lemma discards the whole synset, seven synsets vanished from the converted output: .22_caliber, .38_caliber and .45_caliber from data.adj, 'hood and 's_Gravenhage from data.noun, 'tween and 'tween_decks from data.adv. Allow a single leading dot or apostrophe before the first alphanumeric. The rest of the pattern is unchanged, so genuinely malformed lemmas are still rejected. Found while verifying the satellite fix against WordNet 3.1: with this in place, every raw synset in all four data files converts. --- src/glazing/types.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/glazing/types.py b/src/glazing/types.py index 5ad3e00..f70503d 100644 --- a/src/glazing/types.py +++ b/src/glazing/types.py @@ -161,9 +161,10 @@ VERBNET_KEY_PATTERN = r"^[a-z_-]+#\d+$" # e.g., "give#2" # Name validation patterns -LEMMA_PATTERN = ( - r"^[a-zA-Z0-9][a-zA-Z0-9_\'\-\.\/]*$" # Word lemmas (incl. proper nouns, abbreviations) -) +# Word lemmas (incl. proper nouns and abbreviations). A single leading dot or +# apostrophe is allowed because WordNet lemmatizes calibre terms as +# ".22_caliber" and elided forms as "'hood", "'tween", "'s_Gravenhage". +LEMMA_PATTERN = r"^['.]?[a-zA-Z0-9][a-zA-Z0-9_\'\-\.\/]*$" # Color validation for FrameNet HEX_COLOR_PATTERN = r"^#?[0-9A-Fa-f]{6}$" # 6-digit hex color with optional # prefix From 66c8545963a7ed6de0ad7e9e019fad4504850092 Mon Sep 17 00:00:00 2001 From: Aaron Steven White Date: Tue, 21 Jul 2026 11:19:02 -0400 Subject: [PATCH 3/6] fix(search): build WordNet synset IDs from the canonical key Five call sites formatted synset IDs as f"{synset.offset:08d}{ss_type}", but offset is a str, so the 'd' format code raised ValueError at runtime. In the CLI the exception was swallowed by a broad except and reported as "Relation search failed", leaving `glazing search relations --dataset wordnet` broken outright rather than visibly erroring. Use synset.key, which both avoids the format bug and gives records the same identity that pointers resolve to -- the raw ss_type would have produced "...s" for a satellite record against "...a" for every pointer to it. The merge dict in load_wordnet_from_jsonl becomes correct across the offsets that collide between parts of speech for the same reason. Also route _convert_pos_for_wordnet's WordNet-native codes through normalize_pos so the "s" -> "a" mapping has one source of truth. --- src/glazing/cli/search.py | 2 +- src/glazing/search.py | 24 +++++++++++++----------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/glazing/cli/search.py b/src/glazing/cli/search.py index a178065..6eddac5 100644 --- a/src/glazing/cli/search.py +++ b/src/glazing/cli/search.py @@ -763,7 +763,7 @@ def search_relations( table.add_column("Definition", style="white", no_wrap=False) for synset in synsets[:20]: - synset_id = f"{synset.offset:08d}{synset.ss_type}" + synset_id = synset.key words = ", ".join(w.lemma for w in synset.words[:3]) if len(synset.words) > 3: words += f" (+{len(synset.words) - 3})" diff --git a/src/glazing/search.py b/src/glazing/search.py index 5f4e849..ed09b49 100644 --- a/src/glazing/search.py +++ b/src/glazing/search.py @@ -9,6 +9,7 @@ from dataclasses import dataclass from pathlib import Path +from typing import cast from glazing.framenet.loader import FrameNetLoader from glazing.framenet.models import Frame @@ -36,6 +37,7 @@ from glazing.wordnet.models import Synset from glazing.wordnet.search import WordNetSearch from glazing.wordnet.symbol_parser import filter_by_relation_type +from glazing.wordnet.types import WordNetPOS, normalize_pos @dataclass @@ -395,20 +397,20 @@ def _convert_pos_for_wordnet(self, pos: str | None) -> str | None: return None pos_lower = pos.lower() - pos_map = { - "v": "v", + spelled_out: dict[str, WordNetPOS] = { "verb": "v", - "n": "n", "noun": "n", - "a": "a", "adj": "a", "adjective": "a", - "s": "a", - "r": "r", "adv": "r", "adverb": "r", } - return pos_map.get(pos_lower) + if pos_lower in spelled_out: + return spelled_out[pos_lower] + if pos_lower in ("n", "v", "a", "r", "s"): + # normalize_pos folds satellites ("s") into "a". + return normalize_pos(cast(WordNetPOS, pos_lower)) + return None def _search_propbank_by_lemma(self, lemma: str) -> tuple[list[Frameset], list[Roleset]]: """Search PropBank by lemma. @@ -681,7 +683,7 @@ def search(self, query: str) -> list[SearchResult]: if self.wordnet: synsets = self.wordnet.by_lemma(query) for synset in synsets: - synset_id = f"{synset.offset:08d}{synset.ss_type}" + synset_id = synset.key results.append( SearchResult( dataset="wordnet", @@ -1165,9 +1167,9 @@ def load_wordnet_from_jsonl(self, synsets_path: str, _index_path: str, _pos: str # Merge synsets with existing ones existing_synsets = self.wordnet.get_all_synsets() # Create a dict to merge by offset to avoid duplicates - synset_dict = {f"{s.offset:08d}{s.ss_type}": s for s in existing_synsets} + synset_dict = {s.key: s for s in existing_synsets} for synset in synsets: - synset_id = f"{synset.offset:08d}{synset.ss_type}" + synset_id = synset.key synset_dict[synset_id] = synset # Recreate WordNetSearch with merged synsets self.wordnet = WordNetSearch(list(synset_dict.values())) @@ -1230,7 +1232,7 @@ def search_with_fuzzy( # noqa: C901, PLR0912 for word in synset.words: similarity = levenshtein_ratio(query_normalized, word.lemma.lower()) if similarity >= fuzzy_threshold: - synset_id = f"{synset.offset:08d}{synset.ss_type}" + synset_id = synset.key results.append( SearchResult( dataset="wordnet", From 8c6154a3662f16eaa2b7c210338ea155f0805d59 Mon Sep 17 00:00:00 2001 From: Aaron Steven White Date: Tue, 21 Jul 2026 11:19:30 -0400 Subject: [PATCH 4/6] feat(initialize): version converted data so stale caches re-convert The .initialized marker was created with touch(), carrying no content, and initialize_datasets short-circuited on its mere existence. An upgraded user therefore kept whatever the previous glazing had written forever -- which for this release means a WordNet cache missing all 10,717 satellites, with no signal that anything is wrong. Record a DATA_SCHEMA_VERSION in the marker as JSON and treat a missing, older, or unparseable version as not-current, so `glazing init` re-converts without --force. Conversion runs from the already-downloaded raw files, so this costs no re-download. check_initialization keeps its existing meaning (data present) rather than being overloaded, since a stale cache reported as "not initialized" would be misleading; the new is_data_current answers the staleness question, and importing glazing warns distinctly for each case. --- src/glazing/__init__.py | 12 +++++-- src/glazing/initialize.py | 71 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 76 insertions(+), 7 deletions(-) diff --git a/src/glazing/__init__.py b/src/glazing/__init__.py index 14b8eb3..150ba08 100644 --- a/src/glazing/__init__.py +++ b/src/glazing/__init__.py @@ -31,7 +31,7 @@ import warnings from glazing.__version__ import __version__, __version_info__ -from glazing.initialize import check_initialization, get_default_data_dir +from glazing.initialize import check_initialization, get_default_data_dir, is_data_current def _check_initialization() -> None: @@ -47,8 +47,8 @@ def _check_initialization() -> None: ): return + data_dir = get_default_data_dir() if not check_initialization(): - data_dir = get_default_data_dir() warnings.warn( f"\nGlazing datasets not initialized.\n" f"Run 'glazing init' to download and convert all datasets.\n" @@ -56,6 +56,14 @@ def _check_initialization() -> None: UserWarning, stacklevel=2, ) + elif not is_data_current(): + warnings.warn( + f"\nGlazing datasets were converted by an older version and are out of date.\n" + f"Run 'glazing init' to re-convert them from the downloaded source files.\n" + f"Data location: {data_dir}", + UserWarning, + stacklevel=2, + ) # Check initialization on import (can be disabled via env var) diff --git a/src/glazing/initialize.py b/src/glazing/initialize.py index 9e3fb2b..d1c49c4 100644 --- a/src/glazing/initialize.py +++ b/src/glazing/initialize.py @@ -6,6 +6,7 @@ from __future__ import annotations +import json import os import sys from pathlib import Path @@ -24,6 +25,58 @@ from glazing.verbnet.converter import VerbNetConverter from glazing.wordnet.converter import WordNetConverter +# Version of the converted-data layout. Bump whenever a converter change makes +# previously converted files stale, so that `glazing init` re-converts rather +# than leaving users on silently outdated data. +# +# 2: WordNet adjective satellites are retained (previously all 10,717 were +# dropped) and satellite senses are enriched. +DATA_SCHEMA_VERSION = 2 + + +def _read_schema_version(marker_file: Path) -> int | None: + """Read the data schema version recorded in an initialization marker. + + Parameters + ---------- + marker_file : Path + Path to the ``.initialized`` marker. + + Returns + ------- + int | None + Recorded schema version, or None if absent or unreadable. Markers + written before versioning was introduced are empty and read as None. + """ + try: + payload = json.loads(marker_file.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + version = payload.get("schema_version") if isinstance(payload, dict) else None + return version if isinstance(version, int) else None + + +def is_data_current(data_dir: Path | None = None) -> bool: + """Check whether converted data matches the current schema version. + + Parameters + ---------- + data_dir : Path | None + Data directory to check. If None, uses default. + + Returns + ------- + bool + True if the marker records the current schema version. + """ + if data_dir is None: + data_dir = get_default_data_dir() + + marker_file = Path(data_dir) / ".initialized" + if not marker_file.exists(): + return False + return _read_schema_version(marker_file) == DATA_SCHEMA_VERSION + def get_default_data_dir() -> Path: """Get the default data directory for glazing. @@ -260,12 +313,17 @@ def initialize_datasets( data_dir = Path(data_dir) data_dir.mkdir(parents=True, exist_ok=True) - # Check if already initialized (unless force is True) + # Check if already initialized at the current schema version. Data + # converted by an older glazing is stale even though the marker exists, + # so re-convert rather than leaving the user on outdated files. marker_file = data_dir / ".initialized" if marker_file.exists() and not force: + if _read_schema_version(marker_file) == DATA_SCHEMA_VERSION: + if verbose: + click.echo("Datasets already initialized. Use --force to re-download.") + return True if verbose: - click.echo("Datasets already initialized. Use --force to re-download.") - return True + click.echo("Converted data predates the current schema; re-converting.") if verbose: click.echo(f"Initializing glazing datasets in {data_dir}") @@ -276,9 +334,12 @@ def initialize_datasets( results = [_process_dataset(name, data_dir, verbose) for name in datasets] success = all(results) - # Create marker file + # Create marker file recording the schema the data was converted against if success: - marker_file.touch() + marker_file.write_text( + json.dumps({"schema_version": DATA_SCHEMA_VERSION}), + encoding="utf-8", + ) if verbose: click.echo("\n" + "=" * 60) click.echo("✅ All datasets successfully initialized!") From 28759724457ea88af6ef2967a44c24551cfe8f5e Mon Sep 17 00:00:00 2001 From: Aaron Steven White Date: Tue, 21 Jul 2026 11:19:43 -0400 Subject: [PATCH 5/6] chore(release): bump version to 0.3.0 Minor rather than patch release: WordNetLoader.synsets, lemma_index, and the relation indices change key format from bare offset to SynsetKey, which is a breaking change for anyone reading those mappings directly. Pre-1.0, so a minor bump is the semver-appropriate signal. Also backfills the 0.2.3 changelog link reference, which was never added. --- CHANGELOG.md | 28 +++++++++++++++++++++++++++- docs/api/index.md | 2 +- docs/citation.md | 8 ++++---- docs/index.md | 2 +- pyproject.toml | 2 +- src/glazing/__version__.py | 2 +- 6 files changed, 35 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0bb6772..384bef1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.0] - 2026-07-21 + +### Added + +- **Canonical synset keys.** `Synset.key` and `Pointer.target_key` pair an 8-digit offset with a normalized part of speech (e.g. `"00001740n"`), with `normalize_pos` and `make_synset_key` in `glazing.wordnet.types`. Offsets are byte offsets into a per-POS data file and 333 of them name more than one synset in WordNet 3.1, so an offset alone cannot identify a synset. Satellites normalize to `"a"`, matching the POS WordNet records in every pointer to an adjective, so a synset record and any pointer targeting it now produce the same key +- **Converted-data versioning.** The `.initialized` marker records a `DATA_SCHEMA_VERSION`; `glazing init` re-converts when it predates the current version, and importing `glazing` warns when converted data is stale. Previously the marker was empty, so upgraded users silently kept outdated files + +### Fixed + +- **WordNet adjective satellites** are no longer dropped during conversion; all 10,717 `ss_type="s"` synsets — about 59% of WordNet's adjectives — were discarded while the pointers referencing them were still emitted, leaving ~10,720 dangling references for any consumer following adjective relations (#11) +- **WordNet satellite sense enrichment** now populates `sense_number` and `tag_count` for satellites. Enrichment built a head-less sense key (`lemma%5:LL:II::`), but real satellite keys carry a `head_word:head_id` suffix that the data file does not record; lookups are now keyed on the synset offset from `index.sense`, which is unambiguous where the head-less prefix is not (2,966 of 20,384 satellite keys share one) +- **WordNet synsets sharing an offset across parts of speech** no longer overwrite each other in `WordNetLoader` or raise `ValueError` in `WordNetSearch`, and pointers resolve to the synset they name rather than to whichever synset happened to share the offset +- **WordNet lemmas beginning with a dot or apostrophe** (`.22_caliber`, `'hood`, `'tween`, `'s_Gravenhage`) no longer fail lemma validation, which discarded their entire synsets — 7 in total +- **`glazing search relations --dataset wordnet`** no longer fails with `Unknown format code 'd' for object of type 'str'`; synset IDs were built with `f"{synset.offset:08d}"` against a string offset + +### Changed + +- **BREAKING: `WordNetLoader.synsets`, `lemma_index`, and the four relation indices are keyed by `SynsetKey`** (e.g. `"00001740n"`) rather than by bare offset. `get_synset` still accepts a bare offset and returns `None` when one is ambiguous; pass `pos` or a canonical key to disambiguate. `WordNetSearch.by_offset` accepts either form +- **Adjective satellites are indexed under both `"s"` and `"a"`**, so a query for adjectives returns heads and satellites together while `ss_type` on the record still reports `"s"` + +### Upgrading + +Converted data from earlier versions is missing all 10,717 satellites and cannot be repaired in place. Run `glazing init` to re-convert from the already-downloaded source files; no re-download is needed. + ## [0.2.3] - 2026-06-25 ### Added @@ -224,7 +248,9 @@ Initial release of `glazing`, a package containing unified data models and inter - `tqdm >= 4.60.0` (progress bars) - `rich >= 13.0.0` (CLI formatting) -[Unreleased]: https://github.com/factslab/glazing/compare/v0.2.2...HEAD +[Unreleased]: https://github.com/factslab/glazing/compare/v0.3.0...HEAD +[0.3.0]: https://github.com/factslab/glazing/releases/tag/v0.3.0 +[0.2.3]: https://github.com/factslab/glazing/releases/tag/v0.2.3 [0.2.2]: https://github.com/factslab/glazing/releases/tag/v0.2.2 [0.2.1]: https://github.com/factslab/glazing/releases/tag/v0.2.1 [0.2.0]: https://github.com/factslab/glazing/releases/tag/v0.2.0 diff --git a/docs/api/index.md b/docs/api/index.md index dc0a2e8..602e190 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -118,7 +118,7 @@ except ValidationError as e: ## Version Compatibility -This documentation covers glazing version 0.2.3. Check your installed version: +This documentation covers glazing version 0.3.0. Check your installed version: ```python import glazing diff --git a/docs/citation.md b/docs/citation.md index ffc78f0..360322b 100644 --- a/docs/citation.md +++ b/docs/citation.md @@ -12,22 +12,22 @@ If you use Glazing in your research, please cite our work. title = {Glazing: Unified Data Models and Interfaces for Syntactic and Semantic Frame Ontologies}, year = {2025}, url = {https://github.com/factslab/glazing}, - version = {0.2.3}, + version = {0.3.0}, doi = {10.5281/zenodo.17467082} } ``` ### APA -White, A. S. (2025). *Glazing: Unified Data Models and Interfaces for Syntactic and Semantic Frame Ontologies* (Version 0.2.3) [Computer software]. https://github.com/factslab/glazing +White, A. S. (2025). *Glazing: Unified Data Models and Interfaces for Syntactic and Semantic Frame Ontologies* (Version 0.3.0) [Computer software]. https://github.com/factslab/glazing ### Chicago -White, Aaron Steven. 2025. *Glazing: Unified Data Models and Interfaces for Syntactic and Semantic Frame Ontologies*. Version 0.2.3. https://github.com/factslab/glazing. +White, Aaron Steven. 2025. *Glazing: Unified Data Models and Interfaces for Syntactic and Semantic Frame Ontologies*. Version 0.3.0. https://github.com/factslab/glazing. ### MLA -White, Aaron Steven. *Glazing: Unified Data Models and Interfaces for Syntactic and Semantic Frame Ontologies*. Version 0.2.3, 2025, https://github.com/factslab/glazing. +White, Aaron Steven. *Glazing: Unified Data Models and Interfaces for Syntactic and Semantic Frame Ontologies*. Version 0.3.0, 2025, https://github.com/factslab/glazing. ## Citing Datasets diff --git a/docs/index.md b/docs/index.md index 1b3b235..5fda816 100644 --- a/docs/index.md +++ b/docs/index.md @@ -93,7 +93,7 @@ If you use Glazing in your research, please cite: title = {Glazing: Unified Data Models and Interfaces for Syntactic and Semantic Frame Ontologies}, year = {2025}, url = {https://github.com/factslab/glazing}, - version = {0.2.3}, + version = {0.3.0}, doi = {10.5281/zenodo.17467082} } ``` diff --git a/pyproject.toml b/pyproject.toml index 3870dda..bf6b885 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "glazing" -version = "0.2.3" +version = "0.3.0" description = "Unified data models and interfaces for syntactic and semantic frame ontologies" readme = "README.md" requires-python = ">=3.13" diff --git a/src/glazing/__version__.py b/src/glazing/__version__.py index 6d75b81..2ed8aa7 100644 --- a/src/glazing/__version__.py +++ b/src/glazing/__version__.py @@ -1,4 +1,4 @@ """Version information for the glazing package.""" -__version__ = "0.2.3" +__version__ = "0.3.0" __version_info__ = tuple(int(i) for i in __version__.split(".")) From 4f586f206392b0b91b71e08f615523dc5d8bed1d Mon Sep 17 00:00:00 2001 From: Aaron Steven White Date: Tue, 21 Jul 2026 11:27:40 -0400 Subject: [PATCH 6/6] docs(changelog): replace em-dashes in the 0.3.0 entry Recast the two parenthetical asides as comma clauses and parentheses rather than swapping the character, so the sentences still read naturally. --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 384bef1..c964212 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,10 +16,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- **WordNet adjective satellites** are no longer dropped during conversion; all 10,717 `ss_type="s"` synsets — about 59% of WordNet's adjectives — were discarded while the pointers referencing them were still emitted, leaving ~10,720 dangling references for any consumer following adjective relations (#11) +- **WordNet adjective satellites** are no longer dropped during conversion; all 10,717 `ss_type="s"` synsets, about 59% of WordNet's adjectives, were discarded while the pointers referencing them were still emitted, leaving ~10,720 dangling references for any consumer following adjective relations (#11) - **WordNet satellite sense enrichment** now populates `sense_number` and `tag_count` for satellites. Enrichment built a head-less sense key (`lemma%5:LL:II::`), but real satellite keys carry a `head_word:head_id` suffix that the data file does not record; lookups are now keyed on the synset offset from `index.sense`, which is unambiguous where the head-less prefix is not (2,966 of 20,384 satellite keys share one) - **WordNet synsets sharing an offset across parts of speech** no longer overwrite each other in `WordNetLoader` or raise `ValueError` in `WordNetSearch`, and pointers resolve to the synset they name rather than to whichever synset happened to share the offset -- **WordNet lemmas beginning with a dot or apostrophe** (`.22_caliber`, `'hood`, `'tween`, `'s_Gravenhage`) no longer fail lemma validation, which discarded their entire synsets — 7 in total +- **WordNet lemmas beginning with a dot or apostrophe** (`.22_caliber`, `'hood`, `'tween`, `'s_Gravenhage`) no longer fail lemma validation, which discarded their entire synsets (7 in total) - **`glazing search relations --dataset wordnet`** no longer fails with `Unknown format code 'd' for object of type 'str'`; synset IDs were built with `f"{synset.offset:08d}"` against a string offset ### Changed