From 82cbfc4e51879fdd040af588a1aceda47f2a917c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9tur=20Orri=20Ragnarsson?= Date: Wed, 3 Aug 2022 17:06:22 +0000 Subject: [PATCH] Move trigram generation code from util into the package The code to generate normalized trigrams from text may be useful to users but was not included in the package distribution. It is now available in the icegrams package alongside the Ngrams class. --- .gitignore | 3 + setup.py | 2 +- src/icegrams/__init__.py | 1 + src/icegrams/utils.py | 187 +++++++++++++++++++++++++++++++++++++++ utils/rmh.py | 140 +---------------------------- 5 files changed, 194 insertions(+), 139 deletions(-) create mode 100644 src/icegrams/utils.py diff --git a/.gitignore b/.gitignore index 67a0797..aeaf063 100644 --- a/.gitignore +++ b/.gitignore @@ -81,3 +81,6 @@ $RECYCLE.BIN/ *.msi *.msm *.msp + +# Vim swap files +.*.swp diff --git a/setup.py b/setup.py index c3169c0..3293538 100644 --- a/setup.py +++ b/setup.py @@ -114,7 +114,7 @@ def read(*names, **kwargs): ], keywords=["nlp", "trigram", "ngram", "trigrams", "ngrams", "icelandic"], setup_requires=["cffi>=1.10.0"], - install_requires=["cffi>=1.10.0"], + install_requires=["cffi>=1.10.0", "tokenizer>=3.4.1"], cffi_modules=[ "src/icegrams/trie_build.py:ffibuilder" ], diff --git a/src/icegrams/__init__.py b/src/icegrams/__init__.py index cb58e1c..f5c05ea 100644 --- a/src/icegrams/__init__.py +++ b/src/icegrams/__init__.py @@ -36,6 +36,7 @@ # Expose the icegrams API from .ngrams import Ngrams, MAX_ORDER +from .utils import tokens, trigrams, trigrams_from_tokens __author__ = "Miðeind ehf." __copyright__ = "(C) 2020 Miðeind ehf." diff --git a/src/icegrams/utils.py b/src/icegrams/utils.py new file mode 100644 index 0000000..233f584 --- /dev/null +++ b/src/icegrams/utils.py @@ -0,0 +1,187 @@ +""" + + Icegrams: A trigrams library for Icelandic + + utils.py + + Copyright (C) 2020 Miðeind ehf. + + This software is licensed under the MIT License: + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without restriction, + including without limitation the rights to use, copy, modify, merge, + publish, distribute, sublicense, and/or sell copies of the Software, + and to permit persons to whom the Software is furnished to do so, + subject to the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + + This module contains helper functions for turning text into a token stream + or trigram stream that is normalized according to the same normalization + rules as were used to create the trigram database. + +""" + +from typing import ( + Iterator, Iterable, Tuple, List, Optional, Dict, Set, Callable, Type, Any +) +import os +from itertools import islice, tee +from tokenizer import tokenize, correct_spaces, Tok, TOK + + +# Obtain our path +basepath, _ = os.path.split(os.path.realpath(__file__)) + + +# A set of all words we need to change +CHANGING: Set[str] = set() +# fACE_SK.txt +REPLACING: Dict[str, str] = dict() +# d.txt +DELETING: Set[str] = set() +# fMW.txt +DOUBLING: Dict[str, List[str]] = dict() + + +def load_corrections() -> None: + """ Fills global data structures for correcting tokens """ + with open(os.path.join(basepath, "resources", "correct.txt"), "r", encoding="utf-8") as myfile: + for line in myfile: + key, val = line.strip().split("\t") + REPLACING[key] = val + CHANGING.add(key) + with open(os.path.join(basepath, "resources", "delete.txt"), "r", encoding="utf-8") as myfile: + for line in myfile: + key = line.strip() + DELETING.add(key) + CHANGING.add(key) + with open(os.path.join(basepath, "resources", "split.txt"), "r", encoding="utf-8") as myfile: + for line in myfile: + key, val = line.strip().split("\t") + val = val.strip() + DOUBLING[key] = val.split() + CHANGING.add(key) + if key.islower() and val.islower(): + # Also add uppercase version + key = key.capitalize() + val = val.capitalize() + DOUBLING[key] = val.split() + CHANGING.add(key) + + +load_corrections() + + +def handle_word(token: Tok) -> Iterator[str]: + """ Return the text of a word token, after potential editing """ + t = token.txt + if t in CHANGING: + # We take a closer look + if t in REPLACING: + # Words we simply need to replace + yield REPLACING[t] + elif t in DELETING: + # Words that don't belong in trigrams + pass + elif t in DOUBLING: + # Words incorrectly in one token + for part in DOUBLING[t]: + yield part + else: + # Should not happen + assert False + elif " " in t: + yield from t.split() + else: + yield t + + +def handle_punctuation(token: Tok) -> Iterator[str]: + """ Return the normalized version of punctuation """ + yield token.val[1] + + +def handle_passthrough(token: Tok) -> Iterator[str]: + """ Return the token text unchanged """ + yield token.txt + + +def handle_split(token: Tok) -> Iterator[str]: + """ Split the token text by spaces and return the components """ + yield from token.txt.split() + + +def handle_measurement(token: Tok) -> Iterator[str]: + """ Return a [NUMBER] token followed by a SI unit """ + yield from ("[NUMBER]", token.val[0]) + + +def handle_none(token: Tok) -> Iterator[str]: + """ Empty generator """ + yield from () + + +def handle_other(token: Tok) -> Iterator[str]: + """ Return a standard placeholder for the token kind """ + yield "[" + TOK.descr[token.kind] + "]" + + +# Dispatch the various token types to the appropriate generators, +# defaulting to handle_others +token_dispatch: Dict[int, Callable[[Tok], Iterator[str]]] = { + TOK.WORD: handle_word, + TOK.PUNCTUATION: handle_punctuation, + TOK.MEASUREMENT: handle_measurement, + TOK.MOLECULE: handle_passthrough, + TOK.PERSON: handle_split, + TOK.ENTITY: handle_split, + TOK.COMPANY: handle_split, + TOK.S_BEGIN: handle_none, + TOK.S_END: handle_none, + TOK.P_BEGIN: handle_none, + TOK.P_END: handle_none, + TOK.S_SPLIT: handle_none, +} + + +def tokens(text: str, pad: bool = True) -> Iterator[str]: + """ Generate normalized tokens that are suitable for use with Icegrams. + Optionally pad the token stream at the beginning and end with empty + strings to make trigram generation more convenient. """ + toklist = list(tokenize(text, convert_measurements=True, replace_html_escapes=True)) + if len(toklist) > 1 and all(t.kind != TOK.UNKNOWN for t in toklist): + if pad: + yield "" + yield "" + for t in toklist: + yield from token_dispatch.get(t.kind, handle_other)(t) + if pad: + yield "" + yield "" + + +def trigrams_from_tokens(iterable: Iterable[str]) -> Iterator[Tuple[Any, ...]]: + """ Generate trigrams (tuples of three strings) from the given iterable """ + return zip( + *((islice(seq, i, None) for i, seq in enumerate(tee(iterable, 3)))) + ) + + +def trigrams(text: str) -> Iterator[Tuple[Any, ...]]: + """ Generate normalized trigrams that are suitable for use with Icegrams. """ + return trigrams_from_tokens(tokens(text, pad=True)) + + diff --git a/utils/rmh.py b/utils/rmh.py index ecad04c..d730632 100755 --- a/utils/rmh.py +++ b/utils/rmh.py @@ -41,10 +41,7 @@ Iterator, Iterable, Tuple, List, Optional, Dict, Set, Callable, Type, Any ) -import os import sys -from itertools import islice, tee -import collections import argparse import glob import random @@ -63,26 +60,13 @@ from sqlalchemy.exc import SQLAlchemyError as DatabaseError # type: ignore from sqlalchemy.schema import Table # type: ignore -from tokenizer import tokenize, correct_spaces, Tok, TOK +from icegrams import trigrams -# Obtain our path -basepath, _ = os.path.split(os.path.realpath(__file__)) - # Create the SQLAlchemy ORM Base class Base: Type[Table] = declarative_base() -# A set of all words we need to change -CHANGING: Set[str] = set() -# fACE_SK.txt -REPLACING: Dict[str, str] = dict() -# d.txt -DELETING: Set[str] = set() -# fMW.txt -DOUBLING: Dict[str, List[str]] = dict() - - # Define the command line arguments parser = argparse.ArgumentParser( @@ -270,128 +254,10 @@ def __repr__(self): return "Trigram(t1='{0}', t2='{1}', t3='{2}')".format(self.t1, self.t2, self.t3) -def load_corrections() -> None: - """ Fills global data structures for correcting tokens """ - with open(os.path.join(basepath, "correct.txt"), "r", encoding="utf-8") as myfile: - for line in myfile: - key, val = line.strip().split("\t") - REPLACING[key] = val - CHANGING.add(key) - with open(os.path.join(basepath, "delete.txt"), "r", encoding="utf-8") as myfile: - for line in myfile: - key = line.strip() - DELETING.add(key) - CHANGING.add(key) - with open(os.path.join(basepath, "split.txt"), "r", encoding="utf-8") as myfile: - for line in myfile: - key, val = line.strip().split("\t") - val = val.strip() - DOUBLING[key] = val.split() - CHANGING.add(key) - if key.islower() and val.islower(): - # Also add uppercase version - key = key.capitalize() - val = val.capitalize() - DOUBLING[key] = val.split() - CHANGING.add(key) - - -def handle_word(token: Tok) -> Iterator[str]: - """ Return the text of a word token, after potential editing """ - t = token.txt - if t in CHANGING: - # We take a closer look - if t in REPLACING: - # Words we simply need to replace - yield REPLACING[t] - elif t in DELETING: - # Words that don't belong in trigrams - pass - elif t in DOUBLING: - # Words incorrectly in one token - for part in DOUBLING[t]: - yield part - else: - # Should not happen - assert False - elif " " in t: - yield from t.split() - else: - yield t - - -def handle_punctuation(token: Tok) -> Iterator[str]: - """ Return the normalized version of punctuation """ - yield token.val[1] - - -def handle_passthrough(token: Tok) -> Iterator[str]: - """ Return the token text unchanged """ - yield token.txt - - -def handle_split(token: Tok) -> Iterator[str]: - """ Split the token text by spaces and return the components """ - yield from token.txt.split() - - -def handle_measurement(token: Tok) -> Iterator[str]: - """ Return a [NUMBER] token followed by a SI unit """ - yield from ("[NUMBER]", token.val[0]) - - -def handle_none(token: Tok) -> Iterator[str]: - """ Empty generator """ - yield from () - - -def handle_other(token: Tok) -> Iterator[str]: - """ Return a standard placeholder for the token kind """ - yield "[" + TOK.descr[token.kind] + "]" - - -# Dispatch the various token types to the appropriate generators, -# defaulting to handle_others -token_dispatch: Dict[int, Callable[[Tok], Iterator[str]]] = { - TOK.WORD: handle_word, - TOK.PUNCTUATION: handle_punctuation, - TOK.MEASUREMENT: handle_measurement, - TOK.MOLECULE: handle_passthrough, - TOK.PERSON: handle_split, - TOK.ENTITY: handle_split, - TOK.COMPANY: handle_split, - TOK.S_BEGIN: handle_none, - TOK.S_END: handle_none, - TOK.P_BEGIN: handle_none, - TOK.P_END: handle_none, - TOK.S_SPLIT: handle_none, -} - - -def tokens(text: str) -> Iterator[str]: - """ Generator for token stream """ - toklist = list(tokenize(text, convert_measurements=True, replace_html_escapes=True)) - if len(toklist) > 1 and all(t.kind != TOK.UNKNOWN for t in toklist): - # For each sentence, start and end with empty strings - yield "" - yield "" - for t in toklist: - yield from token_dispatch.get(t.kind, handle_other)(t) - yield "" - yield "" - - -def trigrams(iterable: Iterable[str]) -> Iterator[Tuple[Any, ...]]: - """ Generate trigrams (tuples of three strings) from the given iterable """ - return zip( - *((islice(seq, i, None) for i, seq in enumerate(tee(iterable, 3)))) - ) - - def process(session: SessionContext, text: str) -> int: """ Process a single line of text from an RMH file """ upserted = 0 - for tg in trigrams(tokens(text)): + for tg in trigrams(text): if any(tg): try: Trigram.upsert(session, *tg) @@ -430,8 +296,6 @@ def make_trigrams(path_iterator: Iterable[str], *, limit: int = None): FLUSH_THRESHOLD = 10000 - load_corrections() - with SessionContext(commit=False) as session: # Delete existing trigrams