diff --git a/epitran/_epitran.py b/epitran/_epitran.py index c268d77..c4db2f3 100644 --- a/epitran/_epitran.py +++ b/epitran/_epitran.py @@ -12,16 +12,29 @@ logger.setLevel(logging.WARNING) class Epitran(object): - """Unified interface for IPA transliteration/transcription - - :param code str: ISO 639-3 plus "-" plus ISO 15924 code of the language/script pair that should be loaded - :param preproc bool: apply preprocessors - :param postproc bool: apply postprocessors - :param ligatures bool: use precomposed ligatures instead of standard IPA - :param cedict_filename str: path to file containing the CC-CEDict dictionary - :param rev boolean: use reverse transliteration - :param rev_preproc bool: if True, apply preprocessors when reverse transliterating - :param rev_postproc bool: if True, apply postprocessors when reverse transliterating + """Unified interface for IPA transliteration/transcription. + + Parameters + ---------- + code : str + ISO 639-3 plus "-" plus ISO 15924 code of the language/script pair + that should be loaded. + preproc : bool, optional + Apply preprocessors. Default is True. + postproc : bool, optional + Apply postprocessors. Default is True. + ligatures : bool, optional + Use precomposed ligatures instead of standard IPA. Default is False. + cedict_file : str or None, optional + Path to file containing the CC-CEDict dictionary. Default is None. + rev : bool, optional + Use reverse transliteration. Default is False. + rev_preproc : bool, optional + If True, apply preprocessors when reverse transliterating. Default is True. + rev_postproc : bool, optional + If True, apply postprocessors when reverse transliterating. Default is True. + tones : bool, optional + Handle tone information. Default is False. """ @final special = {'eng-Latn': FliteLexLookup, @@ -31,20 +44,31 @@ class Epitran(object): 'yue-Hant': EpiCanto, } - def __init__(self, code: str, **kwargs): - """Constructor method - - Args: - code (str): ISO 639-3 code and ISO 15924 code joined with a hyphen - **kwargs: Additional parameters passed to the appropriate backend: - preproc (bool): if True, apply preprocessor (default: True) - postproc (bool): if True, apply postprocessors (default: True) - ligatures (bool): if True, use phonetic ligatures (default: False) - cedict_file (str): path to dictionary file for Chinese/Japanese (default: None) - rev (bool): if True, load reverse transliteration (default: False) - rev_preproc (bool): if True, apply preprocessor when reverse transliterating (default: True) - rev_postproc (bool): if True, apply postprocessor when reverse transliterating (default: True) - tones (bool): if True, include tone information (default: False) + def __init__(self, code: str, preproc: bool=True, postproc: bool=True, ligatures: bool=False, + cedict_file: Union[bool, None]=None, rev: bool=False, + rev_preproc: bool=True, rev_postproc: bool=True, tones: bool=False): + """Initialize Epitran transliterator. + + Parameters + ---------- + code : str + ISO 639-3 plus "-" plus ISO 15924 code of the language/script pair. + preproc : bool, optional + Apply preprocessors. Default is True. + postproc : bool, optional + Apply postprocessors. Default is True. + ligatures : bool, optional + Use precomposed ligatures instead of standard IPA. Default is False. + cedict_file : bool or None, optional + Path to CC-CEDict dictionary file. Default is None. + rev : bool, optional + Use reverse transliteration. Default is False. + rev_preproc : bool, optional + Apply preprocessors when reverse transliterating. Default is True. + rev_postproc : bool, optional + Apply postprocessors when reverse transliterating. Default is True. + tones : bool, optional + Handle tone information. Default is False. """ if code in self.special: self.epi = self.special[code](**kwargs) @@ -55,87 +79,154 @@ def __init__(self, code: str, **kwargs): self.puncnorm = PuncNorm() def transliterate(self, word: str, normpunc: bool=False, ligatures: bool=False) -> str: - """Transliterates/transcribes a word into IPA - - :param word str: word to transcribe - :param normpunc bool: if True, normalize punctuation - :param ligatures bool: if True, use precomposed ligatures instead of standard IPA - :return: An IPA string corresponding to the input orthographic string - :rtype: str + """Transliterate/transcribe a word into IPA. + + Parameters + ---------- + word : str + Word to transcribe. + normpunc : bool, optional + If True, normalize punctuation. Default is False. + ligatures : bool, optional + If True, use precomposed ligatures instead of standard IPA. + Default is False. + + Returns + ------- + str + An IPA string corresponding to the input orthographic string. """ return self.epi.transliterate(word, normpunc, ligatures) def reverse_transliterate(self, ipa: str) -> str: - """Reconstructs word from IPA. Does the reverse of transliterate() + """Reconstruct word from IPA. Does the reverse of transliterate(). + + Parameters + ---------- + ipa : str + An IPA representation of a word. - :param ipa str: An IPA representation of a word - :return: An orthographic representation of the word - :rtype: str + Returns + ------- + str + An orthographic representation of the word. """ return self.epi.reverse_transliterate(ipa) def strict_trans(self, word: str, normpunc:bool =False, ligatures: bool=False) -> str: - """Transliterate a word into IPA, ignoring all characters that cannot be recognized. - - :param word str: word to transcribe - :param normpunc bool, optional: if True, normalize punctuation - :param ligatures bool, optional: if True, use precomposed ligatures instead of standard IPA - :return: An IPA string corresponding to the input orthographic string, with all uncoverted characters omitted - :rtype: str + """Transliterate a word into IPA, ignoring unrecognized characters. + + Parameters + ---------- + word : str + Word to transcribe. + normpunc : bool, optional + If True, normalize punctuation. Default is False. + ligatures : bool, optional + If True, use precomposed ligatures instead of standard IPA. + Default is False. + + Returns + ------- + str + An IPA string corresponding to the input orthographic string, + with all unconverted characters omitted. """ return self.epi.strict_trans(word, normpunc, ligatures) def trans_list(self, word: str, normpunc: bool=False, ligatures: bool=False) -> "list[str]": - """Transliterates/transcribes a word into list of IPA phonemes - - :param word str: word to transcribe - :param normpunc bool, optional: if True, normalize punctuation - :param ligatures bool, optional: if True, use precomposed ligatures instead of standard IPA - :return: list of IPA strings, each corresponding to a segment - :rtype: list[str] + """Transliterate/transcribe a word into list of IPA phonemes. + + Parameters + ---------- + word : str + Word to transcribe. + normpunc : bool, optional + If True, normalize punctuation. Default is False. + ligatures : bool, optional + If True, use precomposed ligatures instead of standard IPA. + Default is False. + + Returns + ------- + list of str + List of IPA strings, each corresponding to a segment. """ return self.ft.segs_safe(self.epi.transliterate(word, normpunc, ligatures)) - def trans_delimiter(self, text: str, delimiter: str=str(' '), normpunc: bool=False, ligatures: bool=False) -> str: - """Return IPA transliteration with a delimiter between segments - - :param text str: An orthographic text - :param delimiter str, optional: A string to insert between segments - :param normpunc bool, optional: If True, normalize punctuation - :param ligatures bool, optional: If True, use precomposed ligatures instead of standard IPA - :return: String of IPA phonemes separated by `delimiter` - :rtype: str + def trans_delimiter(self, text: str, delimiter: str=str(' '), normpunc: bool=False, ligatures: bool=False): + """Return IPA transliteration with a delimiter between segments. + + Parameters + ---------- + text : str + An orthographic text. + delimiter : str, optional + A string to insert between segments. Default is ' '. + normpunc : bool, optional + If True, normalize punctuation. Default is False. + ligatures : bool, optional + If True, use precomposed ligatures instead of standard IPA. + Default is False. + + Returns + ------- + str + String of IPA phonemes separated by `delimiter`. """ return delimiter.join(self.trans_list(text, normpunc=normpunc, ligatures=ligatures)) - def xsampa_list(self, word: str, normpunc: bool=False, ligaturize: bool=False) -> "list[str]": - """Transliterates/transcribes a word as X-SAMPA - - :param word str: An orthographic word - :param normpunc bool, optional: If True, normalize punctuation - :param ligatures bool, optional: If True, use precomposed ligatures instead of standard IPA - :return: List of X-SAMPA strings corresponding to `word` - :rtype: list[str] + def xsampa_list(self, word: str, normpunc: bool=False, ligaturize: bool=False): + """Transliterate/transcribe a word as X-SAMPA. + + Parameters + ---------- + word : str + An orthographic word. + normpunc : bool, optional + If True, normalize punctuation. Default is False. + ligaturize : bool, optional + If True, use precomposed ligatures instead of standard IPA. + Default is False. + + Returns + ------- + list of str + List of X-SAMPA strings corresponding to `word`. """ ipa_segs = self.ft.ipa_segs(self.epi.strict_trans(word, normpunc, ligaturize)) return list(map(self.xsampa.ipa2xs, ipa_segs)) - def word_to_tuples(self, word: str, normpunc: bool=False, _ligaturize: bool=False) -> "list[tuple[str, str, str, str, list[int]]]": - """Given a word, returns a list of tuples corresponding to IPA segments. The "feature - vectors" form a list consisting of (segment, vector) pairs. + def word_to_tuples(self, word: str, normpunc: bool=False, _ligaturize: bool=False): + """Convert a word to a list of tuples corresponding to IPA segments. + + The "feature vectors" form a list consisting of (segment, vector) pairs. For IPA segments, segment is a substring of phonetic_form such that the concatenation of all segments in the list is equal to the phonetic_form. The vectors are a sequence of integers drawn from the set {-1, 0, 1} - where -1 corresponds to '-', 0 corresponds to '0', and 1 corresponds to - '+'. - - :param word str: An orthographic word - :param normpunc bool, optional: If True, normalize punctuation - :param ligatures bool, optional: If True, use precomposed ligatures instead of standard IPA - :return: A list of tuples corresponding to IPA segments - :rtype: list[tuple[str, str, str, str, list[int]]] + where -1 corresponds to '-', 0 corresponds to '0', and 1 corresponds to '+'. + + Parameters + ---------- + word : str + An orthographic word. + normpunc : bool, optional + If True, normalize punctuation. Default is False. + _ligaturize : bool, optional + If True, use precomposed ligatures instead of standard IPA. + Default is False. + + Returns + ------- + list of tuple + A list of tuples corresponding to IPA segments. + + Raises + ------ + AttributeError + If method is not implemented for this language-script pair. """ try: return self.epi.word_to_tuples(word, normpunc) diff --git a/epitran/backoff.py b/epitran/backoff.py index c9ce28f..4931201 100644 --- a/epitran/backoff.py +++ b/epitran/backoff.py @@ -15,11 +15,14 @@ class Backoff(object): def __init__(self, lang_script_codes: List[str], cedict_file: Optional[str] = None) -> None: """Construct a Backoff object. - Args: - lang_script_codes (list): codes for languages to try, starting - with the highest priority languages - cedict_file (str): path to the CC-CEdict dictionary file - (necessary only when cmn-Hans or cmn-Hant are used) + Parameters + ---------- + lang_script_codes : list + Codes for languages to try, starting with the highest priority + languages. + cedict_file : str, optional + Path to the CC-CEdict dictionary file (necessary only when + cmn-Hans or cmn-Hant are used). Default is None. """ self.langs = [_epitran.Epitran(c, cedict_file=cedict_file) for c in lang_script_codes] @@ -31,10 +34,16 @@ def __init__(self, lang_script_codes: List[str], cedict_file: Optional[str] = No def transliterate(self, token: str) -> str: """Return IPA transliteration given by first acceptable mode. - Args: - token (unicode): orthographic text - Returns: - str: transliteration as Unicode IPA string + + Parameters + ---------- + token : str + Orthographic text. + + Returns + ------- + str + Transliteration as Unicode IPA string. """ tr_list = [] while token: @@ -64,22 +73,30 @@ def transliterate(self, token: str) -> str: def trans_list(self, token: str) -> List[str]: """Transliterate/transcribe a word into list of IPA phonemes. - Args: - token (unicode): word to transcribe; unicode string + Parameters + ---------- + token : str + Word to transcribe. - Returns: - list: list of IPA unicode strings, each corresponding to a segment + Returns + ------- + list + List of IPA unicode strings, each corresponding to a segment. """ return self.ft.segs_safe(self.transliterate(token)) def xsampa_list(self, token: str) -> Union[str, List[str]]: """Transcribe a word into a list of X-SAMPA phonemes. - Args: - token (unicode): word to transcribe; unicode strings + Parameters + ---------- + token : str + Word to transcribe. - Returns: - list: list of X-SAMPA strings, each corresponding to a segment + Returns + ------- + list + List of X-SAMPA strings, each corresponding to a segment. """ if re.match(r'^\p{Number}+$', token): return '' diff --git a/epitran/bin/connl2engipaspace.py b/epitran/bin/connl2engipaspace.py index d05c5e4..a4e02c5 100644 --- a/epitran/bin/connl2engipaspace.py +++ b/epitran/bin/connl2engipaspace.py @@ -2,12 +2,11 @@ import argparse import codecs +import csv import logging from collections import Counter from typing import List -import csv - import epitran import epitran.flite import panphon @@ -56,7 +55,7 @@ def add_file(flite: epitran.flite.Flite, ft: panphon.FeatureTable, fn: str) -> C def print_space(output: str, space: Counter[str]) -> None: pairs = enumerate(sorted(filter(lambda x: x, space.keys()))) - with open(output, 'wb') as f: + with open(output, 'w', encoding='utf-8', newline='') as f: writer = csv.writer(f) for i, char in pairs: writer.writerow((i, char)) diff --git a/epitran/bin/connl2ipaspace.py b/epitran/bin/connl2ipaspace.py index 30f66d6..692217d 100644 --- a/epitran/bin/connl2ipaspace.py +++ b/epitran/bin/connl2ipaspace.py @@ -74,7 +74,7 @@ def add_file_op(epi: epitran.Epitran, ft: panphon.FeatureTable, fn: str) -> Coun def print_space(output: str, space: Counter[str]) -> None: pairs = enumerate(sorted(filter(lambda x: x, space.keys()))) - with open(output, 'wb') as f: + with open(output, 'w', encoding='utf-8', newline='') as f: writer = csv.writer(f) for i, char in pairs: writer.writerow((i, char)) diff --git a/epitran/bin/migraterules.py b/epitran/bin/migraterules.py index 2aa79a3..15e4f86 100644 --- a/epitran/bin/migraterules.py +++ b/epitran/bin/migraterules.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- +import csv import glob import re from typing import List, Optional @@ -26,7 +27,7 @@ def main() -> None: reader = csv.reader(f) next(reader) for fields in reader: - if re.match('\s*%', fields[0]): + if re.match(r'\s*%', fields[0]): print(','.join([x for x in fields if x]), file=g) else: rule = build_rule(fields) diff --git a/epitran/bin/space2punc.py b/epitran/bin/space2punc.py index 2713e8e..4b305f8 100644 --- a/epitran/bin/space2punc.py +++ b/epitran/bin/space2punc.py @@ -14,7 +14,7 @@ def main(fns: List[str], fnn: str) -> None: for _, s in reader: if len(s) == 1 and unicodedata.category(s)[0] == u'P': punc.add(s) - with open(fnn, 'wb') as f: + with open(fnn, 'w', encoding='utf-8', newline='') as f: writer = csv.writer(f) for mark in sorted(list(punc)): writer.writerow([mark]) diff --git a/epitran/bin/uigtransliterate.py b/epitran/bin/uigtransliterate.py index b28467b..28f9a48 100644 --- a/epitran/bin/uigtransliterate.py +++ b/epitran/bin/uigtransliterate.py @@ -5,5 +5,5 @@ epi = epitran.Epitran('uig-Arab') for line in fileinput.input(): - s = epi.transliterate(line.strip().decode('utf-8')) - print(s.encode('utf-8')) + s = epi.transliterate(line.strip()) + print(s) diff --git a/epitran/data/map/count_phones.py b/epitran/data/map/count_phones.py index b8eab45..e81b150 100644 --- a/epitran/data/map/count_phones.py +++ b/epitran/data/map/count_phones.py @@ -1,8 +1,8 @@ #!/usr/bin/env +import csv import epitran.xsampa import panphon -import csv def main(fn): diff --git a/epitran/flite.py b/epitran/flite.py index 290eabd..492fa80 100644 --- a/epitran/flite.py +++ b/epitran/flite.py @@ -1,33 +1,37 @@ # -*- coding: utf-8 -*- +import csv import logging import os.path import string +import subprocess import unicodedata from typing import Dict, List, Any import regex as re import panphon -import csv from epitran.ligaturize import ligaturize from epitran.puncnorm import PuncNorm -import subprocess - logging.basicConfig(level=logging.CRITICAL) logger = logging.getLogger('epitran') class Flite(object): """English G2P using the Flite speech synthesis system.""" - def __init__(self, arpabet: str = 'arpabet', ligatures: bool = False, **kwargs) -> None: - """Construct a Flite "wrapper" - - Args: - arpabet (str): file containing ARPAbet to IPA mapping - ligatures (bool): if True, use non-standard ligatures instead of - standard IPA + def __init__(self, arpabet='arpabet', ligatures=False, **kwargs): + """Construct a Flite wrapper. + + Parameters + ---------- + arpabet : str, optional + File containing ARPAbet to IPA mapping. Default is 'arpabet'. + ligatures : bool, optional + If True, use non-standard ligatures instead of standard IPA. + Default is False. + **kwargs + Additional keyword arguments. """ arpabet = os.path.join(os.path.dirname(__file__), os.path.join('data', arpabet + '.csv')) self.arpa_map = self._read_arpabet(arpabet) @@ -68,14 +72,23 @@ def english_g2p(self, english: str) -> str: """Stub for English G2P function to be overwritten by subclasses""" return "" - def transliterate(self, text: str, normpunc: bool = False, ligatures: bool = False) -> str: - """Convert English text to IPA transcription - - Args: - text (str): English text - normpunc (bool): if True, normalize punctuation downward - ligatures (bool): if True, use non-standard ligatures instead of - standard IPA + def transliterate(self, text, normpunc=False, ligatures=False): + """Convert English text to IPA transcription. + + Parameters + ---------- + text : str + English text. + normpunc : bool, optional + If True, normalize punctuation downward. Default is False. + ligatures : bool, optional + If True, use non-standard ligatures instead of standard IPA. + Default is False. + + Returns + ------- + str + IPA transcription of the input text. """ text = unicodedata.normalize('NFC', text) acc = [] @@ -92,16 +105,8 @@ def transliterate(self, text: str, normpunc: bool = False, ligatures: bool = Fal def strict_trans(self, text: str, normpunc: bool = False, ligatures: bool = False) -> str: return self.transliterate(text, normpunc, ligatures) - def word_to_tuples(self, word: str, normpunc: bool = False) -> List[Any]: - """Given a word, returns a list of tuples corresponding to IPA segments. - - Args: - word (str): word to transliterate - normpunc (bool): If True, normalizes punctuation to ASCII inventory - - Returns: - list: A list of (category, lettercase, orthographic_form, - phonetic_form, feature_vectors) tuples. + def word_to_tuples(self, word, normpunc=False): + """Convert a word to a list of tuples corresponding to IPA segments. The "feature vectors" form a list consisting of (segment, vector) pairs. For IPA segments, segment is a substring of phonetic_form such that the @@ -109,6 +114,19 @@ def word_to_tuples(self, word: str, normpunc: bool = False) -> List[Any]: The vectors are a sequence of integers drawn from the set {-1, 0, 1} where -1 corresponds to '-', 0 corresponds to '0', and 1 corresponds to '+'. + + Parameters + ---------- + word : str + Word to transliterate. + normpunc : bool, optional + If True, normalizes punctuation to ASCII inventory. Default is False. + + Returns + ------- + list + A list of (category, lettercase, orthographic_form, + phonetic_form, feature_vectors) tuples. """ def cat_and_cap(c): cat, case = tuple(unicodedata.category(c)) diff --git a/epitran/reromanize.py b/epitran/reromanize.py index 41669ba..ae1a13f 100644 --- a/epitran/reromanize.py +++ b/epitran/reromanize.py @@ -1,4 +1,4 @@ - +import csv import os.path import sys from unicodedata import normalize @@ -7,7 +7,6 @@ from importlib import resources import epitran -import csv class ReRomanizer(object): diff --git a/epitran/simple.py b/epitran/simple.py index 0140be8..59bf31e 100644 --- a/epitran/simple.py +++ b/epitran/simple.py @@ -22,40 +22,53 @@ class SimpleEpitran(object): - """The backend object epitran uses for most languages - - :param code str: ISO 639-3 code and ISO 15924 code joined with a hyphen - :param preproc bool, optional: if True, apply preprocessor - :param postproc bool, optional: if True, apply postprocessors - :param ligatures bool, optional: if True, use phonetic ligatures for affricates instead of - standard IPA - :param rev bool, optional: if True, load reverse transliteration - :param rev_preproc bool, optional: if True, applyy preprocessor when reverse transliterating - :param rev_postproc bool, optional: if True, applyy postprocessor when reverse transliterating + """The backend object epitran uses for most languages. + + Parameters + ---------- + code : str + ISO 639-3 code and ISO 15924 code joined with a hyphen. + preproc : bool, optional + If True, apply preprocessor. Default is True. + postproc : bool, optional + If True, apply postprocessors. Default is True. + ligatures : bool, optional + If True, use phonetic ligatures for affricates instead of standard IPA. + Default is False. + rev : bool, optional + If True, load reverse transliteration. Default is False. + rev_preproc : bool, optional + If True, apply preprocessor when reverse transliterating. Default is True. + rev_postproc : bool, optional + If True, apply postprocessor when reverse transliterating. Default is True. + tones : bool, optional + Handle tone information. Default is False. """ - def __init__(self, code: str, **kwargs): - """Constructor - - Args: - code (str): ISO 639-3 code and ISO 15924 code joined with a hyphen - **kwargs: Optional parameters: - preproc (bool): if True, apply preprocessor (default: True) - postproc (bool): if True, apply postprocessors (default: True) - ligatures (bool): if True, use phonetic ligatures (default: False) - rev (bool): if True, load reverse transliteration (default: False) - rev_preproc (bool): if True, apply preprocessor when reverse transliterating (default: True) - rev_postproc (bool): if True, apply postprocessor when reverse transliterating (default: True) - tones (bool): if True, include tone information (default: False) + def __init__(self, code: str, preproc: bool = True, postproc: bool = True, ligatures: bool = False, + rev: bool = False, rev_preproc: bool = True, rev_postproc: bool = True, tones: bool = False): + """Initialize SimpleEpitran transliterator. + + Parameters + ---------- + code : str + ISO 639-3 code and ISO 15924 code joined with a hyphen. + preproc : bool, optional + If True, apply preprocessor. Default is True. + postproc : bool, optional + If True, apply postprocessors. Default is True. + ligatures : bool, optional + If True, use phonetic ligatures for affricates instead of standard IPA. + Default is False. + rev : bool, optional + If True, load reverse transliteration. Default is False. + rev_preproc : bool, optional + If True, apply preprocessor when reverse transliterating. Default is True. + rev_postproc : bool, optional + If True, apply postprocessor when reverse transliterating. Default is True. + tones : bool, optional + Handle tone information. Default is False. """ - # Extract parameters with defaults - preproc = kwargs.get('preproc', True) - postproc = kwargs.get('postproc', True) - ligatures = kwargs.get('ligatures', False) - rev = kwargs.get('rev', False) - rev_preproc = kwargs.get('rev_preproc', True) - rev_postproc = kwargs.get('rev_postproc', True) - tones = kwargs.get('tones', False) self.rev = rev self.tones = tones self.g2p = self._load_g2p_map(code, False) @@ -77,7 +90,7 @@ def __init__(self, code: str, **kwargs): self.rev_preprocessor = PrePostProcessor(code, 'pre', True) self.rev_postprocessor = PrePostProcessor(code, 'post', True) - self.nils = defaultdict(int) + self.nils: "defaultdict[str, int]" = defaultdict(int) def get_tones(self) -> bool: """Returns True if support for tones is turned on. @@ -107,10 +120,23 @@ def _non_deterministic_mappings(self, gr_by_line: "dict[str, list[int]]") -> "li def _load_g2p_map(self, code: str, rev: bool) -> "DefaultDict[str, list[str]]": """Load the code table for the specified language. - :param code str: ISO 639-3 code plus "-" plus ISO 15924 code for the language/script to be loaded - :param rev bool: If True, reverse the table (for reverse transliterating) - :return: A mapping from graphemes to phonemes - :rtype: DefaultDict[str, list[str]] + Parameters + ---------- + code : str + ISO 639-3 code plus "-" plus ISO 15924 code for the language/script + to be loaded. + rev : bool + If True, reverse the table (for reverse transliterating). + + Returns + ------- + DefaultDict[str, list[str]] + A mapping from graphemes to phonemes. + + Raises + ------ + DatafileError + If appropriately-named mapping is not found in data/maps directory. """ g2p = defaultdict(list) gr_by_line = defaultdict(list) @@ -165,16 +191,25 @@ def _construct_regex(self, g2p_keys: Any) -> Any: return regex.compile(f"({r'|'.join(graphemes)})", regex.I) def general_trans(self, text: str, filter_func: "Callable[[tuple[str, bool]], bool]", - normpunc: bool = False, ligatures: bool = False) -> str: - """Transliaterates a word into IPA, filtering with filter_func - - :param text str: word to transcribe; unicode string - :param filter_func Callable[[tuple[str, bool]], bool]: function for filtering - segments; takes a tuple and returns a boolean. - :param normpunct bool: normalize punctuation - :param ligatures bool: use precompsed ligatures instead of standard IPA - :return: IPA string corresponding to the orthographic input `text` - :rtype: str + normpunc: bool = False, ligatures: bool = False): + """Transliterate a word into IPA, filtering with filter_func. + + Parameters + ---------- + text : str + Word to transcribe. + filter_func : Callable[[tuple[str, bool]], bool] + Function for filtering segments; takes a tuple + and returns a boolean. + normpunc : bool, optional + Normalize punctuation. Default is False. + ligatures : bool, optional + Use precomposed ligatures instead of standard IPA. Default is False. + + Returns + ------- + str + IPA string corresponding to the orthographic input `text`. """ text = unicodedata.normalize('NFD', text.lower()) logger.debug('(after norm) text=%s', repr(list(text))) @@ -221,17 +256,26 @@ def is_korean(self, text: str) -> bool: return True return False - def transliterate(self, text: str, normpunc: bool = False, ligatures: bool = False) -> str: - """Transliterates/transcribes a word into IPA. Passes unmapped - characters through to output unchanged. - - :param text str: word to transcribe - :param normpunct bool: if True, normalize punctuation - :param ligatures bool: if True, use precomposed ligatures instead - of standard IPA - :return: IPA string corresponding to the orthographic string `text`. - All unrecognized characters are included. - :rtype: str + def transliterate(self, text: str, normpunc: bool = False, ligatures: bool = False): + """Transliterate/transcribe a word into IPA. + + Passes unmapped characters through to output unchanged. + + Parameters + ---------- + text : str + Word to transcribe. + normpunc : bool, optional + If True, normalize punctuation. Default is False. + ligatures : bool, optional + If True, use precomposed ligatures instead of standard IPA. + Default is False. + + Returns + ------- + str + IPA string corresponding to the orthographic string `text`. + All unrecognized characters are included. """ try: if self.is_korean(text): @@ -242,13 +286,20 @@ def transliterate(self, text: str, normpunc: bool = False, ligatures: bool = Fal return self.general_trans(text, lambda x: True, normpunc, ligatures) - def general_reverse_trans(self, text: str) -> str: - """Reconstructs word from IPA. Does the reverse of transliterate(). + def general_reverse_trans(self, text: str): + """Reconstruct word from IPA. Does the reverse of transliterate(). + Ignores unmapped characters. - :param text str: Transcription to render in orthography - :return: Orthographic string corresponding to `text` - :rtype: str + Parameters + ---------- + text : str + Transcription to render in orthography. + + Returns + ------- + str + Orthographic string corresponding to `text`. """ if self.rev_preproc: text = self.rev_preprocessor.process(text) @@ -276,11 +327,23 @@ def general_reverse_trans(self, text: str) -> str: return unicodedata.normalize('NFC', text) def reverse_transliterate(self, ipa: str) -> str: - """Reconstructs word from IPA. Does the reverse of transliterate() - - :param ipa str: Word transcription in IPA - :return: Reconstruct word in orthography - :rtype: str + """Reconstruct word from IPA. Does the reverse of transliterate(). + + Parameters + ---------- + ipa : str + Word transcription in IPA. + + Returns + ------- + str + Reconstructed word in orthography. + + Raises + ------ + ValueError + If this Epitran object was initialized with no reverse + transliteration loaded. """ if not self.rev: raise ValueError('This Epitran object was initialized' + @@ -288,27 +351,28 @@ def reverse_transliterate(self, ipa: str) -> str: return self.general_reverse_trans(ipa) def strict_trans(self, text: str, normpunc: bool = False, ligatures: bool = False) -> str: - """Transliterates/transcribes a word into IPA, ignoring - umapped characters. - - :param word str: word to transcribe - :param normpunc bool: normalize punctuation - :param ligatures bool: use precomposed ligatures instead of standard IPA - :return: IPA string corresponding to orthographic `word`, ignoring - out-of-mapping characters - :rtype: str + """Transliterate/transcribe a word into IPA, ignoring unmapped characters. + + Parameters + ---------- + text : str + Word to transcribe. + normpunc : bool, optional + Normalize punctuation. Default is False. + ligatures : bool, optional + Use precomposed ligatures instead of standard IPA. Default is False. + + Returns + ------- + str + IPA string corresponding to orthographic `text`, ignoring + out-of-mapping characters. """ return self.general_trans(text, lambda x: x[1], normpunc, ligatures) def word_to_tuples(self, text: str, normpunc: bool = False) -> "list[tuple[str, int, str, str, list[tuple[str, list[int]]]]]": - """Given a word, returns a list of tuples corresponding to IPA segments. - - :param word str: Word to transcribe - :param normpunc bool: Normalize punctuation - :return: Word represented as (category, lettercase, orthographic_form, - phonetic_form, feature_vectors) tuples - :rtype: list[tuple[str, int, str, str, list[tuple[str, list[int]]]]] + """Convert a word to a list of tuples corresponding to IPA segments. The "feature vectors" form a list consisting of (segment, vector) pairs. For IPA segments, segment is a substring of phonetic_form such @@ -316,10 +380,23 @@ def word_to_tuples(self, text: str, normpunc: bool = False) -> "list[tuple[str, the phonetic_form. The vectors are a sequence of integers drawn from the set {-1, 0, 1} where -1 corresponds to '-', 0 corresponds to '0', and 1 corresponds to '+'. + + Parameters + ---------- + text : str + Word to transcribe. + normpunc : bool, optional + Normalize punctuation. Default is False. + + Returns + ------- + list of tuple + Word represented as (category, lettercase, orthographic_form, + phonetic_form, feature_vectors) tuples. """ def cat_and_cap(category: str) -> "tuple[str, int]": - cat, case = tuple(unicodedata.category(category)) - case = 1 if case == 'u' else 0 + cat, case_char = tuple(unicodedata.category(category)) + case = 1 if case_char == 'u' else 0 return cat, case def recode_ft(feature: str) -> int: @@ -354,23 +431,29 @@ def to_vectors(phon: str) -> "list[tuple[str, list[int]]]": phon: str = self.g2p[span.lower()][0] vecs: "list[tuple[str, list[int]]]" = to_vectors(phon) tuples.append(('L', case, span, phon, vecs)) - word: str = word[len(span):] + word = word[len(span):] else: span = word[0] - span: str = self.puncnorm.norm(span) if normpunc else span + span = self.puncnorm.norm(span) if normpunc else span cat, case = cat_and_cap(span) - cat: str = 'P' if normpunc and cat in self.puncnorm else cat - phon: str = '' - vecs: "list[tuple[str, list[int]]]" = to_vectors(phon) + cat = 'P' if normpunc and cat in self.puncnorm else cat + phon = '' + vecs = to_vectors(phon) tuples.append((cat, case, span, phon, vecs)) word = word[1:] return tuples def ipa_segs(self, ipa: str) -> "list[str]": - """Given an IPA string, decompose it into a list of segments + """Decompose an IPA string into a list of segments. + + Parameters + ---------- + ipa : str + A phonetic representation in IPA. - :param ipa str: A phonetic representation in IPA - :return: A list of words corresponding to the segments in `ipa` - :rtype: list[str] + Returns + ------- + list of str + A list of words corresponding to the segments in `ipa`. """ return self.ft.ipa_segs(ipa) diff --git a/epitran/space.py b/epitran/space.py index 68cc9c0..69736cd 100644 --- a/epitran/space.py +++ b/epitran/space.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +import csv import os from typing import Dict, List, Iterator diff --git a/epitran/stripdiacritics.py b/epitran/stripdiacritics.py index 5a9ddc5..371f88b 100644 --- a/epitran/stripdiacritics.py +++ b/epitran/stripdiacritics.py @@ -1,12 +1,11 @@ # -*- coding: utf-8 -*- +import csv import os.path from typing import List from importlib import resources -import csv - class StripDiacritics(object): def __init__(self, code: str) -> None: diff --git a/epitran/vector.py b/epitran/vector.py index 3fe8953..4e70c1f 100644 --- a/epitran/vector.py +++ b/epitran/vector.py @@ -14,34 +14,41 @@ def __init__(self, code: str, space_names: List[str]) -> None: A VectorWithIPASpace object takes orthographic words, via the word_to_segs method, and returns a list of tuples consisting of category - (letter or punctuation), lettercaase, orthographic form, phonetic form, + (letter or punctuation), lettercase, orthographic form, phonetic form, id within an IPA space, and articulatory feature vector. - Args: - code (str): ISO 639-3 code joined to ISO 15924 code with "-" - space_names (list): list of space names consisting of ISO 639-3 - codes joined to ISO 15924 codes with "-" + Parameters + ---------- + code : str + ISO 639-3 code joined to ISO 15924 code with "-". + space_names : list + List of space names consisting of ISO 639-3 codes joined to + ISO 15924 codes with "-". """ self.epi = Epitran(code) self.space = Space(code, space_names) - def word_to_segs(self, word: str, normpunc: bool = False) -> List[Tuple[str, int, str, str, int, List[Optional[int]]]]: - """Returns feature vectors, etc. for segments and punctuation in a word + def word_to_segs(self, word, normpunc=False): + """Return feature vectors, etc. for segments and punctuation in a word. - Args: - word (str): Unicode string representing a word in the - orthography specified when the class is - instantiated - normpunc (bool): normalize punctuation + Parameters + ---------- + word : str + Unicode string representing a word in the orthography specified + when the class is instantiated. + normpunc : bool, optional + Normalize punctuation. Default is False. - Returns: - list: a list of tuples, each representing an IPA segment or a - punctuation character. Tuples consist of . + Returns + ------- + list + A list of tuples, each representing an IPA segment or a punctuation + character. Tuples consist of . - Category consists of the standard Unicode classes (e.g. 'L' - for letter and 'P' for punctuation). Case is binary: 1 for - uppercase and 0 for lowercase. + Category consists of the standard Unicode classes (e.g. 'L' for + letter and 'P' for punctuation). Case is binary: 1 for uppercase + and 0 for lowercase. """ segs = self.epi.word_to_tuples(word, normpunc) new_segs = [] diff --git a/epitran/xsampa.py b/epitran/xsampa.py index 6e29212..0491ef8 100644 --- a/epitran/xsampa.py +++ b/epitran/xsampa.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +import csv import os.path import unicodedata from typing import List @@ -8,7 +9,6 @@ import marisa_trie import panphon -import csv class XSampa(object): @@ -42,15 +42,18 @@ def longest_prefix(self, s: str) -> str: else: return sorted(prefixes, key=len)[-1] # sort by length and return last - def ipa2xs(self, ipa: str) -> str: - """Convert IPA string (unicode) to X-SAMPA string - - Args: - ipa (unicode): An IPA string as unicode + def ipa2xs(self, ipa): + """Convert IPA string to X-SAMPA string. - Returns: - list: a list of strings corresponding to X-SAMPA segments + Parameters + ---------- + ipa : str + An IPA string. + Returns + ------- + str + A string corresponding to X-SAMPA segments. Non-IPA segments are skipped. """ xsampa = []