diff --git a/epitran/_epitran.py b/epitran/_epitran.py index 83ab76f7..14a5c8ff 100644 --- a/epitran/_epitran.py +++ b/epitran/_epitran.py @@ -85,7 +85,7 @@ def trans_list(self, word: str, normpunc: bool=False, ligatures: bool=False) -> """ return self.ft.segs_safe(self.epi.transliterate(word, normpunc, ligatures)) - def trans_delimiter(self, text: str, delimiter: str=str(' '), normpunc: bool=False, ligatures: bool=False): + def trans_delimiter(self, text: str, delimiter: str=str(' '), normpunc: bool=False, ligatures: bool=False) -> str: """Return IPA transliteration with a delimiter between segments :param text str: An orthographic text @@ -98,7 +98,7 @@ def trans_delimiter(self, text: str, delimiter: str=str(' '), normpunc: bool=Fal return delimiter.join(self.trans_list(text, normpunc=normpunc, ligatures=ligatures)) - def xsampa_list(self, word: str, normpunc: bool=False, ligaturize: bool=False): + def xsampa_list(self, word: str, normpunc: bool=False, ligaturize: bool=False) -> "list[str]": """Transliterates/transcribes a word as X-SAMPA :param word str: An orthographic word @@ -111,7 +111,7 @@ def xsampa_list(self, word: str, normpunc: bool=False, ligaturize: bool=False): ligaturize)) return list(map(self.xsampa.ipa2xs, ipa_segs)) - def word_to_tuples(self, word: str, normpunc: bool=False, _ligaturize: bool=False): + def word_to_tuples(self, word: str, normpunc: bool=False, _ligaturize: bool=False) -> "list[tuple[str, str, str, str, list[int]]]": """Given a word, returns a list of tuples corresponding to IPA segments. The "feature vectors" form a list consisting of (segment, vector) pairs. For IPA segments, segment is a substring of phonetic_form such that the diff --git a/epitran/backoff.py b/epitran/backoff.py index aba932db..c9ce28f1 100644 --- a/epitran/backoff.py +++ b/epitran/backoff.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +from typing import List, Optional, Union import regex as re from . import _epitran import panphon.featuretable @@ -11,7 +12,7 @@ class Backoff(object): """Implements rudimentary language ID and backoff.""" - def __init__(self, lang_script_codes, cedict_file=None): + def __init__(self, lang_script_codes: List[str], cedict_file: Optional[str] = None) -> None: """Construct a Backoff object. Args: @@ -28,7 +29,7 @@ def __init__(self, lang_script_codes, cedict_file=None): self.puncnorm = PuncNorm() self.dias = [StripDiacritics(c) for c in lang_script_codes] - def transliterate(self, token): + def transliterate(self, token: str) -> str: """Return IPA transliteration given by first acceptable mode. Args: token (unicode): orthographic text @@ -60,7 +61,7 @@ def transliterate(self, token): token = token[1:] return ''.join(tr_list) - def trans_list(self, token): + def trans_list(self, token: str) -> List[str]: """Transliterate/transcribe a word into list of IPA phonemes. Args: @@ -71,7 +72,7 @@ def trans_list(self, token): """ return self.ft.segs_safe(self.transliterate(token)) - def xsampa_list(self, token): + def xsampa_list(self, token: str) -> Union[str, List[str]]: """Transcribe a word into a list of X-SAMPA phonemes. Args: diff --git a/epitran/bin/connl2engipaspace.py b/epitran/bin/connl2engipaspace.py index 63443662..d05c5e48 100644 --- a/epitran/bin/connl2engipaspace.py +++ b/epitran/bin/connl2engipaspace.py @@ -4,6 +4,7 @@ import codecs import logging from collections import Counter +from typing import List import csv @@ -14,8 +15,8 @@ logger = logging.getLogger('epitran') -def normpunc(flite, s): - def norm(c): +def normpunc(flite: epitran.flite.Flite, s: str) -> str: + def norm(c: str) -> str: if c in flite.puncnorm: return flite.puncnorm[c] else: @@ -23,7 +24,7 @@ def norm(c): return ''.join(map(norm, s)) -def add_record(flite, ft, orth): +def add_record(flite: epitran.flite.Flite, ft: panphon.FeatureTable, orth: str) -> Counter[str]: space = Counter() orth = normpunc(flite, orth) trans = flite.transliterate(orth) @@ -41,7 +42,7 @@ def add_record(flite, ft, orth): return space -def add_file(flite, ft, fn): +def add_file(flite: epitran.flite.Flite, ft: panphon.FeatureTable, fn: str) -> Counter[str]: space = Counter() with codecs.open(fn, 'r', 'utf-8') as f: for line in f: @@ -53,7 +54,7 @@ def add_file(flite, ft, fn): return space -def print_space(output, space): +def print_space(output: str, space: Counter[str]) -> None: pairs = enumerate(sorted(filter(lambda x: x, space.keys()))) with open(output, 'wb') as f: writer = csv.writer(f) @@ -61,7 +62,7 @@ def print_space(output, space): writer.writerow((i, char)) -def main(infiles, output): +def main(infiles: List[str], output: str) -> None: flite = epitran.flite.Flite() ft = panphon.FeatureTable() space = Counter() diff --git a/epitran/bin/connl2ipaspace.py b/epitran/bin/connl2ipaspace.py index 98e98ffe..30f66d63 100644 --- a/epitran/bin/connl2ipaspace.py +++ b/epitran/bin/connl2ipaspace.py @@ -4,6 +4,7 @@ import codecs import logging from collections import Counter +from typing import List import epitran import panphon @@ -12,8 +13,8 @@ logger = logging.getLogger('epitran') -def normpunc(epi, s): - def norm(c): +def normpunc(epi: epitran.Epitran, s: str) -> str: + def norm(c: str) -> str: if c in epi.puncnorm: return epi.puncnorm[c] else: @@ -21,7 +22,7 @@ def norm(c): return ''.join(map(norm, s)) -def add_record_gen(epi, ft, orth): +def add_record_gen(epi: epitran.Epitran, ft: panphon.FeatureTable, orth: str) -> Counter[str]: space = Counter() orth = normpunc(epi, orth) trans = epi.transliterate(orth) @@ -36,7 +37,7 @@ def add_record_gen(epi, ft, orth): return space -def add_file_gen(epi, ft, fn): +def add_file_gen(epi: epitran.Epitran, ft: panphon.FeatureTable, fn: str) -> Counter[str]: space = Counter() with codecs.open(fn, 'r', 'utf-8') as f: for line in f: @@ -48,7 +49,7 @@ def add_file_gen(epi, ft, fn): return space -def add_file_op(epi, ft, fn): +def add_file_op(epi: epitran.Epitran, ft: panphon.FeatureTable, fn: str) -> Counter[str]: space = Counter() with codecs.open(fn, 'r', 'utf-8') as f: for line in f: @@ -71,7 +72,7 @@ def add_file_op(epi, ft, fn): return space -def print_space(output, space): +def print_space(output: str, space: Counter[str]) -> None: pairs = enumerate(sorted(filter(lambda x: x, space.keys()))) with open(output, 'wb') as f: writer = csv.writer(f) @@ -79,7 +80,7 @@ def print_space(output, space): writer.writerow((i, char)) -def main(code, op, infiles, output): +def main(code: str, op: bool, infiles: List[str], output: str) -> None: epi = epitran.Epitran(code) ft = panphon.FeatureTable() space = Counter() diff --git a/epitran/bin/decompose.py b/epitran/bin/decompose.py index 70a0ef69..98ba4277 100644 --- a/epitran/bin/decompose.py +++ b/epitran/bin/decompose.py @@ -4,7 +4,7 @@ import sys -def main(fn): +def main(fn: str) -> None: with open(fn, encoding='utf-8') as f: print(unicodedata.normalize('NFD', f.read())) diff --git a/epitran/bin/detectcaps.py b/epitran/bin/detectcaps.py index 6ed5c860..56195e27 100644 --- a/epitran/bin/detectcaps.py +++ b/epitran/bin/detectcaps.py @@ -5,7 +5,7 @@ import fileinput -def main(): +def main() -> None: for line in fileinput.input(): line = line.decode('utf-8') token = line.strip() diff --git a/epitran/bin/epitranscribe.py b/epitran/bin/epitranscribe.py index 87d00283..f6e00db2 100644 --- a/epitran/bin/epitranscribe.py +++ b/epitran/bin/epitranscribe.py @@ -7,7 +7,7 @@ import argparse -def main(code): +def main(code: str) -> None: epi = epitran.Epitran(code) for line in sys.stdin: # pointless line = line.decode('utf-8') diff --git a/epitran/bin/isbijective.py b/epitran/bin/isbijective.py index a238ae83..c507035e 100644 --- a/epitran/bin/isbijective.py +++ b/epitran/bin/isbijective.py @@ -1,24 +1,25 @@ #!/usr/bin/env pythoh import glob +from typing import List, Tuple import csv -def read_map(fn): +def read_map(fn: str) -> List[Tuple[str, str]]: with open(fn, 'r', encoding='utf-8') as f: reader = csv.reader(f) next(reader) return [(a, b) for [a, b] in reader] -def is_bijection(mapping): +def is_bijection(mapping: List[Tuple[str, str]]) -> bool: a, b = zip(*mapping) distinct_a, distinct_b = set(a), set(b) return len(distinct_a) == len(mapping) and len(distinct_b) == len(mapping) -def main(map_fns): +def main(map_fns: List[str]) -> None: for fn in map_fns: mapping = read_map(fn) is_b = is_bijection(mapping) diff --git a/epitran/bin/ltf2ipaspace.py b/epitran/bin/ltf2ipaspace.py index 34f4ccab..f5fa14a5 100644 --- a/epitran/bin/ltf2ipaspace.py +++ b/epitran/bin/ltf2ipaspace.py @@ -4,6 +4,7 @@ import argparse import glob import os.path +from typing import List, Set from lxml import etree import csv @@ -12,13 +13,13 @@ import panphon.featuretable -def read_tokens(fn): +def read_tokens(fn: str) -> List[str]: tree = etree.parse(fn) root = tree.getroot() return [tok.text for tok in root.findall('.//TOKEN')] -def read_input(input_, langscript): +def read_input(input_: List[List[str]], langscript: str) -> Set[str]: space = set() epi = epitran.Epitran(langscript) ft = panphon.featuretable.FeatureTable() @@ -31,14 +32,14 @@ def read_input(input_, langscript): return space -def write_output(output, space): +def write_output(output: str, space: Set[str]) -> None: with open(output, 'wb') as f: writer = csv.writer(f) for n, ch in enumerate(sorted(list(space))): writer.writerow((n, ch)) -def main(langscript, input_, output): +def main(langscript: str, input_: List[List[str]], output: str) -> None: space = read_input(input_, langscript) write_output(output, space) diff --git a/epitran/bin/migraterules.py b/epitran/bin/migraterules.py index b4b50a9d..09d31969 100644 --- a/epitran/bin/migraterules.py +++ b/epitran/bin/migraterules.py @@ -5,11 +5,12 @@ import glob import re import io +from typing import List, Optional import csv -def build_rule(fields): +def build_rule(fields: List[str]) -> Optional[str]: try: a, b, X, Y = fields b = "0" if not b else b @@ -17,9 +18,10 @@ def build_rule(fields): return '{} -> {} / {} _ {}'.format(a, b, X, Y) except ValueError: print('Malformed rule: {}'.format(','.join(fields))) + return None -def main(): +def main() -> None: for csv in glob.glob('*.csv'): txt = re.match('[A-Za-z-]+', csv).group(0) + '.txt' with open(csv, 'r', encoding='utf-8') as f, open(txt, 'w', encoding='utf-8') as g: diff --git a/epitran/bin/space2punc.py b/epitran/bin/space2punc.py index 93beaa90..2713e8e4 100644 --- a/epitran/bin/space2punc.py +++ b/epitran/bin/space2punc.py @@ -2,10 +2,11 @@ import sys import unicodedata +from typing import List import csv -def main(fns, fnn): +def main(fns: List[str], fnn: str) -> None: punc = set() for fn in fns: with open(fn, 'r', encoding='utf-8') as f: diff --git a/epitran/bin/testvectorgen.py b/epitran/bin/testvectorgen.py index f5a50bee..842bb4e1 100644 --- a/epitran/bin/testvectorgen.py +++ b/epitran/bin/testvectorgen.py @@ -3,11 +3,12 @@ import argparse import codecs +from typing import List import epitran.vector -def main(code, space, infile): +def main(code: str, space: List[str], infile: str) -> None: vec = epitran.vector.VectorsWithIPASpace(code, space) with codecs.open(infile, 'r', 'utf-8') as f: for line in f: diff --git a/epitran/bin/vie-tones.py b/epitran/bin/vie-tones.py index 27e55ffa..95d8a829 100755 --- a/epitran/bin/vie-tones.py +++ b/epitran/bin/vie-tones.py @@ -16,7 +16,7 @@ } -def shuffle_tone(orth, phon): +def shuffle_tone(orth: str, phon: str) -> str: orth = unicodedata.normalize('NFD', orth) if re.search('[aeiouơư]', orth): for tone in tones: @@ -27,7 +27,7 @@ def shuffle_tone(orth, phon): return phon -def main(): +def main() -> None: fnin = sys.argv[1] fnout = os.path.basename(fnin) with open(fnin) as fin, open(fnout, 'w') as fout: diff --git a/epitran/cedict.py b/epitran/cedict.py index 4da55049..c283eb2f 100644 --- a/epitran/cedict.py +++ b/epitran/cedict.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import codecs +from typing import Dict, List, Tuple, Any import marisa_trie import regex as re @@ -9,7 +10,7 @@ class CEDictTrie(object): - def __init__(self, cedict_file, traditional=False): + def __init__(self, cedict_file: str, traditional: bool = False) -> None: """Construct a trie over CC-CEDict Args: @@ -19,7 +20,7 @@ def __init__(self, cedict_file, traditional=False): self.hanzi = self._read_cedict(cedict_file, traditional=traditional) self.trie = self._construct_trie(self.hanzi) - def _read_cedict(self, cedict_file, traditional=False): + def _read_cedict(self, cedict_file: str, traditional: bool = False) -> Dict[str, Tuple[List[str], List[str]]]: comment_re = re.compile(r'\s*#') lemma_re = re.compile(r'(?P[^]]+) \[(?P[^]]+)\] /(?P.+)/') cedict = {} @@ -38,7 +39,7 @@ def _read_cedict(self, cedict_file, traditional=False): cedict[hanzi[1]] = (pinyin, english) # simplified characters only. return cedict - def _construct_trie(self, hanzi): + def _construct_trie(self, hanzi: Dict[str, Tuple[List[str], List[str]]]) -> Any: pairs = [] for hz, df in self.hanzi.items(): py, en = df @@ -47,20 +48,20 @@ def _construct_trie(self, hanzi): trie = marisa_trie.RecordTrie(str('@s'), pairs) return trie - def has_key(self, key): + def has_key(self, key: str) -> bool: return key in self.hanzi - def prefixes(self, s): + def prefixes(self, s: str) -> List[str]: return self.trie.prefixes(s) - def longest_prefix(self, s): + def longest_prefix(self, s: str) -> str: prefixes = self.prefixes(s) if not prefixes: return '' else: return sorted(prefixes, key=len)[-1] # Sort by length and return last. - def tokenize(self, s): + def tokenize(self, s: str) -> List[str]: tokens = [] while s: token = self.longest_prefix(s) @@ -74,7 +75,7 @@ def tokenize(self, s): class CEDictTrieForCantonese(CEDictTrie): - def _read_cedict(self, cedict_file, traditional=False): + def _read_cedict(self, cedict_file: str, traditional: bool = False) -> Dict[str, Tuple[List[str], List[str]]]: comment_re = re.compile(r'\s*#') lemma_re = re.compile(r'(?P[^[]+) \[(?P[^]]+)\] \{(?P[^}]+)\} /(?P.+)/') cedict = {} @@ -104,7 +105,7 @@ def _read_cedict(self, cedict_file, traditional=False): class CEDictTrieForJapanese(object): - def __init__(self, cedict_file): + def __init__(self, cedict_file: str) -> None: """Construct a trie over src Args: @@ -113,7 +114,7 @@ def __init__(self, cedict_file): self.character = self._read_cedict(cedict_file) self.trie = self._construct_trie(self.character) - def _read_cedict(self, cedict_file): + def _read_cedict(self, cedict_file: str) -> Dict[str, str]: cedict = {} with codecs.open(cedict_file, 'r', 'utf-8') as f: for line in f: @@ -126,27 +127,27 @@ def _read_cedict(self, cedict_file): cedict[character] = '' return cedict - def _construct_trie(self, character): + def _construct_trie(self, character: Dict[str, str]) -> Any: pairs = [] for ch, pron in self.character.items(): pairs.append((ch, (pron.encode('utf-8'),))) trie = marisa_trie.RecordTrie(str('@s'), pairs) return trie - def has_key(self, key): + def has_key(self, key: str) -> bool: return key in self.character - def prefixes(self, s): + def prefixes(self, s: str) -> List[str]: return self.trie.prefixes(s) - def longest_prefix(self, s): + def longest_prefix(self, s: str) -> str: prefixes = self.prefixes(s) if not prefixes: return '' else: return sorted(prefixes, key=len)[-1] - def tokenize(self, s): + def tokenize(self, s: str) -> List[str]: tokens = [] while s: token = self.longest_prefix(s) diff --git a/epitran/dictfirst.py b/epitran/dictfirst.py index 549f9275..ae00d83c 100644 --- a/epitran/dictfirst.py +++ b/epitran/dictfirst.py @@ -1,3 +1,4 @@ +from typing import Dict import epitran class DictFirst: @@ -8,16 +9,16 @@ class DictFirst: code2 (str): language-script code for fall-back language dict_fn (str): file path to text file containing dictionary, one word per line """ - def __init__(self, code1, code2, dict_fn): + def __init__(self, code1: str, code2: str, dict_fn: str) -> None: self.epi1 = epitran.Epitran(code1) self.epi2 = epitran.Epitran(code2) self.dictionary = self._read_dictionary(dict_fn) - def _read_dictionary(self, dict_fn): + def _read_dictionary(self, dict_fn: str) -> Dict[str, str]: with open(dict_fn, encoding='utf-8') as f: return {x.strip(): self.epi1.transliterate(x.strip()) for x in f} - def transliterate(self, token): + def transliterate(self, token: str) -> str: """Convert token to IPA, falling back on second language Args: diff --git a/epitran/download.py b/epitran/download.py index 9cf87c42..e7b8cb15 100644 --- a/epitran/download.py +++ b/epitran/download.py @@ -12,12 +12,12 @@ OPENDICT_JA_URL = 'https://github.com/open-dict-data/ipa-dict/raw/refs/heads/master/data/ja.txt' -def base_dir(): +def base_dir() -> str: data_dir = os.path.join(os.path.dirname(__file__), 'epitran_data') os.makedirs(data_dir, exist_ok=True) return data_dir -def cedict(): +def cedict() -> str: txtfilename = os.path.join(base_dir(), 'cedict.txt') if not os.path.exists(txtfilename): @@ -32,7 +32,7 @@ def cedict(): return txtfilename -def cc_canto(): +def cc_canto() -> str: cc_canto_dir = os.path.join(base_dir(), 'cc_canto') cc_canto_txt = os.path.join(cc_canto_dir, 'cccanto-webdist.txt') @@ -43,7 +43,7 @@ def cc_canto(): return cc_canto_txt -def opendict_ja(): +def opendict_ja() -> str: txtfilename = os.path.join(base_dir(), 'ja.txt') if not os.path.exists(txtfilename): diff --git a/epitran/epihan.py b/epitran/epihan.py index df6e4b41..3b2eab06 100644 --- a/epitran/epihan.py +++ b/epitran/epihan.py @@ -1,6 +1,7 @@ # -*- utf-8 -*- import os.path +from typing import Optional, List, Tuple import regex as re @@ -28,8 +29,8 @@ class Epihan(object): (u'\u3011', u']'), ] - def __init__(self, ligatures=False, cedict_file=None, - rules_file='pinyin-to-ipa.txt', tones=False): + def __init__(self, ligatures: bool = False, cedict_file: Optional[str] = None, + rules_file: str = 'pinyin-to-ipa.txt', tones: bool = False) -> None: """Construct epitran object for Chinese Args: @@ -51,7 +52,7 @@ def __init__(self, ligatures=False, cedict_file=None, self.rules = rules.Rules([rules_file]) self.regexp = re.compile(r'\p{Han}') - def normalize_punc(self, text): + def normalize_punc(self, text: str) -> str: """Normalize punctutation in a string Args: @@ -65,7 +66,7 @@ def normalize_punc(self, text): text = text.replace(a, b) return text - def transliterate(self, text, normpunc=False, ligatures=False): + def transliterate(self, text: str, normpunc: bool = False, ligatures: bool = False) -> str: """Transliterates/transcribes a word into IPA Args: @@ -92,12 +93,12 @@ def transliterate(self, text, normpunc=False, ligatures=False): if ligatures else ipa_tokens return u''.join(ipa_tokens) - def strict_trans(self, text, normpunc=False, ligatures=False): + def strict_trans(self, text: str, normpunc: bool = False, ligatures: bool = False) -> str: return self.transliterate(text, normpunc, ligatures) class EpihanTraditional(Epihan): - def __init__(self, ligatures=False, cedict_file=None, tones=False, rules_file='pinyin-to-ipa.txt'): + def __init__(self, ligatures: bool = False, cedict_file: Optional[str] = None, tones: bool = False, rules_file: str = 'pinyin-to-ipa.txt') -> None: """Construct epitran object for Traditional Chinese Args: @@ -118,7 +119,7 @@ def __init__(self, ligatures=False, cedict_file=None, tones=False, rules_file='p self.regexp = re.compile(r'\p{Han}') class EpiCanto(Epihan): - def __init__(self, ligatures=False, cedict_file=None, tones=False, rules_file='jyutping-to-ipa.txt'): + def __init__(self, ligatures: bool = False, cedict_file: Optional[str] = None, tones: bool = False, rules_file: str = 'jyutping-to-ipa.txt') -> None: """Construct epitran object for Cantonese Args: @@ -139,7 +140,7 @@ def __init__(self, ligatures=False, cedict_file=None, tones=False, rules_file='j self.regexp = re.compile(r'\p{Han}') class EpiJpan(object): - def __init__(self, ligatures=False, cedict_file=None, tones=False): + def __init__(self, ligatures: bool = False, cedict_file: Optional[str] = None, tones: bool = False) -> None: """Construct epitran object for Japanese Args: @@ -152,7 +153,7 @@ def __init__(self, ligatures=False, cedict_file=None, tones=False): self.regexp = None self.tones = tones - def transliterate(self, text, normpunc=False, ligatures=False): + def transliterate(self, text: str, normpunc: bool = False, ligatures: bool = False) -> str: tokens = self.cedict.tokenize(text) ipa_tokens = [] for token in tokens: diff --git a/epitran/flite.py b/epitran/flite.py index b1f2cc88..76158e59 100644 --- a/epitran/flite.py +++ b/epitran/flite.py @@ -5,6 +5,7 @@ import string import sys import unicodedata +from typing import Dict, List, Tuple, Any, Optional import regex as re @@ -21,7 +22,7 @@ class Flite(object): """English G2P using the Flite speech synthesis system.""" - def __init__(self, arpabet='arpabet', ligatures=False, **kwargs): + def __init__(self, arpabet: str = 'arpabet', ligatures: bool = False, **kwargs) -> None: """Construct a Flite "wrapper" Args: @@ -40,7 +41,7 @@ def __init__(self, arpabet='arpabet', ligatures=False, **kwargs): self.num_panphon_fts = len(self.ft.names) - def _read_arpabet(self, arpabet): + def _read_arpabet(self, arpabet: str) -> Dict[str, str]: arpa_map = {} with open(arpabet, 'r', encoding='utf-8') as f: reader = csv.reader(f) @@ -48,16 +49,16 @@ def _read_arpabet(self, arpabet): arpa_map[arpa] = ipa return arpa_map - def normalize(self, text): + def normalize(self, text: str) -> str: text = str(text) text = unicodedata.normalize('NFD', text) text = ''.join(filter(lambda x: x in string.printable, text)) return text - def arpa_text_to_list(self, arpa_text): + def arpa_text_to_list(self, arpa_text: str) -> List[str]: return arpa_text.split(' ')[1:-1] - def arpa_to_ipa(self, arpa_text, ligatures=False): + def arpa_to_ipa(self, arpa_text: str, ligatures: bool = False) -> str: arpa_text = arpa_text.strip() arpa_list = self.arpa_text_to_list(arpa_text) arpa_list = map(lambda d: re.sub(r'\d', '', d), arpa_list) @@ -65,11 +66,11 @@ def arpa_to_ipa(self, arpa_text, ligatures=False): text = ''.join(ipa_list) return text - def english_g2p(self, english): + def english_g2p(self, english: str) -> str: """Stub for English G2P function to be overwritten by subclasses""" return "" - def transliterate(self, text, normpunc=False, ligatures=False): + def transliterate(self, text: str, normpunc: bool = False, ligatures: bool = False) -> str: """Convert English text to IPA transcription Args: @@ -90,10 +91,10 @@ def transliterate(self, text, normpunc=False, ligatures=False): text = ligaturize(text) if (ligatures or self.ligatures) else text return text - def strict_trans(self, text, normpunc=False, ligatures=False): + def strict_trans(self, text: str, normpunc: bool = False, ligatures: bool = False) -> str: return self.transliterate(text, normpunc, ligatures) - def word_to_tuples(self, word, normpunc=False): + def word_to_tuples(self, word: str, normpunc: bool = False) -> List[Any]: """Given a word, returns a list of tuples corresponding to IPA segments. Args: @@ -167,7 +168,7 @@ def to_vectors(phon): class FliteT2P(Flite): """Flite G2P using t2p.""" - def english_g2p(self, text): + def english_g2p(self, text: str) -> str: text = self.normalize(text) try: arpa_text = subprocess.check_output(['t2p', '"{}"'.format(text)]) @@ -184,10 +185,10 @@ def english_g2p(self, text): class FliteLexLookup(Flite): """Flite G2P using lex_lookup.""" - def arpa_text_to_list(self, arpa_text): + def arpa_text_to_list(self, arpa_text: str) -> List[str]: return arpa_text[1:-1].split(' ') - def english_g2p(self, text): + def english_g2p(self, text: str) -> str: text = self.normalize(text).lower() try: arpa_text = subprocess.check_output(['lex_lookup', text]) diff --git a/epitran/ligaturize.py b/epitran/ligaturize.py index bc5155ec..3249350a 100644 --- a/epitran/ligaturize.py +++ b/epitran/ligaturize.py @@ -2,7 +2,7 @@ -def ligaturize(text): +def ligaturize(text: str) -> str: """Convert text to employ non-standard ligatures Args: diff --git a/epitran/meta.py b/epitran/meta.py index 0f63dbc1..3aade531 100644 --- a/epitran/meta.py +++ b/epitran/meta.py @@ -1,4 +1,6 @@ -modes = { +from typing import Dict, List, Optional + +modes: Dict[str, List[str]] = { 'aar': ['Latn'], 'amh': ['Ethi-pp', 'Ethi-red', 'Ethi'], 'ara': ['Arab'], @@ -64,11 +66,11 @@ } -def supported_lang(iso639): +def supported_lang(iso639: str) -> bool: return iso639 in modes -def get_default_mode(iso639): +def get_default_mode(iso639: str) -> Optional[str]: try: return '-'.join([iso639, modes[iso639][0]]) except KeyError: diff --git a/epitran/ppprocessor.py b/epitran/ppprocessor.py index 71c0c598..1d126c99 100644 --- a/epitran/ppprocessor.py +++ b/epitran/ppprocessor.py @@ -2,6 +2,8 @@ import logging import os.path +from typing import Union + import pkg_resources from epitran.rules import Rules @@ -10,7 +12,7 @@ class PrePostProcessor(object): - def __init__(self, code, fix, rev): + def __init__(self, code: str, fix: str, rev: bool) -> None: """Constructs a pre/post-processor for orthographic/IPA strings This class reads processor files consisting of context-sensitive rules @@ -25,7 +27,7 @@ def __init__(self, code, fix, rev): """ self.rules = self._read_rules(code, fix, rev) - def _read_rules(self, code, fix, rev): + def _read_rules(self, code: str, fix: str, rev: bool) -> Rules: assert fix in ['pre', 'post'] code += '_rev' if rev else '' fn = os.path.join('data', fix, code + '.txt') @@ -38,7 +40,7 @@ def _read_rules(self, code, fix, rev): else: return Rules([]) - def process(self, word): + def process(self, word: str) -> str: """Apply processor to an input string Args: diff --git a/epitran/puncnorm.py b/epitran/puncnorm.py index 97293726..073dddc8 100644 --- a/epitran/puncnorm.py +++ b/epitran/puncnorm.py @@ -2,14 +2,15 @@ import pkg_resources import csv +from typing import Dict, Iterator class PuncNorm(object): - def __init__(self): + def __init__(self) -> None: """Constructs a punctuation normalization object""" self.puncnorm = self._load_punc_norm_map() - def _load_punc_norm_map(self): + def _load_punc_norm_map(self) -> Dict[str, str]: """Load the map table for normalizing 'down' punctuation.""" path = pkg_resources.resource_filename(__name__, 'data/puncnorm.csv') with open(path, 'r', encoding='utf-8') as f: @@ -17,7 +18,7 @@ def _load_punc_norm_map(self): next(reader) return {punc: norm for (punc, norm) in reader} - def norm(self, text): + def norm(self, text: str) -> str: """Apply punctuation normalization to a string of text Args: @@ -34,8 +35,8 @@ def norm(self, text): new_text.append(c) return ''.join(new_text) - def __iter__(self): + def __iter__(self) -> Iterator[str]: return iter(self.puncnorm) - def __getitem__(self, key): + def __getitem__(self, key: str) -> str: return self.puncnorm[key] diff --git a/epitran/reromanize.py b/epitran/reromanize.py index 4d3ce442..483da245 100644 --- a/epitran/reromanize.py +++ b/epitran/reromanize.py @@ -2,6 +2,7 @@ import os.path import sys from unicodedata import normalize +from typing import Dict, List, Optional import pkg_resources @@ -12,7 +13,7 @@ class ReRomanizer(object): """Converts IPA representations to a readable roman form.""" - def __init__(self, code, table, decompose=True, cedict_file=None): + def __init__(self, code: str, table: str, decompose: bool = True, cedict_file: Optional[str] = None) -> None: """Construct object for re-romanizing Epitran output. This class converts orthographic input, via Epitran, to a more @@ -26,7 +27,7 @@ def __init__(self, code, table, decompose=True, cedict_file=None): self.epi = epitran.Epitran(code, cedict_file=cedict_file) self.mapping = self._load_reromanizer(table, decompose) - def _load_reromanizer(self, table, decompose): + def _load_reromanizer(self, table: str, decompose: bool) -> Dict[str, str]: path = os.path.join('data', 'reromanize', table + '.csv') path = pkg_resources.resource_filename(__name__, path) if os.path.isfile(path): @@ -42,7 +43,7 @@ def _load_reromanizer(self, table, decompose): print('File {} does not exist.'.format(path), file=sys.stderr) return {} - def reromanize_ipa(self, tr_list): + def reromanize_ipa(self, tr_list: List[str]) -> List[str]: re_rom_list = [] for seg in tr_list: if seg in self.mapping: @@ -51,7 +52,7 @@ def reromanize_ipa(self, tr_list): re_rom_list.append(seg) return re_rom_list - def reromanize(self, text): + def reromanize(self, text: str) -> str: """Convert orthographic text to romanized text Arg: diff --git a/epitran/rules.py b/epitran/rules.py index e1947eaa..08b8210e 100644 --- a/epitran/rules.py +++ b/epitran/rules.py @@ -3,6 +3,7 @@ import io import logging import unicodedata +from typing import List, Dict, Optional, Callable, Any import regex as re @@ -11,7 +12,7 @@ logger = logging.getLogger('epitran') -def none2str(x): +def none2str(x: Optional[str]) -> str: return x if x else '' @@ -20,19 +21,19 @@ class RuleFileError(Exception): class Rules(object): - def __init__(self, rule_files): + def __init__(self, rule_files: List[str]) -> None: """Construct an object encoding context-sensitive rules Args: rule_files (list): list of names of rule files """ - self.rules = [] - self.symbols = {} + self.rules: List[Callable[[str], str]] = [] + self.symbols: Dict[str, str] = {} for rule_file in rule_files: rules = self._read_rule_file(rule_file) self.rules = self.rules + rules - def _read_rule_file(self, rule_file): + def _read_rule_file(self, rule_file: str) -> List[Callable[[str], str]]: rules = [] with io.open(rule_file, 'r', encoding='utf-8') as f: for i, line in enumerate(f): @@ -43,7 +44,7 @@ def _read_rule_file(self, rule_file): rules.append(self._read_rule(i, line)) return [rule for rule in rules if rule is not None] - def _sub_symbols(self, line): + def _sub_symbols(self, line: str) -> str: while re.search(r'::\w+::', line): s = re.search(r'::\w+::', line).group(0) if s in self.symbols: @@ -52,7 +53,7 @@ def _sub_symbols(self, line): raise RuleFileError('Undefined symbol: {}'.format(s)) return line - def _read_rule(self, i, line): + def _read_rule(self, i: int, line: str) -> Optional[Callable[[str], str]]: line = line.strip() if line: line = unicodedata.normalize('NFD', line) @@ -76,27 +77,27 @@ def _read_rule(self, i, line): except Exception as e: raise DatafileError('Line {}: "{}" cannot be compiled as regex: ̪{}'.format(i + 1, line, e)) - def _fields_to_function_metathesis(self, a, X, Y): + def _fields_to_function_metathesis(self, a: str, X: str, Y: str) -> Callable[[str], str]: left = r'(?P{}){}(?P{})'.format(X, a, Y) regexp = re.compile(left) - def rewrite(m): + def rewrite(m: Any) -> str: d = {k: none2str(v) for k, v in m.groupdict().items()} return '{}{}{}{}'.format(d['X'], d['sw2'], d['sw1'], d['Y']) return lambda w: regexp.sub(rewrite, w, re.U) - def _fields_to_function(self, a, b, X, Y): + def _fields_to_function(self, a: str, b: str, X: str, Y: str) -> Callable[[str], str]: left = r'(?P{})(?P{})(?P{})'.format(X, a, Y) regexp = re.compile(left) - def rewrite(m): + def rewrite(m: Any) -> str: d = {k: none2str(v) for k, v in m.groupdict().items()} return '{}{}{}'.format(d['X'], b, d['Y']) return lambda w: regexp.sub(rewrite, w, re.U) - def apply(self, text): + def apply(self, text: str) -> str: """Apply rules to input text Args: diff --git a/epitran/simple.py b/epitran/simple.py index 3db89326..e5ece934 100644 --- a/epitran/simple.py +++ b/epitran/simple.py @@ -5,7 +5,7 @@ import csv import unicodedata from collections import defaultdict -from typing import DefaultDict, Callable # pylint: disable=unused-import +from typing import DefaultDict, Callable, Any, Optional # pylint: disable=unused-import import pkg_resources import regex @@ -68,10 +68,10 @@ def get_tones(self) -> bool: """ return self.tones - def __enter__(self): + def __enter__(self) -> "SimpleEpitran": return self - def __exit__(self, _type_, _val, _trace_back): + def __exit__(self, _type_: Any, _val: Any, _trace_back: Any) -> None: for nil, count in self.nils.items(): sys.stderr.write( f'Unknown character "{nil}" occured {count} times.\n') @@ -131,7 +131,7 @@ def _load_g2p_map(self, code: str, rev: bool) -> "DefaultDict[str, list[str]]": raise MappingError(f'Invalid mapping for {code}:\n{message}') return g2p - def _load_punc_norm_map(self): + def _load_punc_norm_map(self) -> "dict[str, str]": """Load the map table for normalizing 'down' punctuation.""" path = os.path.join('data', 'puncnorm.csv') path = pkg_resources.resource_filename(__name__, path) @@ -140,7 +140,7 @@ def _load_punc_norm_map(self): next(reader) return {punc: norm for (punc, norm) in reader} - def _construct_regex(self, g2p_keys): + def _construct_regex(self, g2p_keys: Any) -> Any: """Build a regular expression that will greadily match segments from the mapping table. """ @@ -148,7 +148,7 @@ def _construct_regex(self, g2p_keys): return regex.compile(f"({r'|'.join(graphemes)})", regex.I) def general_trans(self, text: str, filter_func: "Callable[[tuple[str, bool]], bool]", - normpunc: bool = False, ligatures: bool = False): + normpunc: bool = False, ligatures: bool = False) -> str: """Transliaterates a word into IPA, filtering with filter_func :param text str: word to transcribe; unicode string @@ -197,14 +197,14 @@ def general_trans(self, text: str, filter_func: "Callable[[tuple[str, bool]], bo return unicodedata.normalize('NFC', text) # Korean exception handling in transliterate - def is_korean(self, text): + def is_korean(self, text: str) -> bool: """Check if the text contains any Korean characters.""" for char in text: if '\uAC00' <= char <= '\uD7A3': # Checking Korean Unicode return True return False - def transliterate(self, text: str, normpunc: bool = False, ligatures: bool = False): + def transliterate(self, text: str, normpunc: bool = False, ligatures: bool = False) -> str: """Transliterates/transcribes a word into IPA. Passes unmapped characters through to output unchanged. @@ -225,7 +225,7 @@ def transliterate(self, text: str, normpunc: bool = False, ligatures: bool = Fal return self.general_trans(text, lambda x: True, normpunc, ligatures) - def general_reverse_trans(self, text: str): + def general_reverse_trans(self, text: str) -> str: """Reconstructs word from IPA. Does the reverse of transliterate(). Ignores unmapped characters. diff --git a/epitran/space.py b/epitran/space.py index 0a22ae13..4e5ddb47 100644 --- a/epitran/space.py +++ b/epitran/space.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import os +from typing import Dict, List, Iterator import pkg_resources import csv @@ -8,7 +9,7 @@ class Space(object): - def __init__(self, code, space_names): + def __init__(self, code: str, space_names: List[str]) -> None: """Construct a Space object Space objects take strings (corresponding to segments) and return @@ -26,7 +27,7 @@ def __init__(self, code, space_names): self.epi = Epitran(code) self.dict = self._load_space(space_names) - def _load_space(self, space_names): + def _load_space(self, space_names: List[str]) -> Dict[str, int]: segs = set() scripts = list(set([nm.split('-')[1] for nm in space_names])) punc_fns = ['punc-{}.csv'.format(sc) for sc in scripts] @@ -48,10 +49,10 @@ def _load_space(self, space_names): enum = enumerate(sorted(list(segs))) return {seg: num for num, seg in enum} - def __iter__(self): + def __iter__(self) -> Iterator[str]: return iter(self.dict) - def __getitem__(self, key): + def __getitem__(self, key: str) -> int: """Given a string as a key, return the corresponding integer Args: diff --git a/epitran/stripdiacritics.py b/epitran/stripdiacritics.py index 0c3ce96a..ee42d1c9 100644 --- a/epitran/stripdiacritics.py +++ b/epitran/stripdiacritics.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import os.path +from typing import List import pkg_resources @@ -8,7 +9,7 @@ class StripDiacritics(object): - def __init__(self, code): + def __init__(self, code: str) -> None: """Constructs object to strip specified diacritics from text Args: @@ -16,7 +17,7 @@ def __init__(self, code): """ self.diacritics = self._read_diacritics(code) - def _read_diacritics(self, code): + def _read_diacritics(self, code: str) -> List[str]: diacritics = [] fn = os.path.join('data', 'strip', code + '.csv') try: @@ -30,7 +31,7 @@ def _read_diacritics(self, code): diacritics.append(diacritic) return diacritics - def process(self, word): + def process(self, word: str) -> str: """Remove diacritics from an input string Args: diff --git a/epitran/tir2pp.py b/epitran/tir2pp.py index 4efbb10b..2ccbf421 100644 --- a/epitran/tir2pp.py +++ b/epitran/tir2pp.py @@ -7,11 +7,11 @@ class Tir2PP(object): - def __init__(self): + def __init__(self) -> None: fn = os.path.join('data', 'post', 'tir-Ethi-pp.txt') fn = pkg_resources.resource_filename(__name__, fn) self.rules = rules.Rules([fn]) - def apply(self, word): + def apply(self, word: str) -> str: word = word.replace('ɨ', '') return self.rules.apply(word) diff --git a/epitran/vector.py b/epitran/vector.py index 2c6b79af..c03a9979 100644 --- a/epitran/vector.py +++ b/epitran/vector.py @@ -1,5 +1,6 @@ import logging +from typing import List, Tuple, Any, Optional from epitran import Epitran from epitran.space import Space @@ -8,7 +9,7 @@ class VectorsWithIPASpace(object): - def __init__(self, code, space_names): + def __init__(self, code: str, space_names: List[str]) -> None: """Constructs VectorWithIPASpace object A VectorWithIPASpace object takes orthographic words, via the @@ -24,7 +25,7 @@ def __init__(self, code, space_names): self.epi = Epitran(code) self.space = Space(code, space_names) - def word_to_segs(self, word, normpunc=False): + def word_to_segs(self, word: str, normpunc: bool = False) -> List[Tuple[str, int, str, str, int, List[Optional[int]]]]: """Returns feature vectors, etc. for segments and punctuation in a word Args: diff --git a/epitran/xsampa.py b/epitran/xsampa.py index 046da4f7..82d380e0 100644 --- a/epitran/xsampa.py +++ b/epitran/xsampa.py @@ -2,6 +2,7 @@ import os.path import unicodedata +from typing import List import pkg_resources @@ -13,13 +14,13 @@ class XSampa(object): ipa2xs_fn = 'ipa-xsampa.csv' - def __init__(self): + def __init__(self) -> None: """Construct an IPA-XSampa conversion object """ self.trie = self._read_ipa2xs() self.ft = panphon.FeatureTable() - def _read_ipa2xs(self): + def _read_ipa2xs(self) -> marisa_trie.BytesTrie: path = os.path.join('data', self.ipa2xs_fn) path = pkg_resources.resource_filename(__name__, path) pairs = [] @@ -31,17 +32,17 @@ def _read_ipa2xs(self): trie = marisa_trie.BytesTrie(pairs) return trie - def prefixes(self, s): + def prefixes(self, s: str) -> List[str]: return self.trie.prefixes(s) - def longest_prefix(self, s): + def longest_prefix(self, s: str) -> str: prefixes = self.prefixes(s) if not prefixes: return '' else: return sorted(prefixes, key=len)[-1] # sort by length and return last - def ipa2xs(self, ipa): + def ipa2xs(self, ipa: str) -> str: """Convert IPA string (unicode) to X-SAMPA string Args: