diff --git a/README.md b/README.md index b300da82..b3c6f0fc 100644 --- a/README.md +++ b/README.md @@ -38,13 +38,13 @@ It is now possible to use the Epitran class for English, Mandarin Chinese (Simpl The most useful public method of the Epitran class is `transliterate`: -Epitran.**transliterate**(text, normpunc=False, ligatures=False). Convert `text` (in Unicode-encoded orthography of the language specified in the constructor) to IPA, which is returned. `normpunc` enables punctuation normalization and `ligatures` enables non-standard IPA ligatures like "ʤ" and "ʨ". Usage is illustrated below (Python 2): +Epitran.**transliterate**(text, normpunc=False, ligatures=False). Convert `text` (in Unicode-encoded orthography of the language specified in the constructor) to IPA, which is returned. `normpunc` enables punctuation normalization and `ligatures` enables non-standard IPA ligatures like "ʤ" and "ʨ". Usage is illustrated below: ```python ->>> epi.transliterate(u'Düğün') -u'dy\u0270yn' ->>> print(epi.transliterate(u'Düğün')) +>>> epi.transliterate('Düğün') +'dy\u0270yn' +>>> print(epi.transliterate('Düğün')) dyɰyn ``` @@ -71,14 +71,14 @@ The codes for `character_category` are from the initial characters of the two ch ) ``` -Here is an example of an interaction with ```word_to_tuples``` (Python 2): +Here is an example of an interaction with ```word_to_tuples```: ```python >>> import epitran >>> epi = epitran.Epitran('tur-Latn') ->>> epi.word_to_tuples(u'Düğün') -[(u'L', 1, u'D', u'd', [(u'd', [-1, -1, 1, -1, -1, -1, -1, -1, 1, -1, -1, 1, 1, -1, -1, -1, -1, -1, -1, 0, -1])]), (u'L', 0, u'u\u0308', u'y', [(u'y', [1, 1, -1, 1, -1, -1, -1, 0, 1, -1, -1, -1, -1, -1, 1, 1, -1, -1, 1, 1, -1])]), (u'L', 0, u'g\u0306', u'\u0270', [(u'\u0270', [-1, 1, -1, 1, 0, -1, -1, 0, 1, -1, -1, 0, -1, 0, -1, 1, -1, 0, -1, 1, -1])]), (u'L', 0, u'u\u0308', u'y', [(u'y', [1, 1, -1, 1, -1, -1, -1, 0, 1, -1, -1, -1, -1, -1, 1, 1, -1, -1, 1, 1, -1])]), (u'L', 0, u'n', u'n', [(u'n', [-1, 1, 1, -1, -1, -1, 1, -1, 1, -1, -1, 1, 1, -1, -1, -1, -1, -1, -1, 0, -1])])] +>>> epi.word_to_tuples('Düğün') +[('L', 1, 'D', 'd', [('d', [-1, -1, 1, -1, -1, -1, -1, -1, 1, -1, -1, 1, 1, -1, -1, -1, -1, -1, -1, 0, -1])]), ('L', 0, 'ü', 'y', [('y', [1, 1, -1, 1, -1, -1, -1, 0, 1, -1, -1, -1, -1, -1, 1, 1, -1, -1, 1, 1, -1])]), ('L', 0, 'ğ', 'ɰ', [('ɰ', [-1, 1, -1, 1, 0, -1, -1, 0, 1, -1, -1, 0, -1, 0, -1, 1, -1, 0, -1, 1, -1])]), ('L', 0, 'ü', 'y', [('y', [1, 1, -1, 1, -1, -1, -1, 0, 1, -1, -1, -1, -1, -1, 1, 1, -1, -1, 1, 1, -1])]), ('L', 0, 'n', 'n', [('n', [-1, 1, 1, -1, -1, -1, 1, -1, 1, -1, -1, 1, 1, -1, -1, -1, -1, -1, -1, 0, -1])])] ``` ### The Backoff class @@ -87,7 +87,7 @@ Sometimes, when parsing text in more than one script, it is useful to employ a g Backoff(lang_script_codes, cedict_file=None) -Note that the Backoff class does not currently support parameterized preprocessor and postprocessor application and does not support non-standard ligatures. It also does not support punctuation normalization. `lang_script_codes` is a list of codes like `eng-Latn` or `hin-Deva`. For example, if one was transcribing a Hindi text with many English loanwords and some stray characters of Simplified Chinese, one might use the following code (Python 3): +Note that the Backoff class does not currently support parameterized preprocessor and postprocessor application and does not support non-standard ligatures. It also does not support punctuation normalization. `lang_script_codes` is a list of codes like `eng-Latn` or `hin-Deva`. For example, if one was transcribing a Hindi text with many English loanwords and some stray characters of Simplified Chinese, one might use the following code: ```python from epitran.backoff import Backoff @@ -109,7 +109,7 @@ The Backoff class has the following public methods: * **xsampa_list**: returns a list of X-SAMPA (ASCII) strings, each of which is phoneme -Consider the following example (Python 3): +Consider the following example: ```python >>> backoff.transliterate('हिन्दी') @@ -157,13 +157,13 @@ Its principle method is ```word_to_segs```: VectorWithIPASpace.**word_to_segs**(word, normpunc=False). `word` is a Unicode string. If the keyword argument *normpunc* is set to True, punctuation discovered in `word` is normalized to ASCII equivalents. -A typical interaction with the ```VectorsWithIPASpace``` object via the ```word_to_segs``` method is illustrated here (Python 2): +A typical interaction with the ```VectorsWithIPASpace``` object via the ```word_to_segs``` method is illustrated here: ```python >>> import epitran.vector >>> vwis = epitran.vector.VectorsWithIPASpace('uzb-Latn', ['uzb-Latn']) ->>> vwis.word_to_segs(u'darë') -[(u'L', 0, u'd', u'd\u032a', u'40', [-1, -1, 1, -1, -1, -1, -1, -1, 1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 0, -1]), (u'L', 0, u'a', u'a', u'37', [1, 1, -1, 1, -1, -1, -1, 0, 1, -1, -1, -1, -1, -1, -1, -1, 1, 1, -1, 1, -1]), (u'L', 0, u'r', u'r', u'54', [-1, 1, 1, 1, 0, -1, -1, -1, 1, -1, -1, 1, 1, -1, -1, 0, 0, 0, -1, 0, -1]), (u'L', 0, u'e\u0308', u'ja', u'46', [-1, 1, -1, 1, -1, -1, -1, 0, 1, -1, -1, -1, -1, 0, -1, 1, -1, -1, -1, 0, -1]), (u'L', 0, u'e\u0308', u'ja', u'37', [1, 1, -1, 1, -1, -1, -1, 0, 1, -1, -1, -1, -1, -1, -1, -1, 1, 1, -1, 1, -1])] +>>> vwis.word_to_segs('darë') +[('L', 0, 'd', 'd̪', '40', [-1, -1, 1, -1, -1, -1, -1, -1, 1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 0, -1]), ('L', 0, 'a', 'a', '37', [1, 1, -1, 1, -1, -1, -1, 0, 1, -1, -1, -1, -1, -1, -1, -1, 1, 1, -1, 1, -1]), ('L', 0, 'r', 'r', '54', [-1, 1, 1, 1, 0, -1, -1, -1, 1, -1, -1, 1, 1, -1, -1, 0, 0, 0, -1, 0, -1]), ('L', 0, 'ë', 'ja', '46', [-1, 1, -1, 1, -1, -1, -1, 0, 1, -1, -1, -1, -1, 0, -1, 1, -1, -1, -1, 0, -1]), ('L', 0, 'ë', 'ja', '37', [1, 1, -1, 1, -1, -1, -1, 0, 1, -1, -1, -1, -1, -1, -1, -1, 1, 1, -1, 1, -1])] ``` @@ -379,7 +379,7 @@ Note that major languages, including **French**, are missing from this table due ## Installation of Flite (for English G2P) -For use with most languages, Epitran requires no special installation steps. It can be installed as an ordinarary python package, either with `pip` or by running `python setup.py install` in the root of the source directory. However, English G2P in Epitran relies on CMU Flite, a speech synthesis package by Alan Black and other speech researchers at Carnegie Mellon University. For the current version of Epitran, you should follow the installation instructions for `lex_lookup`, which is used as the default G2P interface for Epitran. +For use with most languages, Epitran requires no special installation steps. **Epitran requires Python 3.10 or higher.** It can be installed as an ordinary python package, either with `pip` or by running `python setup.py install` in the root of the source directory. However, English G2P in Epitran relies on CMU Flite, a speech synthesis package by Alan Black and other speech researchers at Carnegie Mellon University. For the current version of Epitran, you should follow the installation instructions for `lex_lookup`, which is used as the default G2P interface for Epitran. ### `t2p` @@ -523,7 +523,7 @@ c -> s / _ (i|e) ``` -For a complete guide to `regex` regular expressions, see the documentation for [`re`](https://docs.python.org/2/library/re.html) and for [`regex`](https://pypi.python.org/pypi/regex), specifically. +For a complete guide to `regex` regular expressions, see the documentation for [`re`](https://docs.python.org/3/library/re.html) and for [`regex`](https://pypi.org/project/regex/), specifically. Fragments of regular expressions can be assigned to symbols and reused throughout a file. For example, symbol for the disjunction of vowels in a language can be used in a rule that changes /u/ into /w/ before vowels: diff --git a/epitran/backoff.py b/epitran/backoff.py index 2554b207..aba932db 100644 --- a/epitran/backoff.py +++ b/epitran/backoff.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import (print_function, absolute_import, - unicode_literals) import regex as re from . import _epitran @@ -35,7 +33,7 @@ def transliterate(self, token): Args: token (unicode): orthographic text Returns: - unicode: transliteration as Unicode IPA string + str: transliteration as Unicode IPA string """ tr_list = [] while token: diff --git a/epitran/bin/connl2engipaspace.py b/epitran/bin/connl2engipaspace.py index 0908afc5..63443662 100644 --- a/epitran/bin/connl2engipaspace.py +++ b/epitran/bin/connl2engipaspace.py @@ -5,7 +5,7 @@ import logging from collections import Counter -import unicodecsv as csv +import csv import epitran import epitran.flite @@ -56,7 +56,7 @@ def add_file(flite, ft, fn): def print_space(output, space): pairs = enumerate(sorted(filter(lambda x: x, space.keys()))) with open(output, 'wb') as f: - writer = csv.writer(f, encoding='utf-8') + writer = csv.writer(f) for i, char in pairs: writer.writerow((i, char)) diff --git a/epitran/bin/connl2ipaspace.py b/epitran/bin/connl2ipaspace.py index b050db59..98e98ffe 100644 --- a/epitran/bin/connl2ipaspace.py +++ b/epitran/bin/connl2ipaspace.py @@ -7,7 +7,7 @@ import epitran import panphon -import unicodecsv as csv +import csv logger = logging.getLogger('epitran') @@ -74,7 +74,7 @@ def add_file_op(epi, ft, fn): def print_space(output, space): pairs = enumerate(sorted(filter(lambda x: x, space.keys()))) with open(output, 'wb') as f: - writer = csv.writer(f, encoding='utf-8') + writer = csv.writer(f) for i, char in pairs: writer.writerow((i, char)) diff --git a/epitran/bin/detectcaps.py b/epitran/bin/detectcaps.py index 47453859..6ed5c860 100644 --- a/epitran/bin/detectcaps.py +++ b/epitran/bin/detectcaps.py @@ -1,6 +1,5 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from __future__ import print_function import unicodedata import fileinput diff --git a/epitran/bin/isbijective.py b/epitran/bin/isbijective.py index 4de7ceb1..a238ae83 100644 --- a/epitran/bin/isbijective.py +++ b/epitran/bin/isbijective.py @@ -1,14 +1,13 @@ #!/usr/bin/env pythoh -from __future__ import print_function import glob -import unicodecsv as csv +import csv def read_map(fn): - with open(fn, 'rb') as f: - reader = csv.reader(f, encoding='utf-8') + with open(fn, 'r', encoding='utf-8') as f: + reader = csv.reader(f) next(reader) return [(a, b) for [a, b] in reader] diff --git a/epitran/bin/ltf2ipaspace.py b/epitran/bin/ltf2ipaspace.py index 51b59676..34f4ccab 100644 --- a/epitran/bin/ltf2ipaspace.py +++ b/epitran/bin/ltf2ipaspace.py @@ -1,13 +1,12 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from __future__ import print_function import argparse import glob import os.path from lxml import etree -import unicodecsv as csv +import csv import epitran import panphon.featuretable @@ -34,7 +33,7 @@ def read_input(input_, langscript): def write_output(output, space): with open(output, 'wb') as f: - writer = csv.writer(f, encoding='utf-8') + writer = csv.writer(f) for n, ch in enumerate(sorted(list(space))): writer.writerow((n, ch)) diff --git a/epitran/bin/migraterules.py b/epitran/bin/migraterules.py index 5b3be649..b4b50a9d 100644 --- a/epitran/bin/migraterules.py +++ b/epitran/bin/migraterules.py @@ -1,13 +1,12 @@ #!/usr/bin/env Python # -*- coding: utf-8 -*- -from __future__ import (print_function, unicode_literals, absolute_import) import glob import re import io -import unicodecsv +import csv def build_rule(fields): @@ -23,8 +22,8 @@ def build_rule(fields): def main(): for csv in glob.glob('*.csv'): txt = re.match('[A-Za-z-]+', csv).group(0) + '.txt' - with open(csv, 'rb') as f, io.open(txt, 'w', encoding='utf-8') as g: - reader = unicodecsv.reader(f, encoding='utf-8') + with open(csv, 'r', encoding='utf-8') as f, open(txt, 'w', encoding='utf-8') as g: + reader = csv.reader(f) next(reader) for fields in reader: if re.match('\s*%', fields[0]): diff --git a/epitran/bin/reromanize.py b/epitran/bin/reromanize.py index 8303432f..26237d15 100644 --- a/epitran/bin/reromanize.py +++ b/epitran/bin/reromanize.py @@ -1,6 +1,5 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 -from __future__ import print_function import epitran.reromanize import argparse diff --git a/epitran/bin/space2punc.py b/epitran/bin/space2punc.py index c3b1666c..93beaa90 100644 --- a/epitran/bin/space2punc.py +++ b/epitran/bin/space2punc.py @@ -2,19 +2,19 @@ import sys import unicodedata -import unicodecsv as csv +import csv def main(fns, fnn): punc = set() for fn in fns: - with open(fn, 'rb') as f: - reader = csv.reader(f, encoding='utf-8') + with open(fn, 'r', encoding='utf-8') as f: + reader = csv.reader(f) for _, s in reader: if len(s) == 1 and unicodedata.category(s)[0] == u'P': punc.add(s) with open(fnn, 'wb') as f: - writer = csv.writer(f, encoding='utf-8') + writer = csv.writer(f) for mark in sorted(list(punc)): writer.writerow([mark]) diff --git a/epitran/bin/testvectorgen.py b/epitran/bin/testvectorgen.py index ac85d880..f5a50bee 100644 --- a/epitran/bin/testvectorgen.py +++ b/epitran/bin/testvectorgen.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -from __future__ import print_function import argparse import codecs diff --git a/epitran/bin/uigtransliterate.py b/epitran/bin/uigtransliterate.py index 90d207ac..b28467b4 100644 --- a/epitran/bin/uigtransliterate.py +++ b/epitran/bin/uigtransliterate.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -from __future__ import print_function import fileinput import epitran diff --git a/epitran/cedict.py b/epitran/cedict.py index 1872c504..4da55049 100644 --- a/epitran/cedict.py +++ b/epitran/cedict.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import (absolute_import, division, print_function, - unicode_literals) import codecs diff --git a/epitran/data/map/count_phones.py b/epitran/data/map/count_phones.py index da4b50ba..b8eab456 100644 --- a/epitran/data/map/count_phones.py +++ b/epitran/data/map/count_phones.py @@ -2,14 +2,14 @@ import epitran.xsampa import panphon -import unicodecsv as csv +import csv def main(fn): ft = panphon.FeatureTable() xs = epitran.xsampa.XSampa() - with open(fn, 'rb') as f: - reader = csv.reader(f, encoding='utf-8') + with open(fn, 'r', encoding='utf-8') as f: + reader = csv.reader(f) next(reader) phones = set() for orth, phon in reader: diff --git a/epitran/epihan.py b/epitran/epihan.py index fa18db6a..df6e4b41 100644 --- a/epitran/epihan.py +++ b/epitran/epihan.py @@ -1,5 +1,4 @@ # -*- utf-8 -*- -from __future__ import print_function, unicode_literals, division, absolute_import import os.path @@ -56,10 +55,10 @@ def normalize_punc(self, text): """Normalize punctutation in a string Args: - text (unicode): an orthographic string + text (str): an orthographic string Return: - unicode: an orthographic string with punctation normalized to + str: an orthographic string with punctation normalized to Western equivalents """ for a, b in self.punc: diff --git a/epitran/flite.py b/epitran/flite.py index bd6e1d61..b1f2cc88 100644 --- a/epitran/flite.py +++ b/epitran/flite.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import (absolute_import, division, print_function, - unicode_literals) import logging import os.path @@ -11,24 +9,16 @@ import regex as re import panphon -import unicodecsv as csv +import csv from epitran.ligaturize import ligaturize from epitran.puncnorm import PuncNorm -if os.name == 'posix' and sys.version_info[0] < 3: - import subprocess32 as subprocess -else: - import subprocess +import subprocess logging.basicConfig(level=logging.CRITICAL) logger = logging.getLogger('epitran') -if sys.version_info[0] == 3: - def unicode(x): - return x - - class Flite(object): """English G2P using the Flite speech synthesis system.""" def __init__(self, arpabet='arpabet', ligatures=False, **kwargs): @@ -52,14 +42,14 @@ def __init__(self, arpabet='arpabet', ligatures=False, **kwargs): def _read_arpabet(self, arpabet): arpa_map = {} - with open(arpabet, 'rb') as f: - reader = csv.reader(f, encoding='utf-8') + with open(arpabet, 'r', encoding='utf-8') as f: + reader = csv.reader(f) for arpa, ipa in reader: arpa_map[arpa] = ipa return arpa_map def normalize(self, text): - text = unicode(text) + text = str(text) text = unicodedata.normalize('NFD', text) text = ''.join(filter(lambda x: x in string.printable, text)) return text @@ -83,7 +73,7 @@ def transliterate(self, text, normpunc=False, ligatures=False): """Convert English text to IPA transcription Args: - text (unicode): English text + text (str): English text normpunc (bool): if True, normalize punctuation downward ligatures (bool): if True, use non-standard ligatures instead of standard IPA @@ -107,7 +97,7 @@ def word_to_tuples(self, word, normpunc=False): """Given a word, returns a list of tuples corresponding to IPA segments. Args: - word (unicode): word to transliterate + word (str): word to transliterate normpunc (bool): If True, normalizes punctuation to ASCII inventory Returns: @@ -124,7 +114,7 @@ def word_to_tuples(self, word, normpunc=False): def cat_and_cap(c): cat, case = tuple(unicodedata.category(c)) case = 1 if case == 'u' else 0 - return unicode(cat), case + return cat, case def recode_ft(ft): try: @@ -145,7 +135,7 @@ def to_vectors(phon): return [to_vector(seg) for seg in self.ft.ipa_segs(phon)] tuples = [] - word = unicode(word) + word = str(word) # word = self.strip_diacritics.process(word) word = unicodedata.normalize('NFKD', word) word = unicodedata.normalize('NFC', word) @@ -210,5 +200,6 @@ def english_g2p(self, text): arpa_text = '' # Split on newlines and take the first element (in case lex_lookup # returns multiple lines). - arpa_text = arpa_text.splitlines()[0] + lines = arpa_text.splitlines() + arpa_text = lines[0] if lines else '' return self.arpa_to_ipa(arpa_text) diff --git a/epitran/ligaturize.py b/epitran/ligaturize.py index 0a852a76..bc5155ec 100644 --- a/epitran/ligaturize.py +++ b/epitran/ligaturize.py @@ -1,16 +1,15 @@ # -*- coding: utf-8 -*- -from __future__ import print_function, unicode_literals, division, absolute_import def ligaturize(text): """Convert text to employ non-standard ligatures Args: - text (unicode): IPA text to Convert + text (str): IPA text to Convert Return: - unicode: non-standard IPA text with phonetic ligatures for affricates + str: non-standard IPA text with phonetic ligatures for affricates """ mapping = [(u't͡s', u'ʦ'), (u't͡ʃ', u'ʧ'), diff --git a/epitran/ppprocessor.py b/epitran/ppprocessor.py index 2acb6850..71c0c598 100644 --- a/epitran/ppprocessor.py +++ b/epitran/ppprocessor.py @@ -1,5 +1,3 @@ -from __future__ import (absolute_import, division, print_function, - unicode_literals) import logging import os.path @@ -44,9 +42,9 @@ def process(self, word): """Apply processor to an input string Args: - word (unicode): input string (orthographic or IPA) + word (str): input string (orthographic or IPA) Returns: - unicode: output string with all rules applied in order + str: output string with all rules applied in order """ return self.rules.apply(word) diff --git a/epitran/puncnorm.py b/epitran/puncnorm.py index 5626180e..97293726 100644 --- a/epitran/puncnorm.py +++ b/epitran/puncnorm.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import pkg_resources -import unicodecsv as csv +import csv class PuncNorm(object): @@ -12,8 +12,8 @@ def __init__(self): def _load_punc_norm_map(self): """Load the map table for normalizing 'down' punctuation.""" path = pkg_resources.resource_filename(__name__, 'data/puncnorm.csv') - with open(path, 'rb') as f: - reader = csv.reader(f, encoding='utf-8', delimiter=str(','), quotechar=str('"')) + with open(path, 'r', encoding='utf-8') as f: + reader = csv.reader(f, delimiter=',', quotechar='"') next(reader) return {punc: norm for (punc, norm) in reader} @@ -21,10 +21,10 @@ def norm(self, text): """Apply punctuation normalization to a string of text Args: - text (unicode): text to normalize_punc + text (str): text to normalize_punc Returns: - unicode: text with normalized punctuation + str: text with normalized punctuation """ new_text = [] for c in text: diff --git a/epitran/reromanize.py b/epitran/reromanize.py index 49afe872..4d3ce442 100644 --- a/epitran/reromanize.py +++ b/epitran/reromanize.py @@ -1,4 +1,3 @@ -from __future__ import print_function, unicode_literals, division, absolute_import import os.path import sys @@ -7,7 +6,7 @@ import pkg_resources import epitran -import unicodecsv as csv +import csv class ReRomanizer(object): @@ -32,8 +31,8 @@ def _load_reromanizer(self, table, decompose): path = pkg_resources.resource_filename(__name__, path) if os.path.isfile(path): mapping = {} - with open(path, 'rb') as f: - reader = csv.reader(f, encoding='utf-8') + with open(path, 'r', encoding='utf-8') as f: + reader = csv.reader(f) next(reader) for ipa, rom in reader: rom = normalize('NFD', rom) if decompose else normalize('NFC', rom) @@ -56,10 +55,10 @@ def reromanize(self, text): """Convert orthographic text to romanized text Arg: - text (unicode): orthographic text + text (str): orthographic text Returns: - unicode: romanized text + str: romanized text """ tr_list = self.epi.trans_list(text) return ''.join(self.reromanize_ipa(tr_list)) diff --git a/epitran/rules.py b/epitran/rules.py index ca168112..e1947eaa 100644 --- a/epitran/rules.py +++ b/epitran/rules.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import (absolute_import, division, print_function, - unicode_literals) import io import logging @@ -102,10 +100,10 @@ def apply(self, text): """Apply rules to input text Args: - text (unicode): input text (e.g. Pinyin) + text (str): input text (e.g. Pinyin) Returns: - unicode: output text (e.g. IPA) + str: output text (e.g. IPA) """ for i, rule in enumerate(self.rules): text = rule(text) diff --git a/epitran/space.py b/epitran/space.py index 43483cc0..0a22ae13 100644 --- a/epitran/space.py +++ b/epitran/space.py @@ -1,11 +1,9 @@ # -*- coding: utf-8 -*- -from __future__ import (absolute_import, division, print_function, - unicode_literals) import os import pkg_resources -import unicodecsv as csv +import csv from epitran import Epitran @@ -35,15 +33,15 @@ def _load_space(self, space_names): for punc_fn in punc_fns: punc_fn = os.path.join('data', 'space', punc_fn) punc_fn = pkg_resources.resource_filename(__name__, punc_fn) - with open(punc_fn, 'rb') as f: - reader = csv.reader(f, encoding='utf-8') + with open(punc_fn, 'r', encoding='utf-8') as f: + reader = csv.reader(f) for (mark,) in reader: segs.add(mark) for name in space_names: fn = os.path.join('data', 'space', name + '.csv') fn = pkg_resources.resource_filename(__name__, fn) - with open(fn, 'rb') as f: - reader = csv.reader(f, encoding='utf-8') + with open(fn, 'r', encoding='utf-8') as f: + reader = csv.reader(f) for _, to_ in reader: for seg in self.epi.ft.ipa_segs(to_): segs.add(seg) diff --git a/epitran/stripdiacritics.py b/epitran/stripdiacritics.py index 392819c8..0c3ce96a 100644 --- a/epitran/stripdiacritics.py +++ b/epitran/stripdiacritics.py @@ -1,12 +1,10 @@ # -*- coding: utf-8 -*- -from __future__ import (absolute_import, division, print_function, - unicode_literals) import os.path import pkg_resources -import unicodecsv as csv +import csv class StripDiacritics(object): @@ -26,8 +24,8 @@ def _read_diacritics(self, code): except KeyError: return [] if os.path.isfile(abs_fn): - with open(abs_fn, 'rb') as f: - reader = csv.reader(f, encoding='utf-8') + with open(abs_fn, 'r', encoding='utf-8') as f: + reader = csv.reader(f) for [diacritic] in reader: diacritics.append(diacritic) return diacritics @@ -36,10 +34,10 @@ def process(self, word): """Remove diacritics from an input string Args: - word (unicode): Unicode IPA string + word (str): Unicode IPA string Returns: - unicode: Unicode IPA string with specified diacritics + str: Unicode IPA string with specified diacritics removed """ # word = unicodedata.normalize('NFD', word) diff --git a/epitran/test/special_test_estonian.py b/epitran/test/special_test_estonian.py index 829f107c..338c88ae 100644 --- a/epitran/test/special_test_estonian.py +++ b/epitran/test/special_test_estonian.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals import unittest diff --git a/epitran/test/special_test_flite.py b/epitran/test/special_test_flite.py index 76ea6f78..4509d4ff 100644 --- a/epitran/test/special_test_flite.py +++ b/epitran/test/special_test_flite.py @@ -1,6 +1,5 @@ #!/usr/bin/env Python # -*- coding: utf-8 -*- -from __future__ import unicode_literals, print_function import unittest import epitran diff --git a/epitran/test/special_test_french.py b/epitran/test/special_test_french.py index b20f87c4..db695c9f 100644 --- a/epitran/test/special_test_french.py +++ b/epitran/test/special_test_french.py @@ -1,6 +1,5 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from __future__ import print_function, unicode_literals import logging import unittest diff --git a/epitran/test/special_test_german.py b/epitran/test/special_test_german.py index 707908d8..d59a8d57 100644 --- a/epitran/test/special_test_german.py +++ b/epitran/test/special_test_german.py @@ -1,6 +1,5 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from __future__ import print_function, unicode_literals import logging import unittest diff --git a/epitran/test/special_test_latvian.py b/epitran/test/special_test_latvian.py index 7683d001..2c83ab2a 100644 --- a/epitran/test/special_test_latvian.py +++ b/epitran/test/special_test_latvian.py @@ -2,7 +2,6 @@ # code base taken from Epitran Bengali test at # https://github.com/dmort27/epitran/blob/master/epitran/test/test_bengali.py -from __future__ import unicode_literals import unittest import epitran diff --git a/epitran/test/special_test_lithuanian.py b/epitran/test/special_test_lithuanian.py index e49e4416..a3ff4b0f 100644 --- a/epitran/test/special_test_lithuanian.py +++ b/epitran/test/special_test_lithuanian.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals import unittest diff --git a/epitran/test/test_afrikaans.py b/epitran/test/test_afrikaans.py index d36bd5de..4a53d16b 100644 --- a/epitran/test/test_afrikaans.py +++ b/epitran/test/test_afrikaans.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals import unittest import epitran diff --git a/epitran/test/test_arabic.py b/epitran/test/test_arabic.py index 0fb84d6f..5ac1ccf8 100644 --- a/epitran/test/test_arabic.py +++ b/epitran/test/test_arabic.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals import unittest diff --git a/epitran/test/test_bengali.py b/epitran/test/test_bengali.py index 6f096b59..31a29d76 100644 --- a/epitran/test/test_bengali.py +++ b/epitran/test/test_bengali.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals import unittest import unicodedata diff --git a/epitran/test/test_bhojpuri.py b/epitran/test/test_bhojpuri.py index 20f8dd3f..243ac67d 100644 --- a/epitran/test/test_bhojpuri.py +++ b/epitran/test/test_bhojpuri.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals, print_function import unicodedata import unittest diff --git a/epitran/test/test_burmese.py b/epitran/test/test_burmese.py index 8e5c991b..208c2092 100644 --- a/epitran/test/test_burmese.py +++ b/epitran/test/test_burmese.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals import unittest diff --git a/epitran/test/test_dutch.py b/epitran/test/test_dutch.py index 08d33f85..088df7f8 100644 --- a/epitran/test/test_dutch.py +++ b/epitran/test/test_dutch.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import print_function, unicode_literals import unittest diff --git a/epitran/test/test_esperanto.py b/epitran/test/test_esperanto.py index d59d2fcb..c8d49a29 100644 --- a/epitran/test/test_esperanto.py +++ b/epitran/test/test_esperanto.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals import unittest diff --git a/epitran/test/test_farsi.py b/epitran/test/test_farsi.py index 36851b1a..a23854ca 100644 --- a/epitran/test/test_farsi.py +++ b/epitran/test/test_farsi.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals import unittest diff --git a/epitran/test/test_finnish.py b/epitran/test/test_finnish.py index a2407529..0412015f 100644 --- a/epitran/test/test_finnish.py +++ b/epitran/test/test_finnish.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals import unittest diff --git a/epitran/test/test_galician.py b/epitran/test/test_galician.py index e75660a8..8f198631 100644 --- a/epitran/test/test_galician.py +++ b/epitran/test/test_galician.py @@ -1,5 +1,4 @@ -from __future__ import unicode_literals import unittest diff --git a/epitran/test/test_hindi.py b/epitran/test/test_hindi.py index 9290e1ed..52e91901 100644 --- a/epitran/test/test_hindi.py +++ b/epitran/test/test_hindi.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals, print_function import unicodedata import unittest diff --git a/epitran/test/test_hungarian.py b/epitran/test/test_hungarian.py index 8e3dfd91..9195cb4a 100644 --- a/epitran/test/test_hungarian.py +++ b/epitran/test/test_hungarian.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals import unittest import unicodedata diff --git a/epitran/test/test_korean.py b/epitran/test/test_korean.py index 6d29294e..b8fd1aa9 100644 --- a/epitran/test/test_korean.py +++ b/epitran/test/test_korean.py @@ -2,7 +2,6 @@ # Examples from Korean pronunciation rulebook, released by the National Institute of Korean Language. # https://korean.go.kr/kornorms/regltn/regltnView.do?regltn_code=0002®ltn_no=346#a346 -from __future__ import unicode_literals import unittest import epitran diff --git a/epitran/test/test_lao.py b/epitran/test/test_lao.py index b1682625..d72c8ece 100644 --- a/epitran/test/test_lao.py +++ b/epitran/test/test_lao.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals import unittest diff --git a/epitran/test/test_luganda.py b/epitran/test/test_luganda.py index d3ad2d64..523e905a 100644 --- a/epitran/test/test_luganda.py +++ b/epitran/test/test_luganda.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # Unit test for Luganda G2P implementation in Epitran -from __future__ import unicode_literals import unittest import epitran diff --git a/epitran/test/test_malayalam.py b/epitran/test/test_malayalam.py index 4922a9b6..1455ab50 100644 --- a/epitran/test/test_malayalam.py +++ b/epitran/test/test_malayalam.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals import unittest import unicodedata diff --git a/epitran/test/test_polish.py b/epitran/test/test_polish.py index b2dbbe39..da487640 100644 --- a/epitran/test/test_polish.py +++ b/epitran/test/test_polish.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals import unittest diff --git a/epitran/test/test_portuguese.py b/epitran/test/test_portuguese.py index 8b54371c..81783eb2 100644 --- a/epitran/test/test_portuguese.py +++ b/epitran/test/test_portuguese.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals import unittest import unicodedata diff --git a/epitran/test/test_punjabi.py b/epitran/test/test_punjabi.py index 9786b68f..13f26401 100644 --- a/epitran/test/test_punjabi.py +++ b/epitran/test/test_punjabi.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals import unittest diff --git a/epitran/test/test_quechua.py b/epitran/test/test_quechua.py index 7c578e9f..3d0d3b2e 100644 --- a/epitran/test/test_quechua.py +++ b/epitran/test/test_quechua.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals import unittest diff --git a/epitran/test/test_russian.py b/epitran/test/test_russian.py index a62ecbdd..90a21b7e 100644 --- a/epitran/test/test_russian.py +++ b/epitran/test/test_russian.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals import unittest diff --git a/epitran/test/test_sardinian.py b/epitran/test/test_sardinian.py index df0f6881..758c2bf0 100644 --- a/epitran/test/test_sardinian.py +++ b/epitran/test/test_sardinian.py @@ -1,4 +1,3 @@ -from __future__ import unicode_literals import unittest import epitran diff --git a/epitran/test/test_serbian.py b/epitran/test/test_serbian.py index 59fd9437..3d9f282a 100644 --- a/epitran/test/test_serbian.py +++ b/epitran/test/test_serbian.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals import unittest diff --git a/epitran/test/test_sinhala.py b/epitran/test/test_sinhala.py index c380fa19..39551af5 100644 --- a/epitran/test/test_sinhala.py +++ b/epitran/test/test_sinhala.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals import unittest import unicodedata diff --git a/epitran/test/test_slovene.py b/epitran/test/test_slovene.py index e79d8e63..84ccd9dd 100644 --- a/epitran/test/test_slovene.py +++ b/epitran/test/test_slovene.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # Examples from the Handbook of the International Phonetic Association (Slovene) -from __future__ import unicode_literals import unittest diff --git a/epitran/test/test_sorani.py b/epitran/test/test_sorani.py index 3ff9f62c..a8a9e13b 100644 --- a/epitran/test/test_sorani.py +++ b/epitran/test/test_sorani.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals import unittest diff --git a/epitran/test/test_spanish.py b/epitran/test/test_spanish.py index bbbeb0d9..57f484bb 100644 --- a/epitran/test/test_spanish.py +++ b/epitran/test/test_spanish.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import print_function, unicode_literals import unittest diff --git a/epitran/test/test_swedish.py b/epitran/test/test_swedish.py index be518458..6d0b43e6 100644 --- a/epitran/test/test_swedish.py +++ b/epitran/test/test_swedish.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals import unittest diff --git a/epitran/test/test_tamil.py b/epitran/test/test_tamil.py index e84b1b3a..ac1b2b7c 100644 --- a/epitran/test/test_tamil.py +++ b/epitran/test/test_tamil.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals import unittest import unicodedata diff --git a/epitran/test/test_telugu.py b/epitran/test/test_telugu.py index a99c24a7..a7e4daa2 100755 --- a/epitran/test/test_telugu.py +++ b/epitran/test/test_telugu.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals import unittest import unicodedata diff --git a/epitran/test/test_thai.py b/epitran/test/test_thai.py index 2e840f46..4c6d2669 100644 --- a/epitran/test/test_thai.py +++ b/epitran/test/test_thai.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals import unittest import unicodedata diff --git a/epitran/test/test_tigrinya.py b/epitran/test/test_tigrinya.py index fa5ca908..23862ba8 100644 --- a/epitran/test/test_tigrinya.py +++ b/epitran/test/test_tigrinya.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals import unittest diff --git a/epitran/test/test_tokipona.py b/epitran/test/test_tokipona.py index d1b52d6e..ba87127d 100644 --- a/epitran/test/test_tokipona.py +++ b/epitran/test/test_tokipona.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals import unittest diff --git a/epitran/test/test_turkish.py b/epitran/test/test_turkish.py index 2e02513e..91b47f28 100644 --- a/epitran/test/test_turkish.py +++ b/epitran/test/test_turkish.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import print_function import logging import unittest diff --git a/epitran/test/test_ukranian.py b/epitran/test/test_ukranian.py index 0955f75a..7f81e61e 100644 --- a/epitran/test/test_ukranian.py +++ b/epitran/test/test_ukranian.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals import unittest diff --git a/epitran/test/test_uzbek.py b/epitran/test/test_uzbek.py index fe4791fc..8a55384d 100644 --- a/epitran/test/test_uzbek.py +++ b/epitran/test/test_uzbek.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import print_function import logging import unittest diff --git a/epitran/test/test_yue.py b/epitran/test/test_yue.py index 7c384810..2a87007d 100755 --- a/epitran/test/test_yue.py +++ b/epitran/test/test_yue.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals import unittest diff --git a/epitran/test/test_zhuang.py b/epitran/test/test_zhuang.py index 58b4cd01..90887e98 100644 --- a/epitran/test/test_zhuang.py +++ b/epitran/test/test_zhuang.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals import unittest diff --git a/epitran/tir2pp.py b/epitran/tir2pp.py index 9bf867a5..4efbb10b 100644 --- a/epitran/tir2pp.py +++ b/epitran/tir2pp.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import, unicode_literals import os.path diff --git a/epitran/vector.py b/epitran/vector.py index e49264c5..2c6b79af 100644 --- a/epitran/vector.py +++ b/epitran/vector.py @@ -1,5 +1,3 @@ -from __future__ import (absolute_import, division, print_function, - unicode_literals) import logging @@ -30,7 +28,7 @@ def word_to_segs(self, word, normpunc=False): """Returns feature vectors, etc. for segments and punctuation in a word Args: - word (unicode): Unicode string representing a word in the + word (str): Unicode string representing a word in the orthography specified when the class is instantiated normpunc (bool): normalize punctuation diff --git a/epitran/xsampa.py b/epitran/xsampa.py index 1e8bd215..046da4f7 100644 --- a/epitran/xsampa.py +++ b/epitran/xsampa.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import (absolute_import, division, print_function, - unicode_literals) import os.path import unicodedata @@ -9,7 +7,7 @@ import marisa_trie import panphon -import unicodecsv as csv +import csv class XSampa(object): @@ -25,8 +23,8 @@ def _read_ipa2xs(self): path = os.path.join('data', self.ipa2xs_fn) path = pkg_resources.resource_filename(__name__, path) pairs = [] - with open(path, 'rb') as f: - reader = csv.reader(f, encoding='utf-8') + with open(path, 'r', encoding='utf-8') as f: + reader = csv.reader(f) next(reader) for ipa, xs, _ in reader: pairs.append((ipa, xs.encode('utf-8'),)) diff --git a/setup.py b/setup.py index d40e8aa6..19c6dee4 100644 --- a/setup.py +++ b/setup.py @@ -11,13 +11,13 @@ author='David R. Mortensen', author_email='dmortens@cs.cmu.edu', license='MIT', + python_requires='>=3.10', install_requires=['setuptools', 'regex', 'panphon>=0.20', 'marisa-trie', 'requests', 'jamo'], - extras_require={':python_version<"3.0"': ['subprocess32']}, scripts=['epitran/bin/epitranscribe.py', 'epitran/bin/uigtransliterate.py', 'epitran/bin/detectcaps.py', @@ -38,6 +38,9 @@ zip_safe=True, classifiers=['Operating System :: OS Independent', 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Topic :: Software Development :: Libraries :: Python Modules', 'Topic :: Text Processing :: Linguistic'] )