Modernize Python 2 compatibility code for Python 3.10+

epitran/cedict.py

@@ -43,9 +43,9 @@ def _construct_trie(self, hanzi: Dict[str, Tuple[List[str], List[str]]]) -> Any:
         pairs = []
         for hz, df in self.hanzi.items():
             py, en = df
-            py = str(''.join(filter(lambda x: x in ASCII_CHARS, ' '.join(py))))
+            py = ''.join(filter(lambda x: x in ASCII_CHARS, ' '.join(py)))
             pairs.append((hz, (py.encode('utf-8'),)))
-        trie = marisa_trie.RecordTrie(str('@s'), pairs)
+        trie = marisa_trie.RecordTrie('@s', pairs)
         return trie
 
     def has_key(self, key: str) -> bool:
@@ -131,7 +131,7 @@ def _construct_trie(self, character: Dict[str, str]) -> Any:
         pairs = []
         for ch, pron in self.character.items():
             pairs.append((ch, (pron.encode('utf-8'),)))
-        trie = marisa_trie.RecordTrie(str('@s'), pairs)
+        trie = marisa_trie.RecordTrie('@s', pairs)
         return trie
 
     def has_key(self, key: str) -> bool:

epitran/flite.py

@@ -3,9 +3,9 @@
 import logging
 import os.path
 import string
 import sys
 import unicodedata
 from typing import Dict, List, Tuple, Any, Optional
 
 import regex as re
 
@@ -50,7 +50,6 @@
         return arpa_map
 
     def normalize(self, text: str) -> str:
-        text = str(text)
         text = unicodedata.normalize('NFD', text)
         text = ''.join(filter(lambda x: x in string.printable, text))
         return text
@@ -136,7 +135,6 @@
                 return [to_vector(seg) for seg in self.ft.ipa_segs(phon)]
 
         tuples = []
-        word = str(word)
         # word = self.strip_diacritics.process(word)
         word = unicodedata.normalize('NFKD', word)
         word = unicodedata.normalize('NFC', word)

epitran/ppprocessor.py

@@ -2,9 +2,9 @@
 import logging
 import os.path
 
 from typing import Union
 
-import pkg_resources
+from importlib import resources
 
 from epitran.rules import Rules
 
@@ -32,12 +32,12 @@
         code += '_rev' if rev else ''
         fn = os.path.join('data', fix, code + '.txt')
         try:
-            abs_fn = pkg_resources.resource_filename(__name__, fn)
-        except KeyError:
-            return Rules([])
-        if os.path.isfile(abs_fn):
-            return Rules([abs_fn])
-        else:
+            resource_path = resources.files(__package__).joinpath(fn)
+            if resource_path.is_file():
+                return Rules([resource_path])
+            else:
+                return Rules([])
+        except (KeyError, FileNotFoundError):
             return Rules([])
 
     def process(self, word: str) -> str:

epitran/puncnorm.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
-import pkg_resources
 import csv
+from importlib import resources
 from typing import Dict, Iterator
 
 
@@ -12,8 +12,7 @@ def __init__(self) -> None:
 
     def _load_punc_norm_map(self) -> Dict[str, str]:
         """Load the map table for normalizing 'down' punctuation."""
-        path = pkg_resources.resource_filename(__name__, 'data/puncnorm.csv')
-        with open(path, 'r', encoding='utf-8') as f:
+        with resources.files(__package__).joinpath('data/puncnorm.csv').open('r', encoding='utf-8') as f:
             reader = csv.reader(f, delimiter=',', quotechar='"')
             next(reader)
             return {punc: norm for (punc, norm) in reader}

epitran/reromanize.py

@@ -4,7 +4,7 @@
 from unicodedata import normalize
 from typing import Dict, List, Optional
 
-import pkg_resources
+from importlib import resources
 
 import epitran
 import csv
@@ -29,10 +29,10 @@ def __init__(self, code: str, table: str, decompose: bool = True, cedict_file: O
 
     def _load_reromanizer(self, table: str, decompose: bool) -> Dict[str, str]:
         path = os.path.join('data', 'reromanize', table + '.csv')
-        path = pkg_resources.resource_filename(__name__, path)
-        if os.path.isfile(path):
+        path = resources.files(__package__).joinpath(path)
+        if path.is_file():
             mapping = {}
-            with open(path, 'r', encoding='utf-8') as f:
+            with path.open('r', encoding='utf-8') as f:
                 reader = csv.reader(f)
                 next(reader)
                 for ipa, rom in reader:

epitran/rules.py

@@ -1,11 +1,12 @@
 # -*- coding: utf-8 -*-
 
-import io
 import logging
 import unicodedata
-from typing import List, Dict, Optional, Callable, Any
+from typing import List, Dict, Optional, Callable, Any, Union
+from pathlib import Path
 
 import regex as re
+from importlib import resources
 
 from epitran.exceptions import DatafileError
 
@@ -21,27 +22,39 @@
 
 
 class Rules(object):
-    def __init__(self, rule_files: List[str]) -> None:
+    def __init__(self, rule_files: List[Union[str, Path]]) -> None:
         """Construct an object encoding context-sensitive rules
 
         Args:
-            rule_files (list): list of names of rule files
+            rule_files (list): list of names of rule files or Path objects
         """
         self.rules: List[Callable[[str], str]] = []
         self.symbols: Dict[str, str] = {}
         for rule_file in rule_files:
             rules = self._read_rule_file(rule_file)
             self.rules = self.rules + rules
 
-    def _read_rule_file(self, rule_file: str) -> List[Callable[[str], str]]:
+    def _read_rule_file(self, rule_file: Union[str, Path]) -> List[Callable[[str], str]]:
         rules = []
-        with io.open(rule_file, 'r', encoding='utf-8') as f:
-            for i, line in enumerate(f):
-                # Normalize the line to decomposed form
-                line = line.strip()
-                line = unicodedata.normalize('NFD', line)
-                if not re.match(r'\s*%', line):
-                    rules.append(self._read_rule(i, line))
+        # Handle both string paths and importlib.resources Path objects
+        if hasattr(rule_file, 'open'):
+            # This is an importlib.resources Path object
+            with rule_file.open('r', encoding='utf-8') as f:
+                for i, line in enumerate(f):
+                    # Normalize the line to decomposed form
+                    line = line.strip()
+                    line = unicodedata.normalize('NFD', line)
+                    if not re.match(r'\s*%', line):
+                        rules.append(self._read_rule(i, line))
+        else:
+            # This is a regular string path
+            with open(rule_file, 'r', encoding='utf-8') as f:
+                for i, line in enumerate(f):
+                    # Normalize the line to decomposed form
+                    line = line.strip()
+                    line = unicodedata.normalize('NFD', line)
+                    if not re.match(r'\s*%', line):
+                        rules.append(self._read_rule(i, line))
         return [rule for rule in rules if rule is not None]
 
     def _sub_symbols(self, line: str) -> str:

epitran/simple.py

@@ -7,7 +7,7 @@
 from collections import defaultdict
 from typing import DefaultDict, Callable, Any, Optional  # pylint: disable=unused-import
 
-import pkg_resources
+from importlib import resources
 import regex
 
 import panphon
@@ -98,28 +98,27 @@ def _load_g2p_map(self, code: str, rev: bool) -> "DefaultDict[str, list[str]]":
         code += '_rev' if rev else ''
         try:
             path = os.path.join('data', 'map', code + '.csv')
-            path = pkg_resources.resource_filename(__name__, path)
-        except IndexError as malformed_data_file:
+            with resources.files(__package__).joinpath(path).open(encoding='utf-8') as f:
+                reader = csv.reader(f)
+                orth, phon = next(reader)
+                if orth != 'Orth' or phon != 'Phon':
+                    raise DatafileError(
+                        f'Header is ["{orth}", "{phon}"] instead of ["Orth", "Phon"].')
+                for (i, fields) in enumerate(reader):
+                    try:
+                        graph, phon = fields
+                    except ValueError as malformed_data_file:
+                        raise DatafileError(
+                            f'Map file is not well formed at line {i + 2}.') from malformed_data_file
+                    graph = unicodedata.normalize('NFD', graph)
+                    phon = unicodedata.normalize('NFD', phon)
+                    if not self.tones:
+                        phon = regex.sub('[˩˨˧˦˥]', '', phon)
+                    g2p[graph].append(phon)
+                    gr_by_line[graph].append(i)
+        except (FileNotFoundError, IndexError) as malformed_data_file:
             raise DatafileError(
                 'Add an appropriately-named mapping to the data/maps directory.') from malformed_data_file
-        with open(path, encoding='utf-8') as f:
-            reader = csv.reader(f)
-            orth, phon = next(reader)
-            if orth != 'Orth' or phon != 'Phon':
-                raise DatafileError(
-                    f'Header is ["{orth}", "{phon}"] instead of ["Orth", "Phon"].')
-            for (i, fields) in enumerate(reader):
-                try:
-                    graph, phon = fields
-                except ValueError as malformed_data_file:
-                    raise DatafileError(
-                        f'Map file is not well formed at line {i + 2}.') from malformed_data_file
-                graph = unicodedata.normalize('NFD', graph)
-                phon = unicodedata.normalize('NFD', phon)
-                if not self.tones:
-                    phon = regex.sub('[˩˨˧˦˥]', '', phon)
-                g2p[graph].append(phon)
-                gr_by_line[graph].append(i)
         nondeterminisms = self._non_deterministic_mappings(gr_by_line)
         if nondeterminisms:
             message = ""
@@ -134,9 +133,8 @@ def _load_g2p_map(self, code: str, rev: bool) -> "DefaultDict[str, list[str]]":
     def _load_punc_norm_map(self) -> "dict[str, str]":
         """Load the map table for normalizing 'down' punctuation."""
         path = os.path.join('data', 'puncnorm.csv')
-        path = pkg_resources.resource_filename(__name__, path)
-        with open(path, encoding='utf-8') as f:
-            reader = csv.reader(f, delimiter=str(','), quotechar=str('"'))
+        with resources.files(__package__).joinpath(path).open(encoding='utf-8') as f:
+            reader = csv.reader(f, delimiter=',', quotechar='"')
             next(reader)
             return {punc: norm for (punc, norm) in reader}
 

epitran/space.py

@@ -3,7 +3,7 @@
 import os
 from typing import Dict, List, Iterator
 
-import pkg_resources
+from importlib import resources
 import csv
 from epitran import Epitran
 
@@ -33,15 +33,15 @@ def _load_space(self, space_names: List[str]) -> Dict[str, int]:
         punc_fns = ['punc-{}.csv'.format(sc) for sc in scripts]
         for punc_fn in punc_fns:
             punc_fn = os.path.join('data', 'space', punc_fn)
-            punc_fn = pkg_resources.resource_filename(__name__, punc_fn)
-            with open(punc_fn, 'r', encoding='utf-8') as f:
+            punc_fn = resources.files(__package__).joinpath(punc_fn)
+            with punc_fn.open('r', encoding='utf-8') as f:
                 reader = csv.reader(f)
                 for (mark,) in reader:
                     segs.add(mark)
         for name in space_names:
             fn = os.path.join('data', 'space', name + '.csv')
-            fn = pkg_resources.resource_filename(__name__, fn)
-            with open(fn, 'r', encoding='utf-8') as f:
+            fn = resources.files(__package__).joinpath(fn)
+            with fn.open('r', encoding='utf-8') as f:
                 reader = csv.reader(f)
                 for _, to_ in reader:
                     for seg in self.epi.ft.ipa_segs(to_):

epitran/stripdiacritics.py

@@ -3,7 +3,7 @@
 import os.path
 from typing import List
 
-import pkg_resources
+from importlib import resources
 
 import csv
 
@@ -21,14 +21,14 @@ def _read_diacritics(self, code: str) -> List[str]:
         diacritics = []
         fn = os.path.join('data', 'strip', code + '.csv')
         try:
-            abs_fn = pkg_resources.resource_filename(__name__, fn)
-        except KeyError:
-            return []
-        if os.path.isfile(abs_fn):
-            with open(abs_fn, 'r', encoding='utf-8') as f:
-                reader = csv.reader(f)
-                for [diacritic] in reader:
-                    diacritics.append(diacritic)
+            resource_path = resources.files(__package__).joinpath(fn)
+            if resource_path.is_file():
+                with resource_path.open('r', encoding='utf-8') as f:
+                    reader = csv.reader(f)
+                    for [diacritic] in reader:
+                        diacritics.append(diacritic)
+        except (KeyError, FileNotFoundError):
+            pass
         return diacritics
 
     def process(self, word: str) -> str:

epitran/tir2pp.py

@@ -2,15 +2,15 @@
 
 import os.path
 
-import pkg_resources
+from importlib import resources
 from . import rules
 
 
 class Tir2PP(object):
     def __init__(self) -> None:
         fn = os.path.join('data', 'post', 'tir-Ethi-pp.txt')
-        fn = pkg_resources.resource_filename(__name__, fn)
-        self.rules = rules.Rules([fn])
+        resource_path = resources.files(__package__).joinpath(fn)
+        self.rules = rules.Rules([resource_path])
 
     def apply(self, word: str) -> str:
         word = word.replace('ɨ', '')

epitran/xsampa.py

@@ -4,7 +4,7 @@
 import unicodedata
 from typing import List
 
-import pkg_resources
+from importlib import resources
 
 import marisa_trie
 import panphon
@@ -22,9 +22,9 @@ def __init__(self) -> None:
 
     def _read_ipa2xs(self) -> marisa_trie.BytesTrie:
         path = os.path.join('data', self.ipa2xs_fn)
-        path = pkg_resources.resource_filename(__name__, path)
+        path = resources.files(__package__).joinpath(path)
         pairs = []
-        with open(path, 'r', encoding='utf-8') as f:
+        with path.open('r', encoding='utf-8') as f:
             reader = csv.reader(f)
             next(reader)
             for ipa, xs, _ in reader: