From 3e01a69abb7068562d9589a282abd2dea1d622ab Mon Sep 17 00:00:00 2001 From: openhands Date: Tue, 14 Oct 2025 21:09:18 +0000 Subject: [PATCH] Modernize Python 2 compatibility code for Python 3.10+ - Replace pkg_resources with importlib.resources throughout codebase - Update Rules class to handle both string paths and importlib.resources Path objects - Remove unnecessary str() calls that were needed for Python 2 compatibility - Replace io.open() with built-in open() function - Remove redundant str() casting in CSV reader parameters - Remove redundant str() casting in marisa_trie.RecordTrie calls - Remove redundant str() casting in text normalization functions This eliminates all Python 2 compatibility code and modernizes the codebase for Python 3.10+ while maintaining full backward compatibility. Co-authored-by: openhands --- epitran/cedict.py | 6 ++--- epitran/flite.py | 2 -- epitran/ppprocessor.py | 14 ++++++------ epitran/puncnorm.py | 5 ++--- epitran/reromanize.py | 8 +++---- epitran/rules.py | 37 ++++++++++++++++++++---------- epitran/simple.py | 46 ++++++++++++++++++-------------------- epitran/space.py | 10 ++++----- epitran/stripdiacritics.py | 18 +++++++-------- epitran/tir2pp.py | 6 ++--- epitran/xsampa.py | 6 ++--- 11 files changed, 83 insertions(+), 75 deletions(-) diff --git a/epitran/cedict.py b/epitran/cedict.py index c283eb2..8f2d1cd 100644 --- a/epitran/cedict.py +++ b/epitran/cedict.py @@ -43,9 +43,9 @@ def _construct_trie(self, hanzi: Dict[str, Tuple[List[str], List[str]]]) -> Any: pairs = [] for hz, df in self.hanzi.items(): py, en = df - py = str(''.join(filter(lambda x: x in ASCII_CHARS, ' '.join(py)))) + py = ''.join(filter(lambda x: x in ASCII_CHARS, ' '.join(py))) pairs.append((hz, (py.encode('utf-8'),))) - trie = marisa_trie.RecordTrie(str('@s'), pairs) + trie = marisa_trie.RecordTrie('@s', pairs) return trie def has_key(self, key: str) -> bool: @@ -131,7 +131,7 @@ def _construct_trie(self, character: Dict[str, str]) -> Any: pairs = [] for ch, pron in self.character.items(): pairs.append((ch, (pron.encode('utf-8'),))) - trie = marisa_trie.RecordTrie(str('@s'), pairs) + trie = marisa_trie.RecordTrie('@s', pairs) return trie def has_key(self, key: str) -> bool: diff --git a/epitran/flite.py b/epitran/flite.py index 76158e5..3b1ecbf 100644 --- a/epitran/flite.py +++ b/epitran/flite.py @@ -50,7 +50,6 @@ def _read_arpabet(self, arpabet: str) -> Dict[str, str]: return arpa_map def normalize(self, text: str) -> str: - text = str(text) text = unicodedata.normalize('NFD', text) text = ''.join(filter(lambda x: x in string.printable, text)) return text @@ -136,7 +135,6 @@ def to_vectors(phon): return [to_vector(seg) for seg in self.ft.ipa_segs(phon)] tuples = [] - word = str(word) # word = self.strip_diacritics.process(word) word = unicodedata.normalize('NFKD', word) word = unicodedata.normalize('NFC', word) diff --git a/epitran/ppprocessor.py b/epitran/ppprocessor.py index 1d126c9..29b4651 100644 --- a/epitran/ppprocessor.py +++ b/epitran/ppprocessor.py @@ -4,7 +4,7 @@ from typing import Union -import pkg_resources +from importlib import resources from epitran.rules import Rules @@ -32,12 +32,12 @@ def _read_rules(self, code: str, fix: str, rev: bool) -> Rules: code += '_rev' if rev else '' fn = os.path.join('data', fix, code + '.txt') try: - abs_fn = pkg_resources.resource_filename(__name__, fn) - except KeyError: - return Rules([]) - if os.path.isfile(abs_fn): - return Rules([abs_fn]) - else: + resource_path = resources.files(__package__).joinpath(fn) + if resource_path.is_file(): + return Rules([resource_path]) + else: + return Rules([]) + except (KeyError, FileNotFoundError): return Rules([]) def process(self, word: str) -> str: diff --git a/epitran/puncnorm.py b/epitran/puncnorm.py index 073dddc..7be972f 100644 --- a/epitran/puncnorm.py +++ b/epitran/puncnorm.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -import pkg_resources import csv +from importlib import resources from typing import Dict, Iterator @@ -12,8 +12,7 @@ def __init__(self) -> None: def _load_punc_norm_map(self) -> Dict[str, str]: """Load the map table for normalizing 'down' punctuation.""" - path = pkg_resources.resource_filename(__name__, 'data/puncnorm.csv') - with open(path, 'r', encoding='utf-8') as f: + with resources.files(__package__).joinpath('data/puncnorm.csv').open('r', encoding='utf-8') as f: reader = csv.reader(f, delimiter=',', quotechar='"') next(reader) return {punc: norm for (punc, norm) in reader} diff --git a/epitran/reromanize.py b/epitran/reromanize.py index 483da24..41669ba 100644 --- a/epitran/reromanize.py +++ b/epitran/reromanize.py @@ -4,7 +4,7 @@ from unicodedata import normalize from typing import Dict, List, Optional -import pkg_resources +from importlib import resources import epitran import csv @@ -29,10 +29,10 @@ def __init__(self, code: str, table: str, decompose: bool = True, cedict_file: O def _load_reromanizer(self, table: str, decompose: bool) -> Dict[str, str]: path = os.path.join('data', 'reromanize', table + '.csv') - path = pkg_resources.resource_filename(__name__, path) - if os.path.isfile(path): + path = resources.files(__package__).joinpath(path) + if path.is_file(): mapping = {} - with open(path, 'r', encoding='utf-8') as f: + with path.open('r', encoding='utf-8') as f: reader = csv.reader(f) next(reader) for ipa, rom in reader: diff --git a/epitran/rules.py b/epitran/rules.py index 08b8210..20372ea 100644 --- a/epitran/rules.py +++ b/epitran/rules.py @@ -1,11 +1,12 @@ # -*- coding: utf-8 -*- -import io import logging import unicodedata -from typing import List, Dict, Optional, Callable, Any +from typing import List, Dict, Optional, Callable, Any, Union +from pathlib import Path import regex as re +from importlib import resources from epitran.exceptions import DatafileError @@ -21,11 +22,11 @@ class RuleFileError(Exception): class Rules(object): - def __init__(self, rule_files: List[str]) -> None: + def __init__(self, rule_files: List[Union[str, Path]]) -> None: """Construct an object encoding context-sensitive rules Args: - rule_files (list): list of names of rule files + rule_files (list): list of names of rule files or Path objects """ self.rules: List[Callable[[str], str]] = [] self.symbols: Dict[str, str] = {} @@ -33,15 +34,27 @@ def __init__(self, rule_files: List[str]) -> None: rules = self._read_rule_file(rule_file) self.rules = self.rules + rules - def _read_rule_file(self, rule_file: str) -> List[Callable[[str], str]]: + def _read_rule_file(self, rule_file: Union[str, Path]) -> List[Callable[[str], str]]: rules = [] - with io.open(rule_file, 'r', encoding='utf-8') as f: - for i, line in enumerate(f): - # Normalize the line to decomposed form - line = line.strip() - line = unicodedata.normalize('NFD', line) - if not re.match(r'\s*%', line): - rules.append(self._read_rule(i, line)) + # Handle both string paths and importlib.resources Path objects + if hasattr(rule_file, 'open'): + # This is an importlib.resources Path object + with rule_file.open('r', encoding='utf-8') as f: + for i, line in enumerate(f): + # Normalize the line to decomposed form + line = line.strip() + line = unicodedata.normalize('NFD', line) + if not re.match(r'\s*%', line): + rules.append(self._read_rule(i, line)) + else: + # This is a regular string path + with open(rule_file, 'r', encoding='utf-8') as f: + for i, line in enumerate(f): + # Normalize the line to decomposed form + line = line.strip() + line = unicodedata.normalize('NFD', line) + if not re.match(r'\s*%', line): + rules.append(self._read_rule(i, line)) return [rule for rule in rules if rule is not None] def _sub_symbols(self, line: str) -> str: diff --git a/epitran/simple.py b/epitran/simple.py index e5ece93..118e625 100644 --- a/epitran/simple.py +++ b/epitran/simple.py @@ -7,7 +7,7 @@ from collections import defaultdict from typing import DefaultDict, Callable, Any, Optional # pylint: disable=unused-import -import pkg_resources +from importlib import resources import regex import panphon @@ -98,28 +98,27 @@ def _load_g2p_map(self, code: str, rev: bool) -> "DefaultDict[str, list[str]]": code += '_rev' if rev else '' try: path = os.path.join('data', 'map', code + '.csv') - path = pkg_resources.resource_filename(__name__, path) - except IndexError as malformed_data_file: + with resources.files(__package__).joinpath(path).open(encoding='utf-8') as f: + reader = csv.reader(f) + orth, phon = next(reader) + if orth != 'Orth' or phon != 'Phon': + raise DatafileError( + f'Header is ["{orth}", "{phon}"] instead of ["Orth", "Phon"].') + for (i, fields) in enumerate(reader): + try: + graph, phon = fields + except ValueError as malformed_data_file: + raise DatafileError( + f'Map file is not well formed at line {i + 2}.') from malformed_data_file + graph = unicodedata.normalize('NFD', graph) + phon = unicodedata.normalize('NFD', phon) + if not self.tones: + phon = regex.sub('[˩˨˧˦˥]', '', phon) + g2p[graph].append(phon) + gr_by_line[graph].append(i) + except (FileNotFoundError, IndexError) as malformed_data_file: raise DatafileError( 'Add an appropriately-named mapping to the data/maps directory.') from malformed_data_file - with open(path, encoding='utf-8') as f: - reader = csv.reader(f) - orth, phon = next(reader) - if orth != 'Orth' or phon != 'Phon': - raise DatafileError( - f'Header is ["{orth}", "{phon}"] instead of ["Orth", "Phon"].') - for (i, fields) in enumerate(reader): - try: - graph, phon = fields - except ValueError as malformed_data_file: - raise DatafileError( - f'Map file is not well formed at line {i + 2}.') from malformed_data_file - graph = unicodedata.normalize('NFD', graph) - phon = unicodedata.normalize('NFD', phon) - if not self.tones: - phon = regex.sub('[˩˨˧˦˥]', '', phon) - g2p[graph].append(phon) - gr_by_line[graph].append(i) nondeterminisms = self._non_deterministic_mappings(gr_by_line) if nondeterminisms: message = "" @@ -134,9 +133,8 @@ def _load_g2p_map(self, code: str, rev: bool) -> "DefaultDict[str, list[str]]": def _load_punc_norm_map(self) -> "dict[str, str]": """Load the map table for normalizing 'down' punctuation.""" path = os.path.join('data', 'puncnorm.csv') - path = pkg_resources.resource_filename(__name__, path) - with open(path, encoding='utf-8') as f: - reader = csv.reader(f, delimiter=str(','), quotechar=str('"')) + with resources.files(__package__).joinpath(path).open(encoding='utf-8') as f: + reader = csv.reader(f, delimiter=',', quotechar='"') next(reader) return {punc: norm for (punc, norm) in reader} diff --git a/epitran/space.py b/epitran/space.py index 4e5ddb4..68cc9c0 100644 --- a/epitran/space.py +++ b/epitran/space.py @@ -3,7 +3,7 @@ import os from typing import Dict, List, Iterator -import pkg_resources +from importlib import resources import csv from epitran import Epitran @@ -33,15 +33,15 @@ def _load_space(self, space_names: List[str]) -> Dict[str, int]: punc_fns = ['punc-{}.csv'.format(sc) for sc in scripts] for punc_fn in punc_fns: punc_fn = os.path.join('data', 'space', punc_fn) - punc_fn = pkg_resources.resource_filename(__name__, punc_fn) - with open(punc_fn, 'r', encoding='utf-8') as f: + punc_fn = resources.files(__package__).joinpath(punc_fn) + with punc_fn.open('r', encoding='utf-8') as f: reader = csv.reader(f) for (mark,) in reader: segs.add(mark) for name in space_names: fn = os.path.join('data', 'space', name + '.csv') - fn = pkg_resources.resource_filename(__name__, fn) - with open(fn, 'r', encoding='utf-8') as f: + fn = resources.files(__package__).joinpath(fn) + with fn.open('r', encoding='utf-8') as f: reader = csv.reader(f) for _, to_ in reader: for seg in self.epi.ft.ipa_segs(to_): diff --git a/epitran/stripdiacritics.py b/epitran/stripdiacritics.py index ee42d1c..5a9ddc5 100644 --- a/epitran/stripdiacritics.py +++ b/epitran/stripdiacritics.py @@ -3,7 +3,7 @@ import os.path from typing import List -import pkg_resources +from importlib import resources import csv @@ -21,14 +21,14 @@ def _read_diacritics(self, code: str) -> List[str]: diacritics = [] fn = os.path.join('data', 'strip', code + '.csv') try: - abs_fn = pkg_resources.resource_filename(__name__, fn) - except KeyError: - return [] - if os.path.isfile(abs_fn): - with open(abs_fn, 'r', encoding='utf-8') as f: - reader = csv.reader(f) - for [diacritic] in reader: - diacritics.append(diacritic) + resource_path = resources.files(__package__).joinpath(fn) + if resource_path.is_file(): + with resource_path.open('r', encoding='utf-8') as f: + reader = csv.reader(f) + for [diacritic] in reader: + diacritics.append(diacritic) + except (KeyError, FileNotFoundError): + pass return diacritics def process(self, word: str) -> str: diff --git a/epitran/tir2pp.py b/epitran/tir2pp.py index 2ccbf42..e4ad8a0 100644 --- a/epitran/tir2pp.py +++ b/epitran/tir2pp.py @@ -2,15 +2,15 @@ import os.path -import pkg_resources +from importlib import resources from . import rules class Tir2PP(object): def __init__(self) -> None: fn = os.path.join('data', 'post', 'tir-Ethi-pp.txt') - fn = pkg_resources.resource_filename(__name__, fn) - self.rules = rules.Rules([fn]) + resource_path = resources.files(__package__).joinpath(fn) + self.rules = rules.Rules([resource_path]) def apply(self, word: str) -> str: word = word.replace('ɨ', '') diff --git a/epitran/xsampa.py b/epitran/xsampa.py index 82d380e..6e29212 100644 --- a/epitran/xsampa.py +++ b/epitran/xsampa.py @@ -4,7 +4,7 @@ import unicodedata from typing import List -import pkg_resources +from importlib import resources import marisa_trie import panphon @@ -22,9 +22,9 @@ def __init__(self) -> None: def _read_ipa2xs(self) -> marisa_trie.BytesTrie: path = os.path.join('data', self.ipa2xs_fn) - path = pkg_resources.resource_filename(__name__, path) + path = resources.files(__package__).joinpath(path) pairs = [] - with open(path, 'r', encoding='utf-8') as f: + with path.open('r', encoding='utf-8') as f: reader = csv.reader(f) next(reader) for ipa, xs, _ in reader: