Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
245 changes: 168 additions & 77 deletions epitran/_epitran.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,39 +12,63 @@
logger.setLevel(logging.WARNING)

class Epitran(object):
"""Unified interface for IPA transliteration/transcription

:param code str: ISO 639-3 plus "-" plus ISO 15924 code of the language/script pair that should be loaded
:param preproc bool: apply preprocessors
:param postproc bool: apply postprocessors
:param ligatures bool: use precomposed ligatures instead of standard IPA
:param cedict_filename str: path to file containing the CC-CEDict dictionary
:param rev boolean: use reverse transliteration
:param rev_preproc bool: if True, apply preprocessors when reverse transliterating
:param rev_postproc bool: if True, apply postprocessors when reverse transliterating
"""Unified interface for IPA transliteration/transcription.

Parameters
----------
code : str
ISO 639-3 plus "-" plus ISO 15924 code of the language/script pair
that should be loaded.
preproc : bool, optional
Apply preprocessors. Default is True.
postproc : bool, optional
Apply postprocessors. Default is True.
ligatures : bool, optional
Use precomposed ligatures instead of standard IPA. Default is False.
cedict_file : str or None, optional
Path to file containing the CC-CEDict dictionary. Default is None.
rev : bool, optional
Use reverse transliteration. Default is False.
rev_preproc : bool, optional
If True, apply preprocessors when reverse transliterating. Default is True.
rev_postproc : bool, optional
If True, apply postprocessors when reverse transliterating. Default is True.
tones : bool, optional
Handle tone information. Default is False.
"""
@final
special = {'eng-Latn': FliteLexLookup,

Check failure on line 40 in epitran/_epitran.py

View workflow job for this annotation

GitHub Actions / ruff

Ruff (invalid-syntax)

epitran/_epitran.py:40:5: invalid-syntax: Expected class, function definition or async function definition after decorator
'cmn-Hans': Epihan,
'cmn-Hant': EpihanTraditional,
'jpn-Jpan': EpiJpan,
'yue-Hant': EpiCanto,
}

def __init__(self, code: str, **kwargs):
"""Constructor method

Args:
code (str): ISO 639-3 code and ISO 15924 code joined with a hyphen
**kwargs: Additional parameters passed to the appropriate backend:
preproc (bool): if True, apply preprocessor (default: True)
postproc (bool): if True, apply postprocessors (default: True)
ligatures (bool): if True, use phonetic ligatures (default: False)
cedict_file (str): path to dictionary file for Chinese/Japanese (default: None)
rev (bool): if True, load reverse transliteration (default: False)
rev_preproc (bool): if True, apply preprocessor when reverse transliterating (default: True)
rev_postproc (bool): if True, apply postprocessor when reverse transliterating (default: True)
tones (bool): if True, include tone information (default: False)
def __init__(self, code: str, preproc: bool=True, postproc: bool=True, ligatures: bool=False,
cedict_file: Union[bool, None]=None, rev: bool=False,
rev_preproc: bool=True, rev_postproc: bool=True, tones: bool=False):
"""Initialize Epitran transliterator.

Parameters
----------
code : str
ISO 639-3 plus "-" plus ISO 15924 code of the language/script pair.
preproc : bool, optional
Apply preprocessors. Default is True.
postproc : bool, optional
Apply postprocessors. Default is True.
ligatures : bool, optional
Use precomposed ligatures instead of standard IPA. Default is False.
cedict_file : bool or None, optional
Path to CC-CEDict dictionary file. Default is None.
rev : bool, optional
Use reverse transliteration. Default is False.
rev_preproc : bool, optional
Apply preprocessors when reverse transliterating. Default is True.
rev_postproc : bool, optional
Apply postprocessors when reverse transliterating. Default is True.
tones : bool, optional
Handle tone information. Default is False.
"""
if code in self.special:
self.epi = self.special[code](**kwargs)
Expand All @@ -55,87 +79,154 @@
self.puncnorm = PuncNorm()

def transliterate(self, word: str, normpunc: bool=False, ligatures: bool=False) -> str:
"""Transliterates/transcribes a word into IPA

:param word str: word to transcribe
:param normpunc bool: if True, normalize punctuation
:param ligatures bool: if True, use precomposed ligatures instead of standard IPA
:return: An IPA string corresponding to the input orthographic string
:rtype: str
"""Transliterate/transcribe a word into IPA.

Parameters
----------
word : str
Word to transcribe.
normpunc : bool, optional
If True, normalize punctuation. Default is False.
ligatures : bool, optional
If True, use precomposed ligatures instead of standard IPA.
Default is False.

Returns
-------
str
An IPA string corresponding to the input orthographic string.
"""
return self.epi.transliterate(word, normpunc, ligatures)

def reverse_transliterate(self, ipa: str) -> str:
"""Reconstructs word from IPA. Does the reverse of transliterate()
"""Reconstruct word from IPA. Does the reverse of transliterate().

Parameters
----------
ipa : str
An IPA representation of a word.

:param ipa str: An IPA representation of a word
:return: An orthographic representation of the word
:rtype: str
Returns
-------
str
An orthographic representation of the word.
"""
return self.epi.reverse_transliterate(ipa)

def strict_trans(self, word: str, normpunc:bool =False, ligatures: bool=False) -> str:
"""Transliterate a word into IPA, ignoring all characters that cannot be recognized.

:param word str: word to transcribe
:param normpunc bool, optional: if True, normalize punctuation
:param ligatures bool, optional: if True, use precomposed ligatures instead of standard IPA
:return: An IPA string corresponding to the input orthographic string, with all uncoverted characters omitted
:rtype: str
"""Transliterate a word into IPA, ignoring unrecognized characters.

Parameters
----------
word : str
Word to transcribe.
normpunc : bool, optional
If True, normalize punctuation. Default is False.
ligatures : bool, optional
If True, use precomposed ligatures instead of standard IPA.
Default is False.

Returns
-------
str
An IPA string corresponding to the input orthographic string,
with all unconverted characters omitted.
"""
return self.epi.strict_trans(word, normpunc, ligatures)

def trans_list(self, word: str, normpunc: bool=False, ligatures: bool=False) -> "list[str]":
"""Transliterates/transcribes a word into list of IPA phonemes

:param word str: word to transcribe
:param normpunc bool, optional: if True, normalize punctuation
:param ligatures bool, optional: if True, use precomposed ligatures instead of standard IPA
:return: list of IPA strings, each corresponding to a segment
:rtype: list[str]
"""Transliterate/transcribe a word into list of IPA phonemes.

Parameters
----------
word : str
Word to transcribe.
normpunc : bool, optional
If True, normalize punctuation. Default is False.
ligatures : bool, optional
If True, use precomposed ligatures instead of standard IPA.
Default is False.

Returns
-------
list of str
List of IPA strings, each corresponding to a segment.
"""
return self.ft.segs_safe(self.epi.transliterate(word, normpunc, ligatures))

def trans_delimiter(self, text: str, delimiter: str=str(' '), normpunc: bool=False, ligatures: bool=False) -> str:
"""Return IPA transliteration with a delimiter between segments

:param text str: An orthographic text
:param delimiter str, optional: A string to insert between segments
:param normpunc bool, optional: If True, normalize punctuation
:param ligatures bool, optional: If True, use precomposed ligatures instead of standard IPA
:return: String of IPA phonemes separated by `delimiter`
:rtype: str
def trans_delimiter(self, text: str, delimiter: str=str(' '), normpunc: bool=False, ligatures: bool=False):
"""Return IPA transliteration with a delimiter between segments.

Parameters
----------
text : str
An orthographic text.
delimiter : str, optional
A string to insert between segments. Default is ' '.
normpunc : bool, optional
If True, normalize punctuation. Default is False.
ligatures : bool, optional
If True, use precomposed ligatures instead of standard IPA.
Default is False.

Returns
-------
str
String of IPA phonemes separated by `delimiter`.
"""
return delimiter.join(self.trans_list(text, normpunc=normpunc,
ligatures=ligatures))

def xsampa_list(self, word: str, normpunc: bool=False, ligaturize: bool=False) -> "list[str]":
"""Transliterates/transcribes a word as X-SAMPA

:param word str: An orthographic word
:param normpunc bool, optional: If True, normalize punctuation
:param ligatures bool, optional: If True, use precomposed ligatures instead of standard IPA
:return: List of X-SAMPA strings corresponding to `word`
:rtype: list[str]
def xsampa_list(self, word: str, normpunc: bool=False, ligaturize: bool=False):
"""Transliterate/transcribe a word as X-SAMPA.

Parameters
----------
word : str
An orthographic word.
normpunc : bool, optional
If True, normalize punctuation. Default is False.
ligaturize : bool, optional
If True, use precomposed ligatures instead of standard IPA.
Default is False.

Returns
-------
list of str
List of X-SAMPA strings corresponding to `word`.
"""
ipa_segs = self.ft.ipa_segs(self.epi.strict_trans(word, normpunc,
ligaturize))
return list(map(self.xsampa.ipa2xs, ipa_segs))

def word_to_tuples(self, word: str, normpunc: bool=False, _ligaturize: bool=False) -> "list[tuple[str, str, str, str, list[int]]]":
"""Given a word, returns a list of tuples corresponding to IPA segments. The "feature
vectors" form a list consisting of (segment, vector) pairs.
def word_to_tuples(self, word: str, normpunc: bool=False, _ligaturize: bool=False):
"""Convert a word to a list of tuples corresponding to IPA segments.

The "feature vectors" form a list consisting of (segment, vector) pairs.
For IPA segments, segment is a substring of phonetic_form such that the
concatenation of all segments in the list is equal to the phonetic_form.
The vectors are a sequence of integers drawn from the set {-1, 0, 1}
where -1 corresponds to '-', 0 corresponds to '0', and 1 corresponds to
'+'.

:param word str: An orthographic word
:param normpunc bool, optional: If True, normalize punctuation
:param ligatures bool, optional: If True, use precomposed ligatures instead of standard IPA
:return: A list of tuples corresponding to IPA segments
:rtype: list[tuple[str, str, str, str, list[int]]]
where -1 corresponds to '-', 0 corresponds to '0', and 1 corresponds to '+'.

Parameters
----------
word : str
An orthographic word.
normpunc : bool, optional
If True, normalize punctuation. Default is False.
_ligaturize : bool, optional
If True, use precomposed ligatures instead of standard IPA.
Default is False.

Returns
-------
list of tuple
A list of tuples corresponding to IPA segments.

Raises
------
AttributeError
If method is not implemented for this language-script pair.
"""
try:
return self.epi.word_to_tuples(word, normpunc)
Expand Down
51 changes: 34 additions & 17 deletions epitran/backoff.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,14 @@ class Backoff(object):
def __init__(self, lang_script_codes: List[str], cedict_file: Optional[str] = None) -> None:
"""Construct a Backoff object.

Args:
lang_script_codes (list): codes for languages to try, starting
with the highest priority languages
cedict_file (str): path to the CC-CEdict dictionary file
(necessary only when cmn-Hans or cmn-Hant are used)
Parameters
----------
lang_script_codes : list
Codes for languages to try, starting with the highest priority
languages.
cedict_file : str, optional
Path to the CC-CEdict dictionary file (necessary only when
cmn-Hans or cmn-Hant are used). Default is None.
"""
self.langs = [_epitran.Epitran(c, cedict_file=cedict_file)
for c in lang_script_codes]
Expand All @@ -31,10 +34,16 @@ def __init__(self, lang_script_codes: List[str], cedict_file: Optional[str] = No

def transliterate(self, token: str) -> str:
"""Return IPA transliteration given by first acceptable mode.
Args:
token (unicode): orthographic text
Returns:
str: transliteration as Unicode IPA string

Parameters
----------
token : str
Orthographic text.

Returns
-------
str
Transliteration as Unicode IPA string.
"""
tr_list = []
while token:
Expand Down Expand Up @@ -64,22 +73,30 @@ def transliterate(self, token: str) -> str:
def trans_list(self, token: str) -> List[str]:
"""Transliterate/transcribe a word into list of IPA phonemes.

Args:
token (unicode): word to transcribe; unicode string
Parameters
----------
token : str
Word to transcribe.

Returns:
list: list of IPA unicode strings, each corresponding to a segment
Returns
-------
list
List of IPA unicode strings, each corresponding to a segment.
"""
return self.ft.segs_safe(self.transliterate(token))

def xsampa_list(self, token: str) -> Union[str, List[str]]:
"""Transcribe a word into a list of X-SAMPA phonemes.

Args:
token (unicode): word to transcribe; unicode strings
Parameters
----------
token : str
Word to transcribe.

Returns:
list: list of X-SAMPA strings, each corresponding to a segment
Returns
-------
list
List of X-SAMPA strings, each corresponding to a segment.
"""
if re.match(r'^\p{Number}+$', token):
return ''
Expand Down
5 changes: 2 additions & 3 deletions epitran/bin/connl2engipaspace.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,11 @@

import argparse
import codecs
import csv
import logging
from collections import Counter
from typing import List

import csv

import epitran
import epitran.flite
import panphon
Expand Down Expand Up @@ -56,7 +55,7 @@ def add_file(flite: epitran.flite.Flite, ft: panphon.FeatureTable, fn: str) -> C

def print_space(output: str, space: Counter[str]) -> None:
pairs = enumerate(sorted(filter(lambda x: x, space.keys())))
with open(output, 'wb') as f:
with open(output, 'w', encoding='utf-8', newline='') as f:
writer = csv.writer(f)
for i, char in pairs:
writer.writerow((i, char))
Expand Down
Loading
Loading