From 262d2c2c28942383f3ed484a4078262159f7c0b6 Mon Sep 17 00:00:00 2001 From: Sourcery AI <> Date: Sun, 21 Aug 2022 17:03:40 +0000 Subject: [PATCH 1/2] 'Refactored by Sourcery' --- apps/col_data.py | 3 +- apps/col_dictionary.py | 10 ++- ...ol_dictionary_export_from_elasticsearch.py | 21 +++-- .../col_dictionary_import_to_elasticsearch.py | 4 +- apps/col_streamlit.py | 4 +- .../scripts/correct_v1.0.alpha.py | 3 +- datasets/DI_Vietnamese-UVD/scripts/data.py | 10 +-- .../scripts/underthesea_v170_dictionary.py | 10 +-- .../DI_Vietnamese-UVD/scripts/validate.py | 8 +- .../scripts/validate_with_corpus.py | 57 +++++++++++--- .../UD_Vietnamese-COL/scripts/make_corpus.py | 3 +- examples/chatbot/baggage_claim/run_tests.py | 4 +- examples/ner/preprocess_data.py | 2 +- examples/ner/run_ner_pl.py | 16 ++-- examples/ner/tasks.py | 44 +++++------ examples/ner/utils_ner.py | 47 +++++------ examples/sentiment/data.py | 11 +-- examples/sentiment/train_bert.py | 31 ++++---- examples/sentiment/train_gpt2.py | 7 +- examples/text_normalize/compare_tool.py | 4 +- examples/text_normalize/evaluation.py | 3 +- examples/text_normalize/normalize.py | 62 +++++++++++---- examples/text_normalize/tools/nvh.py | 13 +--- examples/text_normalize/tools/vtm.py | 3 +- examples/text_normalize/tools/vtt.py | 3 +- .../underthesea_core/lab_underthesea_core.py | 2 +- tests/featurizers/benchmark_featurizers.py | 2 +- travis_pypi_setup.py | 17 ++-- underthesea/__init__.py | 2 +- underthesea/corpus/data.py | 12 +-- underthesea/corpus/tagged_corpus.py | 3 +- underthesea/corpus/util.py | 22 ++---- underthesea/corpus/validate_corpus.py | 12 +-- underthesea/corpus/word_tokenize_corpus.py | 3 +- underthesea/corpus/ws_corpus.py | 7 +- underthesea/data_fetcher.py | 12 ++- .../datasets/uit_absa_hotel/uit_absa_hotel.py | 15 ++-- .../uit_absa_restaurant.py | 15 ++-- underthesea/datasets/vlsp2013_wtk/revise_1.py | 10 +-- underthesea/datasets/vlsp2013_wtk/revise_2.py | 7 +- .../datasets/vlsp2013_wtk/revise_corpus.py | 2 +- underthesea/dictionary/__init__.py | 4 +- underthesea/feature_engineering/feature.py | 48 ++++-------- underthesea/file_utils.py | 19 ++--- underthesea/model_fetcher.py | 5 +- underthesea/models/crf_sequence_tagger.py | 3 +- underthesea/models/text_classifier.py | 12 +-- underthesea/modules/base.py | 2 +- underthesea/modules/embeddings.py | 8 +- underthesea/pipeline/chunking/__init__.py | 3 +- underthesea/pipeline/chunking/model_crf.py | 4 +- .../pipeline/chunking/tagged_feature.py | 48 ++++-------- .../pipeline/classification/text_features.py | 77 +++++++++++++++---- underthesea/pipeline/ner/__init__.py | 3 +- underthesea/pipeline/ner/model_crf.py | 7 +- underthesea/pipeline/ner/tagged_feature.py | 48 ++++-------- underthesea/pipeline/pos_tag/__init__.py | 3 +- .../pipeline/pos_tag/tagged_feature.py | 48 ++++-------- .../pipeline/sent_tokenize/__init__.py | 3 +- .../pipeline/sentiment/bank/text_features.py | 77 +++++++++++++++---- .../pipeline/sentiment/general/__init__.py | 3 +- .../sentiment/general/text_features.py | 77 +++++++++++++++---- .../pipeline/text_normalize/__init__.py | 8 +- .../text_normalize/token_normalize.py | 4 +- .../pipeline/word_tokenize/__init__.py | 6 +- underthesea/pipeline/word_tokenize/model.py | 3 +- underthesea/pipeline/word_tokenize/nightly.py | 5 +- .../pipeline/word_tokenize/regex_tokenize.py | 13 ++-- underthesea/transformer/number.py | 5 +- underthesea/transformer/tagged.py | 23 +++--- underthesea/transformer/tagged_feature.py | 12 +-- underthesea/transformer/word_vector.py | 3 +- underthesea/transforms/conll.py | 13 ++-- underthesea/util/__init__.py | 14 ++-- underthesea/utils/__init__.py | 24 ++++-- underthesea/utils/col_analyzer.py | 28 +++---- underthesea/utils/col_dictionary.py | 15 ++-- underthesea/utils/col_external_dictionary.py | 20 ++--- underthesea/utils/col_lyrics.py | 13 ++-- underthesea/utils/col_script.py | 19 ++--- underthesea/utils/col_sketchengine.py | 2 +- underthesea/utils/col_stopwords.py | 2 +- underthesea/utils/col_wiki_clean.py | 6 +- underthesea/utils/col_wiki_ud.py | 10 +-- underthesea/utils/sp_config.py | 16 ++-- underthesea/utils/sp_data.py | 4 +- underthesea/utils/sp_embedding.py | 4 +- underthesea/utils/sp_field.py | 5 +- underthesea/utils/sp_vocab.py | 8 +- 89 files changed, 662 insertions(+), 651 deletions(-) diff --git a/apps/col_data.py b/apps/col_data.py index 5565dccf4..35e5da555 100644 --- a/apps/col_data.py +++ b/apps/col_data.py @@ -27,8 +27,7 @@ def get_word(self, word): return {} print(word) body = self.es.get(index=self.es_index, id=word) - data = body["_source"] - return data + return body["_source"] def save(self, word, data): self.es.index(index=self.es_index, id=word, body=data) diff --git a/apps/col_dictionary.py b/apps/col_dictionary.py index e1c2b8683..587ea598c 100644 --- a/apps/col_dictionary.py +++ b/apps/col_dictionary.py @@ -40,8 +40,11 @@ def switch_word(word): """, unsafe_allow_html=True) placeholder = st.sidebar.empty() - search_text_box = placeholder.text_input('Word', value=st.session_state['current_word'], key='sidebar_text_input') - if search_text_box: + if search_text_box := placeholder.text_input( + 'Word', + value=st.session_state['current_word'], + key='sidebar_text_input', + ): switch_word(search_text_box) buttons = {} @@ -55,6 +58,5 @@ def switch_word(word): output_data = json_viewer(json_object=data, label=0) - save_button = st.button('Save') - if save_button: + if save_button := st.button('Save'): dictionary.save(st.session_state['current_word'], output_data) diff --git a/apps/col_dictionary_export_from_elasticsearch.py b/apps/col_dictionary_export_from_elasticsearch.py index b580af885..4140dfbee 100644 --- a/apps/col_dictionary_export_from_elasticsearch.py +++ b/apps/col_dictionary_export_from_elasticsearch.py @@ -37,14 +37,13 @@ def extract_dictionary_from_elasticsearch(data): index_name = "dictionary" with open(DICTIONARY_FILE, "w") as f: f.write("") - f = open(DICTIONARY_FILE, "a") - i = 0 - for res in query_doc(es, index_name): - docs = res["hits"]["hits"] - print(len(docs)) - doc_content = extract_dictionary_from_elasticsearch(docs) - if len(doc_content) > 1: - dict_content = yaml.dump(doc_content, allow_unicode=True, sort_keys=True) - f.write(dict_content) - i += 1 - f.close() + with open(DICTIONARY_FILE, "a") as f: + i = 0 + for res in query_doc(es, index_name): + docs = res["hits"]["hits"] + print(len(docs)) + doc_content = extract_dictionary_from_elasticsearch(docs) + if len(doc_content) > 1: + dict_content = yaml.dump(doc_content, allow_unicode=True, sort_keys=True) + f.write(dict_content) + i += 1 diff --git a/apps/col_dictionary_import_to_elasticsearch.py b/apps/col_dictionary_import_to_elasticsearch.py index 894f9711e..c38d18139 100644 --- a/apps/col_dictionary_import_to_elasticsearch.py +++ b/apps/col_dictionary_import_to_elasticsearch.py @@ -9,9 +9,7 @@ es = Elasticsearch() words = yaml.safe_load(content) - i = 0 - for word_key, value in words.items(): - i += 1 + for i, (word_key, value) in enumerate(words.items(), start=1): data = { "headword": word_key, "senses": value diff --git a/apps/col_streamlit.py b/apps/col_streamlit.py index ef5c1cb77..f578c9950 100644 --- a/apps/col_streamlit.py +++ b/apps/col_streamlit.py @@ -42,9 +42,7 @@ def find_word(ud_dataset, word) -> list: st.write('Loaded corpus: wiki') st.write(f'Loaded dictionary: {dictionary_n_words} words (202108.yaml)') -word = st.text_input('Word') - -if word: +if word := st.text_input('Word'): sentences = find_word(ud_dataset, word) st.write(f"## {word}") st.write("Output:") diff --git a/datasets/DI_Vietnamese-UVD/scripts/correct_v1.0.alpha.py b/datasets/DI_Vietnamese-UVD/scripts/correct_v1.0.alpha.py index 76c21ae7f..a93671022 100644 --- a/datasets/DI_Vietnamese-UVD/scripts/correct_v1.0.alpha.py +++ b/datasets/DI_Vietnamese-UVD/scripts/correct_v1.0.alpha.py @@ -36,8 +36,7 @@ def tag_correct(data): tag = node['tag'] if tag in TAG_MAP: correct_data[word][i]['tag'] = TAG_MAP[tag] - # case for word 'khối' - if word == 'khối' and node['tag'] == 'S': + if word == 'khối' and tag == 'S': correct_data[word][i]['tag'] = 'adjective' return correct_data diff --git a/datasets/DI_Vietnamese-UVD/scripts/data.py b/datasets/DI_Vietnamese-UVD/scripts/data.py index ad7987781..26dc97cc4 100644 --- a/datasets/DI_Vietnamese-UVD/scripts/data.py +++ b/datasets/DI_Vietnamese-UVD/scripts/data.py @@ -24,20 +24,14 @@ def add(self, word): def load(file): with open(file) as f: yaml.safe_load(f) - new_dict = Dictionary() - return new_dict + return Dictionary() def to_data(self): data = {} words = sorted(self.words) for text in words: word = self.words[text] - if word.data is None: - content = '' - elif len(word.data) == 0: - content = '' - else: - content = word.data + content = '' if word.data is None or len(word.data) == 0 else word.data data[text] = content return data diff --git a/datasets/DI_Vietnamese-UVD/scripts/underthesea_v170_dictionary.py b/datasets/DI_Vietnamese-UVD/scripts/underthesea_v170_dictionary.py index 194883cfb..f0d623c50 100644 --- a/datasets/DI_Vietnamese-UVD/scripts/underthesea_v170_dictionary.py +++ b/datasets/DI_Vietnamese-UVD/scripts/underthesea_v170_dictionary.py @@ -24,13 +24,7 @@ 'n': 'NOUN', 'S': 'NOUN' # khối } -TEMP_IGNORE_POS = set([ - 'R', # phụ từ tiếng Việt - 'X', # không phân loại - 'Z', # yếu tố cấu tạo từ - 'D', # không có định nghĩa (ví dụ: chút ít) - 'O', # úi chà -]) +TEMP_IGNORE_POS = {'R', 'X', 'Z', 'D', 'O'} logger.info("Start loading") dict = Dictionary() pos_count = {} @@ -45,7 +39,7 @@ defs = [] pos_tags = {} text = key - for definition in data[key]: + for definition in data[text]: pos_tag = definition['pos'] if pos_tag not in pos_tags: i = len(pos_tags) diff --git a/datasets/DI_Vietnamese-UVD/scripts/validate.py b/datasets/DI_Vietnamese-UVD/scripts/validate.py index 43d2488e0..fc6eb9f67 100644 --- a/datasets/DI_Vietnamese-UVD/scripts/validate.py +++ b/datasets/DI_Vietnamese-UVD/scripts/validate.py @@ -6,10 +6,8 @@ def warn(file, line_number, message, type=None): global total_errors - text = "" - if type: - text = f"[{type}] " - text += basename(file) + ":" + str(line_number) + text = f"[{type}] " if type else "" + text += f"{basename(file)}:{str(line_number)}" if total_errors < MAX_SHOW_ERRORS: print(colored(text, 'red'), colored(message, 'red')) @@ -73,6 +71,6 @@ def stats(): print("\n# DICTIONARY STATISTICS") global NUM_WORDS print("* Number of words:", NUM_WORDS) - print("* Tags:", len(VALID_TAGS), "(" + str(sorted(VALID_TAGS))[1:-1] + ")") + print("* Tags:", len(VALID_TAGS), f"({str(sorted(VALID_TAGS))[1:-1]})") stats() diff --git a/datasets/DI_Vietnamese-UVD/scripts/validate_with_corpus.py b/datasets/DI_Vietnamese-UVD/scripts/validate_with_corpus.py index c025c9a97..c7430999c 100644 --- a/datasets/DI_Vietnamese-UVD/scripts/validate_with_corpus.py +++ b/datasets/DI_Vietnamese-UVD/scripts/validate_with_corpus.py @@ -13,16 +13,33 @@ MAX_SHOW_ERRORS = 100 total_errors = 0 -punct = {"!", "/", ",", ".", "...", "?", "-", "\"", "-", ":", "(", ")", "–", "&", ";", "‘", "’", "+"} +punct = { + "!", + "/", + ",", + ".", + "...", + "?", + "\"", + "-", + ":", + "(", + ")", + "–", + "&", + ";", + "‘", + "’", + "+", +} + specials = {"rbkt", "lbkt"} def warn(file, line_number, message, type=None): global total_errors - text = "" - if type: - text = f"[{type}] " - text += basename(file) + ":" + str(line_number) + text = f"[{type}] " if type else "" + text += f"{basename(file)}:{str(line_number)}" if total_errors < MAX_SHOW_ERRORS: print(colored(text, 'red'), colored(message, 'red')) @@ -30,8 +47,7 @@ def warn(file, line_number, message, type=None): def load_dictionary(dict_file): - dict = joblib.load(dict_file) - return dict + return joblib.load(dict_file) def load_corpus(file): @@ -60,12 +76,29 @@ def load_corpus(file): continue tags = list(words[word]) ignores_tags = { - "ADJ", "ADV", "INTJ", "NOUN", "PROPN", "PRON", "SYM", "X", "N:G", "VERB:G", "NY", - "N", "NB", "NNPy", - "NNP", "NNPy", - "V", "VERB", - "Num", "NUMx", "NUM", "NUMX" + "ADJ", + "ADV", + "INTJ", + "NOUN", + "PROPN", + "PRON", + "SYM", + "X", + "N:G", + "VERB:G", + "NY", + "N", + "NB", + "NNP", + "NNPy", + "V", + "VERB", + "Num", + "NUMx", + "NUM", + "NUMX", } + if tags[0] in ignores_tags: continue if word not in dict: diff --git a/datasets/UD_Vietnamese-COL/scripts/make_corpus.py b/datasets/UD_Vietnamese-COL/scripts/make_corpus.py index c6666a7fc..c2c2a6938 100644 --- a/datasets/UD_Vietnamese-COL/scripts/make_corpus.py +++ b/datasets/UD_Vietnamese-COL/scripts/make_corpus.py @@ -17,8 +17,7 @@ def add_lemma_column(sentence): tokens.insert(1, lemma) new_row = "\t".join(tokens) result.append(new_row) - new_sentence = "\n".join(result) - return new_sentence + return "\n".join(result) filepath = join(dirname(dirname(__file__)), "corpus", "ud", "202108.txt") diff --git a/examples/chatbot/baggage_claim/run_tests.py b/examples/chatbot/baggage_claim/run_tests.py index 02427e42c..31cd3e17c 100644 --- a/examples/chatbot/baggage_claim/run_tests.py +++ b/examples/chatbot/baggage_claim/run_tests.py @@ -42,9 +42,7 @@ def send(self, message): payload = {'sender': self.id, 'message': 'hey there'} headers = {'content-type': 'application/json'} r = requests.post('http://localhost:5005/webhooks/rest/webhook', json=payload, headers=headers) - if r is None: - return None - return r.json() + return None if r is None else r.json() if __name__ == '__main__': diff --git a/examples/ner/preprocess_data.py b/examples/ner/preprocess_data.py index e257b0b28..2f855c01d 100644 --- a/examples/ner/preprocess_data.py +++ b/examples/ner/preprocess_data.py @@ -26,9 +26,9 @@ def transform_sentence(sentence): f.write(content) +max_len = 0 for file in ["train.txt", "test.txt", "dev.txt"]: print(file) - max_len = 0 src_folder = join("bert-ner", "data", "vlsp2016") dest_folder = join("bert-ner", "data") transform(src_file=join(src_folder, file), dest_file=join(dest_folder, file)) diff --git a/examples/ner/run_ner_pl.py b/examples/ner/run_ner_pl.py index d8d051006..fad179d55 100644 --- a/examples/ner/run_ner_pl.py +++ b/examples/ner/run_ner_pl.py @@ -25,14 +25,13 @@ def encode(self, labels): if type(labels) == list: return [self.encode(label) for label in labels] label = labels # label is a string - if label not in self.label2index: - index = self.vocab_size - self.label2index[label] = index - self.index2label[index] = label - self.vocab_size += 1 - return index - else: + if label in self.label2index: return self.label2index[label] + index = self.vocab_size + self.label2index[label] = index + self.index2label[index] = label + self.vocab_size += 1 + return index class TokenClassificationCorpus: @@ -187,8 +186,7 @@ def test_step(self, batch, batch_idx): self.log('test_loss', loss) def configure_optimizers(self): - optimizer = AdamW(self.parameters(), lr=2e-5) - return optimizer + return AdamW(self.parameters(), lr=2e-5) def main(): diff --git a/examples/ner/tasks.py b/examples/ner/tasks.py index 409be0715..0aa893ab7 100644 --- a/examples/ner/tasks.py +++ b/examples/ner/tasks.py @@ -51,20 +51,19 @@ def write_predictions_to_file(self, writer: TextIO, test_input_reader: TextIO, p if not preds_list[example_id]: example_id += 1 elif preds_list[example_id]: - output_line = line.split()[0] + " " + preds_list[example_id].pop(0) + "\n" + output_line = f"{line.split()[0]} {preds_list[example_id].pop(0)}" + "\n" writer.write(output_line) else: logger.warning("Maximum sequence length exceeded: No prediction for '%s'.", line.split()[0]) def get_labels(self, path: str) -> List[str]: - if path: - with open(path, "r") as f: - labels = f.read().splitlines() - if "O" not in labels: - labels = ["O"] + labels - return labels - else: + if not path: return ["O", "B-MISC", "I-MISC", "B-PER", "I-PER", "B-ORG", "I-ORG", "B-LOC", "I-LOC"] + with open(path, "r") as f: + labels = f.read().splitlines() + if "O" not in labels: + labels = ["O"] + labels + return labels class Chunk(NER): @@ -73,13 +72,7 @@ def __init__(self): super().__init__(label_idx=-2) def get_labels(self, path: str) -> List[str]: - if path: - with open(path, "r") as f: - labels = f.read().splitlines() - if "O" not in labels: - labels = ["O"] + labels - return labels - else: + if not path: return [ "O", "B-ADVP", @@ -103,6 +96,11 @@ def get_labels(self, path: str) -> List[str]: "I-CONJP", "I-PP", ] + with open(path, "r") as f: + labels = f.read().splitlines() + if "O" not in labels: + labels = ["O"] + labels + return labels class POS(TokenClassificationTask): @@ -127,15 +125,17 @@ def read_examples_from_file(self, data_dir, mode: Union[Split, str]) -> List[Inp return examples def write_predictions_to_file(self, writer: TextIO, test_input_reader: TextIO, preds_list: List): - example_id = 0 - for sentence in parse_incr(test_input_reader): + for example_id, sentence in enumerate(parse_incr(test_input_reader)): s_p = preds_list[example_id] - out = "" - for token in sentence: - out += f'{token["form"]} ({token["upos"]}|{s_p.pop(0)}) ' - out += "\n" + out = ( + "".join( + f'{token["form"]} ({token["upos"]}|{s_p.pop(0)}) ' + for token in sentence + ) + + "\n" + ) + writer.write(out) - example_id += 1 def get_labels(self, path: str) -> List[str]: if path: diff --git a/examples/ner/utils_ner.py b/examples/ner/utils_ner.py index 99cd982c7..c83fe783a 100644 --- a/examples/ner/utils_ner.py +++ b/examples/ner/utils_ner.py @@ -215,12 +215,13 @@ def __init__( # Load data features from cache or dataset file cached_features_file = os.path.join( data_dir, - "cached_{}_{}_{}".format(mode.value, tokenizer.__class__.__name__, str(max_seq_length)), + f"cached_{mode.value}_{tokenizer.__class__.__name__}_{str(max_seq_length)}", ) + # Make sure only the first process in distributed training processes the dataset, # and the others will use the cache. - lock_path = cached_features_file + ".lock" + lock_path = f"{cached_features_file}.lock" with FileLock(lock_path): if os.path.exists(cached_features_file) and not overwrite_cache: @@ -230,23 +231,24 @@ def __init__( logger.info(f"Creating features from dataset file at {data_dir}") examples = token_classification_task.read_examples_from_file(data_dir, mode) # TODO clean up all this to leverage built-in features of tokenizers - self.features = token_classification_task.convert_examples_to_features( - examples, - labels, - max_seq_length, - tokenizer, - cls_token_at_end=bool(model_type in ["xlnet"]), - # xlnet has a cls token at the end - cls_token=tokenizer.cls_token, - cls_token_segment_id=2 if model_type in ["xlnet"] else 0, - sep_token=tokenizer.sep_token, - sep_token_extra=False, - # roberta uses an extra separator b/w pairs of sentences, cf. github.com/pytorch/fairseq/commit/1684e166e3da03f5b600dbb7855cb98ddfcd0805 - pad_on_left=bool(tokenizer.padding_side == "left"), - pad_token=tokenizer.pad_token_id, - pad_token_segment_id=tokenizer.pad_token_type_id, - pad_token_label_id=self.pad_token_label_id, + self.features = ( + token_classification_task.convert_examples_to_features( + examples, + labels, + max_seq_length, + tokenizer, + cls_token_at_end=model_type in {"xlnet"}, + cls_token=tokenizer.cls_token, + cls_token_segment_id=2 if model_type in {"xlnet"} else 0, + sep_token=tokenizer.sep_token, + sep_token_extra=False, + pad_on_left=tokenizer.padding_side == "left", + pad_token=tokenizer.pad_token_id, + pad_token_segment_id=tokenizer.pad_token_type_id, + pad_token_label_id=self.pad_token_label_id, + ) ) + logger.info(f"Saving features into cached file {cached_features_file}") torch.save(self.features, cached_features_file) @@ -289,19 +291,18 @@ def __init__( labels, max_seq_length, tokenizer, - cls_token_at_end=bool(model_type in ["xlnet"]), - # xlnet has a cls token at the end + cls_token_at_end=model_type in {"xlnet"}, cls_token=tokenizer.cls_token, - cls_token_segment_id=2 if model_type in ["xlnet"] else 0, + cls_token_segment_id=2 if model_type in {"xlnet"} else 0, sep_token=tokenizer.sep_token, sep_token_extra=False, - # roberta uses an extra separator b/w pairs of sentences, cf. github.com/pytorch/fairseq/commit/1684e166e3da03f5b600dbb7855cb98ddfcd0805 - pad_on_left=bool(tokenizer.padding_side == "left"), + pad_on_left=tokenizer.padding_side == "left", pad_token=tokenizer.pad_token_id, pad_token_segment_id=tokenizer.pad_token_type_id, pad_token_label_id=self.pad_token_label_id, ) + def gen(): for ex in self.features: if ex.token_type_ids is None: diff --git a/examples/sentiment/data.py b/examples/sentiment/data.py index 45d7a0bc4..c434158c6 100644 --- a/examples/sentiment/data.py +++ b/examples/sentiment/data.py @@ -31,12 +31,7 @@ def __getitem__(self, item): else: hot_encoding.append(0.0) label_hot_encoding = torch.FloatTensor(hot_encoding) - output = dict( - text=text, - label=label_hot_encoding, - input_ids=input_ids - ) - return output + return dict(text=text, label=label_hot_encoding, input_ids=input_ids) class MultiLabelClassificationDatamodule(pl.LightningDataModule): @@ -45,9 +40,7 @@ def __init__(self, corpus, tokenizer, **kwargs): self.corpus = corpus # num_labels = corpus.num_labels num_labels = corpus.num_aspect_labels - samples = None - if "samples" in kwargs: - samples = kwargs["samples"] + samples = kwargs.get("samples") del kwargs["samples"] self.dataset_kwargs = { "tokenizer": tokenizer, diff --git a/examples/sentiment/train_bert.py b/examples/sentiment/train_bert.py index 5040053f4..81921ece5 100644 --- a/examples/sentiment/train_bert.py +++ b/examples/sentiment/train_bert.py @@ -41,12 +41,11 @@ def __getitem__(self, index: int): else: hot_encoding.append(0.0) label_hot_encoding = torch.FloatTensor(hot_encoding) - output = dict( + return dict( input_ids=encoded["input_ids"], attention_mask=encoded["attention_mask"], labels=label_hot_encoding, ) - return output class UITABSADataModule(pl.LightningDataModule): @@ -58,12 +57,14 @@ def __init__(self, train_dataset, test_dataset, val_dataset, batch_size=24): self.batch_size = batch_size def train_dataloader(self): - output = DataLoader(self.train_dataset, self.batch_size, shuffle=True, drop_last=True) - return output + return DataLoader( + self.train_dataset, self.batch_size, shuffle=True, drop_last=True + ) def val_dataloader(self): - output = DataLoader(self.val_dataset, self.batch_size, shuffle=True, drop_last=True) - return output + return DataLoader( + self.val_dataset, self.batch_size, shuffle=True, drop_last=True + ) def test_dataloader(self): return DataLoader(self.test_dataset, self.batch_size, shuffle=False, drop_last=True) @@ -80,11 +81,12 @@ def __init__(self, config): def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, start_positions=None, end_positions=None): - outputs = self.roberta(input_ids, - attention_mask=attention_mask, - position_ids=position_ids, - head_mask=head_mask) - return outputs + return self.roberta( + input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + head_mask=head_mask, + ) class BertForMultilabelClassification(pl.LightningModule): @@ -103,9 +105,7 @@ def forward(self, input_ids, attention_mask, labels=None): output = self.roberta(input_ids.squeeze()) output = self.classifier(output.pooler_output) output = torch.sigmoid(output) - loss = 0 - if labels is not None: - loss = self.criterion(output, labels) + loss = self.criterion(output, labels) if labels is not None else 0 return loss, output def training_step(self, batch, batch_idx): @@ -139,8 +139,7 @@ def test_step(self, batch, batch_idx): return loss def configure_optimizers(self): - optimizer = AdamW(self.parameters(), lr=2e-5) - return optimizer + return AdamW(self.parameters(), lr=2e-5) def main(): diff --git a/examples/sentiment/train_gpt2.py b/examples/sentiment/train_gpt2.py index f62b80b8b..bf79cb5e3 100644 --- a/examples/sentiment/train_gpt2.py +++ b/examples/sentiment/train_gpt2.py @@ -22,14 +22,12 @@ def __init__(self, gpt2, num_labels): self.valid_f1 = F1(mdmc_average='global') def forward(self, input_ids, labels=None): - loss = 0 gpt2_outputs = self.gpt2(input_ids) hidden_states = gpt2_outputs[0].squeeze() logits = self.logit(self.linear(hidden_states)) batch_size, sequence_length = input_ids.shape[:2] logits = logits[range(batch_size), sequence_length] - if labels is not None: - loss = self.criterion(logits, labels) + loss = self.criterion(logits, labels) if labels is not None else 0 return loss, logits def training_step(self, batch, batch_idx): @@ -50,8 +48,7 @@ def validation_step(self, batch, batch_idx): self.log('valid_f1', self.valid_f1, on_step=True, on_epoch=True) def configure_optimizers(self): - optimizer = SGD(self.parameters(), lr=1e-6) - return optimizer + return SGD(self.parameters(), lr=1e-6) @hydra.main(config_path="configs/", config_name="config.yaml") diff --git a/examples/text_normalize/compare_tool.py b/examples/text_normalize/compare_tool.py index a4954c683..f301165e9 100644 --- a/examples/text_normalize/compare_tool.py +++ b/examples/text_normalize/compare_tool.py @@ -18,13 +18,13 @@ def compare_two_tools(): df = pd.DataFrame(columns=["word", "lower", "other", "uts", "group", "miss_spell"]) with open(TOKENS_ANALYSE_FILE) as f: data = {} - for i, line in enumerate(f): + for line in f: word, freq = line.split("\t\t") other_words = Normalizer.normalize(word) uts_words = text_normalize(word) if word != "nghiêng" and len(word) > 6: continue - if other_words != word and other_words != uts_words: + if other_words not in [word, uts_words]: analysable_word = AnalysableWord(word) data[analysable_word] = { "other": other_words, diff --git a/examples/text_normalize/evaluation.py b/examples/text_normalize/evaluation.py index 868793cf1..89544df51 100644 --- a/examples/text_normalize/evaluation.py +++ b/examples/text_normalize/evaluation.py @@ -30,8 +30,7 @@ def predict_sentence(s, conf): print(f"{tokens[0]} -> {tokens[1]}") new_row = "\t".join(tokens) result.append(new_row) - new_s = "\n".join(result) - return new_s + return "\n".join(result) def predict(cfg): diff --git a/examples/text_normalize/normalize.py b/examples/text_normalize/normalize.py index dc5ce0224..6014ed98b 100644 --- a/examples/text_normalize/normalize.py +++ b/examples/text_normalize/normalize.py @@ -78,21 +78,52 @@ def count_normalize(): "y": "ỷỳỵỵỹý" } syllable_map = {} -for key in syllable_map_r: - items = syllable_map_r[key] +for key, items in syllable_map_r.items(): for item in items: syllable_map[item] = key -NONE_DIACRITIC_SINGLE_VOWELS = set(["a", "e", "i", "o", "u", "y"]) -NONE_DIACRITIC_DOUBLE_VOWELS = set([ - "ai", "ao", "au", "ay", - "eo", "eu", "ia", "ie", "iu", "oa", "oe", "oi", "oo", - "ua", "ue", "ui", "uo", "uu", "uy", "ye" -]) -NONE_DIACRITIC_TRIPLE_VOWELS = set([ - "iai", "ieu", "iua", "oai", "oao", "oay", "oeo", - "uao", "uai", "uay", "uoi", "uou", "uya", "uye", "uyu", - "yeu" -]) +NONE_DIACRITIC_SINGLE_VOWELS = {"a", "e", "i", "o", "u", "y"} +NONE_DIACRITIC_DOUBLE_VOWELS = { + "ai", + "ao", + "au", + "ay", + "eo", + "eu", + "ia", + "ie", + "iu", + "oa", + "oe", + "oi", + "oo", + "ua", + "ue", + "ui", + "uo", + "uu", + "uy", + "ye", +} + +NONE_DIACRITIC_TRIPLE_VOWELS = { + "iai", + "ieu", + "iua", + "oai", + "oao", + "oay", + "oeo", + "uao", + "uai", + "uay", + "uoi", + "uou", + "uya", + "uye", + "uyu", + "yeu", +} + NONE_DIACRITIC_VOWELS = NONE_DIACRITIC_SINGLE_VOWELS | NONE_DIACRITIC_DOUBLE_VOWELS | NONE_DIACRITIC_TRIPLE_VOWELS @@ -108,10 +139,7 @@ def __init__(self, word): normalize_syllable = syllable if normalize_syllable in NONE_DIACRITIC_SINGLE_VOWELS: group += normalize_syllable - if group in NONE_DIACRITIC_VOWELS: - miss_spell = False - else: - miss_spell = True + miss_spell = group not in NONE_DIACRITIC_VOWELS self.group = group self.miss_spell = miss_spell diff --git a/examples/text_normalize/tools/nvh.py b/examples/text_normalize/tools/nvh.py index 8c134880d..c0218dad1 100644 --- a/examples/text_normalize/tools/nvh.py +++ b/examples/text_normalize/tools/nvh.py @@ -7,14 +7,11 @@ def loaddicchar(): - dic = {} char1252 = 'à|á|ả|ã|ạ|ầ|ấ|ẩ|ẫ|ậ|ằ|ắ|ẳ|ẵ|ặ|è|é|ẻ|ẽ|ẹ|ề|ế|ể|ễ|ệ|ì|í|ỉ|ĩ|ị|ò|ó|ỏ|õ|ọ|ồ|ố|ổ|ỗ|ộ|ờ|ớ|ở|ỡ|ợ|ù|ú|ủ|ũ|ụ|ừ|ứ|ử|ữ|ự|ỳ|ý|ỷ|ỹ|ỵ|À|Á|Ả|Ã|Ạ|Ầ|Ấ|Ẩ|Ẫ|Ậ|Ằ|Ắ|Ẳ|Ẵ|Ặ|È|É|Ẻ|Ẽ|Ẹ|Ề|Ế|Ể|Ễ|Ệ|Ì|Í|Ỉ|Ĩ|Ị|Ò|Ó|Ỏ|Õ|Ọ|Ồ|Ố|Ổ|Ỗ|Ộ|Ờ|Ớ|Ở|Ỡ|Ợ|Ù|Ú|Ủ|Ũ|Ụ|Ừ|Ứ|Ử|Ữ|Ự|Ỳ|Ý|Ỷ|Ỹ|Ỵ'.split( '|') charutf8 = "à|á|ả|ã|ạ|ầ|ấ|ẩ|ẫ|ậ|ằ|ắ|ẳ|ẵ|ặ|è|é|ẻ|ẽ|ẹ|ề|ế|ể|ễ|ệ|ì|í|ỉ|ĩ|ị|ò|ó|ỏ|õ|ọ|ồ|ố|ổ|ỗ|ộ|ờ|ớ|ở|ỡ|ợ|ù|ú|ủ|ũ|ụ|ừ|ứ|ử|ữ|ự|ỳ|ý|ỷ|ỹ|ỵ|À|Á|Ả|Ã|Ạ|Ầ|Ấ|Ẩ|Ẫ|Ậ|Ằ|Ắ|Ẳ|Ẵ|Ặ|È|É|Ẻ|Ẽ|Ẹ|Ề|Ế|Ể|Ễ|Ệ|Ì|Í|Ỉ|Ĩ|Ị|Ò|Ó|Ỏ|Õ|Ọ|Ồ|Ố|Ổ|Ỗ|Ộ|Ờ|Ớ|Ở|Ỡ|Ợ|Ù|Ú|Ủ|Ũ|Ụ|Ừ|Ứ|Ử|Ữ|Ự|Ỳ|Ý|Ỷ|Ỹ|Ỵ".split( '|') - for i in range(len(char1252)): - dic[char1252[i]] = charutf8[i] - return dic + return {char1252[i]: charutf8[i] for i in range(len(char1252))} dicchar = loaddicchar() @@ -130,7 +127,7 @@ def chuan_hoa_dau_tu_tieng_viet(word): for index in nguyen_am_index: x, y = nguyen_am_to_ids[chars[index]] - if x == 4 or x == 8: # ê, ơ + if x in [4, 8]: # ê, ơ chars[index] = bang_nguyen_am[x][dau_cau] # for index2 in nguyen_am_index: # if index2 != index: @@ -165,12 +162,10 @@ def is_valid_vietnam_word(word): for index, char in enumerate(chars): x, y = nguyen_am_to_ids.get(char, (-1, -1)) if x != -1: - if nguyen_am_index == -1: + if nguyen_am_index == -1 or index - nguyen_am_index == 1: nguyen_am_index = index else: - if index - nguyen_am_index != 1: - return False - nguyen_am_index = index + return False return True diff --git a/examples/text_normalize/tools/vtm.py b/examples/text_normalize/tools/vtm.py index f7d1dd88d..e37979169 100644 --- a/examples/text_normalize/tools/vtm.py +++ b/examples/text_normalize/tools/vtm.py @@ -2,5 +2,4 @@ def normalize(word): - text = VietnameseTextNormalizer.Normalize(word) - return text + return VietnameseTextNormalizer.Normalize(word) diff --git a/examples/text_normalize/tools/vtt.py b/examples/text_normalize/tools/vtt.py index 6a69e7d83..df35cd098 100644 --- a/examples/text_normalize/tools/vtt.py +++ b/examples/text_normalize/tools/vtt.py @@ -2,5 +2,4 @@ def normalize(word): - text = viet_text_tools.normalize_diacritics(word) - return text + return viet_text_tools.normalize_diacritics(word) diff --git a/extensions/underthesea_core/lab_underthesea_core.py b/extensions/underthesea_core/lab_underthesea_core.py index 200ddfabc..0876fc4eb 100644 --- a/extensions/underthesea_core/lab_underthesea_core.py +++ b/extensions/underthesea_core/lab_underthesea_core.py @@ -12,7 +12,7 @@ "T[0]", "T[1]", "T[2]", "T[0,1].is_in_dict" ] -dictionary = set(["Chàng", "trai", "trường học"]) +dictionary = {"Chàng", "trai", "trường học"} # print("Call featurizer function") # print(featurizer(sentences, feature_configs, dictionary)) diff --git a/tests/featurizers/benchmark_featurizers.py b/tests/featurizers/benchmark_featurizers.py index 85fe0e108..f76bb4f8a 100644 --- a/tests/featurizers/benchmark_featurizers.py +++ b/tests/featurizers/benchmark_featurizers.py @@ -32,7 +32,7 @@ max_tokens = 200 words = ["Messi", "đạt", "giải", "quả", "bóng", "vàng"] sentences = [] -for i in range(n_sentences): +for _ in range(n_sentences): word = choice(words) sentence = [[word, "X"]] * max_tokens sentences.append(sentence) diff --git a/travis_pypi_setup.py b/travis_pypi_setup.py index 6eb3c25f9..17d066302 100644 --- a/travis_pypi_setup.py +++ b/travis_pypi_setup.py @@ -60,8 +60,11 @@ def fetch_public_key(repo): keyurl = 'https://api.travis-ci.org/repos/{0}/key'.format(repo) data = json.loads(urlopen(keyurl).read().decode()) if 'key' not in data: - errmsg = "Could not find public key for repo: {}.\n".format(repo) - errmsg += "Have you already added your GitHub repo to Travis?" + errmsg = ( + f"Could not find public key for repo: {repo}.\n" + + "Have you already added your GitHub repo to Travis?" + ) + raise ValueError(errmsg) return data['key'] @@ -110,11 +113,15 @@ def main(args): print("Wrote encrypted password to .travis.yml -- you're ready to deploy") -if '__main__' == __name__: +if __name__ == '__main__': import argparse parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--repo', default=GITHUB_REPO, - help='GitHub repo (default: %s)' % GITHUB_REPO) + parser.add_argument( + '--repo', + default=GITHUB_REPO, + help=f'GitHub repo (default: {GITHUB_REPO})', + ) + parser.add_argument('--password', help='PyPI password (will prompt if not provided)') diff --git a/underthesea/__init__.py b/underthesea/__init__.py index 7accffc1d..983e14524 100644 --- a/underthesea/__init__.py +++ b/underthesea/__init__.py @@ -25,7 +25,7 @@ except NameError: __version__ = 'unknown (running code interactively?)' except IOError as ex: - __version__ = "unknown (%s)" % ex + __version__ = f"unknown ({ex})" ########################################################### # TOP-LEVEL MODULES diff --git a/underthesea/corpus/data.py b/underthesea/corpus/data.py index 30649e1e4..cec0ad30c 100644 --- a/underthesea/corpus/data.py +++ b/underthesea/corpus/data.py @@ -20,16 +20,13 @@ def value(self, value): @score.setter def score(self, score): - if 0.0 <= score <= 1.0: - self._score = score - else: - self._score = 1.0 + self._score = score if 0.0 <= score <= 1.0 else 1.0 def __str__(self): - return "{} ({})".format(self._value, self._score) + return f"{self._value} ({self._score})" def __repr__(self): - return "{} ({})".format(self._value, self._score) + return f"{self._value} ({self._score})" class Sentence: @@ -49,8 +46,7 @@ def __repr__(self) -> str: def to_text_classification_format(self) -> str: labels_text = " ".join([f"__label__{label.value}" for label in self.labels]) - output = f"{labels_text} {self.text}" - return output + return f"{labels_text} {self.text}" def add_labels(self, labels: Union[List[Label], List[str]]): for label in labels: diff --git a/underthesea/corpus/tagged_corpus.py b/underthesea/corpus/tagged_corpus.py index 639b76cb5..95133ddbb 100644 --- a/underthesea/corpus/tagged_corpus.py +++ b/underthesea/corpus/tagged_corpus.py @@ -19,8 +19,7 @@ class DataReader: def load_tagged_corpus(data_folder, train_file=None, test_file=None): train = DataReader.__read_tagged_data(join(data_folder, train_file)) test = DataReader.__read_tagged_data(join(data_folder, test_file)) - tagged_corpus = TaggedCorpus(train, test) - return tagged_corpus + return TaggedCorpus(train, test) @staticmethod def __read_tagged_data(data_file): diff --git a/underthesea/corpus/util.py b/underthesea/corpus/util.py index e40650b92..87dd5a6c2 100644 --- a/underthesea/corpus/util.py +++ b/underthesea/corpus/util.py @@ -13,16 +13,11 @@ def __init__(self, path, parent_path, is_last): self.path = Path(str(path)) self.parent = parent_path self.is_last = is_last - if self.parent: - self.depth = self.parent.depth + 1 - else: - self.depth = 0 + self.depth = self.parent.depth + 1 if self.parent else 0 @property def displayname(self): - if self.path.is_dir(): - return self.path.name + '/' - return self.path.name + return f'{self.path.name}/' if self.path.is_dir() else self.path.name @classmethod def make_tree(cls, root, parent=None, is_last=False, criteria=None): @@ -32,12 +27,12 @@ def make_tree(cls, root, parent=None, is_last=False, criteria=None): displayable_root = cls(root, parent, is_last) yield displayable_root - children = sorted(list(path - for path in root.iterdir() - if criteria(path)), - key=lambda s: str(s).lower()) - count = 1 - for path in children: + children = sorted( + [path for path in root.iterdir() if criteria(path)], + key=lambda s: str(s).lower(), + ) + + for count, path in enumerate(children, start=1): is_last = count == len(children) if path.is_dir(): yield from cls.make_tree(path, @@ -46,7 +41,6 @@ def make_tree(cls, root, parent=None, is_last=False, criteria=None): criteria=criteria) else: yield cls(path, displayable_root, is_last) - count += 1 @classmethod def _default_criteria(cls, path): diff --git a/underthesea/corpus/validate_corpus.py b/underthesea/corpus/validate_corpus.py index fbef4dfd9..de6b5c9b6 100644 --- a/underthesea/corpus/validate_corpus.py +++ b/underthesea/corpus/validate_corpus.py @@ -7,7 +7,7 @@ from underthesea.file_utils import DATASETS_FOLDER from underthesea.pipeline.word_tokenize import tokenize -SUPPORTED_CORPUS_TYPE = set(["TOKENIZE"]) +SUPPORTED_CORPUS_TYPE = {"TOKENIZE"} error_count = 0 DEFAULT_MAX_ERROR = 30 @@ -65,7 +65,7 @@ def validate_utf8(file): else: break results = charset_normalizer.from_bytes(text).best() - if not (results.encoding == "utf-8" and results.coherence >= 0.99): + if results.encoding != "utf-8" or results.coherence < 0.99: warn(message=f"File {file} should encoding with UTF-8", level=1) sys.exit(1) with open(file, "r") as f: @@ -151,13 +151,13 @@ def validate_token(sentence, i_end, file): tokenized_tokens = tokenize(text) if tokenized_tokens != content_tokens: if len(text) > 53: - message = "tokenized error for text: " + text[:50] + "..." + message = f"tokenized error for text: {text[:50]}..." else: - message = "tokenized error for text: " + text + message = f"tokenized error for text: {text}" message += '\n' - message += ' Actual:' + str(content_tokens) + message += f' Actual:{content_tokens}' message += '\n' - message += 'Expected:' + str(tokenized_tokens) + message += f'Expected:{str(tokenized_tokens)}' warn(message=message, error_type="Format tokenize-error", file=base_name, line_number=i_start, level=3, sent_id=sent_id) diff --git a/underthesea/corpus/word_tokenize_corpus.py b/underthesea/corpus/word_tokenize_corpus.py index bef1d2a8c..55fef6942 100644 --- a/underthesea/corpus/word_tokenize_corpus.py +++ b/underthesea/corpus/word_tokenize_corpus.py @@ -8,8 +8,7 @@ class WordTokenizeCorpusReader: def read(data_folder, train_file=None, test_file=None): train = WordTokenizeCorpusReader.__read_data(join(data_folder, train_file)) test = WordTokenizeCorpusReader.__read_data(join(data_folder, test_file)) - tagged_corpus = TaggedCorpus(train, test) - return tagged_corpus + return TaggedCorpus(train, test) @staticmethod def __read_data(data_file): diff --git a/underthesea/corpus/ws_corpus.py b/underthesea/corpus/ws_corpus.py index 6cae9fc26..09ff30bd0 100644 --- a/underthesea/corpus/ws_corpus.py +++ b/underthesea/corpus/ws_corpus.py @@ -12,11 +12,8 @@ def __init__(self): def map_token(self, token): if token[1] == "B_W": - return " " + token[0] - if token[1] == "I_W": - return "_" + token[0] - else: - return " " + token[0] + return f" {token[0]}" + return f"_{token[0]}" if token[1] == "I_W" else f" {token[0]}" def _column_to_text(self, sentence): s = sentence diff --git a/underthesea/data_fetcher.py b/underthesea/data_fetcher.py index 64ba2d715..f1080e97b 100644 --- a/underthesea/data_fetcher.py +++ b/underthesea/data_fetcher.py @@ -70,7 +70,7 @@ def download_data(data, url): print(f"Resource {data} is already existed in: {filepath}") return - if data in set(["VNESES", "VNTQ_SMALL", "VNTQ_BIG"]): + if data in {"VNESES", "VNTQ_SMALL", "VNTQ_BIG"}: DataFetcher.download_raw_file_to_cache(repo_data) zip_datasets = [ @@ -98,9 +98,8 @@ def list(all): license = REPO[key]["license"] year = REPO[key]["year"] directory = REPO[key]["cache_dir"] - if not all: - if license == "Close": - continue + if not all and license == "Close": + continue if license == "Close": license = "Close*" datasets.append([name, type, license, year, directory]) @@ -183,8 +182,7 @@ def load_classification_corpus(data_folder) -> CategorizedCorpus: else: sentences_train, sentences_dev = DataFetcher.__sample(sentences_train) sentences_test: List[Sentence] = DataFetcher.read_text_classification_file(test_file) - corpus = CategorizedCorpus(sentences_train, sentences_dev, sentences_test) - return corpus + return CategorizedCorpus(sentences_train, sentences_dev, sentences_test) @staticmethod def __sample(data: List[Sentence], percentage: float = 0.1): @@ -200,8 +198,8 @@ def read_text_classification_file(path_to_file) -> List[Sentence]: sentences = [] with open(path_to_file) as f: lines = f.read().splitlines() + label_pattern = r"__label__(?P