Sourcery refactored master branch

apps/col_data.py

@@ -27,8 +27,7 @@ def get_word(self, word):
             return {}
         print(word)
         body = self.es.get(index=self.es_index, id=word)
-        data = body["_source"]
-        return data
+        return body["_source"]
 
     def save(self, word, data):
         self.es.index(index=self.es_index, id=word, body=data)

apps/col_dictionary.py

@@ -40,8 +40,11 @@ def switch_word(word):
     </style>""", unsafe_allow_html=True)
 
     placeholder = st.sidebar.empty()
-    search_text_box = placeholder.text_input('Word', value=st.session_state['current_word'], key='sidebar_text_input')
-    if search_text_box:
+    if search_text_box := placeholder.text_input(
+        'Word',
+        value=st.session_state['current_word'],
+        key='sidebar_text_input',
+    ):
         switch_word(search_text_box)
 
     buttons = {}
@@ -55,6 +58,5 @@ def switch_word(word):
 
     output_data = json_viewer(json_object=data, label=0)
 
-    save_button = st.button('Save')
-    if save_button:
+    if save_button := st.button('Save'):
         dictionary.save(st.session_state['current_word'], output_data)

apps/col_dictionary_export_from_elasticsearch.py

@@ -37,14 +37,13 @@ def extract_dictionary_from_elasticsearch(data):
     index_name = "dictionary"
     with open(DICTIONARY_FILE, "w") as f:
         f.write("")
-    f = open(DICTIONARY_FILE, "a")
-    i = 0
-    for res in query_doc(es, index_name):
-        docs = res["hits"]["hits"]
-        print(len(docs))
-        doc_content = extract_dictionary_from_elasticsearch(docs)
-        if len(doc_content) > 1:
-            dict_content = yaml.dump(doc_content, allow_unicode=True, sort_keys=True)
-            f.write(dict_content)
-            i += 1
-    f.close()
+    with open(DICTIONARY_FILE, "a") as f:
+        i = 0
+        for res in query_doc(es, index_name):
+            docs = res["hits"]["hits"]
+            print(len(docs))
+            doc_content = extract_dictionary_from_elasticsearch(docs)
+            if len(doc_content) > 1:
+                dict_content = yaml.dump(doc_content, allow_unicode=True, sort_keys=True)
+                f.write(dict_content)
+                i += 1

apps/col_dictionary_import_to_elasticsearch.py

@@ -9,9 +9,7 @@
     es = Elasticsearch()
 
     words = yaml.safe_load(content)
-    i = 0
-    for word_key, value in words.items():
-        i += 1
+    for i, (word_key, value) in enumerate(words.items(), start=1):
         data = {
             "headword": word_key,
             "senses": value

apps/col_streamlit.py

@@ -42,9 +42,7 @@ def find_word(ud_dataset, word) -> list:
 st.write('Loaded corpus: wiki')
 st.write(f'Loaded dictionary: {dictionary_n_words} words (202108.yaml)')
 
-word = st.text_input('Word')
-
-if word:
+if word := st.text_input('Word'):
     sentences = find_word(ud_dataset, word)
     st.write(f"## {word}")
     st.write("Output:")

datasets/DI_Vietnamese-UVD/scripts/correct_v1.0.alpha.py

@@ -36,8 +36,7 @@ def tag_correct(data):
             tag = node['tag']
             if tag in TAG_MAP:
                 correct_data[word][i]['tag'] = TAG_MAP[tag]
-            # case for word 'khối'
-            if word == 'khối' and node['tag'] == 'S':
+            if word == 'khối' and tag == 'S':
                 correct_data[word][i]['tag'] = 'adjective'
     return correct_data
 

datasets/DI_Vietnamese-UVD/scripts/data.py

@@ -24,20 +24,14 @@ def add(self, word):
     def load(file):
         with open(file) as f:
             yaml.safe_load(f)
-            new_dict = Dictionary()
-            return new_dict
+            return Dictionary()
 
     def to_data(self):
         data = {}
         words = sorted(self.words)
         for text in words:
             word = self.words[text]
-            if word.data is None:
-                content = ''
-            elif len(word.data) == 0:
-                content = ''
-            else:
-                content = word.data
+            content = '' if word.data is None or len(word.data) == 0 else word.data
             data[text] = content
         return data
 

datasets/DI_Vietnamese-UVD/scripts/underthesea_v170_dictionary.py

@@ -24,13 +24,7 @@
     'n': 'NOUN',
     'S': 'NOUN'  # khối
 }
-TEMP_IGNORE_POS = set([
-    'R',  # phụ từ tiếng Việt
-    'X',  # không phân loại
-    'Z',  # yếu tố cấu tạo từ
-    'D',  # không có định nghĩa (ví dụ: chút ít)
-    'O',  # úi chà
-])
+TEMP_IGNORE_POS = {'R', 'X', 'Z', 'D', 'O'}
 logger.info("Start loading")
 dict = Dictionary()
 pos_count = {}
@@ -45,7 +39,7 @@
     defs = []
     pos_tags = {}
     text = key
-    for definition in data[key]:
+    for definition in data[text]:
         pos_tag = definition['pos']
         if pos_tag not in pos_tags:
             i = len(pos_tags)

datasets/DI_Vietnamese-UVD/scripts/validate.py

@@ -6,10 +6,8 @@
 
 def warn(file, line_number, message, type=None):
     global total_errors
-    text = ""
-    if type:
-        text = f"[{type}] "
-    text += basename(file) + ":" + str(line_number)
+    text = f"[{type}] " if type else ""
+    text += f"{basename(file)}:{str(line_number)}"
     if total_errors < MAX_SHOW_ERRORS:
         print(colored(text, 'red'), colored(message, 'red'))
 
@@ -73,6 +71,6 @@ def stats():
         print("\n# DICTIONARY STATISTICS")
         global NUM_WORDS
         print("* Number of words:", NUM_WORDS)
-        print("* Tags:", len(VALID_TAGS), "(" + str(sorted(VALID_TAGS))[1:-1] + ")")
+        print("* Tags:", len(VALID_TAGS), f"({str(sorted(VALID_TAGS))[1:-1]})")
 
     stats()

datasets/DI_Vietnamese-UVD/scripts/validate_with_corpus.py

@@ -13,25 +13,41 @@
 MAX_SHOW_ERRORS = 100
 total_errors = 0
 
-punct = {"!", "/", ",", ".", "...", "?", "-", "\"", "-", ":", "(", ")", "–", "&", ";", "‘", "’", "+"}
+punct = {
+    "!",
+    "/",
+    ",",
+    ".",
+    "...",
+    "?",
+    "\"",
+    "-",
+    ":",
+    "(",
+    ")",
+    "–",
+    "&",
+    ";",
+    "‘",
+    "’",
+    "+",
+}
+
 specials = {"rbkt", "lbkt"}
 
 
 def warn(file, line_number, message, type=None):
     global total_errors
-    text = ""
-    if type:
-        text = f"[{type}] "
-    text += basename(file) + ":" + str(line_number)
+    text = f"[{type}] " if type else ""
+    text += f"{basename(file)}:{str(line_number)}"
     if total_errors < MAX_SHOW_ERRORS:
         print(colored(text, 'red'), colored(message, 'red'))
 
     total_errors += 1
 
 
 def load_dictionary(dict_file):
-    dict = joblib.load(dict_file)
-    return dict
+    return joblib.load(dict_file)
 
 
 def load_corpus(file):
@@ -60,12 +76,29 @@ def load_corpus(file):
             continue
         tags = list(words[word])
         ignores_tags = {
-            "ADJ", "ADV", "INTJ", "NOUN", "PROPN", "PRON", "SYM", "X", "N:G", "VERB:G", "NY",
-            "N", "NB", "NNPy",
-            "NNP", "NNPy",
-            "V", "VERB",
-            "Num", "NUMx", "NUM", "NUMX"
+            "ADJ",
+            "ADV",
+            "INTJ",
+            "NOUN",
+            "PROPN",
+            "PRON",
+            "SYM",
+            "X",
+            "N:G",
+            "VERB:G",
+            "NY",
+            "N",
+            "NB",
+            "NNP",
+            "NNPy",
+            "V",
+            "VERB",
+            "Num",
+            "NUMx",
+            "NUM",
+            "NUMX",
         }
+
         if tags[0] in ignores_tags:
             continue
         if word not in dict:

datasets/UD_Vietnamese-COL/scripts/make_corpus.py

@@ -17,8 +17,7 @@ def add_lemma_column(sentence):
             tokens.insert(1, lemma)
             new_row = "\t".join(tokens)
             result.append(new_row)
-    new_sentence = "\n".join(result)
-    return new_sentence
+    return "\n".join(result)
 
 
 filepath = join(dirname(dirname(__file__)), "corpus", "ud", "202108.txt")

examples/chatbot/baggage_claim/run_tests.py

@@ -42,9 +42,7 @@ def send(self, message):
         payload = {'sender': self.id, 'message': 'hey there'}
         headers = {'content-type': 'application/json'}
         r = requests.post('http://localhost:5005/webhooks/rest/webhook', json=payload, headers=headers)
-        if r is None:
-            return None
-        return r.json()
+        return None if r is None else r.json()
 
 
 if __name__ == '__main__':

examples/ner/preprocess_data.py

@@ -26,9 +26,9 @@ def transform_sentence(sentence):
         f.write(content)
 
 
+max_len = 0
 for file in ["train.txt", "test.txt", "dev.txt"]:
     print(file)
-    max_len = 0
     src_folder = join("bert-ner", "data", "vlsp2016")
     dest_folder = join("bert-ner", "data")
     transform(src_file=join(src_folder, file), dest_file=join(dest_folder, file))

examples/ner/run_ner_pl.py

@@ -25,14 +25,13 @@ def encode(self, labels):
         if type(labels) == list:
             return [self.encode(label) for label in labels]
         label = labels  # label is a string
-        if label not in self.label2index:
-            index = self.vocab_size
-            self.label2index[label] = index
-            self.index2label[index] = label
-            self.vocab_size += 1
-            return index
-        else:
+        if label in self.label2index:
             return self.label2index[label]
+        index = self.vocab_size
+        self.label2index[label] = index
+        self.index2label[index] = label
+        self.vocab_size += 1
+        return index
 
 
 class TokenClassificationCorpus:
@@ -187,8 +186,7 @@ def test_step(self, batch, batch_idx):
         self.log('test_loss', loss)
 
     def configure_optimizers(self):
-        optimizer = AdamW(self.parameters(), lr=2e-5)
-        return optimizer
+        return AdamW(self.parameters(), lr=2e-5)
 
 
 def main():

examples/ner/tasks.py

@@ -51,20 +51,19 @@ def write_predictions_to_file(self, writer: TextIO, test_input_reader: TextIO, p
                 if not preds_list[example_id]:
                     example_id += 1
             elif preds_list[example_id]:
-                output_line = line.split()[0] + " " + preds_list[example_id].pop(0) + "\n"
+                output_line = f"{line.split()[0]} {preds_list[example_id].pop(0)}" + "\n"
                 writer.write(output_line)
             else:
                 logger.warning("Maximum sequence length exceeded: No prediction for '%s'.", line.split()[0])
 
     def get_labels(self, path: str) -> List[str]:
-        if path:
-            with open(path, "r") as f:
-                labels = f.read().splitlines()
-            if "O" not in labels:
-                labels = ["O"] + labels
-            return labels
-        else:
+        if not path:
             return ["O", "B-MISC", "I-MISC", "B-PER", "I-PER", "B-ORG", "I-ORG", "B-LOC", "I-LOC"]
+        with open(path, "r") as f:
+            labels = f.read().splitlines()
+        if "O" not in labels:
+            labels = ["O"] + labels
+        return labels
 
 
 class Chunk(NER):
@@ -73,13 +72,7 @@ def __init__(self):
         super().__init__(label_idx=-2)
 
     def get_labels(self, path: str) -> List[str]:
-        if path:
-            with open(path, "r") as f:
-                labels = f.read().splitlines()
-            if "O" not in labels:
-                labels = ["O"] + labels
-            return labels
-        else:
+        if not path:
             return [
                 "O",
                 "B-ADVP",
@@ -103,6 +96,11 @@ def get_labels(self, path: str) -> List[str]:
                 "I-CONJP",
                 "I-PP",
             ]
+        with open(path, "r") as f:
+            labels = f.read().splitlines()
+        if "O" not in labels:
+            labels = ["O"] + labels
+        return labels
 
 
 class POS(TokenClassificationTask):
@@ -127,15 +125,17 @@ def read_examples_from_file(self, data_dir, mode: Union[Split, str]) -> List[Inp
         return examples
 
     def write_predictions_to_file(self, writer: TextIO, test_input_reader: TextIO, preds_list: List):
-        example_id = 0
-        for sentence in parse_incr(test_input_reader):
+        for example_id, sentence in enumerate(parse_incr(test_input_reader)):
             s_p = preds_list[example_id]
-            out = ""
-            for token in sentence:
-                out += f'{token["form"]} ({token["upos"]}|{s_p.pop(0)}) '
-            out += "\n"
+            out = (
+                "".join(
+                    f'{token["form"]} ({token["upos"]}|{s_p.pop(0)}) '
+                    for token in sentence
+                )
+                + "\n"
+            )
+
             writer.write(out)
-            example_id += 1
 
     def get_labels(self, path: str) -> List[str]:
         if path: