document-statistics/zipf_graph.py at master · norbert-acedanski/document-statistics · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
from fileinput import FileInput
import matplotlib.pyplot as plt
import re

file_name = "process_file.txt"
start_index = 0
amount_of_words_to_display = 100
list_of_characters_to_cut = ["‘", "’", "'", "–", "-"]


def load_data():
    try:
        with open(file_name, "r", encoding='utf-8') as input_file:
            data = input_file.readlines()
            for (line_number, line) in enumerate(data):
                data[line_number] = line.replace('\n', '')
    except OSError:
        raise FileNotFoundError(f'File "{file_name}" could not be opened. Check file name or file path.')
    if len(data) == 0:
        raise Exception("No data found in given a file!")
    return data


def process_data(data):
    word_count = {}
    for line in data:
        for word in line.split():
            isolated_word = re.sub('[!?@#$\[\]<>%^&*():;]', '', word).lower()
            isolated_word = isolated_word.replace(',', '.') if ',' in isolated_word else isolated_word
            try:
                word_with_numbers_handled = float(isolated_word)
            except ValueError:
                word_with_numbers_handled = re.sub('[,.]', '', isolated_word)
                if word_with_numbers_handled[0] in list_of_characters_to_cut:
                    word_with_numbers_handled = word_with_numbers_handled[1:]
                if word_with_numbers_handled == "":
                    continue
                if word_with_numbers_handled[-1] in list_of_characters_to_cut:
                    word_with_numbers_handled = word_with_numbers_handled[:-1]
            if word_with_numbers_handled in word_count:
                word_count[str(word_with_numbers_handled)] += 1
            else:
                word_count[str(word_with_numbers_handled)] = 1
    sorted_word_count = sorted(word_count.items(), key=lambda x: x[1], reverse=True)
    words_list, number_of_word_instances_list = [], []
    for pair in sorted_word_count:
        words_list.append(pair[0])
        number_of_word_instances_list.append(pair[1])
    return words_list, number_of_word_instances_list


def save_to_files(words_list, number_of_word_instances_list):
    with open("words_list.txt", "w", encoding='utf-8') as words_list_file, \
            open("number_of_word_instances_list.txt", "w", encoding='utf-8') as number_of_word_instances_list_file:
        for count in range(len(words_list)):
            words_list_file.write(words_list[count] + '\n')
            number_of_word_instances_list_file.write(str(number_of_word_instances_list[count]) + '\n')


def print_number_of_words_in_given_file():
    print(f"Number of words in a given file: {len(words_list)}")


def plot_zipf(words_list, number_of_word_instances_list):
    for (number, word) in enumerate(words_list):
        words_list[number] = str(number + 1) + ". " + str(word)
    continue_graph = "y"
    multiply_factor = 1
    plt.figure("Zipf Graph")
    plt.ion()
    plt.show()
    while continue_graph == "y":
        if start_index + (multiply_factor - 1)*amount_of_words_to_display > len(words_list):
            print("No more words to graph")
            input("To close window, press Enter")
            break
        plt.clf()
        plt.title("Zipf Graph")
        plt.xlabel("Word in given document")
        plt.ylabel("Amount of times the word occurs in the document")
        plt.plot(words_list[start_index + (multiply_factor - 1)*amount_of_words_to_display:
                            start_index + multiply_factor*amount_of_words_to_display],
                 number_of_word_instances_list[start_index + (multiply_factor - 1)*amount_of_words_to_display:
                                               start_index + multiply_factor*amount_of_words_to_display], marker="o")
        plt.xticks(rotation=90)
        plt.grid(True)
        plt.draw()
        continue_graph = input("Show next " + str(amount_of_words_to_display) + " words? (y/N): ").lower()
        while continue_graph != "y" and continue_graph != "n":
            continue_graph = input("Unknown command. Try again!: ")
        multiply_factor += 1
    print("Closing window")


if __name__ == '__main__':
    data = load_data()
    words_list, number_of_word_instances_list = process_data(data)
    save_to_files(words_list, number_of_word_instances_list)
    print_number_of_words_in_given_file()
    plot_zipf(words_list, number_of_word_instances_list)