binary-classifiers/perceptron.py at main · shivchander/binary-classifiers · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
#!/usr/bin/env python3
__author__ = "Shivchander Sudalairaj"
__license__ = "MIT"

'''
Perceptron classifier
'''
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
np.random.seed(0)


# Find the min and max values for each column
def dataset_minmax(dataset):
    minmax = list()
    for i in range(len(dataset[0])):
        col_values = [row[i] for row in dataset]
        value_min = min(col_values)
        value_max = max(col_values)
        minmax.append([value_min, value_max])
    return minmax


# Rescale dataset columns to the range 0-1
def normalize_dataset(dataset):
    minmax = dataset_minmax(dataset)
    for row in dataset:
        for i in range(len(row)):
            row[i] = (row[i] - minmax[i][0]) / (minmax[i][1] - minmax[i][0])
    return dataset


def train_test_split(X, y, test_size):

    df = pd.DataFrame(X, columns=['P', 'N'])
    df['label'] = y
    # Shuffle dataset
    shuffle_df = df.sample(frac=1)

    # Define a size for your train set
    train_size = int((1-test_size) * len(df))

    # Split your dataset
    train_set = shuffle_df[:train_size]
    test_set = shuffle_df[train_size:]

    X_train = train_set.to_numpy()[:, 0:-1]
    y_train = train_set.to_numpy()[:, -1]
    X_test = test_set.to_numpy()[:, 0:-1]
    y_test = test_set.to_numpy()[:, -1]

    return X_train, X_test, y_train, y_test


def balanced_acc(y_true, y_pred):

    tp, tn, fp, fn = 0, 0, 0, 0

    for y, y_hat in zip(y_true, y_pred):
        if y == y_hat == 1:
            tp += 1
        elif y == y_hat == 0:
            tn += 1
        elif y_hat == 1 and y == 0:
            fp += 1
        else:
            fn += 1
    sensitivity = tp / (tp + fn)
    specificity = tn / (fp + tn)
    balanced_acc = (sensitivity + specificity) / 2

    return round(balanced_acc, 3)


class Perceptron:
    def __init__(self):
        self.w = None
        self.b = None

    def threshold_function(self, x):
        fx = np.dot(self.w, x)+self.b
        return 1 if fx > 0 else 0

    def predict(self, X):
        y_pred = []
        X = normalize_dataset(X)
        for x in X:
            y_pred.append(self.threshold_function(x))

        return y_pred

    def fit(self, X, y, epochs=100, alpha=0.1, validation_split=0.2, weight_init='random',epoch_checkpoint=5,
            verbose=True, do_plot=True, do_validation=True, normalize=True, training_error_track=False):

        if normalize:
            X = np.array(normalize_dataset(X))

        if do_validation:
            # train-validation split
            X_train, X_val, y_train, y_val = train_test_split(X, y, test_size= validation_split)
        else:
            X_train = X
            y_train = y

        # weight initialization
        if weight_init == 'random':
            self.w = np.random.rand(X.shape[1])      # random init of weights to braak symmetry
        if weight_init == 'zeros':
            self.w = np.zeros(X.shape[1])
        if weight_init == 'ones':
            self.w = np.ones(X.shape[1])

        self.b = 0                                  # initializing bias to 0
        weights_history = []
        bias_history = []
        train_errors = {}
        val_errors = {}
        for i in range(epochs):
            for xi, yi in zip(X_train, y_train):
                y_pred = self.threshold_function(xi)
                # weight update
                self.w = self.w + (alpha * (yi - y_pred)) * xi
                # bias update
                self.b = self.b + (alpha * (yi - y_pred)) * 1

            weights_history.append(self.w)
            bias_history.append(self.b)

            if do_validation:
                if i % epoch_checkpoint == 0:        # checkpoint to track error
                    _y_train_preds = self.predict(X_train)
                    _y_val_preds = self.predict(X_val)
                    train_err_i = 1 - balanced_acc(y_train, _y_train_preds)
                    val_err_i = 1 - balanced_acc(y_val, _y_val_preds)
                    train_errors[i] = train_err_i
                    val_errors[i] = val_err_i

                    if verbose:
                        print("Epoch %d: \n\t Training Error: %0.3f \t Validation Error: %0.3f"%(i, train_err_i,
                                                                                                       val_err_i))

            if training_error_track:
                if i % epoch_checkpoint == 0:
                    _y_preds = self.predict(X_train)
                    train_err_i = 1 - balanced_acc(y_train, _y_preds)
                    train_errors[i] = train_err_i

        if do_validation:
            if do_plot:
                plt.plot(list(train_errors.keys()), list(train_errors.values()), label='E_train')
                plt.plot(list(val_errors.keys()), list(val_errors.values()), label='E_test')
                plt.xlabel('epochs')
                plt.ylabel('Error (1- balanced acc')
                plt.title('Training and Test Error of Perceptron with epochs')
                plt.legend(loc="upper left")
                # plt.xticks(list(train_errors.keys()))
                plt.savefig('figs/perceptron.pdf')
                plt.clf()

        if training_error_track:
            return train_errors