CNN Model

src/framework__data_set.py

@@ -13,6 +13,7 @@
 from os.path import isfile, join
 import numpy as np
 
+
 """
 ***********************************************************************************************************************
     TimeSeriesDataSet Class
@@ -29,13 +30,17 @@ def __init__(self, list_of_df):
         self.__is_data_scaled = False
         self.__mean = None
         self.__std = None
+        self.merged_df = None
 
     """
     *******************************************************************************************************************
         Helper functions
     *******************************************************************************************************************
     """
 
+    def get_list(self):
+        return self.__list_of_df
+
     def __get_mean_and_std(self):
         """
         calculates mean and std of all samples
@@ -59,6 +64,26 @@ def __getitem__(self, key):
     def __len__(self):
         return len(self.__list_of_df)
 
+    def merge_df(self):
+        """
+        concat all the dataframes of the same application
+        """
+        # todo: fix the merging of same apps on different pods and namespaces
+        self.merged_df = pd.concat(self.__list_of_df)
+
+    def sort_by_time(self):
+        """
+        concat all the dataframes of the same application
+        and sort them by time
+        """
+        self.merge_df()
+        self.merged_df = self.merged_df.sort_values(by="time")
+        self.merged_df = self.merged_df.groupby(['time'], as_index=False).max().reset_index()
+
+    def get_marged(self):
+        return self.merged_df
+
+
     def sub_sample_data(self, sub_sample_rate):
         """
         creates sub sampling according to the rate (if for example rate = 5, then every 5 samples, the one with the
@@ -74,6 +99,33 @@ def sub_sample_data(self, sub_sample_rate):
 
         self.__list_of_df = new_list_of_df
 
+    def add_features(self):  # 2022-04-21 02:50:00   - example
+        """
+        Adding to the DataFrame "hour" and "day of week" columns for using those columns as features later
+        """
+        # todo: figure out if one-hot encoding can be good here
+        new_list_of_df = []
+        for df in self:
+            df['hour'] = df['time'].apply(lambda x: int((str(x).split(' ')[1].split(':')[0])))
+            df['day'] = df['time'].apply(lambda x: pd.Timestamp(str(x).split(' ')[0]).day_of_week) # or dayofweek
+        self.__list_of_df = new_list_of_df
+
+    def mean_sub_sample_data(self, sub_sample_rate):
+        """
+        creates sub sampling according to the rate (if for example rate = 5, then every 5 samples, the one with the
+        mean value is chosen to be in the data set).
+        @param sub_sample_rate:
+        """
+        # todo: fix the bug where the "time" column is disappear
+        new_list_of_df = []
+
+        for df in self:
+            sub_sampled_data = df.groupby(df.index // sub_sample_rate).mean()
+            assert len(sub_sampled_data) == ((len(df) + sub_sample_rate - 1) // sub_sample_rate)
+            new_list_of_df.append(sub_sampled_data)
+
+        self.__list_of_df = new_list_of_df
+
     def filter_data_that_is_too_short(self, data_length_limit):
         """
         filters the data samples. all data samples that have a length that is lower than data_length_limit will be
@@ -88,7 +140,38 @@ def filter_data_that_is_too_short(self, data_length_limit):
 
         self.__list_of_df = new_list_of_df
 
-    def plot_dataset(self, number_of_samples):
+
+    def filter_series_with_zeros(self):
+        """
+        filters the data samples with zeros.
+        """
+        new_list_of_df = []
+
+        for df in self:
+            # check if there is sample in the dataframe that contains some zero value
+            if not df.isin([0]).any().any():
+                new_list_of_df.append(df)
+
+        self.__list_of_df = new_list_of_df
+
+
+    def filter_series_extreme_values(self, n):
+        """
+        filter the first and last element from every dataframe
+        """
+        new_list_of_df = []
+
+        for df in self:
+            print(len(df))
+            new_list_of_df.append(df.iloc[n:-n])
+            print(len(df.iloc[n:-n]))
+            assert len(new_list_of_df[-1]) == len(df) - 2*n
+
+
+        self.__list_of_df = new_list_of_df
+
+
+    def plot_dataset(self, number_of_samples, title):
         """
         randomly selects samples from the data sets and plots . x-axis is time and y-axis is the value
         @param number_of_samples: number of randomly selected samples
@@ -99,8 +182,47 @@ def plot_dataset(self, number_of_samples):
             ts = df["sample"].copy()
             ts.index = [time for time in df["time"]]
             ts.plot()
+            plt.ylabel(title)
+            plt.xlabel('time stamp')
             plt.show()
 
+
+    def plot_group(self):
+        """
+        randomly selects samples from the data sets and plots . x-axis is time and y-axis is the value
+        @param number_of_samples: number of randomly selected samples
+        """
+        df = self.merged_df
+        # plt.close("all")
+        ts = df["sample"].copy()
+        ts.index = [time for time in df["time"]]
+        f = plt.figure()
+        f.set_figwidth(20)
+        f.set_figheight(10)
+        ts.plot()
+        plt.ylabel("group plot")
+        plt.xlabel('time stamp')
+        plt.show()
+
+
+
+    def plot_not_random_dataset(self, number_of_samples):
+        """
+        not randomly selects samples from the data sets and plots . x-axis is time and y-axis is the value
+        @param number_of_samples: number of selected samples
+        """
+        print("Totam number of TS are: ", len(self.__list_of_df))
+        counter = 0
+        for df in self.__list_of_df:
+            if counter == number_of_samples:
+                break
+            ts = df["sample"].copy()
+            ts.index = [time for time in df["time"]]
+            ts.plot()
+            plt.show()
+            counter += 1
+
+
     def scale_data(self):
         """
         rescaling the distribution of values so that the mean of observed values is 0, and the std is 1.
@@ -109,7 +231,7 @@ def scale_data(self):
         assert not self.__is_data_scaled
         self.__is_data_scaled = True
         self.__mean, self.__std = self.__get_mean_and_std()
-        # print(f"self.__mean = {self.__mean}, self.__std = {self.__std}", )
+        # print(f"self.__mean = {self.__mean}, self.__std = {self.__std}" , )
         # print("max_sample = ", max_sample, " min_sample = ", min_sample)
         for df in self:
             standardized_sample_column = (df["sample"] - self.__mean) / self.__std
@@ -295,27 +417,37 @@ def main():
     if test == 0:
         print("Getting DataSet.")
         dataset = get_data_set(
-            metric="container_mem",
-            application_name="bridge-marker",
+            metric="container_cpu",
+            application_name="cni-plugins",
             path_to_data="../data/"
         )
+        # dataset.sort_by_time()
+        # dataset.plot_group()
+        # exit()
+
+        dataset.filter_data_that_is_too_short(data_length_limit=20)
+        dataset.filter_series_extreme_values(3)
+        dataset.plot_dataset(number_of_samples=20, title="Value before normalization")
+        print("number of series before filter zeros: ", len(dataset.get_list()))
+        dataset.filter_series_with_zeros()
+        print("number of series after filter zeros: ", len(dataset.get_list()))
+        exit()
+
+        print("Splitting DataSet.")
+        dataset.sub_sample_data(sub_sample_rate=5)
         print("Plotting.")
-        dataset.plot_dataset(number_of_samples=3)
-        print("Subsampling.")
-        dataset.sub_sample_data(sub_sample_rate=60)
-        print("Plotting.")
-        dataset.plot_dataset(number_of_samples=3)
+        dataset.plot_dataset(number_of_samples=3, title="Value before normalization")
         print("Normalizing.")
         dataset.scale_data()
         print("Plotting.")
-        dataset.plot_dataset(number_of_samples=3)
+        dataset.plot_dataset(number_of_samples=3, title="Value after normalization")
         print("Filtering time series that are too short.")
-        dataset.filter_data_that_is_too_short(data_length_limit=2 * length_to_predict)
+        dataset.filter_data_that_is_too_short(data_length_limit=30)
         print("Splitting.")
         train, test = dataset.split_to_train_and_test(length_to_predict=length_to_predict)
         print("Plotting.")
-        train.plot_dataset(number_of_samples=10)
-        test.plot_dataset(number_of_samples=10)
+        train.plot_dataset(number_of_samples=3, title="train value")
+        test.plot_dataset(number_of_samples=3, title="test value")
     else:
         hist = get_amount_of_data_per_application(
             metric="container_mem",