dataExtractor.py

import pandas as pd
# from sklearn.decomposition import PCA
from sklearn.cluster import AffinityPropagation
from sklearn.cluster import DBSCAN
from sklearn.cluster import OPTICS
from sklearn.cluster import MeanShift
from sklearn.cluster import AgglomerativeClustering
from sklearn.cluster import Birch
from sklearn.cluster import KMeans
from sklearn.cluster import MiniBatchKMeans
from sklearn.cluster import SpectralClustering
from sklearn.mixture import GaussianMixture
from sklearn.preprocessing import StandardScaler
from prince import MCA #Categorical PCA
from helperFuncs import co_association_matrix
import numpy as np
import sys
import os
# from os.path import exists

def validIDs(features):
    valid_ids = set()
    for path in features:
        directories = path.split('/')
        search_dir = directories[0] + "/" + directories[1]
        files = os.listdir(search_dir)
        for file in files:
            if not os.path.isfile(search_dir+"/"+file):
                continue
            participant_id = file.split('_')[1][:-4]
            valid_ids.add(participant_id)
    valid_ids = list(valid_ids)
    valid_ids.sort()
    return valid_ids

def sigLocExtract(sigLocFilePath, meta):
    df = pd.read_csv(sigLocFilePath+meta['pid']+'.csv')
    M, Y, _ = tuple(meta['f_opt'])
    df = df.loc[(pd.DatetimeIndex(df.start).month == M) & (pd.DatetimeIndex(df.start).year == Y)]
    grouped = df.groupby('start')
    data = [meta]
    for name, group in grouped:
        dic = {}
        dic['ActivityDate'] = name.split(' ')[0]
        cols = ['latitude','longitude','proportion']
        locs = []
        for i in range(len(group)):
            locDet = {}
            for col in cols:
                locDet[col] = list(group[col])[i]
            locs.append(locDet)
        dic['locations'] = locs
        data.append(dic)
    return data

#Meta format - [category, method of imputing data]
def surveyMCA(filePath, meta):
    df = pd.read_csv(filePath+meta['pid']+'.csv')
    pdf = pd.DataFrame()
    pdf['ActivityDate'] = pd.to_datetime(df['ActivityDate'],dayfirst=True)
    pdf['ActivityDate'] = pdf['ActivityDate'].dt.strftime("%d/%m/%Y")
    category = meta['f_opt'][0]
    columnSubSet = []
    for column in list(df.columns):
        if(column[:-1] == category):
            columnSubSet.append(column)
    df.replace({0: "False", 1: "True"}, inplace = True) #This line was added for MCA
    data = df[columnSubSet].values
    mca = MCA(n_components=2, engine='sklearn', random_state=67)
    pData = np.array(mca.fit_transform(data))
    pVal = pData[:,[0]] #MCA 1
    pVal = [val[0] for val in pVal]
    pdf[category] = pVal
    returnData = []
    vizMeta = {} #This data is rendered by d3.js
    vizMeta['c0'] = category
    vizMeta['panelId'] = meta['panelId']
    returnData.append(vizMeta)
    for _, row in pdf.iterrows():
        dic = {}
        dic['ActivityDate'] = row['ActivityDate'] #Constant column
        dic[category] = row[category]
        returnData.append(dic)
    return returnData


#Meta format - [Aggregate/Complete, Number of clusters]
def surveyPerformance(surveyFilePath, meta):
    df = pd.read_csv(surveyFilePath+meta['pid']+'.csv')
    df['ActivityDate'] = pd.to_datetime(df['ActivityDate'],dayfirst=True)
    df.sort_values(by='ActivityDate', inplace=True)
    df['ActivityDate'] = df['ActivityDate'].dt.strftime("%d/%m/%Y")
    #Aggregating features
    if(meta['f_opt'][0] == "Aggregate"):
        df["social"] = (df['social1'] + df['social2'] + df['social3'] + df['social4'] + df['social5'])/5
        df["mood"] = (df['mood1'] + df['mood2'] + df['mood3'] + df['mood4'] + df['mood5'] + df['mood6'] + df['mood7'] + df['mood8'] + df['mood9'])/9
        df["sleep"] = (df["sleep1"] +  df["sleep2"] + df["sleep3"])/3
        df["psychosis"] = (df['psychosis1'] + df['psychosis2'] + df['psychosis3'] + df['psychosis4'] + df['psychosis5'])/5
        df["anxiety"] = (df['anxiety1'] + df['anxiety2'] + df['anxiety3'] + df['anxiety4'] + df['anxiety5'] + df['anxiety6'] + df['anxiety7'])/7
    data = df.drop(columns=["ActivityDate","Unnamed: 0"]).to_numpy()
    days = df.shape[0]
    matrixDat = np.zeros((days,days))
    print(meta['f_opt'])
    if(meta['f_opt'][1] == "Natural clustering"):
        save_stdout = sys.stdout
        sys.stdout = open('trash', 'w')
        model = AffinityPropagation(damping=0.5)
        matrixDat += co_association_matrix(model, data, days)
        model = DBSCAN(eps=0.1, min_samples=3)
        matrixDat += co_association_matrix(model, data, days)
        model = MeanShift()
        matrixDat += co_association_matrix(model, data, days)
        model = OPTICS(eps=0.1, min_samples=3)
        matrixDat += co_association_matrix(model, data, days)
        matrixDat /= 4.0
        sys.stdout = save_stdout
    else:
        save_stdout = sys.stdout
        try:
            clusters = int(meta['f_opt'][1])
            sys.stdout = open('trash', 'w')
            model = AgglomerativeClustering(n_clusters=clusters)
            matrixDat += co_association_matrix(model, data, days)
            model = Birch(threshold=0.01, n_clusters=clusters)
            matrixDat += co_association_matrix(model, data, days)
            model = KMeans(n_clusters=clusters)
            matrixDat += co_association_matrix(model, data, days)
            model = MiniBatchKMeans(n_clusters=clusters)
            matrixDat += co_association_matrix(model, data, days)
            model = SpectralClustering(n_clusters=clusters)
            matrixDat += co_association_matrix(model, data, days)
            model = GaussianMixture(n_components=clusters)
            matrixDat += co_association_matrix(model, data, days)
            matrixDat /= 6.0
            sys.stdout = save_stdout
        except:
            raise Warning("Couldn't convert string to int")   
    returnData = [{'panelId':meta['panelId']},{'dates':list(df['ActivityDate'].values)},{'data': list(matrixDat.flatten())}]
    return returnData            

def scatterPca_1_2(surveyFilePath, meta):
    df = pd.read_csv(surveyFilePath+meta['pid']+'.csv')
    df.replace({0: "False", 1: "True"}, inplace = True) #This line was added for MCA
    cols = [col for col in df if col.startswith(meta['f_opt'][0])]
    patient_df = df[cols + ['ActivityDate']]
    cat_df = patient_df[cols + ["ActivityDate"]]
    try:
        mca = MCA(n_components=2, engine='sklearn', random_state=67)
        X = cat_df[cols].to_numpy()
        X_pca = np.array(mca.fit_transform(X))
        pca_cols = pd.DataFrame(X_pca, columns=[meta['f_opt'][0] + '_pca_1', meta['f_opt'][0] + '_pca_2'])
        pca_df = pd.concat([pca_cols.reset_index(drop=True),patient_df[['ActivityDate']].reset_index(drop=True)], axis=1)
    except:
        #Not enough data available for Categorical PCA with 2 components
        return [{'panelId':meta['panelId']},{'category': meta['f_opt'][0]}, {'data': []}]
    
    returnData = []
    for idx, row in pca_df.iterrows():
        dic = {}
        for column in pca_df.columns:
            dic[column] = row[column]
        returnData.append(dic)
    return [{'panelId':meta['panelId']},{'category': meta['f_opt'][0]}, {'data': returnData},]

def eigGap(surveyFilePath, meta):
    df = pd.read_csv(surveyFilePath+meta['pid']+'.csv')
    df.replace({0: "False", 1: "True"}, inplace = True) #This line was added for MCA
    categories = ['mood', 'social', 'anxiety', 'psychosis', 'sleep']
    returnData = []
    for category in categories:
        eigenValueDict = {}
        cat_cols = [col for col in df if col.startswith(category)]
        cat_df = df[cat_cols] #Select columns of certain category
        mca = MCA(n_components=3, engine='sklearn', random_state=67) #Increase components to get more eigen values
        mca.fit_transform(cat_df.values)
        eigenValueDict['category'] = category

        if(meta['f_opt'][0] == 'firstEigenGap'): #Based on first or second eigen gaps return data
            eigenValueDict['eigenValues'] = mca.eigenvalues_[:2]
        else:
            eigenValueDict['eigenValues'] = mca.eigenvalues_[1:]
        returnData.append(eigenValueDict)
    if(meta['f_opt'][0] == 'firstEigenGap'): #Based on first or second eigen gaps return data
        eigValues = [1,2]
    else:
        eigValues = [2,3]
    return [{'panelId':meta['panelId'], 'eigValues':eigValues}, {'data': returnData},]