diff --git a/LASSO_REGRESSION.md b/LASSO_REGRESSION.md new file mode 100644 index 0000000..214d50b --- /dev/null +++ b/LASSO_REGRESSION.md @@ -0,0 +1,23 @@ +Implementing GZip Lasso Regression (with kNN=0) shows worse performance then using normal base regression with kNN. +The alpha used was 0.03. I used GridSearch to find the best alpha. However, I only did it +for freesolv in this iteration. + +# Results Gzip-based Molecular Lasso-Regression +|Data Set|Split |AUROC/RMSE (Valid)|F1/MAE (Valid) |AUROC/RMSE (Test)| F1/MAE (Test) | +|--------|------|------------------|---------------|-----------------|---------------| +|freesolv|random|0.751 +/- 0.174 |0.341 +/- 0.174|0.653 +/- 0.174 |0.367 +/- 0.174| +|delaney |random|1.562 +/- 0.051 |1.236 +/- 0.051|1.693 +/- 0.051 |1.337 +/- 0.051| +|lipo |random|1.105 +/- 0.035 |0.925 +/- 0.035|1.134 +/- 0.035 |0.954 +/- 0.035| + +Also, tried adding in categorical information from what unique smiles x1 and x2 came from to form the compressed +sequence. It definitely hurt the model. Still wanted to try it out. +freesolv alpha = 0.02 +delaney alpha = 0.0 (Lasso doesn't converge well with an alpha of 0.0) +lip alpha = 0.01 +# Results Gzip-based Molecular Lasso-Regression w/ OneHot +|Data Set|Split |AUROC/RMSE (Valid)|F1/MAE (Valid) |AUROC/RMSE (Test)| F1/MAE (Test) | +|--------|------|------------------|---------------|-----------------|---------------| +|freesolv|random|0.626 +/- 0.076 |0.331 +/- 0.076|0.674 +/- 0.076 |0.328 +/- 0.076| +|delaney |random|1.705 +/- 0.094 |1.33 +/- 0.094 |1.706 +/- 0.094 |1.334 +/- 0.094| +|lipo |random|1.132 +/- 0.022 |0.949 +/- 0.022|1.124 +/- 0.022 |0.93 +/- 0.022| + diff --git a/gzip_compression_test/RMSE_TEST_CLASSIFICATION.png b/gzip_compression_test/RMSE_TEST_CLASSIFICATION.png new file mode 100644 index 0000000..8a75b50 Binary files /dev/null and b/gzip_compression_test/RMSE_TEST_CLASSIFICATION.png differ diff --git a/gzip_compression_test/RMSE_VALID_CLASSIFICATION.png b/gzip_compression_test/RMSE_VALID_CLASSIFICATION.png new file mode 100644 index 0000000..9efd569 Binary files /dev/null and b/gzip_compression_test/RMSE_VALID_CLASSIFICATION.png differ diff --git a/gzip_compression_test/total_results.csv b/gzip_compression_test/total_results.csv new file mode 100644 index 0000000..127a0c3 --- /dev/null +++ b/gzip_compression_test/total_results.csv @@ -0,0 +1,81 @@ +Compression,DataSet,Split,RMSE_Valid,RMSE_STD_VALID,MAE_Valid,MAE_STD_VALID,RMSE_Test,RMSE_STD_TEST,MAE_Test,MAE_STD_TEST, +0,hiv,random,0.493,0.015,0.835,0.015,0.499,0.015,0.832,0.015, +0,sider,random,0.507,0.011,0.52,0.011,0.396,0.011,0.514,0.011, +0,bbbp,scaffold,0.313,0.0,0.338,0.0,0.447,0.0,0.451,0.0, +0,bace_classification,random,0.518,0.008,0.555,0.008,0.501,0.008,0.516,0.008, +0,clintox,random,0.539,0.067,0.792,0.067,0.504,0.067,0.8,0.067, +0,tox21,random,0.519,0.013,0.129,0.013,0.517,0.013,0.129,0.013, +0,muv,random,0.375,0.216,0.0,0.216,0.125,0.216,0.0,0.216, +0,schneider,random,0.0,0.0,0.144,0.0,0.0,0.0,0.144,0.0, +1,hiv,random,0.753,0.013,0.9,0.013,0.753,0.013,0.897,0.013, +1,sider,random,0.442,0.256,0.648,0.256,0.591,0.256,0.658,0.256, +1,bbbp,scaffold,0.916,0.0,0.917,0.0,0.629,0.0,0.618,0.0, +1,bace_classification,random,0.826,0.019,0.828,0.019,0.795,0.019,0.794,0.019, +1,clintox,random,0.913,0.017,0.908,0.017,0.895,0.017,0.915,0.017, +1,tox21,random,0.685,0.003,0.289,0.003,0.691,0.003,0.278,0.003, +1,muv,random,0.407,0.235,0.03,0.235,0.415,0.235,0.039,0.235, +1,schneider,random,0.0,0.0,0.795,0.0,0.0,0.0,0.795,0.0, +2,hiv,random,0.751,0.008,0.896,0.008,0.754,0.008,0.898,0.008, +2,sider,random,0.587,0.015,0.652,0.015,0.595,0.015,0.66,0.015, +2,bbbp,scaffold,0.925,0.0,0.926,0.0,0.65,0.0,0.637,0.0, +2,bace_classification,random,0.804,0.031,0.798,0.031,0.815,0.031,0.816,0.031, +2,clintox,random,0.923,0.03,0.908,0.03,0.884,0.03,0.893,0.03, +2,tox21,random,0.702,0.02,0.3,0.02,0.687,0.02,0.301,0.02, +2,muv,random,0.131,0.228,0.041,0.228,0.28,0.228,0.038,0.228, +2,schneider,random,0.0,0.0,0.798,0.0,0.0,0.0,0.798,0.0, +3,hiv,random,0.761,0.015,0.896,0.015,0.762,0.015,0.897,0.015, +3,sider,random,0.448,0.259,0.661,0.259,0.59,0.259,0.648,0.259, +3,bbbp,scaffold,0.914,0.0,0.917,0.0,0.644,0.0,0.632,0.0, +3,bace_classification,random,0.833,0.028,0.828,0.028,0.794,0.028,0.793,0.028, +3,clintox,random,0.881,0.041,0.893,0.041,0.878,0.041,0.909,0.041, +3,tox21,random,0.695,0.015,0.293,0.015,0.693,0.015,0.294,0.015, +3,muv,random,0.269,0.269,0.027,0.269,0.261,0.269,0.034,0.269, +3,schneider,random,0.0,0.0,0.797,0.0,0.0,0.0,0.797,0.0, +4,hiv,random,0.748,0.018,0.898,0.018,0.762,0.018,0.896,0.018, +4,sider,random,0.583,0.011,0.641,0.011,0.578,0.011,0.646,0.011, +4,bbbp,scaffold,0.931,0.0,0.931,0.0,0.648,0.0,0.637,0.0, +4,bace_classification,random,0.783,0.023,0.785,0.023,0.797,0.023,0.794,0.023, +4,clintox,random,0.899,0.026,0.9,0.026,0.943,0.026,0.918,0.026, +4,tox21,random,0.698,0.015,0.301,0.015,0.686,0.015,0.287,0.015, +4,muv,random,0.279,0.279,0.044,0.279,0.271,0.279,0.029,0.279, +4,schneider,random,0.0,0.0,0.798,0.0,0.0,0.0,0.798,0.0, +5,hiv,random,0.734,0.023,0.894,0.023,0.761,0.023,0.898,0.023, +5,sider,random,0.606,0.006,0.659,0.006,0.591,0.006,0.653,0.006, +5,bbbp,scaffold,0.931,0.0,0.931,0.0,0.634,0.0,0.623,0.0, +5,bace_classification,random,0.809,0.011,0.808,0.011,0.808,0.011,0.806,0.011, +5,clintox,random,0.93,0.008,0.916,0.008,0.856,0.008,0.892,0.008, +5,tox21,random,0.69,0.011,0.296,0.011,0.693,0.011,0.289,0.011, +5,muv,random,0.284,0.284,0.046,0.284,0.278,0.284,0.049,0.284, +5,schneider,random,0.0,0.0,0.802,0.0,0.0,0.0,0.802,0.0, +6,hiv,random,0.747,0.016,0.899,0.016,0.743,0.016,0.899,0.016, +6,sider,random,0.296,0.297,0.658,0.297,0.433,0.297,0.648,0.297, +6,bbbp,scaffold,0.931,0.0,0.931,0.0,0.634,0.0,0.623,0.0, +6,bace_classification,random,0.797,0.018,0.798,0.018,0.799,0.018,0.798,0.018, +6,clintox,random,0.925,0.047,0.926,0.047,0.908,0.047,0.895,0.047, +6,tox21,random,0.7,0.013,0.292,0.013,0.701,0.013,0.305,0.013, +6,muv,random,0.139,0.241,0.037,0.241,0.283,0.241,0.052,0.241, +6,schneider,random,0.0,0.0,0.802,0.0,0.0,0.0,0.802,0.0, +7,hiv,random,0.754,0.015,0.895,0.015,0.729,0.015,0.897,0.015, +7,sider,random,0.583,0.009,0.649,0.009,0.598,0.009,0.646,0.009, +7,bbbp,scaffold,0.931,0.0,0.931,0.0,0.634,0.0,0.623,0.0, +7,bace_classification,random,0.792,0.006,0.79,0.006,0.817,0.006,0.814,0.006, +7,clintox,random,0.938,0.033,0.923,0.033,0.894,0.033,0.911,0.033, +7,tox21,random,0.705,0.014,0.295,0.014,0.693,0.014,0.287,0.014, +7,muv,random,0.156,0.27,0.031,0.27,0.547,0.27,0.035,0.27, +7,schneider,random,0.0,0.0,0.801,0.0,0.0,0.0,0.801,0.0, +8,hiv,random,0.765,0.008,0.899,0.008,0.748,0.008,0.896,0.008, +8,sider,random,0.592,0.016,0.644,0.016,0.602,0.016,0.656,0.016, +8,bbbp,scaffold,0.931,0.0,0.931,0.0,0.634,0.0,0.623,0.0, +8,bace_classification,random,0.798,0.037,0.798,0.037,0.799,0.037,0.798,0.037, +8,clintox,random,0.888,0.025,0.899,0.025,0.904,0.025,0.919,0.025, +8,tox21,random,0.685,0.01,0.281,0.01,0.704,0.01,0.304,0.01, +8,muv,random,0.146,0.252,0.035,0.252,0.411,0.252,0.032,0.252, +8,schneider,random,0.0,0.0,0.801,0.0,0.0,0.0,0.801,0.0, +9,hiv,random,0.75,0.014,0.898,0.014,0.763,0.014,0.898,0.014, +9,sider,random,0.446,0.258,0.663,0.258,0.589,0.258,0.643,0.258, +9,bbbp,scaffold,0.931,0.0,0.931,0.0,0.634,0.0,0.623,0.0, +9,bace_classification,random,0.788,0.045,0.788,0.045,0.828,0.045,0.826,0.045, +9,clintox,random,0.904,0.032,0.92,0.032,0.916,0.032,0.892,0.032, +9,tox21,random,0.699,0.004,0.288,0.004,0.698,0.004,0.305,0.004, +9,muv,random,0.128,0.221,0.038,0.221,0.138,0.221,0.036,0.221, +9,schneider,random,0.0,0.0,0.801,0.0,0.0,0.0,0.801,0.0, diff --git a/gzip_lasso_regressor.py b/gzip_lasso_regressor.py new file mode 100644 index 0000000..f7b777e --- /dev/null +++ b/gzip_lasso_regressor.py @@ -0,0 +1,261 @@ +import gzip +import multiprocessing +from typing import Iterable, Tuple +from collections import Counter +from functools import partial +import numpy as np +from tqdm import tqdm +from scipy.linalg import solve +from sklearn.metrics.pairwise import pairwise_distances +from sklearn.linear_model import Lasso +from sklearn.model_selection import GridSearchCV +from sklearn.model_selection import RepeatedKFold + +def genonehot_smiles(smiles: list, onehot_dict: dict=dict()) -> dict: + """Convert smiles into a categorical onehot dictionary. + The model needs to be able to track what it is looking + at not only NCD + + PARAMETERS + ---------- + smiles : List[str] + List containing smiles. + + onehot_dict : dict (default is dict()) + This is either an empty dict or one that needs to be added + to. The keys are the smiles and the values are its column + in the onehot matrix. + + RETURNS + ------- + onehot_dict : dict + The keys are the smiles and the values are its columns + in the onehot matrix. + """ + # determine if empty dict or not + if onehot_dict == {}: + # Pull out unique smiles + unique_smiles = set(smiles) + # make a numpy array that goes from 0 to len + Nunique_smiles = np.arange(len(unique_smiles)) + onehot_dict = dict(zip( + unique_smiles, + Nunique_smiles + )) + else: + # pull out old smiles + key_smiles = list(onehot_dict.keys()) + # combine the list + key_smiles.extend(smiles) + # Pull out unique smiles + unique_smiles = set(key_smiles) + # make a numpy array that goes from 0 to len + Nunique_smiles = np.arange(len(unique_smiles)) + onehot_dict = dict( + zip( + unique_smiles, + Nunique_smiles + ) + ) + return onehot_dict + + +def regress_lasso_traindata_(x1: str, + X_train: np.array, + k: int, + onehot_dict:dict) -> Tuple[np.array, np.array]: + """Generate a byte compression matrix from x1 based upon + passed X_train input concats + + PARAMETERS + ---------- + x1 : str + Input smiles molecule string + + X_train : tuple | list (iterable) + list or tuple of smiles moleclue strings + + y_train : list (iterable) + y labels for each element in X_train + + k: int + Number of Neighbors to consider in k-NN + + RETURNS + ------- + distance_from_x1 : np.array + NCD values associated with each Cx1x2 pair + + k_pairs : np.array + An Array of size (K, 2). The top k Cx1x2 distances + in kNN, but the indices + """ + # Compress encoded (utf-8) x1 passed in to fun + # take length + Cx1 = len(gzip.compress(x1.encode())) + distance_from_x1 = [] + smile_list = [] + + # Loop through x2 to generate + # normalized compression distances from Cx1x2 based + # upon Cx1 and Cx2. We normalize by longest compression + # sequence + for x2 in X_train: + Cx2 = len(gzip.compress(x2.encode())) + x1x2 = " ".join([x1, x2]) + Cx1x2 = len(gzip.compress(x1x2.encode())) + ncd = (Cx1x2 - min(Cx1, Cx2)) / max(Cx1, Cx2) + distance_from_x1.append(ncd) + smile_list.append(x2) + + # Return all of the distances from x1 + distance_from_x1 = np.array(distance_from_x1) + + # if kNN is set then apply neighbors only + if k != 0: + # Sort distances and pull out top k neighbors + sorted_idx = np.argsort(distance_from_x1) + distance_from_x1 = distance_from_x1[sorted_idx[:k]] + top_k_distance = [smile_list[i] for i in sorted_idx[:k]] + # pull out top 5 indexes per row + # k_pairs = sorted_idx[:5] + vectorize_keys = np.vectorize(lambda key1, key2: (onehot_dict[key1], + onehot_dict[key2])) + k_pairs = vectorize_keys([x1]*k, top_k_distance) + k_pairs = np.vstack( + (k_pairs[0], k_pairs[1]), + ) + else: + k_pairs = np.array(0) + + return distance_from_x1, k_pairs.T + + +def regress_lasso(X_train, y_train, X_test, + k, onehot_dict) -> np.array: + """Perform Lasso based Regression on dataset to + train and predict + + PARAMETERS + ---------- + X_train : list | tuple (iterable) + Smiles train dataset + + y_train : list | tuple (iterable) + Particular y prediction values associated with + regression task specified + + X_test : list | tuple (iterable) + Smiles testing dataset + + k : int + Determine number of neighbors to consider in kNN + RETURNS + ------- + preds : np.array + Array of predicted y values for the X_test dataset + """ + # dataset_train = [] + # preds = [] + # k_pairs = [] + + cpu_count = multiprocessing.cpu_count() + + with multiprocessing.Pool(cpu_count) as p: + train_tuple = p.map( + partial( + regress_lasso_traindata_, + X_train=X_train, + k=k, + onehot_dict=onehot_dict, + ), + X_train, + ) + dataset_train, k_pairs_train = zip(*train_tuple) + + with multiprocessing.Pool(cpu_count) as p: + test_tuple = p.map( + partial( + regress_lasso_traindata_, + X_train=X_train, + k=k, + onehot_dict=onehot_dict + ), + X_test + ) + dataset_test, k_pairs_test = zip(*test_tuple) + + + # Convert both datasets to np.arrays for fitting + dataset_train = np.array(dataset_train) + dataset_test = np.array(dataset_test) + + # grab length + train_len = dataset_train.shape[0] + test_len = dataset_test.shape[0] + + # Full dataset + full_dataset = np.concatenate(( + dataset_train, + dataset_test, + )) + + # Number of pairs in data + pairs = np.concatenate(( + np.concatenate(k_pairs_train), + np.concatenate(k_pairs_test) + )) + + # encode string pairs into categorical data + onehot_mat = np.zeros((pairs.shape[0], len(onehot_dict))) + onehot_mat[np.arange(pairs.shape[0]), pairs[:, 0]] = 1. + onehot_mat[np.arange(pairs.shape[0]), pairs[:, 1]] = 1. + + # combine distances and onehote pairs + X_full = np.concatenate( + (onehot_mat, full_dataset.flatten().reshape(-1,1)), + axis=1 + ) + + if k != 0: + train_X = X_full[:train_len*k,:] + test_X = X_full[train_len*k:,:] + # reformat y to fit train_X + y_reshape = np.repeat(y_train, k, axis=0) + else: + train_X = X_full[:train_len*train_len,:] + test_X = X_full[train_len*train_len:,:] + # reformat y to fit train_X + y_reshape = np.repeat(y_train, train_len, axis=0) + + gridsearch_alpha(train_X, y_reshape) + + # Init lasso model + # alpha generated from GridSearchCV + model = Lasso(alpha=0.01) + + # fit and predict values + model.fit(train_X, y_reshape) + preds = model.predict(test_X) + + return np.array(preds) + +def gridsearch_alpha(X_train, y_train) -> int: + print('\n') + print("*********STARTING GRIDSEARCH***********") + model = Lasso() + + # define model evaluation method + cv = RepeatedKFold(n_splits=10, n_repeats=3, random_state=1) + # define grid + grid = dict() + grid['alpha'] = np.linspace(0, 1, 21) + # define search + search = GridSearchCV(model, grid, scoring='neg_mean_absolute_error', cv=cv, n_jobs=-1) + # perform the search + results = search.fit(X_train, y_train) + # summarize + print('MAE: %.3f' % results.best_score_) + print('Config: %s' % results.best_params_) + + return 0 diff --git a/gzip_regressor.py b/gzip_regressor.py index a17a9ca..2dbb49a 100644 --- a/gzip_regressor.py +++ b/gzip_regressor.py @@ -10,9 +10,38 @@ def regress_(x1, X_train, y_train, k): + """Generate a byte compression matrix from x1 based upon + passed X_train input concats + + PARAMETERS + ---------- + x1 : str + Input smiles molecule string + + X_train : tuple | list (iterable) + list or tuple of smiles moleclue strings + + y_train : list (iterable) + y labels for each element in X_train + + k : int + Number of neighbors to collect in K-NN + + + RETURNS + ------- + task_preds : list + + """ + # Compress encoded (utf-8) x1 passed in to fun + # take length Cx1 = len(gzip.compress(x1.encode())) distance_from_x1 = [] + # Loop through x2 to generate + # normalized compression distances from Cx1x2 based + # upon Cx1 and Cx2. We normalize by longest compression + # sequence for x2 in X_train: Cx2 = len(gzip.compress(x2.encode())) x1x2 = " ".join([x1, x2]) @@ -20,12 +49,16 @@ def regress_(x1, X_train, y_train, k): ncd = (Cx1x2 - min(Cx1, Cx2)) / max(Cx1, Cx2) distance_from_x1.append(ncd) + # Sort ncd indexes in ascending order based upon distance + # These are used to grab top K-NN for both values and dists distance_from_x1 = np.array(distance_from_x1) sorted_idx = np.argsort(distance_from_x1) top_k_values = y_train[sorted_idx[:k]] top_k_dists = distance_from_x1[sorted_idx[:k]] task_preds = [] + # Loop through both top_k_value.T and top_k_dists.T + # Scale the predictions by the total distance found within a sample for vals, dists in zip(np.array(top_k_values).T, np.array(top_k_dists).T): dists = 1 - dists task_preds.append(np.mean(vals * dists) / np.sum(dists)) @@ -34,6 +67,28 @@ def regress_(x1, X_train, y_train, k): def regress(X_train, y_train, X_test, k): + """Perform Regression based task on dataset + + PARAMETERS + ---------- + X_train : list | tuple (iterable) + Smiles train dataset + + y_train : list | tuple (iterable) + Particular y prediction values associated with + regression task specified + + X_test : list | tuple (iterable) + Smiles testing dataset + + k : int + Amount of neighbors to consider in K-NN + + RETURNS + ------- + preds : np.array + Array of predicted y values for the X_test dataset + """ preds = [] cpu_count = multiprocessing.cpu_count() @@ -53,6 +108,19 @@ def regress(X_train, y_train, X_test, k): def compute_pairwise_ncd(pair): + """Compute the normalized compression pairwise distance + between two smiles + + PARAMTERS + --------- + pair : tuple + Smiles to compare + + RETURNS + ------- + ncd : float + Normalized Compression Distance + """ x1, x2 = pair Cx1 = len(gzip.compress(x1.encode())) Cx2 = len(gzip.compress(x2.encode()))