From 34d86c908f7146e15e61d21cc4c0b2660a4b8908 Mon Sep 17 00:00:00 2001 From: Marvin Thielk Date: Fri, 26 Apr 2019 10:33:06 -0700 Subject: [PATCH] format data --- notebooks/3.0-mjt-neuralTDA.ipynb | 206 ++++++++++++++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 notebooks/3.0-mjt-neuralTDA.ipynb diff --git a/notebooks/3.0-mjt-neuralTDA.ipynb b/notebooks/3.0-mjt-neuralTDA.ipynb new file mode 100644 index 0000000..7db72bc --- /dev/null +++ b/notebooks/3.0-mjt-neuralTDA.ipynb @@ -0,0 +1,206 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "ExecuteTime": { + "end_time": "2019-04-26T08:07:25.962574Z", + "start_time": "2019-04-26T08:07:24.044637Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/anaconda/envs/morphs36/lib/python3.6/site-packages/matplotlib/__init__.py:886: MatplotlibDeprecationWarning: \n", + "examples.directory is deprecated; in the future, examples will be found relative to the 'datapath' directory.\n", + " \"found relative to the 'datapath' directory.\".format(key))\n", + "/usr/local/anaconda/envs/morphs36/lib/python3.6/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n", + " \"This module will be removed in 0.20.\", DeprecationWarning)\n" + ] + } + ], + "source": [ + "import morphs\n", + "import numpy as np\n", + "import pickle" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "ExecuteTime": { + "end_time": "2019-04-26T08:07:25.969156Z", + "start_time": "2019-04-26T08:07:25.965418Z" + } + }, + "outputs": [], + "source": [ + "def rasterize(spike_times, num_bins, window_size=.01):\n", + " raster = np.zeros(num_bins)\n", + " for spike_time in spike_times:\n", + " raster[int(spike_time / window_size)] += 1\n", + " return raster" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "ExecuteTime": { + "end_time": "2019-04-26T09:22:59.514385Z", + "start_time": "2019-04-26T08:07:25.971988Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/mnt/cube/mthielk/analysis/B1101/kwik/Pen02_Lft_AP2501_ML500__Site03_Z2222__B1101_cat_P02_S03_1\n", + "/mnt/cube/mthielk/analysis/B1101/kwik/Pen02_Lft_AP2501_ML500__Site04_Z2410__B1101_cat_P02_S04_3\n", + "/mnt/cube/mthielk/analysis/B1101/kwik/Pen01_Lft_AP2500_ML750__Site04_Z2300__B1101_cat_P01_S04_3\n", + "/mnt/cube/mthielk/analysis/B1101/kwik/Pen01_Lft_AP2500_ML750__Site02_Z1880__B1101_cat_P01_S02_1\n", + "/mnt/cube/mthielk/analysis/B1101/kwik/Pen01_Lft_AP2500_ML750__Site03_Z2000__B1101_cat_P01_S03_2\n", + "/mnt/cube/mthielk/analysis/B1101/kwik/Pen02_Lft_AP2501_ML500__Site05_Z3072__B1101_cat_P02_S05_4\n", + "/mnt/cube/mthielk/analysis/B1101/kwik/Pen02_Lft_AP2501_ML500__Site02_Z2050__B1101_cat_P02_S02_1\n", + "/mnt/cube/mthielk/analysis/B1218/kwik/Pen01_Lft_AP2500_ML160__Site06_Z3368__B1218_cat_P01_S06_5\n", + "/mnt/cube/mthielk/analysis/B1218/kwik/Pen01_Lft_AP2500_ML160__Site05_Z2950__B1218_cat_P01_S05_3\n", + "/mnt/cube/mthielk/analysis/B1218/kwik/Pen01_Lft_AP2500_ML160__Site08_Z4323__B1218_cat_P01_S08_7\n", + "/mnt/cube/mthielk/analysis/B1218/kwik/Pen01_Lft_AP2500_ML160__Site06_Z3368__B1218_cat_P01_S06_4\n", + "/mnt/cube/mthielk/analysis/B1218/kwik/Pen01_Lft_AP2500_ML160__Site07_Z3721__B1218_cat_P01_S07_6\n", + "/mnt/cube/mthielk/analysis/B1218/kwik/Pen01_Lft_AP2500_ML160__Site02_Z2337__B1218_cat_P01_S02_1\n", + "/mnt/cube/mthielk/analysis/B1218/kwik/Pen01_Lft_AP2500_ML160__Site04_Z2583__B1218_cat_P01_S04_2\n", + "/mnt/cube/mthielk/analysis/B1134/kwik/Pen01_Lft_AP2500_ML500__Site02_Z3100__B1134_cat_P01_S02_1\n", + "/mnt/cube/mthielk/analysis/B1134/kwik/Pen01_Lft_AP2500_ML500__Site03_Z3200__B1134_cat_P01_S03_1\n", + "/mnt/cube/mthielk/analysis/B1088/kwik/Pen01_Lft_AP2500_ML120__Site04_Z2650__B1088_cat_P01_S04_3\n", + "/mnt/cube/mthielk/analysis/B1088/kwik/Pen01_Lft_AP2500_ML120__Site02_Z1950__B1088_cat_P01_S02_1\n", + "/mnt/cube/mthielk/analysis/B1088/kwik/Pen01_Lft_AP2500_ML120__Site03_Z2377__B1088_cat_P01_S03_2\n", + "/mnt/cube/mthielk/analysis/B1088/kwik/Pen01_Lft_AP2500_ML120__Site05_Z3000__B1088_cat_P01_S05_4\n", + "/mnt/cube/mthielk/analysis/B1096/kwik/Pen04_Lft_AP2500_ML50__Site02_Z2500__B1096_cat_P04_S02_1\n", + "/mnt/cube/mthielk/analysis/B1096/kwik/Pen04_Lft_AP2500_ML50__Site03_Z2850__B1096_cat_P04_S03_2\n", + "/mnt/cube/mthielk/analysis/B1096/kwik/Pen07_Rgt_AP2450_ML100__Site01_Z2500__B1096_cat_P07_S01_9\n", + "/mnt/cube/mthielk/analysis/B1096/kwik/Pen04_Lft_AP2500_ML50__Site04_Z3400__B1096_cat_P04_S04_3\n", + "/mnt/cube/mthielk/analysis/B1096/kwik/Pen05_Rgt_AP2500_ML100__Site03_Z3000__B1096_cat_P05_S03_5\n", + "/mnt/cube/mthielk/analysis/B1096/kwik/Pen07_Rgt_AP2450_ML100__Site02_Z3000__B1096_cat_P07_S02_10\n", + "/mnt/cube/mthielk/analysis/B1096/kwik/Pen06_Rgt_AP2550_ML100__Site02_Z2500__B1096_cat_P06_S02_6\n", + "/mnt/cube/mthielk/analysis/B1096/kwik/Pen05_Rgt_AP2500_ML100__Site02_Z2500__B1096_cat_P05_S02_4\n", + "/mnt/cube/mthielk/analysis/B1096/kwik/Pen06_Rgt_AP2550_ML100__Site04_Z3000__B1096_cat_P06_S04_8\n", + "/mnt/cube/mthielk/analysis/B1096/kwik/Pen06_Rgt_AP2550_ML100__Site03_Z2720__B1096_cat_P06_S03_7\n", + "/mnt/cube/mthielk/analysis/B1229/kwik/Pen01_Rgt_AP2500_ML150__Site02_Z2100__B1229_cat_P01_S02_1\n", + "/mnt/cube/mthielk/analysis/B1229/kwik/Pen01_Rgt_AP2500_ML150__Site04_Z2915__B1229_cat_P01_S04_3\n", + "/mnt/cube/mthielk/analysis/B1229/kwik/Pen01_Rgt_AP2500_ML150__Site03_Z2500__B1229_cat_P01_S03_2\n", + "/mnt/cube/mthielk/analysis/B1229/kwik/Pen02_Rgt_AP2501_ML150__Site02_Z2400__B1229_cat_P02_S02_4\n", + "/mnt/cube/mthielk/analysis/B1082/kwik/Pen02_Lft_AP2510_ML200__Site03_Z2250__B1082_cat_P02_S03_3\n", + "/mnt/cube/mthielk/analysis/B1082/kwik/Pen02_Lft_AP2510_ML200__Site02_Z2000__B1082_cat_P02_S02_2\n", + "/mnt/cube/mthielk/analysis/B1082/kwik/Pen03_Lft_AP2590_ML200__Site01_Z2000__B1082_cat_P03_S01_7\n", + "/mnt/cube/mthielk/analysis/B1082/kwik/Pen02_Lft_AP2510_ML200__Site06_Z3000__B1082_cat_P02_S06_6\n", + "/mnt/cube/mthielk/analysis/B1082/kwik/Pen01_Lft_AP2500_ML200__Site02_Z2250__B1082_cat_P01_S02_1\n", + "/mnt/cube/mthielk/analysis/B1082/kwik/Pen02_Lft_AP2510_ML200__Site05_Z2700__B1082_cat_P02_S05_5\n", + "/mnt/cube/mthielk/analysis/B1082/kwik/Pen03_Lft_AP2590_ML200__Site02_Z2250__B1082_cat_P03_S02_8\n", + "/mnt/cube/mthielk/analysis/B1082/kwik/Pen02_Lft_AP2510_ML200__Site04_Z2450__B1082_cat_P02_S04_4\n", + "/mnt/cube/mthielk/analysis/B1183/kwik/Pen01_Lft_AP2500_ML500__Site04_Z2500__B1183_cat_P01_S04_3\n", + "/mnt/cube/mthielk/analysis/B1183/kwik/Pen01_Lft_AP2500_ML500__Site03_Z2250__B1183_cat_P01_S03_2\n", + "/mnt/cube/mthielk/analysis/B1183/kwik/Pen01_Lft_AP2500_ML500__Site06_Z3000__B1183_cat_P01_S06_5\n", + "/mnt/cube/mthielk/analysis/B1183/kwik/Pen01_Lft_AP2500_ML500__Site05_Z2750__B1183_cat_P01_S05_4\n", + "/mnt/cube/mthielk/analysis/B1183/kwik/Pen01_Lft_AP2500_ML500__Site02_Z1500__B1183_cat_P01_S02_1\n" + ] + } + ], + "source": [ + "window_size = .01 # seconds\n", + "stim_length = .4\n", + "num_bins = int(stim_length / window_size)\n", + "\n", + "pop_tensors = {}\n", + "labels = {}\n", + "for block_path in morphs.config.paths.blocks():\n", + " print(block_path)\n", + " pop_tensors[block_path] = {}\n", + " labels[block_path] = {}\n", + " spikes = morphs.load.ephys_data(block_path, collapse_endpoints=False)\n", + " spikes = spikes[\n", + " (spikes[\"stim_aligned_time\"] >= 0) & (spikes[\"stim_aligned_time\"] < stim_length)\n", + " ]\n", + " cluster_map = {cluster:i for i, cluster in enumerate(np.sort(spikes['cluster'].unique()))}\n", + " num_clusters = len(cluster_map)\n", + " for morph_dim, morph_dim_group in spikes.groupby(\"morph_dim\"):\n", + " trial_gb = morph_dim_group.groupby([\"morph_pos\", \"stim_presentation\"])\n", + " num_trials = len(trial_gb)\n", + " pop_tensor = np.zeros((num_clusters, num_bins, num_trials))\n", + " pos_list = np.zeros(num_trials)\n", + " for i, ((morph_pos, stim_presentation), trial_block) in enumerate(trial_gb):\n", + " pos_list[i] = morph_pos\n", + " for cluster, cluster_group in trial_block.groupby(\"cluster\"):\n", + " pop_tensor[cluster_map[cluster], :, i] = rasterize(\n", + " cluster_group[\"stim_aligned_time\"].values,\n", + " num_bins,\n", + " window_size=window_size,\n", + " )\n", + " pop_tensors[block_path][morph_dim] = pop_tensor" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "ExecuteTime": { + "end_time": "2019-04-26T09:23:20.373838Z", + "start_time": "2019-04-26T09:22:59.516474Z" + } + }, + "outputs": [], + "source": [ + "savepath = morphs.paths.PROCESSED_DIR / \"TDA.pkl\"\n", + "with open(savepath.as_posix(), \"wb\") as f:\n", + " pickle.dump({\"pop_tensors\": pop_tensors, \"labels\": labels}, f)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "ExecuteTime": { + "end_time": "2019-04-26T09:23:20.380817Z", + "start_time": "2019-04-26T09:23:20.376063Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/usr/local/home/mthielk/GitHub/morphs/data/processed/TDA.pkl\n" + ] + } + ], + "source": [ + "print(savepath.as_posix())" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python3.6 (morphs)", + "language": "python", + "name": "py36" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.7" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}