From 1d7b3a804c8e4aa39c8248637e194bda3b6cc8b1 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Sat, 27 Aug 2016 23:51:29 -0700 Subject: [PATCH 1/6] Added function to save ArrayedImage to an OpenMSI HDF5 file --- omaat_lib.ipy | 84 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 80 insertions(+), 4 deletions(-) diff --git a/omaat_lib.ipy b/omaat_lib.ipy index f9e247a..8d0bc78 100644 --- a/omaat_lib.ipy +++ b/omaat_lib.ipy @@ -344,7 +344,7 @@ class ArrayedImage(object): location. If there are spots missing in the grid, you may want to set this. Default is 0. :return: no return value """ - + if self.xCenters is None: raise ValueError("You can't run the optimizer because there are no defined spots") @@ -586,6 +586,81 @@ class ArrayedImage(object): putWindowOnTop() plt.show() + def saveToOpenMSIFile(self, fileName=None): + """ + Save the data from this ArrayedImage to and OpenMSI HDF5 file + :param fileName: + :return: + """ + from omsi.analysis.generic import analysis_generic + from omsi.dataformat.omsi_file.main_file import omsi_file + import h5py + + if fileName is None: + from tempfile import NamedTemporaryFile + tempfile = NamedTemporaryFile(mode='a') + fileName = tempfile.name + tempfile.seek(0) + + + dt = datetime.datetime.now() + curr_ana = analysis_generic(name_key="openmsi_arrayed_analysis_results_{:d}-{:d}-{:d}_{:d}h{:d}.csv".format(dt.year,dt.month,dt.day,dt.hour,dt.minute)) + dtypes = curr_ana.get_default_dtypes() + groups = curr_ana.get_default_parameter_groups() + + curr_ana.real_analysis_type = 'omaat_lib.ArrayedImage' + #curr_ana.add_parameter(name='__analysis_function', + # help='The analysis function we want to execute', + # dtype=str) + #curr_ana['_analysis_function'] = cloudpickle.dumps(self) + curr_ana.data_names = ['spotLocations', 'spotList', 'xCenters', 'yCenters', 'baseImage', 'imStack'] + curr_ana['spotLocations'] = self.spotLocations + curr_ana['spotList'] = self.spotList + curr_ana['xCenters'] = self.xCenters + curr_ana['yCenters'] = self.yCenters + curr_ana['baseImage'] = self.baseImage + curr_ana['imStack'] = self.imStack + curr_ana.add_parameter(name='Nrows', + help='Number of rows', + dtype=dtypes['int'], + required=True, + default=12, + choices=None, + group=groups['settings'], + data=self.Nrows) + curr_ana.add_parameter(name='Ncolumns', + help='Number of columns', + dtype=dtypes['int'], + required=True, + default=12, + choices=None, + group=groups['settings'], + data=self.Ncolumns) + curr_ana.add_parameter(name='mz', + help='The m/z axis of the input data', + dtype=dtypes['ndarray'], + required=True, + group=groups['input'], + data=self.mz) + curr_ana.add_parameter(name='ions', + help='List of ions used', + dtype=dtypes['ndarray'], + required=True, + group=groups['input'], + data=self.ions) + curr_ana.add_parameter(name='originalSize', + help='Original size', + dtype=dtypes['ndarray'], + required=True, + group=groups['input'], + data=self.originalSize) + + outfile = omsi_file(h5py.File(fileName)) + exp = outfile.create_experiment(exp_identifier='OMAAT store') + exp.create_analysis(curr_ana) + outfile.flush() + return outfile + def writeResultTable(self,fileName="",spotList=None,minPixelIntensity=0,alphaRows=False): """ @@ -1036,7 +1111,8 @@ def login(username=""): r.raise_for_status() csrftoken = newOpenMSIsession.requests_session.cookies['csrftoken'] login_data = dict(username=arrayed_analysis_default_username, password=password, csrfmiddlewaretoken=csrftoken) - result = newOpenMSIsession.requests_session.post(authURL, data=login_data, headers=dict(Referer=authURL)).url[-5:] + tmp = newOpenMSIsession.requests_session.post(authURL, data=login_data, headers=dict(Referer=authURL)) + result = tmp.url[-5:] IPython.display.clear_output() if(result=="login"): print("Password for user \"" + arrayed_analysis_default_username + "\" was likely wrong, re-run this cell to try again") @@ -1273,10 +1349,10 @@ def barycentric_trapezoidial_interpolation(Nx,Ny,p,hexagonalOffset=0.5): # xi,yi = openmsi.barycentric_trapezoidial_interpolation(Nx,Ny,newCoords) # a.plot(xi,yi,'.',markersize=12) # plt.show() - + x_basis = np.linspace(0,1,Nx) y_basis = np.linspace(0,1,Ny) - + px = [[p[0,0], p[2,0]],[p[1,0], p[3,0]]] #these are the [2,2] x-coordinates py = [[p[0,1], p[2,1]],[p[1,1], p[3,1]]] #these are the [2,2] x-coordinates #fx = interpolate.interp2d([1,0], [1,0], px, kind='linear') From 9996c2fed836794c4054073f32ab3f839c278f19 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Sun, 28 Aug 2016 03:32:34 -0700 Subject: [PATCH 2/6] Updated save to OMSI and added function to upload OMAAT data to OpenMSI --- omaat_lib.ipy | 66 ++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 57 insertions(+), 9 deletions(-) diff --git a/omaat_lib.ipy b/omaat_lib.ipy index 8d0bc78..a9212ed 100644 --- a/omaat_lib.ipy +++ b/omaat_lib.ipy @@ -589,19 +589,15 @@ class ArrayedImage(object): def saveToOpenMSIFile(self, fileName=None): """ Save the data from this ArrayedImage to and OpenMSI HDF5 file - :param fileName: - :return: + :param fileName: Name of the output file + :return: Instance of omsi.dataformat.omsi_file.main_file.omsi_file of the generated output OpenMSI file """ from omsi.analysis.generic import analysis_generic from omsi.dataformat.omsi_file.main_file import omsi_file import h5py if fileName is None: - from tempfile import NamedTemporaryFile - tempfile = NamedTemporaryFile(mode='a') - fileName = tempfile.name - tempfile.seek(0) - + fileName = input("Output filename: ") dt = datetime.datetime.now() curr_ana = analysis_generic(name_key="openmsi_arrayed_analysis_results_{:d}-{:d}-{:d}_{:d}h{:d}.csv".format(dt.year,dt.month,dt.day,dt.hour,dt.minute)) @@ -613,13 +609,31 @@ class ArrayedImage(object): # help='The analysis function we want to execute', # dtype=str) #curr_ana['_analysis_function'] = cloudpickle.dumps(self) - curr_ana.data_names = ['spotLocations', 'spotList', 'xCenters', 'yCenters', 'baseImage', 'imStack'] + curr_ana.data_names = ['spotLocations', 'xCenters', 'yCenters', 'baseImage', 'imStack', 'spectraDF'] curr_ana['spotLocations'] = self.spotLocations - curr_ana['spotList'] = self.spotList + if isinstance(self.spotList, list): + for spot_index, spot in enumerate(self.spotList): + spot_name = 'spot_'+str(spot_index) + curr_ana.data_names.append(spot_name) + curr_ana[spot_name] = spot curr_ana['xCenters'] = self.xCenters curr_ana['yCenters'] = self.yCenters curr_ana['baseImage'] = self.baseImage curr_ana['imStack'] = self.imStack + curr_ana['spectraDF'] = self.spectra_df + if curr_ana['spectraDF'] is not None: + try: + curr_ana['spectraDF'] = curr_ana['spectraDF'].as_matrix() + except: + curr_ana['spectraDF'] = np.asarray(curr_ana['spectraDF']) + if curr_ana['spectraDF'].dtype == np.dtype('O'): + curr_ana['spectraDF'] = None + print("Unknown format for spectraDF. Ignore") + print(self.spectra_df) + for i in curr_ana.data_names: + print((i, curr_ana[i])) + + self.spectra_df=None curr_ana.add_parameter(name='Nrows', help='Number of rows', dtype=dtypes['int'], @@ -661,6 +675,40 @@ class ArrayedImage(object): outfile.flush() return outfile + def upload_to_openmsi(self, fileName=None, username=None, session=None, machine='cori'): + """ + Upload the given file to the OpenMSI website. The file will be generate + using saveToOpenMSIFile if it does not exist + :param fileName: The name of the file to be uploaded (and generated) + :param username: NERSC username + :param session: The request NERSC session with NEWT + :param machine: The machine at NERSC to which the file should be uploaded. (default='cori') + :return: + """ + from omsi.shared.omsi_web_helper import WebHelper + + if fileName is None: + fileName = input("Output filename: ") + + # Save this ArrayedImage to file if necessary + if not os.path.exists(fileName): + print("Saving Arrayed Image to file") + outfile = self.saveToOpenMSIFile(fileName=fileName) + outfile.close_file() + # Upload the file to nersc and register it with OpenMSIs + try: + print("Uploading file to NERSC and adding it to OpenMSI") + result, result2, added, temps = WebHelper.upload_file_to_nersc(filepath=fileName, + register=True, + session=session, + machine=machine) + except ValueError as e: + if e.message == "Authentication failed.": + print("Authentication failed. Please try again.") + return {} + else: + raise + print((result, result2, added, temps)) def writeResultTable(self,fileName="",spotList=None,minPixelIntensity=0,alphaRows=False): """ From f0fdd5b08be73d3b83ddaf53830c9d45445a01e0 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Sun, 28 Aug 2016 04:03:24 -0700 Subject: [PATCH 3/6] Removed extra print statements left from debugging --- omaat_lib.ipy | 2 -- 1 file changed, 2 deletions(-) diff --git a/omaat_lib.ipy b/omaat_lib.ipy index a9212ed..7c2fc94 100644 --- a/omaat_lib.ipy +++ b/omaat_lib.ipy @@ -630,8 +630,6 @@ class ArrayedImage(object): curr_ana['spectraDF'] = None print("Unknown format for spectraDF. Ignore") print(self.spectra_df) - for i in curr_ana.data_names: - print((i, curr_ana[i])) self.spectra_df=None curr_ana.add_parameter(name='Nrows', From 8203fd7faa5d8cadcbcef7268b04d5c17fdbe8d3 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Sun, 28 Aug 2016 04:03:53 -0700 Subject: [PATCH 4/6] Added code cells to illustrate how we can save and upload OMAAT results to OpenMSI --- omaat_notebook.ipynb | 98 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 88 insertions(+), 10 deletions(-) diff --git a/omaat_notebook.ipynb b/omaat_notebook.ipynb index 7bd0203..3407774 100644 --- a/omaat_notebook.ipynb +++ b/omaat_notebook.ipynb @@ -366,17 +366,46 @@ "source": [ "spectra_df=openMSIsession.getSpotSpectra(img,verbose=True) #Loads the spectra from the OpenMSI server\n", " #It's lot of data, so save the desulting dataframe\n", - " #so that you dont have to run this method repeatedly.\n", - " \n", - "A01_spectrum=spectra_df[\"O08\"] # get only the spectrum for the spot at location A01\n", + " #so that you dont have to run this method repeatedly." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "A01_spectrum=spectra_df[\"A01\"] # get only the spectrum for the spot at location A01\n", "A01_spectrum.plot() #plot the entire spectrum for spot A01\n", "plt.xlabel(\"m/z\")\n", "plt.ylabel(\"intensity\")\n", - "plt.show()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ "A01_spectrum[900:1950].plot() #plot only the m/z values between 900 and 1200\n", "plt.xlabel(\"m/z\")\n", "plt.ylabel(\"intensity\")\n", - "plt.show()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ "spectra_df.plot() # plot ALL the spectra that are loaded on top of each other.\n", " #Depending on how many spots are in your image, this can be\n", " #a LOT of data, if you don't have a good computer this might crash.\n", @@ -385,6 +414,39 @@ "plt.show()" ] }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": false + }, + "source": [ + "

Cell 13. Uploading results to OpenMSI (Python 2 only)

\n", + "\n", + "You can save the results from the spot analysis to an OpenMSI HDF5 file and upload the file to OpenMSI to easily share results with others.\n", + "\n", + "If you only want to save Arrayed Image to an OpenMSI HDF5 file then do:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# Save the ArrayedImage to an OpenMSI File\n", + "omsi_out_file = img.saveToOpenMSIFile() \n", + "# fileName : If not given than a dialog will be shown to ask for a filename to be used" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To upload our ArrayedImage to OpenMSI we simply call:" + ] + }, { "cell_type": "code", "execution_count": null, @@ -392,26 +454,42 @@ "collapsed": false }, "outputs": [], + "source": [ + "img.upload_to_openmsi()\n", + "# fileName: If we set the fileName than the existing file will be used otherwise we'll ask for a name\n", + "# username: We can set the username or we'll ask for it\n", + "# session: The NERSC NEWT session to be used (not the OpenMSI seesion). We'll create it if not given\n", + "# machine: The NERSC machine we should use for the upload, e.g., 'cori' or 'edison'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], "source": [] } ], "metadata": { + "anaconda-cloud": {}, "kernelspec": { - "display_name": "Python 3", + "display_name": "Python [conda root]", "language": "python", - "name": "python3" + "name": "conda-root-py" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 3 + "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.1" + "pygments_lexer": "ipython2", + "version": "2.7.11" } }, "nbformat": 4, From fa60043373543a3d4310aaef891db7965c1de4f9 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Tue, 20 Sep 2016 14:37:11 -0700 Subject: [PATCH 5/6] Store arrayedImage and spotSpectra as pickle. Move upload function to OpenMSISession. Added functions to download data from OpenMSI. Added function to restore arrayed image and spot spectra from file --- omaat_lib.ipy | 223 ++++++++++++++------ omaat_notebook.ipynb | 472 +++++++++++++++++++++++++++++++++++++++---- 2 files changed, 595 insertions(+), 100 deletions(-) diff --git a/omaat_lib.ipy b/omaat_lib.ipy index 7c2fc94..d4b51f9 100644 --- a/omaat_lib.ipy +++ b/omaat_lib.ipy @@ -110,7 +110,6 @@ class ArrayedImage(object): self.spotLocations=None self.Nrows=None self.Ncolumns=None - self.spectra_df=None def __str__(self): @@ -586,18 +585,19 @@ class ArrayedImage(object): putWindowOnTop() plt.show() - def saveToOpenMSIFile(self, fileName=None): + def saveToOpenMSIFile(self, filename=None, spotSpectra=None): """ Save the data from this ArrayedImage to and OpenMSI HDF5 file - :param fileName: Name of the output file + :param filename: Name of the output file :return: Instance of omsi.dataformat.omsi_file.main_file.omsi_file of the generated output OpenMSI file """ from omsi.analysis.generic import analysis_generic from omsi.dataformat.omsi_file.main_file import omsi_file import h5py + import pickle - if fileName is None: - fileName = input("Output filename: ") + if filename is None: + filename = input("Output filename: ") dt = datetime.datetime.now() curr_ana = analysis_generic(name_key="openmsi_arrayed_analysis_results_{:d}-{:d}-{:d}_{:d}h{:d}.csv".format(dt.year,dt.month,dt.day,dt.hour,dt.minute)) @@ -609,7 +609,7 @@ class ArrayedImage(object): # help='The analysis function we want to execute', # dtype=str) #curr_ana['_analysis_function'] = cloudpickle.dumps(self) - curr_ana.data_names = ['spotLocations', 'xCenters', 'yCenters', 'baseImage', 'imStack', 'spectraDF'] + curr_ana.data_names = ['spotLocations', 'xCenters', 'yCenters', 'baseImage', 'imStack', 'spectraDF', 'arrayedImageP', 'spotSpectraP', 'spotSpectra'] curr_ana['spotLocations'] = self.spotLocations if isinstance(self.spotList, list): for spot_index, spot in enumerate(self.spotList): @@ -620,18 +620,23 @@ class ArrayedImage(object): curr_ana['yCenters'] = self.yCenters curr_ana['baseImage'] = self.baseImage curr_ana['imStack'] = self.imStack - curr_ana['spectraDF'] = self.spectra_df - if curr_ana['spectraDF'] is not None: - try: - curr_ana['spectraDF'] = curr_ana['spectraDF'].as_matrix() - except: - curr_ana['spectraDF'] = np.asarray(curr_ana['spectraDF']) - if curr_ana['spectraDF'].dtype == np.dtype('O'): - curr_ana['spectraDF'] = None - print("Unknown format for spectraDF. Ignore") - print(self.spectra_df) - - self.spectra_df=None + try: + curr_ana['spectraDF'] = self.spectra_df + except AttributeError: + curr_ana['spectraDF'] = None + curr_ana['arrayedImageP'] = pickle.dumps(self) + if curr_ana['spectraDF'] is None: + curr_ana['spectraDF'] = self.resultsDataFrame() + if spotSpectra is not None: + if isinstance(spotSpectra, pd.DataFrame): + curr_ana['spotSpectra'] = spotSpectra.as_matrix() + curr_ana['spotSpectraP'] = pickle.dumps(spotSpectra) + else: + curr_ana['spotSpectra'] = np.asarray(spotSpectra) + if curr_ana['spotSpectra'].dtype == np.dtype('O'): + curr_ana.pop('spotSpectra') + raise ValueError("Unsupported format for spotSpectra. Conversion to numpy resulted in type 'O'") + curr_ana.add_parameter(name='Nrows', help='Number of rows', dtype=dtypes['int'], @@ -666,58 +671,41 @@ class ArrayedImage(object): required=True, group=groups['input'], data=self.originalSize) + curr_ana.add_parameter(name='filename', + help='The name of the input file', + dtype=dtypes['unicode'], + required=True, + group=groups['input'], + data=self.filename) + curr_ana.add_parameter(name='expIndex', + help='The index of the experiment in the input file', + dtype=dtypes['int'], + required=True, + group=groups['input'], + data=self.expIndex) + curr_ana.add_parameter(name='dataIndex', + help='The index of the dataset in the input file', + dtype=dtypes['int'], + required=True, + group=groups['input'], + data=self.dataIndex) - outfile = omsi_file(h5py.File(fileName)) + outfile = omsi_file(h5py.File(filename)) exp = outfile.create_experiment(exp_identifier='OMAAT store') exp.create_analysis(curr_ana) outfile.flush() return outfile - def upload_to_openmsi(self, fileName=None, username=None, session=None, machine='cori'): - """ - Upload the given file to the OpenMSI website. The file will be generate - using saveToOpenMSIFile if it does not exist - :param fileName: The name of the file to be uploaded (and generated) - :param username: NERSC username - :param session: The request NERSC session with NEWT - :param machine: The machine at NERSC to which the file should be uploaded. (default='cori') - :return: + def writeResultTable(self,filename="",spotList=None,minPixelIntensity=0,alphaRows=False): """ - from omsi.shared.omsi_web_helper import WebHelper - if fileName is None: - fileName = input("Output filename: ") - - # Save this ArrayedImage to file if necessary - if not os.path.exists(fileName): - print("Saving Arrayed Image to file") - outfile = self.saveToOpenMSIFile(fileName=fileName) - outfile.close_file() - # Upload the file to nersc and register it with OpenMSIs - try: - print("Uploading file to NERSC and adding it to OpenMSI") - result, result2, added, temps = WebHelper.upload_file_to_nersc(filepath=fileName, - register=True, - session=session, - machine=machine) - except ValueError as e: - if e.message == "Authentication failed.": - print("Authentication failed. Please try again.") - return {} - else: - raise - print((result, result2, added, temps)) - - def writeResultTable(self,fileName="",spotList=None,minPixelIntensity=0,alphaRows=False): - """ - - :param fileName: filename to write to. will automatically be appended with a .csv extension. + :param filename: filename to write to. will automatically be appended with a .csv extension. Default is will use current date and time :return: """ - if fileName: - actualFileName="{}.csv".format(fileName) + if filename: + actualFileName="{}.csv".format(filename) else: dt = datetime.datetime.now() actualFileName="openmsi_arrayed_analysis_results_{:d}-{:d}-{:d}_{:d}h{:d}.csv".format(dt.year,dt.month,dt.day,dt.hour,dt.minute) @@ -866,8 +854,6 @@ class OpenMSIsession(object): return list(fileList.keys()) - - def fileSelector(self): """ @@ -1074,7 +1060,7 @@ class OpenMSIsession(object): If True, sets the column names of the data frame to strings with an alphabetical identifier. alphaRows=False sets the column names to 2-tuples (row,column). Default is True, ignored if spotList is defined. - :param verbose: If Ture, ouput which spot's spectrum just finished the loading process + :param verbose: If True, ouput which spot's spectrum just finished the loading process :return: A dataframe with intensities at various m/z values. Row indexes are m/z values and columns correspond to different spots (Be aware this is different from how the resultsDataFrame is laid out!!) @@ -1128,8 +1114,123 @@ class OpenMSIsession(object): #dataframe[i]= #df['coords'].append(coords) #df['spectra'].append(data) + return dataframe + def restore_omaat_results(self, filename, expIndex=0, anaIndex=0, localFile=False): + """ + Download the results from a previous OMAAT analysis and restore them + + :param filename: Name of the data file + :param expIndex: The index of the experiment with the results + :param anaIndex: The index of the analysis with the omaat results + :param infile: The input HDF5 file if available. If not available than the function will + download the data it needs from OpenMSI + + :return: Tuple with: + * Instance of ArrayedImage with all results + * Pandas dataframe with the spot spectra if available (or None) + + """ + import h5py + import pickle + spotSpectra = None + arrayedImage = None + if filename is None: + filename = self.filename + if localFile: + infile = h5py.File(filename, 'r') + exp = infile['entry_'+str(expIndex)] + ana = exp['analysis_'+str(anaIndex)] + + if 'spotSpectraP' in ana.keys(): + spotSpectra = pickle.loads(ana['spotSpectraP'][()]) + if 'arrayedImageP' in ana.keys(): + arrayedImage = pickle.loads(ana['arrayedImageP'][()]) + else: + payload = {'format': 'JSON', + 'file': filename, + 'expIndex':expIndex, + 'anaIndex': anaIndex, + 'anaDataName': 'spotSpectraP'} + url = 'https://openmsi.nersc.gov/openmsi/qcube' + r = self.requests_session.get(url, params=payload, stream=True) + r.raise_for_status() + data = np.asarray(json.loads(r.content.decode('utf-8'))) + spotSpectra = pickle.loads(data[0]) + + payload['anaDataName'] = 'arrayedImageP' + r = self.requests_session.get(url, params=payload, stream=True) + r.raise_for_status() + data = np.asarray(json.loads(r.content.decode('utf-8'))) + arrayedImage = pickle.loads(data[0]) + + return spotSpectra, arrayedImage + + + def download_file(self, filename, saveFilename=None): + """ + Download the indicated file from OpenMSI + + :param filename: The name of the file to be downloaded + :param saveFilename: The name of the file saved on disk + :return: Name of the file saved to disk + """ + saveFilename = saveFilename if saveFilename is not None else os.path.basename(filename) + payload = {'format': 'HDF5', 'file': filename} + url = 'https://openmsi.nersc.gov/openmsi/qcube' + r = self.requests_session.get(url, params=payload, stream=True) + with open(saveFilename, 'wb') as outfile: + for chunk in r.iter_content(chunk_size=1024): + if chunk: # filter out keep-alive new chunks + outfile.write(chunk) + return saveFilename + + + def upload_omaat_results(self, filename=None, session=None, machine='cori'): + """ + Upload the given file to the OpenMSI website. The file will be generate + using saveToOpenMSIFile if it does not exist + :param filename: The name of the file to be uploaded. A dialog will be shown if set to None. + :param username: NERSC username + :param session: The request NERSC session with NEWT + :param machine: The machine at NERSC to which the file should be uploaded. (default='cori') + :return: Tuple with the: + * boolean indicating whether the file upload was successful, + * boolean indicating whether the setting of file permissions was successful + * boolean indicating whether the registration with OpenMSI was successful + * the NEWT session (or None if persist_session is False) + """ + from omsi.shared.omsi_web_helper import WebHelper + from omsi.dataformat.omsi_file.main_file import omsi_file + + # Determine the filename + if filename is None: + filename = input("Output filename: ") + + # Check that the file is valid + if not omsi_file.is_valid_dataset(filename): + if os.path.exists(filename): + raise ValueError("The file is not a valid OpenMSI file") + else: + raise ValueError("The file does not seem to exsit.") + + # Upload the file to nersc and register it with OpenMSIs + try: + upload_successful, permissions_successfull, register_successful, temps = \ + WebHelper.upload_file_to_nersc(filepath=filename, + username=self.username, + register=True, + session=session, + persist_session=(session is not None), + machine=machine) + except ValueError as e: + if e.message == "Authentication failed.": + print("Authentication failed. Please try again.") + return {} + else: + raise + return upload_successful, permissions_successfull, register_successful, temps def login(username=""): """ diff --git a/omaat_notebook.ipynb b/omaat_notebook.ipynb index 3407774..6b6d6c5 100644 --- a/omaat_notebook.ipynb +++ b/omaat_notebook.ipynb @@ -47,11 +47,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": { "collapsed": false }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Completed loading OpenMSI Arrayed Analysis Toolkit\n" + ] + } + ], "source": [ "#load the code. Since it's specialized ipython notebook code, use '%run' rather than 'import'\n", "%run omaat_lib.ipy" @@ -69,11 +77,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": { "collapsed": false }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Login appears to be successful!\n" + ] + } + ], "source": [ "#log into OpenMSI.nersc.gov\n", "openMSIsession = login()" @@ -97,11 +113,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": { "collapsed": false }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Stored 'arrayed_analysis_default_filename' (unicode)\n", + "Loading image...\n", + "loading ion 1 of 3. m/z = 1141.350000 +/- 0.500000\n", + "Time to load ion: 0.166079998016 seconds\n", + "loading ion 2 of 3. m/z = 1143.050000 +/- 0.500000\n", + "Time to load ion: 0.160965919495 seconds\n", + "loading ion 3 of 3. m/z = 1241.250000 +/- 0.500000\n", + "Time to load ion: 0.169579029083 seconds\n", + "Image has been loaded.\n", + "Image has been saved in the global 'img' variable.\n" + ] + } + ], "source": [ "if \"openMSIsession\" not in locals():\n", " openMSIsession=OpenMSIsession()\n", @@ -120,11 +153,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": { "collapsed": false }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The window should be open now. If you can't see it, check to see if it's behind another window\n" + ] + } + ], "source": [ "color_map=\"hot_r\" #want to try a different color map? change it here, run the cell, and all functions below will use it\n", "marker_color=\"blue\"\n", @@ -148,11 +189,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": { "collapsed": false }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of columns? leave blank for default (\"2\") \n", + "Number of rows? leave blank for default (\"2\") \n", + "Hexagonal Offset? This shifts every other line by this many spots. leave blank for default (\"0.000000\") \n", + "Stored 'arrayed_analysis_columns' (int)\n", + "Stored 'arrayed_analysis_rows' (int)\n", + "Stored 'arrayed_analysis_offset' (float)\n", + "The window should be open now. If you can't see it, check to see if it's behind another window\n", + "new spot x and y locations have been saved.\n" + ] + } + ], "source": [ "#define spot centers as a trapezoid.\n", "#if you want to pass rows/columns as arguments, or choose the colormap, use the img.roughPosition() method in stead.\n", @@ -163,7 +219,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "

Cell 6. Automatic spot optimization

\n", + "##

Cell 6. Automatic spot optimization

\n", "In the next cell, the Jupyter notebook will optimze the marker position. For detials on the optimization algorithm, see the method section in the [manuscript](link to paper). For this tutorial, perform the automatic spot optimization.\n", "
    \n", "
  • First, put in the integration radius for the individual markers in the mask. For the tutorial, put in x. \n", @@ -180,11 +236,24 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": { "collapsed": false }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Stored 'arrayed_analysis_radius' (float)\n", + "Stored 'arrayed_analysis_minScore' (float)\n", + "done with optimization round 1 of 3\n", + "done with optimization round 2 of 3\n", + "done with optimization round 3 of 3\n", + "optimization routine completed. new spot x and y positions saved.\n" + ] + } + ], "source": [ "#automagically optimize the spot centers to correspond to the actual spots on the image\n", "img.optimizeSpots_with_dialogs()" @@ -203,11 +272,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": { "collapsed": false }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The window should be open now. If you can't see it, check to see if it's behind another window\n", + "new spot x and y locations have been saved.\n" + ] + } + ], "source": [ "#check the positions of the spots and manually adjust them if need be\n", "radius=arrayed_analysis_radius if (\"arrayed_analysis_radius\" in locals()) else 2\n", @@ -225,11 +303,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": { "collapsed": false }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Done saving.\n" + ] + } + ], "source": [ "#Optional: Save the ArrayedImage into a pickle file.\n", "filename=\"filename.arrayed_img\"\n", @@ -249,11 +335,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": { "collapsed": false }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ArrayedImage based on 20120913_nimzyme.h5\n", + "Ions loaded: [1141.35, 1143.05, 1241.25]\n", + "# of spot locations defined: 4\n", + "# of spot pixel masks defined: None\n" + ] + } + ], "source": [ "#Optional: Load an ArrayedImage from a pickle file. This way you can work off-line\n", "filename=\"filename.arrayed_img\"\n", @@ -274,11 +371,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": { "collapsed": false }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4 spots generated. number of spots with N pixels:{11: 1, 13: 3}\n", + "The window should be open now. If you can't see it, check to see if it's behind another window\n" + ] + } + ], "source": [ "#You'll need to call this function. It returns a list of spots (where each spot is a list of pixels),\n", "#which is also stored inside the object.\n", @@ -300,17 +406,34 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": { "collapsed": false, "scrolled": true }, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "Click the link below to access the results file:
    openmsi_arrayed_analysis_results_2016-9-20_14h31.csv
    Open it in Microsoft Excel or your favorite data analysis program.
    \n", + " You should now be done using this tool, but if you decide you want to\n", + " further optimize your spot placement, make sure to re-run the\n", + " notebook cells to update your results.\n", + "

    Thank you for using the OpenMSI Arrayed Analysis tool!" + ], + "text/plain": [ + "/Users/oruebel/Devel/OpenMSI_Arrayed_Analysis_Tools/openmsi_arrayed_analysis_results_2016-9-20_14h31.csv" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "#Write results to a file\n", "#if you don't pass it an explicit spotList it will use the spot set stored in the ArrayedImage\n", "filename=\"\" #.csv extension will be automatically added\n", - "img.writeResultTable(fileName=filename,alphaRows=True)" + "img.writeResultTable(filename=filename,alphaRows=True)" ] }, { @@ -324,11 +447,168 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": { "collapsed": false }, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
    \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    ion1141.351143.051241.25
    descriptorsummeanmedianminmaxnum_pixelssummeanmedianminmaxnum_pixelssummeanmedianminmaxnum_pixels
    A0135893.03263.0000003349.0321.05566.011.04522.0411.090909408.016.0724.011.0111574.010143.09090911977.01029.015032.011.0
    A0244428.03417.5384622990.098.08148.013.05664.0435.692308370.05.01029.013.093948.07226.7692318213.0315.013905.013.0
    B0121354.01642.6153851590.087.04008.013.02434.0187.230769179.05.0539.013.0136714.010516.4615387009.01023.022521.013.0
    B0214681.01129.307692855.010.03191.013.01564.0120.30769265.05.0419.013.0103822.07986.30769210098.027.016508.013.0
    \n", + "
    " + ], + "text/plain": [ + "ion 1141.35 1143.05 \\\n", + "descriptor sum mean median min max num_pixels sum \n", + "A01 35893.0 3263.000000 3349.0 321.0 5566.0 11.0 4522.0 \n", + "A02 44428.0 3417.538462 2990.0 98.0 8148.0 13.0 5664.0 \n", + "B01 21354.0 1642.615385 1590.0 87.0 4008.0 13.0 2434.0 \n", + "B02 14681.0 1129.307692 855.0 10.0 3191.0 13.0 1564.0 \n", + "\n", + "ion 1241.25 \\\n", + "descriptor mean median min max num_pixels sum \n", + "A01 411.090909 408.0 16.0 724.0 11.0 111574.0 \n", + "A02 435.692308 370.0 5.0 1029.0 13.0 93948.0 \n", + "B01 187.230769 179.0 5.0 539.0 13.0 136714.0 \n", + "B02 120.307692 65.0 5.0 419.0 13.0 103822.0 \n", + "\n", + "ion \n", + "descriptor mean median min max num_pixels \n", + "A01 10143.090909 11977.0 1029.0 15032.0 11.0 \n", + "A02 7226.769231 8213.0 315.0 13905.0 13.0 \n", + "B01 10516.461538 7009.0 1023.0 22521.0 13.0 \n", + "B02 7986.307692 10098.0 27.0 16508.0 13.0 " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/oruebel/anaconda/lib/python2.7/site-packages/ipykernel/__main__.py:7: FutureWarning: sort is deprecated, use sort_values(inplace=True) for INPLACE sorting\n" + ] + } + ], "source": [ "df=img.resultsDataFrame(minPixelIntensity=0,alphaRows=True) #generate the dataframe\n", "IPython.display.display(df)\n", @@ -358,11 +638,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": { "collapsed": false }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Finished loading spectrum 1 out of 4\n", + "Finished loading spectrum 2 out of 4\n", + "Finished loading spectrum 3 out of 4\n", + "Finished loading spectrum 4 out of 4\n" + ] + } + ], "source": [ "spectra_df=openMSIsession.getSpotSpectra(img,verbose=True) #Loads the spectra from the OpenMSI server\n", " #It's lot of data, so save the desulting dataframe\n", @@ -371,7 +662,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": { "collapsed": true }, @@ -386,7 +677,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": { "collapsed": true }, @@ -400,7 +691,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": { "collapsed": false }, @@ -429,39 +720,142 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Save the ArrayedImage to an OpenMSI File\n", - "omsi_out_file = img.saveToOpenMSIFile() \n", - "# fileName : If not given than a dialog will be shown to ask for a filename to be used" + "import sys\n", + "sys.path.append('/Users/oruebel/Devel/BASTet-git/bastet')\n", + "save_filename = 'save_omaat2.h5'\n", + "omsi_out_file = img.saveToOpenMSIFile(filename=save_filename, spotSpectra=spectra_df) \n", + "# filename : If None, than a dialog will be shown to ask for a filename to be used" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "To upload our ArrayedImage to OpenMSI we simply call:" + "We can now easily restore our analysis from the local file at any time via:" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": { - "collapsed": false + "collapsed": true }, "outputs": [], "source": [ - "img.upload_to_openmsi()\n", - "# fileName: If we set the fileName than the existing file will be used otherwise we'll ask for a name\n", + "spotSpectra_resored, arrayedImage_restored = openMSIsession.restore_omaat_results(\n", + " filename=save_filename,\n", + " localFile=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To upload our ArrayedImage to OpenMSI we simply call:" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Please enter your NERSC Password\n", + "Enter password for user \"oruebel\" \n", + "········\n" + ] + }, + { + "data": { + "text/plain": [ + "(True, True, True, None)" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "openMSIsession.upload_omaat_results(filename=save_filename, machine='edison')\n", + "# filename: If we set the fileName to None, then we'll ask the user for a filename via a dialog\n", "# username: We can set the username or we'll ask for it\n", "# session: The NERSC NEWT session to be used (not the OpenMSI seesion). We'll create it if not given\n", "# machine: The NERSC machine we should use for the upload, e.g., 'cori' or 'edison'" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can naturally also download the complete HDF5 file from OpenMSI simply via:" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "remote_file = os.path.join(openMSIsession.username, os.path.basename(save_filename))\n", + "download_filename = openMSIsession.download_file(filename=remote_file)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can now again restore our analysis from the downloaded file in the same way as before via:" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "spotSpectra_resored, arrayedImage_restored = openMSIsession.restore_omaat_results(\n", + " filename=download_filename,\n", + " localFile=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can also restore our arrayed image and spot spectra data from the remote file stored on OpenMSI directly via" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "#remote_file = os.path.join(openMSIsession.username, os.path.basename(save_filename))\n", + "#spotSpectra_resored, arrayedImage_restored = openMSIsession.restore_omaat_results(\n", + "# filename=remote_file,\n", + "# localFile=False)" + ] + }, { "cell_type": "code", "execution_count": null, From aa11554f37b5d59e07cda013d9279960c5741f43 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Tue, 20 Sep 2016 14:43:16 -0700 Subject: [PATCH 6/6] Removed sys path append from notebook --- omaat_notebook.ipynb | 385 ++++--------------------------------------- 1 file changed, 36 insertions(+), 349 deletions(-) diff --git a/omaat_notebook.ipynb b/omaat_notebook.ipynb index 6b6d6c5..ddc2c06 100644 --- a/omaat_notebook.ipynb +++ b/omaat_notebook.ipynb @@ -47,19 +47,11 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": { "collapsed": false }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Completed loading OpenMSI Arrayed Analysis Toolkit\n" - ] - } - ], + "outputs": [], "source": [ "#load the code. Since it's specialized ipython notebook code, use '%run' rather than 'import'\n", "%run omaat_lib.ipy" @@ -77,19 +69,11 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": { "collapsed": false }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Login appears to be successful!\n" - ] - } - ], + "outputs": [], "source": [ "#log into OpenMSI.nersc.gov\n", "openMSIsession = login()" @@ -113,28 +97,11 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": { "collapsed": false }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Stored 'arrayed_analysis_default_filename' (unicode)\n", - "Loading image...\n", - "loading ion 1 of 3. m/z = 1141.350000 +/- 0.500000\n", - "Time to load ion: 0.166079998016 seconds\n", - "loading ion 2 of 3. m/z = 1143.050000 +/- 0.500000\n", - "Time to load ion: 0.160965919495 seconds\n", - "loading ion 3 of 3. m/z = 1241.250000 +/- 0.500000\n", - "Time to load ion: 0.169579029083 seconds\n", - "Image has been loaded.\n", - "Image has been saved in the global 'img' variable.\n" - ] - } - ], + "outputs": [], "source": [ "if \"openMSIsession\" not in locals():\n", " openMSIsession=OpenMSIsession()\n", @@ -153,19 +120,11 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": { "collapsed": false }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The window should be open now. If you can't see it, check to see if it's behind another window\n" - ] - } - ], + "outputs": [], "source": [ "color_map=\"hot_r\" #want to try a different color map? change it here, run the cell, and all functions below will use it\n", "marker_color=\"blue\"\n", @@ -189,26 +148,11 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": { "collapsed": false }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Number of columns? leave blank for default (\"2\") \n", - "Number of rows? leave blank for default (\"2\") \n", - "Hexagonal Offset? This shifts every other line by this many spots. leave blank for default (\"0.000000\") \n", - "Stored 'arrayed_analysis_columns' (int)\n", - "Stored 'arrayed_analysis_rows' (int)\n", - "Stored 'arrayed_analysis_offset' (float)\n", - "The window should be open now. If you can't see it, check to see if it's behind another window\n", - "new spot x and y locations have been saved.\n" - ] - } - ], + "outputs": [], "source": [ "#define spot centers as a trapezoid.\n", "#if you want to pass rows/columns as arguments, or choose the colormap, use the img.roughPosition() method in stead.\n", @@ -236,24 +180,11 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": { "collapsed": false }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Stored 'arrayed_analysis_radius' (float)\n", - "Stored 'arrayed_analysis_minScore' (float)\n", - "done with optimization round 1 of 3\n", - "done with optimization round 2 of 3\n", - "done with optimization round 3 of 3\n", - "optimization routine completed. new spot x and y positions saved.\n" - ] - } - ], + "outputs": [], "source": [ "#automagically optimize the spot centers to correspond to the actual spots on the image\n", "img.optimizeSpots_with_dialogs()" @@ -272,20 +203,11 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": { "collapsed": false }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The window should be open now. If you can't see it, check to see if it's behind another window\n", - "new spot x and y locations have been saved.\n" - ] - } - ], + "outputs": [], "source": [ "#check the positions of the spots and manually adjust them if need be\n", "radius=arrayed_analysis_radius if (\"arrayed_analysis_radius\" in locals()) else 2\n", @@ -303,19 +225,11 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": { "collapsed": false }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Done saving.\n" - ] - } - ], + "outputs": [], "source": [ "#Optional: Save the ArrayedImage into a pickle file.\n", "filename=\"filename.arrayed_img\"\n", @@ -335,22 +249,11 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": { "collapsed": false }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "ArrayedImage based on 20120913_nimzyme.h5\n", - "Ions loaded: [1141.35, 1143.05, 1241.25]\n", - "# of spot locations defined: 4\n", - "# of spot pixel masks defined: None\n" - ] - } - ], + "outputs": [], "source": [ "#Optional: Load an ArrayedImage from a pickle file. This way you can work off-line\n", "filename=\"filename.arrayed_img\"\n", @@ -371,20 +274,11 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": { "collapsed": false }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "4 spots generated. number of spots with N pixels:{11: 1, 13: 3}\n", - "The window should be open now. If you can't see it, check to see if it's behind another window\n" - ] - } - ], + "outputs": [], "source": [ "#You'll need to call this function. It returns a list of spots (where each spot is a list of pixels),\n", "#which is also stored inside the object.\n", @@ -406,29 +300,12 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": { "collapsed": false, "scrolled": true }, - "outputs": [ - { - "data": { - "text/html": [ - "Click the link below to access the results file:
    openmsi_arrayed_analysis_results_2016-9-20_14h31.csv
    Open it in Microsoft Excel or your favorite data analysis program.
    \n", - " You should now be done using this tool, but if you decide you want to\n", - " further optimize your spot placement, make sure to re-run the\n", - " notebook cells to update your results.\n", - "

    Thank you for using the OpenMSI Arrayed Analysis tool!" - ], - "text/plain": [ - "/Users/oruebel/Devel/OpenMSI_Arrayed_Analysis_Tools/openmsi_arrayed_analysis_results_2016-9-20_14h31.csv" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "#Write results to a file\n", "#if you don't pass it an explicit spotList it will use the spot set stored in the ArrayedImage\n", @@ -447,168 +324,11 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": { "collapsed": false }, - "outputs": [ - { - "data": { - "text/html": [ - "
    \n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    ion1141.351143.051241.25
    descriptorsummeanmedianminmaxnum_pixelssummeanmedianminmaxnum_pixelssummeanmedianminmaxnum_pixels
    A0135893.03263.0000003349.0321.05566.011.04522.0411.090909408.016.0724.011.0111574.010143.09090911977.01029.015032.011.0
    A0244428.03417.5384622990.098.08148.013.05664.0435.692308370.05.01029.013.093948.07226.7692318213.0315.013905.013.0
    B0121354.01642.6153851590.087.04008.013.02434.0187.230769179.05.0539.013.0136714.010516.4615387009.01023.022521.013.0
    B0214681.01129.307692855.010.03191.013.01564.0120.30769265.05.0419.013.0103822.07986.30769210098.027.016508.013.0
    \n", - "
    " - ], - "text/plain": [ - "ion 1141.35 1143.05 \\\n", - "descriptor sum mean median min max num_pixels sum \n", - "A01 35893.0 3263.000000 3349.0 321.0 5566.0 11.0 4522.0 \n", - "A02 44428.0 3417.538462 2990.0 98.0 8148.0 13.0 5664.0 \n", - "B01 21354.0 1642.615385 1590.0 87.0 4008.0 13.0 2434.0 \n", - "B02 14681.0 1129.307692 855.0 10.0 3191.0 13.0 1564.0 \n", - "\n", - "ion 1241.25 \\\n", - "descriptor mean median min max num_pixels sum \n", - "A01 411.090909 408.0 16.0 724.0 11.0 111574.0 \n", - "A02 435.692308 370.0 5.0 1029.0 13.0 93948.0 \n", - "B01 187.230769 179.0 5.0 539.0 13.0 136714.0 \n", - "B02 120.307692 65.0 5.0 419.0 13.0 103822.0 \n", - "\n", - "ion \n", - "descriptor mean median min max num_pixels \n", - "A01 10143.090909 11977.0 1029.0 15032.0 11.0 \n", - "A02 7226.769231 8213.0 315.0 13905.0 13.0 \n", - "B01 10516.461538 7009.0 1023.0 22521.0 13.0 \n", - "B02 7986.307692 10098.0 27.0 16508.0 13.0 " - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/oruebel/anaconda/lib/python2.7/site-packages/ipykernel/__main__.py:7: FutureWarning: sort is deprecated, use sort_values(inplace=True) for INPLACE sorting\n" - ] - } - ], + "outputs": [], "source": [ "df=img.resultsDataFrame(minPixelIntensity=0,alphaRows=True) #generate the dataframe\n", "IPython.display.display(df)\n", @@ -638,22 +358,11 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": { "collapsed": false }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Finished loading spectrum 1 out of 4\n", - "Finished loading spectrum 2 out of 4\n", - "Finished loading spectrum 3 out of 4\n", - "Finished loading spectrum 4 out of 4\n" - ] - } - ], + "outputs": [], "source": [ "spectra_df=openMSIsession.getSpotSpectra(img,verbose=True) #Loads the spectra from the OpenMSI server\n", " #It's lot of data, so save the desulting dataframe\n", @@ -662,7 +371,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": { "collapsed": true }, @@ -677,7 +386,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": { "collapsed": true }, @@ -691,7 +400,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": { "collapsed": false }, @@ -720,15 +429,13 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Save the ArrayedImage to an OpenMSI File\n", - "import sys\n", - "sys.path.append('/Users/oruebel/Devel/BASTet-git/bastet')\n", "save_filename = 'save_omaat2.h5'\n", "omsi_out_file = img.saveToOpenMSIFile(filename=save_filename, spotSpectra=spectra_df) \n", "# filename : If None, than a dialog will be shown to ask for a filename to be used" @@ -743,7 +450,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": { "collapsed": true }, @@ -763,31 +470,11 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "metadata": { "collapsed": false }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Please enter your NERSC Password\n", - "Enter password for user \"oruebel\" \n", - "········\n" - ] - }, - { - "data": { - "text/plain": [ - "(True, True, True, None)" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "openMSIsession.upload_omaat_results(filename=save_filename, machine='edison')\n", "# filename: If we set the fileName to None, then we'll ask the user for a filename via a dialog\n", @@ -805,7 +492,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "metadata": { "collapsed": false }, @@ -824,7 +511,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "metadata": { "collapsed": false }, @@ -844,7 +531,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "metadata": { "collapsed": false },