From 63e500acb8d2360624c974741ab0c272d0d4e5d8 Mon Sep 17 00:00:00 2001 From: Jinyong Shin Date: Thu, 23 Feb 2017 12:20:53 -0500 Subject: [PATCH 01/42] first week --- ndio/remote/neurodata.py | 2494 +++++++++++++++++++------------------- 1 file changed, 1249 insertions(+), 1245 deletions(-) diff --git a/ndio/remote/neurodata.py b/ndio/remote/neurodata.py index 37f42b5..6ee9182 100644 --- a/ndio/remote/neurodata.py +++ b/ndio/remote/neurodata.py @@ -18,9 +18,9 @@ from functools import wraps try: - import urllib.request as urllib2 + import urllib.request as urllib2 except ImportError: - import urllib2 + import urllib2 DEFAULT_HOSTNAME = "openconnecto.me" DEFAULT_SUFFIX = "nd" @@ -29,1246 +29,1250 @@ class neurodata(Remote): - """ - The NeuroData remote, for interfacing with ndstore, ndlims, and friends. - """ - - # SECTION: - # Enumerables - IMAGE = IMG = 'image' - ANNOTATION = ANNO = 'annotation' - - def __init__(self, - user_token, - hostname=DEFAULT_HOSTNAME, - protocol=DEFAULT_PROTOCOL, - meta_root="http://lims.neurodata.io/", - meta_protocol=DEFAULT_PROTOCOL, **kwargs): - """ - Initializer for the neurodata remote class. - - Arguments: - hostname (str: "openconnecto.me"): The hostname to connect to - protocol (str: "http"): The protocol (http or https) to use - meta_root (str: "http://lims.neurodata.io/"): The metadata server - meta_protocol (str: "http"): The protocol to use for the md server - check_tokens (boolean: False): Whether functions that take `token` - as an argument should check for the existance of that token and - fail immediately if it is not found. This is a good idea for - functions that take more time to complete, and might not fail - until the very end otherwise. - chunk_threshold (int: 1e9 / 4): The maximum size of a numpy array - that will be uploaded in one HTTP request. If you find that - your data requests are commonly timing out, try reducing this. - Default is 1e9 / 4, or a 0.25GiB. - suffix (str: "ocp"): The URL suffix to specify ndstore/microns. If - you aren't sure what to do with this, don't specify one. - """ - self._check_tokens = kwargs.get('check_tokens', False) - self._chunk_threshold = kwargs.get('chunk_threshold', 1E9 / 4) - self._ext = kwargs.get('suffix', DEFAULT_SUFFIX) - self._known_tokens = [] - self._user_token = user_token - - # Prepare meta url - self.meta_root = meta_root - if not self.meta_root.endswith('/'): - self.meta_root = self.meta_root + "/" - if self.meta_root.startswith('https'): - self.meta_root = self.meta_root[self.meta_root.index('://') + 3:] - self.meta_protocol = meta_protocol - - super(neurodata, self).__init__(hostname, protocol) - - # SECTION: - # Decorators - def _check_token(f): - @wraps(f) - def wrapped(self, *args, **kwargs): - if self._check_tokens: - if 'token' in kwargs: - token = kwargs['token'] - else: - token = args[0] - if token not in self._known_tokens: - if self.ping('{}/info/'.format(token)) != 200: - raise RemoteDataNotFoundError("Bad token {}".format( - token)) - else: - self._known_tokens.append(token) - return f(self, *args, **kwargs) - return wrapped - - # SECTION: - # Utilities - def ping(self, suffix='public_tokens/'): - """ - Return the status-code of the API (estimated using the public-tokens - lookup page). - - Arguments: - suffix (str : 'public_tokens/'): The url endpoint to check - - Returns: - int: status code - """ - return super(neurodata, self).ping(suffix) - - def url(self, suffix=""): - """ - Return a constructed URL, appending an optional suffix (uri path). - - Arguments: - suffix (str : ""): The suffix to append to the end of the URL - - Returns: - str: The complete URL - """ - return super(neurodata, self).url('{}/sd/'.format(self._ext) + suffix) - - def meta_url(self, suffix=""): - """ - Return a constructed URL, appending an optional suffix (uri path), - for the metadata server. (Should be temporary, until the LIMS shim - is fixed.) - - Arguments: - suffix (str : ""): The suffix to append to the end of the URL - - Returns: - str: The complete URL - """ - return self.meta_protocol + "://" + self.meta_root + suffix - - def __repr__(self): - """ - Return a string representation that can be used to reproduce this - instance. `eval(repr(this))` should return an identical copy. - - Arguments: - None - - Returns: - str: Representation of reproducible instance. - """ - return "ndio.remote.neurodata('{}', '{}')".format( - self.hostname, - self.protocol, - self.meta_url, - self.meta_protocol - ) - - # SECTION: - # Metadata - def get_public_tokens(self): - """ - Get a list of public tokens available on this server. - - Arguments: - None - - Returns: - str[]: list of public tokens - """ - r = getURL(self.url() + "public_tokens/") - return r.json() - - def get_public_datasets(self): - """ - NOTE: VERY SLOW! - Get a list of public datasets. Different than public tokens! - - Arguments: - None - - Returns: - str[]: list of public datasets - """ - return list(self.get_public_datasets_and_tokens().keys()) - - def get_public_datasets_and_tokens(self): - """ - NOTE: VERY SLOW! - Get a dictionary relating key:dataset to value:[tokens] that rely - on that dataset. - - Arguments: - None - - Returns: - dict: relating key:dataset to value:[tokens] - """ - datasets = {} - tokens = self.get_public_tokens() - for t in tokens: - dataset = self.get_token_dataset(t) - if dataset in datasets: - datasets[dataset].append(t) - else: - datasets[dataset] = [t] - return datasets - - @_check_token - def get_token_dataset(self, token): - """ - Get the dataset for a given token. - - Arguments: - token (str): The token to inspect - - Returns: - str: The name of the dataset - """ - return self.get_proj_info(token)['dataset']['description'] - - @_check_token - def get_proj_info(self, token): - """ - Return the project info for a given token. - - Arguments: - token (str): Token to return information for - - Returns: - JSON: representation of proj_info - """ - r = getURL(self.url() + "{}/info/".format(token)) - return r.json() - - @_check_token - def get_metadata(self, token): - """ - An alias for get_proj_info. - """ - return self.get_proj_info(token) - - @_check_token - def get_channels(self, token): - """ - Wraps get_proj_info to return a dictionary of just the channels of - a given project. - - Arguments: - token (str): Token to return channels for - - Returns: - JSON: dictionary of channels. - """ - return self.get_proj_info(token)['channels'] - - @_check_token - def get_image_size(self, token, resolution=0): - """ - Return the size of the volume (3D). Convenient for when you want - to download the entirety of a dataset. - - Arguments: - token (str): The token for which to find the dataset image bounds - resolution (int : 0): The resolution at which to get image bounds. - Defaults to 0, to get the largest area available. - - Returns: - int[3]: The size of the bounds. Should == get_volume.shape - - Raises: - RemoteDataNotFoundError: If the token is invalid, or if the - metadata at that resolution is unavailable in projinfo. - """ - info = self.get_proj_info(token) - res = str(resolution) - if res not in info['dataset']['imagesize']: - raise RemoteDataNotFoundError("Resolution " + res + - " is not available.") - return info['dataset']['imagesize'][str(resolution)] - - @_check_token - def set_metadata(self, token, data): - """ - Insert new metadata into the OCP metadata database. - - Arguments: - token (str): Token of the datum to set - data (str): A dictionary to insert as metadata. Include `secret`. - - Returns: - json: Info of the inserted ID (convenience) or an error message. - - Throws: - RemoteDataUploadError: If the token is already populated, or if - there is an issue with your specified `secret` key. - """ - req = requests.post(self.meta_url("metadata/ocp/set/" + token), - json=data) - - if req.status_code != 200: - raise RemoteDataUploadError( - "Could not upload metadata: " + req.json()['message'] - ) - return req.json() - - @_check_token - def get_subvolumes(self, token): - """ - Return a list of subvolumes taken from LIMS, if available. - - Arguments: - token (str): The token to read from in LIMS - - Returns: - dict: or None if unavailable - """ - md = self.get_metadata(token)['metadata'] - if 'subvolumes' in md: - return md['subvolumes'] - else: - return None - - @_check_token - def add_subvolume(self, token, channel, secret, - x_start, x_stop, - y_start, y_stop, - z_start, z_stop, - resolution, title, notes): - """ - Adds a new subvolume to a token/channel. - - Arguments: - token (str): The token to write to in LIMS - channel (str): Channel to add in the subvolume. Can be `None` - x_start (int): Start in x dimension - x_stop (int): Stop in x dimension - y_start (int): Start in y dimension - y_stop (int): Stop in y dimension - z_start (int): Start in z dimension - z_stop (int): Stop in z dimension - resolution (int): The resolution at which this subvolume is seen - title (str): The title to set for the subvolume - notes (str): Optional extra thoughts on the subvolume - - Returns: - boolean: success - """ - md = self.get_metadata(token)['metadata'] - if 'subvolumes' in md: - subvols = md['subvolumes'] - else: - subvols = [] - - subvols.append({ - 'token': token, - 'channel': channel, - 'x_start': x_start, - 'x_stop': x_stop, - 'y_start': y_start, - 'y_stop': y_stop, - 'z_start': z_start, - 'z_stop': z_stop, - 'resolution': resolution, - 'title': title, - 'notes': notes - }) - - return self.set_metadata(token, { - 'secret': secret, - 'subvolumes': subvols - }) - - # SECTION: - # Data Download - @_check_token - def get_block_size(self, token, resolution=None): - """ - Gets the block-size for a given token at a given resolution. - - Arguments: - token (str): The token to inspect - resolution (int : None): The resolution at which to inspect data. - If none is specified, uses the minimum available. - - Returns: - int[3]: The xyz blocksize. - """ - cdims = self.get_metadata(token)['dataset']['cube_dimension'] - if resolution is None: - resolution = min(cdims.keys()) - return cdims[str(resolution)] - - @_check_token - def get_image_offset(self, token, resolution=0): - """ - Gets the image offset for a given token at a given resolution. For - instance, the `kasthuri11` dataset starts at (0, 0, 1), so its 1850th - slice is slice 1850, not 1849. When downloading a full dataset, the - result of this function should be your x/y/z starts. - - Arguments: - token (str): The token to inspect - resolution (int : 0): The resolution at which to gather the offset - - Returns: - int[3]: The origin of the dataset, as a list - """ - info = self.get_proj_info(token) - res = str(resolution) - if res not in info['dataset']['offset']: - raise RemoteDataNotFoundError("Resolution " + res + - " is not available.") - return info['dataset']['offset'][str(resolution)] - - @_check_token - def get_xy_slice(self, token, channel, - x_start, x_stop, - y_start, y_stop, - z_index, - resolution=0): - """ - Return a binary-encoded, decompressed 2d image. You should - specify a 'token' and 'channel' pair. For image data, users - should use the channel 'image.' - - Arguments: - token (str): Token to identify data to download - channel (str): Channel - resolution (int): Resolution level - Q_start (int):` The lower bound of dimension 'Q' - Q_stop (int): The upper bound of dimension 'Q' - z_index (int): The z-slice to image - - Returns: - str: binary image data - """ - vol = self.get_cutout(token, channel, x_start, x_stop, y_start, - y_stop, z_index, z_index + 1, resolution) - - vol = numpy.squeeze(vol) # 3D volume to 2D slice - - return vol - - @_check_token - def get_image(self, token, channel, - x_start, x_stop, - y_start, y_stop, - z_index, - resolution=0): - """ - Alias for the `get_xy_slice` function for backwards compatibility. - """ - return self.get_xy_slice(token, channel, - x_start, x_stop, - y_start, y_stop, - z_index, - resolution) - - @_check_token - def get_volume(self, token, channel, - x_start, x_stop, - y_start, y_stop, - z_start, z_stop, - resolution=1, - block_size=DEFAULT_BLOCK_SIZE, - neariso=False): - """ - Get a RAMONVolume volumetric cutout from the neurodata server. - - Arguments: - token (str): Token to identify data to download - channel (str): Channel - resolution (int): Resolution level - Q_start (int): The lower bound of dimension 'Q' - Q_stop (int): The upper bound of dimension 'Q' - block_size (int[3]): Block size of this dataset - neariso (bool : False): Passes the 'neariso' param to the cutout. - If you don't know what this means, ignore it! - - Returns: - ndio.ramon.RAMONVolume: Downloaded data. - """ - size = (x_stop - x_start) * (y_stop - y_start) * (z_stop - z_start) - volume = ramon.RAMONVolume() - volume.xyz_offset = [x_start, y_start, z_start] - volume.resolution = resolution - - volume.cutout = self.get_cutout(token, channel, x_start, - x_stop, y_start, y_stop, - z_start, z_stop, - resolution=resolution, - block_size=block_size, - neariso=neariso) - return volume - - @_check_token - def get_cutout(self, token, channel, - x_start, x_stop, - y_start, y_stop, - z_start, z_stop, - resolution=1, - block_size=DEFAULT_BLOCK_SIZE, - neariso=False): - """ - Get volumetric cutout data from the neurodata server. - - Arguments: - token (str): Token to identify data to download - channel (str): Channel - resolution (int): Resolution level - Q_start (int): The lower bound of dimension 'Q' - Q_stop (int): The upper bound of dimension 'Q' - block_size (int[3]): Block size of this dataset. If not provided, - ndio uses the metadata of this tokenchannel to set. If you find - that your downloads are timing out or otherwise failing, it may - be wise to start off by making this smaller. - neariso (bool : False): Passes the 'neariso' param to the cutout. - If you don't know what this means, ignore it! - - Returns: - numpy.ndarray: Downloaded data. - """ - if block_size is None: - # look up block size from metadata - block_size = self.get_block_size(token, resolution) - - origin = self.get_image_offset(token, resolution) - - # If z_stop - z_start is < 16, backend still pulls minimum 16 slices - if (z_stop - z_start) < 16: - z_slices = 16 - else: - z_slices = z_stop - z_start - - # Calculate size of the data to be downloaded. - size = (x_stop - x_start) * (y_stop - y_start) * z_slices * 4 - - # Switch which download function to use based on which libraries are - # available in this version of python. - if six.PY2: - dl_func = self._get_cutout_blosc_no_chunking - elif six.PY3: - dl_func = self._get_cutout_no_chunking - else: - raise ValueError("Invalid Python version.") - - if size < self._chunk_threshold: - vol = dl_func(token, channel, resolution, - x_start, x_stop, - y_start, y_stop, - z_start, z_stop, neariso=neariso) - vol = numpy.rollaxis(vol, 1) - vol = numpy.rollaxis(vol, 2) - return vol - else: - from ndio.utils.parallel import block_compute - blocks = block_compute(x_start, x_stop, - y_start, y_stop, - z_start, z_stop, - origin, block_size) - - vol = numpy.zeros(((z_stop - z_start), - (y_stop - y_start), - (x_stop - x_start))) - for b in blocks: - - data = dl_func(token, channel, resolution, - b[0][0], b[0][1], - b[1][0], b[1][1], - b[2][0], b[2][1], neariso=neariso) - - if b == blocks[0]: # first block - vol = numpy.zeros(((z_stop - z_start), - (y_stop - y_start), - (x_stop - x_start)), dtype=data.dtype) - - vol[b[2][0] - z_start: b[2][1] - z_start, - b[1][0] - y_start: b[1][1] - y_start, - b[0][0] - x_start: b[0][1] - x_start] = data - - vol = numpy.rollaxis(vol, 1) - vol = numpy.rollaxis(vol, 2) - return vol - - def _get_cutout_no_chunking(self, token, channel, resolution, - x_start, x_stop, y_start, y_stop, - z_start, z_stop, neariso=False): - url = self.url() + "{}/{}/hdf5/{}/{},{}/{},{}/{},{}/".format( - token, channel, resolution, - x_start, x_stop, - y_start, y_stop, - z_start, z_stop - ) - - if neariso: - url += "neariso/" - - req = getURL(url) - if req.status_code is not 200: - raise IOError("Bad server response for {}: {}: {}".format( - url, - req.status_code, - req.text)) - - with tempfile.NamedTemporaryFile() as tmpfile: - tmpfile.write(req.content) - tmpfile.seek(0) - h5file = h5py.File(tmpfile.name, "r") - return h5file.get(channel).get('CUTOUT')[:] - raise IOError("Failed to make tempfile.") - - def _get_cutout_blosc_no_chunking(self, token, channel, resolution, - x_start, x_stop, y_start, y_stop, - z_start, z_stop, neariso=False): - - url = self.url() + "{}/{}/blosc/{}/{},{}/{},{}/{},{}/".format( - token, channel, resolution, - x_start, x_stop, - y_start, y_stop, - z_start, z_stop - ) - - if neariso: - url += "neariso/" - - req = getURL(url) - if req.status_code is not 200: - raise IOError("Bad server response for {}: {}: {}".format( - url, - req.status_code, - req.text)) - - # This will need modification for >3D blocks - return blosc.unpack_array(req.content)[0] - - raise IOError("Failed to retrieve blosc cutout.") - - # SECTION: - # Data Upload - - @_check_token - def post_cutout(self, token, channel, - x_start, - y_start, - z_start, - data, - resolution=0): - """ - Post a cutout to the server. - - Arguments: - token (str) - channel (str) - x_start (int) - y_start (int) - z_start (int) - data (numpy.ndarray): A numpy array of data. Pass in (x, y, z) - resolution (int : 0): Resolution at which to insert the data - - Returns: - bool: True on success - - Raises: - RemoteDataUploadError: if there's an issue during upload. - """ - datatype = self.get_proj_info(token)['channels'][channel]['datatype'] - if data.dtype.name != datatype: - data = data.astype(datatype) - - data = numpy.rollaxis(data, 1) - data = numpy.rollaxis(data, 2) - - if six.PY3 or data.nbytes > 1.5e9: - ul_func = self._post_cutout_no_chunking_npz - else: - ul_func = self._post_cutout_no_chunking_blosc - - if data.size < self._chunk_threshold: - return ul_func(token, channel, x_start, - y_start, z_start, data, - resolution) - - return self._post_cutout_with_chunking(token, channel, - x_start, y_start, z_start, data, - resolution, ul_func) - - def _post_cutout_with_chunking(self, token, channel, x_start, - y_start, z_start, data, - resolution, ul_func): - # must chunk first - from ndio.utils.parallel import block_compute - blocks = block_compute(x_start, x_start + data.shape[2], - y_start, y_start + data.shape[1], - z_start, z_start + data.shape[0]) - for b in blocks: - # data coordinate relative to the size of the arra - subvol = data[b[2][0] - z_start: b[2][1] - z_start, - b[1][0] - y_start: b[1][1] - y_start, - b[0][0] - x_start: b[0][1] - x_start] - # upload the chunk: - # upload coordinate relative to x_start, y_start, z_start - ul_func(token, channel, b[0][0], - b[1][0], b[2][0], subvol, - resolution) - return True - - def _post_cutout_no_chunking_npz(self, token, channel, - x_start, y_start, z_start, - data, resolution): - - data = numpy.expand_dims(data, axis=0) - tempfile = BytesIO() - numpy.save(tempfile, data) - compressed = zlib.compress(tempfile.getvalue()) - - url = self.url("{}/{}/npz/{}/{},{}/{},{}/{},{}/".format( - token, channel, - resolution, - x_start, x_start + data.shape[3], - y_start, y_start + data.shape[2], - z_start, z_start + data.shape[1] - )) - - req = requests.post(url, data=compressed, headers={ - 'Content-Type': 'application/octet-stream' - }) - - if req.status_code is not 200: - raise RemoteDataUploadError(req.text) - else: - return True - - def _post_cutout_no_chunking_blosc(self, token, channel, - x_start, y_start, z_start, - data, resolution): - """ - Accepts data in zyx. !!! - """ - data = numpy.expand_dims(data, axis=0) - blosc_data = blosc.pack_array(data) - - url = self.url("{}/{}/blosc/{}/{},{}/{},{}/{},{}/".format( - token, channel, - resolution, - x_start, x_start + data.shape[3], - y_start, y_start + data.shape[2], - z_start, z_start + data.shape[1] - )) - req = requests.post(url, data=blosc_data, headers={ - 'Content-Type': 'application/octet-stream' - }) - - if req.status_code is not 200: - raise RemoteDataUploadError(req.text) - else: - return True - - # SECTION: - # RAMON Download - - @_check_token - def get_ramon_bounding_box(self, token, channel, r_id, resolution=0): - """ - Get the bounding box for a RAMON object (specified by ID). - - Arguments: - token (str): Project to use - channel (str): Channel to use - r_id (int): Which ID to get a bounding box - resolution (int : 0): The resolution at which to download - - Returns: - (x_start, x_stop, y_start, y_stop, z_start, z_stop): ints - """ - url = self.url('{}/{}/{}/boundingbox/{}/'.format(token, channel, - r_id, resolution)) - - r_id = str(r_id) - res = getURL(url) - - if res.status_code != 200: - rt = self.get_ramon_metadata(token, channel, r_id)[r_id]['type'] - if rt in ['neuron']: - raise ValueError("ID {} is of type '{}'".format(r_id, rt)) - raise RemoteDataNotFoundError("No such ID {}".format(r_id)) - - with tempfile.NamedTemporaryFile() as tmpfile: - tmpfile.write(res.content) - tmpfile.seek(0) - h5file = h5py.File(tmpfile.name, "r") - origin = h5file["{}/XYZOFFSET".format(r_id)][()] - size = h5file["{}/XYZDIMENSION".format(r_id)][()] - return (origin[0], origin[0] + size[0], - origin[1], origin[1] + size[1], - origin[2], origin[2] + size[2]) - - @_check_token - def get_ramon_ids(self, token, channel, ramon_type=None): - """ - Return a list of all IDs available for download from this token and - channel. - - Arguments: - token (str): Project to use - channel (str): Channel to use - ramon_type (int : None): Optional. If set, filters IDs and only - returns those of RAMON objects of the requested type. - - Returns: - int[]: A list of the ids of the returned RAMON objects - - Raises: - RemoteDataNotFoundError: If the channel or token is not found - """ - url = self.url("{}/{}/query/".format(token, channel)) - if ramon_type is not None: - # User is requesting a specific ramon_type. - if type(ramon_type) is not int: - ramon_type = ramon.AnnotationType.get_int(ramon_type) - url += "type/{}/".format(str(ramon_type)) - - req = getURL(url) - - if req.status_code is not 200: - raise RemoteDataNotFoundError('No query results for token {}.' - .format(token)) - else: - with tempfile.NamedTemporaryFile() as tmpfile: - tmpfile.write(req.content) - tmpfile.seek(0) - h5file = h5py.File(tmpfile.name, "r") - if 'ANNOIDS' not in h5file: - return [] - return [i for i in h5file['ANNOIDS']] - raise IOError("Could not successfully mock HDF5 file for parsing.") - - @_check_token - def get_ramon(self, token, channel, ids, resolution=0, - include_cutout=False, sieve=None, batch_size=100): - """ - Download a RAMON object by ID. - - Arguments: - token (str): Project to use - channel (str): The channel to use - ids (int, str, int[], str[]): The IDs of a RAMON object to gather. - Can be int (3), string ("3"), int[] ([3, 4, 5]), or string - (["3", "4", "5"]). - resolution (int : None): Resolution. Defaults to the most granular - resolution (0 for now) - include_cutout (bool : False): If True, r.cutout is populated - sieve (function : None): A function that accepts a single ramon - and returns True or False depending on whether you want that - ramon object to be included in your response or not. - For example, - ``` - def is_even_id(ramon): - return ramon.id % 2 == 0 - ``` - You can then pass this to get_ramon like this: - ``` - ndio.remote.neurodata.get_ramon( . . . , sieve=is_even_id) - ``` - batch_size (int : 100): The amount of RAMON objects to download at - a time. If this is greater than 100, we anticipate things going - very poorly for you. So if you set it <100, ndio will use it. - If >=100, set it to 100. - - Returns: - ndio.ramon.RAMON[]: A list of returned RAMON objects. - - Raises: - RemoteDataNotFoundError: If the requested ids cannot be found. - """ - b_size = min(100, batch_size) - - _return_first_only = False - if type(ids) is not list: - _return_first_only = True - ids = [ids] - ids = [str(i) for i in ids] - - rs = [] - id_batches = [ids[i:i + b_size] for i in range(0, len(ids), b_size)] - for batch in id_batches: - rs.extend(self._get_ramon_batch(token, channel, batch, resolution)) - - rs = self._filter_ramon(rs, sieve) - - if include_cutout: - rs = [self._add_ramon_cutout(token, channel, r, resolution) - for r in rs] - - if _return_first_only: - return rs[0] - - return sorted(rs, key=lambda x: ids.index(x.id)) - - def _filter_ramon(self, rs, sieve): - if sieve is not None: - return [r for r in rs if sieve(r)] - return rs - - def _add_ramon_cutout(self, token, channel, ramon, resolution): - origin = ramon.xyz_offset - # Get the bounding box (cube-aligned) - bbox = self.get_ramon_bounding_box(token, channel, - ramon.id, resolution=resolution) - # Get the cutout (cube-aligned) - cutout = self.get_cutout(token, channel, - *bbox, resolution=resolution) - cutout[cutout != int(ramon.id)] = 0 - - # Compute upper offset and crop - bounds = numpy.argwhere(cutout) - mins = [min([i[dim] for i in bounds]) for dim in range(3)] - maxs = [max([i[dim] for i in bounds]) for dim in range(3)] - - ramon.cutout = cutout[ - mins[0]:maxs[0], - mins[1]:maxs[1], - mins[2]:maxs[2] - ] - - return ramon - - def _get_ramon_batch(self, token, channel, ids, resolution): - ids = [str(i) for i in ids] - url = self.url("{}/{}/{}/json/".format(token, channel, ",".join(ids))) - req = getURL(url) - - if req.status_code is not 200: - raise RemoteDataNotFoundError('No data for id {}.'.format(ids)) - else: - return ramon.from_json(req.json()) - - @_check_token - def get_ramon_metadata(self, token, channel, anno_id): - """ - Download a RAMON object by ID. `anno_id` can be a string `"123"`, an - int `123`, an array of ints `[123, 234, 345]`, an array of strings - `["123", "234", "345"]`, or a comma-separated string list - `"123,234,345"`. - - Arguments: - token (str): Project to use - channel (str): The channel to use - anno_id: An int, a str, or a list of ids to gather - - Returns: - JSON. If you pass a single id in str or int, returns a single datum - If you pass a list of int or str or a comma-separated string, will - return a dict with keys from the list and the values are the JSON - returned from the server. - - Raises: - RemoteDataNotFoundError: If the data cannot be found on the Remote - """ - if type(anno_id) in [int, numpy.uint32]: - # there's just one ID to download - return self._get_single_ramon_metadata(token, channel, - str(anno_id)) - elif type(anno_id) is str: - # either "id" or "id,id,id": - if (len(anno_id.split(',')) > 1): - results = {} - for i in anno_id.split(','): - results[i] = self._get_single_ramon_metadata( - token, channel, anno_id.strip() - ) - return results - else: - # "id" - return self._get_single_ramon_metadata(token, channel, - anno_id.strip()) - elif type(anno_id) is list: - # [id, id] or ['id', 'id'] - results = [] - for i in anno_id: - results.append(self._get_single_ramon_metadata(token, channel, - str(i))) - return results - - def _get_single_ramon_metadata(self, token, channel, anno_id): - req = getURL(self.url() + - "{}/{}/{}/json/".format(token, channel, anno_id)) - if req.status_code is not 200: - raise RemoteDataNotFoundError('No data for id {}.'.format(anno_id)) - return req.json() - - @_check_token - def delete_ramon(self, token, channel, anno): - """ - Deletes an annotation from the server. Probably you should be careful - with this function, it seems dangerous. - - Arguments: - token (str): The token to inspect - channel (str): The channel to inspect - anno (int OR list(int) OR RAMON): The annotation to delete. If a - RAMON object is supplied, the remote annotation will be deleted - by an ID lookup. If an int is supplied, the annotation will be - deleted for that ID. If a list of ints are provided, they will - all be deleted. - - Returns: - bool: Success - """ - if type(anno) is int: - a = anno - if type(anno) is str: - a = int(anno) - if type(anno) is list: - a = ",".join(anno) - else: - a = anno.id - - req = requests.delete(self.url("{}/{}/{}/".format(token, channel, a))) - if req.status_code is not 200: - raise RemoteDataNotFoundError("Could not delete id {}: {}" - .format(a, req.text)) - else: - return True - - @_check_token - def post_ramon(self, token, channel, r, batch_size=100): - """ - Posts a RAMON object to the Remote. - - Arguments: - token (str): Project to use - channel (str): The channel to use - r (RAMON or RAMON[]): The annotation(s) to upload - batch_size (int : 100): The number of RAMONs to post simultaneously - at maximum in one file. If len(r) > batch_size, the batch will - be split and uploaded automatically. Must be less than 100. - - Returns: - bool: Success = True - - Throws: - RemoteDataUploadError: if something goes wrong - """ - # Max out batch-size at 100. - b_size = min(100, batch_size) - - # Coerce incoming IDs to a list. - if type(r) is not list: - r = [r] - - # If there are too many to fit in one batch, split here and call this - # function recursively. - if len(r) > batch_size: - batches = [r[i:i + b_size] for i in range(0, len(r), b_size)] - for batch in batches: - self.post_ramon(token, channel, batch, b_size) - return - - with tempfile.NamedTemporaryFile(delete=False) as tmpfile: - for i in r: - tmpfile = ramon.to_hdf5(i, tmpfile) - - url = self.url("{}/{}/overwrite/".format(token, channel)) - req = urllib2.Request(url, tmpfile.read()) - res = urllib2.urlopen(req) - - if res.code != 200: - raise RemoteDataUploadError('[{}] Could not upload {}' - .format(res.code, str(r))) - - rets = res.read() - if six.PY3: - rets = rets.decode() - return_ids = [int(rid) for rid in rets.split(',')] - - # Now post the cutout separately: - for ri in r: - if 'cutout' in dir(ri) and ri.cutout is not None: - orig = ri.xyz_offset - self.post_cutout(token, channel, - orig[0], orig[1], orig[2], - ri.cutout, resolution=r.resolution) - return return_ids - return True - - # SECTION: - # ID Manipulation - - @_check_token - def reserve_ids(self, token, channel, quantity): - """ - Requests a list of next-available-IDs from the server. - - Arguments: - quantity (int): The number of IDs to reserve - - Returns: - int[quantity]: List of IDs you've been granted - """ - quantity = str(quantity) - url = self.url("{}/{}/reserve/{}/".format(token, channel, quantity)) - req = getURL(url) - if req.status_code is not 200: - raise RemoteDataNotFoundError('Invalid req: ' + req.status_code) - out = req.json() - return [out[0] + i for i in range(out[1])] - - @_check_token - def merge_ids(self, token, channel, ids, delete=False): - """ - Call the restful endpoint to merge two RAMON objects into one. - - Arguments: - token (str): The token to inspect - channel (str): The channel to inspect - ids (int[]): the list of the IDs to merge - delete (bool : False): Whether to delete after merging. - - Returns: - json: The ID as returned by ndstore - """ - req = getURL(self.url() + "/merge/{}/" - .format(','.join([str(i) for i in ids]))) - if req.status_code is not 200: - raise RemoteDataUploadError('Could not merge ids {}'.format( - ','.join([str(i) for i in ids]))) - if delete: - self.delete_ramon(token, channel, ids[1:]) - return True - - # SECTION: - # Channels - - def _check_channel(self, channel): - for c in channel: - if not c.isalnum(): - raise ValueError( - "Channel name cannot contain character {}.".format(c) - ) - return True - - @_check_token - def create_channel(self, token, name, channel_type, dtype, readonly): - """ - Create a new channel on the Remote, using channel_data. - - Arguments: - token (str): The token the new channel should be added to - name (str): The name of the channel to add - channel_type (str): Type of the channel (e.g. `neurodata.IMAGE`) - dtype (str): The datatype of the channel's data (e.g. `uint8`) - readonly (bool): Can others write to this channel? - - Returns: - bool: `True` if successful, `False` otherwise. - - Raises: - ValueError: If your args were bad :( - RemoteDataUploadError: If the channel data is valid but upload - fails for some other reason. - """ - self._check_channel(name) - - if channel_type not in ['image', 'annotation']: - raise ValueError('Channel type must be ' + - 'neurodata.IMAGE or neurodata.ANNOTATION.') - - if readonly * 1 not in [0, 1]: - raise ValueError("readonly must be 0 (False) or 1 (True).") - - # Good job! You supplied very nice arguments. - req = requests.post(self.url("{}/createChannel/".format(token)), json={ - "channels": { - name: { - "channel_name": name, - "channel_type": channel_type, - "datatype": dtype, - "readonly": readonly * 1 - } - } - }) - - if req.status_code is not 200: - raise RemoteDataUploadError('Could not upload {}'.format(req.text)) - else: - return True - - @_check_token - def delete_channel(self, token, name): - """ - Delete an existing channel on the Remote. Be careful! - - Arguments: - token (str): The token the new channel should be deleted from - name (str): The name of the channel to delete - - Returns: - bool: True if successful, False otherwise. - - Raises: - RemoteDataUploadError: If the upload fails for some reason. - """ - req = requests.post(self.url("{}/deleteChannel/".format(token)), json={ - "channels": [name] - }) - - if req.status_code is not 200: - raise RemoteDataUploadError('Could not delete {}'.format(req.text)) - if req.content == "SUCCESS": - return True - else: - return False - - # Propagation - - @_check_token - def propagate(self, token, channel): - """ - Kick off the propagate function on the remote server. - - Arguments: - token (str): The token to propagate - channel (str): The channel to propagate - - Returns: - boolean: Success - """ - if self.get_propagate_status(token, channel) != u'0': - return - url = self.url('{}/{}/setPropagate/1/'.format(token, channel)) - req = getURL(url) - if req.status_code is not 200: - raise RemoteDataUploadError('Propagate fail: {}'.format(req.text)) - return True - - @_check_token - def get_propagate_status(self, token, channel): - """ - Get the propagate status for a token/channel pair. - - Arguments: - token (str): The token to check - channel (str): The channel to check - - Returns: - str: The status code - """ - url = self.url('{}/{}/getPropagate/'.format(token, channel)) - req = getURL(url) - if req.status_code is not 200: - raise ValueError('Bad pair: {}/{}'.format(token, channel)) - return req.text - - def getURL(self, url, token=''): - """ - Get the propagate status for a token/channel pair. - - Arguments: - url (str): The url make a get to - token (str): The authentication token - - Returns: - obj: The response object - """ - if (token == ''): - token = self._user_token - - return requests.get(url, - headers={ - 'Authorization': 'Token {}'.format(token)}, - verify=False) + """ + The NeuroData remote, for interfacing with ndstore, ndlims, and friends. + """ + + # SECTION: + # Enumerables + IMAGE = IMG = 'image' + ANNOTATION = ANNO = 'annotation' + + def __init__(self, + user_token, + hostname=DEFAULT_HOSTNAME, + protocol=DEFAULT_PROTOCOL, + meta_root="http://lims.neurodata.io/", + meta_protocol=DEFAULT_PROTOCOL, **kwargs): + """ + Initializer for the neurodata remote class. + + Arguments: + hostname (str: "openconnecto.me"): The hostname to connect to + protocol (str: "http"): The protocol (http or https) to use + meta_root (str: "http://lims.neurodata.io/"): The metadata server + meta_protocol (str: "http"): The protocol to use for the md server + check_tokens (boolean: False): Whether functions that take `token` + as an argument should check for the existance of that token and + fail immediately if it is not found. This is a good idea for + functions that take more time to complete, and might not fail + until the very end otherwise. + chunk_threshold (int: 1e9 / 4): The maximum size of a numpy array + that will be uploaded in one HTTP request. If you find that + your data requests are commonly timing out, try reducing this. + Default is 1e9 / 4, or a 0.25GiB. + suffix (str: "ocp"): The URL suffix to specify ndstore/microns. If + you aren't sure what to do with this, don't specify one. + """ + self._check_tokens = kwargs.get('check_tokens', False) + self._chunk_threshold = kwargs.get('chunk_threshold', 1E9 / 4) + self._ext = kwargs.get('suffix', DEFAULT_SUFFIX) + self._known_tokens = [] + self._user_token = user_token + + # Prepare meta url + self.meta_root = meta_root + if not self.meta_root.endswith('/'): + self.meta_root = self.meta_root + "/" + if self.meta_root.startswith('https'): + self.meta_root = self.meta_root[self.meta_root.index('://') + 3:] + self.meta_protocol = meta_protocol + + super(neurodata, self).__init__(hostname, protocol) + + # SECTION: + # Decorators + def _check_token(f): + @wraps(f) + def wrapped(self, *args, **kwargs): + if self._check_tokens: + if 'token' in kwargs: + token = kwargs['token'] + else: + token = args[0] + if token not in self._known_tokens: + if self.ping('{}/info/'.format(token)) != 200: + raise RemoteDataNotFoundError("Bad token {}".format( + token)) + else: + self._known_tokens.append(token) + return f(self, *args, **kwargs) + return wrapped + + # SECTION: + # Utilities + def ping(self, suffix='public_tokens/'): + """ + Return the status-code of the API (estimated using the public-tokens + lookup page). + + Arguments: + suffix (str : 'public_tokens/'): The url endpoint to check + + Returns: + int: status code + """ + return super(neurodata, self).ping(suffix) + + def url(self, suffix=""): + """ + Return a constructed URL, appending an optional suffix (uri path). + + Arguments: + suffix (str : ""): The suffix to append to the end of the URL + + Returns: + str: The complete URL + """ + return super(neurodata, self).url('{}/sd/'.format(self._ext) + suffix) + + def meta_url(self, suffix=""): + """ + Return a constructed URL, appending an optional suffix (uri path), + for the metadata server. (Should be temporary, until the LIMS shim + is fixed.) + + Arguments: + suffix (str : ""): The suffix to append to the end of the URL + + Returns: + str: The complete URL + """ + return self.meta_protocol + "://" + self.meta_root + suffix + + def __repr__(self): + """ + Return a string representation that can be used to reproduce this + instance. `eval(repr(this))` should return an identical copy. + + Arguments: + None + + Returns: + str: Representation of reproducible instance. + """ + return "ndio.remote.neurodata('{}', '{}')".format( + self.hostname, + self.protocol, + self.meta_url, + self.meta_protocol + ) + + # SECTION: + # Metadata + def get_public_tokens(self): + """ + Get a list of public tokens available on this server. + + Arguments: + None + + Returns: + str[]: list of public tokens + """ + r = getURL(self.url() + "public_tokens/") + return r.json() + + def get_public_datasets(self): + """ + NOTE: VERY SLOW! + Get a list of public datasets. Different than public tokens! + + Arguments: + None + + Returns: + str[]: list of public datasets + """ + return list(self.get_public_datasets_and_tokens().keys()) + + def get_public_datasets_and_tokens(self): + """ + NOTE: VERY SLOW! + Get a dictionary relating key:dataset to value:[tokens] that rely + on that dataset. + + Arguments: + None + + Returns: + dict: relating key:dataset to value:[tokens] + """ + datasets = {} + tokens = self.get_public_tokens() + for t in tokens: + dataset = self.get_token_dataset(t) + if dataset in datasets: + datasets[dataset].append(t) + else: + datasets[dataset] = [t] + return datasets + + @_check_token + def get_token_dataset(self, token): + """ + Get the dataset for a given token. + + Arguments: + token (str): The token to inspect + + Returns: + str: The name of the dataset + """ + return self.get_proj_info(token)['dataset']['description'] + + @_check_token + def get_proj_info(self, token): + """ + Return the project info for a given token. + + Arguments: + token (str): Token to return information for + + Returns: + JSON: representation of proj_info + """ + r = getURL(self.url() + "{}/info/".format(token)) + return r.json() + + @_check_token + def get_metadata(self, token): + """ + An alias for get_proj_info. + """ + return self.get_proj_info(token) + + @_check_token + def get_channels(self, token): + """ + Wraps get_proj_info to return a dictionary of just the channels of + a given project. + + Arguments: + token (str): Token to return channels for + + Returns: + JSON: dictionary of channels. + """ + return self.get_proj_info(token)['channels'] + + @_check_token + def get_image_size(self, token, resolution=0): + """ + Return the size of the volume (3D). Convenient for when you want + to download the entirety of a dataset. + + Arguments: + token (str): The token for which to find the dataset image bounds + resolution (int : 0): The resolution at which to get image bounds. + Defaults to 0, to get the largest area available. + + Returns: + int[3]: The size of the bounds. Should == get_volume.shape + + Raises: + RemoteDataNotFoundError: If the token is invalid, or if the + metadata at that resolution is unavailable in projinfo. + """ + info = self.get_proj_info(token) + res = str(resolution) + if res not in info['dataset']['imagesize']: + raise RemoteDataNotFoundError("Resolution " + res + + " is not available.") + return info['dataset']['imagesize'][str(resolution)] + + @_check_token + def set_metadata(self, token, data): + """ + Insert new metadata into the OCP metadata database. + + Arguments: + token (str): Token of the datum to set + data (str): A dictionary to insert as metadata. Include `secret`. + + Returns: + json: Info of the inserted ID (convenience) or an error message. + + Throws: + RemoteDataUploadError: If the token is already populated, or if + there is an issue with your specified `secret` key. + """ + req = requests.post(self.meta_url("metadata/ocp/set/" + token), + json=data) + + if req.status_code != 200: + raise RemoteDataUploadError( + "Could not upload metadata: " + req.json()['message'] + ) + return req.json() + + @_check_token + def get_subvolumes(self, token): + """ + Return a list of subvolumes taken from LIMS, if available. + + Arguments: + token (str): The token to read from in LIMS + + Returns: + dict: or None if unavailable + """ + md = self.get_metadata(token)['metadata'] + if 'subvolumes' in md: + return md['subvolumes'] + else: + return None + + @_check_token + def add_subvolume(self, token, channel, secret, + x_start, x_stop, + y_start, y_stop, + z_start, z_stop, + resolution, title, notes): + """ + Adds a new subvolume to a token/channel. + + Arguments: + token (str): The token to write to in LIMS + channel (str): Channel to add in the subvolume. Can be `None` + x_start (int): Start in x dimension + x_stop (int): Stop in x dimension + y_start (int): Start in y dimension + y_stop (int): Stop in y dimension + z_start (int): Start in z dimension + z_stop (int): Stop in z dimension + resolution (int): The resolution at which this subvolume is seen + title (str): The title to set for the subvolume + notes (str): Optional extra thoughts on the subvolume + + Returns: + boolean: success + """ + md = self.get_metadata(token)['metadata'] + if 'subvolumes' in md: + subvols = md['subvolumes'] + else: + subvols = [] + + subvols.append({ + 'token': token, + 'channel': channel, + 'x_start': x_start, + 'x_stop': x_stop, + 'y_start': y_start, + 'y_stop': y_stop, + 'z_start': z_start, + 'z_stop': z_stop, + 'resolution': resolution, + 'title': title, + 'notes': notes + }) + + return self.set_metadata(token, { + 'secret': secret, + 'subvolumes': subvols + }) + + # SECTION: + # Data Download + @_check_token + def get_block_size(self, token, resolution=None): + """ + Gets the block-size for a given token at a given resolution. + + Arguments: + token (str): The token to inspect + resolution (int : None): The resolution at which to inspect data. + If none is specified, uses the minimum available. + + Returns: + int[3]: The xyz blocksize. + """ + cdims = self.get_metadata(token)['dataset']['cube_dimension'] + if resolution is None: + resolution = min(cdims.keys()) + return cdims[str(resolution)] + + @_check_token + def get_image_offset(self, token, resolution=0): + """ + Gets the image offset for a given token at a given resolution. For + instance, the `kasthuri11` dataset starts at (0, 0, 1), so its 1850th + slice is slice 1850, not 1849. When downloading a full dataset, the + result of this function should be your x/y/z starts. + + Arguments: + token (str): The token to inspect + resolution (int : 0): The resolution at which to gather the offset + + Returns: + int[3]: The origin of the dataset, as a list + """ + info = self.get_proj_info(token) + res = str(resolution) + if res not in info['dataset']['offset']: + raise RemoteDataNotFoundError("Resolution " + res + + " is not available.") + return info['dataset']['offset'][str(resolution)] + + @_check_token + def get_xy_slice(self, token, channel, + x_start, x_stop, + y_start, y_stop, + z_index, + resolution=0): + """ + Return a binary-encoded, decompressed 2d image. You should + specify a 'token' and 'channel' pair. For image data, users + should use the channel 'image.' + + Arguments: + token (str): Token to identify data to download + channel (str): Channel + resolution (int): Resolution level + Q_start (int):` The lower bound of dimension 'Q' + Q_stop (int): The upper bound of dimension 'Q' + z_index (int): The z-slice to image + + Returns: + str: binary image data + """ + vol = self.get_cutout(token, channel, x_start, x_stop, y_start, + y_stop, z_index, z_index + 1, resolution) + + vol = numpy.squeeze(vol) # 3D volume to 2D slice + + return vol + + @_check_token + def get_image(self, token, channel, + x_start, x_stop, + y_start, y_stop, + z_index, + resolution=0): + """ + Alias for the `get_xy_slice` function for backwards compatibility. + """ + return self.get_xy_slice(token, channel, + x_start, x_stop, + y_start, y_stop, + z_index, + resolution) + + @_check_token + def get_volume(self, token, channel, + x_start, x_stop, + y_start, y_stop, + z_start, z_stop, + resolution=1, + block_size=DEFAULT_BLOCK_SIZE, + neariso=False): + """ + Get a RAMONVolume volumetric cutout from the neurodata server. + + Arguments: + token (str): Token to identify data to download + channel (str): Channel + resolution (int): Resolution level + Q_start (int): The lower bound of dimension 'Q' + Q_stop (int): The upper bound of dimension 'Q' + block_size (int[3]): Block size of this dataset + neariso (bool : False): Passes the 'neariso' param to the cutout. + If you don't know what this means, ignore it! + + Returns: + ndio.ramon.RAMONVolume: Downloaded data. + """ + size = (x_stop - x_start) * (y_stop - y_start) * (z_stop - z_start) + volume = ramon.RAMONVolume() + volume.xyz_offset = [x_start, y_start, z_start] + volume.resolution = resolution + + volume.cutout = self.get_cutout(token, channel, x_start, + x_stop, y_start, y_stop, + z_start, z_stop, + resolution=resolution, + block_size=block_size, + neariso=neariso) + return volume + + @_check_token + def get_cutout(self, token, channel, + x_start, x_stop, + y_start, y_stop, + z_start, z_stop, + resolution=1, + block_size=DEFAULT_BLOCK_SIZE, + neariso=False): + """ + Get volumetric cutout data from the neurodata server. + + Arguments: + token (str): Token to identify data to download + channel (str): Channel + resolution (int): Resolution level + Q_start (int): The lower bound of dimension 'Q' + Q_stop (int): The upper bound of dimension 'Q' + block_size (int[3]): Block size of this dataset. If not provided, + ndio uses the metadata of this tokenchannel to set. If you find + that your downloads are timing out or otherwise failing, it may + be wise to start off by making this smaller. + neariso (bool : False): Passes the 'neariso' param to the cutout. + If you don't know what this means, ignore it! + + Returns: + numpy.ndarray: Downloaded data. + """ + if block_size is None: + # look up block size from metadata + block_size = self.get_block_size(token, resolution) + + origin = self.get_image_offset(token, resolution) + + # If z_stop - z_start is < 16, backend still pulls minimum 16 slices + if (z_stop - z_start) < 16: + z_slices = 16 + else: + z_slices = z_stop - z_start + + # Calculate size of the data to be downloaded. + size = (x_stop - x_start) * (y_stop - y_start) * z_slices * 4 + + # Switch which download function to use based on which libraries are + # available in this version of python. + if six.PY2: + dl_func = self._get_cutout_blosc_no_chunking + elif six.PY3: + dl_func = self._get_cutout_no_chunking + else: + raise ValueError("Invalid Python version.") + + if size < self._chunk_threshold: + vol = dl_func(token, channel, resolution, + x_start, x_stop, + y_start, y_stop, + z_start, z_stop, neariso=neariso) + vol = numpy.rollaxis(vol, 1) + vol = numpy.rollaxis(vol, 2) + return vol + else: + from ndio.utils.parallel import block_compute + blocks = block_compute(x_start, x_stop, + y_start, y_stop, + z_start, z_stop, + origin, block_size) + + vol = numpy.zeros(((z_stop - z_start), + (y_stop - y_start), + (x_stop - x_start))) + for b in blocks: + + data = dl_func(token, channel, resolution, + b[0][0], b[0][1], + b[1][0], b[1][1], + b[2][0], b[2][1], neariso=neariso) + + if b == blocks[0]: # first block + vol = numpy.zeros(((z_stop - z_start), + (y_stop - y_start), + (x_stop - x_start)), dtype=data.dtype) + + vol[b[2][0] - z_start: b[2][1] - z_start, + b[1][0] - y_start: b[1][1] - y_start, + b[0][0] - x_start: b[0][1] - x_start] = data + + vol = numpy.rollaxis(vol, 1) + vol = numpy.rollaxis(vol, 2) + return vol + + def _get_cutout_no_chunking(self, token, channel, resolution, + x_start, x_stop, y_start, y_stop, + z_start, z_stop, neariso=False): + url = self.url() + "{}/{}/hdf5/{}/{},{}/{},{}/{},{}/".format( + token, channel, resolution, + x_start, x_stop, + y_start, y_stop, + z_start, z_stop + ) + + if neariso: + url += "neariso/" + + req = getURL(url) + if req.status_code is not 200: + raise IOError("Bad server response for {}: {}: {}".format( + url, + req.status_code, + req.text)) + + with tempfile.NamedTemporaryFile() as tmpfile: + tmpfile.write(req.content) + tmpfile.seek(0) + h5file = h5py.File(tmpfile.name, "r") + return h5file.get(channel).get('CUTOUT')[:] + raise IOError("Failed to make tempfile.") + + def _get_cutout_blosc_no_chunking(self, token, channel, resolution, + x_start, x_stop, y_start, y_stop, + z_start, z_stop, neariso=False): + + url = self.url() + "{}/{}/blosc/{}/{},{}/{},{}/{},{}/".format( + token, channel, resolution, + x_start, x_stop, + y_start, y_stop, + z_start, z_stop + ) + + if neariso: + url += "neariso/" + + req = getURL(url) + if req.status_code is not 200: + raise IOError("Bad server response for {}: {}: {}".format( + url, + req.status_code, + req.text)) + + # This will need modification for >3D blocks + return blosc.unpack_array(req.content)[0] + + raise IOError("Failed to retrieve blosc cutout.") + + # SECTION: + # Data Upload + + @_check_token + def post_cutout(self, token, channel, + x_start, + y_start, + z_start, + data, + resolution=0): + """ + Post a cutout to the server. + + Arguments: + token (str) + channel (str) + x_start (int) + y_start (int) + z_start (int) + data (numpy.ndarray): A numpy array of data. Pass in (x, y, z) + resolution (int : 0): Resolution at which to insert the data + + Returns: + bool: True on success + + Raises: + RemoteDataUploadError: if there's an issue during upload. + """ + datatype = self.get_proj_info(token)['channels'][channel]['datatype'] + if data.dtype.name != datatype: + data = data.astype(datatype) + + data = numpy.rollaxis(data, 1) + data = numpy.rollaxis(data, 2) + + if six.PY3 or data.nbytes > 1.5e9: + ul_func = self._post_cutout_no_chunking_npz + else: + ul_func = self._post_cutout_no_chunking_blosc + + if data.size < self._chunk_threshold: + return ul_func(token, channel, x_start, + y_start, z_start, data, + resolution) + + return self._post_cutout_with_chunking(token, channel, + x_start, y_start, z_start, data, + resolution, ul_func) + + def _post_cutout_with_chunking(self, token, channel, x_start, + y_start, z_start, data, + resolution, ul_func): + # must chunk first + from ndio.utils.parallel import block_compute + blocks = block_compute(x_start, x_start + data.shape[2], + y_start, y_start + data.shape[1], + z_start, z_start + data.shape[0]) + for b in blocks: + # data coordinate relative to the size of the arra + subvol = data[b[2][0] - z_start: b[2][1] - z_start, + b[1][0] - y_start: b[1][1] - y_start, + b[0][0] - x_start: b[0][1] - x_start] + # upload the chunk: + # upload coordinate relative to x_start, y_start, z_start + ul_func(token, channel, b[0][0], + b[1][0], b[2][0], subvol, + resolution) + return True + + def _post_cutout_no_chunking_npz(self, token, channel, + x_start, y_start, z_start, + data, resolution): + + data = numpy.expand_dims(data, axis=0) + tempfile = BytesIO() + numpy.save(tempfile, data) + compressed = zlib.compress(tempfile.getvalue()) + + url = self.url("{}/{}/npz/{}/{},{}/{},{}/{},{}/".format( + token, channel, + resolution, + x_start, x_start + data.shape[3], + y_start, y_start + data.shape[2], + z_start, z_start + data.shape[1] + )) + + req = requests.post(url, data=compressed, headers={ + 'Content-Type': 'application/octet-stream' + }) + + if req.status_code is not 200: + raise RemoteDataUploadError(req.text) + else: + return True + + def _post_cutout_no_chunking_blosc(self, token, channel, + x_start, y_start, z_start, + data, resolution): + """ + Accepts data in zyx. !!! + """ + data = numpy.expand_dims(data, axis=0) + blosc_data = blosc.pack_array(data) + + url = self.url("{}/{}/blosc/{}/{},{}/{},{}/{},{}/".format( + token, channel, + resolution, + x_start, x_start + data.shape[3], + y_start, y_start + data.shape[2], + z_start, z_start + data.shape[1] + )) + req = requests.post(url, data=blosc_data, headers={ + 'Content-Type': 'application/octet-stream' + }) + + if req.status_code is not 200: + raise RemoteDataUploadError(req.text) + else: + return True + + # SECTION: + # RAMON Download + + @_check_token + def get_ramon_bounding_box(self, token, channel, r_id, resolution=0): + """ + Get the bounding box for a RAMON object (specified by ID). + + Arguments: + token (str): Project to use + channel (str): Channel to use + r_id (int): Which ID to get a bounding box + resolution (int : 0): The resolution at which to download + + Returns: + (x_start, x_stop, y_start, y_stop, z_start, z_stop): ints + """ + url = self.url('{}/{}/{}/boundingbox/{}/'.format(token, channel, + r_id, resolution)) + + r_id = str(r_id) + res = getURL(url) + + if res.status_code != 200: + rt = self.get_ramon_metadata(token, channel, r_id)[r_id]['type'] + if rt in ['neuron']: + raise ValueError("ID {} is of type '{}'".format(r_id, rt)) + raise RemoteDataNotFoundError("No such ID {}".format(r_id)) + + with tempfile.NamedTemporaryFile() as tmpfile: + tmpfile.write(res.content) + tmpfile.seek(0) + h5file = h5py.File(tmpfile.name, "r") + origin = h5file["{}/XYZOFFSET".format(r_id)][()] + size = h5file["{}/XYZDIMENSION".format(r_id)][()] + return (origin[0], origin[0] + size[0], + origin[1], origin[1] + size[1], + origin[2], origin[2] + size[2]) + + @_check_token + def get_ramon_ids(self, token, channel, ramon_type=None): + """ + Return a list of all IDs available for download from this token and + channel. + + Arguments: + token (str): Project to use + channel (str): Channel to use + ramon_type (int : None): Optional. If set, filters IDs and only + returns those of RAMON objects of the requested type. + + Returns: + int[]: A list of the ids of the returned RAMON objects + + Raises: + RemoteDataNotFoundError: If the channel or token is not found + """ + url = self.url("{}/{}/query/".format(token, channel)) + if ramon_type is not None: + # User is requesting a specific ramon_type. + if type(ramon_type) is not int: + ramon_type = ramon.AnnotationType.get_int(ramon_type) + url += "type/{}/".format(str(ramon_type)) + + req = getURL(url) + + if req.status_code is not 200: + raise RemoteDataNotFoundError('No query results for token {}.' + .format(token)) + else: + with tempfile.NamedTemporaryFile() as tmpfile: + tmpfile.write(req.content) + tmpfile.seek(0) + h5file = h5py.File(tmpfile.name, "r") + if 'ANNOIDS' not in h5file: + return [] + return [i for i in h5file['ANNOIDS']] + raise IOError("Could not successfully mock HDF5 file for parsing.") + + @_check_token + def get_ramon(self, token, channel, ids, resolution=0, + include_cutout=False, sieve=None, batch_size=100): + """ + Download a RAMON object by ID. + + Arguments: + token (str): Project to use + channel (str): The channel to use + ids (int, str, int[], str[]): The IDs of a RAMON object to gather. + Can be int (3), string ("3"), int[] ([3, 4, 5]), or string + (["3", "4", "5"]). + resolution (int : None): Resolution. Defaults to the most granular + resolution (0 for now) + include_cutout (bool : False): If True, r.cutout is populated + sieve (function : None): A function that accepts a single ramon + and returns True or False depending on whether you want that + ramon object to be included in your response or not. + For example, + ``` + def is_even_id(ramon): + return ramon.id % 2 == 0 + ``` + You can then pass this to get_ramon like this: + ``` + ndio.remote.neurodata.get_ramon( . . . , sieve=is_even_id) + ``` + batch_size (int : 100): The amount of RAMON objects to download at + a time. If this is greater than 100, we anticipate things going + very poorly for you. So if you set it <100, ndio will use it. + If >=100, set it to 100. + + Returns: + ndio.ramon.RAMON[]: A list of returned RAMON objects. + + Raises: + RemoteDataNotFoundError: If the requested ids cannot be found. + """ + b_size = min(100, batch_size) + + _return_first_only = False + if type(ids) is not list: + _return_first_only = True + ids = [ids] + ids = [str(i) for i in ids] + + rs = [] + id_batches = [ids[i:i + b_size] for i in range(0, len(ids), b_size)] + for batch in id_batches: + rs.extend(self._get_ramon_batch(token, channel, batch, resolution)) + + rs = self._filter_ramon(rs, sieve) + + if include_cutout: + rs = [self._add_ramon_cutout(token, channel, r, resolution) + for r in rs] + + if _return_first_only: + return rs[0] + + return sorted(rs, key=lambda x: ids.index(x.id)) + + def _filter_ramon(self, rs, sieve): + if sieve is not None: + return [r for r in rs if sieve(r)] + return rs + + def _add_ramon_cutout(self, token, channel, ramon, resolution): + origin = ramon.xyz_offset + # Get the bounding box (cube-aligned) + bbox = self.get_ramon_bounding_box(token, channel, + ramon.id, resolution=resolution) + # Get the cutout (cube-aligned) + cutout = self.get_cutout(token, channel, + *bbox, resolution=resolution) + cutout[cutout != int(ramon.id)] = 0 + + # Compute upper offset and crop + bounds = numpy.argwhere(cutout) + mins = [min([i[dim] for i in bounds]) for dim in range(3)] + maxs = [max([i[dim] for i in bounds]) for dim in range(3)] + + ramon.cutout = cutout[ + mins[0]:maxs[0], + mins[1]:maxs[1], + mins[2]:maxs[2] + ] + + return ramon + + def _get_ramon_batch(self, token, channel, ids, resolution): + ids = [str(i) for i in ids] + url = self.url("{}/{}/{}/json/".format(token, channel, ",".join(ids))) + req = getURL(url) + + if req.status_code is not 200: + raise RemoteDataNotFoundError('No data for id {}.'.format(ids)) + else: + return ramon.from_json(req.json()) + + @_check_token + def get_ramon_metadata(self, token, channel, anno_id): + """ + Download a RAMON object by ID. `anno_id` can be a string `"123"`, an + int `123`, an array of ints `[123, 234, 345]`, an array of strings + `["123", "234", "345"]`, or a comma-separated string list + `"123,234,345"`. + + Arguments: + token (str): Project to use + channel (str): The channel to use + anno_id: An int, a str, or a list of ids to gather + + Returns: + JSON. If you pass a single id in str or int, returns a single datum + If you pass a list of int or str or a comma-separated string, will + return a dict with keys from the list and the values are the JSON + returned from the server. + + Raises: + RemoteDataNotFoundError: If the data cannot be found on the Remote + """ + if type(anno_id) in [int, numpy.uint32]: + # there's just one ID to download + return self._get_single_ramon_metadata(token, channel, + str(anno_id)) + elif type(anno_id) is str: + # either "id" or "id,id,id": + if (len(anno_id.split(',')) > 1): + results = {} + for i in anno_id.split(','): + results[i] = self._get_single_ramon_metadata( + token, channel, anno_id.strip() + ) + return results + else: + # "id" + return self._get_single_ramon_metadata(token, channel, + anno_id.strip()) + elif type(anno_id) is list: + # [id, id] or ['id', 'id'] + results = [] + for i in anno_id: + results.append(self._get_single_ramon_metadata(token, channel, + str(i))) + return results + + def _get_single_ramon_metadata(self, token, channel, anno_id): + req = getURL(self.url() + + "{}/{}/{}/json/".format(token, channel, anno_id)) + if req.status_code is not 200: + raise RemoteDataNotFoundError('No data for id {}.'.format(anno_id)) + return req.json() + + @_check_token + def delete_ramon(self, token, channel, anno): + """ + Deletes an annotation from the server. Probably you should be careful + with this function, it seems dangerous. + + Arguments: + token (str): The token to inspect + channel (str): The channel to inspect + anno (int OR list(int) OR RAMON): The annotation to delete. If a + RAMON object is supplied, the remote annotation will be deleted + by an ID lookup. If an int is supplied, the annotation will be + deleted for that ID. If a list of ints are provided, they will + all be deleted. + + Returns: + bool: Success + """ + if type(anno) is int: + a = anno + if type(anno) is str: + a = int(anno) + if type(anno) is list: + a = ",".join(anno) + else: + a = anno.id + + req = requests.delete(self.url("{}/{}/{}/".format(token, channel, a))) + if req.status_code is not 200: + raise RemoteDataNotFoundError("Could not delete id {}: {}" + .format(a, req.text)) + else: + return True + + @_check_token + def post_ramon(self, token, channel, r, batch_size=100): + """ + Posts a RAMON object to the Remote. + + Arguments: + token (str): Project to use + channel (str): The channel to use + r (RAMON or RAMON[]): The annotation(s) to upload + batch_size (int : 100): The number of RAMONs to post simultaneously + at maximum in one file. If len(r) > batch_size, the batch will + be split and uploaded automatically. Must be less than 100. + + Returns: + bool: Success = True + + Throws: + RemoteDataUploadError: if something goes wrong + """ + # Max out batch-size at 100. + b_size = min(100, batch_size) + + # Coerce incoming IDs to a list. + if type(r) is not list: + r = [r] + + # If there are too many to fit in one batch, split here and call this + # function recursively. + if len(r) > batch_size: + batches = [r[i:i + b_size] for i in range(0, len(r), b_size)] + for batch in batches: + self.post_ramon(token, channel, batch, b_size) + return + + with tempfile.NamedTemporaryFile(delete=False) as tmpfile: + for i in r: + tmpfile = ramon.to_hdf5(i, tmpfile) + + url = self.url("{}/{}/overwrite/".format(token, channel)) + req = urllib2.Request(url, tmpfile.read()) + res = urllib2.urlopen(req) + + if res.code != 200: + raise RemoteDataUploadError('[{}] Could not upload {}' + .format(res.code, str(r))) + + rets = res.read() + if six.PY3: + rets = rets.decode() + return_ids = [int(rid) for rid in rets.split(',')] + + # Now post the cutout separately: + for ri in r: + if 'cutout' in dir(ri) and ri.cutout is not None: + orig = ri.xyz_offset + self.post_cutout(token, channel, + orig[0], orig[1], orig[2], + ri.cutout, resolution=r.resolution) + return return_ids + return True + + # SECTION: + # ID Manipulation + + @_check_token + def reserve_ids(self, token, channel, quantity): + """ + Requests a list of next-available-IDs from the server. + + Arguments: + quantity (int): The number of IDs to reserve + + Returns: + int[quantity]: List of IDs you've been granted + """ + quantity = str(quantity) + url = self.url("{}/{}/reserve/{}/".format(token, channel, quantity)) + req = getURL(url) + if req.status_code is not 200: + raise RemoteDataNotFoundError('Invalid req: ' + req.status_code) + out = req.json() + return [out[0] + i for i in range(out[1])] + + @_check_token + def merge_ids(self, token, channel, ids, delete=False): + """ + Call the restful endpoint to merge two RAMON objects into one. + + Arguments: + token (str): The token to inspect + channel (str): The channel to inspect + ids (int[]): the list of the IDs to merge + delete (bool : False): Whether to delete after merging. + + Returns: + json: The ID as returned by ndstore + """ + req = getURL(self.url() + "/merge/{}/" + .format(','.join([str(i) for i in ids]))) + if req.status_code is not 200: + raise RemoteDataUploadError('Could not merge ids {}'.format( + ','.join([str(i) for i in ids]))) + if delete: + self.delete_ramon(token, channel, ids[1:]) + return True + + # SECTION: + # Channels + + def _check_channel(self, channel): + for c in channel: + if not c.isalnum(): + raise ValueError( + "Channel name cannot contain character {}.".format(c) + ) + return True + + @_check_token + def create_channel(self, token, name, channel_type, dtype, readonly): + """ + Create a new channel on the Remote, using channel_data. + + Arguments: + token (str): The token the new channel should be added to + name (str): The name of the channel to add + channel_type (str): Type of the channel (e.g. `neurodata.IMAGE`) + dtype (str): The datatype of the channel's data (e.g. `uint8`) + readonly (bool): Can others write to this channel? + + Returns: + bool: `True` if successful, `False` otherwise. + + Raises: + ValueError: If your args were bad :( + RemoteDataUploadError: If the channel data is valid but upload + fails for some other reason. + """ + self._check_channel(name) + + if channel_type not in ['image', 'annotation']: + raise ValueError('Channel type must be ' + + 'neurodata.IMAGE or neurodata.ANNOTATION.') + + if readonly * 1 not in [0, 1]: + raise ValueError("readonly must be 0 (False) or 1 (True).") + + # Good job! You supplied very nice arguments. + req = requests.post(self.url("{}/createChannel/".format(token)), json={ + "channels": { + name: { + "channel_name": name, + "channel_type": channel_type, + "datatype": dtype, + "readonly": readonly * 1 + } + } + }) + + if req.status_code is not 200: + raise RemoteDataUploadError('Could not upload {}'.format(req.text)) + else: + return True + + @_check_token + def delete_channel(self, token, name): + """ + Delete an existing channel on the Remote. Be careful! + + Arguments: + token (str): The token the new channel should be deleted from + name (str): The name of the channel to delete + + Returns: + bool: True if successful, False otherwise. + + Raises: + RemoteDataUploadError: If the upload fails for some reason. + """ + req = requests.post(self.url("{}/deleteChannel/".format(token)), json={ + "channels": [name] + }) + + if req.status_code is not 200: + raise RemoteDataUploadError('Could not delete {}'.format(req.text)) + if req.content == "SUCCESS": + return True + else: + return False + + # Propagation + + @_check_token + def propagate(self, token, channel): + """ + Kick off the propagate function on the remote server. + + Arguments: + token (str): The token to propagate + channel (str): The channel to propagate + + Returns: + boolean: Success + """ + if self.get_propagate_status(token, channel) != u'0': + return + url = self.url('{}/{}/setPropagate/1/'.format(token, channel)) + req = getURL(url) + if req.status_code is not 200: + raise RemoteDataUploadError('Propagate fail: {}'.format(req.text)) + return True + + @_check_token + def get_propagate_status(self, token, channel): + """ + Get the propagate status for a token/channel pair. + + Arguments: + token (str): The token to check + channel (str): The channel to check + + Returns: + str: The status code + """ + url = self.url('{}/{}/getPropagate/'.format(token, channel)) + req = getURL(url) + if req.status_code is not 200: + raise ValueError('Bad pair: {}/{}'.format(token, channel)) + return req.text + + def getURL(self, url): + """ + Get the propagate status for a token/channel pair. + + Arguments: + url (str): The url make a get to + + Returns: + obj: The response object + """ + try: + req = requests.get(url, + headers={ + 'Authorization': 'Token {}'.format(token) + }, + verify=True) + return req + except requests.exceptions.ConnectionError as e: + if str(e) == "403 Client Error: Forbidden": + raise ValueError('Access Denied') + else: + raise From 63671ddd77775d53ac836c98980a247f0172f431 Mon Sep 17 00:00:00 2001 From: Jinyong Shin Date: Thu, 23 Feb 2017 17:06:58 -0500 Subject: [PATCH 02/42] without indent --- ndio/remote/neurodata.py | 2497 +++++++++++++++++++------------------- 1 file changed, 1248 insertions(+), 1249 deletions(-) diff --git a/ndio/remote/neurodata.py b/ndio/remote/neurodata.py index 6ee9182..604f815 100644 --- a/ndio/remote/neurodata.py +++ b/ndio/remote/neurodata.py @@ -18,9 +18,9 @@ from functools import wraps try: - import urllib.request as urllib2 + import urllib.request as urllib2 except ImportError: - import urllib2 + import urllib2 DEFAULT_HOSTNAME = "openconnecto.me" DEFAULT_SUFFIX = "nd" @@ -29,1250 +29,1249 @@ class neurodata(Remote): - """ - The NeuroData remote, for interfacing with ndstore, ndlims, and friends. - """ - - # SECTION: - # Enumerables - IMAGE = IMG = 'image' - ANNOTATION = ANNO = 'annotation' - - def __init__(self, - user_token, - hostname=DEFAULT_HOSTNAME, - protocol=DEFAULT_PROTOCOL, - meta_root="http://lims.neurodata.io/", - meta_protocol=DEFAULT_PROTOCOL, **kwargs): - """ - Initializer for the neurodata remote class. - - Arguments: - hostname (str: "openconnecto.me"): The hostname to connect to - protocol (str: "http"): The protocol (http or https) to use - meta_root (str: "http://lims.neurodata.io/"): The metadata server - meta_protocol (str: "http"): The protocol to use for the md server - check_tokens (boolean: False): Whether functions that take `token` - as an argument should check for the existance of that token and - fail immediately if it is not found. This is a good idea for - functions that take more time to complete, and might not fail - until the very end otherwise. - chunk_threshold (int: 1e9 / 4): The maximum size of a numpy array - that will be uploaded in one HTTP request. If you find that - your data requests are commonly timing out, try reducing this. - Default is 1e9 / 4, or a 0.25GiB. - suffix (str: "ocp"): The URL suffix to specify ndstore/microns. If - you aren't sure what to do with this, don't specify one. - """ - self._check_tokens = kwargs.get('check_tokens', False) - self._chunk_threshold = kwargs.get('chunk_threshold', 1E9 / 4) - self._ext = kwargs.get('suffix', DEFAULT_SUFFIX) - self._known_tokens = [] - self._user_token = user_token - - # Prepare meta url - self.meta_root = meta_root - if not self.meta_root.endswith('/'): - self.meta_root = self.meta_root + "/" - if self.meta_root.startswith('https'): - self.meta_root = self.meta_root[self.meta_root.index('://') + 3:] - self.meta_protocol = meta_protocol - - super(neurodata, self).__init__(hostname, protocol) - - # SECTION: - # Decorators - def _check_token(f): - @wraps(f) - def wrapped(self, *args, **kwargs): - if self._check_tokens: - if 'token' in kwargs: - token = kwargs['token'] - else: - token = args[0] - if token not in self._known_tokens: - if self.ping('{}/info/'.format(token)) != 200: - raise RemoteDataNotFoundError("Bad token {}".format( - token)) - else: - self._known_tokens.append(token) - return f(self, *args, **kwargs) - return wrapped - - # SECTION: - # Utilities - def ping(self, suffix='public_tokens/'): - """ - Return the status-code of the API (estimated using the public-tokens - lookup page). - - Arguments: - suffix (str : 'public_tokens/'): The url endpoint to check - - Returns: - int: status code - """ - return super(neurodata, self).ping(suffix) - - def url(self, suffix=""): - """ - Return a constructed URL, appending an optional suffix (uri path). - - Arguments: - suffix (str : ""): The suffix to append to the end of the URL - - Returns: - str: The complete URL - """ - return super(neurodata, self).url('{}/sd/'.format(self._ext) + suffix) - - def meta_url(self, suffix=""): - """ - Return a constructed URL, appending an optional suffix (uri path), - for the metadata server. (Should be temporary, until the LIMS shim - is fixed.) - - Arguments: - suffix (str : ""): The suffix to append to the end of the URL - - Returns: - str: The complete URL - """ - return self.meta_protocol + "://" + self.meta_root + suffix - - def __repr__(self): - """ - Return a string representation that can be used to reproduce this - instance. `eval(repr(this))` should return an identical copy. - - Arguments: - None - - Returns: - str: Representation of reproducible instance. - """ - return "ndio.remote.neurodata('{}', '{}')".format( - self.hostname, - self.protocol, - self.meta_url, - self.meta_protocol - ) - - # SECTION: - # Metadata - def get_public_tokens(self): - """ - Get a list of public tokens available on this server. - - Arguments: - None - - Returns: - str[]: list of public tokens - """ - r = getURL(self.url() + "public_tokens/") - return r.json() - - def get_public_datasets(self): - """ - NOTE: VERY SLOW! - Get a list of public datasets. Different than public tokens! - - Arguments: - None - - Returns: - str[]: list of public datasets - """ - return list(self.get_public_datasets_and_tokens().keys()) - - def get_public_datasets_and_tokens(self): - """ - NOTE: VERY SLOW! - Get a dictionary relating key:dataset to value:[tokens] that rely - on that dataset. - - Arguments: - None - - Returns: - dict: relating key:dataset to value:[tokens] - """ - datasets = {} - tokens = self.get_public_tokens() - for t in tokens: - dataset = self.get_token_dataset(t) - if dataset in datasets: - datasets[dataset].append(t) - else: - datasets[dataset] = [t] - return datasets - - @_check_token - def get_token_dataset(self, token): - """ - Get the dataset for a given token. - - Arguments: - token (str): The token to inspect - - Returns: - str: The name of the dataset - """ - return self.get_proj_info(token)['dataset']['description'] - - @_check_token - def get_proj_info(self, token): - """ - Return the project info for a given token. - - Arguments: - token (str): Token to return information for - - Returns: - JSON: representation of proj_info - """ - r = getURL(self.url() + "{}/info/".format(token)) - return r.json() - - @_check_token - def get_metadata(self, token): - """ - An alias for get_proj_info. - """ - return self.get_proj_info(token) - - @_check_token - def get_channels(self, token): - """ - Wraps get_proj_info to return a dictionary of just the channels of - a given project. - - Arguments: - token (str): Token to return channels for - - Returns: - JSON: dictionary of channels. - """ - return self.get_proj_info(token)['channels'] - - @_check_token - def get_image_size(self, token, resolution=0): - """ - Return the size of the volume (3D). Convenient for when you want - to download the entirety of a dataset. - - Arguments: - token (str): The token for which to find the dataset image bounds - resolution (int : 0): The resolution at which to get image bounds. - Defaults to 0, to get the largest area available. - - Returns: - int[3]: The size of the bounds. Should == get_volume.shape - - Raises: - RemoteDataNotFoundError: If the token is invalid, or if the - metadata at that resolution is unavailable in projinfo. - """ - info = self.get_proj_info(token) - res = str(resolution) - if res not in info['dataset']['imagesize']: - raise RemoteDataNotFoundError("Resolution " + res + - " is not available.") - return info['dataset']['imagesize'][str(resolution)] - - @_check_token - def set_metadata(self, token, data): - """ - Insert new metadata into the OCP metadata database. - - Arguments: - token (str): Token of the datum to set - data (str): A dictionary to insert as metadata. Include `secret`. - - Returns: - json: Info of the inserted ID (convenience) or an error message. - - Throws: - RemoteDataUploadError: If the token is already populated, or if - there is an issue with your specified `secret` key. - """ - req = requests.post(self.meta_url("metadata/ocp/set/" + token), - json=data) - - if req.status_code != 200: - raise RemoteDataUploadError( - "Could not upload metadata: " + req.json()['message'] - ) - return req.json() - - @_check_token - def get_subvolumes(self, token): - """ - Return a list of subvolumes taken from LIMS, if available. - - Arguments: - token (str): The token to read from in LIMS - - Returns: - dict: or None if unavailable - """ - md = self.get_metadata(token)['metadata'] - if 'subvolumes' in md: - return md['subvolumes'] - else: - return None - - @_check_token - def add_subvolume(self, token, channel, secret, - x_start, x_stop, - y_start, y_stop, - z_start, z_stop, - resolution, title, notes): - """ - Adds a new subvolume to a token/channel. - - Arguments: - token (str): The token to write to in LIMS - channel (str): Channel to add in the subvolume. Can be `None` - x_start (int): Start in x dimension - x_stop (int): Stop in x dimension - y_start (int): Start in y dimension - y_stop (int): Stop in y dimension - z_start (int): Start in z dimension - z_stop (int): Stop in z dimension - resolution (int): The resolution at which this subvolume is seen - title (str): The title to set for the subvolume - notes (str): Optional extra thoughts on the subvolume - - Returns: - boolean: success - """ - md = self.get_metadata(token)['metadata'] - if 'subvolumes' in md: - subvols = md['subvolumes'] - else: - subvols = [] - - subvols.append({ - 'token': token, - 'channel': channel, - 'x_start': x_start, - 'x_stop': x_stop, - 'y_start': y_start, - 'y_stop': y_stop, - 'z_start': z_start, - 'z_stop': z_stop, - 'resolution': resolution, - 'title': title, - 'notes': notes - }) - - return self.set_metadata(token, { - 'secret': secret, - 'subvolumes': subvols - }) - - # SECTION: - # Data Download - @_check_token - def get_block_size(self, token, resolution=None): - """ - Gets the block-size for a given token at a given resolution. - - Arguments: - token (str): The token to inspect - resolution (int : None): The resolution at which to inspect data. - If none is specified, uses the minimum available. - - Returns: - int[3]: The xyz blocksize. - """ - cdims = self.get_metadata(token)['dataset']['cube_dimension'] - if resolution is None: - resolution = min(cdims.keys()) - return cdims[str(resolution)] - - @_check_token - def get_image_offset(self, token, resolution=0): - """ - Gets the image offset for a given token at a given resolution. For - instance, the `kasthuri11` dataset starts at (0, 0, 1), so its 1850th - slice is slice 1850, not 1849. When downloading a full dataset, the - result of this function should be your x/y/z starts. - - Arguments: - token (str): The token to inspect - resolution (int : 0): The resolution at which to gather the offset - - Returns: - int[3]: The origin of the dataset, as a list - """ - info = self.get_proj_info(token) - res = str(resolution) - if res not in info['dataset']['offset']: - raise RemoteDataNotFoundError("Resolution " + res + - " is not available.") - return info['dataset']['offset'][str(resolution)] - - @_check_token - def get_xy_slice(self, token, channel, - x_start, x_stop, - y_start, y_stop, - z_index, - resolution=0): - """ - Return a binary-encoded, decompressed 2d image. You should - specify a 'token' and 'channel' pair. For image data, users - should use the channel 'image.' - - Arguments: - token (str): Token to identify data to download - channel (str): Channel - resolution (int): Resolution level - Q_start (int):` The lower bound of dimension 'Q' - Q_stop (int): The upper bound of dimension 'Q' - z_index (int): The z-slice to image - - Returns: - str: binary image data - """ - vol = self.get_cutout(token, channel, x_start, x_stop, y_start, - y_stop, z_index, z_index + 1, resolution) - - vol = numpy.squeeze(vol) # 3D volume to 2D slice - - return vol - - @_check_token - def get_image(self, token, channel, - x_start, x_stop, - y_start, y_stop, - z_index, - resolution=0): - """ - Alias for the `get_xy_slice` function for backwards compatibility. - """ - return self.get_xy_slice(token, channel, - x_start, x_stop, - y_start, y_stop, - z_index, - resolution) - - @_check_token - def get_volume(self, token, channel, - x_start, x_stop, - y_start, y_stop, - z_start, z_stop, - resolution=1, - block_size=DEFAULT_BLOCK_SIZE, - neariso=False): - """ - Get a RAMONVolume volumetric cutout from the neurodata server. - - Arguments: - token (str): Token to identify data to download - channel (str): Channel - resolution (int): Resolution level - Q_start (int): The lower bound of dimension 'Q' - Q_stop (int): The upper bound of dimension 'Q' - block_size (int[3]): Block size of this dataset - neariso (bool : False): Passes the 'neariso' param to the cutout. - If you don't know what this means, ignore it! - - Returns: - ndio.ramon.RAMONVolume: Downloaded data. - """ - size = (x_stop - x_start) * (y_stop - y_start) * (z_stop - z_start) - volume = ramon.RAMONVolume() - volume.xyz_offset = [x_start, y_start, z_start] - volume.resolution = resolution - - volume.cutout = self.get_cutout(token, channel, x_start, - x_stop, y_start, y_stop, - z_start, z_stop, - resolution=resolution, - block_size=block_size, - neariso=neariso) - return volume - - @_check_token - def get_cutout(self, token, channel, - x_start, x_stop, - y_start, y_stop, - z_start, z_stop, - resolution=1, - block_size=DEFAULT_BLOCK_SIZE, - neariso=False): - """ - Get volumetric cutout data from the neurodata server. - - Arguments: - token (str): Token to identify data to download - channel (str): Channel - resolution (int): Resolution level - Q_start (int): The lower bound of dimension 'Q' - Q_stop (int): The upper bound of dimension 'Q' - block_size (int[3]): Block size of this dataset. If not provided, - ndio uses the metadata of this tokenchannel to set. If you find - that your downloads are timing out or otherwise failing, it may - be wise to start off by making this smaller. - neariso (bool : False): Passes the 'neariso' param to the cutout. - If you don't know what this means, ignore it! - - Returns: - numpy.ndarray: Downloaded data. - """ - if block_size is None: - # look up block size from metadata - block_size = self.get_block_size(token, resolution) - - origin = self.get_image_offset(token, resolution) - - # If z_stop - z_start is < 16, backend still pulls minimum 16 slices - if (z_stop - z_start) < 16: - z_slices = 16 - else: - z_slices = z_stop - z_start - - # Calculate size of the data to be downloaded. - size = (x_stop - x_start) * (y_stop - y_start) * z_slices * 4 - - # Switch which download function to use based on which libraries are - # available in this version of python. - if six.PY2: - dl_func = self._get_cutout_blosc_no_chunking - elif six.PY3: - dl_func = self._get_cutout_no_chunking - else: - raise ValueError("Invalid Python version.") - - if size < self._chunk_threshold: - vol = dl_func(token, channel, resolution, - x_start, x_stop, - y_start, y_stop, - z_start, z_stop, neariso=neariso) - vol = numpy.rollaxis(vol, 1) - vol = numpy.rollaxis(vol, 2) - return vol - else: - from ndio.utils.parallel import block_compute - blocks = block_compute(x_start, x_stop, - y_start, y_stop, - z_start, z_stop, - origin, block_size) - - vol = numpy.zeros(((z_stop - z_start), - (y_stop - y_start), - (x_stop - x_start))) - for b in blocks: - - data = dl_func(token, channel, resolution, - b[0][0], b[0][1], - b[1][0], b[1][1], - b[2][0], b[2][1], neariso=neariso) - - if b == blocks[0]: # first block - vol = numpy.zeros(((z_stop - z_start), - (y_stop - y_start), - (x_stop - x_start)), dtype=data.dtype) - - vol[b[2][0] - z_start: b[2][1] - z_start, - b[1][0] - y_start: b[1][1] - y_start, - b[0][0] - x_start: b[0][1] - x_start] = data - - vol = numpy.rollaxis(vol, 1) - vol = numpy.rollaxis(vol, 2) - return vol - - def _get_cutout_no_chunking(self, token, channel, resolution, - x_start, x_stop, y_start, y_stop, - z_start, z_stop, neariso=False): - url = self.url() + "{}/{}/hdf5/{}/{},{}/{},{}/{},{}/".format( - token, channel, resolution, - x_start, x_stop, - y_start, y_stop, - z_start, z_stop - ) - - if neariso: - url += "neariso/" - - req = getURL(url) - if req.status_code is not 200: - raise IOError("Bad server response for {}: {}: {}".format( - url, - req.status_code, - req.text)) - - with tempfile.NamedTemporaryFile() as tmpfile: - tmpfile.write(req.content) - tmpfile.seek(0) - h5file = h5py.File(tmpfile.name, "r") - return h5file.get(channel).get('CUTOUT')[:] - raise IOError("Failed to make tempfile.") - - def _get_cutout_blosc_no_chunking(self, token, channel, resolution, - x_start, x_stop, y_start, y_stop, - z_start, z_stop, neariso=False): - - url = self.url() + "{}/{}/blosc/{}/{},{}/{},{}/{},{}/".format( - token, channel, resolution, - x_start, x_stop, - y_start, y_stop, - z_start, z_stop - ) - - if neariso: - url += "neariso/" - - req = getURL(url) - if req.status_code is not 200: - raise IOError("Bad server response for {}: {}: {}".format( - url, - req.status_code, - req.text)) - - # This will need modification for >3D blocks - return blosc.unpack_array(req.content)[0] - - raise IOError("Failed to retrieve blosc cutout.") - - # SECTION: - # Data Upload - - @_check_token - def post_cutout(self, token, channel, - x_start, - y_start, - z_start, - data, - resolution=0): - """ - Post a cutout to the server. - - Arguments: - token (str) - channel (str) - x_start (int) - y_start (int) - z_start (int) - data (numpy.ndarray): A numpy array of data. Pass in (x, y, z) - resolution (int : 0): Resolution at which to insert the data - - Returns: - bool: True on success - - Raises: - RemoteDataUploadError: if there's an issue during upload. - """ - datatype = self.get_proj_info(token)['channels'][channel]['datatype'] - if data.dtype.name != datatype: - data = data.astype(datatype) - - data = numpy.rollaxis(data, 1) - data = numpy.rollaxis(data, 2) - - if six.PY3 or data.nbytes > 1.5e9: - ul_func = self._post_cutout_no_chunking_npz - else: - ul_func = self._post_cutout_no_chunking_blosc - - if data.size < self._chunk_threshold: - return ul_func(token, channel, x_start, - y_start, z_start, data, - resolution) - - return self._post_cutout_with_chunking(token, channel, - x_start, y_start, z_start, data, - resolution, ul_func) - - def _post_cutout_with_chunking(self, token, channel, x_start, - y_start, z_start, data, - resolution, ul_func): - # must chunk first - from ndio.utils.parallel import block_compute - blocks = block_compute(x_start, x_start + data.shape[2], - y_start, y_start + data.shape[1], - z_start, z_start + data.shape[0]) - for b in blocks: - # data coordinate relative to the size of the arra - subvol = data[b[2][0] - z_start: b[2][1] - z_start, - b[1][0] - y_start: b[1][1] - y_start, - b[0][0] - x_start: b[0][1] - x_start] - # upload the chunk: - # upload coordinate relative to x_start, y_start, z_start - ul_func(token, channel, b[0][0], - b[1][0], b[2][0], subvol, - resolution) - return True - - def _post_cutout_no_chunking_npz(self, token, channel, - x_start, y_start, z_start, - data, resolution): - - data = numpy.expand_dims(data, axis=0) - tempfile = BytesIO() - numpy.save(tempfile, data) - compressed = zlib.compress(tempfile.getvalue()) - - url = self.url("{}/{}/npz/{}/{},{}/{},{}/{},{}/".format( - token, channel, - resolution, - x_start, x_start + data.shape[3], - y_start, y_start + data.shape[2], - z_start, z_start + data.shape[1] - )) - - req = requests.post(url, data=compressed, headers={ - 'Content-Type': 'application/octet-stream' - }) - - if req.status_code is not 200: - raise RemoteDataUploadError(req.text) - else: - return True - - def _post_cutout_no_chunking_blosc(self, token, channel, - x_start, y_start, z_start, - data, resolution): - """ - Accepts data in zyx. !!! - """ - data = numpy.expand_dims(data, axis=0) - blosc_data = blosc.pack_array(data) - - url = self.url("{}/{}/blosc/{}/{},{}/{},{}/{},{}/".format( - token, channel, - resolution, - x_start, x_start + data.shape[3], - y_start, y_start + data.shape[2], - z_start, z_start + data.shape[1] - )) - req = requests.post(url, data=blosc_data, headers={ - 'Content-Type': 'application/octet-stream' - }) - - if req.status_code is not 200: - raise RemoteDataUploadError(req.text) - else: - return True - - # SECTION: - # RAMON Download - - @_check_token - def get_ramon_bounding_box(self, token, channel, r_id, resolution=0): - """ - Get the bounding box for a RAMON object (specified by ID). - - Arguments: - token (str): Project to use - channel (str): Channel to use - r_id (int): Which ID to get a bounding box - resolution (int : 0): The resolution at which to download - - Returns: - (x_start, x_stop, y_start, y_stop, z_start, z_stop): ints - """ - url = self.url('{}/{}/{}/boundingbox/{}/'.format(token, channel, - r_id, resolution)) - - r_id = str(r_id) - res = getURL(url) - - if res.status_code != 200: - rt = self.get_ramon_metadata(token, channel, r_id)[r_id]['type'] - if rt in ['neuron']: - raise ValueError("ID {} is of type '{}'".format(r_id, rt)) - raise RemoteDataNotFoundError("No such ID {}".format(r_id)) - - with tempfile.NamedTemporaryFile() as tmpfile: - tmpfile.write(res.content) - tmpfile.seek(0) - h5file = h5py.File(tmpfile.name, "r") - origin = h5file["{}/XYZOFFSET".format(r_id)][()] - size = h5file["{}/XYZDIMENSION".format(r_id)][()] - return (origin[0], origin[0] + size[0], - origin[1], origin[1] + size[1], - origin[2], origin[2] + size[2]) - - @_check_token - def get_ramon_ids(self, token, channel, ramon_type=None): - """ - Return a list of all IDs available for download from this token and - channel. - - Arguments: - token (str): Project to use - channel (str): Channel to use - ramon_type (int : None): Optional. If set, filters IDs and only - returns those of RAMON objects of the requested type. - - Returns: - int[]: A list of the ids of the returned RAMON objects - - Raises: - RemoteDataNotFoundError: If the channel or token is not found - """ - url = self.url("{}/{}/query/".format(token, channel)) - if ramon_type is not None: - # User is requesting a specific ramon_type. - if type(ramon_type) is not int: - ramon_type = ramon.AnnotationType.get_int(ramon_type) - url += "type/{}/".format(str(ramon_type)) - - req = getURL(url) - - if req.status_code is not 200: - raise RemoteDataNotFoundError('No query results for token {}.' - .format(token)) - else: - with tempfile.NamedTemporaryFile() as tmpfile: - tmpfile.write(req.content) - tmpfile.seek(0) - h5file = h5py.File(tmpfile.name, "r") - if 'ANNOIDS' not in h5file: - return [] - return [i for i in h5file['ANNOIDS']] - raise IOError("Could not successfully mock HDF5 file for parsing.") - - @_check_token - def get_ramon(self, token, channel, ids, resolution=0, - include_cutout=False, sieve=None, batch_size=100): - """ - Download a RAMON object by ID. - - Arguments: - token (str): Project to use - channel (str): The channel to use - ids (int, str, int[], str[]): The IDs of a RAMON object to gather. - Can be int (3), string ("3"), int[] ([3, 4, 5]), or string - (["3", "4", "5"]). - resolution (int : None): Resolution. Defaults to the most granular - resolution (0 for now) - include_cutout (bool : False): If True, r.cutout is populated - sieve (function : None): A function that accepts a single ramon - and returns True or False depending on whether you want that - ramon object to be included in your response or not. - For example, - ``` - def is_even_id(ramon): - return ramon.id % 2 == 0 - ``` - You can then pass this to get_ramon like this: - ``` - ndio.remote.neurodata.get_ramon( . . . , sieve=is_even_id) - ``` - batch_size (int : 100): The amount of RAMON objects to download at - a time. If this is greater than 100, we anticipate things going - very poorly for you. So if you set it <100, ndio will use it. - If >=100, set it to 100. - - Returns: - ndio.ramon.RAMON[]: A list of returned RAMON objects. - - Raises: - RemoteDataNotFoundError: If the requested ids cannot be found. - """ - b_size = min(100, batch_size) - - _return_first_only = False - if type(ids) is not list: - _return_first_only = True - ids = [ids] - ids = [str(i) for i in ids] - - rs = [] - id_batches = [ids[i:i + b_size] for i in range(0, len(ids), b_size)] - for batch in id_batches: - rs.extend(self._get_ramon_batch(token, channel, batch, resolution)) - - rs = self._filter_ramon(rs, sieve) - - if include_cutout: - rs = [self._add_ramon_cutout(token, channel, r, resolution) - for r in rs] - - if _return_first_only: - return rs[0] - - return sorted(rs, key=lambda x: ids.index(x.id)) - - def _filter_ramon(self, rs, sieve): - if sieve is not None: - return [r for r in rs if sieve(r)] - return rs - - def _add_ramon_cutout(self, token, channel, ramon, resolution): - origin = ramon.xyz_offset - # Get the bounding box (cube-aligned) - bbox = self.get_ramon_bounding_box(token, channel, - ramon.id, resolution=resolution) - # Get the cutout (cube-aligned) - cutout = self.get_cutout(token, channel, - *bbox, resolution=resolution) - cutout[cutout != int(ramon.id)] = 0 - - # Compute upper offset and crop - bounds = numpy.argwhere(cutout) - mins = [min([i[dim] for i in bounds]) for dim in range(3)] - maxs = [max([i[dim] for i in bounds]) for dim in range(3)] - - ramon.cutout = cutout[ - mins[0]:maxs[0], - mins[1]:maxs[1], - mins[2]:maxs[2] - ] - - return ramon - - def _get_ramon_batch(self, token, channel, ids, resolution): - ids = [str(i) for i in ids] - url = self.url("{}/{}/{}/json/".format(token, channel, ",".join(ids))) - req = getURL(url) - - if req.status_code is not 200: - raise RemoteDataNotFoundError('No data for id {}.'.format(ids)) - else: - return ramon.from_json(req.json()) - - @_check_token - def get_ramon_metadata(self, token, channel, anno_id): - """ - Download a RAMON object by ID. `anno_id` can be a string `"123"`, an - int `123`, an array of ints `[123, 234, 345]`, an array of strings - `["123", "234", "345"]`, or a comma-separated string list - `"123,234,345"`. - - Arguments: - token (str): Project to use - channel (str): The channel to use - anno_id: An int, a str, or a list of ids to gather - - Returns: - JSON. If you pass a single id in str or int, returns a single datum - If you pass a list of int or str or a comma-separated string, will - return a dict with keys from the list and the values are the JSON - returned from the server. - - Raises: - RemoteDataNotFoundError: If the data cannot be found on the Remote - """ - if type(anno_id) in [int, numpy.uint32]: - # there's just one ID to download - return self._get_single_ramon_metadata(token, channel, - str(anno_id)) - elif type(anno_id) is str: - # either "id" or "id,id,id": - if (len(anno_id.split(',')) > 1): - results = {} - for i in anno_id.split(','): - results[i] = self._get_single_ramon_metadata( - token, channel, anno_id.strip() - ) - return results - else: - # "id" - return self._get_single_ramon_metadata(token, channel, - anno_id.strip()) - elif type(anno_id) is list: - # [id, id] or ['id', 'id'] - results = [] - for i in anno_id: - results.append(self._get_single_ramon_metadata(token, channel, - str(i))) - return results - - def _get_single_ramon_metadata(self, token, channel, anno_id): - req = getURL(self.url() + - "{}/{}/{}/json/".format(token, channel, anno_id)) - if req.status_code is not 200: - raise RemoteDataNotFoundError('No data for id {}.'.format(anno_id)) - return req.json() - - @_check_token - def delete_ramon(self, token, channel, anno): - """ - Deletes an annotation from the server. Probably you should be careful - with this function, it seems dangerous. - - Arguments: - token (str): The token to inspect - channel (str): The channel to inspect - anno (int OR list(int) OR RAMON): The annotation to delete. If a - RAMON object is supplied, the remote annotation will be deleted - by an ID lookup. If an int is supplied, the annotation will be - deleted for that ID. If a list of ints are provided, they will - all be deleted. - - Returns: - bool: Success - """ - if type(anno) is int: - a = anno - if type(anno) is str: - a = int(anno) - if type(anno) is list: - a = ",".join(anno) - else: - a = anno.id - - req = requests.delete(self.url("{}/{}/{}/".format(token, channel, a))) - if req.status_code is not 200: - raise RemoteDataNotFoundError("Could not delete id {}: {}" - .format(a, req.text)) - else: - return True - - @_check_token - def post_ramon(self, token, channel, r, batch_size=100): - """ - Posts a RAMON object to the Remote. - - Arguments: - token (str): Project to use - channel (str): The channel to use - r (RAMON or RAMON[]): The annotation(s) to upload - batch_size (int : 100): The number of RAMONs to post simultaneously - at maximum in one file. If len(r) > batch_size, the batch will - be split and uploaded automatically. Must be less than 100. - - Returns: - bool: Success = True - - Throws: - RemoteDataUploadError: if something goes wrong - """ - # Max out batch-size at 100. - b_size = min(100, batch_size) - - # Coerce incoming IDs to a list. - if type(r) is not list: - r = [r] - - # If there are too many to fit in one batch, split here and call this - # function recursively. - if len(r) > batch_size: - batches = [r[i:i + b_size] for i in range(0, len(r), b_size)] - for batch in batches: - self.post_ramon(token, channel, batch, b_size) - return - - with tempfile.NamedTemporaryFile(delete=False) as tmpfile: - for i in r: - tmpfile = ramon.to_hdf5(i, tmpfile) - - url = self.url("{}/{}/overwrite/".format(token, channel)) - req = urllib2.Request(url, tmpfile.read()) - res = urllib2.urlopen(req) - - if res.code != 200: - raise RemoteDataUploadError('[{}] Could not upload {}' - .format(res.code, str(r))) - - rets = res.read() - if six.PY3: - rets = rets.decode() - return_ids = [int(rid) for rid in rets.split(',')] - - # Now post the cutout separately: - for ri in r: - if 'cutout' in dir(ri) and ri.cutout is not None: - orig = ri.xyz_offset - self.post_cutout(token, channel, - orig[0], orig[1], orig[2], - ri.cutout, resolution=r.resolution) - return return_ids - return True - - # SECTION: - # ID Manipulation - - @_check_token - def reserve_ids(self, token, channel, quantity): - """ - Requests a list of next-available-IDs from the server. - - Arguments: - quantity (int): The number of IDs to reserve - - Returns: - int[quantity]: List of IDs you've been granted - """ - quantity = str(quantity) - url = self.url("{}/{}/reserve/{}/".format(token, channel, quantity)) - req = getURL(url) - if req.status_code is not 200: - raise RemoteDataNotFoundError('Invalid req: ' + req.status_code) - out = req.json() - return [out[0] + i for i in range(out[1])] - - @_check_token - def merge_ids(self, token, channel, ids, delete=False): - """ - Call the restful endpoint to merge two RAMON objects into one. - - Arguments: - token (str): The token to inspect - channel (str): The channel to inspect - ids (int[]): the list of the IDs to merge - delete (bool : False): Whether to delete after merging. - - Returns: - json: The ID as returned by ndstore - """ - req = getURL(self.url() + "/merge/{}/" - .format(','.join([str(i) for i in ids]))) - if req.status_code is not 200: - raise RemoteDataUploadError('Could not merge ids {}'.format( - ','.join([str(i) for i in ids]))) - if delete: - self.delete_ramon(token, channel, ids[1:]) - return True - - # SECTION: - # Channels - - def _check_channel(self, channel): - for c in channel: - if not c.isalnum(): - raise ValueError( - "Channel name cannot contain character {}.".format(c) - ) - return True - - @_check_token - def create_channel(self, token, name, channel_type, dtype, readonly): - """ - Create a new channel on the Remote, using channel_data. - - Arguments: - token (str): The token the new channel should be added to - name (str): The name of the channel to add - channel_type (str): Type of the channel (e.g. `neurodata.IMAGE`) - dtype (str): The datatype of the channel's data (e.g. `uint8`) - readonly (bool): Can others write to this channel? - - Returns: - bool: `True` if successful, `False` otherwise. - - Raises: - ValueError: If your args were bad :( - RemoteDataUploadError: If the channel data is valid but upload - fails for some other reason. - """ - self._check_channel(name) - - if channel_type not in ['image', 'annotation']: - raise ValueError('Channel type must be ' + - 'neurodata.IMAGE or neurodata.ANNOTATION.') - - if readonly * 1 not in [0, 1]: - raise ValueError("readonly must be 0 (False) or 1 (True).") - - # Good job! You supplied very nice arguments. - req = requests.post(self.url("{}/createChannel/".format(token)), json={ - "channels": { - name: { - "channel_name": name, - "channel_type": channel_type, - "datatype": dtype, - "readonly": readonly * 1 - } - } - }) - - if req.status_code is not 200: - raise RemoteDataUploadError('Could not upload {}'.format(req.text)) - else: - return True - - @_check_token - def delete_channel(self, token, name): - """ - Delete an existing channel on the Remote. Be careful! - - Arguments: - token (str): The token the new channel should be deleted from - name (str): The name of the channel to delete - - Returns: - bool: True if successful, False otherwise. - - Raises: - RemoteDataUploadError: If the upload fails for some reason. - """ - req = requests.post(self.url("{}/deleteChannel/".format(token)), json={ - "channels": [name] - }) - - if req.status_code is not 200: - raise RemoteDataUploadError('Could not delete {}'.format(req.text)) - if req.content == "SUCCESS": - return True - else: - return False - - # Propagation - - @_check_token - def propagate(self, token, channel): - """ - Kick off the propagate function on the remote server. - - Arguments: - token (str): The token to propagate - channel (str): The channel to propagate - - Returns: - boolean: Success - """ - if self.get_propagate_status(token, channel) != u'0': - return - url = self.url('{}/{}/setPropagate/1/'.format(token, channel)) - req = getURL(url) - if req.status_code is not 200: - raise RemoteDataUploadError('Propagate fail: {}'.format(req.text)) - return True - - @_check_token - def get_propagate_status(self, token, channel): - """ - Get the propagate status for a token/channel pair. - - Arguments: - token (str): The token to check - channel (str): The channel to check - - Returns: - str: The status code - """ - url = self.url('{}/{}/getPropagate/'.format(token, channel)) - req = getURL(url) - if req.status_code is not 200: - raise ValueError('Bad pair: {}/{}'.format(token, channel)) - return req.text - - def getURL(self, url): - """ - Get the propagate status for a token/channel pair. - - Arguments: - url (str): The url make a get to - - Returns: - obj: The response object - """ - try: - req = requests.get(url, - headers={ - 'Authorization': 'Token {}'.format(token) - }, - verify=True) - return req - except requests.exceptions.ConnectionError as e: - if str(e) == "403 Client Error: Forbidden": - raise ValueError('Access Denied') - else: - raise + """ + The NeuroData remote, for interfacing with ndstore, ndlims, and friends. + """ + + # SECTION: + # Enumerables + IMAGE = IMG = 'image' + ANNOTATION = ANNO = 'annotation' + + def __init__(self, + user_token, + hostname=DEFAULT_HOSTNAME, + protocol=DEFAULT_PROTOCOL, + meta_root="http://lims.neurodata.io/", + meta_protocol=DEFAULT_PROTOCOL, **kwargs): + """ + Initializer for the neurodata remote class. + + Arguments: + hostname (str: "openconnecto.me"): The hostname to connect to + protocol (str: "http"): The protocol (http or https) to use + meta_root (str: "http://lims.neurodata.io/"): The metadata server + meta_protocol (str: "http"): The protocol to use for the md server + check_tokens (boolean: False): Whether functions that take `token` + as an argument should check for the existance of that token and + fail immediately if it is not found. This is a good idea for + functions that take more time to complete, and might not fail + until the very end otherwise. + chunk_threshold (int: 1e9 / 4): The maximum size of a numpy array + that will be uploaded in one HTTP request. If you find that + your data requests are commonly timing out, try reducing this. + Default is 1e9 / 4, or a 0.25GiB. + suffix (str: "ocp"): The URL suffix to specify ndstore/microns. If + you aren't sure what to do with this, don't specify one. + """ + self._check_tokens = kwargs.get('check_tokens', False) + self._chunk_threshold = kwargs.get('chunk_threshold', 1E9 / 4) + self._ext = kwargs.get('suffix', DEFAULT_SUFFIX) + self._known_tokens = [] + self._user_token = user_token + + # Prepare meta url + self.meta_root = meta_root + if not self.meta_root.endswith('/'): + self.meta_root = self.meta_root + "/" + if self.meta_root.startswith('https'): + self.meta_root = self.meta_root[self.meta_root.index('://') + 3:] + self.meta_protocol = meta_protocol + + super(neurodata, self).__init__(hostname, protocol) + + # SECTION: + # Decorators + def _check_token(f): + @wraps(f) + def wrapped(self, *args, **kwargs): + if self._check_tokens: + if 'token' in kwargs: + token = kwargs['token'] + else: + token = args[0] + if token not in self._known_tokens: + if self.ping('{}/info/'.format(token)) != 200: + raise RemoteDataNotFoundError("Bad token {}".format( + token)) + else: + self._known_tokens.append(token) + return f(self, *args, **kwargs) + return wrapped + + # SECTION: + # Utilities + def ping(self, suffix='public_tokens/'): + """ + Return the status-code of the API (estimated using the public-tokens + lookup page). + + Arguments: + suffix (str : 'public_tokens/'): The url endpoint to check + + Returns: + int: status code + """ + return super(neurodata, self).ping(suffix) + + def url(self, suffix=""): + """ + Return a constructed URL, appending an optional suffix (uri path). + + Arguments: + suffix (str : ""): The suffix to append to the end of the URL + + Returns: + str: The complete URL + """ + return super(neurodata, self).url('{}/sd/'.format(self._ext) + suffix) + + def meta_url(self, suffix=""): + """ + Return a constructed URL, appending an optional suffix (uri path), + for the metadata server. (Should be temporary, until the LIMS shim + is fixed.) + + Arguments: + suffix (str : ""): The suffix to append to the end of the URL + + Returns: + str: The complete URL + """ + return self.meta_protocol + "://" + self.meta_root + suffix + + def __repr__(self): + """ + Return a string representation that can be used to reproduce this + instance. `eval(repr(this))` should return an identical copy. + + Arguments: + None + + Returns: + str: Representation of reproducible instance. + """ + return "ndio.remote.neurodata('{}', '{}')".format( + self.hostname, + self.protocol, + self.meta_url, + self.meta_protocol + ) + + # SECTION: + # Metadata + def get_public_tokens(self): + """ + Get a list of public tokens available on this server. + + Arguments: + None + + Returns: + str[]: list of public tokens + """ + r = getURL(self.url() + "public_tokens/") + return r.json() + + def get_public_datasets(self): + """ + NOTE: VERY SLOW! + Get a list of public datasets. Different than public tokens! + + Arguments: + None + + Returns: + str[]: list of public datasets + """ + return list(self.get_public_datasets_and_tokens().keys()) + + def get_public_datasets_and_tokens(self): + """ + NOTE: VERY SLOW! + Get a dictionary relating key:dataset to value:[tokens] that rely + on that dataset. + + Arguments: + None + + Returns: + dict: relating key:dataset to value:[tokens] + """ + datasets = {} + tokens = self.get_public_tokens() + for t in tokens: + dataset = self.get_token_dataset(t) + if dataset in datasets: + datasets[dataset].append(t) + else: + datasets[dataset] = [t] + return datasets + + @_check_token + def get_token_dataset(self, token): + """ + Get the dataset for a given token. + + Arguments: + token (str): The token to inspect + + Returns: + str: The name of the dataset + """ + return self.get_proj_info(token)['dataset']['description'] + + @_check_token + def get_proj_info(self, token): + """ + Return the project info for a given token. + + Arguments: + token (str): Token to return information for + + Returns: + JSON: representation of proj_info + """ + r = getURL(self.url() + "{}/info/".format(token)) + return r.json() + + @_check_token + def get_metadata(self, token): + """ + An alias for get_proj_info. + """ + return self.get_proj_info(token) + + @_check_token + def get_channels(self, token): + """ + Wraps get_proj_info to return a dictionary of just the channels of + a given project. + + Arguments: + token (str): Token to return channels for + + Returns: + JSON: dictionary of channels. + """ + return self.get_proj_info(token)['channels'] + + @_check_token + def get_image_size(self, token, resolution=0): + """ + Return the size of the volume (3D). Convenient for when you want + to download the entirety of a dataset. + + Arguments: + token (str): The token for which to find the dataset image bounds + resolution (int : 0): The resolution at which to get image bounds. + Defaults to 0, to get the largest area available. + + Returns: + int[3]: The size of the bounds. Should == get_volume.shape + + Raises: + RemoteDataNotFoundError: If the token is invalid, or if the + metadata at that resolution is unavailable in projinfo. + """ + info = self.get_proj_info(token) + res = str(resolution) + if res not in info['dataset']['imagesize']: + raise RemoteDataNotFoundError("Resolution " + res + + " is not available.") + return info['dataset']['imagesize'][str(resolution)] + + @_check_token + def set_metadata(self, token, data): + """ + Insert new metadata into the OCP metadata database. + + Arguments: + token (str): Token of the datum to set + data (str): A dictionary to insert as metadata. Include `secret`. + + Returns: + json: Info of the inserted ID (convenience) or an error message. + + Throws: + RemoteDataUploadError: If the token is already populated, or if + there is an issue with your specified `secret` key. + """ + req = requests.post(self.meta_url("metadata/ocp/set/" + token), + json=data) + + if req.status_code != 200: + raise RemoteDataUploadError( + "Could not upload metadata: " + req.json()['message'] + ) + return req.json() + + @_check_token + def get_subvolumes(self, token): + """ + Return a list of subvolumes taken from LIMS, if available. + + Arguments: + token (str): The token to read from in LIMS + + Returns: + dict: or None if unavailable + """ + md = self.get_metadata(token)['metadata'] + if 'subvolumes' in md: + return md['subvolumes'] + else: + return None + + @_check_token + def add_subvolume(self, token, channel, secret, + x_start, x_stop, + y_start, y_stop, + z_start, z_stop, + resolution, title, notes): + """ + Adds a new subvolume to a token/channel. + + Arguments: + token (str): The token to write to in LIMS + channel (str): Channel to add in the subvolume. Can be `None` + x_start (int): Start in x dimension + x_stop (int): Stop in x dimension + y_start (int): Start in y dimension + y_stop (int): Stop in y dimension + z_start (int): Start in z dimension + z_stop (int): Stop in z dimension + resolution (int): The resolution at which this subvolume is seen + title (str): The title to set for the subvolume + notes (str): Optional extra thoughts on the subvolume + + Returns: + boolean: success + """ + md = self.get_metadata(token)['metadata'] + if 'subvolumes' in md: + subvols = md['subvolumes'] + else: + subvols = [] + + subvols.append({ + 'token': token, + 'channel': channel, + 'x_start': x_start, + 'x_stop': x_stop, + 'y_start': y_start, + 'y_stop': y_stop, + 'z_start': z_start, + 'z_stop': z_stop, + 'resolution': resolution, + 'title': title, + 'notes': notes + }) + + return self.set_metadata(token, { + 'secret': secret, + 'subvolumes': subvols + }) + + # SECTION: + # Data Download + @_check_token + def get_block_size(self, token, resolution=None): + """ + Gets the block-size for a given token at a given resolution. + + Arguments: + token (str): The token to inspect + resolution (int : None): The resolution at which to inspect data. + If none is specified, uses the minimum available. + + Returns: + int[3]: The xyz blocksize. + """ + cdims = self.get_metadata(token)['dataset']['cube_dimension'] + if resolution is None: + resolution = min(cdims.keys()) + return cdims[str(resolution)] + + @_check_token + def get_image_offset(self, token, resolution=0): + """ + Gets the image offset for a given token at a given resolution. For + instance, the `kasthuri11` dataset starts at (0, 0, 1), so its 1850th + slice is slice 1850, not 1849. When downloading a full dataset, the + result of this function should be your x/y/z starts. + + Arguments: + token (str): The token to inspect + resolution (int : 0): The resolution at which to gather the offset + + Returns: + int[3]: The origin of the dataset, as a list + """ + info = self.get_proj_info(token) + res = str(resolution) + if res not in info['dataset']['offset']: + raise RemoteDataNotFoundError("Resolution " + res + + " is not available.") + return info['dataset']['offset'][str(resolution)] + + @_check_token + def get_xy_slice(self, token, channel, + x_start, x_stop, + y_start, y_stop, + z_index, + resolution=0): + """ + Return a binary-encoded, decompressed 2d image. You should + specify a 'token' and 'channel' pair. For image data, users + should use the channel 'image.' + + Arguments: + token (str): Token to identify data to download + channel (str): Channel + resolution (int): Resolution level + Q_start (int):` The lower bound of dimension 'Q' + Q_stop (int): The upper bound of dimension 'Q' + z_index (int): The z-slice to image + + Returns: + str: binary image data + """ + vol = self.get_cutout(token, channel, x_start, x_stop, y_start, + y_stop, z_index, z_index + 1, resolution) + + vol = numpy.squeeze(vol) # 3D volume to 2D slice + + return vol + + @_check_token + def get_image(self, token, channel, + x_start, x_stop, + y_start, y_stop, + z_index, + resolution=0): + """ + Alias for the `get_xy_slice` function for backwards compatibility. + """ + return self.get_xy_slice(token, channel, + x_start, x_stop, + y_start, y_stop, + z_index, + resolution) + + @_check_token + def get_volume(self, token, channel, + x_start, x_stop, + y_start, y_stop, + z_start, z_stop, + resolution=1, + block_size=DEFAULT_BLOCK_SIZE, + neariso=False): + """ + Get a RAMONVolume volumetric cutout from the neurodata server. + + Arguments: + token (str): Token to identify data to download + channel (str): Channel + resolution (int): Resolution level + Q_start (int): The lower bound of dimension 'Q' + Q_stop (int): The upper bound of dimension 'Q' + block_size (int[3]): Block size of this dataset + neariso (bool : False): Passes the 'neariso' param to the cutout. + If you don't know what this means, ignore it! + + Returns: + ndio.ramon.RAMONVolume: Downloaded data. + """ + size = (x_stop - x_start) * (y_stop - y_start) * (z_stop - z_start) + volume = ramon.RAMONVolume() + volume.xyz_offset = [x_start, y_start, z_start] + volume.resolution = resolution + + volume.cutout = self.get_cutout(token, channel, x_start, + x_stop, y_start, y_stop, + z_start, z_stop, + resolution=resolution, + block_size=block_size, + neariso=neariso) + return volume + + @_check_token + def get_cutout(self, token, channel, + x_start, x_stop, + y_start, y_stop, + z_start, z_stop, + resolution=1, + block_size=DEFAULT_BLOCK_SIZE, + neariso=False): + """ + Get volumetric cutout data from the neurodata server. + + Arguments: + token (str): Token to identify data to download + channel (str): Channel + resolution (int): Resolution level + Q_start (int): The lower bound of dimension 'Q' + Q_stop (int): The upper bound of dimension 'Q' + block_size (int[3]): Block size of this dataset. If not provided, + ndio uses the metadata of this tokenchannel to set. If you find + that your downloads are timing out or otherwise failing, it may + be wise to start off by making this smaller. + neariso (bool : False): Passes the 'neariso' param to the cutout. + If you don't know what this means, ignore it! + + Returns: + numpy.ndarray: Downloaded data. + """ + if block_size is None: + # look up block size from metadata + block_size = self.get_block_size(token, resolution) + + origin = self.get_image_offset(token, resolution) + + # If z_stop - z_start is < 16, backend still pulls minimum 16 slices + if (z_stop - z_start) < 16: + z_slices = 16 + else: + z_slices = z_stop - z_start + + # Calculate size of the data to be downloaded. + size = (x_stop - x_start) * (y_stop - y_start) * z_slices * 4 + + # Switch which download function to use based on which libraries are + # available in this version of python. + if six.PY2: + dl_func = self._get_cutout_blosc_no_chunking + elif six.PY3: + dl_func = self._get_cutout_no_chunking + else: + raise ValueError("Invalid Python version.") + + if size < self._chunk_threshold: + vol = dl_func(token, channel, resolution, + x_start, x_stop, + y_start, y_stop, + z_start, z_stop, neariso=neariso) + vol = numpy.rollaxis(vol, 1) + vol = numpy.rollaxis(vol, 2) + return vol + else: + from ndio.utils.parallel import block_compute + blocks = block_compute(x_start, x_stop, + y_start, y_stop, + z_start, z_stop, + origin, block_size) + + vol = numpy.zeros(((z_stop - z_start), + (y_stop - y_start), + (x_stop - x_start))) + for b in blocks: + + data = dl_func(token, channel, resolution, + b[0][0], b[0][1], + b[1][0], b[1][1], + b[2][0], b[2][1], neariso=neariso) + + if b == blocks[0]: # first block + vol = numpy.zeros(((z_stop - z_start), + (y_stop - y_start), + (x_stop - x_start)), dtype=data.dtype) + + vol[b[2][0] - z_start: b[2][1] - z_start, + b[1][0] - y_start: b[1][1] - y_start, + b[0][0] - x_start: b[0][1] - x_start] = data + + vol = numpy.rollaxis(vol, 1) + vol = numpy.rollaxis(vol, 2) + return vol + + def _get_cutout_no_chunking(self, token, channel, resolution, + x_start, x_stop, y_start, y_stop, + z_start, z_stop, neariso=False): + url = self.url() + "{}/{}/hdf5/{}/{},{}/{},{}/{},{}/".format( + token, channel, resolution, + x_start, x_stop, + y_start, y_stop, + z_start, z_stop + ) + + if neariso: + url += "neariso/" + + req = getURL(url) + if req.status_code is not 200: + raise IOError("Bad server response for {}: {}: {}".format( + url, + req.status_code, + req.text)) + + with tempfile.NamedTemporaryFile() as tmpfile: + tmpfile.write(req.content) + tmpfile.seek(0) + h5file = h5py.File(tmpfile.name, "r") + return h5file.get(channel).get('CUTOUT')[:] + raise IOError("Failed to make tempfile.") + + def _get_cutout_blosc_no_chunking(self, token, channel, resolution, + x_start, x_stop, y_start, y_stop, + z_start, z_stop, neariso=False): + + url = self.url() + "{}/{}/blosc/{}/{},{}/{},{}/{},{}/".format( + token, channel, resolution, + x_start, x_stop, + y_start, y_stop, + z_start, z_stop + ) + + if neariso: + url += "neariso/" + + req = getURL(url) + if req.status_code is not 200: + raise IOError("Bad server response for {}: {}: {}".format( + url, + req.status_code, + req.text)) + + # This will need modification for >3D blocks + return blosc.unpack_array(req.content)[0] + + raise IOError("Failed to retrieve blosc cutout.") + + # SECTION: + # Data Upload + + @_check_token + def post_cutout(self, token, channel, + x_start, + y_start, + z_start, + data, + resolution=0): + """ + Post a cutout to the server. + + Arguments: + token (str) + channel (str) + x_start (int) + y_start (int) + z_start (int) + data (numpy.ndarray): A numpy array of data. Pass in (x, y, z) + resolution (int : 0): Resolution at which to insert the data + + Returns: + bool: True on success + + Raises: + RemoteDataUploadError: if there's an issue during upload. + """ + datatype = self.get_proj_info(token)['channels'][channel]['datatype'] + if data.dtype.name != datatype: + data = data.astype(datatype) + + data = numpy.rollaxis(data, 1) + data = numpy.rollaxis(data, 2) + + if six.PY3 or data.nbytes > 1.5e9: + ul_func = self._post_cutout_no_chunking_npz + else: + ul_func = self._post_cutout_no_chunking_blosc + + if data.size < self._chunk_threshold: + return ul_func(token, channel, x_start, + y_start, z_start, data, + resolution) + + return self._post_cutout_with_chunking(token, channel, + x_start, y_start, z_start, data, + resolution, ul_func) + + def _post_cutout_with_chunking(self, token, channel, x_start, + y_start, z_start, data, + resolution, ul_func): + # must chunk first + from ndio.utils.parallel import block_compute + blocks = block_compute(x_start, x_start + data.shape[2], + y_start, y_start + data.shape[1], + z_start, z_start + data.shape[0]) + for b in blocks: + # data coordinate relative to the size of the arra + subvol = data[b[2][0] - z_start: b[2][1] - z_start, + b[1][0] - y_start: b[1][1] - y_start, + b[0][0] - x_start: b[0][1] - x_start] + # upload the chunk: + # upload coordinate relative to x_start, y_start, z_start + ul_func(token, channel, b[0][0], + b[1][0], b[2][0], subvol, + resolution) + return True + + def _post_cutout_no_chunking_npz(self, token, channel, + x_start, y_start, z_start, + data, resolution): + + data = numpy.expand_dims(data, axis=0) + tempfile = BytesIO() + numpy.save(tempfile, data) + compressed = zlib.compress(tempfile.getvalue()) + + url = self.url("{}/{}/npz/{}/{},{}/{},{}/{},{}/".format( + token, channel, + resolution, + x_start, x_start + data.shape[3], + y_start, y_start + data.shape[2], + z_start, z_start + data.shape[1] + )) + + req = requests.post(url, data=compressed, headers={ + 'Content-Type': 'application/octet-stream' + }) + + if req.status_code is not 200: + raise RemoteDataUploadError(req.text) + else: + return True + + def _post_cutout_no_chunking_blosc(self, token, channel, + x_start, y_start, z_start, + data, resolution): + """ + Accepts data in zyx. !!! + """ + data = numpy.expand_dims(data, axis=0) + blosc_data = blosc.pack_array(data) + + url = self.url("{}/{}/blosc/{}/{},{}/{},{}/{},{}/".format( + token, channel, + resolution, + x_start, x_start + data.shape[3], + y_start, y_start + data.shape[2], + z_start, z_start + data.shape[1] + )) + req = requests.post(url, data=blosc_data, headers={ + 'Content-Type': 'application/octet-stream' + }) + + if req.status_code is not 200: + raise RemoteDataUploadError(req.text) + else: + return True + + # SECTION: + # RAMON Download + + @_check_token + def get_ramon_bounding_box(self, token, channel, r_id, resolution=0): + """ + Get the bounding box for a RAMON object (specified by ID). + + Arguments: + token (str): Project to use + channel (str): Channel to use + r_id (int): Which ID to get a bounding box + resolution (int : 0): The resolution at which to download + + Returns: + (x_start, x_stop, y_start, y_stop, z_start, z_stop): ints + """ + url = self.url('{}/{}/{}/boundingbox/{}/'.format(token, channel, + r_id, resolution)) + + r_id = str(r_id) + res = getURL(url) + + if res.status_code != 200: + rt = self.get_ramon_metadata(token, channel, r_id)[r_id]['type'] + if rt in ['neuron']: + raise ValueError("ID {} is of type '{}'".format(r_id, rt)) + raise RemoteDataNotFoundError("No such ID {}".format(r_id)) + + with tempfile.NamedTemporaryFile() as tmpfile: + tmpfile.write(res.content) + tmpfile.seek(0) + h5file = h5py.File(tmpfile.name, "r") + origin = h5file["{}/XYZOFFSET".format(r_id)][()] + size = h5file["{}/XYZDIMENSION".format(r_id)][()] + return (origin[0], origin[0] + size[0], + origin[1], origin[1] + size[1], + origin[2], origin[2] + size[2]) + + @_check_token + def get_ramon_ids(self, token, channel, ramon_type=None): + """ + Return a list of all IDs available for download from this token and + channel. + + Arguments: + token (str): Project to use + channel (str): Channel to use + ramon_type (int : None): Optional. If set, filters IDs and only + returns those of RAMON objects of the requested type. + + Returns: + int[]: A list of the ids of the returned RAMON objects + + Raises: + RemoteDataNotFoundError: If the channel or token is not found + """ + url = self.url("{}/{}/query/".format(token, channel)) + if ramon_type is not None: + # User is requesting a specific ramon_type. + if type(ramon_type) is not int: + ramon_type = ramon.AnnotationType.get_int(ramon_type) + url += "type/{}/".format(str(ramon_type)) + + req = getURL(url) + + if req.status_code is not 200: + raise RemoteDataNotFoundError('No query results for token {}.' + .format(token)) + else: + with tempfile.NamedTemporaryFile() as tmpfile: + tmpfile.write(req.content) + tmpfile.seek(0) + h5file = h5py.File(tmpfile.name, "r") + if 'ANNOIDS' not in h5file: + return [] + return [i for i in h5file['ANNOIDS']] + raise IOError("Could not successfully mock HDF5 file for parsing.") + + @_check_token + def get_ramon(self, token, channel, ids, resolution=0, + include_cutout=False, sieve=None, batch_size=100): + """ + Download a RAMON object by ID. + + Arguments: + token (str): Project to use + channel (str): The channel to use + ids (int, str, int[], str[]): The IDs of a RAMON object to gather. + Can be int (3), string ("3"), int[] ([3, 4, 5]), or string + (["3", "4", "5"]). + resolution (int : None): Resolution. Defaults to the most granular + resolution (0 for now) + include_cutout (bool : False): If True, r.cutout is populated + sieve (function : None): A function that accepts a single ramon + and returns True or False depending on whether you want that + ramon object to be included in your response or not. + For example, + ``` + def is_even_id(ramon): + return ramon.id % 2 == 0 + ``` + You can then pass this to get_ramon like this: + ``` + ndio.remote.neurodata.get_ramon( . . . , sieve=is_even_id) + ``` + batch_size (int : 100): The amount of RAMON objects to download at + a time. If this is greater than 100, we anticipate things going + very poorly for you. So if you set it <100, ndio will use it. + If >=100, set it to 100. + + Returns: + ndio.ramon.RAMON[]: A list of returned RAMON objects. + + Raises: + RemoteDataNotFoundError: If the requested ids cannot be found. + """ + b_size = min(100, batch_size) + + _return_first_only = False + if type(ids) is not list: + _return_first_only = True + ids = [ids] + ids = [str(i) for i in ids] + + rs = [] + id_batches = [ids[i:i + b_size] for i in range(0, len(ids), b_size)] + for batch in id_batches: + rs.extend(self._get_ramon_batch(token, channel, batch, resolution)) + + rs = self._filter_ramon(rs, sieve) + + if include_cutout: + rs = [self._add_ramon_cutout(token, channel, r, resolution) + for r in rs] + + if _return_first_only: + return rs[0] + + return sorted(rs, key=lambda x: ids.index(x.id)) + + def _filter_ramon(self, rs, sieve): + if sieve is not None: + return [r for r in rs if sieve(r)] + return rs + + def _add_ramon_cutout(self, token, channel, ramon, resolution): + origin = ramon.xyz_offset + # Get the bounding box (cube-aligned) + bbox = self.get_ramon_bounding_box(token, channel, + ramon.id, resolution=resolution) + # Get the cutout (cube-aligned) + cutout = self.get_cutout(token, channel, + *bbox, resolution=resolution) + cutout[cutout != int(ramon.id)] = 0 + + # Compute upper offset and crop + bounds = numpy.argwhere(cutout) + mins = [min([i[dim] for i in bounds]) for dim in range(3)] + maxs = [max([i[dim] for i in bounds]) for dim in range(3)] + + ramon.cutout = cutout[ + mins[0]:maxs[0], + mins[1]:maxs[1], + mins[2]:maxs[2] + ] + + return ramon + + def _get_ramon_batch(self, token, channel, ids, resolution): + ids = [str(i) for i in ids] + url = self.url("{}/{}/{}/json/".format(token, channel, ",".join(ids))) + req = getURL(url) + + if req.status_code is not 200: + raise RemoteDataNotFoundError('No data for id {}.'.format(ids)) + else: + return ramon.from_json(req.json()) + + @_check_token + def get_ramon_metadata(self, token, channel, anno_id): + """ + Download a RAMON object by ID. `anno_id` can be a string `"123"`, an + int `123`, an array of ints `[123, 234, 345]`, an array of strings + `["123", "234", "345"]`, or a comma-separated string list + `"123,234,345"`. + + Arguments: + token (str): Project to use + channel (str): The channel to use + anno_id: An int, a str, or a list of ids to gather + + Returns: + JSON. If you pass a single id in str or int, returns a single datum + If you pass a list of int or str or a comma-separated string, will + return a dict with keys from the list and the values are the JSON + returned from the server. + + Raises: + RemoteDataNotFoundError: If the data cannot be found on the Remote + """ + if type(anno_id) in [int, numpy.uint32]: + # there's just one ID to download + return self._get_single_ramon_metadata(token, channel, + str(anno_id)) + elif type(anno_id) is str: + # either "id" or "id,id,id": + if (len(anno_id.split(',')) > 1): + results = {} + for i in anno_id.split(','): + results[i] = self._get_single_ramon_metadata( + token, channel, anno_id.strip() + ) + return results + else: + # "id" + return self._get_single_ramon_metadata(token, channel, + anno_id.strip()) + elif type(anno_id) is list: + # [id, id] or ['id', 'id'] + results = [] + for i in anno_id: + results.append(self._get_single_ramon_metadata(token, channel, + str(i))) + return results + + def _get_single_ramon_metadata(self, token, channel, anno_id): + req = getURL(self.url() + + "{}/{}/{}/json/".format(token, channel, anno_id)) + if req.status_code is not 200: + raise RemoteDataNotFoundError('No data for id {}.'.format(anno_id)) + return req.json() + + @_check_token + def delete_ramon(self, token, channel, anno): + """ + Deletes an annotation from the server. Probably you should be careful + with this function, it seems dangerous. + + Arguments: + token (str): The token to inspect + channel (str): The channel to inspect + anno (int OR list(int) OR RAMON): The annotation to delete. If a + RAMON object is supplied, the remote annotation will be deleted + by an ID lookup. If an int is supplied, the annotation will be + deleted for that ID. If a list of ints are provided, they will + all be deleted. + + Returns: + bool: Success + """ + if type(anno) is int: + a = anno + if type(anno) is str: + a = int(anno) + if type(anno) is list: + a = ",".join(anno) + else: + a = anno.id + + req = requests.delete(self.url("{}/{}/{}/".format(token, channel, a))) + if req.status_code is not 200: + raise RemoteDataNotFoundError("Could not delete id {}: {}" + .format(a, req.text)) + else: + return True + + @_check_token + def post_ramon(self, token, channel, r, batch_size=100): + """ + Posts a RAMON object to the Remote. + + Arguments: + token (str): Project to use + channel (str): The channel to use + r (RAMON or RAMON[]): The annotation(s) to upload + batch_size (int : 100): The number of RAMONs to post simultaneously + at maximum in one file. If len(r) > batch_size, the batch will + be split and uploaded automatically. Must be less than 100. + + Returns: + bool: Success = True + + Throws: + RemoteDataUploadError: if something goes wrong + """ + # Max out batch-size at 100. + b_size = min(100, batch_size) + + # Coerce incoming IDs to a list. + if type(r) is not list: + r = [r] + + # If there are too many to fit in one batch, split here and call this + # function recursively. + if len(r) > batch_size: + batches = [r[i:i + b_size] for i in range(0, len(r), b_size)] + for batch in batches: + self.post_ramon(token, channel, batch, b_size) + return + + with tempfile.NamedTemporaryFile(delete=False) as tmpfile: + for i in r: + tmpfile = ramon.to_hdf5(i, tmpfile) + + url = self.url("{}/{}/overwrite/".format(token, channel)) + req = urllib2.Request(url, tmpfile.read()) + res = urllib2.urlopen(req) + + if res.code != 200: + raise RemoteDataUploadError('[{}] Could not upload {}' + .format(res.code, str(r))) + + rets = res.read() + if six.PY3: + rets = rets.decode() + return_ids = [int(rid) for rid in rets.split(',')] + + # Now post the cutout separately: + for ri in r: + if 'cutout' in dir(ri) and ri.cutout is not None: + orig = ri.xyz_offset + self.post_cutout(token, channel, + orig[0], orig[1], orig[2], + ri.cutout, resolution=r.resolution) + return return_ids + return True + + # SECTION: + # ID Manipulation + + @_check_token + def reserve_ids(self, token, channel, quantity): + """ + Requests a list of next-available-IDs from the server. + + Arguments: + quantity (int): The number of IDs to reserve + + Returns: + int[quantity]: List of IDs you've been granted + """ + quantity = str(quantity) + url = self.url("{}/{}/reserve/{}/".format(token, channel, quantity)) + req = getURL(url) + if req.status_code is not 200: + raise RemoteDataNotFoundError('Invalid req: ' + req.status_code) + out = req.json() + return [out[0] + i for i in range(out[1])] + + @_check_token + def merge_ids(self, token, channel, ids, delete=False): + """ + Call the restful endpoint to merge two RAMON objects into one. + + Arguments: + token (str): The token to inspect + channel (str): The channel to inspect + ids (int[]): the list of the IDs to merge + delete (bool : False): Whether to delete after merging. + + Returns: + json: The ID as returned by ndstore + """ + req = getURL(self.url() + "/merge/{}/" + .format(','.join([str(i) for i in ids]))) + if req.status_code is not 200: + raise RemoteDataUploadError('Could not merge ids {}'.format( + ','.join([str(i) for i in ids]))) + if delete: + self.delete_ramon(token, channel, ids[1:]) + return True + + # SECTION: + # Channels + + def _check_channel(self, channel): + for c in channel: + if not c.isalnum(): + raise ValueError( + "Channel name cannot contain character {}.".format(c) + ) + return True + + @_check_token + def create_channel(self, token, name, channel_type, dtype, readonly): + """ + Create a new channel on the Remote, using channel_data. + + Arguments: + token (str): The token the new channel should be added to + name (str): The name of the channel to add + channel_type (str): Type of the channel (e.g. `neurodata.IMAGE`) + dtype (str): The datatype of the channel's data (e.g. `uint8`) + readonly (bool): Can others write to this channel? + + Returns: + bool: `True` if successful, `False` otherwise. + + Raises: + ValueError: If your args were bad :( + RemoteDataUploadError: If the channel data is valid but upload + fails for some other reason. + """ + self._check_channel(name) + + if channel_type not in ['image', 'annotation']: + raise ValueError('Channel type must be ' + + 'neurodata.IMAGE or neurodata.ANNOTATION.') + + if readonly * 1 not in [0, 1]: + raise ValueError("readonly must be 0 (False) or 1 (True).") + + # Good job! You supplied very nice arguments. + req = requests.post(self.url("{}/createChannel/".format(token)), json={ + "channels": { + name: { + "channel_name": name, + "channel_type": channel_type, + "datatype": dtype, + "readonly": readonly * 1 + } + } + }) + + if req.status_code is not 200: + raise RemoteDataUploadError('Could not upload {}'.format(req.text)) + else: + return True + + @_check_token + def delete_channel(self, token, name): + """ + Delete an existing channel on the Remote. Be careful! + + Arguments: + token (str): The token the new channel should be deleted from + name (str): The name of the channel to delete + + Returns: + bool: True if successful, False otherwise. + + Raises: + RemoteDataUploadError: If the upload fails for some reason. + """ + req = requests.post(self.url("{}/deleteChannel/".format(token)), json={ + "channels": [name] + }) + + if req.status_code is not 200: + raise RemoteDataUploadError('Could not delete {}'.format(req.text)) + if req.content == "SUCCESS": + return True + else: + return False + + # Propagation + + @_check_token + def propagate(self, token, channel): + """ + Kick off the propagate function on the remote server. + + Arguments: + token (str): The token to propagate + channel (str): The channel to propagate + + Returns: + boolean: Success + """ + if self.get_propagate_status(token, channel) != u'0': + return + url = self.url('{}/{}/setPropagate/1/'.format(token, channel)) + req = getURL(url) + if req.status_code is not 200: + raise RemoteDataUploadError('Propagate fail: {}'.format(req.text)) + return True + + @_check_token + def get_propagate_status(self, token, channel): + """ + Get the propagate status for a token/channel pair. + + Arguments: + token (str): The token to check + channel (str): The channel to check + + Returns: + str: The status code + """ + url = self.url('{}/{}/getPropagate/'.format(token, channel)) + req = getURL(url) + if req.status_code is not 200: + raise ValueError('Bad pair: {}/{}'.format(token, channel)) + return req.text + + def getURL(self, url): + """ + Get the propagate status for a token/channel pair. + + Arguments: + url (str): The url make a get to + + Returns: + obj: The response object + """ + try: + req = requests.get(url, + headers={ + 'Authorization': 'Token {}'.format(token)}, + verify=True) + return req + except requests.exceptions.ConnectionError as e: + if str(e) == "403 Client Error: Forbidden": + raise ValueError('Access Denied') + else: + raise From c63ecae9cc9cee890950a3f7a59da10cd515fd61 Mon Sep 17 00:00:00 2001 From: Jinyong Shin Date: Fri, 24 Feb 2017 09:53:15 -0500 Subject: [PATCH 03/42] white space change --- ndio/remote/neurodata.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ndio/remote/neurodata.py b/ndio/remote/neurodata.py index 604f815..2696a01 100644 --- a/ndio/remote/neurodata.py +++ b/ndio/remote/neurodata.py @@ -1265,9 +1265,8 @@ def getURL(self, url): obj: The response object """ try: - req = requests.get(url, - headers={ - 'Authorization': 'Token {}'.format(token)}, + req = requests.get(url, + headers={'Authorization': 'Token {}'.format(token)}, verify=True) return req except requests.exceptions.ConnectionError as e: From baae77a1d8467e2e13dd9d3e01c35223c840892a Mon Sep 17 00:00:00 2001 From: Jinyong Shin Date: Fri, 24 Feb 2017 10:37:43 -0500 Subject: [PATCH 04/42] indent --- ndio/remote/neurodata.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ndio/remote/neurodata.py b/ndio/remote/neurodata.py index 2696a01..fd0f6db 100644 --- a/ndio/remote/neurodata.py +++ b/ndio/remote/neurodata.py @@ -1266,7 +1266,9 @@ def getURL(self, url): """ try: req = requests.get(url, - headers={'Authorization': 'Token {}'.format(token)}, + headers={ + 'Authorization': 'Token {}'.format(token) + }, verify=True) return req except requests.exceptions.ConnectionError as e: From 7137a2ae9319f59f92eedc27eaf98bd2cee1e215 Mon Sep 17 00:00:00 2001 From: Jinyong Shin Date: Fri, 24 Feb 2017 12:23:01 -0500 Subject: [PATCH 05/42] indent --- ndio/remote/neurodata.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ndio/remote/neurodata.py b/ndio/remote/neurodata.py index fd0f6db..8430e58 100644 --- a/ndio/remote/neurodata.py +++ b/ndio/remote/neurodata.py @@ -1266,7 +1266,8 @@ def getURL(self, url): """ try: req = requests.get(url, - headers={ + headers = + { 'Authorization': 'Token {}'.format(token) }, verify=True) From 1e9c97fd2b572481175b242edea2f879cdaf82d8 Mon Sep 17 00:00:00 2001 From: Jinyong Shin Date: Fri, 24 Feb 2017 13:02:38 -0500 Subject: [PATCH 06/42] change --- ndio/remote/neurodata.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ndio/remote/neurodata.py b/ndio/remote/neurodata.py index 8430e58..bd134a1 100644 --- a/ndio/remote/neurodata.py +++ b/ndio/remote/neurodata.py @@ -1266,11 +1266,9 @@ def getURL(self, url): """ try: req = requests.get(url, - headers = - { - 'Authorization': 'Token {}'.format(token) - }, - verify=True) + headers ={'Authorization': 'Token {}'.format(token) + }, + verify=True) return req except requests.exceptions.ConnectionError as e: if str(e) == "403 Client Error: Forbidden": From dc8b72b0643c49eee05a67d129f5978496f7dcd7 Mon Sep 17 00:00:00 2001 From: Jinyong Shin Date: Fri, 24 Feb 2017 13:17:08 -0500 Subject: [PATCH 07/42] change --- ndio/remote/neurodata.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ndio/remote/neurodata.py b/ndio/remote/neurodata.py index bd134a1..d03364f 100644 --- a/ndio/remote/neurodata.py +++ b/ndio/remote/neurodata.py @@ -1266,9 +1266,10 @@ def getURL(self, url): """ try: req = requests.get(url, - headers ={'Authorization': 'Token {}'.format(token) - }, - verify=True) + headers={ + 'Authorization': 'Token {}'.format(token)}, + verify=True + ) return req except requests.exceptions.ConnectionError as e: if str(e) == "403 Client Error: Forbidden": From e4927395905a75d44b6bc53808c0c7120ac8eba6 Mon Sep 17 00:00:00 2001 From: Jinyong Shin Date: Fri, 24 Feb 2017 13:25:41 -0500 Subject: [PATCH 08/42] change --- ndio/remote/neurodata.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ndio/remote/neurodata.py b/ndio/remote/neurodata.py index d03364f..b833a27 100644 --- a/ndio/remote/neurodata.py +++ b/ndio/remote/neurodata.py @@ -1266,10 +1266,10 @@ def getURL(self, url): """ try: req = requests.get(url, - headers={ + headers={ 'Authorization': 'Token {}'.format(token)}, - verify=True - ) + verify=True + ) return req except requests.exceptions.ConnectionError as e: if str(e) == "403 Client Error: Forbidden": From 50281211bf3b299b1a6c37091ba021a4ce45467a Mon Sep 17 00:00:00 2001 From: Jinyong Shin Date: Fri, 24 Feb 2017 13:37:22 -0500 Subject: [PATCH 09/42] change --- ndio/remote/neurodata.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ndio/remote/neurodata.py b/ndio/remote/neurodata.py index b833a27..2bd61b3 100644 --- a/ndio/remote/neurodata.py +++ b/ndio/remote/neurodata.py @@ -1265,11 +1265,12 @@ def getURL(self, url): obj: The response object """ try: - req = requests.get(url, - headers={ - 'Authorization': 'Token {}'.format(token)}, - verify=True - ) + req = requests.get(url, + header={ + 'Authorization': 'Token {}'.format(token) + }, + verify=True + ) return req except requests.exceptions.ConnectionError as e: if str(e) == "403 Client Error: Forbidden": From 5e78c5ca4b80f8752449462eb8b599667fee5f68 Mon Sep 17 00:00:00 2001 From: Jinyong Shin Date: Fri, 24 Feb 2017 13:43:43 -0500 Subject: [PATCH 10/42] change --- ndio/remote/neurodata.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ndio/remote/neurodata.py b/ndio/remote/neurodata.py index 2bd61b3..545f914 100644 --- a/ndio/remote/neurodata.py +++ b/ndio/remote/neurodata.py @@ -1265,11 +1265,10 @@ def getURL(self, url): obj: The response object """ try: - req = requests.get(url, - header={ + req = requests.get(url, + header={ 'Authorization': 'Token {}'.format(token) - }, - verify=True + }, verify=True ) return req except requests.exceptions.ConnectionError as e: From 1beb70de97a40bbef968973f7cea879199780047 Mon Sep 17 00:00:00 2001 From: Jinyong Shin Date: Fri, 24 Feb 2017 13:45:08 -0500 Subject: [PATCH 11/42] change --- ndio/remote/neurodata.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ndio/remote/neurodata.py b/ndio/remote/neurodata.py index 545f914..2c09a4c 100644 --- a/ndio/remote/neurodata.py +++ b/ndio/remote/neurodata.py @@ -1265,11 +1265,9 @@ def getURL(self, url): obj: The response object """ try: - req = requests.get(url, - header={ - 'Authorization': 'Token {}'.format(token) - }, verify=True - ) + req = requests.get(url, verify = True, header={ + 'Authorization': 'Token {}'.format(token) + }) return req except requests.exceptions.ConnectionError as e: if str(e) == "403 Client Error: Forbidden": From d285aa34533e688011f7d5add5d88370755da2ab Mon Sep 17 00:00:00 2001 From: Jinyong Shin Date: Fri, 24 Feb 2017 14:04:40 -0500 Subject: [PATCH 12/42] change --- ndio/remote/neurodata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ndio/remote/neurodata.py b/ndio/remote/neurodata.py index 2c09a4c..dca47c5 100644 --- a/ndio/remote/neurodata.py +++ b/ndio/remote/neurodata.py @@ -1265,7 +1265,7 @@ def getURL(self, url): obj: The response object """ try: - req = requests.get(url, verify = True, header={ + req = requests.get(url, verify=True, header={ 'Authorization': 'Token {}'.format(token) }) return req From 67edc3ab7437063e844195ecd48dd3f418a1e757 Mon Sep 17 00:00:00 2001 From: Jinyong Shin Date: Tue, 28 Feb 2017 02:12:20 -0500 Subject: [PATCH 13/42] add getURL --- ndio/remote/neurodata.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ndio/remote/neurodata.py b/ndio/remote/neurodata.py index dca47c5..ba7e3e6 100644 --- a/ndio/remote/neurodata.py +++ b/ndio/remote/neurodata.py @@ -1265,9 +1265,11 @@ def getURL(self, url): obj: The response object """ try: - req = requests.get(url, verify=True, header={ - 'Authorization': 'Token {}'.format(token) - }) + req = requests.get(url, headers={ + 'Authorization': 'Token {}'.format(self._user_token) + }, + verify=False, + ) return req except requests.exceptions.ConnectionError as e: if str(e) == "403 Client Error: Forbidden": From 68b9a3089765948f904fa060cd09673d73626da4 Mon Sep 17 00:00:00 2001 From: Jinyong Shin Date: Wed, 1 Mar 2017 15:03:34 -0500 Subject: [PATCH 14/42] test update --- ndio/remote/neurodata.py | 13 +++++----- tests/test_getURL.py | 52 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 6 deletions(-) create mode 100644 tests/test_getURL.py diff --git a/ndio/remote/neurodata.py b/ndio/remote/neurodata.py index ba7e3e6..05dd9f5 100644 --- a/ndio/remote/neurodata.py +++ b/ndio/remote/neurodata.py @@ -1265,12 +1265,13 @@ def getURL(self, url): obj: The response object """ try: - req = requests.get(url, headers={ - 'Authorization': 'Token {}'.format(self._user_token) - }, - verify=False, - ) - return req + req = requests.get(url, headers={ + 'Authorization':'Token {}'.format(self._user_token) + }, verify = False) + if req.status_code is 403: + raise ValueError("Access Denied") + else: + return req except requests.exceptions.ConnectionError as e: if str(e) == "403 Client Error: Forbidden": raise ValueError('Access Denied') diff --git a/tests/test_getURL.py b/tests/test_getURL.py new file mode 100644 index 0000000..eb585c2 --- /dev/null +++ b/tests/test_getURL.py @@ -0,0 +1,52 @@ +import unittest +import ndio.remote.neurodata as neurodata + +neurodata_token = "1524d74ec9a915536061af8fd5b51f2cde7de9a5" +test_token = "b1876fab697e519e0cfe4acc9ca0496bf2889e17" +Default_token = "" + +class TestgetURL(unittest.TestCase): + + def setUp(self): + self.neurodata = neurodata(user_token = neurodata_token) + self.public = neurodata(user_token = "") + self.test = neurodata(user_token = Default_token) + + def test_masterToken(self): + try: + req = self.neurodata.getURL("https://localhost/nd/sd/private_neuro/info/") + self.assertEqual(req.status_code, 200) + except ValueError as e: + print(e) + try: + req2 = self.neurodata.getURL("https://localhost/nd/sd/temp_neuro/info/") + self.assertEqual(req2.status_code, 200) + except ValueError as e: + print(e) + + def test_testToken(self): + try: + req = self.neurodata.getURL("https://localhost/nd/sd/private_neuro/info/") + self.assertEqual(req.status_code, 403) + except ValueError as e: + print(e) + try: + req2 = self.neurodata.getURL("https://localhost/nd/sd/temp_neuro/info/") + self.assertEqual(req2.status_code, 200) + except ValueError as e: + print(e) + + def test_publicToken(self): + try: + req = self.neurodata.getURL("https://localhost/nd/sd/private_neuro/info/") + self.assertEqual(req.status_code, 403) + except ValueError as e: + print(e) + try: + req2 = self.neurodata.getURL("https://localhost/nd/sd/temp_neuro/info/") + self.assertEqual(req2.status_code, 200) + except ValueError as e: + print(e) + +if __name__ == '__main__': + unittest.main() From 30fa3faa36f8b84d6173a99ae491c11e623566ad Mon Sep 17 00:00:00 2001 From: Jinyong Shin Date: Wed, 1 Mar 2017 15:05:58 -0500 Subject: [PATCH 15/42] test --- ndio/remote/neurodata.py | 28 ++++++++++++++-------------- tests/test_sanity_check.py | 11 ----------- 2 files changed, 14 insertions(+), 25 deletions(-) delete mode 100644 tests/test_sanity_check.py diff --git a/ndio/remote/neurodata.py b/ndio/remote/neurodata.py index 05dd9f5..36ccbfb 100644 --- a/ndio/remote/neurodata.py +++ b/ndio/remote/neurodata.py @@ -170,7 +170,7 @@ def get_public_tokens(self): Returns: str[]: list of public tokens """ - r = getURL(self.url() + "public_tokens/") + r = self.getURL(self.url() + "public_tokens/") return r.json() def get_public_datasets(self): @@ -232,7 +232,7 @@ def get_proj_info(self, token): Returns: JSON: representation of proj_info """ - r = getURL(self.url() + "{}/info/".format(token)) + r = self.getURL(self.url() + "{}/info/".format(token)) return r.json() @_check_token @@ -598,7 +598,7 @@ def _get_cutout_no_chunking(self, token, channel, resolution, if neariso: url += "neariso/" - req = getURL(url) + req = self.getURL(url) if req.status_code is not 200: raise IOError("Bad server response for {}: {}: {}".format( url, @@ -626,7 +626,7 @@ def _get_cutout_blosc_no_chunking(self, token, channel, resolution, if neariso: url += "neariso/" - req = getURL(url) + req = self.getURL(url) if req.status_code is not 200: raise IOError("Bad server response for {}: {}: {}".format( url, @@ -779,7 +779,7 @@ def get_ramon_bounding_box(self, token, channel, r_id, resolution=0): r_id, resolution)) r_id = str(r_id) - res = getURL(url) + res = self.getURL(url) if res.status_code != 200: rt = self.get_ramon_metadata(token, channel, r_id)[r_id]['type'] @@ -822,7 +822,7 @@ def get_ramon_ids(self, token, channel, ramon_type=None): ramon_type = ramon.AnnotationType.get_int(ramon_type) url += "type/{}/".format(str(ramon_type)) - req = getURL(url) + req = self.getURL(url) if req.status_code is not 200: raise RemoteDataNotFoundError('No query results for token {}.' @@ -930,7 +930,7 @@ def _add_ramon_cutout(self, token, channel, ramon, resolution): def _get_ramon_batch(self, token, channel, ids, resolution): ids = [str(i) for i in ids] url = self.url("{}/{}/{}/json/".format(token, channel, ",".join(ids))) - req = getURL(url) + req = self.getURL(url) if req.status_code is not 200: raise RemoteDataNotFoundError('No data for id {}.'.format(ids)) @@ -985,8 +985,8 @@ def get_ramon_metadata(self, token, channel, anno_id): return results def _get_single_ramon_metadata(self, token, channel, anno_id): - req = getURL(self.url() + - "{}/{}/{}/json/".format(token, channel, anno_id)) + req = self.getURL(self.url() + + "{}/{}/{}/json/".format(token, channel, anno_id)) if req.status_code is not 200: raise RemoteDataNotFoundError('No data for id {}.'.format(anno_id)) return req.json() @@ -1102,7 +1102,7 @@ def reserve_ids(self, token, channel, quantity): """ quantity = str(quantity) url = self.url("{}/{}/reserve/{}/".format(token, channel, quantity)) - req = getURL(url) + req = self.getURL(url) if req.status_code is not 200: raise RemoteDataNotFoundError('Invalid req: ' + req.status_code) out = req.json() @@ -1122,8 +1122,8 @@ def merge_ids(self, token, channel, ids, delete=False): Returns: json: The ID as returned by ndstore """ - req = getURL(self.url() + "/merge/{}/" - .format(','.join([str(i) for i in ids]))) + req = self.getURL(self.url() + "/merge/{}/" + .format(','.join([str(i) for i in ids]))) if req.status_code is not 200: raise RemoteDataUploadError('Could not merge ids {}'.format( ','.join([str(i) for i in ids]))) @@ -1231,7 +1231,7 @@ def propagate(self, token, channel): if self.get_propagate_status(token, channel) != u'0': return url = self.url('{}/{}/setPropagate/1/'.format(token, channel)) - req = getURL(url) + req = self.getURL(url) if req.status_code is not 200: raise RemoteDataUploadError('Propagate fail: {}'.format(req.text)) return True @@ -1249,7 +1249,7 @@ def get_propagate_status(self, token, channel): str: The status code """ url = self.url('{}/{}/getPropagate/'.format(token, channel)) - req = getURL(url) + req = self.getURL(url) if req.status_code is not 200: raise ValueError('Bad pair: {}/{}'.format(token, channel)) return req.text diff --git a/tests/test_sanity_check.py b/tests/test_sanity_check.py deleted file mode 100644 index 47d0049..0000000 --- a/tests/test_sanity_check.py +++ /dev/null @@ -1,11 +0,0 @@ -import unittest - - -class TestSanityCheck(unittest.TestCase): - - def test_sanity(self): - self.assertEqual(1, 1) - self.assertNotEqual(1, 2) - -if __name__ == '__main__': - unittest.main() From adfde46be21f4170e6f6a4d06d923839b33f0833 Mon Sep 17 00:00:00 2001 From: Jinyong Shin Date: Thu, 2 Mar 2017 22:26:08 -0500 Subject: [PATCH 16/42] changes --- ndio/remote/neurodata.py | 4 ++-- tests/test_getURL.py | 21 ++++++++++++--------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/ndio/remote/neurodata.py b/ndio/remote/neurodata.py index c4310a2..d2c0c55 100644 --- a/ndio/remote/neurodata.py +++ b/ndio/remote/neurodata.py @@ -1266,8 +1266,8 @@ def getURL(self, url): """ try: req = requests.get(url, headers={ - 'Authorization':'Token {}'.format(self._user_token) - }, verify = False) + 'Authorization': 'Token {}'.format(self._user_token) + }, verify=False) if req.status_code is 403: raise ValueError("Access Denied") else: diff --git a/tests/test_getURL.py b/tests/test_getURL.py index eb585c2..3d9f0a5 100644 --- a/tests/test_getURL.py +++ b/tests/test_getURL.py @@ -5,45 +5,48 @@ test_token = "b1876fab697e519e0cfe4acc9ca0496bf2889e17" Default_token = "" + class TestgetURL(unittest.TestCase): def setUp(self): - self.neurodata = neurodata(user_token = neurodata_token) - self.public = neurodata(user_token = "") - self.test = neurodata(user_token = Default_token) + self.neurodata = neurodata(user_token=neurodata_token) + self.public = neurodata(user_token="") + self.test = neurodata(user_token=Default_token) + self.private_URL = "https://localhost/nd/sd/private_neuro/info/" + self.public_URL = "https://localhost/nd/sd/temp_neuro/info/" def test_masterToken(self): try: - req = self.neurodata.getURL("https://localhost/nd/sd/private_neuro/info/") + req = self.neurodata.getURL(self.private_URL) self.assertEqual(req.status_code, 200) except ValueError as e: print(e) try: - req2 = self.neurodata.getURL("https://localhost/nd/sd/temp_neuro/info/") + req2 = self.neurodata.getURL(self.public_URL) self.assertEqual(req2.status_code, 200) except ValueError as e: print(e) def test_testToken(self): try: - req = self.neurodata.getURL("https://localhost/nd/sd/private_neuro/info/") + req = self.neurodata.getURL(self.private_URL) self.assertEqual(req.status_code, 403) except ValueError as e: print(e) try: - req2 = self.neurodata.getURL("https://localhost/nd/sd/temp_neuro/info/") + req2 = self.neurodata.getURL(self.public_URL) self.assertEqual(req2.status_code, 200) except ValueError as e: print(e) def test_publicToken(self): try: - req = self.neurodata.getURL("https://localhost/nd/sd/private_neuro/info/") + req = self.neurodata.getURL(self.private_URL) self.assertEqual(req.status_code, 403) except ValueError as e: print(e) try: - req2 = self.neurodata.getURL("https://localhost/nd/sd/temp_neuro/info/") + req2 = self.neurodata.getURL(self.public_URL) self.assertEqual(req2.status_code, 200) except ValueError as e: print(e) From 3ea3282a7737220518b0e8ddcec0c5e28c4a1739 Mon Sep 17 00:00:00 2001 From: Jin Date: Fri, 3 Mar 2017 14:20:58 -0500 Subject: [PATCH 17/42] neurodata.py getURL update --- .cache/v/cache/lastfailed | 4 +++ ndio/remote/neurodata.py | 11 ++++++--- ndio/remote/pass.txt | 1 + test_getURL.py | 52 +++++++++++++++++++++++++++++++++++++++ test_getURL.py~ | 35 ++++++++++++++++++++++++++ 5 files changed, 99 insertions(+), 4 deletions(-) create mode 100644 .cache/v/cache/lastfailed create mode 100644 ndio/remote/pass.txt create mode 100644 test_getURL.py create mode 100644 test_getURL.py~ diff --git a/.cache/v/cache/lastfailed b/.cache/v/cache/lastfailed new file mode 100644 index 0000000..7222840 --- /dev/null +++ b/.cache/v/cache/lastfailed @@ -0,0 +1,4 @@ +{ + "test_getURL.py::TestgetURL::test_publicToken": true, + "test_getURL.py::TestgetURL::test_testToken": true +} \ No newline at end of file diff --git a/ndio/remote/neurodata.py b/ndio/remote/neurodata.py index dca47c5..05dd9f5 100644 --- a/ndio/remote/neurodata.py +++ b/ndio/remote/neurodata.py @@ -1265,10 +1265,13 @@ def getURL(self, url): obj: The response object """ try: - req = requests.get(url, verify=True, header={ - 'Authorization': 'Token {}'.format(token) - }) - return req + req = requests.get(url, headers={ + 'Authorization':'Token {}'.format(self._user_token) + }, verify = False) + if req.status_code is 403: + raise ValueError("Access Denied") + else: + return req except requests.exceptions.ConnectionError as e: if str(e) == "403 Client Error: Forbidden": raise ValueError('Access Denied') diff --git a/ndio/remote/pass.txt b/ndio/remote/pass.txt new file mode 100644 index 0000000..3dde6bb --- /dev/null +++ b/ndio/remote/pass.txt @@ -0,0 +1 @@ +1524d74ec9a915536061af8fd5b51f2cde7de9a5 diff --git a/test_getURL.py b/test_getURL.py new file mode 100644 index 0000000..eb585c2 --- /dev/null +++ b/test_getURL.py @@ -0,0 +1,52 @@ +import unittest +import ndio.remote.neurodata as neurodata + +neurodata_token = "1524d74ec9a915536061af8fd5b51f2cde7de9a5" +test_token = "b1876fab697e519e0cfe4acc9ca0496bf2889e17" +Default_token = "" + +class TestgetURL(unittest.TestCase): + + def setUp(self): + self.neurodata = neurodata(user_token = neurodata_token) + self.public = neurodata(user_token = "") + self.test = neurodata(user_token = Default_token) + + def test_masterToken(self): + try: + req = self.neurodata.getURL("https://localhost/nd/sd/private_neuro/info/") + self.assertEqual(req.status_code, 200) + except ValueError as e: + print(e) + try: + req2 = self.neurodata.getURL("https://localhost/nd/sd/temp_neuro/info/") + self.assertEqual(req2.status_code, 200) + except ValueError as e: + print(e) + + def test_testToken(self): + try: + req = self.neurodata.getURL("https://localhost/nd/sd/private_neuro/info/") + self.assertEqual(req.status_code, 403) + except ValueError as e: + print(e) + try: + req2 = self.neurodata.getURL("https://localhost/nd/sd/temp_neuro/info/") + self.assertEqual(req2.status_code, 200) + except ValueError as e: + print(e) + + def test_publicToken(self): + try: + req = self.neurodata.getURL("https://localhost/nd/sd/private_neuro/info/") + self.assertEqual(req.status_code, 403) + except ValueError as e: + print(e) + try: + req2 = self.neurodata.getURL("https://localhost/nd/sd/temp_neuro/info/") + self.assertEqual(req2.status_code, 200) + except ValueError as e: + print(e) + +if __name__ == '__main__': + unittest.main() diff --git a/test_getURL.py~ b/test_getURL.py~ new file mode 100644 index 0000000..39c4d73 --- /dev/null +++ b/test_getURL.py~ @@ -0,0 +1,35 @@ +import unittest +import ndio.remote.neurodata as neurodata + +neurodata_token = "1524d74ec9a915536061af8fd5b51f2cde7de9a5" +test_token = "b1876fab697e519e0cfe4acc9ca0496bf2889e17" +Default_token = "" + +class TestgetURL(unittest.TestCase): + + def setUp(self): + self.neurodata = neurodata(user_token = neurodata_token) + self.public = neurodata(user_token = "") + self.test = neurodata(user_token = Default_token) + + def test_masterToken(self): + + req = self.neurodata.getURL("https://localhost/nd/sd/private_neuro/info/") + self.assertEqual(req.status_code, 200) + req2 = self.neurodata.getURL("https://localhost/nd/sd/temp_neuro/info/") + self.assertEqual(req2.status_code, 200) + + def test_testToken(self): + req = self.neurodata.getURL("https://localhost/nd/sd/private_neuro/info/") + self.assertEqual(req.status_code, 403) + req2 = self.neurodata.getURL("https://localhost/nd/sd/temp_neuro/info/") + self.assertEqual(req2.status_code, 200) + + def test_publicToken(self): + req = self.neurodata.getURL("https://localhost/nd/sd/private_neuro/info/") + self.assertEqual(req.status_code, 403) + req2 = self.neurodata.getURL("https://localhost/nd/sd/temp_neuro/info/") + self.assertEqual(req2.status_code, 200) + +if __name__ == '__main__': + unittest.main() From a003a96b8fbe7e440a3c1a78609c9c3da62748e6 Mon Sep 17 00:00:00 2001 From: Jin Date: Fri, 3 Mar 2017 14:45:27 -0500 Subject: [PATCH 18/42] update getURL --- ndio/remote/neurodata.py | 4 ++-- test_getURL.py | 21 +++++++++++---------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/ndio/remote/neurodata.py b/ndio/remote/neurodata.py index c4310a2..d2c0c55 100644 --- a/ndio/remote/neurodata.py +++ b/ndio/remote/neurodata.py @@ -1266,8 +1266,8 @@ def getURL(self, url): """ try: req = requests.get(url, headers={ - 'Authorization':'Token {}'.format(self._user_token) - }, verify = False) + 'Authorization': 'Token {}'.format(self._user_token) + }, verify=False) if req.status_code is 403: raise ValueError("Access Denied") else: diff --git a/test_getURL.py b/test_getURL.py index eb585c2..fc91f8a 100644 --- a/test_getURL.py +++ b/test_getURL.py @@ -9,41 +9,42 @@ class TestgetURL(unittest.TestCase): def setUp(self): self.neurodata = neurodata(user_token = neurodata_token) - self.public = neurodata(user_token = "") + self.public = neurodata() self.test = neurodata(user_token = Default_token) - + self.private_project = "https://localhost/nd/sd/private_neuro/info/" + self.public_project = "https://localhost/nd/sd/temp_neuro/info/" def test_masterToken(self): try: - req = self.neurodata.getURL("https://localhost/nd/sd/private_neuro/info/") + req = self.neurodata.getURL(self.private_project) self.assertEqual(req.status_code, 200) except ValueError as e: - print(e) + print(req.content) try: - req2 = self.neurodata.getURL("https://localhost/nd/sd/temp_neuro/info/") + req2 = self.neurodata.getURL(self.public_project) self.assertEqual(req2.status_code, 200) except ValueError as e: print(e) def test_testToken(self): try: - req = self.neurodata.getURL("https://localhost/nd/sd/private_neuro/info/") + req = self.neurodata.getURL(self.private_project) self.assertEqual(req.status_code, 403) except ValueError as e: print(e) try: - req2 = self.neurodata.getURL("https://localhost/nd/sd/temp_neuro/info/") + req2 = self.neurodata.getURL(self.public_project) self.assertEqual(req2.status_code, 200) except ValueError as e: print(e) def test_publicToken(self): try: - req = self.neurodata.getURL("https://localhost/nd/sd/private_neuro/info/") + req = self.neurodata.getURL(self.private_project) self.assertEqual(req.status_code, 403) except ValueError as e: - print(e) + print(req.content) try: - req2 = self.neurodata.getURL("https://localhost/nd/sd/temp_neuro/info/") + req2 = self.neurodata.getURL(self.public_project) self.assertEqual(req2.status_code, 200) except ValueError as e: print(e) From e7e18efa8456a121288797c76f1ef91a028a518f Mon Sep 17 00:00:00 2001 From: jyshin93 Date: Mon, 6 Mar 2017 15:44:09 -0500 Subject: [PATCH 19/42] add test_getURL.py --- .cache/v/cache/lastfailed | 3 +-- test_getURL.py | 53 --------------------------------------- test_getURL.py~ | 35 -------------------------- tests/test_getURL.py | 44 ++++++++++++++++++++------------ 4 files changed, 29 insertions(+), 106 deletions(-) delete mode 100644 test_getURL.py delete mode 100644 test_getURL.py~ diff --git a/.cache/v/cache/lastfailed b/.cache/v/cache/lastfailed index 7222840..b7ddf61 100644 --- a/.cache/v/cache/lastfailed +++ b/.cache/v/cache/lastfailed @@ -1,4 +1,3 @@ { - "test_getURL.py::TestgetURL::test_publicToken": true, - "test_getURL.py::TestgetURL::test_testToken": true + "test_getURL.py::TestgetURL::test_masterToken": true } \ No newline at end of file diff --git a/test_getURL.py b/test_getURL.py deleted file mode 100644 index fc91f8a..0000000 --- a/test_getURL.py +++ /dev/null @@ -1,53 +0,0 @@ -import unittest -import ndio.remote.neurodata as neurodata - -neurodata_token = "1524d74ec9a915536061af8fd5b51f2cde7de9a5" -test_token = "b1876fab697e519e0cfe4acc9ca0496bf2889e17" -Default_token = "" - -class TestgetURL(unittest.TestCase): - - def setUp(self): - self.neurodata = neurodata(user_token = neurodata_token) - self.public = neurodata() - self.test = neurodata(user_token = Default_token) - self.private_project = "https://localhost/nd/sd/private_neuro/info/" - self.public_project = "https://localhost/nd/sd/temp_neuro/info/" - def test_masterToken(self): - try: - req = self.neurodata.getURL(self.private_project) - self.assertEqual(req.status_code, 200) - except ValueError as e: - print(req.content) - try: - req2 = self.neurodata.getURL(self.public_project) - self.assertEqual(req2.status_code, 200) - except ValueError as e: - print(e) - - def test_testToken(self): - try: - req = self.neurodata.getURL(self.private_project) - self.assertEqual(req.status_code, 403) - except ValueError as e: - print(e) - try: - req2 = self.neurodata.getURL(self.public_project) - self.assertEqual(req2.status_code, 200) - except ValueError as e: - print(e) - - def test_publicToken(self): - try: - req = self.neurodata.getURL(self.private_project) - self.assertEqual(req.status_code, 403) - except ValueError as e: - print(req.content) - try: - req2 = self.neurodata.getURL(self.public_project) - self.assertEqual(req2.status_code, 200) - except ValueError as e: - print(e) - -if __name__ == '__main__': - unittest.main() diff --git a/test_getURL.py~ b/test_getURL.py~ deleted file mode 100644 index 39c4d73..0000000 --- a/test_getURL.py~ +++ /dev/null @@ -1,35 +0,0 @@ -import unittest -import ndio.remote.neurodata as neurodata - -neurodata_token = "1524d74ec9a915536061af8fd5b51f2cde7de9a5" -test_token = "b1876fab697e519e0cfe4acc9ca0496bf2889e17" -Default_token = "" - -class TestgetURL(unittest.TestCase): - - def setUp(self): - self.neurodata = neurodata(user_token = neurodata_token) - self.public = neurodata(user_token = "") - self.test = neurodata(user_token = Default_token) - - def test_masterToken(self): - - req = self.neurodata.getURL("https://localhost/nd/sd/private_neuro/info/") - self.assertEqual(req.status_code, 200) - req2 = self.neurodata.getURL("https://localhost/nd/sd/temp_neuro/info/") - self.assertEqual(req2.status_code, 200) - - def test_testToken(self): - req = self.neurodata.getURL("https://localhost/nd/sd/private_neuro/info/") - self.assertEqual(req.status_code, 403) - req2 = self.neurodata.getURL("https://localhost/nd/sd/temp_neuro/info/") - self.assertEqual(req2.status_code, 200) - - def test_publicToken(self): - req = self.neurodata.getURL("https://localhost/nd/sd/private_neuro/info/") - self.assertEqual(req.status_code, 403) - req2 = self.neurodata.getURL("https://localhost/nd/sd/temp_neuro/info/") - self.assertEqual(req2.status_code, 200) - -if __name__ == '__main__': - unittest.main() diff --git a/tests/test_getURL.py b/tests/test_getURL.py index 3d9f0a5..f4e8213 100644 --- a/tests/test_getURL.py +++ b/tests/test_getURL.py @@ -5,48 +5,60 @@ test_token = "b1876fab697e519e0cfe4acc9ca0496bf2889e17" Default_token = "" - class TestgetURL(unittest.TestCase): def setUp(self): - self.neurodata = neurodata(user_token=neurodata_token) - self.public = neurodata(user_token="") - self.test = neurodata(user_token=Default_token) - self.private_URL = "https://localhost/nd/sd/private_neuro/info/" - self.public_URL = "https://localhost/nd/sd/temp_neuro/info/" - + self.neurodata = neurodata(user_token = neurodata_token) + self.public = neurodata() + self.test = neurodata(user_token = Default_token) + self.private_project = "https://localhost/nd/sd/private_neuro/info/" + self.public_project = "https://localhost/nd/sd/public_neuro/info/" + self.private_test_project = "https://localhost/nd/sd/private_test_neuro/info/" def test_masterToken(self): try: - req = self.neurodata.getURL(self.private_URL) + req = self.neurodata.getURL(self.private_project) + print(req.content) self.assertEqual(req.status_code, 200) except ValueError as e: - print(e) + print(req.content) try: - req2 = self.neurodata.getURL(self.public_URL) + req2 = self.neurodata.getURL(self.public_project) self.assertEqual(req2.status_code, 200) except ValueError as e: print(e) - + + try: + req3 = self.neurodata.getURL(self.private_test_project) + self.assertEqual(req3.status_code, 200) + except ValueError as e: + print(e) + def test_testToken(self): try: - req = self.neurodata.getURL(self.private_URL) + req = self.test.getURL(self.private_project) self.assertEqual(req.status_code, 403) except ValueError as e: print(e) try: - req2 = self.neurodata.getURL(self.public_URL) + req2 = self.test.getURL(self.public_project) self.assertEqual(req2.status_code, 200) except ValueError as e: print(e) + try: + req3 = self.test.getURL(self.private_test_project) + self.assertEqual(req3.status_code, 200) + except ValueError as e: + print(e) + def test_publicToken(self): try: - req = self.neurodata.getURL(self.private_URL) + req = self.public.getURL(self.private_project) self.assertEqual(req.status_code, 403) except ValueError as e: - print(e) + print(req.content) try: - req2 = self.neurodata.getURL(self.public_URL) + req2 = self.public.getURL(self.public_project) self.assertEqual(req2.status_code, 200) except ValueError as e: print(e) From 37211540a80c220b1395ed399addf563cd14cde7 Mon Sep 17 00:00:00 2001 From: jyshin93 Date: Mon, 6 Mar 2017 15:51:35 -0500 Subject: [PATCH 20/42] update test_getURL.py --- test_getURL.py | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 test_getURL.py diff --git a/test_getURL.py b/test_getURL.py new file mode 100644 index 0000000..3d0cb99 --- /dev/null +++ b/test_getURL.py @@ -0,0 +1,67 @@ +import unittest +import ndio.remote.neurodata as neurodata + +neurodata_token = "1524d74ec9a915536061af8fd5b51f2cde7de9a5" +test_token = "b1876fab697e519e0cfe4acc9ca0496bf2889e17" +Default_token = "" + +class TestgetURL(unittest.TestCase): + + def setUp(self): + self.neurodata = neurodata(user_token=neurodata_token) + self.public = neurodata(user_token=Default_token) + self.test = neurodata(user_token=test_token) + self.private_project = "https://localhost/nd/sd/private_neuro/info/" + self.public_project = "https://localhost/nd/sd/public_neuro/info/" + self.private_test_project = "https://localhost/nd/sd/private_test_neuro/info/" + def test_masterToken(self): + try: + req = self.neurodata.getURL(self.private_project) + print(req.content) + self.assertEqual(req.status_code, 200) + except ValueError as e: + print(req.content) + try: + req2 = self.neurodata.getURL(self.public_project) + self.assertEqual(req2.status_code, 200) + except ValueError as e: + print(e) + + try: + req3 = self.neurodata.getURL(self.private_test_project) + self.assertEqual(req3.status_code, 200) + except ValueError as e: + print(e) + + def test_testToken(self): + try: + req = self.test.getURL(self.private_project) + self.assertEqual(req.status_code, 403) + except ValueError as e: + print(e) + try: + req2 = self.test.getURL(self.public_project) + self.assertEqual(req2.status_code, 200) + except ValueError as e: + print(e) + try: + req3 = self.test.getURL(self.private_test_project) + self.assertEqual(req3.status_code, 200) + except ValueError as e: + print(e) + + + def test_publicToken(self): + try: + req = self.public.getURL(self.private_project) + self.assertEqual(req.status_code, 403) + except ValueError as e: + print(req.content) + try: + req2 = self.public.getURL(self.public_project) + self.assertEqual(req2.status_code, 200) + except ValueError as e: + print(e) + +if __name__ == '__main__': + unittest.main() From 8ea52c79f1cd7c87ad3fe8ecb3ad9fd1f1db89a6 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 6 Mar 2017 21:00:14 +0000 Subject: [PATCH 21/42] Forcing use of hostname from settings --- tests/test_getURL.py | 14 ++++++++------ tests/test_settings.py | 4 ++-- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/test_getURL.py b/tests/test_getURL.py index f4e8213..c026946 100644 --- a/tests/test_getURL.py +++ b/tests/test_getURL.py @@ -1,19 +1,21 @@ import unittest import ndio.remote.neurodata as neurodata +import test_settings neurodata_token = "1524d74ec9a915536061af8fd5b51f2cde7de9a5" test_token = "b1876fab697e519e0cfe4acc9ca0496bf2889e17" Default_token = "" +HOSTNAME = hostname=test_settings.HOSTNAME class TestgetURL(unittest.TestCase): def setUp(self): - self.neurodata = neurodata(user_token = neurodata_token) - self.public = neurodata() - self.test = neurodata(user_token = Default_token) - self.private_project = "https://localhost/nd/sd/private_neuro/info/" - self.public_project = "https://localhost/nd/sd/public_neuro/info/" - self.private_test_project = "https://localhost/nd/sd/private_test_neuro/info/" + self.neurodata = neurodata(user_token = neurodata_token, hostname=HOSTNAME) + self.public = neurodata(hostname=HOSTNAME) + self.test = neurodata(user_token = Default_token, hostname=HOSTNAME) + self.private_project = "https://{}/sd/private_neuro/info/".format(HOSTNAME) + self.public_project = "https://{}/sd/public_neuro/info/".format(HOSTNAME) + self.private_test_project = "https://{}/sd/private_test_neuro/info/".format(HOSTNAME) def test_masterToken(self): try: req = self.neurodata.getURL(self.private_project) diff --git a/tests/test_settings.py b/tests/test_settings.py index 6365340..4ada4da 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -11,5 +11,5 @@ DEV_MODE = True # server to check against -# SITE_HOST = 'localhost:8080' -HOSTNAME = 'localhost/nd' +HOSTNAME = 'localhost:8080' +#HOSTNAME = 'localhost/nd' From f54b6cb1004185f4dd4fe061ece92860695212f0 Mon Sep 17 00:00:00 2001 From: jyshin93 Date: Thu, 16 Mar 2017 18:06:25 -0400 Subject: [PATCH 22/42] update from the review on test_getURL.py and neurodata.py getURL function --- .cache/v/cache/lastfailed | 3 ++- ndio/remote/neurodata.py | 5 +---- test_getURL.py | 5 +++++ tests/test_getURL.py | 15 ++++++++------- tests/test_settings.py | 6 ++++-- 5 files changed, 20 insertions(+), 14 deletions(-) diff --git a/.cache/v/cache/lastfailed b/.cache/v/cache/lastfailed index b7ddf61..1433162 100644 --- a/.cache/v/cache/lastfailed +++ b/.cache/v/cache/lastfailed @@ -1,3 +1,4 @@ { - "test_getURL.py::TestgetURL::test_masterToken": true + "tests/test_getURL.py::TestgetURL::test_masterToken": true, + "tests/test_getURL.py::TestgetURL::test_testToken": true } \ No newline at end of file diff --git a/ndio/remote/neurodata.py b/ndio/remote/neurodata.py index c532135..33124c3 100644 --- a/ndio/remote/neurodata.py +++ b/ndio/remote/neurodata.py @@ -227,10 +227,7 @@ def getURL(self, url): else: return req except requests.exceptions.ConnectionError as e: - if str(e) == "403 Client Error: Forbidden": - raise ValueError('Access Denied') - else: - raise + raise e def post_url(self, url, token='', json={}): """ diff --git a/test_getURL.py b/test_getURL.py index 3d0cb99..4151314 100644 --- a/test_getURL.py +++ b/test_getURL.py @@ -62,6 +62,11 @@ def test_publicToken(self): self.assertEqual(req2.status_code, 200) except ValueError as e: print(e) + try: + req3 = self.test.getURL(self.private_test_project) + self.assertEqual(req3.status_code, 403) + except ValueError as e: + print(e) if __name__ == '__main__': unittest.main() diff --git a/tests/test_getURL.py b/tests/test_getURL.py index c026946..1b94ada 100644 --- a/tests/test_getURL.py +++ b/tests/test_getURL.py @@ -1,18 +1,19 @@ -import unittest import ndio.remote.neurodata as neurodata import test_settings -neurodata_token = "1524d74ec9a915536061af8fd5b51f2cde7de9a5" -test_token = "b1876fab697e519e0cfe4acc9ca0496bf2889e17" -Default_token = "" -HOSTNAME = hostname=test_settings.HOSTNAME + + + +HOSTNAME = test_settings.HOSTNAME +NEURODATA = test_settings.NEURODATA +TEST = test_settings.TEST class TestgetURL(unittest.TestCase): def setUp(self): - self.neurodata = neurodata(user_token = neurodata_token, hostname=HOSTNAME) + self.neurodata = neurodata(user_token=NEURODATA, hostname=HOSTNAME) self.public = neurodata(hostname=HOSTNAME) - self.test = neurodata(user_token = Default_token, hostname=HOSTNAME) + self.test = neurodata(user_token=TEST, hostname=HOSTNAME) self.private_project = "https://{}/sd/private_neuro/info/".format(HOSTNAME) self.public_project = "https://{}/sd/public_neuro/info/".format(HOSTNAME) self.private_test_project = "https://{}/sd/private_test_neuro/info/".format(HOSTNAME) diff --git a/tests/test_settings.py b/tests/test_settings.py index 4ada4da..5859051 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -11,5 +11,7 @@ DEV_MODE = True # server to check against -HOSTNAME = 'localhost:8080' -#HOSTNAME = 'localhost/nd' +# HOSTNAME = 'localhost:8080' +HOSTNAME = 'localhost/nd' +NEURODATA = "1524d74ec9a915536061af8fd5b51f2cde7de9a5" +TEST = "b1876fab697e519e0cfe4acc9ca0496bf2889e17" From 9cd6ff25f9212ddf6ab9ce95c8a52243d9b29716 Mon Sep 17 00:00:00 2001 From: jyshin93 Date: Fri, 17 Mar 2017 13:11:37 -0400 Subject: [PATCH 23/42] updated test_getURL.py --- .cache/v/cache/lastfailed | 4 --- ndio/remote/neurodata.py | 5 ++- test_getURL.py | 72 --------------------------------------- tests/test_getURL.py | 6 ++-- tests/test_settings.py | 4 +-- 5 files changed, 9 insertions(+), 82 deletions(-) delete mode 100644 .cache/v/cache/lastfailed delete mode 100644 test_getURL.py diff --git a/.cache/v/cache/lastfailed b/.cache/v/cache/lastfailed deleted file mode 100644 index 1433162..0000000 --- a/.cache/v/cache/lastfailed +++ /dev/null @@ -1,4 +0,0 @@ -{ - "tests/test_getURL.py::TestgetURL::test_masterToken": true, - "tests/test_getURL.py::TestgetURL::test_testToken": true -} \ No newline at end of file diff --git a/ndio/remote/neurodata.py b/ndio/remote/neurodata.py index 33124c3..2b5538d 100644 --- a/ndio/remote/neurodata.py +++ b/ndio/remote/neurodata.py @@ -227,7 +227,10 @@ def getURL(self, url): else: return req except requests.exceptions.ConnectionError as e: - raise e + if str(e) == '403 Client Error: Forbidden': + raise ValueError('Access Denied') + else: + raise e def post_url(self, url, token='', json={}): """ diff --git a/test_getURL.py b/test_getURL.py deleted file mode 100644 index 4151314..0000000 --- a/test_getURL.py +++ /dev/null @@ -1,72 +0,0 @@ -import unittest -import ndio.remote.neurodata as neurodata - -neurodata_token = "1524d74ec9a915536061af8fd5b51f2cde7de9a5" -test_token = "b1876fab697e519e0cfe4acc9ca0496bf2889e17" -Default_token = "" - -class TestgetURL(unittest.TestCase): - - def setUp(self): - self.neurodata = neurodata(user_token=neurodata_token) - self.public = neurodata(user_token=Default_token) - self.test = neurodata(user_token=test_token) - self.private_project = "https://localhost/nd/sd/private_neuro/info/" - self.public_project = "https://localhost/nd/sd/public_neuro/info/" - self.private_test_project = "https://localhost/nd/sd/private_test_neuro/info/" - def test_masterToken(self): - try: - req = self.neurodata.getURL(self.private_project) - print(req.content) - self.assertEqual(req.status_code, 200) - except ValueError as e: - print(req.content) - try: - req2 = self.neurodata.getURL(self.public_project) - self.assertEqual(req2.status_code, 200) - except ValueError as e: - print(e) - - try: - req3 = self.neurodata.getURL(self.private_test_project) - self.assertEqual(req3.status_code, 200) - except ValueError as e: - print(e) - - def test_testToken(self): - try: - req = self.test.getURL(self.private_project) - self.assertEqual(req.status_code, 403) - except ValueError as e: - print(e) - try: - req2 = self.test.getURL(self.public_project) - self.assertEqual(req2.status_code, 200) - except ValueError as e: - print(e) - try: - req3 = self.test.getURL(self.private_test_project) - self.assertEqual(req3.status_code, 200) - except ValueError as e: - print(e) - - - def test_publicToken(self): - try: - req = self.public.getURL(self.private_project) - self.assertEqual(req.status_code, 403) - except ValueError as e: - print(req.content) - try: - req2 = self.public.getURL(self.public_project) - self.assertEqual(req2.status_code, 200) - except ValueError as e: - print(e) - try: - req3 = self.test.getURL(self.private_test_project) - self.assertEqual(req3.status_code, 403) - except ValueError as e: - print(e) - -if __name__ == '__main__': - unittest.main() diff --git a/tests/test_getURL.py b/tests/test_getURL.py index 1b94ada..b167c71 100644 --- a/tests/test_getURL.py +++ b/tests/test_getURL.py @@ -1,6 +1,6 @@ import ndio.remote.neurodata as neurodata import test_settings - +import unittest @@ -20,7 +20,7 @@ def setUp(self): def test_masterToken(self): try: req = self.neurodata.getURL(self.private_project) - print(req.content) + print(req.status_code) self.assertEqual(req.status_code, 200) except ValueError as e: print(req.content) @@ -57,7 +57,7 @@ def test_testToken(self): def test_publicToken(self): try: req = self.public.getURL(self.private_project) - self.assertEqual(req.status_code, 403) + self.assertEqual(req.status_code, 401) except ValueError as e: print(req.content) try: diff --git a/tests/test_settings.py b/tests/test_settings.py index 5859051..57061d9 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -13,5 +13,5 @@ # server to check against # HOSTNAME = 'localhost:8080' HOSTNAME = 'localhost/nd' -NEURODATA = "1524d74ec9a915536061af8fd5b51f2cde7de9a5" -TEST = "b1876fab697e519e0cfe4acc9ca0496bf2889e17" +NEURODATA = open('/tmp/neurotoken.txt', 'r').read() +TEST = open('/tmp/testtoken.txt', 'r').read() From dbb1a5daa094070f4d11098677becf439f2b3404 Mon Sep 17 00:00:00 2001 From: jyshin93 Date: Fri, 17 Mar 2017 13:26:07 -0400 Subject: [PATCH 24/42] changed token textfile name --- tests/test_getURL.py | 2 -- tests/test_settings.py | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_getURL.py b/tests/test_getURL.py index b167c71..9e6cb20 100644 --- a/tests/test_getURL.py +++ b/tests/test_getURL.py @@ -2,8 +2,6 @@ import test_settings import unittest - - HOSTNAME = test_settings.HOSTNAME NEURODATA = test_settings.NEURODATA TEST = test_settings.TEST diff --git a/tests/test_settings.py b/tests/test_settings.py index 57061d9..874187b 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -13,5 +13,5 @@ # server to check against # HOSTNAME = 'localhost:8080' HOSTNAME = 'localhost/nd' -NEURODATA = open('/tmp/neurotoken.txt', 'r').read() -TEST = open('/tmp/testtoken.txt', 'r').read() +NEURODATA = open('/tmp/token_super', 'r').read() +TEST = open('/tmp/token_user', 'r').read() From a19837f981f3caa562feb7661c52c233656ff17c Mon Sep 17 00:00:00 2001 From: jyshin93 Date: Mon, 27 Mar 2017 18:09:53 -0400 Subject: [PATCH 25/42] resources api implemented in neurodata.py and test 4 methods --- ndio/remote/neurodata.py | 108 ++++++++++++++++++++++++++++++++++++ tests/test_resources_api.py | 58 ++++++++++++++++++- 2 files changed, 165 insertions(+), 1 deletion(-) diff --git a/ndio/remote/neurodata.py b/ndio/remote/neurodata.py index 2b5538d..03ddb13 100644 --- a/ndio/remote/neurodata.py +++ b/ndio/remote/neurodata.py @@ -286,6 +286,114 @@ def get_token_dataset(self, token): """ return self.get_proj_info(token)['dataset']['description'] + ################################## + #TOKEN# + ################################## + def create_token(self, + token_name, + project_name, + dataset_name, + is_public): + """ + Creates a token with the given parameters. + + Arguments: + project_name (str): Project name + dataset_name (str): Dataset name project is based on + token_name (str): Token name + is_public (int): 1 is public. 0 is not public + + Returns: + bool: True if project created, false if not created. + """ + + url = self.url()[:-4] + '/nd/resource/dataset/{}'.format( + dataset_name) + '/project/{}'.format(project_name) + \ + '/token/{}/'.format(token_name) + + json = { + "token_name": token_name, + "public": is_public + } + + req = self.post_url(url, json=json) + + if req.status_code is not 201: + raise RemoteDataUploadError('Cout not upload {}:'.format(req.text)) + if req.content == "" or req.content == b'': + return True + else: + return False + + def get_token(self, + token_name, + project_name, + dataset_name): + """ + Get a token with the given parameters. + + Arguments: + project_name (str): Project name + dataset_name (str): Dataset name project is based on + token_name (str): Token name + + Returns: + dict: Token info + """ + url = self.url()[:-4] + "/nd/resource/dataset/{}".format(dataset_name)\ + + "/project/{}".format(project_name) + "/token/{}/".format(token_name) + req = self.getURL(url) + + if req.status_code is not 200: + raise RemoteDataUploadError('Could not find {}'.format(req.text)) + else: + return req.json() + + def delete_token(self, + token_name, + project_name, + dataset_name): + """ + Delete a token with the given parameters. + + Arguments: + project_name (str): Project name + dataset_name (str): Dataset name project is based on + token_name (str): Token name + channel_name (str): Channel name project is based on + + Returns: + bool: True if project deleted, false if not deleted. + """ + url = self.url()[:-4] + "/nd/resource/dataset/{}".format(dataset_name)\ + + "/project/{}".format(project_name) + "/token/{}/".format(token_name) + req = self.delete_url(url) + + if req.status_code is not 204: + raise RemoteDataUploadError("Could not delete {}".format(req.text)) + if req.content == "" or req.content == b'': + return True + else: + return False + + def list_tokens(self): + """ + Lists a set of tokens that are public in Neurodata. + + Arguments: + + Returns: + dict: Public tokens found in Neurodata + """ + url = self.url()[:-4] + "/nd/resource/public/token/" + req = self.getURL(url) + + if req.status_code is not 200: + raise RemoteDataNotFoundError('Coud not find {}'.format(req.text)) + else: + return req.json() + + def create_project(self, project_name, dataset_name, diff --git a/tests/test_resources_api.py b/tests/test_resources_api.py index 196782e..5e7f011 100644 --- a/tests/test_resources_api.py +++ b/tests/test_resources_api.py @@ -10,7 +10,7 @@ class TestResourcesApi(unittest.TestCase): def setUp(self): self.token_user = '043813bf95d9d5be2bb19448d5a9337db086c559' hostname = 'localhost' - self.nd = nd(self.token_user, hostname=hostname) + self.nd = nd(user_token=self.token_user, hostname=hostname) def test_create_dataset(self): result = self.nd.create_dataset('test', 1, 1, 1, 1.0, 1.0, 1.0) @@ -169,5 +169,61 @@ def test_delete_channel(self): self.nd.delete_project('testp', 'test') self.nd.delete_dataset('test') + def test_create_token(self): + self.nd.create_dataset('test', 1, 1, 1, 1.0, 1.0, 1.0, 0) + self.nd.create_project( + 'testp', 'test', 'localhost', 1, 1, 'localhost', 'Redis') + result = self.nd.create_token('testt', 'testp', 'test', 1) + self.assertEqual(result, True) + self.nd.delete_token('testt', 'testp', 'test') + self.nd.delete_project('testp', 'test') + self.nd.delete_dataset('test') + + def test_get_token(self): + self.nd.create_dataset('test', 1, 1, 1, 1.0, 1.0, 1.0, 0) + self.nd.create_project( + 'testp', 'test', 'localhost', 1, 1, 'localhost', 'Redis') + self.nd.create_token('testt', 'testp', 'test', 1) + result = self.nd.get_token('testt', 'testp', 'test') + compare_dict = {u'token_name': 'testt', + u'public': 1} + for key in compare_dict: + self.assertEqual(result[key], compare_dict[key]) + self.nd.delete_token('testt', 'testp', 'test') + self.nd.delete_project('testp', 'test') + self.nd.delete_dataset('test') + + def test_delete_token(self): + self.nd.create_dataset('test', 1, 1, 1, 1.0, 1.0, 1.0, 0) + self.nd.create_project( + 'testp', 'test', 'localhost', 1, 1, 'localhost', 'Redis') + self.nd.create_token('testt', 'testp', 'test', 1) + result = self.nd.delete_token('testt', 'testp', 'test') + self.assertEqual(result, True) + self.nd.delete_project('testp', 'test') + self.nd.delete_dataset('test') + + def test_list_token(self): + self.nd.create_dataset('test', 1, 1, 1, 1.0, 1.0, 1.0, 0) + test_token_names = [u'test1t', u'test2t', u'test3t'] + self.nd.create_project( + 'test1p', 'test', 'localhost', 1, 1, 'localhost', 'Redis') + self.nd.create_project( + 'test2p', 'test', 'localhost', 1, 1, 'localhost', 'Redis') + self.nd.create_token('test1t', 'test1p', 'test', 1) + self.nd.create_token('test2t', 'test1p', 'test', 1) + self.nd.create_token('test3t', 'test2p', 'test', 1) + test_token_list = self.nd.list_tokens() + for name in test_token_names: + self.assertEqual(name in test_token_list, True) + self.nd.delete_token('test1t', 'test1p', 'test') + self.nd.delete_token('test2t', 'test1p', 'test') + self.nd.delete_token('test3t', 'test2p', 'test') + self.nd.delete_project('test1p', 'test') + self.nd.delete_project('test2p', 'test') + self.nd.delete_dataset('test') + + + if __name__ == '__main__': unittest.main() From 7da3ee403cdd048ca560ee9d16c30a9e8c56d9d8 Mon Sep 17 00:00:00 2001 From: Alex Eusman Date: Mon, 3 Apr 2017 13:03:44 -0400 Subject: [PATCH 26/42] Added the set up for the dataset/project/token so we can test. One fails because for some reason backend doesn't consider it authenticatied --- tests/test_getURL.py | 34 +++++++++++++++++++++++++++++++--- tests/test_settings.py | 8 ++++---- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/tests/test_getURL.py b/tests/test_getURL.py index 9e6cb20..48209f4 100644 --- a/tests/test_getURL.py +++ b/tests/test_getURL.py @@ -12,9 +12,36 @@ def setUp(self): self.neurodata = neurodata(user_token=NEURODATA, hostname=HOSTNAME) self.public = neurodata(hostname=HOSTNAME) self.test = neurodata(user_token=TEST, hostname=HOSTNAME) - self.private_project = "https://{}/sd/private_neuro/info/".format(HOSTNAME) - self.public_project = "https://{}/sd/public_neuro/info/".format(HOSTNAME) - self.private_test_project = "https://{}/sd/private_test_neuro/info/".format(HOSTNAME) + self.private_project = "https://{}/nd/sd/private_neuro/info/".format(HOSTNAME) + self.public_project = "https://{}/nd/sd/public_neuro/info/".format(HOSTNAME) + self.private_test_project = "https://{}/nd/sd/private_test_neuro/info/".format(HOSTNAME) + + #Create the projects/tokens that are required for this + self.neurodata.create_dataset('private_neuro', 10, 10, 10, 1, 1, 1) + self.neurodata.create_project('private_neuro', 'private_neuro', 'localhost', 0) + self.neurodata.create_token('private_neuro', 'private_neuro', 'private_neuro', 0) + + self.neurodata.create_dataset('public_neuro', 10, 10, 10, 1, 1, 1) + self.neurodata.create_project('public_neuro', 'public_neuro', 'localhost', 1) + self.neurodata.create_token('public_neuro', 'public_neuro', 'public_neuro', 1) + + self.test.create_dataset('private_test_neuro', 10, 10, 10, 1, 1, 1) + self.test.create_project('private_test_neuro', 'private_test_neuro', 'localhost', 0) + self.test.create_token('private_test_neuro', 'private_test_neuro', 'private_test_neuro', 0) + + def tearDown(self): + self.neurodata.delete_token('private_neuro', 'private_neuro', 'private_neuro') + self.neurodata.delete_project('private_neuro', 'private_neuro') + self.neurodata.delete_dataset('private_neuro') + + self.neurodata.delete_token('public_neuro', 'public_neuro', 'public_neuro') + self.neurodata.delete_project('public_neuro', 'public_neuro') + self.neurodata.delete_dataset('public_neuro') + + self.neurodata.delete_token('private_test_neuro', 'private_test_neuro', 'private_test_neuro') + self.neurodata.delete_project('private_test_neuro', 'private_test_neuro') + self.neurodata.delete_dataset('private_test_neuro') + def test_masterToken(self): try: req = self.neurodata.getURL(self.private_project) @@ -35,6 +62,7 @@ def test_masterToken(self): print(e) def test_testToken(self): + import pdb; pdb.set_trace() try: req = self.test.getURL(self.private_project) self.assertEqual(req.status_code, 403) diff --git a/tests/test_settings.py b/tests/test_settings.py index 874187b..a34e38b 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -11,7 +11,7 @@ DEV_MODE = True # server to check against -# HOSTNAME = 'localhost:8080' -HOSTNAME = 'localhost/nd' -NEURODATA = open('/tmp/token_super', 'r').read() -TEST = open('/tmp/token_user', 'r').read() +#HOSTNAME = 'localhost:8080' +HOSTNAME = 'localhost' +NEURODATA = open('/tmp/token_super', 'r').read().replace('\n', '') +TEST = open('/tmp/token_user', 'r').read().replace('\n', '') From b9328f29dfb20ac9429bd9864447216e802cae96 Mon Sep 17 00:00:00 2001 From: Alex Eusman Date: Mon, 3 Apr 2017 14:22:48 -0400 Subject: [PATCH 27/42] Corrections to resources test --- tests/test_resources_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_resources_api.py b/tests/test_resources_api.py index 5e7f011..9d45836 100644 --- a/tests/test_resources_api.py +++ b/tests/test_resources_api.py @@ -8,7 +8,7 @@ class TestResourcesApi(unittest.TestCase): def setUp(self): - self.token_user = '043813bf95d9d5be2bb19448d5a9337db086c559' + self.token_user = test_settings.NEURODATA hostname = 'localhost' self.nd = nd(user_token=self.token_user, hostname=hostname) From 1364718688eec045f11edd36ad0dc97ca430ae57 Mon Sep 17 00:00:00 2001 From: Alex Eusman Date: Mon, 3 Apr 2017 15:16:22 -0400 Subject: [PATCH 28/42] Fixing checkstyle issues --- ndio/remote/neurodata.py | 15 ++++++++------- tests/test_getURL.py | 1 - 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ndio/remote/neurodata.py b/ndio/remote/neurodata.py index a934da2..ae14457 100644 --- a/ndio/remote/neurodata.py +++ b/ndio/remote/neurodata.py @@ -320,7 +320,6 @@ def create_token(self, Returns: bool: True if project created, false if not created. """ - url = self.url()[:-4] + '/nd/resource/dataset/{}'.format( dataset_name) + '/project/{}'.format(project_name) + \ '/token/{}/'.format(token_name) @@ -355,7 +354,8 @@ def get_token(self, dict: Token info """ url = self.url()[:-4] + "/nd/resource/dataset/{}".format(dataset_name)\ - + "/project/{}".format(project_name) + "/token/{}/".format(token_name) + + "/project/{}".format(project_name)\ + + "/token/{}/".format(token_name) req = self.getURL(url) if req.status_code is not 200: @@ -380,7 +380,8 @@ def delete_token(self, bool: True if project deleted, false if not deleted. """ url = self.url()[:-4] + "/nd/resource/dataset/{}".format(dataset_name)\ - + "/project/{}".format(project_name) + "/token/{}/".format(token_name) + + "/project/{}".format(project_name)\ + + "/token/{}/".format(token_name) req = self.delete_url(url) if req.status_code is not 204: @@ -395,7 +396,7 @@ def list_tokens(self): Lists a set of tokens that are public in Neurodata. Arguments: - + Returns: dict: Public tokens found in Neurodata """ @@ -407,7 +408,6 @@ def list_tokens(self): else: return req.json() - def create_project(self, project_name, dataset_name, @@ -1318,7 +1318,8 @@ def delete_ramon(self, token, channel, anno): else: a = anno.id - req = requests.delete(self.url("{}/{}/{}/".format(token, channel, a)), verify=False) + req = requests.delete(self.url("{}/{}/{}/".format(token, channel, a)), + verify=False) if req.status_code is not 200: raise RemoteDataNotFoundError("Could not delete id {}: {}" .format(a, req.text)) @@ -1769,4 +1770,4 @@ def get_propagate_status(self, token, channel): req = self.getURL(url) if req.status_code is not 200: raise ValueError('Bad pair: {}/{}'.format(token, channel)) - return req.text \ No newline at end of file + return req.text diff --git a/tests/test_getURL.py b/tests/test_getURL.py index 48209f4..de6e9d2 100644 --- a/tests/test_getURL.py +++ b/tests/test_getURL.py @@ -62,7 +62,6 @@ def test_masterToken(self): print(e) def test_testToken(self): - import pdb; pdb.set_trace() try: req = self.test.getURL(self.private_project) self.assertEqual(req.status_code, 403) From ce5d589218e6b741d724ba8e4ac0a92e40cd7d77 Mon Sep 17 00:00:00 2001 From: Alex Eusman Date: Mon, 3 Apr 2017 15:19:35 -0400 Subject: [PATCH 29/42] Correcting pep8 in convert --- ndio/convert/convert.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ndio/convert/convert.py b/ndio/convert/convert.py index 36ecea1..3412065 100644 --- a/ndio/convert/convert.py +++ b/ndio/convert/convert.py @@ -118,9 +118,9 @@ def convert(in_file, out_file, in_fmt="", out_fmt=""): # Get formats, either by explicitly naming them or by guessing. # TODO: It'd be neat to check here if an explicit fmt matches the guess. in_fmt = in_fmt.lower() or _guess_format_from_extension( - in_file.split('.')[-1].lower()) + in_file.split('.')[-1].lower()) out_fmt = out_fmt.lower() or _guess_format_from_extension( - out_file.split('.')[-1].lower()) + out_file.split('.')[-1].lower()) if not in_fmt or not out_fmt: raise ValueError("Cannot determine conversion formats.") From c96d2285389799571aba69aebae8f56525b73c3e Mon Sep 17 00:00:00 2001 From: jyshin93 Date: Mon, 3 Apr 2017 19:34:32 -0400 Subject: [PATCH 30/42] pull --- tests/test_getURL.py | 22 ++++++++++++++++++++++ tests/test_resources_api.py | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/tests/test_getURL.py b/tests/test_getURL.py index 9e6cb20..e532d88 100644 --- a/tests/test_getURL.py +++ b/tests/test_getURL.py @@ -64,5 +64,27 @@ def test_publicToken(self): except ValueError as e: print(e) + def test_newToken(self): + test = neurodata(user_token=TEST, hostname='localhost') + test.create_dataset('test', 1, 1, 1, 1.0, 1.0, 1.0, 0) + test.create_project('testp', 'test', 'localhost', 1, 1, 'localhost', 'Redis') + test.create_token('testt', 'testp', 'test', 1) + new_token = neurodata(user_token='testt', hostname=HOSTNAME) + new_token_project = "https://{}/sd/testp/info/".format(HOSTNAME) + # try: + # req = new_token.getURL(new_token_project) + # self.assertEqual(req.status_code, 200) + # except ValueError as e: + # print(e) + # try: + # req2 = new_token.getURL(self.private_project) + # self.assertEqual(req2.status_code, 401) + # except ValueError as e: + # print(e) + + test.delete_token('testt', 'testp', 'test') + test.delete_project('testp', 'test') + test.delete_dataset('test') + if __name__ == '__main__': unittest.main() diff --git a/tests/test_resources_api.py b/tests/test_resources_api.py index 5e7f011..38e699f 100644 --- a/tests/test_resources_api.py +++ b/tests/test_resources_api.py @@ -8,7 +8,7 @@ class TestResourcesApi(unittest.TestCase): def setUp(self): - self.token_user = '043813bf95d9d5be2bb19448d5a9337db086c559' + self.token_user = test_settings.TEST hostname = 'localhost' self.nd = nd(user_token=self.token_user, hostname=hostname) From 52c8babe253b07470f7a4abb4ef7e97004babd1b Mon Sep 17 00:00:00 2001 From: jyshin93 Date: Fri, 7 Apr 2017 00:18:10 -0400 Subject: [PATCH 31/42] test_propagate old -> test_propagate --- tests/test_propagate.py | 44 +++++++++++++++++++++++++++++++++++++ tests/test_propagate.py.old | 25 --------------------- 2 files changed, 44 insertions(+), 25 deletions(-) create mode 100644 tests/test_propagate.py delete mode 100644 tests/test_propagate.py.old diff --git a/tests/test_propagate.py b/tests/test_propagate.py new file mode 100644 index 0000000..329790c --- /dev/null +++ b/tests/test_propagate.py @@ -0,0 +1,44 @@ +import unittest +import ndio.remote.neurodata as neurodata +import ndio.remote.errors +import test_settings +import numpy +import h5py +import os + + +class TestPropagate(unittest.TestCase): + + def setUp(self): + hostname = test_settings.HOSTNAME + self.token_user = test_settings.NEURODATA + self.nd = neurodata(user_token=self.token_user, hostname=hostname) + + def test_propagate_status_fails_on_bad_token(self): + token = 'this is not a token' + with self.assertRaises(ValueError): + self.nd.get_propagate_status(token, 'channel') + + + #Channel is by default not allowed to be propagated. But pass the test + def test_propagated(self): + self.nd.create_dataset('test', 1, 1, 1, 1.0, 1.0, 1.0, 0) + self.nd.create_project( + 'testp', 'test', 'localhost', 1, 1, 'localhost', 'Redis') + self.nd.create_channel( + 'testc', 'testp', 'test', 'image', 'uint8', 0, 500, 0) + self.nd.create_token('testt', 'testp', 'test', 1) + + self.assertEqual(self.nd.propagate('testt', 'testc'), True) + #Is it supposed to be 1? + self.assertEqual(self.nd.get_propagate_status('testt', 'testc'), '1') + + self.nd.delete_token('testt', 'testp', 'test') + self.nd.delete_channel('testc', 'testp', 'test') + self.nd.delete_project('testp', 'test') + self.nd.delete_dataset('test') + + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_propagate.py.old b/tests/test_propagate.py.old deleted file mode 100644 index 15cab7f..0000000 --- a/tests/test_propagate.py.old +++ /dev/null @@ -1,25 +0,0 @@ -import unittest -import ndio.remote.neurodata as neurodata -import ndio.remote.errors -import numpy -import h5py -import os - - -class TestPropagate(unittest.TestCase): - - def setUp(self): - self.nd = neurodata(check_tokens=True) - - def test_propagate_status_fails_on_bad_token(self): - token = 'this is not a token' - with self.assertRaises(ndio.remote.errors.RemoteDataNotFoundError): - self.nd.get_propagate_status(token, 'channel') - - def test_kasthuri11_is_propagated(self): - token = 'kasthuri11' - self.assertEqual(self.nd.get_propagate_status(token, 'image'), '2') - - -if __name__ == '__main__': - unittest.main() From fb878a3e8762bf8fc5702742c2abace69a23ac7f Mon Sep 17 00:00:00 2001 From: jyshin93 Date: Sat, 8 Apr 2017 18:25:10 -0400 Subject: [PATCH 32/42] test_convert.old --> test_conver.py update --- tests/test_convert_image_export.py | 50 +++++++++++++++++++++++ tests/test_convert_image_export.py.old | 56 -------------------------- tests/test_propagate.py | 4 +- tests/test_ramon_import_export.py | 43 ++++++++++++++++++++ tests/test_ramon_import_export.py.old | 28 ------------- tests/trash/.ignore_me | 2 - 6 files changed, 95 insertions(+), 88 deletions(-) create mode 100644 tests/test_convert_image_export.py delete mode 100644 tests/test_convert_image_export.py.old create mode 100644 tests/test_ramon_import_export.py delete mode 100644 tests/test_ramon_import_export.py.old delete mode 100644 tests/trash/.ignore_me diff --git a/tests/test_convert_image_export.py b/tests/test_convert_image_export.py new file mode 100644 index 0000000..be7afc1 --- /dev/null +++ b/tests/test_convert_image_export.py @@ -0,0 +1,50 @@ +import unittest +import test_settings +import ndio.remote.neurodata as neurodata +import ndio.ramon +import ndio.convert.png as ndpng +import ndio.convert.tiff as ndtiff +import numpy as np + + +class TestDownload(unittest.TestCase): + + def setUp(self): + self.hostname = test_settings.HOSTNAME + self.token_user = test_settings.NEURODATA + self.nd = neurodata(user_token=self.token_user, hostname=self.hostname) + self.image_data = np.zeros((512,512), dtype=np.uint8) + + # self.image_data = (255.0 / data.max() * (data - data.min())).astype(np.uint8) + + def test_export_load_png(self): + + # if returns string, successful export + self.assertEqual( + ndpng.save("trash/download.png", self.image_data), + "trash/download.png") + + # now confirm import works too + self.assertEqual(ndpng.load("trash/download.png")[0][0], + self.image_data[0][0]) + self.assertEqual(ndpng.load("trash/download.png")[10][10], + self.image_data[10][10]) + + + def test_export_load_tiff(self): + # if returns string, successful export + self.assertEqual( + ndtiff.save("trash/download-1.tiff", + self.image_data), + "trash/download-1.tiff") + + # now confirm import works too + self.assertEqual( + ndtiff.load("trash/download-1.tiff")[0][0], + self.image_data[0][0]) + self.assertEqual( + ndtiff.load("trash/download-1.tiff")[10][10], + self.image_data[10][10]) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_convert_image_export.py.old b/tests/test_convert_image_export.py.old deleted file mode 100644 index 672d023..0000000 --- a/tests/test_convert_image_export.py.old +++ /dev/null @@ -1,56 +0,0 @@ -import unittest -import ndio.remote.neurodata as neurodata -import ndio.ramon -import ndio.convert.png as ndpng -import ndio.convert.tiff as ndtiff -import numpy - - -class TestDownload(unittest.TestCase): - - def setUp(self): - self.oo = neurodata() - - def test_export_load(self): - # kasthuri11/image/xy/3/1000,1100/1000,1100/1000/ - image_download = self.oo.get_image('kasthuri11', 'image', - 1000, 1100, - 1000, 1100, - 1000, - resolution=3) - - # if returns string, successful export - self.assertEqual( - ndpng.export_png("tests/trash/download.png", image_download), - "tests/trash/download.png") - - # now confirm import works too - self.assertEqual(ndpng.load("tests/trash/download.png")[0][0], - image_download[0][0]) - self.assertEqual(ndpng.load("tests/trash/download.png")[10][10], - image_download[10][10]) - - def test_export_load(self): - # kasthuri11/image/xy/3/1000,1100/1000,1100/1000/ - image_download = self.oo.get_image('kasthuri11', 'image', - 1000, 1100, - 1000, 1100, - 1000, - resolution=3) - - # if returns string, successful export - self.assertEqual( - ndtiff.save("tests/trash/download-1.tiff", - image_download), - "tests/trash/download-1.tiff") - - # now confirm import works too - self.assertEqual( - ndtiff.load("tests/trash/download-1.tiff")[0][0], - image_download[0][0]) - self.assertEqual( - ndtiff.load("tests/trash/download-1.tiff")[10][10], - image_download[10][10]) - -if __name__ == '__main__': - unittest.main() diff --git a/tests/test_propagate.py b/tests/test_propagate.py index 329790c..3e4f13f 100644 --- a/tests/test_propagate.py +++ b/tests/test_propagate.py @@ -10,8 +10,8 @@ class TestPropagate(unittest.TestCase): def setUp(self): - hostname = test_settings.HOSTNAME - self.token_user = test_settings.NEURODATA + hostname = test_settings.HOSTNAME + self.token_user = test_settings.NEURODATA self.nd = neurodata(user_token=self.token_user, hostname=hostname) def test_propagate_status_fails_on_bad_token(self): diff --git a/tests/test_ramon_import_export.py b/tests/test_ramon_import_export.py new file mode 100644 index 0000000..246b5cd --- /dev/null +++ b/tests/test_ramon_import_export.py @@ -0,0 +1,43 @@ +import unittest +import ndio.remote.neurodata as neurodata +import ndio.ramon as ramon +import test_settings +import numpy +import h5py +import os +import random +import json + + +class TestRAMON(unittest.TestCase): + + def setUp(self): + self.user = test_settings.NEURODATA + self.hostname = test_settings.HOSTNAME + self.nd = neurodata(user_token=self.user, hostname=self.hostname) + self.ramon_id = 1 + + def test_json_export(self): + self.nd.create_dataset('test', 1, 1, 1, 1.0, 1.0, 1.0, 0) + self.nd.create_project( + 'testp', 'test', 'localhost', 1, 1, 'localhost', 'Redis') + self.nd.delete_channel('testc', 'testp', 'test') + self.nd.create_channel( + 'testc', 'testp', 'test', 'image', 'uint8', 0, 500, 0) + self.nd.create_token('testt', 'testp', 'test', 1) + + r = self.nd.get_ramon('testt', 'testc', self.ramon_id) + + self.assertEqual( + str(self.ramon_id), + json.loads(ramon.to_json(r))['1']['id'] + ) + + self.nd.delete_token('testt', 'testp', 'test') + self.nd.delete_channel('testc', 'testp', 'test') + self.nd.delete_project('testp', 'test') + self.nd.delete_dataset('test') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_ramon_import_export.py.old b/tests/test_ramon_import_export.py.old deleted file mode 100644 index 765f553..0000000 --- a/tests/test_ramon_import_export.py.old +++ /dev/null @@ -1,28 +0,0 @@ -import unittest -import ndio.remote.neurodata as neurodata -import ndio.ramon as ramon -import numpy -import h5py -import os -import random -import json - - -class TestRAMON(unittest.TestCase): - - def setUp(self): - self.nd = neurodata() - self.t = 'kasthuri2015_ramon_v4' - self.c = 'neurons' - self.ramon_id = 1 - - def test_json_export(self): - r = self.nd.get_ramon(self.t, self.c, self.ramon_id) - self.assertEqual( - str(self.ramon_id), - json.loads(ramon.to_json(r))['1']['id'] - ) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/trash/.ignore_me b/tests/trash/.ignore_me deleted file mode 100644 index d6b7ef3..0000000 --- a/tests/trash/.ignore_me +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore From 4ded9a889e37c4d721d44acb1b2c7ced5c6189ad Mon Sep 17 00:00:00 2001 From: jyshin93 Date: Thu, 13 Apr 2017 12:06:00 -0400 Subject: [PATCH 33/42] u --- tests/test_convert_image_export.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_convert_image_export.py b/tests/test_convert_image_export.py index be7afc1..42d6e4c 100644 --- a/tests/test_convert_image_export.py +++ b/tests/test_convert_image_export.py @@ -20,6 +20,7 @@ def setUp(self): def test_export_load_png(self): # if returns string, successful export + # python remove self.assertEqual( ndpng.save("trash/download.png", self.image_data), "trash/download.png") From b001d9fc500b1523213f4841c2068b4e95272afb Mon Sep 17 00:00:00 2001 From: jyshin93 Date: Thu, 13 Apr 2017 12:11:38 -0400 Subject: [PATCH 34/42] update conver_image_export_import --- tests/data/test.graphml | 209 ------------------ tests/test_convert_image_export.py | 21 +- ...ort.py => test_ramon_import_export.py.old} | 0 3 files changed, 11 insertions(+), 219 deletions(-) delete mode 100644 tests/data/test.graphml rename tests/{test_ramon_import_export.py => test_ramon_import_export.py.old} (100%) diff --git a/tests/data/test.graphml b/tests/data/test.graphml deleted file mode 100644 index afa0b0d..0000000 --- a/tests/data/test.graphml +++ /dev/null @@ -1,209 +0,0 @@ - - - - - - - - - Erdos renyi (gnp) graph - gnp - false - 0.4 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/tests/test_convert_image_export.py b/tests/test_convert_image_export.py index 42d6e4c..90dcdb0 100644 --- a/tests/test_convert_image_export.py +++ b/tests/test_convert_image_export.py @@ -5,6 +5,7 @@ import ndio.convert.png as ndpng import ndio.convert.tiff as ndtiff import numpy as np +import os class TestDownload(unittest.TestCase): @@ -20,32 +21,32 @@ def setUp(self): def test_export_load_png(self): # if returns string, successful export - # python remove self.assertEqual( - ndpng.save("trash/download.png", self.image_data), - "trash/download.png") + ndpng.save("download.png", self.image_data), + "download.png") # now confirm import works too - self.assertEqual(ndpng.load("trash/download.png")[0][0], + self.assertEqual(ndpng.load("download.png")[0][0], self.image_data[0][0]) - self.assertEqual(ndpng.load("trash/download.png")[10][10], + self.assertEqual(ndpng.load("download.png")[10][10], self.image_data[10][10]) - + os.system("rm ./download.png") def test_export_load_tiff(self): # if returns string, successful export self.assertEqual( - ndtiff.save("trash/download-1.tiff", + ndtiff.save("download-1.tiff", self.image_data), - "trash/download-1.tiff") + "download-1.tiff") # now confirm import works too self.assertEqual( - ndtiff.load("trash/download-1.tiff")[0][0], + ndtiff.load("download-1.tiff")[0][0], self.image_data[0][0]) self.assertEqual( - ndtiff.load("trash/download-1.tiff")[10][10], + ndtiff.load("download-1.tiff")[10][10], self.image_data[10][10]) + os.system("rm ./download-1.tiff") if __name__ == '__main__': unittest.main() diff --git a/tests/test_ramon_import_export.py b/tests/test_ramon_import_export.py.old similarity index 100% rename from tests/test_ramon_import_export.py rename to tests/test_ramon_import_export.py.old From f1166fc39450a207c5ddd2397e12a9d0dbbf2386 Mon Sep 17 00:00:00 2001 From: jyshin93 Date: Wed, 19 Apr 2017 00:15:31 -0400 Subject: [PATCH 35/42] commit before pull --- ndio/remote/neurodata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ndio/remote/neurodata.py b/ndio/remote/neurodata.py index ae14457..e8d2b8e 100644 --- a/ndio/remote/neurodata.py +++ b/ndio/remote/neurodata.py @@ -450,7 +450,7 @@ def create_project(self, } req = self.post_url(url, json=json) - + import pdb; pdb.set_trace() if req.status_code is not 201: raise RemoteDataUploadError('Could not upload {}'.format(req.text)) if req.content == "" or req.content == b'': From 1556aaac870f5da322fbc154b5d176b53e8a16a1 Mon Sep 17 00:00:00 2001 From: jyshin93 Date: Wed, 19 Apr 2017 00:46:19 -0400 Subject: [PATCH 36/42] pudate post cutout to upload 4D but no time series yet --- ndio/remote/data.py | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/ndio/remote/data.py b/ndio/remote/data.py index a7ed369..392ac42 100644 --- a/ndio/remote/data.py +++ b/ndio/remote/data.py @@ -345,6 +345,7 @@ def post_cutout(self, token, channel, x_start, y_start, z_start, + t_start=0, data, resolution=0): """ @@ -371,6 +372,7 @@ def post_cutout(self, token, channel, data = numpy.rollaxis(data, 1) data = numpy.rollaxis(data, 2) + data = numpy.rollaxis(data, 3) if six.PY3 or data.nbytes > 1.5e9: ul_func = self._post_cutout_no_chunking_npz @@ -379,48 +381,51 @@ def post_cutout(self, token, channel, if data.size < self._chunk_threshold: return ul_func(token, channel, x_start, - y_start, z_start, data, - resolution) + y_start, z_start, t_start, + data, resolution) return self._post_cutout_with_chunking(token, channel, - x_start, y_start, z_start, data, - resolution, ul_func) + x_start, y_start, z_start, t_start, + data, resolution, ul_func) def _post_cutout_with_chunking(self, token, channel, x_start, - y_start, z_start, data, + y_start, z_start, t_start, data, resolution, ul_func): # must chunk first from ndio.utils.parallel import block_compute - blocks = block_compute(x_start, x_start + data.shape[2], - y_start, y_start + data.shape[1], - z_start, z_start + data.shape[0]) + blocks = block_compute(x_start, x_start + data.shape[3], + y_start, y_start + data.shape[2], + z_start, z_start + data.shape[1], + t_start, t_start + data.shape[0]) for b in blocks: # data coordinate relative to the size of the arra - subvol = data[b[2][0] - z_start: b[2][1] - z_start, + subvol = data[b[3][0] - t_start: b[3][1] - t_start, + b[2][0] - z_start: b[2][1] - z_start, b[1][0] - y_start: b[1][1] - y_start, b[0][0] - x_start: b[0][1] - x_start] # upload the chunk: # upload coordinate relative to x_start, y_start, z_start ul_func(token, channel, b[0][0], - b[1][0], b[2][0], subvol, - resolution) + b[1][0], b[2][0], b[3][0], + subvol, resolution) return True def _post_cutout_no_chunking_npz(self, token, channel, x_start, y_start, z_start, - data, resolution): + t_start, data, resolution): data = numpy.expand_dims(data, axis=0) tempfile = BytesIO() numpy.save(tempfile, data) compressed = zlib.compress(tempfile.getvalue()) - url = self.url("{}/{}/npz/{}/{},{}/{},{}/{},{}/".format( + url = self.url("{}/{}/npz/{}/{},{}/{},{}/{},{}/{},{}/".format( token, channel, resolution, x_start, x_start + data.shape[3], y_start, y_start + data.shape[2], - z_start, z_start + data.shape[1] + z_start, z_start + data.shape[1], + t_start, t_start + data.shape[0] )) req = self.remote_utils.post_url(url, data=compressed, headers={ @@ -434,18 +439,19 @@ def _post_cutout_no_chunking_npz(self, token, channel, def _post_cutout_no_chunking_blosc(self, token, channel, x_start, y_start, z_start, - data, resolution): + t_start, data, resolution): """ Accepts data in zyx. !!! """ data = numpy.expand_dims(data, axis=0) blosc_data = blosc.pack_array(data) - url = self.url("{}/{}/blosc/{}/{},{}/{},{}/{},{}/0,0/".format( + url = self.url("{}/{}/blosc/{}/{},{}/{},{}/{},{}/{},{}/0,0/".format( token, channel, resolution, x_start, x_start + data.shape[3], y_start, y_start + data.shape[2], - z_start, z_start + data.shape[1] + z_start, z_start + data.shape[1], + t_start, t_start + data.shape[0] )) req = self.remote_utils.post_url(url, data=blosc_data, headers={ From 0c9301244a2d3931983e4ae57fd9165cb0287fa8 Mon Sep 17 00:00:00 2001 From: jyshin93 Date: Thu, 11 May 2017 23:00:22 -0400 Subject: [PATCH 37/42] middle of data --- tests/test_data.py | 50 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 tests/test_data.py diff --git a/tests/test_data.py b/tests/test_data.py new file mode 100644 index 0000000..6f0656e --- /dev/null +++ b/tests/test_data.py @@ -0,0 +1,50 @@ +import unittest +from ndio.remote.neurodata import neurodata as neurodata +import ndio.remote.errors +import numpy +import h5py +import os +import test_settings + +HOSTNAME = test_settings.HOSTNAME +NEURODATA = test_settings.NEURODATA +TEST = test_settings.TEST + +class TestData(unittest.TestCase): + + def setUp(self): + self.token_user = NEURODATA + hostname = HOSTNAME + self.nd = neurodata(self.token_user, + hostname=hostname, + check_tokens=True) + self.ramon_id = 1 + dataset_name = 'demo1' + project_name = 'ndio_demos' + self.t = 'test_token' + self.c = 'test_channel' + self.nd.create_dataset(dataset_name, 100, 100, 100, 1.0, + 1.0, + 1.0) + self.nd.create_project(project_name, dataset_name, + 'localhost', + 1, 1, 'localhost', 'Redis') + self.nd.create_token(self.t, project_name, dataset_name, 1) + try: + self.nd.create_channel(self.c, project_name, + dataset_name, 'timeseries', + 'uint8', 0, 500, propagate=1) + except: + pass + + #random 3-d array + self.data = numpy.arange(30).reshape(2,3,5) + + def test_post_cutout(self): + self.nd.post_cutout(self.t, self.c, 0, 0, 0, 0, self.data) + + + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file From dd734147420ce0a08030aa95b2ee65e27ceae5e0 Mon Sep 17 00:00:00 2001 From: jyshin93 Date: Thu, 11 May 2017 23:04:33 -0400 Subject: [PATCH 38/42] middle --- tests/test_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_data.py b/tests/test_data.py index 6f0656e..d693ab2 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -1,5 +1,5 @@ import unittest -from ndio.remote.neurodata import neurodata as neurodata +import ndio.remote.neurodata as neurodata import ndio.remote.errors import numpy import h5py From 35c34d9142f93a797ca0145275593c3c4f7611eb Mon Sep 17 00:00:00 2001 From: jyshin93 Date: Thu, 11 May 2017 23:36:49 -0400 Subject: [PATCH 39/42] data --- ndio/remote/data.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/ndio/remote/data.py b/ndio/remote/data.py index 392ac42..9a8cdf0 100644 --- a/ndio/remote/data.py +++ b/ndio/remote/data.py @@ -345,7 +345,7 @@ def post_cutout(self, token, channel, x_start, y_start, z_start, - t_start=0, + t_start, data, resolution=0): """ @@ -372,7 +372,6 @@ def post_cutout(self, token, channel, data = numpy.rollaxis(data, 1) data = numpy.rollaxis(data, 2) - data = numpy.rollaxis(data, 3) if six.PY3 or data.nbytes > 1.5e9: ul_func = self._post_cutout_no_chunking_npz @@ -393,21 +392,21 @@ def _post_cutout_with_chunking(self, token, channel, x_start, resolution, ul_func): # must chunk first from ndio.utils.parallel import block_compute - blocks = block_compute(x_start, x_start + data.shape[3], - y_start, y_start + data.shape[2], - z_start, z_start + data.shape[1], - t_start, t_start + data.shape[0]) + blocks = block_compute(x_start, x_start + data.shape[2], + y_start, y_start + data.shape[1], + z_start, z_start + data.shape[0]) + t_interval = 0 for b in blocks: # data coordinate relative to the size of the arra - subvol = data[b[3][0] - t_start: b[3][1] - t_start, - b[2][0] - z_start: b[2][1] - z_start, + subvol = data[b[2][0] - z_start: b[2][1] - z_start, b[1][0] - y_start: b[1][1] - y_start, b[0][0] - x_start: b[0][1] - x_start] # upload the chunk: # upload coordinate relative to x_start, y_start, z_start ul_func(token, channel, b[0][0], - b[1][0], b[2][0], b[3][0], + b[1][0], b[2][0], t_interval, subvol, resolution) + t_interval += 1 return True def _post_cutout_no_chunking_npz(self, token, channel, @@ -425,7 +424,7 @@ def _post_cutout_no_chunking_npz(self, token, channel, x_start, x_start + data.shape[3], y_start, y_start + data.shape[2], z_start, z_start + data.shape[1], - t_start, t_start + data.shape[0] + t_start, t_start + 1 )) req = self.remote_utils.post_url(url, data=compressed, headers={ @@ -451,7 +450,7 @@ def _post_cutout_no_chunking_blosc(self, token, channel, x_start, x_start + data.shape[3], y_start, y_start + data.shape[2], z_start, z_start + data.shape[1], - t_start, t_start + data.shape[0] + t_start, t_start + 1 )) req = self.remote_utils.post_url(url, data=blosc_data, headers={ From 31acd936089b6e3096730d179f2590368f2d2f67 Mon Sep 17 00:00:00 2001 From: jyshin93 Date: Fri, 12 May 2017 12:18:23 -0400 Subject: [PATCH 40/42] Rebase abort and recommit --- ndio/remote/data.py | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/ndio/remote/data.py b/ndio/remote/data.py index 9a8cdf0..a7ed369 100644 --- a/ndio/remote/data.py +++ b/ndio/remote/data.py @@ -345,7 +345,6 @@ def post_cutout(self, token, channel, x_start, y_start, z_start, - t_start, data, resolution=0): """ @@ -380,22 +379,21 @@ def post_cutout(self, token, channel, if data.size < self._chunk_threshold: return ul_func(token, channel, x_start, - y_start, z_start, t_start, - data, resolution) + y_start, z_start, data, + resolution) return self._post_cutout_with_chunking(token, channel, - x_start, y_start, z_start, t_start, - data, resolution, ul_func) + x_start, y_start, z_start, data, + resolution, ul_func) def _post_cutout_with_chunking(self, token, channel, x_start, - y_start, z_start, t_start, data, + y_start, z_start, data, resolution, ul_func): # must chunk first from ndio.utils.parallel import block_compute blocks = block_compute(x_start, x_start + data.shape[2], y_start, y_start + data.shape[1], z_start, z_start + data.shape[0]) - t_interval = 0 for b in blocks: # data coordinate relative to the size of the arra subvol = data[b[2][0] - z_start: b[2][1] - z_start, @@ -404,27 +402,25 @@ def _post_cutout_with_chunking(self, token, channel, x_start, # upload the chunk: # upload coordinate relative to x_start, y_start, z_start ul_func(token, channel, b[0][0], - b[1][0], b[2][0], t_interval, - subvol, resolution) - t_interval += 1 + b[1][0], b[2][0], subvol, + resolution) return True def _post_cutout_no_chunking_npz(self, token, channel, x_start, y_start, z_start, - t_start, data, resolution): + data, resolution): data = numpy.expand_dims(data, axis=0) tempfile = BytesIO() numpy.save(tempfile, data) compressed = zlib.compress(tempfile.getvalue()) - url = self.url("{}/{}/npz/{}/{},{}/{},{}/{},{}/{},{}/".format( + url = self.url("{}/{}/npz/{}/{},{}/{},{}/{},{}/".format( token, channel, resolution, x_start, x_start + data.shape[3], y_start, y_start + data.shape[2], - z_start, z_start + data.shape[1], - t_start, t_start + 1 + z_start, z_start + data.shape[1] )) req = self.remote_utils.post_url(url, data=compressed, headers={ @@ -438,19 +434,18 @@ def _post_cutout_no_chunking_npz(self, token, channel, def _post_cutout_no_chunking_blosc(self, token, channel, x_start, y_start, z_start, - t_start, data, resolution): + data, resolution): """ Accepts data in zyx. !!! """ data = numpy.expand_dims(data, axis=0) blosc_data = blosc.pack_array(data) - url = self.url("{}/{}/blosc/{}/{},{}/{},{}/{},{}/{},{}/0,0/".format( + url = self.url("{}/{}/blosc/{}/{},{}/{},{}/{},{}/0,0/".format( token, channel, resolution, x_start, x_start + data.shape[3], y_start, y_start + data.shape[2], - z_start, z_start + data.shape[1], - t_start, t_start + 1 + z_start, z_start + data.shape[1] )) req = self.remote_utils.post_url(url, data=blosc_data, headers={ From 988696c0af0ac02126fa6f4b648e4bb99b8a5f65 Mon Sep 17 00:00:00 2001 From: jyshin93 Date: Fri, 12 May 2017 12:20:03 -0400 Subject: [PATCH 41/42] rebase master neurodata --- ndio/remote/neurodata.py | 611 --------------------------------------- 1 file changed, 611 deletions(-) diff --git a/ndio/remote/neurodata.py b/ndio/remote/neurodata.py index 4652bd5..d20c80a 100644 --- a/ndio/remote/neurodata.py +++ b/ndio/remote/neurodata.py @@ -65,616 +65,6 @@ def __init__(self, suffix (str: "ocp"): The URL suffix to specify ndstore/microns. If you aren't sure what to do with this, don't specify one. """ - - self._check_tokens = kwargs.get('check_tokens', False) - self._chunk_threshold = kwargs.get('chunk_threshold', 1E9 / 4) - self._ext = kwargs.get('suffix', DEFAULT_SUFFIX) - self._known_tokens = [] - self._user_token = user_token - - # Prepare meta url - self.meta_root = meta_root - if not self.meta_root.endswith('/'): - self.meta_root = self.meta_root + "/" - if self.meta_root.startswith('https'): - self.meta_root = self.meta_root[self.meta_root.index('://') + 3:] - self.meta_protocol = meta_protocol - - super(neurodata, self).__init__(hostname, protocol) - - # SECTION: - # Decorators - def _check_token(f): - @wraps(f) - def wrapped(self, *args, **kwargs): - if self._check_tokens: - if 'token' in kwargs: - token = kwargs['token'] - else: - token = args[0] - if token not in self._known_tokens: - if self.ping('{}/info/'.format(token)) != 200: - raise RemoteDataNotFoundError("Bad token {}".format( - token)) - else: - self._known_tokens.append(token) - return f(self, *args, **kwargs) - return wrapped - - # SECTION: - # Utilities - def ping(self, suffix='public_tokens/'): - """ - Return the status-code of the API (estimated using the public-tokens - lookup page). - - Arguments: - suffix (str : 'public_tokens/'): The url endpoint to check - - Returns: - int: status code - """ - return super(neurodata, self).ping(suffix) - - def url(self, suffix=""): - """ - Return a constructed URL, appending an optional suffix (uri path). - - Arguments: - suffix (str : ""): The suffix to append to the end of the URL - - Returns: - str: The complete URL - """ - return super(neurodata, self).url('{}/sd/'.format(self._ext) + suffix) - - def meta_url(self, suffix=""): - """ - Return a constructed URL, appending an optional suffix (uri path), - for the metadata server. (Should be temporary, until the LIMS shim - is fixed.) - - Arguments: - suffix (str : ""): The suffix to append to the end of the URL - - Returns: - str: The complete URL - """ - return self.meta_protocol + "://" + self.meta_root + suffix - - def __repr__(self): - """ - Return a string representation that can be used to reproduce this - instance. `eval(repr(this))` should return an identical copy. - - Arguments: - None - - Returns: - str: Representation of reproducible instance. - """ - return "ndio.remote.neurodata('{}', '{}')".format( - self.hostname, - self.protocol, - self.meta_url, - self.meta_protocol - ) - - # SECTION: - # Metadata - def get_public_tokens(self): - """ - Get a list of public tokens available on this server. - - Arguments: - None - - Returns: - str[]: list of public tokens - """ - r = self.getURL(self.url() + "public_tokens/") - return r.json() - - def get_public_datasets(self): - """ - NOTE: VERY SLOW! - Get a list of public datasets. Different than public tokens! - - Arguments: - None - - Returns: - str[]: list of public datasets - """ - return list(self.get_public_datasets_and_tokens().keys()) - - def get_public_datasets_and_tokens(self): - """ - NOTE: VERY SLOW! - Get a dictionary relating key:dataset to value:[tokens] that rely - on that dataset. - - Arguments: - None - - Returns: - dict: relating key:dataset to value:[tokens] - """ - datasets = {} - tokens = self.get_public_tokens() - for t in tokens: - dataset = self.get_token_dataset(t) - if dataset in datasets: - datasets[dataset].append(t) - else: - datasets[dataset] = [t] - return datasets - - def getURL(self, url): - """ - Get the propagate status for a token/channel pair. - - Arguments: - url (str): The url make a get to - - Returns: - obj: The response object - """ - try: - req = requests.get(url, headers={ - 'Authorization': 'Token {}'.format(self._user_token) - }, verify=False) - if req.status_code is 403: - raise ValueError("Access Denied") - else: - return req - except requests.exceptions.ConnectionError as e: - if str(e) == '403 Client Error: Forbidden': - raise ValueError('Access Denied') - else: - raise e - - def post_url(self, url, token='', json=None, data=None, headers=None): - """ - Returns a post resquest object taking in a url, user token, and - possible json information. - - Arguments: - url (str): The url to make post to - token (str): The authentication token - json (dict): json info to send - - Returns: - obj: Post request object - """ - if (token == ''): - token = self._user_token - - if headers: - headers.update({'Authorization': 'Token {}'.format(token)}) - else: - headers = {'Authorization': 'Token {}'.format(token)} - - if json: - return requests.post(url, - headers=headers, - json=json, - verify=False) - if data: - return requests.post(url, - headers=headers, - data=data, - verify=False) - - return requests.post(url, - headers=headers, - verify=False) - - def delete_url(self, url, token=''): - """ - Returns a delete resquest object taking in a url and user token. - - Arguments: - url (str): The url to make post to - token (str): The authentication token - - Returns: - obj: Delete request object - """ - if (token == ''): - token = self._user_token - - return requests.delete(url, - headers={ - 'Authorization': 'Token {}'.format(token)}, - verify=False,) - - @_check_token - def get_token_dataset(self, token): - """ - Get the dataset for a given token. - - Arguments: - token (str): The token to inspect - - Returns: - str: The name of the dataset - """ - return self.get_proj_info(token)['dataset']['description'] - - ################################## - #TOKEN# - ################################## - def create_token(self, - token_name, - project_name, - dataset_name, - is_public): - """ - Creates a token with the given parameters. - - Arguments: - project_name (str): Project name - dataset_name (str): Dataset name project is based on - token_name (str): Token name - is_public (int): 1 is public. 0 is not public - - Returns: - bool: True if project created, false if not created. - """ - url = self.url()[:-4] + '/nd/resource/dataset/{}'.format( - dataset_name) + '/project/{}'.format(project_name) + \ - '/token/{}/'.format(token_name) - - json = { - "token_name": token_name, - "public": is_public - } - - req = self.post_url(url, json=json) - - if req.status_code is not 201: - raise RemoteDataUploadError('Cout not upload {}:'.format(req.text)) - if req.content == "" or req.content == b'': - return True - else: - return False - - def get_token(self, - token_name, - project_name, - dataset_name): - """ - Get a token with the given parameters. - - Arguments: - project_name (str): Project name - dataset_name (str): Dataset name project is based on - token_name (str): Token name - - Returns: - dict: Token info - """ - url = self.url()[:-4] + "/nd/resource/dataset/{}".format(dataset_name)\ - + "/project/{}".format(project_name)\ - + "/token/{}/".format(token_name) - req = self.getURL(url) - - if req.status_code is not 200: - raise RemoteDataUploadError('Could not find {}'.format(req.text)) - else: - return req.json() - - def delete_token(self, - token_name, - project_name, - dataset_name): - """ - Delete a token with the given parameters. - - Arguments: - project_name (str): Project name - dataset_name (str): Dataset name project is based on - token_name (str): Token name - channel_name (str): Channel name project is based on - - Returns: - bool: True if project deleted, false if not deleted. - """ - url = self.url()[:-4] + "/nd/resource/dataset/{}".format(dataset_name)\ - + "/project/{}".format(project_name)\ - + "/token/{}/".format(token_name) - req = self.delete_url(url) - - if req.status_code is not 204: - raise RemoteDataUploadError("Could not delete {}".format(req.text)) - if req.content == "" or req.content == b'': - return True - else: - return False - - def list_tokens(self): - """ - Lists a set of tokens that are public in Neurodata. - - Arguments: - - Returns: - dict: Public tokens found in Neurodata - """ - url = self.url()[:-4] + "/nd/resource/public/token/" - req = self.getURL(url) - - if req.status_code is not 200: - raise RemoteDataNotFoundError('Coud not find {}'.format(req.text)) - else: - return req.json() - - def create_project(self, - project_name, - dataset_name, - hostname, - is_public, - s3backend=0, - kvserver='localhost', - kvengine='MySQL', - mdengine='MySQL', - description=''): - """ - Creates a project with the given parameters. - - Arguments: - project_name (str): Project name - dataset_name (str): Dataset name project is based on - hostname (str): Hostname - s3backend (str): S3 region to save the data in - is_public (int): 1 is public. 0 is not public. - kvserver (str): Server to store key value pairs in - kvengine (str): Database to store key value pairs in - mdengine (str): ??? - description (str): Description for your project - - Returns: - bool: True if project created, false if not created. - """ - url = self.url()[:-4] + "/nd/resource/dataset/{}".format( - dataset_name) + "/project/{}/".format(project_name) - - json = { - "project_name": project_name, - "host": hostname, - "s3backend": s3backend, - "public": is_public, - "kvserver": kvserver, - "kvengine": kvengine, - "mdengine": mdengine, - "project_description": description - } - - req = self.post_url(url, json=json) - import pdb; pdb.set_trace() - if req.status_code is not 201: - raise RemoteDataUploadError('Could not upload {}'.format(req.text)) - if req.content == "" or req.content == b'': - return True - else: - return False - - def get_project(self, project_name, dataset_name): - """ - Get details regarding a project given its name and dataset its linked - to. - - Arguments: - project_name (str): Project name - dataset_name (str): Dataset name project is based on - - Returns: - dict: Project info - """ - url = self.url()[:-4] + "/nd/resource/dataset/{}".format(dataset_name)\ - + "/project/{}/".format(project_name) - req = self.getURL(url) - - if req.status_code is not 200: - raise RemoteDataNotFoundError('Could not find {}'.format(req.text)) - else: - return req.json() - - def delete_project(self, project_name, dataset_name): - """ - Deletes a project once given its name and its related dataset. - - Arguments: - project_name (str): Project name - dataset_name (str): Dataset name project is based on - - Returns: - bool: True if project deleted, False if not. - """ - url = self.url()[:-4] + "/nd/resource/dataset/{}".format(dataset_name)\ - + "/project/{}/".format(project_name) - req = self.delete_url(url) - - if req.status_code is not 204: - raise RemoteDataUploadError('Could not delete {}'.format(req.text)) - if req.content == "" or req.content == b'': - return True - else: - return False - - def list_projects(self, dataset_name): - """ - Lists a set of projects related to a dataset. - - Arguments: - dataset_name (str): Dataset name to search projects for - - Returns: - dict: Projects found based on dataset query - """ - url = self.url()[:-4] + "/nd/resource/dataset/{}".format(dataset_name)\ - + "/project/" - - req = self.getURL(url) - - if req.status_code is not 200: - raise RemoteDataNotFoundError('Could not find {}'.format(req.text)) - else: - return req.json() - - @_check_token - def get_proj_info(self, token): - """ - Return the project info for a given token. - - Arguments: - token (str): Token to return information for - - Returns: - JSON: representation of proj_info - """ - r = self.getURL(self.url() + "{}/info/".format(token)) - return r.json() - - @_check_token - def get_metadata(self, token): - """ - An alias for get_proj_info. - """ - return self.get_proj_info(token) - - @_check_token - def get_channels(self, token): - """ - Wraps get_proj_info to return a dictionary of just the channels of - a given project. - - Arguments: - token (str): Token to return channels for - - Returns: - JSON: dictionary of channels. - """ - return self.get_proj_info(token)['channels'] - - @_check_token - def get_image_size(self, token, resolution=0): - """ - Return the size of the volume (3D). Convenient for when you want - to download the entirety of a dataset. - - Arguments: - token (str): The token for which to find the dataset image bounds - resolution (int : 0): The resolution at which to get image bounds. - Defaults to 0, to get the largest area available. - - Returns: - int[3]: The size of the bounds. Should == get_volume.shape - - Raises: - RemoteDataNotFoundError: If the token is invalid, or if the - metadata at that resolution is unavailable in projinfo. - """ - info = self.get_proj_info(token) - res = str(resolution) - if res not in info['dataset']['imagesize']: - raise RemoteDataNotFoundError("Resolution " + res + - " is not available.") - return info['dataset']['imagesize'][str(resolution)] - - @_check_token - def set_metadata(self, token, data): - """ - Insert new metadata into the OCP metadata database. - - Arguments: - token (str): Token of the datum to set - data (str): A dictionary to insert as metadata. Include `secret`. - - Returns: - json: Info of the inserted ID (convenience) or an error message. - - Throws: - RemoteDataUploadError: If the token is already populated, or if - there is an issue with your specified `secret` key. - """ - req = requests.post(self.meta_url("metadata/ocp/set/" + token), - json=data, verify=False) - - if req.status_code != 200: - raise RemoteDataUploadError( - "Could not upload metadata: " + req.json()['message'] - ) - return req.json() - - @_check_token - def get_subvolumes(self, token): - """ - Return a list of subvolumes taken from LIMS, if available. - - Arguments: - token (str): The token to read from in LIMS - - Returns: - dict: or None if unavailable - """ - md = self.get_metadata(token)['metadata'] - if 'subvolumes' in md: - return md['subvolumes'] - else: - return None - - @_check_token - def add_subvolume(self, token, channel, secret, - x_start, x_stop, - y_start, y_stop, - z_start, z_stop, - resolution, title, notes): - """ - Adds a new subvolume to a token/channel. - - Arguments: - token (str): The token to write to in LIMS - channel (str): Channel to add in the subvolume. Can be `None` - x_start (int): Start in x dimension - x_stop (int): Stop in x dimension - y_start (int): Start in y dimension - y_stop (int): Stop in y dimension - z_start (int): Start in z dimension - z_stop (int): Stop in z dimension - resolution (int): The resolution at which this subvolume is seen - title (str): The title to set for the subvolume - notes (str): Optional extra thoughts on the subvolume - - Returns: - boolean: success - """ - md = self.get_metadata(token)['metadata'] - if 'subvolumes' in md: - subvols = md['subvolumes'] - else: - subvols = [] - - subvols.append({ - 'token': token, - 'channel': channel, - 'x_start': x_start, - 'x_stop': x_stop, - 'y_start': y_start, - 'y_stop': y_stop, - 'z_start': z_start, - 'z_stop': z_stop, - 'resolution': resolution, - 'title': title, - 'notes': notes - }) - - return self.set_metadata(token, { - 'secret': secret, - 'subvolumes': subvols - }) -======= self.data = data(user_token, hostname, protocol, @@ -695,7 +85,6 @@ def add_subvolume(self, token, channel, secret, protocol, meta_root, meta_protocol, **kwargs) ->>>>>>> 071c7cab971b6269b9a398c3bf6a773e8eee9370 # SECTION: # Data Download From de4c375be28e8072d884065cea15a09840b03704 Mon Sep 17 00:00:00 2001 From: Alex Eusman Date: Tue, 23 May 2017 10:23:54 -0400 Subject: [PATCH 42/42] Update test_ramon_import_export.py --- tests/test_ramon_import_export.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_ramon_import_export.py b/tests/test_ramon_import_export.py index d71a359..73cbc03 100644 --- a/tests/test_ramon_import_export.py +++ b/tests/test_ramon_import_export.py @@ -22,7 +22,6 @@ def test_json_export(self): r = ramon.AnnotationType.get_class(anno_type)() # print json.loads(ramon.to_json(r)) self.nd.create_dataset('test', 1, 1, 1, 1.0, 1.0, 1.0, 0) - import pdb; pdb.set_trace() self.nd.create_project( 'testp', 'test', 'localhost', 1, 1, 'localhost', 'MySQL') self.nd.delete_channel('testc', 'testp', 'test')