From 3079c8c7a820ecd5474ef3c8ada71c725620b2b6 Mon Sep 17 00:00:00 2001 From: tomdele Date: Tue, 17 Mar 2020 18:12:14 +0100 Subject: [PATCH 1/6] fix #40 --- bluepysnap/edges.py | 4 ++-- bluepysnap/nodes.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bluepysnap/edges.py b/bluepysnap/edges.py index 0ededf38..eb390029 100644 --- a/bluepysnap/edges.py +++ b/bluepysnap/edges.py @@ -50,7 +50,7 @@ def __init__(self, config, circuit): self._circuit = circuit self._populations = {} - @cached_property + @property def storage(self): """Access to the libsonata edge storage.""" return libsonata.EdgeStorage(self._h5_filepath) @@ -109,7 +109,7 @@ def __init__(self, edge_storage, population_name): self._edge_storage = edge_storage self.name = population_name - @cached_property + @property def _population(self): return self._edge_storage.storage.open_population(self.name) diff --git a/bluepysnap/nodes.py b/bluepysnap/nodes.py index bb289bb1..d99a8b15 100644 --- a/bluepysnap/nodes.py +++ b/bluepysnap/nodes.py @@ -54,7 +54,7 @@ def __init__(self, config, circuit): self._circuit = circuit self._populations = {} - @cached_property + @property def storage(self): """Access to the libsonata node storage.""" return libsonata.NodeStorage(self._h5_filepath) @@ -172,7 +172,7 @@ def _data(self): """Collected data for the node population as a pandas.DataFrame.""" return self._node_storage.load_population_data(self.name) - @cached_property + @property def _population(self): return self._node_storage.storage.open_population(self.name) From 00fd7d6e23d874a11a951bee0f8fb4ee2f3d22f5 Mon Sep 17 00:00:00 2001 From: tomdele Date: Wed, 18 Mar 2020 17:36:17 +0100 Subject: [PATCH 2/6] close_context function --- bluepysnap/circuit.py | 9 +++++++++ bluepysnap/edges.py | 10 +++++++--- bluepysnap/nodes.py | 7 ++++++- tests/test_circuit.py | 30 ++++++++++++++++++++++++------ tests/test_edges.py | 1 + tests/test_nodes.py | 3 ++- 6 files changed, 49 insertions(+), 11 deletions(-) diff --git a/bluepysnap/circuit.py b/bluepysnap/circuit.py index 1e991fcb..bed0e0b5 100644 --- a/bluepysnap/circuit.py +++ b/bluepysnap/circuit.py @@ -70,3 +70,12 @@ def edges(self): self._config['networks']['edges'], lambda cfg: EdgeStorage(cfg, self) ) + + def close_contexts(self): + """Close the context for all populations.""" + if self.nodes: + for population in self.nodes.values(): + population.close_context() + if self.edges: + for population in self.edges.values(): + population.close_context() diff --git a/bluepysnap/edges.py b/bluepysnap/edges.py index eb390029..f09a4861 100644 --- a/bluepysnap/edges.py +++ b/bluepysnap/edges.py @@ -109,11 +109,16 @@ def __init__(self, edge_storage, population_name): self._edge_storage = edge_storage self.name = population_name - @property + @cached_property def _population(self): return self._edge_storage.storage.open_population(self.name) - @property + def close_context(self): + """Close the h5 context for edge population.""" + if "_population" in self.__dict__: + del self.__dict__["_population"] + + @cached_property def size(self): """Population size.""" return self._population.size @@ -396,7 +401,6 @@ def _optimal_direction(): secondary_node_ids = np.unique(secondary_node_ids) secondary_node_ids_used = set() - for key_node_id in primary_node_ids: connected_node_ids = get_connected_node_ids(key_node_id, unique=False) connected_node_ids_with_count = np.stack( diff --git a/bluepysnap/nodes.py b/bluepysnap/nodes.py index d99a8b15..6f2368a7 100644 --- a/bluepysnap/nodes.py +++ b/bluepysnap/nodes.py @@ -172,10 +172,15 @@ def _data(self): """Collected data for the node population as a pandas.DataFrame.""" return self._node_storage.load_population_data(self.name) - @property + @cached_property def _population(self): return self._node_storage.storage.open_population(self.name) + def close_context(self): + """Close the h5 context for node population.""" + if "_population" in self.__dict__: + del self.__dict__["_population"] + @cached_property def size(self): """Node population size.""" diff --git a/tests/test_circuit.py b/tests/test_circuit.py index 10972d44..4fae8ed4 100644 --- a/tests/test_circuit.py +++ b/tests/test_circuit.py @@ -1,4 +1,5 @@ import pytest +import h5py from bluepysnap.nodes import NodePopulation from bluepysnap.edges import EdgePopulation @@ -11,12 +12,12 @@ def test_all(): circuit = test_module.Circuit( str(TEST_DATA_DIR / 'circuit_config.json')) - assert( - circuit.config['networks']['nodes'][0] == - { - 'nodes_file': str(TEST_DATA_DIR / 'nodes.h5'), - 'node_types_file': None, - } + assert ( + circuit.config['networks']['nodes'][0] == + { + 'nodes_file': str(TEST_DATA_DIR / 'nodes.h5'), + 'node_types_file': None, + } ) assert isinstance(circuit.nodes, dict) assert isinstance(circuit.edges, dict) @@ -34,3 +35,20 @@ def test_duplicate_population(): with pytest.raises(BluepySnapError): circuit.nodes + +def test_close_contexts(): + circuit = test_module.Circuit( + str(TEST_DATA_DIR / 'circuit_config.json') + ) + circuit.edges['default'].size + circuit.nodes['default'].size + node_file = circuit.config['networks']['nodes'][0]['nodes_file'] + edge_file = circuit.config['networks']['edges'][0]['edges_file'] + + circuit.close_contexts() + + with h5py.File(node_file, "r+") as h5: + list(h5) + + with h5py.File(edge_file, "r+") as h5: + list(h5) diff --git a/tests/test_edges.py b/tests/test_edges.py index 366b3f03..e820ba43 100644 --- a/tests/test_edges.py +++ b/tests/test_edges.py @@ -4,6 +4,7 @@ import pandas as pd import pandas.testing as pdt import pytest +import h5py import libsonata from mock import Mock diff --git a/tests/test_nodes.py b/tests/test_nodes.py index 5870fda6..377d9c4c 100644 --- a/tests/test_nodes.py +++ b/tests/test_nodes.py @@ -1,8 +1,9 @@ import numpy as np import numpy.testing as npt import pandas as pd -import pandas.util.testing as pdt +import pandas.testing as pdt import pytest +import h5py import libsonata from mock import Mock From ac101d8a9df7b66c575c93b3bcdeac8378664bf0 Mon Sep 17 00:00:00 2001 From: tomdele Date: Thu, 19 Mar 2020 16:50:37 +0100 Subject: [PATCH 3/6] Only in circuit --- bluepysnap/circuit.py | 10 ++++++++-- bluepysnap/edges.py | 5 ----- bluepysnap/nodes.py | 5 ----- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/bluepysnap/circuit.py b/bluepysnap/circuit.py index bed0e0b5..da36e070 100644 --- a/bluepysnap/circuit.py +++ b/bluepysnap/circuit.py @@ -73,9 +73,15 @@ def edges(self): def close_contexts(self): """Close the context for all populations.""" + + def _close_context(pop): + """Close the h5 context for population.""" + if "_population" in pop.__dict__: + del pop.__dict__["_population"] + if self.nodes: for population in self.nodes.values(): - population.close_context() + _close_context(population) if self.edges: for population in self.edges.values(): - population.close_context() + _close_context(population) diff --git a/bluepysnap/edges.py b/bluepysnap/edges.py index f09a4861..ea3b9798 100644 --- a/bluepysnap/edges.py +++ b/bluepysnap/edges.py @@ -113,11 +113,6 @@ def __init__(self, edge_storage, population_name): def _population(self): return self._edge_storage.storage.open_population(self.name) - def close_context(self): - """Close the h5 context for edge population.""" - if "_population" in self.__dict__: - del self.__dict__["_population"] - @cached_property def size(self): """Population size.""" diff --git a/bluepysnap/nodes.py b/bluepysnap/nodes.py index 6f2368a7..700b2fb9 100644 --- a/bluepysnap/nodes.py +++ b/bluepysnap/nodes.py @@ -176,11 +176,6 @@ def _data(self): def _population(self): return self._node_storage.storage.open_population(self.name) - def close_context(self): - """Close the h5 context for node population.""" - if "_population" in self.__dict__: - del self.__dict__["_population"] - @cached_property def size(self): """Node population size.""" From 3b8aad91a2fdd003c77c95a7593ba2a641925672 Mon Sep 17 00:00:00 2001 From: tomdele Date: Fri, 20 Mar 2020 11:30:28 +0100 Subject: [PATCH 4/6] Context manager for circuits --- bluepysnap/circuit.py | 32 ++++++++++++++++++++++---------- tests/test_circuit.py | 28 +++++++++++++++++++--------- 2 files changed, 41 insertions(+), 19 deletions(-) diff --git a/bluepysnap/circuit.py b/bluepysnap/circuit.py index da36e070..7f114695 100644 --- a/bluepysnap/circuit.py +++ b/bluepysnap/circuit.py @@ -49,6 +49,7 @@ def __init__(self, config): Circuit: A Circuit object. """ self._config = Config(config).resolve() + self._open = True @property def config(self): @@ -58,20 +59,24 @@ def config(self): @cached_property def nodes(self): """Access to node population(s). See :py:class:`~bluepysnap.nodes.NodePopulation`.""" - return _collect_populations( - self._config['networks']['nodes'], - lambda cfg: NodeStorage(cfg, self) - ) + if self._open: + return _collect_populations( + self._config['networks']['nodes'], + lambda cfg: NodeStorage(cfg, self) + ) + raise BluepySnapError("I/O error. Cannot access the h5 files with closed context.") @cached_property def edges(self): """Access to edge population(s). See :py:class:`~bluepysnap.edges.EdgePopulation`.""" - return _collect_populations( - self._config['networks']['edges'], - lambda cfg: EdgeStorage(cfg, self) - ) - - def close_contexts(self): + if self._open: + return _collect_populations( + self._config['networks']['edges'], + lambda cfg: EdgeStorage(cfg, self) + ) + raise BluepySnapError("I/O error. Cannot access the h5 files with closed context.") + + def __exit__(self, exc_type, exc_val, exc_tb): """Close the context for all populations.""" def _close_context(pop): @@ -85,3 +90,10 @@ def _close_context(pop): if self.edges: for population in self.edges.values(): _close_context(population) + + del self.__dict__["nodes"] + del self.__dict__["edges"] + self._open = False + + def __enter__(self): + return self diff --git a/tests/test_circuit.py b/tests/test_circuit.py index 4fae8ed4..4189f2ba 100644 --- a/tests/test_circuit.py +++ b/tests/test_circuit.py @@ -37,18 +37,28 @@ def test_duplicate_population(): def test_close_contexts(): - circuit = test_module.Circuit( - str(TEST_DATA_DIR / 'circuit_config.json') - ) - circuit.edges['default'].size - circuit.nodes['default'].size - node_file = circuit.config['networks']['nodes'][0]['nodes_file'] - edge_file = circuit.config['networks']['edges'][0]['edges_file'] - - circuit.close_contexts() + with test_module.Circuit(str(TEST_DATA_DIR / 'circuit_config.json')) as circuit: + edge = circuit.edges['default'] + edge.size + node = circuit.nodes['default'] + node.size + node_file = circuit.config['networks']['nodes'][0]['nodes_file'] + edge_file = circuit.config['networks']['edges'][0]['edges_file'] with h5py.File(node_file, "r+") as h5: list(h5) with h5py.File(edge_file, "r+") as h5: list(h5) + + +def test_close_contexts_error(): + with test_module.Circuit(str(TEST_DATA_DIR / 'circuit_config.json')) as circuit: + circuit.edges['default'].size + circuit.nodes['default'].size + + with pytest.raises(BluepySnapError): + circuit.nodes + + with pytest.raises(BluepySnapError): + circuit.edges From e64174accfdfaa6d586c685b782f87368ccccc86 Mon Sep 17 00:00:00 2001 From: tomdele Date: Fri, 20 Mar 2020 11:50:16 +0100 Subject: [PATCH 5/6] Need to propagate to the nodes and edges... --- bluepysnap/circuit.py | 9 +++++---- bluepysnap/edges.py | 4 +++- bluepysnap/nodes.py | 4 +++- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/bluepysnap/circuit.py b/bluepysnap/circuit.py index 7f114695..30e203f8 100644 --- a/bluepysnap/circuit.py +++ b/bluepysnap/circuit.py @@ -49,7 +49,7 @@ def __init__(self, config): Circuit: A Circuit object. """ self._config = Config(config).resolve() - self._open = True + self.is_open = True @property def config(self): @@ -59,7 +59,7 @@ def config(self): @cached_property def nodes(self): """Access to node population(s). See :py:class:`~bluepysnap.nodes.NodePopulation`.""" - if self._open: + if self.is_open: return _collect_populations( self._config['networks']['nodes'], lambda cfg: NodeStorage(cfg, self) @@ -69,7 +69,7 @@ def nodes(self): @cached_property def edges(self): """Access to edge population(s). See :py:class:`~bluepysnap.edges.EdgePopulation`.""" - if self._open: + if self.is_open: return _collect_populations( self._config['networks']['edges'], lambda cfg: EdgeStorage(cfg, self) @@ -93,7 +93,8 @@ def _close_context(pop): del self.__dict__["nodes"] del self.__dict__["edges"] - self._open = False + self.is_open = False def __enter__(self): + """Enter the context manager for circuit.""" return self diff --git a/bluepysnap/edges.py b/bluepysnap/edges.py index ea3b9798..a889e001 100644 --- a/bluepysnap/edges.py +++ b/bluepysnap/edges.py @@ -111,7 +111,9 @@ def __init__(self, edge_storage, population_name): @cached_property def _population(self): - return self._edge_storage.storage.open_population(self.name) + if self._edge_storage.circuit.is_open: + return self._edge_storage.storage.open_population(self.name) + raise BluepySnapError("I/O error. Cannot access the h5 files with closed context.") @cached_property def size(self): diff --git a/bluepysnap/nodes.py b/bluepysnap/nodes.py index 700b2fb9..4dc8bde7 100644 --- a/bluepysnap/nodes.py +++ b/bluepysnap/nodes.py @@ -174,7 +174,9 @@ def _data(self): @cached_property def _population(self): - return self._node_storage.storage.open_population(self.name) + if self._node_storage.circuit.is_open: + return self._node_storage.storage.open_population(self.name) + raise BluepySnapError("I/O error. Cannot access the h5 files with closed context.") @cached_property def size(self): From 78d3f8c785331c860b292376daca81dc6549b862 Mon Sep 17 00:00:00 2001 From: tomdele Date: Fri, 20 Mar 2020 11:55:19 +0100 Subject: [PATCH 6/6] Fix tests --- tests/test_circuit.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/test_circuit.py b/tests/test_circuit.py index 4189f2ba..c6d0a722 100644 --- a/tests/test_circuit.py +++ b/tests/test_circuit.py @@ -51,14 +51,21 @@ def test_close_contexts(): with h5py.File(edge_file, "r+") as h5: list(h5) - -def test_close_contexts_error(): +def test_close_contexts_errors_extracted_obj(): with test_module.Circuit(str(TEST_DATA_DIR / 'circuit_config.json')) as circuit: - circuit.edges['default'].size - circuit.nodes['default'].size + edges = circuit.edges['default'] + edges.size + nodes = circuit.nodes['default'] + nodes.size with pytest.raises(BluepySnapError): circuit.nodes with pytest.raises(BluepySnapError): circuit.edges + + with pytest.raises(BluepySnapError): + nodes.property_names + + with pytest.raises(BluepySnapError): + edges.property_names