diff --git a/docs/source/reference/loaders.rst b/docs/source/reference/loaders.rst index c8a9274ec..082c8570e 100644 --- a/docs/source/reference/loaders.rst +++ b/docs/source/reference/loaders.rst @@ -101,43 +101,52 @@ angles dataset in the input hdf5/NeXus file doesn't exist, or cannot be used. To configure the loader to handle such cases, please refer to :ref:`user_defined_angles`. -Data with Separate Darks and/or Flats -===================================== -It can sometimes be the case that darks and flats are written to separate -hdf5/NeXuS files, rather than written to the same hdf5/NeXus file as the -projections. - -Omitting the image key -++++++++++++++++++++++ +Loading the separate darks and flats +==================================== -In such cases, there is no image key dataset in the hdf5/NeXus file containing the -projections (because there are only projections in the dataset, rather than -projections + darks + flats, so there's no need to have an image key). Due to this, -one difference to the previously shown configuration to handle this case is that -the :code:`image_key_path` parameter is omitted. +It can be the case that darks and flats are written to separate +hdf5/NeXuS files. HTTomo currently supports two options, detailed in the two subsections below. -Loading the separate darks and flats +Files that do not contain image keys ++++++++++++++++++++++++++++++++++++ -Additionally, there is a need to specify: +These are the files without the image keys that contain only flats or darks in two separate files. +Here one needs to add :code:`darks` and :code:`flats` parameters to the loader parameters with the following fields: -- the path to the hdf5/NeXuS file containing the darks/flats -- the dataset within the given hdf5/NeXus file that contains the darks/flats data +- :code:`file`, the path to the hdf5/NeXus file containing the darks/flats +- :code:`data_path`, the dataset within the hdf5/NeXus file that contains the + darks/flats -In order to specify this information for both darks and flats, there is the -:code:`darks` and :code:`flats` parameters, see the following as an example: +as shown in the following code example: .. literalinclude:: ../../../tests/samples/pipeline_template_examples/DLS/03_i12_separate_darks_flats.yaml :language: yaml :emphasize-lines: 10-15 -Both parameters have two fields that needs to be specified: +Files with image keys ++++++++++++++++++++++ + +This can be the case when a new scan is performed, which contains the required image keys. Therefore the keys +in the older scan should be ignored. In this instance, we need to provide a parameter :code:`image_key_path` in addition to +:code:`file` and :code:`data_path` fields. + +.. code-block:: yaml + :emphasize-lines:7,11 -- :code:`file`, the path to the hdf5/NeXus file containing the darks/flats -- :code:`data_path`, the dataset within the hdf5/NeXus file that contains the - darks/flats + - method: standard_tomo + module_path: httomo.data.hdf.loaders + parameters: + darks: + file: path/to/new/file.nxs + data_path: /entry1/tomo_entry/data/data + image_key_path: /entry1/tomo_entry/instrument/detector/image_key + flats: + file: path/to/new/file.nxs + data_path: /entry1/tomo_entry/data/data + image_key_path: /entry1/tomo_entry/instrument/detector/image_key + .. _user_defined_angles: Providing/Overriding Angles Data diff --git a/httomo/darks_flats.py b/httomo/darks_flats.py index b6ac93751..5dd736c95 100644 --- a/httomo/darks_flats.py +++ b/httomo/darks_flats.py @@ -20,7 +20,7 @@ class DarksFlatsFileConfig(NamedTuple): Notes ----- - There are currently two supported configurations for where dark-field or flat-field images + There are currently three supported configurations for where dark-field or flat-field images can be loaded from: 1. Dark-field and flat-field images are stored in the same file as the projection images, @@ -43,6 +43,10 @@ class DarksFlatsFileConfig(NamedTuple): Therefore, an image key is not needed to distinguish between projection, dark-field, or flat-field images. For this case, the `image_key_path` should be given as `None`. + + 3. Dark-field and flat-field images are stored in separate files with own unique or identical image keys. + This can be a new dataset or two different dataset. Therefore, the image_key_path parameter should be provided + for both flats and darks. """ file: Path diff --git a/httomo/transform_loader_params.py b/httomo/transform_loader_params.py index b7121c11f..b974e87a6 100644 --- a/httomo/transform_loader_params.py +++ b/httomo/transform_loader_params.py @@ -284,6 +284,7 @@ class DarksFlatsParam(TypedDict): file: str data_path: str + image_key_path: Optional[str] def parse_darks_flats( @@ -299,6 +300,7 @@ def parse_darks_flats( if isinstance(in_file, str): in_file = Path(in_file) data_path = config["data_path"] if config is not None else data_config.data_path + image_key_path = config["image_key_path"] if config is not None else image_key_path return DarksFlatsFileConfig( file=in_file, data_path=data_path, image_key_path=image_key_path ) @@ -380,12 +382,16 @@ def parse_config( angles_config = parse_angles(config["rotation_angles"]) data_config = DataConfig(in_file=input_file, data_path=str(data_path)) - darks_config = parse_darks_flats( - data_config, image_key_path, config.get("darks", None) - ) - flats_config = parse_darks_flats( - data_config, image_key_path, config.get("flats", None) - ) + + darks_value = config.get("darks", None) + if darks_value is not None and "image_key_path" not in darks_value: + darks_value["image_key_path"] = None + darks_config = parse_darks_flats(data_config, image_key_path, darks_value) + flats_value = config.get("flats", None) + if flats_value is not None and "image_key_path" not in flats_value: + flats_value["image_key_path"] = None + flats_config = parse_darks_flats(data_config, image_key_path, flats_value) + return ( data_config, image_key_path, diff --git a/tests/loaders/test_standard_tomo_loader.py b/tests/loaders/test_standard_tomo_loader.py index 3581c1d03..73f48b7ed 100644 --- a/tests/loaders/test_standard_tomo_loader.py +++ b/tests/loaders/test_standard_tomo_loader.py @@ -484,6 +484,45 @@ def test_standard_tomo_loader_read_block_two_procs( np.testing.assert_array_equal(block.data, projs) +def test_standard_tomo_loader_read_flats_darks_other_data( + standard_data_path: str, + standard_image_key_path: str, +): + IN_FILE_PATH = Path(__file__).parent.parent / "test_data/tomo_standard.nxs" + IN_FILE2_PATH = ( + Path(__file__).parent.parent / "test_data/tomo_standard_mod_flatsdarks.nxs" + ) + DARKS_FLATS_CONFIG = DarksFlatsFileConfig( + file=IN_FILE2_PATH, + data_path=standard_data_path, + image_key_path=standard_image_key_path, + ) + ANGLES_CONFIG = RawAngles(data_path="/entry1/tomo_entry/data/rotation_angle") + SLICING_DIM: SlicingDimType = 0 + COMM = MPI.COMM_WORLD + + PREVIEW_CONFIG = PreviewConfig( + angles=PreviewDimConfig(start=0, stop=180), + detector_y=PreviewDimConfig(start=0, stop=128), + detector_x=PreviewDimConfig(start=0, stop=160), + ) + loader = StandardTomoLoader( + in_file=IN_FILE_PATH, + data_path=DARKS_FLATS_CONFIG.data_path, + image_key_path=DARKS_FLATS_CONFIG.image_key_path, + darks=DARKS_FLATS_CONFIG, + flats=DARKS_FLATS_CONFIG, + angles=ANGLES_CONFIG, + preview_config=PREVIEW_CONFIG, + slicing_dim=SLICING_DIM, + comm=COMM, + ) + IN_FILE2_FLATS_SUM = 599897507 + IN_FILE2_DARKS_SUM = 409600 + assert loader.flats.sum() == IN_FILE2_FLATS_SUM + assert loader.darks.sum() == IN_FILE2_DARKS_SUM + + def test_standard_tomo_loader_read_block_adjust_for_darks_flats_single_proc() -> None: IN_FILE_PATH = Path(__file__).parent.parent / "test_data/k11_diad/k11-18014.nxs" DATA_PATH = "/entry/imaging/data" diff --git a/tests/test_data/tomo_standard_mod_flatsdarks.nxs b/tests/test_data/tomo_standard_mod_flatsdarks.nxs new file mode 100644 index 000000000..7eda8973c Binary files /dev/null and b/tests/test_data/tomo_standard_mod_flatsdarks.nxs differ diff --git a/tests/test_transform_loader_params.py b/tests/test_transform_loader_params.py index bf11eb726..7d0b9ba5d 100644 --- a/tests/test_transform_loader_params.py +++ b/tests/test_transform_loader_params.py @@ -448,15 +448,37 @@ def test_parse_data(): ( DataConfig(Path("/some/path/to/data.nxs"), "/entry1/tomo_entry/data/data"), None, - {"file": "/some/other/path/to/data.h5", "data_path": "/data"}, + { + "file": "/some/other/path/to/data.h5", + "data_path": "/data", + "image_key_path": None, + }, DarksFlatsFileConfig( file=Path("/some/other/path/to/data.h5"), data_path="/data", image_key_path=None, ), ), + ( + DataConfig(Path("/some/path/to/data.nxs"), "/entry1/tomo_entry/data/data"), + "/path/to/keys/data_one", + { + "file": "/some/path/to/data2.nxs", + "data_path": "/data", + "image_key_path": "/path/to/keys/data_two", + }, + DarksFlatsFileConfig( + file=Path("/some/path/to/data2.nxs"), + data_path="/data", + image_key_path="/path/to/keys/data_two", + ), + ), + ], + ids=[ + "darks/flats-in-input-file", + "darks/flats-in-separate-file", + "darks/flats-in-separate-file-with-image-key", ], - ids=["darks/flats-in-input-file", "darks/flats-in-separate-file"], ) def test_parse_darks_flats_( data_config: DataConfig,