reusing flats/darks from a different scan

docs/source/reference/loaders.rst

@@ -101,43 +101,52 @@ angles dataset in the input hdf5/NeXus file doesn't exist, or cannot be used.
 To configure the loader to handle such cases, please refer to
 :ref:`user_defined_angles`.
 
-Data with Separate Darks and/or Flats
-=====================================
 
-It can sometimes be the case that darks and flats are written to separate
-hdf5/NeXuS files, rather than written to the same hdf5/NeXus file as the
-projections.
-
-Omitting the image key
-++++++++++++++++++++++
+Loading the separate darks and flats
+====================================
 
-In such cases, there is no image key dataset in the hdf5/NeXus file containing the
-projections (because there are only projections in the dataset, rather than
-projections + darks + flats, so there's no need to have an image key). Due to this,
-one difference to the previously shown configuration to handle this case is that
-the :code:`image_key_path` parameter is omitted.
+It can be the case that darks and flats are written to separate
+hdf5/NeXuS files. HTTomo currently supports two options, detailed in the two subsections below.
 
-Loading the separate darks and flats
+Files that do not contain image keys
 ++++++++++++++++++++++++++++++++++++
 
-Additionally, there is a need to specify:
+These are the files without the image keys that contain only flats or darks in two separate files.
+Here one needs to add :code:`darks` and :code:`flats` parameters to the loader parameters with the following fields: 
 
-- the path to the hdf5/NeXuS file containing the darks/flats
-- the dataset within the given hdf5/NeXus file that contains the darks/flats data
+- :code:`file`, the path to the hdf5/NeXus file containing the darks/flats
+- :code:`data_path`, the dataset within the hdf5/NeXus file that contains the
+  darks/flats
 
-In order to specify this information for both darks and flats, there is the
-:code:`darks` and :code:`flats` parameters, see the following as an example:
+as shown in the following code example:
 
 .. literalinclude:: ../../../tests/samples/pipeline_template_examples/DLS/03_i12_separate_darks_flats.yaml
    :language: yaml
    :emphasize-lines: 10-15
 
-Both parameters have two fields that needs to be specified:
+Files with image keys
++++++++++++++++++++++
+
+This can be the case when a new scan is performed, which contains the required image keys. Therefore the keys
+in the older scan should be ignored. In this instance, we need to provide a parameter :code:`image_key_path` in addition to 
+:code:`file` and :code:`data_path` fields.
+
+.. code-block:: yaml
+   :emphasize-lines:7,11
 
-- :code:`file`, the path to the hdf5/NeXus file containing the darks/flats
-- :code:`data_path`, the dataset within the hdf5/NeXus file that contains the
-  darks/flats
 
+    - method: standard_tomo
+      module_path: httomo.data.hdf.loaders
+      parameters:
+        darks:
+          file: path/to/new/file.nxs
+          data_path: /entry1/tomo_entry/data/data
+          image_key_path: /entry1/tomo_entry/instrument/detector/image_key
+        flats:
+          file: path/to/new/file.nxs
+          data_path: /entry1/tomo_entry/data/data
+          image_key_path: /entry1/tomo_entry/instrument/detector/image_key          
+
 .. _user_defined_angles:
 
 Providing/Overriding Angles Data

httomo/darks_flats.py

@@ -20,7 +20,7 @@ class DarksFlatsFileConfig(NamedTuple):
     Notes
     -----
 
-    There are currently two supported configurations for where dark-field or flat-field images
+    There are currently three supported configurations for where dark-field or flat-field images
     can be loaded from:
 
     1. Dark-field and flat-field images are stored in the same file as the projection images,
@@ -43,6 +43,10 @@ class DarksFlatsFileConfig(NamedTuple):
 
     Therefore, an image key is not needed to distinguish between projection, dark-field, or
     flat-field images. For this case, the `image_key_path` should be given as `None`.
+
+    3. Dark-field and flat-field images are stored in separate files with own unique or identical image keys.
+    This can be a new dataset or two different dataset. Therefore, the image_key_path parameter should be provided
+    for both flats and darks.
     """
 
     file: Path

httomo/transform_loader_params.py

@@ -284,6 +284,7 @@ class DarksFlatsParam(TypedDict):
 
     file: str
     data_path: str
+    image_key_path: Optional[str]
 
 
 def parse_darks_flats(
@@ -299,6 +300,7 @@ def parse_darks_flats(
     if isinstance(in_file, str):
         in_file = Path(in_file)
     data_path = config["data_path"] if config is not None else data_config.data_path
+    image_key_path = config["image_key_path"] if config is not None else image_key_path
     return DarksFlatsFileConfig(
         file=in_file, data_path=data_path, image_key_path=image_key_path
     )
@@ -380,12 +382,16 @@ def parse_config(
         angles_config = parse_angles(config["rotation_angles"])
 
     data_config = DataConfig(in_file=input_file, data_path=str(data_path))
-    darks_config = parse_darks_flats(
-        data_config, image_key_path, config.get("darks", None)
-    )
-    flats_config = parse_darks_flats(
-        data_config, image_key_path, config.get("flats", None)
-    )
+
+    darks_value = config.get("darks", None)
+    if darks_value is not None and "image_key_path" not in darks_value:
+        darks_value["image_key_path"] = None
+    darks_config = parse_darks_flats(data_config, image_key_path, darks_value)
+    flats_value = config.get("flats", None)
+    if flats_value is not None and "image_key_path" not in flats_value:
+        flats_value["image_key_path"] = None
+    flats_config = parse_darks_flats(data_config, image_key_path, flats_value)
+
     return (
         data_config,
         image_key_path,

tests/loaders/test_standard_tomo_loader.py

@@ -484,6 +484,45 @@ def test_standard_tomo_loader_read_block_two_procs(
     np.testing.assert_array_equal(block.data, projs)
 
 
+def test_standard_tomo_loader_read_flats_darks_other_data(
+    standard_data_path: str,
+    standard_image_key_path: str,
+):
+    IN_FILE_PATH = Path(__file__).parent.parent / "test_data/tomo_standard.nxs"
+    IN_FILE2_PATH = (
+        Path(__file__).parent.parent / "test_data/tomo_standard_mod_flatsdarks.nxs"
+    )
+    DARKS_FLATS_CONFIG = DarksFlatsFileConfig(
+        file=IN_FILE2_PATH,
+        data_path=standard_data_path,
+        image_key_path=standard_image_key_path,
+    )
+    ANGLES_CONFIG = RawAngles(data_path="/entry1/tomo_entry/data/rotation_angle")
+    SLICING_DIM: SlicingDimType = 0
+    COMM = MPI.COMM_WORLD
+
+    PREVIEW_CONFIG = PreviewConfig(
+        angles=PreviewDimConfig(start=0, stop=180),
+        detector_y=PreviewDimConfig(start=0, stop=128),
+        detector_x=PreviewDimConfig(start=0, stop=160),
+    )
+    loader = StandardTomoLoader(
+        in_file=IN_FILE_PATH,
+        data_path=DARKS_FLATS_CONFIG.data_path,
+        image_key_path=DARKS_FLATS_CONFIG.image_key_path,
+        darks=DARKS_FLATS_CONFIG,
+        flats=DARKS_FLATS_CONFIG,
+        angles=ANGLES_CONFIG,
+        preview_config=PREVIEW_CONFIG,
+        slicing_dim=SLICING_DIM,
+        comm=COMM,
+    )
+    IN_FILE2_FLATS_SUM = 599897507
+    IN_FILE2_DARKS_SUM = 409600
+    assert loader.flats.sum() == IN_FILE2_FLATS_SUM
+    assert loader.darks.sum() == IN_FILE2_DARKS_SUM
+
+
 def test_standard_tomo_loader_read_block_adjust_for_darks_flats_single_proc() -> None:
     IN_FILE_PATH = Path(__file__).parent.parent / "test_data/k11_diad/k11-18014.nxs"
     DATA_PATH = "/entry/imaging/data"

tests/test_transform_loader_params.py

@@ -448,15 +448,37 @@ def test_parse_data():
         (
             DataConfig(Path("/some/path/to/data.nxs"), "/entry1/tomo_entry/data/data"),
             None,
-            {"file": "/some/other/path/to/data.h5", "data_path": "/data"},
+            {
+                "file": "/some/other/path/to/data.h5",
+                "data_path": "/data",
+                "image_key_path": None,
+            },
             DarksFlatsFileConfig(
                 file=Path("/some/other/path/to/data.h5"),
                 data_path="/data",
                 image_key_path=None,
             ),
         ),
+        (
+            DataConfig(Path("/some/path/to/data.nxs"), "/entry1/tomo_entry/data/data"),
+            "/path/to/keys/data_one",
+            {
+                "file": "/some/path/to/data2.nxs",
+                "data_path": "/data",
+                "image_key_path": "/path/to/keys/data_two",
+            },
+            DarksFlatsFileConfig(
+                file=Path("/some/path/to/data2.nxs"),
+                data_path="/data",
+                image_key_path="/path/to/keys/data_two",
+            ),
+        ),
+    ],
+    ids=[
+        "darks/flats-in-input-file",
+        "darks/flats-in-separate-file",
+        "darks/flats-in-separate-file-with-image-key",
     ],
-    ids=["darks/flats-in-input-file", "darks/flats-in-separate-file"],
 )
 def test_parse_darks_flats_(
     data_config: DataConfig,