diff --git a/eregion/configs/__init__.py b/eregion/configs/__init__.py index e69de29..d085c3a 100644 --- a/eregion/configs/__init__.py +++ b/eregion/configs/__init__.py @@ -0,0 +1 @@ +from .config import * \ No newline at end of file diff --git a/eregion/configs/config.py b/eregion/configs/config.py index 47cbe2d..84a835d 100644 --- a/eregion/configs/config.py +++ b/eregion/configs/config.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod import yaml -from utils.misc_utils import configure_logger +from utils import configure_logger # A yaml constructor for slice objects def slice_constructor(loader, node): @@ -16,6 +16,14 @@ def slice_constructor(loader, node): start, stop, step = values case _: raise ValueError("Invalid number of arguments for slice.") + + if step is None: + if start > stop: + step = -1 + else: + step = 1 + if stop == -1: + stop = None return slice(start, stop, step) yaml.add_constructor('!slice', slice_constructor) diff --git a/eregion/configs/detectors/basic_ccd.yaml b/eregion/configs/detectors/basic_ccd.yaml index ae91744..7441de9 100644 --- a/eregion/configs/detectors/basic_ccd.yaml +++ b/eregion/configs/detectors/basic_ccd.yaml @@ -1,19 +1,22 @@ -## Config file for creating instances of an image class (here DetImage) for a basic CCD detector -## The DetImage instance contains the outputs from a "single" detector. -## The "image" can be stored in one FITS file with one extension that has all the detectors/outputs, -## OR in multiple FITS extensions, one per detector/output, OR in multiple FITS files, one per detector/output. -## The config structure should describe how to load one FITS file. +## Config file for creating instances of DetImage image class for a basic CCD detector +## The DetImage instance is designed to contain the outputs of a SINGLE detector (not mosaic). +## The flexibility provided by YAML allows instantiating image(s) from one FITS file with one extension that has all the detectors/outputs, +## OR from multiple FITS extensions, one per detector/output, OR from multiple FITS files, one per detector/output. +## The config structure should describe what's contained per FITS file. + +## The following example describes a single CCD detector with two outputs (channels) that are read out from the same FITS extension. +## The data for each output is defined by a slice of the FITS extension (ext_slice), and the location of that data in the full DetImage data array is also defined by a slice (data_slice). --- description: Basic single CCD detector, one channel detector_type: CCD # (specify the type of detector, e.g., CCD, CMOS, H2RG, etc.) -detector_output_class: CCDOutput +detector_output_class: CCDOutput # (use the correct class here for the detector output type) -# List of class objects, each containing a list of outputs. +# List of Output class objects, each containing a list of outputs. # An output is described by the FITS file it is in, the extension in that file, and the slices that define its data. objects: - name: 'det_1' class: DetImage - filename_format: '*.fits*' # Use wildcard to indicate the filename + filename_format: '*.fits*' # Use wildcard to pattern match filename if needed properties: x_size: 2048 y_size: 4096 @@ -22,27 +25,27 @@ objects: outputs: - id: 'chan_1' ext_id: 1 # FITS extension ID - ext_slice: [!slice [0, 4096], !slice [0, 1024]] # Slice of the ext_id that has the data for this output - data_slice: [!slice [0, 4096], !slice [0, 1024]] # Slice of the full DetImage data array where this output's data will go - serial_prescan: !slice [0, 0] - serial_overscan: !slice [1024, 1024] - parallel_prescan: !slice [0, 0] - parallel_overscan: !slice [4096, 4096] + ext_slice: [!slice [0, 4096], !slice [0, 1024]] # Slice of the ext_id that has the data for this output (left half) + data_slice: [!slice [0, 4096], !slice [0, 1024]] # Slice of the full DetImage data array where this output's data will go (left half) + serial_prescan: !slice [0, 20] # 20 prescan columns w.r.t. the data_slice + serial_overscan: !slice [1004, 1024] # 20 overscan columns w.r.t. the data_slice + parallel_prescan: !slice [0, 20] # 20 prescan rows w.r.t. the data_slice + parallel_overscan: !slice [4076, 4096] # 20 overscan rows w.r.t. the data_slice parallel_axis: 'y' # First axis in the data array (rows) represent parallel readout direction - readout_pixel: [0, 0] # Readout starts at (0,0) + readout_pixel: [0, 0] # Readout of this amplifier (assumed to be away from this corner in both directions) gain: 1.0 # electrons/ADU read_noise: 5.0 # electrons bias_level: 1000 # ADU - id: 'chan_2' - ext_id: 1 # FITS extension ID - ext_slice: [!slice [0, 4096], !slice [1024, 2048]] # Slice of the ext_id that has the data for this output - data_slice: [!slice [0, 4096], !slice [1024, 2048]] # Slice of the full DetImage data array where this output's data will go - serial_prescan: !slice [0, 0] - serial_overscan: !slice [1024, 1024] - parallel_prescan: !slice [0, 0] - parallel_overscan: !slice [4096, 4096] - parallel_axis: 'y' # First axis in the data array (rows) represent parallel readout direction - readout_pixel: [0, 2047] # Readout starts at (0,2047) + ext_id: 1 + ext_slice: [!slice [0, 4096], !slice [1024, 2048]] #(right half) + data_slice: [!slice [0, 4096], !slice [1024, 2048]] #(right half) + serial_prescan: !slice [2048, 2027] # 20 prescan columns w.r.t. the data_slice (on the right since readout is from the right) + serial_overscan: !slice [1044, 1023] # 20 overscan columns w.r.t. the data_slice + parallel_prescan: !slice [0, 20] # 20 prescan rows w.r.t. the data_slice + parallel_overscan: !slice [4076, 4096] # 20 overscan rows w.r.t. the data_slice + parallel_axis: 'y' + readout_pixel: [0, 2047] # Readout of this amplifier (assumed to be away from this corner in both directions) gain: 1.0 # electrons/ADU read_noise: 5.0 # electrons bias_level: 1000 # ADU diff --git a/eregion/configs/detectors/deimos.yaml b/eregion/configs/detectors/deimos.yaml index aa657f7..a8044cf 100644 --- a/eregion/configs/detectors/deimos.yaml +++ b/eregion/configs/detectors/deimos.yaml @@ -1,6 +1,6 @@ -## Example detector definition file for the DEIMOS instrument CCD array -## A single FITS file contains 8 extensions, one per detector, so 8 DetImage instances are created from this file. -## Each DetImage instance has two outputs (two channels per detector). +## Example detector definition file for the OLD DEIMOS full array. +## A single FITS file contains (at least) 8 extensions, one per detector (both outputs in same extension). +## 8 DetImage instances are defined below. --- description: DEIMOS CCD detector array, 8 detectors with two channel readout each detector_type: CCD # (specify the type of detector, e.g., CCD, CMOS, H2RG, etc.) @@ -46,6 +46,8 @@ objects: x_cen: -46.08 # mm y_cen: 30.72 # mm angle: 0.0 # degrees + flip_x: false # flip to place in focal plane assuming 0,0 is bottom left and +y is up, +x is right + flip_y: true - name: 'det_2' class: DetImage @@ -86,6 +88,8 @@ objects: x_cen: -15.36 # mm y_cen: 30.72 # mm angle: 0.0 # degrees + flip_x: false + flip_y: true - name: 'det_3' class: DetImage @@ -126,6 +130,8 @@ objects: x_cen: 15.36 # mm y_cen: 30.72 # mm angle: 0.0 # degrees + flip_x: false + flip_y: true - name: 'det_4' class: DetImage @@ -166,6 +172,8 @@ objects: x_cen: 46.08 # mm y_cen: 30.72 # mm angle: 0.0 # degrees + flip_x: false + flip_y: true - name: 'det_5' class: DetImage @@ -203,9 +211,11 @@ objects: read_noise: 5.0 # electrons bias_level: 1000 # ADU focal_plane_position: - x_cen: 46.08 # mm + x_cen: -46.08 # mm y_cen: -30.72 # mm angle: 0.0 # degrees + flip_x: true + flip_y: false - name: 'det_6' class: DetImage @@ -243,9 +253,11 @@ objects: read_noise: 5.0 # electrons bias_level: 1000 # ADU focal_plane_position: - x_cen: 15.36 # mm + x_cen: -15.36 # mm y_cen: -30.72 # mm angle: 0.0 # degrees + flip_x: true + flip_y: false - name: 'det_7' class: DetImage @@ -283,9 +295,11 @@ objects: read_noise: 5.0 # electrons bias_level: 1000 # ADU focal_plane_position: - x_cen: -15.36 # mm + x_cen: 15.36 # mm y_cen: -30.72 # mm angle: 0.0 # degrees + flip_x: true + flip_y: false - name: 'det_8' class: DetImage @@ -323,10 +337,11 @@ objects: read_noise: 5.0 # electrons bias_level: 1000 # ADU focal_plane_position: - x_cen: -46.08 # mm + x_cen: 46.08 # mm y_cen: -30.72 # mm angle: 0.0 # degrees - + flip_x: true + flip_y: false diff --git a/eregion/configs/detectors/deimos_sci.yaml b/eregion/configs/detectors/deimos_sci.yaml new file mode 100644 index 0000000..7c66f5e --- /dev/null +++ b/eregion/configs/detectors/deimos_sci.yaml @@ -0,0 +1,348 @@ +## NEW DEIMOS Science detectors full array config. +## 8 fits files for full array, one per detector, each with 2 extensions for the two channels of that detector. +## Complete config to load all 8 detectors correctly from 8 separate files defined below. +--- +description: DEIMOS CCD detector array, 8 detectors with two channel readout each +detector_type: CCD +detector_output_class: CCDOutput + +objects: + - name: 'det_1' + class: DetImage + filename_format: '*SCI_1_*' # Use wildcard to indicate the filename + properties: + x_size: 2188 + y_size: 4124 + saturation_level: 65535 # ADU + pixel_size: 0.015 # mm + outputs: + - id: 'E' + ext_id: 1 + ext_slice: [!slice [0, 4124], !slice [0, 1094]] # Slice of the ext_id that has the data for this output + data_slice: [!slice [0, 4124], !slice [0, 1094]] # Slice of the full DetImage data array where this output's data will go + serial_prescan: !slice [0, 50] + serial_overscan: !slice [1074, 1094] + parallel_prescan: !slice [0, 0] + parallel_overscan: !slice [4104, 4124] + parallel_axis: 'y' + readout_pixel: [0, 0] + gain: 1.0 # electrons/ADU + read_noise: # electrons + bias_level: # ADU + - id: 'F' + ext_id: 4 # FITS extension ID + ext_slice: [!slice [0, 4124], !slice [0, 1094]] # Slice of the ext_id that has the data for this output + data_slice: [!slice [0, 4124], !slice [1094, 2188]] # Slice of the full DetImage data array where this output's data will go + serial_prescan: !slice [2187, 2137] + serial_overscan: !slice [1113, 1093] + parallel_prescan: !slice [0, 0] + parallel_overscan: !slice [4104, 4124] + parallel_axis: 'y' + readout_pixel: [0, 2187] + gain: 1.0 # electrons/ADU + read_noise: # electrons + bias_level: # ADU + focal_plane_position: + x_cen: 49.275 # mm + y_cen: 30.945 # mm + angle: 180.0 # degrees (assuming origin is bottom left and angle is CCW positive) + flip_x: false + flip_y: false + + - name: 'det_2' + class: DetImage + filename_format: '*SCI_2_*' # Use wildcard to indicate the filename + properties: + x_size: 2188 + y_size: 4124 + saturation_level: 65535 # ADU + pixel_size: 0.015 # mm + outputs: + - id: 'E' + ext_id: 1 + ext_slice: [!slice [0, 4124], !slice [0, 1094]] # Slice of the ext_id that has the data for this output + data_slice: [!slice [0, 4124], !slice [0, 1094]] # Slice of the full DetImage data array where this output's data will go + serial_prescan: !slice [0, 50] + serial_overscan: !slice [1074, 1094] + parallel_prescan: !slice [0, 0] + parallel_overscan: !slice [4104, 4124] + parallel_axis: 'y' + readout_pixel: [0, 0] + gain: 1.0 # electrons/ADU + read_noise: # electrons + bias_level: # ADU + - id: 'F' + ext_id: 4 # FITS extension ID + ext_slice: [!slice [0, 4124], !slice [0, 1094]] # Slice of the ext_id that has the data for this output + data_slice: [!slice [0, 4124], !slice [1094, 2188]] # Slice of the full DetImage data array where this output's data will go + serial_prescan: !slice [2187, 2137] + serial_overscan: !slice [1113, 1093] + parallel_prescan: !slice [0, 0] + parallel_overscan: !slice [4104, 4124] + parallel_axis: 'y' + readout_pixel: [0, 2187] + gain: 1.0 # electrons/ADU + read_noise: # electrons + bias_level: # ADU + focal_plane_position: + x_cen: 16.425 # mm + y_cen: 30.945 # mm + angle: 180.0 # degrees (assuming origin is bottom left and angle is CCW positive) + flip_x: false + flip_y: false + + - name: 'det_3' + class: DetImage + filename_format: '*SCI_3_*' # Use wildcard to indicate the filename + properties: + x_size: 2188 + y_size: 4124 + saturation_level: 65535 # ADU + pixel_size: 0.015 # mm + outputs: + - id: 'E' + ext_id: 1 + ext_slice: [!slice [0, 4124], !slice [0, 1094]] # Slice of the ext_id that has the data for this output + data_slice: [!slice [0, 4124], !slice [0, 1094]] # Slice of the full DetImage data array where this output's data will go + serial_prescan: !slice [0, 50] + serial_overscan: !slice [1074, 1094] + parallel_prescan: !slice [0, 0] + parallel_overscan: !slice [4104, 4124] + parallel_axis: 'y' + readout_pixel: [0, 0] + gain: 1.0 # electrons/ADU + read_noise: # electrons + bias_level: # ADU + - id: 'F' + ext_id: 4 # FITS extension ID + ext_slice: [!slice [0, 4124], !slice [0, 1094]] # Slice of the ext_id that has the data for this output + data_slice: [!slice [0, 4124], !slice [1094, 2188]] # Slice of the full DetImage data array where this output's data will go + serial_prescan: !slice [2187, 2137] + serial_overscan: !slice [1113, 1093] + parallel_prescan: !slice [0, 0] + parallel_overscan: !slice [4104, 4124] + parallel_axis: 'y' + readout_pixel: [0, 2187] + gain: 1.0 # electrons/ADU + read_noise: # electrons + bias_level: # ADU + focal_plane_position: + x_cen: -16.425 # mm + y_cen: 30.945 # mm + angle: 180.0 # degrees (assuming origin is bottom left and angle is CCW positive) + flip_x: false + flip_y: false + + - name: 'det_4' + class: DetImage + filename_format: '*SCI_4_*' # Use wildcard to indicate the filename + properties: + x_size: 2188 + y_size: 4124 + saturation_level: 65535 # ADU + pixel_size: 0.015 # mm + outputs: + - id: 'E' + ext_id: 1 + ext_slice: [!slice [0, 4124], !slice [0, 1094]] # Slice of the ext_id that has the data for this output + data_slice: [!slice [0, 4124], !slice [0, 1094]] # Slice of the full DetImage data array where this output's data will go + serial_prescan: !slice [0, 50] + serial_overscan: !slice [1074, 1094] + parallel_prescan: !slice [0, 0] + parallel_overscan: !slice [4104, 4124] + parallel_axis: 'y' + readout_pixel: [0, 0] + gain: 1.0 # electrons/ADU + read_noise: # electrons + bias_level: # ADU + - id: 'F' + ext_id: 4 # FITS extension ID + ext_slice: [!slice [0, 4124], !slice [0, 1094]] # Slice of the ext_id that has the data for this output + data_slice: [!slice [0, 4124], !slice [1094, 2188]] # Slice of the full DetImage data array where this output's data will go + serial_prescan: !slice [2187, 2137] + serial_overscan: !slice [1113, 1093] + parallel_prescan: !slice [0, 0] + parallel_overscan: !slice [4104, 4124] + parallel_axis: 'y' + readout_pixel: [0, 2187] + gain: 1.0 # electrons/ADU + read_noise: # electrons + bias_level: # ADU + focal_plane_position: + x_cen: -49.275 # mm + y_cen: 30.945 # mm + angle: 180.0 # degrees (assuming origin is bottom left and angle is CCW positive) + flip_x: false + flip_y: false + + - name: 'det_5' + class: DetImage + filename_format: '*SCI_5_*' # Use wildcard to indicate the filename + properties: + x_size: 2188 + y_size: 4124 + saturation_level: 65535 # ADU + pixel_size: 0.015 # mm + outputs: + - id: 'E' + ext_id: 1 + ext_slice: [!slice [0, 4124], !slice [0, 1094]] # Slice of the ext_id that has the data for this output + data_slice: [!slice [0, 4124], !slice [0, 1094]] # Slice of the full DetImage data array where this output's data will go + serial_prescan: !slice [0, 50] + serial_overscan: !slice [1074, 1094] + parallel_prescan: !slice [0, 0] + parallel_overscan: !slice [4104, 4124] + parallel_axis: 'y' + readout_pixel: [0, 0] + gain: 1.0 # electrons/ADU + read_noise: # electrons + bias_level: # ADU + - id: 'F' + ext_id: 4 # FITS extension ID + ext_slice: [!slice [0, 4124], !slice [0, 1094]] # Slice of the ext_id that has the data for this output + data_slice: [!slice [0, 4124], !slice [1094, 2188]] # Slice of the full DetImage data array where this output's data will go + serial_prescan: !slice [2187, 2137] + serial_overscan: !slice [1113, 1093] + parallel_prescan: !slice [0, 0] + parallel_overscan: !slice [4104, 4124] + parallel_axis: 'y' + readout_pixel: [0, 2187] + gain: 1.0 # electrons/ADU + read_noise: # electrons + bias_level: # ADU + focal_plane_position: + x_cen: -49.275 # mm + y_cen: -30.945 # mm + angle: 0.0 # degrees (assuming origin is bottom left and angle is CCW positive) + flip_x: false + flip_y: false + + - name: 'det_6' + class: DetImage + filename_format: '*SCI_6_*' # Use wildcard to indicate the filename + properties: + x_size: 2188 + y_size: 4124 + saturation_level: 65535 # ADU + pixel_size: 0.015 # mm + outputs: + - id: 'E' + ext_id: 1 + ext_slice: [!slice [0, 4124], !slice [0, 1094]] # Slice of the ext_id that has the data for this output + data_slice: [!slice [0, 4124], !slice [0, 1094]] # Slice of the full DetImage data array where this output's data will go + serial_prescan: !slice [0, 50] + serial_overscan: !slice [1074, 1094] + parallel_prescan: !slice [0, 0] + parallel_overscan: !slice [4104, 4124] + parallel_axis: 'y' + readout_pixel: [0, 0] + gain: 1.0 # electrons/ADU + read_noise: # electrons + bias_level: # ADU + - id: 'F' + ext_id: 4 # FITS extension ID + ext_slice: [!slice [0, 4124], !slice [0, 1094]] # Slice of the ext_id that has the data for this output + data_slice: [!slice [0, 4124], !slice [1094, 2188]] # Slice of the full DetImage data array where this output's data will go + serial_prescan: !slice [2187, 2137] + serial_overscan: !slice [1113, 1093] + parallel_prescan: !slice [0, 0] + parallel_overscan: !slice [4104, 4124] + parallel_axis: 'y' + readout_pixel: [0, 2187] + gain: 1.0 # electrons/ADU + read_noise: # electrons + bias_level: # ADU + focal_plane_position: + x_cen: -16.425 # mm + y_cen: -30.945 # mm + angle: 0.0 # degrees + flip_x: false + flip_y: false + + - name: 'det_7' + class: DetImage + filename_format: '*SCI_7_*' # Use wildcard to indicate the filename + properties: + x_size: 2188 + y_size: 4124 + saturation_level: 65535 # ADU + pixel_size: 0.015 # mm + outputs: + - id: 'E' + ext_id: 1 + ext_slice: [!slice [0, 4124], !slice [0, 1094]] # Slice of the ext_id that has the data for this output + data_slice: [!slice [0, 4124], !slice [0, 1094]] # Slice of the full DetImage data array where this output's data will go + serial_prescan: !slice [0, 50] + serial_overscan: !slice [1074, 1094] + parallel_prescan: !slice [0, 0] + parallel_overscan: !slice [4104, 4124] + parallel_axis: 'y' + readout_pixel: [0, 0] + gain: 1.0 # electrons/ADU + read_noise: # electrons + bias_level: # ADU + - id: 'F' + ext_id: 4 # FITS extension ID + ext_slice: [!slice [0, 4124], !slice [0, 1094]] # Slice of the ext_id that has the data for this output + data_slice: [!slice [0, 4124], !slice [1094, 2188]] # Slice of the full DetImage data array where this output's data will go + serial_prescan: !slice [2187, 2137] + serial_overscan: !slice [1113, 1093] + parallel_prescan: !slice [0, 0] + parallel_overscan: !slice [4104, 4124] + parallel_axis: 'y' + readout_pixel: [0, 2187] + gain: 1.0 # electrons/ADU + read_noise: # electrons + bias_level: # ADU + focal_plane_position: + x_cen: 16.425 # mm + y_cen: -30.945 # mm + angle: 0.0 # degrees + flip_x: false + flip_y: false + + - name: 'det_8' + class: DetImage + filename_format: '*SCI_8_*' # Use wildcard to indicate the filename + properties: + x_size: 2188 + y_size: 4124 + saturation_level: 65535 # ADU + pixel_size: 0.015 # mm + outputs: + - id: 'E' + ext_id: 1 + ext_slice: [!slice [0, 4124], !slice [0, 1094]] # Slice of the ext_id that has the data for this output + data_slice: [!slice [0, 4124], !slice [0, 1094]] # Slice of the full DetImage data array where this output's data will go + serial_prescan: !slice [0, 50] + serial_overscan: !slice [1074, 1094] + parallel_prescan: !slice [0, 0] + parallel_overscan: !slice [4104, 4124] + parallel_axis: 'y' + readout_pixel: [0, 0] + gain: 1.0 # electrons/ADU + read_noise: # electrons + bias_level: # ADU + - id: 'F' + ext_id: 4 # FITS extension ID + ext_slice: [!slice [0, 4124], !slice [0, 1094]] # Slice of the ext_id that has the data for this output + data_slice: [!slice [0, 4124], !slice [1094, 2188]] # Slice of the full DetImage data array where this output's data will go + serial_prescan: !slice [2187, 2137] + serial_overscan: !slice [1113, 1093] + parallel_prescan: !slice [0, 0] + parallel_overscan: !slice [4104, 4124] + parallel_axis: 'y' + readout_pixel: [0, 2187] + gain: 1.0 # electrons/ADU + read_noise: # electrons + bias_level: # ADU + focal_plane_position: + x_cen: 49.275 # mm + y_cen: -30.945 # mm + angle: 0.0 # degrees + flip_x: false + flip_y: false + + + + diff --git a/eregion/configs/detectors/deimos_sci_raw.yaml b/eregion/configs/detectors/deimos_sci_raw.yaml new file mode 100644 index 0000000..321530e --- /dev/null +++ b/eregion/configs/detectors/deimos_sci_raw.yaml @@ -0,0 +1,298 @@ +## NEW DEIMOS science CCD config, first pass version +## FITS with 1 extension containing full focal plane image with 8 detectors -> 8 DetImage instances to be created +--- +description: DEIMOS CCD detector array, 8 detectors with two channel readout each +detector_type: CCD +detector_output_class: CCDOutput + +objects: + - name: 'det_1' + class: DetImage + filename_format: '*.fits*' + properties: + x_size: 2188 + y_size: 4125 + saturation_level: 65535 # ADU + pixel_size: 0.015 # mm + outputs: + - id: 'E1' + ext_id: 0 + ext_slice: [!slice [8249, 4124], !slice [0, 1094]] + data_slice: [!slice [0, 4125], !slice [0, 1094]] + serial_prescan: !slice [0, 50] + serial_overscan: !slice [1074, 1094] + parallel_prescan: !slice [0, 1] + parallel_overscan: !slice [4105, 4125] + parallel_axis: 'y' + readout_pixel: [0, 0] + - id: 'F1' + ext_id: 0 + ext_slice: [!slice [8249, 4124], !slice [1094, 2188]] + data_slice: [!slice [0, 4125], !slice [1094, 2188]] + serial_prescan: !slice [2187, 2137] + serial_overscan: !slice [1113, 1093] + parallel_prescan: !slice [0, 1] + parallel_overscan: !slice [4105, 4125] + parallel_axis: 'y' + readout_pixel: [0, 2187] + focal_plane_position: + x_cen: -49.23 # mm + y_cen: 30.9375 # mm + angle: 0.0 # degrees + flip_x: false + flip_y: true + + - name: 'det_2' + class: DetImage + filename_format: '*.fits*' # Use wildcard to indicate the filename + properties: + x_size: 2188 + y_size: 4125 + saturation_level: 65535 # ADU + pixel_size: 0.015 # mm + outputs: + - id: 'E2' + ext_id: 0 + ext_slice: [!slice [8249, 4124], !slice [2188, 3282]] + data_slice: [!slice [0, 4125], !slice [0, 1094]] + serial_prescan: !slice [0, 50] + serial_overscan: !slice [1074, 1094] + parallel_prescan: !slice [0, 1] + parallel_overscan: !slice [4105, 4125] + parallel_axis: 'y' + readout_pixel: [0, 0] + - id: 'F2' + ext_id: 0 + ext_slice: [!slice [8249, 4124], !slice [3282, 4376]] + data_slice: [!slice [0, 4125], !slice [1094, 2188]] + serial_prescan: !slice [2187, 2137] + serial_overscan: !slice [1113, 1093] + parallel_prescan: !slice [0, 1] + parallel_overscan: !slice [4105, 4125] + parallel_axis: 'y' + readout_pixel: [0, 2187] + focal_plane_position: + x_cen: -16.41 # mm + y_cen: 30.9375 # mm + angle: 0.0 # degrees + flip_x: false + flip_y: true + + - name: 'det_3' + class: DetImage + filename_format: '*.fits*' # Use wildcard to indicate the filename + properties: + x_size: 2188 + y_size: 4125 + saturation_level: 65535 # ADU + pixel_size: 0.015 # mm + outputs: + - id: 'E3' + ext_id: 0 + ext_slice: [!slice [8249, 4124], !slice [4376, 5470]] + data_slice: [!slice [0, 4125], !slice [0, 1094]] + serial_prescan: !slice [0, 50] + serial_overscan: !slice [1074, 1094] + parallel_prescan: !slice [0, 1] + parallel_overscan: !slice [4105, 4125] + parallel_axis: 'y' + readout_pixel: [0, 0] + - id: 'F3' + ext_id: 0 + ext_slice: [!slice [8249, 4124], !slice [5470, 6564]] + data_slice: [!slice [0, 4125], !slice [1094, 2188]] + serial_prescan: !slice [2187, 2137] + serial_overscan: !slice [1113, 1093] + parallel_prescan: !slice [0, 1] + parallel_overscan: !slice [4105, 4125] + parallel_axis: 'y' + readout_pixel: [0, 2187] + focal_plane_position: + x_cen: 16.41 # mm + y_cen: 30.9375 # mm + angle: 0.0 # degrees + flip_x: false + flip_y: true + + - name: 'det_4' + class: DetImage + filename_format: '*.fits*' # Use wildcard to indicate the filename + properties: + x_size: 2188 + y_size: 4125 + saturation_level: 65535 # ADU + pixel_size: 0.015 # mm + outputs: + - id: 'E4' + ext_id: 0 + ext_slice: [!slice [8249, 4124], !slice [6564, 7658]] + data_slice: [!slice [0, 4125], !slice [0, 1094]] + serial_prescan: !slice [0, 50] + serial_overscan: !slice [1074, 1094] + parallel_prescan: !slice [0, 1] + parallel_overscan: !slice [4105, 4125] + parallel_axis: 'y' + readout_pixel: [0, 0] + - id: 'F4' + ext_id: 0 + ext_slice: [!slice [8249, 4124], !slice [7658, 8752]] + data_slice: [!slice [0, 4125], !slice [1094, 2188]] + serial_prescan: !slice [2187, 2137] + serial_overscan: !slice [1113, 1093] + parallel_prescan: !slice [0, 1] + parallel_overscan: !slice [4105, 4125] + parallel_axis: 'y' + readout_pixel: [0, 2187] + focal_plane_position: + x_cen: 49.23 # mm + y_cen: 30.9375 # mm + angle: 0.0 # degrees + flip_x: false + flip_y: true + + - name: 'det_5' + class: DetImage + filename_format: '*.fits*' # Use wildcard to indicate the filename + properties: + x_size: 2188 + y_size: 4125 + saturation_level: 65535 # ADU + pixel_size: 0.015 # mm + outputs: + - id: 'E5' + ext_id: 0 + ext_slice: [!slice [0, 4125], !slice [8751, 7657]] + data_slice: [!slice [0, 4125], !slice [0, 1094]] + serial_prescan: !slice [0, 50] + serial_overscan: !slice [1074, 1094] + parallel_prescan: !slice [0, 1] + parallel_overscan: !slice [4105, 4125] + parallel_axis: 'y' + readout_pixel: [0, 0] + - id: 'F5' + ext_id: 0 + ext_slice: [!slice [0, 4125], !slice [7657, 6563]] + data_slice: [!slice [0, 4125], !slice [1094, 2188]] + serial_prescan: !slice [2187, 2137] + serial_overscan: !slice [1113, 1093] + parallel_prescan: !slice [0, 1] + parallel_overscan: !slice [4105, 4125] + parallel_axis: 'y' + readout_pixel: [0, 2187] + focal_plane_position: + x_cen: 49.23 # mm + y_cen: -30.9375 # mm + angle: 0.0 # degrees + flip_x: true + flip_y: false + + - name: 'det_6' + class: DetImage + filename_format: '*.fits*' # Use wildcard to indicate the filename + properties: + x_size: 2188 + y_size: 4125 + saturation_level: 65535 # ADU + pixel_size: 0.015 # mm + outputs: + - id: 'E6' + ext_id: 0 + ext_slice: [!slice [0, 4125], !slice [6563, 5469]] + data_slice: [!slice [0, 4125], !slice [0, 1094]] + serial_prescan: !slice [0, 50] + serial_overscan: !slice [1074, 1094] + parallel_prescan: !slice [0, 1] + parallel_overscan: !slice [4105, 4125] + parallel_axis: 'y' + readout_pixel: [0, 0] + - id: 'F6' + ext_id: 0 + ext_slice: [!slice [0, 4125], !slice [5469, 4375]] + data_slice: [!slice [0, 4125], !slice [1094, 2188]] + serial_prescan: !slice [2187, 2137] + serial_overscan: !slice [1113, 1093] + parallel_prescan: !slice [0, 1] + parallel_overscan: !slice [4105, 4125] + parallel_axis: 'y' + readout_pixel: [0, 2187] + focal_plane_position: + x_cen: 16.41 # mm + y_cen: -30.9375 # mm + angle: 0.0 # degrees + flip_x: true + flip_y: false + + - name: 'det_7' + class: DetImage + filename_format: '*.fits*' # Use wildcard to indicate the filename + properties: + x_size: 2188 + y_size: 4125 + saturation_level: 65535 # ADU + pixel_size: 0.015 # mm + outputs: + - id: 'E7' + ext_id: 0 + ext_slice: [!slice [0, 4125], !slice [4375, 3281]] + data_slice: [!slice [0, 4125], !slice [0, 1094]] + serial_prescan: !slice [0, 50] + serial_overscan: !slice [1074, 1094] + parallel_prescan: !slice [0, 1] + parallel_overscan: !slice [4105, 4125] + parallel_axis: 'y' + readout_pixel: [0, 0] + - id: 'F7' + ext_id: 0 + ext_slice: [!slice [0, 4125], !slice [3281, 2187]] + data_slice: [!slice [0, 4125], !slice [1094, 2188]] + serial_prescan: !slice [2187, 2137] + serial_overscan: !slice [1113, 1093] + parallel_prescan: !slice [0, 1] + parallel_overscan: !slice [4105, 4125] + parallel_axis: 'y' + readout_pixel: [0, 2187] + focal_plane_position: + x_cen: -16.41 # mm + y_cen: -30.9375 # mm + angle: 0.0 # degrees + flip_x: true + flip_y: false + + - name: 'det_8' + class: DetImage + filename_format: '*.fits*' # Use wildcard to indicate the filename + properties: + x_size: 2188 + y_size: 4125 + saturation_level: 65535 # ADU + pixel_size: 0.015 # mm + outputs: + - id: 'E8' + ext_id: 0 + ext_slice: [!slice [0, 4125], !slice [2187, 1093]] + data_slice: [!slice [0, 4125], !slice [0, 1094]] + serial_prescan: !slice [0, 50] + serial_overscan: !slice [1074, 1094] + parallel_prescan: !slice [0, 1] + parallel_overscan: !slice [4105, 4125] + parallel_axis: 'y' + readout_pixel: [0, 0] + - id: 'F8' + ext_id: 0 + ext_slice: [!slice [0, 4125], !slice [1093, -1]] + data_slice: [!slice [0, 4125], !slice [1094, 2188]] + serial_prescan: !slice [2187, 2137] + serial_overscan: !slice [1113, 1093] + parallel_prescan: !slice [0, 1] + parallel_overscan: !slice [4105, 4125] + parallel_axis: 'y' + readout_pixel: [0, 2187] + focal_plane_position: + x_cen: -49.23 # mm + y_cen: -30.9375 # mm + angle: 0.0 # degrees + flip_x: true + flip_y: false + + + diff --git a/eregion/configs/detectors/deimos_singledet.yaml b/eregion/configs/detectors/deimos_singledet.yaml index 8903d4d..4d2b8ea 100644 --- a/eregion/configs/detectors/deimos_singledet.yaml +++ b/eregion/configs/detectors/deimos_singledet.yaml @@ -1,6 +1,6 @@ -## Example detector definition file for the DEIMOS instrument CCD array -## A single FITS file contains 8 extensions, one per detector, so 8 DetImage instances are created from this file. -## Each DetImage instance has two outputs (two channels per detector). +## Example detector definition file for DEIMOS single detector. +## A single FITS file contains data for one detector, with the 2 outputs in 2 separate extensions. +## One DetImage instance is defined below. --- description: DEIMOS CCD detector single detector_type: CCD # (specify the type of detector, e.g., CCD, CMOS, H2RG, etc.) @@ -22,7 +22,7 @@ objects: data_slice: [!slice [0, 4125], !slice [0, 1094]] # Slice of the full DetImage data array where this output's data will go serial_prescan: !slice [0, 50] serial_overscan: !slice [1074, 1094] - parallel_prescan: !slice [0, 50] + parallel_prescan: !slice [0, 1] parallel_overscan: !slice [4105, 4125] parallel_axis: 'y' # First axis in the data array (rows) represent parallel readout direction readout_pixel: [0, 0] # top left pixel @@ -33,18 +33,18 @@ objects: ext_id: 2 # FITS extension ID ext_slice: [!slice [0, 4125], !slice [0, 1094]] # Slice of the ext_id that has the data for this output data_slice: [!slice [0, 4125], !slice [1094, 2188]] # Slice of the full DetImage data array where this output's data will go - serial_prescan: !slice [1094, 1044] - serial_overscan: !slice [20, 0] - parallel_prescan: !slice [0, 50] + serial_prescan: !slice [2187, 2137] + serial_overscan: !slice [1113, 1093] + parallel_prescan: !slice [0, 1] parallel_overscan: !slice [4105, 4125] parallel_axis: 'y' # First axis in the data array (rows) represent parallel readout direction - readout_pixel: [0, 1094] # top right pixel + readout_pixel: [0, 2187] # top right pixel gain: 1.0 # electrons/ADU read_noise: 5.0 # electrons bias_level: 1000 # ADU focal_plane_position: - x_cen: -46.08 # mm - y_cen: 30.72 # mm + x_cen: 0 # mm + y_cen: 0 # mm angle: 0.0 # degrees diff --git a/eregion/configs/pipeline_flows/example.yaml b/eregion/configs/pipeline_flows/example.yaml index 8dc630c..f9dd1eb 100644 --- a/eregion/configs/pipeline_flows/example.yaml +++ b/eregion/configs/pipeline_flows/example.yaml @@ -10,20 +10,21 @@ pipelines: nodes: # List of tasks (nodes) in the pipeline flow - name: TASK_1 # Name of the task node, required task: package.module.class # Path to the Class of the task to run, must be a subclass of `Task` defined in tasks.task - init: # Initialization parameters (for Task.__init__) inputs: # Specify any args needed from outputs of other tasks in this config - arg_1: pipe_name.node_name.data.key # Output of tasks are wrapped in TaskResult objects by the engine, and the data produced by the task is in the TaskResult.data dict; specify the path to the data you want to use as input for this task + arg_1: pipe_name.node_name.data.key # Output of tasks are wrapped in TaskResult objects by the engine, and the data + # produced by the task is in the TaskResult.data dict; specify the path to the + # data you want to use as input for this task # etc. params: # Specify any additional kwargs (which are not task outputs) needed; refer to the task documentation for required and optional params and kwargs param_1: value param_2: value # etc. - - run: # Run-time (Task.run() or Task.lazy_run()) inputs and parameters, as above, use `inputs` to specify data coming from outputs of other tasks, and `params` for any additional parameters + run: # Run-time (Task.run() or Task.lazy_run()) inputs and parameters, as above, use `inputs` to specify data coming from + # outputs of other tasks, and `params` for any additional parameters inputs: arg_1: pipe_name.node_name.data.key - # etc. + # etc. params: param_1: value param_2: value @@ -43,7 +44,6 @@ pipelines: params: param_1: value # etc. - depends_on: [TASK_1] # should be specified if this task depends on the output of another task; ensures correct execution order in the pipeline flow - name: PIPE_2 diff --git a/eregion/configs/pipeline_flows/masterbias_example.yaml b/eregion/configs/pipeline_flows/masterbias_example.yaml index 1ab6fc0..ebc8107 100644 --- a/eregion/configs/pipeline_flows/masterbias_example.yaml +++ b/eregion/configs/pipeline_flows/masterbias_example.yaml @@ -15,6 +15,7 @@ pipelines: params: input_source: "/Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/*_bias_*.fits" identifier_func: tasks.custom.guess_image_type_from_filename_DEIMOS + fitsloader_func: tasks.custom.load_image_fits_DEIMOS - name: master_bias task: tasks.calibration.MasterBias @@ -39,6 +40,7 @@ pipelines: params: input_source: "/Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/*flat_0.000*.fits" identifier_func: tasks.custom.guess_image_type_from_filename_DEIMOS + fitsloader_func: tasks.custom.load_image_fits_DEIMOS - name: bias_subtraction task: tasks.preprocessing.BiasSubtraction diff --git a/eregion/core/image_operations.py b/eregion/core/image_operations.py index c859bd3..98eda8b 100644 --- a/eregion/core/image_operations.py +++ b/eregion/core/image_operations.py @@ -90,4 +90,27 @@ def sigma_clip_image(image: np.ndarray | np.ma.MaskedArray, sigma: float, axis: The sigma-clipped image. """ masked = sigma_clip(image, sigma=sigma, axis=axis, **kwargs) - return masked \ No newline at end of file + return masked + +def flip_and_rotate(image: np.ndarray, angle: float, flip_x: bool=False, flip_y: bool=False) -> np.ndarray: + """ + Flip and rotate an image. Rotation angle is assumed to be in degrees and positive for counter-clockwise direction, + and has to be a multiple of 90. + :param image: 2D numpy array + :param angle: in degrees + :param flip_x: True to flip left-right + :param flip_y: True to flip up-down + :return: flipped and rotated image + """ + if image.ndim != 2: + raise ValueError('Input image is not a 2D array.') + if flip_y: + image = np.flipud(image) + if flip_x: + image = np.fliplr(image) + if angle % 90 != 0: + raise ValueError('Angle must be a multiple of 90 degrees.') + else: + k = (angle // 90) % 4 + image = np.rot90(image, int(k)) + return image \ No newline at end of file diff --git a/eregion/datamodels/__init__.py b/eregion/datamodels/__init__.py index e69de29..bdcb698 100644 --- a/eregion/datamodels/__init__.py +++ b/eregion/datamodels/__init__.py @@ -0,0 +1 @@ +from .image import * \ No newline at end of file diff --git a/eregion/datamodels/image.py b/eregion/datamodels/image.py index 0e3c5b9..0e9d4c6 100644 --- a/eregion/datamodels/image.py +++ b/eregion/datamodels/image.py @@ -9,12 +9,11 @@ import matplotlib.pyplot as plt from astropy.io import fits -from utils.image_utils import ensure_dataarray, slice_data, ensure_numpy -from utils.misc_utils import configure_logger +from utils import ensure_dataarray, slice_data, ensure_numpy, configure_logger +from core.image_operations import flip_and_rotate logger = configure_logger(__name__) - class DetectorProperties(BaseModel): """ Physical and sampling properties for a detector tile. @@ -123,14 +122,12 @@ def set_data_in_parent(self, new_data: xr.DataArray | np.ndarray): # Convert new_data to numpy array if it's an xarray DataArray, to ensure compatibility with parent data array. new_data_np = ensure_numpy(new_data) # Assign new data to the appropriate slice in the parent DetImage - self.parent.data.values[self.output_slice] = new_data_np + self.parent.data[self.output_slice] = new_data_np def show(self, ax=None, save=None, **imshow_kwargs): if ax is None: _, ax = plt.subplots(1,1, figsize=(6, 6), tight_layout=True) - image = self.data - im = ax.imshow(image, **imshow_kwargs) - ax.figure.colorbar(im, ax=ax) + im = self.data.plot.imshow(ax=ax, **imshow_kwargs) if save is not None: ax.figure.savefig(save) return ax @@ -161,42 +158,40 @@ def serial_axis(self) -> str: def get_prescan(self, kind: str) -> xr.DataArray: slc = self.serial_prescan if kind == "serial" else self.parallel_prescan axis = self.serial_axis if kind == "serial" else self.parallel_axis - return self.data.isel(**{axis: slc}) + return slice_data(self.data, {axis: slc}) def get_overscan(self, kind: str) -> xr.DataArray: slc = self.serial_overscan if kind == "serial" else self.parallel_overscan axis = self.serial_axis if kind == "serial" else self.parallel_axis - return self.data.isel(**{axis: slc}) + return self.data.sel(**{axis: slc}) def show(self, ax=None, shade_regions=False, save=None, **imshow_kwargs): - if ax is None: - _, ax = plt.subplots(1,1, figsize=(6, 6), tight_layout=True) - image = self.data - im = ax.imshow(image.values, **imshow_kwargs) + ax = super().show(ax=ax, save=None, **imshow_kwargs) if shade_regions: ## Shade the prescan and overscan regions - spandict = {0: ax.axvspan, 1: ax.axhspan} - def _bounds(s: slice, n: int) -> tuple[int, int]: - s0 = 0 if s.start is None else (n + s.start if s.start < 0 else s.start) - s1 = n if s.stop is None else (n + s.stop if s.stop < 0 else s.stop) + s0 = s.start if s.start is not None else (0 if s.step > 0 else n) + s1 = (s.stop-1 if s.step>0 else s.stop+1) if s.stop is not None else (n if s.step>0 else 0) return s0, s1 + spandict = {0: ax.axvspan, 1: ax.axhspan} regions = [ - (self.serial_prescan, self.parallel_axis, "gold", "Serial Prescan"), - (self.serial_overscan, self.parallel_axis, "red", "Serial Overscan"), - (self.parallel_prescan, self.serial_axis, "cyan", "Parallel Prescan"), - (self.parallel_overscan, self.serial_axis, "blue", "Parallel Overscan"), + (self.serial_prescan, self.parallel_axis, "gold", "S Prescan"), + (self.serial_overscan, self.parallel_axis, "red", "S Overscan"), + (self.parallel_prescan, self.serial_axis, "cyan", "P Prescan"), + (self.parallel_overscan, self.serial_axis, "blue", "P Overscan"), ] - shape = image.shape + shape = self.data.shape for s, axis, color, label in regions: axis_idx = 0 if axis=='y' else 1 a, b = _bounds(s, shape[axis_idx]) spandict[axis_idx](a, b, color=color, alpha=0.3, label=label) - ax.legend(loc=(0.01, 1.01), fontsize=8) + ax.scatter(self.readout_pixel[1], + self.readout_pixel[0], + marker='x', color='red', s=100) + ax.legend(loc=(0.55, 1.05), fontsize=7) - ax.figure.colorbar(im, ax=ax) if save is not None: ax.figure.savefig(save) return ax @@ -251,6 +246,7 @@ def __init__( self.meta = {} # Merge any additional kwargs into meta self.meta.update(kwargs) + self.id = self.meta.name # Outputs @@ -293,8 +289,7 @@ def show(self, ax=None, save=None, **imshow_kwargs): raise ValueError("DetImage has no data to show.") if ax is None: _, ax = plt.subplots(1, 1, figsize=(6, 6), tight_layout=True) - im = ax.imshow(self.data.values, **imshow_kwargs) - ax.figure.colorbar(im, ax=ax) + im = self.data.plot.imshow(ax=ax, **imshow_kwargs) if save is not None: ax.figure.savefig(save) return ax @@ -305,9 +300,9 @@ class FocalPlaneImage: :param num_detectors: int Number of detector tiles expected. :param dim: tuple[int, int] - Dimensions of the focal plane image (height, width) in pixels. + Dimensions of the focal plane image (height, width) in mm. :param fp_center: Optional[Tuple[float, float]] - Pixels coordinates (y, x) corresponding to the focal plane center with respect to the image array origin (top-left). + Coordinates (y, x) in mm corresponding to the focal plane center with respect to the image array origin (top-left). Default is dim/2 i.e. (ydim/2, xdim/2). :param det_images: Optional[List[DetImage]] List of DetImage objects to place on the focal plane. @@ -316,64 +311,81 @@ class FocalPlaneImage: def __init__( self, num_detectors: int, - dim: tuple[int, int], - fp_center: Optional[tuple[float, float]] = None, + dim: Optional[tuple[int, int]] = None, det_images: Optional[list[DetImage]] = None, **kwargs, ): self.meta: dict = {} if kwargs: self.meta.update(kwargs) - self.num_detectors = int(num_detectors) - self.dim = tuple(dim) - self.fp_cen_pix = fp_center if fp_center is not None else (dim[0] / 2, dim[1] / 2) - self.det_images: list[DetImage] = [] - if det_images: - for di in det_images: - self.add_DetImage(di) - if self.det_images: - self.construct_focal_plane_image() + self.num_detectors = int(num_detectors) + self.dim_mm = tuple(dim) if dim is not None else None + self.pixel_size = None + self.data: xr.DataArray | None = None # To hold det image data in one array + self.table: pd.DataFrame | None = None # To keep track of det_image data position within focal-plane data array + + self.det_images: dict[str, DetImage] = {} + if det_images is not None: + self.update(det_images) + + def update(self, det_images: list[DetImage]) -> None: + self.add_det_images(det_images) + self.construct_focal_plane_image() + + def add_det_images(self, det_images): + det_images = det_images if len(det_images) > 0 else [det_images] + for det_image in det_images: + if len(self.det_images) == self.num_detectors: + logger.error( + f"Number of DetImages added have reached number of detectors present in this focal-plane.") + break + self.validate_det_image(det_image) + det_image.focal_plane = self + self.det_images[det_image.id] = det_image def validate_det_image(self, det_image: DetImage) -> None: """ Accepts either validated DetImageMeta or legacy dict with required keys. """ - if isinstance(det_image.meta, DetImageMeta): - return - if not isinstance(det_image.meta, dict): + import datamodels + if not (isinstance(det_image.meta, datamodels.DetImageMeta) or isinstance(det_image.meta, dict)): raise ValueError("DetImage.meta must be DetImageMeta or dict.") required = {"properties", "focal_plane_position"} if not required.issubset(det_image.meta.keys()): raise ValueError("DetImage.meta missing required keys for focal-plane placement.") # attempt to coerce for consistent downstream access - det_image.meta = DetImageMeta.model_validate(det_image.meta) + det_image.meta = datamodels.DetImageMeta.model_validate(det_image.meta) + + # check pixel size is same for all det images + pixsize = det_image.meta.properties.pixel_size + if self.pixel_size is None: + self.pixel_size = pixsize + elif pixsize != self.pixel_size: + raise ValueError("All DetImage objects must have the same pixel_size for focal-plane assembly.") def construct_focal_plane_image(self): - if not self.det_images: + if len(self.det_images) == 0: raise ValueError("No DetImage objects to assemble.") - # Cast all to validated meta - for di in self.det_images: - self.validate_det_image(di) - - pixsize = self.det_images[0].meta.properties.pixel_size # type: ignore[union-attr] frames = [] - for det_image in self.det_images: + for det_id, det_image in self.det_images.items(): props = det_image.meta.properties # type: ignore[union-attr] pos = det_image.meta.focal_plane_position # type: ignore[union-attr] xhalf, yhalf = props.x_size / 2, props.y_size / 2 frames.append( { - "det_id": getattr(det_image.meta, "name", None) or "unknown", # type: ignore[arg-type] - "x_min": pos.x_cen / pixsize - xhalf, - "x_max": pos.x_cen / pixsize + xhalf, - "y_min": pos.y_cen / pixsize - yhalf, - "y_max": pos.y_cen / pixsize + yhalf, + "det_id": det_id, # type: ignore[arg-type] + "x_min": int(pos.x_cen / self.pixel_size - xhalf), + "x_max": int(pos.x_cen / self.pixel_size - xhalf) + props.x_size, + "y_min": int(pos.y_cen / self.pixel_size - yhalf), + "y_max": int(pos.y_cen / self.pixel_size - yhalf) + props.y_size, + "angle": pos.angle if hasattr(pos, "angle") else None, + "flip_x": pos.flip_x if hasattr(pos, "flip_x") else None, + "flip_y": pos.flip_y if hasattr(pos, "flip_y") else None, } ) - - frames_df = pd.DataFrame(frames, index=range(len(self.det_images))) + frames_df = pd.DataFrame(frames) # Verify that there are no overlapping detectors, i.e. area covered inside corners should not overlap for i in range(len(frames_df)): @@ -386,63 +398,56 @@ def construct_focal_plane_image(self): raise ValueError(f"Detectors {a['det_id']} and {b['det_id']} overlap in focal plane.") # Verify the size of the focal plane image - fp_ymin = int(frames_df["y_min"].min()) - fp_ymax = int(frames_df["y_max"].max()) - fp_xmin = int(frames_df["x_min"].min()) - fp_xmax = int(frames_df["x_max"].max()) - calc_dim = (fp_ymax - fp_ymin, fp_xmax - fp_xmin) - if calc_dim != self.dim: - logger.warning("Provided dim %s != computed dim %s.", self.dim, calc_dim) - - # Calculate the positions to place each det_image in the focal plane array - # Flip y-axis and shift origin (specified by self.fp_cen_pix) to top-left corner - frames_df["y_min_fp"] = (self.fp_cen_pix[0] - frames_df["y_max"]).astype(int) - frames_df["y_max_fp"] = (self.fp_cen_pix[0] - frames_df["y_min"]).astype(int) - frames_df["x_min_fp"] = (frames_df["x_min"] + self.fp_cen_pix[1]).astype(int) - frames_df["x_max_fp"] = (frames_df["x_max"] + self.fp_cen_pix[1]).astype(int) + calc_dim = np.array([frames_df["y_max"].max() - frames_df["y_min"].min(), + frames_df["x_max"].max() - frames_df["x_min"].min()]) + if self.dim_mm is not None: + dim_pix = np.array(self.dim_mm) / self.pixel_size + if any(calc_dim > dim_pix): + logger.warning("Provided dim %s < computed dim %s.", dim_pix, calc_dim) + else: + dim_pix = calc_dim.astype(int) # Initialize DataArray - self.data: xr.DataArray = xr.DataArray( - np.zeros(self.dim, dtype=float), + self.data = xr.DataArray( + np.zeros(dim_pix, dtype=float), dims=("y", "x"), - coords={"y": np.arange(self.dim[0]), "x": np.arange(self.dim[1])}, + coords={"y": np.arange(frames_df["y_min"].min(), frames_df["y_max"].max(), 1), + "x": np.arange(frames_df["x_min"].min(), frames_df["x_max"].max(), 1)} ) # Place tiles - for i, det_image in enumerate(self.det_images): - if det_image.data is None: + for i in range(len(frames_df)): + row = frames_df.iloc[i] + di = self.det_images[row["det_id"]] + if di.data is None: raise ValueError(f"DetImage at index {i} has no data.") - yslc = slice(int(frames_df.loc[i, "y_min_fp"]), int(frames_df.loc[i, "y_max_fp"])) - xslc = slice(int(frames_df.loc[i, "x_min_fp"]), int(frames_df.loc[i, "x_max_fp"])) - self.data[yslc, xslc] = det_image.data.values + else: + imdata = flip_and_rotate(di.data.values, angle=row['angle'], flip_x=row['flip_x'], + flip_y=row['flip_y']) - self.frames_df = frames_df + slc = {'y':slice(row['y_min'], row['y_max']-1), 'x':slice(row['x_min'], row['x_max']-1)} + self.data.loc[slc] = imdata - def add_DetImage(self, det_image: DetImage): - if len(self.det_images) >= self.num_detectors: - raise ValueError(f"Number of det_images ({len(self.det_images)}) reached limit ({self.num_detectors}).") - self.validate_det_image(det_image) - det_image.focal_plane = self - self.det_images.append(det_image) + self.table = frames_df - def show(self, ax=None, save=None, **imshow_kwargs): + def show(self, ax=None, save=None, show_det_id=False, **imshow_kwargs): if ax is None: _, ax = plt.subplots(1,1, figsize=(8, 8), tight_layout=True) - im = ax.imshow(self.data.values, **imshow_kwargs) + im = self.data.plot.imshow(ax=ax, **imshow_kwargs) # Draw detector boundaries - if hasattr(self, "frames_df"): - for _, row in self.frames_df.iterrows(): + if hasattr(self, "table"): + for _, row in self.table.iterrows(): rect = plt.Rectangle( - (row["x_min_fp"], row["y_min_fp"]), - row["x_max_fp"] - row["x_min_fp"], - row["y_max_fp"] - row["y_min_fp"], + (row["x_min"], row["y_min"]), + row["x_max"] - row["x_min"], + row["y_max"] - row["y_min"], linewidth=1, edgecolor="r", facecolor="none", ) ax.add_patch(rect) - ax.text(row["x_min_fp"] + 150, row["y_min_fp"] + 150, str(row["det_id"]), color="white", fontsize=8) - ax.figure.colorbar(im, ax=ax) + if show_det_id: + ax.text(row["x_min"] + 150, row["y_min"] + 150, str(row["det_id"]), color="white", fontsize=8) if save is not None: ax.figure.savefig(save) return ax diff --git a/eregion/pipeline/engine.py b/eregion/pipeline/engine.py index 29e576c..3d2d0c3 100644 --- a/eregion/pipeline/engine.py +++ b/eregion/pipeline/engine.py @@ -2,10 +2,9 @@ from copy import deepcopy import graphlib -from tasks.task import TaskResult -from configs.config import PipelineConfig -from utils.misc_utils import configure_logger -from utils.misc_utils import load_class +from tasks import TaskResult +from configs import PipelineConfig +from utils import configure_logger, load_class from prefect import task, flow from prefect.futures import wait diff --git a/eregion/tasks/__init__.py b/eregion/tasks/__init__.py index e69de29..0c6c257 100644 --- a/eregion/tasks/__init__.py +++ b/eregion/tasks/__init__.py @@ -0,0 +1,6 @@ +from .task import * +from .custom import * +from .imagegen import * +from .calibration import * +from .preprocessing import * +from .analysis import * diff --git a/eregion/tasks/calibration.py b/eregion/tasks/calibration.py index b1a3b2b..9851de0 100644 --- a/eregion/tasks/calibration.py +++ b/eregion/tasks/calibration.py @@ -1,9 +1,8 @@ import numpy as np -from tasks.task import Task -from datamodels.image import DetImage -from utils.image_utils import ensure_dataarray -from utils.misc_utils import load_class +from tasks import Task +from datamodels import DetImage +from utils import ensure_dataarray, load_class # Task to generate master bias @@ -11,7 +10,7 @@ class MasterBias(Task): def __init__(self, name=None, **kwargs): super().__init__(name=name, **kwargs) - def run(self, bias_images: list[DetImage]) -> dict[str,list[DetImage]]: + def run(self, bias_images: list[DetImage], **kwargs) -> dict[str,list[DetImage]]: """ Generate master bias frames from a list of bias DetImage objects. :param bias_images: list of DetImage @@ -22,13 +21,9 @@ def run(self, bias_images: list[DetImage]) -> dict[str,list[DetImage]]: # Check that bias_images are DetImage instances if not isinstance(bias_images, list) or not all(isinstance(img, DetImage) for img in bias_images): raise ValueError("bias_images must be a list of DetImage instances.") - # Verify that all input images are of 'bias' image_type - # TODO: Add a init kwarg to specify the expected image_type of input bias frames, and check against that instead of hardcoding 'bias' - for img in bias_images: - if 'bias' not in img.image_type.lower(): - raise ValueError(f"All input images must have image_type 'bias'. Found '{img.image_type}' in {img.meta.filename}.") # Group bias images by detector name + # TODO: add a generic group_by utility bias_dict = {} for img in bias_images: det_name = img.meta.name if 'name' in img.meta else 'default' @@ -51,7 +46,7 @@ def run(self, bias_images: list[DetImage]) -> dict[str,list[DetImage]]: master_bias.meta.update({'filenames':', '.join([img.meta.filename for img in imgs])}) master_bias.outputs.update(imgs[0].outputs) master_biases.append(master_bias) - return {'master_biases': master_biases} + return {'master_bias': master_biases} def _create_masterbias(self, biases: list[np.ndarray], method='median')-> np.ndarray: """ diff --git a/eregion/tasks/custom.py b/eregion/tasks/custom.py index 725a94d..be33bd1 100644 --- a/eregion/tasks/custom.py +++ b/eregion/tasks/custom.py @@ -1,5 +1,10 @@ ### File for custom input functions/tasks defined by the user. ### import os +import numpy as np +from typing import Any +from astropy.io import fits + +from utils import load_image_fits def guess_image_type_from_filename_DEIMOS(filename: str) -> str: """ @@ -20,3 +25,10 @@ def guess_image_type_from_filename_DEIMOS(filename: str) -> str: return 'science' else: return 'unknown' + +def load_image_fits_DEIMOS(filename: str) -> tuple[list[Any], list[fits.Header]]: + input_data_array, input_headers = load_image_fits(filename) + for i, hdr in enumerate(input_headers): + if "TAPOFFS" in hdr: + input_data_array[i] = (input_data_array[i].astype(np.int64) >> 12) - hdr["TAPOFFS"] + return input_data_array, input_headers \ No newline at end of file diff --git a/eregion/tasks/imagegen.py b/eregion/tasks/imagegen.py index 91a0a01..52da0c2 100644 --- a/eregion/tasks/imagegen.py +++ b/eregion/tasks/imagegen.py @@ -1,15 +1,13 @@ import os import glob2 import time -import importlib -from typing import Iterator, Generator, Callable, Iterable +import numpy as np +from typing import Iterator, Generator, Callable, Iterable, Optional, Any from joblib import Parallel, delayed -from datamodels.image import * -from utils.image_utils import ensure_dataarray -from configs.config import DetectorConfig -from tasks.task import LazyTask -from utils.io_utils import load_image_fits, parse_list_of_files, guess_image_type_from_header +from configs import DetectorConfig +from tasks import LazyTask +from utils import load_image_fits, parse_list_of_files, guess_image_type_from_header, load_class, ensure_dataarray ## Classes to handle image generation from configuration files @@ -44,8 +42,7 @@ def set_identifier(self, func: str | Callable[..., str]) -> None: """ if not callable(func): try: - module, cls = func.rsplit('.', 1) - func = getattr(importlib.import_module(module), cls) + func = load_class(func) except Exception as e: raise ValueError(f"Error loading identifier function '{func}': {e}") self._identifier_task = func @@ -59,8 +56,7 @@ def set_fitsloader(self, func: str | Callable[..., str]) -> None: """ if not callable(func): try: - module, cls = func.rsplit('.', 1) - func = getattr(importlib.import_module(module), cls) + func = load_class(func) except Exception as e: raise ValueError(f"Error loading FITS loader function '{func}': {e}") self._fitsloader_task = func @@ -160,8 +156,11 @@ def _build_single_image_object(self, outputs = obj.pop('outputs') self.logger.info("Building object %s with %s %s outputs and type %s from file %s", obj['class'], len(outputs), output_class, image_type, filename or 'array input') - ImageClass = globals()[obj.pop('class')] - OutputClass = globals()[output_class] + imclass = obj.pop('class') + imclass = "datamodels."+imclass if "datamodels" not in imclass else imclass + ImageClass = load_class(imclass) + outclass = "datamodels."+output_class if "datamodels" not in output_class else output_class + OutputClass = load_class(outclass) # instantiate image object image = ImageClass(image_type=image_type, **obj, filename=filename or 'none') @@ -174,6 +173,13 @@ def _build_single_image_object(self, image_data_size[1] = max(image_data_size[1], output_obj.output_slice[1].stop) image.add_output(output_obj) + # verify that calculated image size is consistent with set size in obj properties (if given) + if 'properties' in obj and 'x_size' in obj['properties'] and 'y_size' in obj['properties']: + if image_data_size != [obj['properties']['y_size'], obj['properties']['x_size']]: + self.logger.error(f"Calculated image size {image_data_size} does not match specified size in config" + f" {obj['properties']['y_size'], obj['properties']['x_size']} for {obj['name']}") + raise ValueError("Calculated image size does not match specified size in config") + # Assemble full image data from outputs image_data = np.zeros(image_data_size) for output_id in image.outputs: @@ -236,6 +242,7 @@ def lazy_run(self, for filename in file_batch: self.logger.info("Processing file %s", filename) # Load FITS data + # TODO: fitsloader and identifier can be combined if self._fitsloader_task is not None: input_data_array, input_headers = self._fitsloader_task(filename=filename, **fitsloader_kwargs) else: diff --git a/eregion/tasks/preprocessing.py b/eregion/tasks/preprocessing.py index 7d66795..89eed3d 100644 --- a/eregion/tasks/preprocessing.py +++ b/eregion/tasks/preprocessing.py @@ -3,15 +3,13 @@ from typing import Optional, Iterable, Iterator, Any import numpy as np import xarray as xr +from joblib import Parallel, delayed -from tasks.task import LazyTask -from datamodels.image import DetImage, Output -from utils.image_utils import ensure_dataarray, ensure_numpy -from utils.misc_utils import load_class +from tasks import LazyTask +from datamodels import DetImage, Output +from utils import ensure_dataarray, ensure_numpy, load_class from core.image_operations import subtract_from_image, sigma_clip_image -from joblib import Parallel, delayed - ########### BasePreprocessingTask ########### class BasePreprocessingTask(LazyTask): """ diff --git a/eregion/tasks/task.py b/eregion/tasks/task.py index 3af6d9e..5563038 100644 --- a/eregion/tasks/task.py +++ b/eregion/tasks/task.py @@ -8,8 +8,7 @@ from astropy.time import Time import inspect -from utils.io_utils import configure_logger -from utils.misc_utils import load_class +from utils import configure_logger, load_class # Base abstract class for tasks, should have a call method for direct execution and a run method for pipeline workflows class Task(ABC): diff --git a/eregion/utils/__init__.py b/eregion/utils/__init__.py index e69de29..d66c827 100644 --- a/eregion/utils/__init__.py +++ b/eregion/utils/__init__.py @@ -0,0 +1,3 @@ +from .image_utils import * +from .io_utils import * +from .misc_utils import * \ No newline at end of file diff --git a/eregion/utils/image_utils.py b/eregion/utils/image_utils.py index 3b9dd1c..5501bbe 100644 --- a/eregion/utils/image_utils.py +++ b/eregion/utils/image_utils.py @@ -69,18 +69,35 @@ def ensure_numpy(data: xr.DataArray | np.ndarray) -> np.ndarray: case _: raise TypeError("data must be an xarray.DataArray, or numpy.ndarray") -def slice_data(data: xr.DataArray, slicer: tuple[slice, ...]) -> xr.DataArray: +def slice_data(data: xr.DataArray, slicer: tuple[slice, ...] | dict[str, slice]) -> xr.DataArray: """ Slice a 2D or 3D DataArray using ('y','x','t) positional slices. """ - if not isinstance(slicer, tuple) or not all(isinstance(s, slice) for s in slicer): - raise ValueError("slicer must be a tuple of slice objects.") - - match (data.ndim, data.dims): - case (2, ("y", "x")): - return data.isel(y=slicer[0], x=slicer[1]) - case (3, ("y", "x", "t")): - return data.isel(y=slicer[0], x=slicer[1], t=slicer[2]) + match slicer: + case tuple(): + if not all(isinstance(s, slice) for s in slicer): + raise ValueError("All elements of slicer tuple must be slices.") + match (data.ndim, data.dims): + case (2, ("y", "x")): + slicer = {"y": slicer[0], "x": slicer[1]} + case (3, ("y", "x", "t")): + slicer = {"y": slicer[0], "x": slicer[1], "t": slicer[2]} + case _: + raise ValueError("DataArray must be 2D with dims ('y','x') or 3D with dims ('y','x','t').") + case dict(): + if not all(isinstance(s, slice) for s in slicer.values()): + raise ValueError("All values of slicer dict must be slices.") case _: - raise ValueError("DataArray must be 2D with dims ('y','x') or 3D with dims ('y','x','t').") + raise ValueError("slicer must be a tuple of slices or a dict of {dim: slice}.") + + # hack to not include last element of slices but still use .sel() which includes the stop index + for k, sl in slicer.items(): + if sl.step > 0: + slicer[k] = slice(sl.start, sl.stop - 1, sl.step) + else: + slicer[k] = slice(sl.start, sl.stop + 1, sl.step) + + return data.sel(**slicer) + + diff --git a/playground/Example_pipeline_flows.ipynb b/playground/Example_pipeline_flows.ipynb index 562270e..aa7217f 100644 --- a/playground/Example_pipeline_flows.ipynb +++ b/playground/Example_pipeline_flows.ipynb @@ -6,8 +6,8 @@ "metadata": { "collapsed": true, "ExecuteTime": { - "end_time": "2026-03-11T00:40:57.497533Z", - "start_time": "2026-03-11T00:40:55.959788Z" + "end_time": "2026-05-27T00:49:09.737739Z", + "start_time": "2026-05-27T00:49:08.400947Z" } }, "source": "from pipeline.engine import PipelineEngine", @@ -17,8 +17,8 @@ { "metadata": { "ExecuteTime": { - "end_time": "2026-03-11T00:40:57.895771Z", - "start_time": "2026-03-11T00:40:57.507592Z" + "end_time": "2026-05-27T00:49:11.211991Z", + "start_time": "2026-05-27T00:49:11.166829Z" } }, "cell_type": "code", @@ -29,11 +29,23 @@ "name": "stderr", "output_type": "stream", "text": [ - "2026-03-10 17:40:57,510 - PipelineConfig - INFO - Config loaded from file '../eregion/configs/pipeline_flows/masterbias_example.yaml'\n", - "2026-03-10 17:40:57,511 - pipeline.engine - INFO - Number of pipelines defined: 2\n", - "2026-03-10 17:40:57,893 - pipeline.engine - INFO - Pipeline order: [('calib_flow',), ('preproc_flow',)]\n", - "2026-03-10 17:40:57,894 - pipeline.engine - INFO - Pipeline calib_flow order: [('calib_flow.image_creator',), ('calib_flow.master_bias',)]\n", - "2026-03-10 17:40:57,894 - pipeline.engine - INFO - Pipeline preproc_flow order: [('preproc_flow.image_creator', 'calib_flow.master_bias'), ('preproc_flow.bias_subtraction',), ('preproc_flow.overscan_subtraction',), ('preproc_flow.badpixel_masking',)]\n" + "2026-05-26 17:49:11,177 - PipelineConfig - INFO - Config loaded from file '../eregion/configs/pipeline_flows/masterbias_example.yaml'\n", + "2026-05-26 17:49:11,177 - pipeline.engine - INFO - Number of pipelines defined: 2\n", + "2026-05-26 17:49:11,178 - pipeline.engine - INFO - Pipeline order: [('calib_flow',), ('preproc_flow',)]\n", + "2026-05-26 17:49:11,178 - pipeline.engine - INFO - Pipeline calib_flow order: [('calib_flow.image_creator',), ('calib_flow.master_bias',)]\n", + "2026-05-26 17:49:11,179 - pipeline.engine - INFO - Pipeline preproc_flow order: [('preproc_flow.image_creator', 'calib_flow.master_bias'), ('preproc_flow.bias_subtraction',), ('preproc_flow.overscan_subtraction',), ('preproc_flow.badpixel_masking',)]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tasks.imagegen ImageCreator\n", + "tasks.calibration MasterBias\n", + "tasks.imagegen ImageCreator\n", + "tasks.preprocessing BiasSubtraction\n", + "tasks.preprocessing ScanSubtraction\n", + "tasks.preprocessing SigmaClipMasking\n" ] } ], @@ -42,8 +54,8 @@ { "metadata": { "ExecuteTime": { - "end_time": "2026-03-11T00:41:23.004102Z", - "start_time": "2026-03-11T00:40:57.906266Z" + "end_time": "2026-05-27T00:49:33.516303Z", + "start_time": "2026-05-27T00:49:12.063781Z" } }, "cell_type": "code", @@ -53,180 +65,199 @@ { "data": { "text/plain": [ - "17:40:58.639 | \u001B[36mINFO\u001B[0m | prefect - Starting temporary server on \u001B[94mhttp://127.0.0.1:8485\u001B[0m\n", + "17:49:12.825 | \u001B[36mINFO\u001B[0m | prefect - Starting temporary server on \u001B[94mhttp://127.0.0.1:8657\u001B[0m\n", "See \u001B[94mhttps://docs.prefect.io/v3/concepts/server#how-to-guides\u001B[0m for more information on running a dedicated Prefect server.\n" ], "text/html": [ - "
17:40:58.639 | INFO    | prefect - Starting temporary server on http://127.0.0.1:8485\n",
+       "
17:49:12.825 | INFO    | prefect - Starting temporary server on http://127.0.0.1:8657\n",
        "See https://docs.prefect.io/v3/concepts/server#how-to-guides for more information on running a dedicated Prefect server.\n",
        "
\n"
       ]
      },
      "metadata": {},
-     "output_type": "display_data",
-     "jetTransient": {
-      "display_id": null
-     }
+     "output_type": "display_data"
     },
     {
      "data": {
       "text/plain": [
-       "17:41:00.506 | \u001B[36mINFO\u001B[0m    | Flow run\u001B[35m 'unbiased-ara'\u001B[0m - Beginning flow run\u001B[35m 'unbiased-ara'\u001B[0m for flow\u001B[1;35m 'calib_flow'\u001B[0m\n"
+       "17:49:14.683 | \u001B[36mINFO\u001B[0m    | Flow run\u001B[35m 'jolly-barracuda'\u001B[0m - Beginning flow run\u001B[35m 'jolly-barracuda'\u001B[0m for flow\u001B[1;35m 'calib_flow'\u001B[0m\n"
       ],
       "text/html": [
-       "
17:41:00.506 | INFO    | Flow run 'unbiased-ara' - Beginning flow run 'unbiased-ara' for flow 'calib_flow'\n",
+       "
17:49:14.683 | INFO    | Flow run 'jolly-barracuda' - Beginning flow run 'jolly-barracuda' for flow 'calib_flow'\n",
        "
\n"
       ]
      },
      "metadata": {},
-     "output_type": "display_data",
-     "jetTransient": {
-      "display_id": null
-     }
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2026-05-26 17:49:14,712 - DetectorConfig - INFO - Config loaded from file '/Users/yashvi/Desktop/Detector Characterization Tools/eregion/eregion/configs/detectors/deimos_singledet.yaml'\n",
+      "2026-05-26 17:49:14,712 - calib_flow.image_creator - INFO - Item /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/*_bias_*.fits is a glob pattern\n",
+      "2026-05-26 17:49:14,714 - utils.io_utils - INFO - Found FITS file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_2_2025-08-12T101455.893.fits.\n",
+      "2026-05-26 17:49:14,715 - utils.io_utils - INFO - Found FITS file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_5_2025-08-12T101626.632.fits.\n",
+      "2026-05-26 17:49:14,716 - utils.io_utils - INFO - Found FITS file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_4_2025-08-12T101556.386.fits.\n",
+      "2026-05-26 17:49:14,716 - utils.io_utils - INFO - Found FITS file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_3_2025-08-12T101526.139.fits.\n",
+      "2026-05-26 17:49:14,717 - utils.io_utils - INFO - Found FITS file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_9_2025-08-12T101827.618.fits.\n",
+      "2026-05-26 17:49:14,718 - utils.io_utils - INFO - Found FITS file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_0_2025-08-12T101355.400.fits.\n",
+      "2026-05-26 17:49:14,719 - utils.io_utils - INFO - Found FITS file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_6_2025-08-12T101656.879.fits.\n",
+      "2026-05-26 17:49:14,721 - utils.io_utils - INFO - Found FITS file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_1_2025-08-12T101425.647.fits.\n",
+      "2026-05-26 17:49:14,722 - utils.io_utils - INFO - Found FITS file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_7_2025-08-12T101727.125.fits.\n",
+      "2026-05-26 17:49:14,722 - utils.io_utils - INFO - Found FITS file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_8_2025-08-12T101757.371.fits.\n",
+      "2026-05-26 17:49:14,722 - calib_flow.image_creator - INFO - Processing file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_0_2025-08-12T101355.400.fits\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "tasks.custom guess_image_type_from_filename_DEIMOS\n",
+      "tasks.custom load_image_fits_DEIMOS\n"
+     ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "2026-03-10 17:41:00,538 - DetectorConfig - INFO - Config loaded from file '/Users/yashvi/Desktop/Detector Characterization Tools/eregion/eregion/configs/detectors/deimos_singledet.yaml'\n",
-      "2026-03-10 17:41:00,539 - calib_flow.image_creator - INFO - Item /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/*_bias_*.fits is a glob pattern\n",
-      "2026-03-10 17:41:00,540 - utils.io_utils - INFO - Found FITS file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_2_2025-08-12T101455.893.fits.\n",
-      "2026-03-10 17:41:00,541 - utils.io_utils - INFO - Found FITS file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_5_2025-08-12T101626.632.fits.\n",
-      "2026-03-10 17:41:00,541 - utils.io_utils - INFO - Found FITS file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_4_2025-08-12T101556.386.fits.\n",
-      "2026-03-10 17:41:00,542 - utils.io_utils - INFO - Found FITS file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_3_2025-08-12T101526.139.fits.\n",
-      "2026-03-10 17:41:00,543 - utils.io_utils - INFO - Found FITS file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_9_2025-08-12T101827.618.fits.\n",
-      "2026-03-10 17:41:00,543 - utils.io_utils - INFO - Found FITS file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_0_2025-08-12T101355.400.fits.\n",
-      "2026-03-10 17:41:00,543 - utils.io_utils - INFO - Found FITS file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_6_2025-08-12T101656.879.fits.\n",
-      "2026-03-10 17:41:00,544 - utils.io_utils - INFO - Found FITS file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_1_2025-08-12T101425.647.fits.\n",
-      "2026-03-10 17:41:00,544 - utils.io_utils - INFO - Found FITS file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_7_2025-08-12T101727.125.fits.\n",
-      "2026-03-10 17:41:00,544 - utils.io_utils - INFO - Found FITS file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_8_2025-08-12T101757.371.fits.\n",
-      "2026-03-10 17:41:00,545 - calib_flow.image_creator - INFO - Processing file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_0_2025-08-12T101355.400.fits\n",
-      "2026-03-10 17:41:01,522 - calib_flow.image_creator - INFO - Processing file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_1_2025-08-12T101425.647.fits\n",
-      "2026-03-10 17:41:02,369 - calib_flow.image_creator - INFO - Processing file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_2_2025-08-12T101455.893.fits\n",
-      "2026-03-10 17:41:03,217 - calib_flow.image_creator - INFO - Processing file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_3_2025-08-12T101526.139.fits\n",
-      "2026-03-10 17:41:04,074 - calib_flow.image_creator - INFO - Processing file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_4_2025-08-12T101556.386.fits\n",
-      "2026-03-10 17:41:04,486 - calib_flow.image_creator - INFO - Processing file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_5_2025-08-12T101626.632.fits\n",
-      "2026-03-10 17:41:04,914 - calib_flow.image_creator - INFO - Processing file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_6_2025-08-12T101656.879.fits\n",
-      "2026-03-10 17:41:05,352 - calib_flow.image_creator - INFO - Processing file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_7_2025-08-12T101727.125.fits\n",
-      "2026-03-10 17:41:05,798 - calib_flow.image_creator - INFO - Processing file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_8_2025-08-12T101757.371.fits\n",
-      "2026-03-10 17:41:06,672 - calib_flow.image_creator - INFO - Processing file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_9_2025-08-12T101827.618.fits\n"
+      "2026-05-26 17:49:15,815 - calib_flow.image_creator - INFO - Processing file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_1_2025-08-12T101425.647.fits\n",
+      "2026-05-26 17:49:16,743 - calib_flow.image_creator - INFO - Processing file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_2_2025-08-12T101455.893.fits\n",
+      "2026-05-26 17:49:17,674 - calib_flow.image_creator - INFO - Processing file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_3_2025-08-12T101526.139.fits\n",
+      "2026-05-26 17:49:18,078 - calib_flow.image_creator - INFO - Processing file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_4_2025-08-12T101556.386.fits\n",
+      "2026-05-26 17:49:18,515 - calib_flow.image_creator - INFO - Processing file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_5_2025-08-12T101626.632.fits\n",
+      "2026-05-26 17:49:19,440 - calib_flow.image_creator - INFO - Processing file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_6_2025-08-12T101656.879.fits\n",
+      "2026-05-26 17:49:20,365 - calib_flow.image_creator - INFO - Processing file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_7_2025-08-12T101727.125.fits\n",
+      "2026-05-26 17:49:20,777 - calib_flow.image_creator - INFO - Processing file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_8_2025-08-12T101757.371.fits\n",
+      "2026-05-26 17:49:21,214 - calib_flow.image_creator - INFO - Processing file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_bias_9_2025-08-12T101827.618.fits\n"
      ]
     },
     {
      "data": {
       "text/plain": [
-       "17:41:07.067 | \u001B[36mINFO\u001B[0m    | Task run 'calib_flow.image_creator-9fe' - Finished in state \u001B[32mCompleted\u001B[0m()\n"
+       "17:49:21.663 | \u001B[36mINFO\u001B[0m    | Task run 'calib_flow.image_creator-fd5' - Finished in state \u001B[32mCompleted\u001B[0m()\n"
       ],
       "text/html": [
-       "
17:41:07.067 | INFO    | Task run 'calib_flow.image_creator-9fe' - Finished in state Completed()\n",
+       "
17:49:21.663 | INFO    | Task run 'calib_flow.image_creator-fd5' - Finished in state Completed()\n",
        "
\n"
       ]
      },
      "metadata": {},
-     "output_type": "display_data",
-     "jetTransient": {
-      "display_id": null
-     }
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "core.image_operations median_combine\n"
+     ]
     },
     {
      "data": {
       "text/plain": [
-       "17:41:09.365 | \u001B[36mINFO\u001B[0m    | Task run 'calib_flow.master_bias-17d' - Finished in state \u001B[32mCompleted\u001B[0m()\n"
+       "17:49:23.969 | \u001B[36mINFO\u001B[0m    | Task run 'calib_flow.master_bias-d7a' - Finished in state \u001B[32mCompleted\u001B[0m()\n"
       ],
       "text/html": [
-       "
17:41:09.365 | INFO    | Task run 'calib_flow.master_bias-17d' - Finished in state Completed()\n",
+       "
17:49:23.969 | INFO    | Task run 'calib_flow.master_bias-d7a' - Finished in state Completed()\n",
        "
\n"
       ]
      },
      "metadata": {},
-     "output_type": "display_data",
-     "jetTransient": {
-      "display_id": null
-     }
+     "output_type": "display_data"
     },
     {
      "data": {
       "text/plain": [
-       "17:41:09.382 | \u001B[36mINFO\u001B[0m    | Flow run\u001B[35m 'unbiased-ara'\u001B[0m - Finished in state \u001B[32mCompleted\u001B[0m()\n"
+       "17:49:23.982 | \u001B[36mINFO\u001B[0m    | Flow run\u001B[35m 'jolly-barracuda'\u001B[0m - Finished in state \u001B[32mCompleted\u001B[0m()\n"
       ],
       "text/html": [
-       "
17:41:09.382 | INFO    | Flow run 'unbiased-ara' - Finished in state Completed()\n",
+       "
17:49:23.982 | INFO    | Flow run 'jolly-barracuda' - Finished in state Completed()\n",
        "
\n"
       ]
      },
      "metadata": {},
-     "output_type": "display_data",
-     "jetTransient": {
-      "display_id": null
-     }
+     "output_type": "display_data"
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "2026-03-10 17:41:09,384 - pipeline.engine - INFO - Pipeline 'calib_flow' complete\n"
+      "2026-05-26 17:49:23,983 - pipeline.engine - INFO - Pipeline 'calib_flow' complete\n"
      ]
     },
     {
      "data": {
       "text/plain": [
-       "17:41:09.427 | \u001B[36mINFO\u001B[0m    | Flow run\u001B[35m 'awesome-eel'\u001B[0m - Beginning flow run\u001B[35m 'awesome-eel'\u001B[0m for flow\u001B[1;35m 'preproc_flow'\u001B[0m\n"
+       "17:49:24.022 | \u001B[36mINFO\u001B[0m    | Flow run\u001B[35m 'voracious-mongrel'\u001B[0m - Beginning flow run\u001B[35m 'voracious-mongrel'\u001B[0m for flow\u001B[1;35m 'preproc_flow'\u001B[0m\n"
       ],
       "text/html": [
-       "
17:41:09.427 | INFO    | Flow run 'awesome-eel' - Beginning flow run 'awesome-eel' for flow 'preproc_flow'\n",
+       "
17:49:24.022 | INFO    | Flow run 'voracious-mongrel' - Beginning flow run 'voracious-mongrel' for flow 'preproc_flow'\n",
        "
\n"
       ]
      },
      "metadata": {},
-     "output_type": "display_data",
-     "jetTransient": {
-      "display_id": null
-     }
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2026-05-26 17:49:24,041 - DetectorConfig - INFO - Config loaded from file '/Users/yashvi/Desktop/Detector Characterization Tools/eregion/eregion/configs/detectors/deimos_singledet.yaml'\n",
+      "2026-05-26 17:49:24,041 - DetectorConfig - INFO - Config loaded from file '/Users/yashvi/Desktop/Detector Characterization Tools/eregion/eregion/configs/detectors/deimos_singledet.yaml'\n",
+      "2026-05-26 17:49:24,042 - preproc_flow.image_creator - INFO - Item /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/*flat_0.000*.fits is a glob pattern\n",
+      "2026-05-26 17:49:24,044 - utils.io_utils - INFO - Found FITS file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_flat_0.000_0_2025-08-12T102203.192.fits.\n",
+      "2026-05-26 17:49:24,045 - utils.io_utils - INFO - Found FITS file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_flat_0.000_1_2025-08-12T102233.989.fits.\n",
+      "2026-05-26 17:49:24,046 - preproc_flow.image_creator - INFO - Processing file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_flat_0.000_0_2025-08-12T102203.192.fits\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "tasks.custom guess_image_type_from_filename_DEIMOS\n",
+      "tasks.custom load_image_fits_DEIMOS\n"
+     ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "2026-03-10 17:41:09,450 - DetectorConfig - INFO - Config loaded from file '/Users/yashvi/Desktop/Detector Characterization Tools/eregion/eregion/configs/detectors/deimos_singledet.yaml'\n",
-      "2026-03-10 17:41:09,450 - DetectorConfig - INFO - Config loaded from file '/Users/yashvi/Desktop/Detector Characterization Tools/eregion/eregion/configs/detectors/deimos_singledet.yaml'\n",
-      "2026-03-10 17:41:09,451 - preproc_flow.image_creator - INFO - Item /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/*flat_0.000*.fits is a glob pattern\n",
-      "2026-03-10 17:41:09,452 - utils.io_utils - INFO - Found FITS file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_flat_0.000_0_2025-08-12T102203.192.fits.\n",
-      "2026-03-10 17:41:09,454 - utils.io_utils - INFO - Found FITS file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_flat_0.000_1_2025-08-12T102233.989.fits.\n",
-      "2026-03-10 17:41:09,454 - preproc_flow.image_creator - INFO - Processing file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_flat_0.000_0_2025-08-12T102203.192.fits\n",
-      "2026-03-10 17:41:10,302 - preproc_flow.image_creator - INFO - Processing file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_flat_0.000_1_2025-08-12T102233.989.fits\n"
+      "2026-05-26 17:49:24,466 - preproc_flow.image_creator - INFO - Processing file /Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/DTU_DT-SCI_2_flat_0.000_1_2025-08-12T102233.989.fits\n"
      ]
     },
     {
      "data": {
       "text/plain": [
-       "17:41:10.711 | \u001B[36mINFO\u001B[0m    | Task run 'preproc_flow.image_creator-532' - Finished in state \u001B[32mCompleted\u001B[0m()\n"
+       "17:49:24.889 | \u001B[36mINFO\u001B[0m    | Task run 'preproc_flow.image_creator-4bb' - Finished in state \u001B[32mCompleted\u001B[0m()\n"
       ],
       "text/html": [
-       "
17:41:10.711 | INFO    | Task run 'preproc_flow.image_creator-532' - Finished in state Completed()\n",
+       "
17:49:24.889 | INFO    | Task run 'preproc_flow.image_creator-4bb' - Finished in state Completed()\n",
        "
\n"
       ]
      },
      "metadata": {},
-     "output_type": "display_data",
-     "jetTransient": {
-      "display_id": null
-     }
+     "output_type": "display_data"
     },
     {
      "data": {
       "text/plain": [
-       "17:41:13.745 | \u001B[36mINFO\u001B[0m    | Task run 'preproc_flow.bias_subtraction-d2a' - Finished in state \u001B[32mCompleted\u001B[0m()\n"
+       "17:49:28.006 | \u001B[36mINFO\u001B[0m    | Task run 'preproc_flow.bias_subtraction-103' - Finished in state \u001B[32mCompleted\u001B[0m()\n"
       ],
       "text/html": [
-       "
17:41:13.745 | INFO    | Task run 'preproc_flow.bias_subtraction-d2a' - Finished in state Completed()\n",
+       "
17:49:28.006 | INFO    | Task run 'preproc_flow.bias_subtraction-103' - Finished in state Completed()\n",
        "
\n"
       ]
      },
      "metadata": {},
-     "output_type": "display_data",
-     "jetTransient": {
-      "display_id": null
-     }
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "core.image_operations median_by_axis\n",
+      "core.image_operations median_by_axis\n"
+     ]
     },
     {
      "name": "stderr",
@@ -235,7 +266,21 @@
       "/Users/yashvi/Desktop/Detector Characterization Tools/eregion/.venv/lib/python3.13/site-packages/numpy/_core/fromnumeric.py:3860: RuntimeWarning: Mean of empty slice.\n",
       "  return _methods._mean(a, axis=axis, dtype=dtype,\n",
       "/Users/yashvi/Desktop/Detector Characterization Tools/eregion/.venv/lib/python3.13/site-packages/numpy/_core/_methods.py:136: RuntimeWarning: invalid value encountered in divide\n",
-      "  ret = um.true_divide(\n",
+      "  ret = um.true_divide(\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "core.image_operations median_by_axis\n",
+      "core.image_operations median_by_axis\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
       "/Users/yashvi/Desktop/Detector Characterization Tools/eregion/.venv/lib/python3.13/site-packages/numpy/_core/fromnumeric.py:3860: RuntimeWarning: Mean of empty slice.\n",
       "  return _methods._mean(a, axis=axis, dtype=dtype,\n",
       "/Users/yashvi/Desktop/Detector Characterization Tools/eregion/.venv/lib/python3.13/site-packages/numpy/_core/_methods.py:136: RuntimeWarning: invalid value encountered in divide\n",
@@ -245,95 +290,76 @@
     {
      "data": {
       "text/plain": [
-       "17:41:16.396 | \u001B[36mINFO\u001B[0m    | Task run 'preproc_flow.overscan_subtraction-3d0' - Finished in state \u001B[32mCompleted\u001B[0m()\n"
+       "17:49:28.826 | \u001B[36mINFO\u001B[0m    | Task run 'preproc_flow.overscan_subtraction-e23' - Finished in state \u001B[32mCompleted\u001B[0m()\n"
       ],
       "text/html": [
-       "
17:41:16.396 | INFO    | Task run 'preproc_flow.overscan_subtraction-3d0' - Finished in state Completed()\n",
+       "
17:49:28.826 | INFO    | Task run 'preproc_flow.overscan_subtraction-e23' - Finished in state Completed()\n",
        "
\n"
       ]
      },
      "metadata": {},
-     "output_type": "display_data",
-     "jetTransient": {
-      "display_id": null
-     }
+     "output_type": "display_data"
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
       "WARNING: Input data contains invalid values (NaNs or infs), which were automatically clipped. [astropy.stats.sigma_clipping]\n",
-      "2026-03-10 17:41:18,317 - datamodels.image - WARNING - Output with id chan_1 already exists, overwrite is set to True.\n",
-      "2026-03-10 17:41:18,317 - datamodels.image - WARNING - Output with id chan_2 already exists, overwrite is set to True.\n",
+      "2026-05-26 17:49:30,698 - datamodels.image - WARNING - Output with id chan_1 already exists, overwrite is set to True.\n",
+      "2026-05-26 17:49:30,698 - datamodels.image - WARNING - Output with id chan_2 already exists, overwrite is set to True.\n",
       "WARNING: Input data contains invalid values (NaNs or infs), which were automatically clipped. [astropy.stats.sigma_clipping]\n",
-      "2026-03-10 17:41:21,860 - datamodels.image - WARNING - Output with id chan_1 already exists, overwrite is set to True.\n",
-      "2026-03-10 17:41:21,860 - datamodels.image - WARNING - Output with id chan_2 already exists, overwrite is set to True.\n"
+      "2026-05-26 17:49:33,293 - datamodels.image - WARNING - Output with id chan_1 already exists, overwrite is set to True.\n",
+      "2026-05-26 17:49:33,293 - datamodels.image - WARNING - Output with id chan_2 already exists, overwrite is set to True.\n"
      ]
     },
     {
      "data": {
       "text/plain": [
-       "17:41:22.968 | \u001B[36mINFO\u001B[0m    | Task run 'preproc_flow.badpixel_masking-59b' - Finished in state \u001B[32mCompleted\u001B[0m()\n"
+       "17:49:33.467 | \u001B[36mINFO\u001B[0m    | Task run 'preproc_flow.badpixel_masking-44b' - Finished in state \u001B[32mCompleted\u001B[0m()\n"
       ],
       "text/html": [
-       "
17:41:22.968 | INFO    | Task run 'preproc_flow.badpixel_masking-59b' - Finished in state Completed()\n",
+       "
17:49:33.467 | INFO    | Task run 'preproc_flow.badpixel_masking-44b' - Finished in state Completed()\n",
        "
\n"
       ]
      },
      "metadata": {},
-     "output_type": "display_data",
-     "jetTransient": {
-      "display_id": null
-     }
+     "output_type": "display_data"
     },
     {
      "data": {
       "text/plain": [
-       "17:41:22.998 | \u001B[36mINFO\u001B[0m    | Flow run\u001B[35m 'awesome-eel'\u001B[0m - Finished in state \u001B[32mCompleted\u001B[0m()\n"
+       "17:49:33.481 | \u001B[36mINFO\u001B[0m    | Flow run\u001B[35m 'voracious-mongrel'\u001B[0m - Finished in state \u001B[32mCompleted\u001B[0m()\n"
       ],
       "text/html": [
-       "
17:41:22.998 | INFO    | Flow run 'awesome-eel' - Finished in state Completed()\n",
+       "
17:49:33.481 | INFO    | Flow run 'voracious-mongrel' - Finished in state Completed()\n",
        "
\n"
       ]
      },
      "metadata": {},
-     "output_type": "display_data",
-     "jetTransient": {
-      "display_id": null
-     }
+     "output_type": "display_data"
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "2026-03-10 17:41:23,000 - pipeline.engine - INFO - Pipeline 'preproc_flow' complete\n"
+      "2026-05-26 17:49:33,482 - pipeline.engine - INFO - Pipeline 'preproc_flow' complete\n"
      ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "{'calib_flow.image_creator': TaskResult(task_name='calib_flow.image_creator', data={'bias': [, , , , , , , , , ]}, params={'init': {'detector_config': '/Users/yashvi/Desktop/Detector Characterization Tools/eregion/eregion/configs/detectors/deimos_singledet.yaml'}, 'run': {'input_source': '/Users/yashvi/Desktop/Detector Characterization Tools/DTU_dettest/DTU_singledet_acceptance/PTC/SCI/20250812-101350/*_bias_*.fits', 'identifier_func': 'tasks.custom.guess_image_type_from_filename_DEIMOS'}}, upstream=[], timestamp=