-
Notifications
You must be signed in to change notification settings - Fork 3
Open
Labels
Milestone
Description
the L0 processing step for generating QL spectra data is not order independent of the parent LB fits files.
in general the GenericProduct.add method should be symmetric a + b = b + a but this is not the case as the end2end testing shows the finial L0 product fits file is dependent of the order the parent LB files are processed.
needs fixing and investigation if this needs a reprocessing of existing FITS files.
example input:
example script to investigate:
import zipfile
import warnings
import urllib.request
from pprint import pformat
from pathlib import Path
import pytest
from astropy.io.fits.diff import FITSDiff
from stixcore.config.config import CONFIG
from stixcore.data.test import test_data
from stixcore.ephemeris.manager import Spice, SpiceKernelManager
from stixcore.idb.manager import IDBManager
from stixcore.io.RidLutManager import RidLutManager
from stixcore.processing.LBtoL0 import Level0
from stixcore.processing.pipeline import PipelineStatus
from stixcore.products.product import Product
from stixcore.soop.manager import SOOPManager
from stixcore.util.logging import get_logger
from stixcore.util.scripts.end2end_testing import end2end_pipeline
def test_e2e_21_6_32(out_dir):
_spm = SpiceKernelManager(test_data.ephemeris.KERNELS_DIR)
Spice.instance = Spice(_spm.get_latest_mk())
# pinpoint the api files location
CONFIG.set('SOOP', 'soop_files_download', str(test_data.soop.DIR))
SOOPManager.instance = SOOPManager(test_data.soop.DIR, mock_api=True)
idbpath = Path(__file__).parent.parent.parent / "data" / "idb"
IDBManager.instance = IDBManager(idbpath) # force_version="2.26.35")
RidLutManager.instance = RidLutManager(Path(CONFIG.get('Publish', 'rid_lut_file')), update=True)
PipelineStatus.log_setup()
f1 = Path("/data/stix/out/test/e2e_21_6_32/solo_LB_stix-21-6-32_0678153600_V02.fits")
f2 = Path("/data/stix/out/test/e2e_21_6_32/solo_LB_stix-21-6-32_0678240000_V02.fits")
out_dir_o1 = out_dir / "o1"
out_dir_o2 = out_dir / "o2"
l0_proc_o1 = Level0(out_dir_o1, out_dir_o1)
l0_files_o1 = l0_proc_o1.process_fits_files(files=[f2, f1])
l0_proc_o2 = Level0(out_dir_o2, out_dir_o2)
l0_files_o2 = l0_proc_o2.process_fits_files(files=[f1, f2])
for fits_1 in l0_files_o1:
# find corresponding original file
fits_2 = next(ofits for ofits in l0_files_o2 if ofits.name == fits_1.name)
diff = FITSDiff(fits_1, fits_2, atol=0.00001, rtol=0.00001,
ignore_keywords=['CHECKSUM', 'DATASUM', 'DATE',
'VERS_SW', 'VERS_CFG', 'HISTORY'])
if not diff.identical:
print(diff.report())
assert diff.identical