diff --git a/bin/atms_dr_runner.py b/bin/atms_dr_runner.py new file mode 100644 index 0000000..86d82cb --- /dev/null +++ b/bin/atms_dr_runner.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2022, 2023 Pytroll developers + +# Author(s): + +# Adam Dybbroe + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Level-1 processing for Direct Readout S-NPP/JPSS ATMS data. + +Using the CSPP level-1 processor from the SSEC, Wisconsin, based on the ADL +software from NASA. Listen for pytroll messages of ready RDR files trigger +processing on direct readout RDR data (granules or full swaths). + +""" + +import argparse +import logging +import os +import sys +import logging.handlers + +from cspp_runner.atms_rdr2sdr_runner import AtmsSdrRunner +from cspp_runner.logger import setup_logging + +CSPP_SDR_HOME = os.environ.get("CSPP_SDR_HOME", '') + +#: Default time format +_DEFAULT_TIME_FORMAT = '%Y-%m-%d %H:%M:%S' + +#: Default log format +_DEFAULT_LOG_FORMAT = '[%(levelname)s: %(asctime)s : %(name)s] %(message)s' + + +LOG = logging.getLogger(__name__) + + +def get_parser(): + """Get parser for commandline-arguments.""" + parser = argparse.ArgumentParser() + parser.add_argument("-c", "--config-file", + required=True, + dest="config_file", + type=str, + default=None, + help="The file containing configuration parameters.") + parser.add_argument("-l", "--log-config", + help="Log config file to use instead of the standard logging.") + parser.add_argument("-v", "--verbose", dest="verbosity", action="count", default=0, + help="Verbosity (between 1 and 2 occurrences with more leading to more " + "verbose logging). WARN=0, INFO=1, " + "DEBUG=2. This is overridden by the log config file if specified.") + return parser + + +def parse_args(): + """Parse command-line arguments.""" + parser = get_parser() + return parser.parse_args() + + +def main(): + """Start the CSPP ATMS runner.""" + cmd_args = parse_args() + print("Read config from", cmd_args.config_file) + + setup_logging(cmd_args) + + try: + atms = AtmsSdrRunner(cmd_args.config_file) + except Exception as err: + LOG.error('ATMS RDR to SDR processing crashed: %s', str(err)) + sys.exit(1) + try: + atms.start() + atms.join() + except KeyboardInterrupt: + LOG.debug("Interrupting") + finally: + atms.close() + + +if __name__ == "__main__": + main() diff --git a/cspp_runner/atms_rdr2sdr_runner.py b/cspp_runner/atms_rdr2sdr_runner.py new file mode 100644 index 0000000..516cb96 --- /dev/null +++ b/cspp_runner/atms_rdr2sdr_runner.py @@ -0,0 +1,352 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2023 Pytroll developers + +# Author(s): + +# Adam Dybbroe + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Run the ATMS RDR to SDR processing withy CSPP on incoming DR data.""" + + +import socket +from trollsift import Parser, globify +import os +import shutil + +import pathlib +import tempfile +from glob import glob +from datetime import datetime + +import subprocess +from urllib.parse import urlparse +import logging +import time + +import signal +from queue import Empty +from threading import Thread +from posttroll.listener import ListenerContainer +from posttroll.message import Message +from posttroll.publisher import NoisyPublisher +from cspp_runner.config import read_config +from cspp_runner.runner import check_environment +from cspp_runner.constants import (PLATFORM_SHORTNAMES, + PLATFORM_LONGNAMES) + + +logger = logging.getLogger(__name__) + + +class AtmsSdrRunner(Thread): + """The ATMS CSPP runner to process RDR to SDR files.""" + + def __init__(self, configfile): + """Initialize the runner object.""" + super().__init__() + + self.configfile = configfile + self.options = {} + config = read_config(self.configfile) + self.options = config + + self.host = socket.gethostname() + + self.sdr_file_patterns = self.options['sdr_file_patterns'] + self._sdr_home = self.options['level1_home'] + + self.input_topics = self.options['subscribe_topics'] + self.output_topics = self.options['publish_topics'] + + self._atms_sdr_call = self.options['atms_sdr_call'] + self._atms_sdr_options = self.options['atms_sdr_options'] + + self.listener = None + self.publisher = None + self.loop = False + self._setup_and_start_communication() + + def _setup_and_start_communication(self): + """Set up the Posttroll communication and start the publisher.""" + logger.info("Starting up ATMS DR Runner") + logger.debug("Input topics:") + for top in self.input_topics: + logger.debug("{topic}".format(topic=str(top))) + + self.listener = ListenerContainer(topics=self.input_topics) + self.publisher = NoisyPublisher("atms-rdr2sdr-runner") + self.publisher.start() + self.loop = True + signal.signal(signal.SIGTERM, self.signal_shutdown) + + def signal_shutdown(self, *args, **kwargs): + """Shutdown the ATMS rdr2sdr runner.""" + self.close() + + def run(self): + """Run the CSPP ATMS RDR2SDR processing.""" + while self.loop: + try: + msg = self.listener.output_queue.get(timeout=1) + logger.debug("Message: %s", str(msg.data)) + except Empty: + continue + else: + if msg.type not in ['file', 'collection', 'dataset']: + logger.debug("Message type not supported: %s", str(msg.type)) + continue + + wrkdir = run_atms_from_message(msg, self._atms_sdr_call, self._atms_sdr_options) + + logger.info("ATMS RDR to SDR processing finished") + logger.debug("Start packing the files and publish") + + sdr_filepaths = get_filepaths(wrkdir, msg.data, self.sdr_file_patterns) + logger.debug("Files: %s", str(sdr_filepaths)) + if len(sdr_filepaths) == 0: + logger.warning("No ATMS files - cspp processing failed! " + + "No files to move. Continue.") + continue + + dest_sdr_files = move_files_to_destination(sdr_filepaths, + self.sdr_file_patterns, self._sdr_home) + logger.debug("Files after having been moved: %s", str(dest_sdr_files)) + + orbit_number = _fix_orbit_number(dest_sdr_files, self.sdr_file_patterns) + output_messages = self._get_output_messages(dest_sdr_files, msg, orbit_number) + + for output_msg in output_messages: + if output_msg: + logger.debug("Sending message: %s", str(output_msg)) + self.publisher.send(str(output_msg)) + + def _get_output_messages(self, sdr_files, input_msg, orbit_number): + """Generate output messages from SDR files and input message, and return.""" + out_messages = [] + for topic in self.output_topics: + to_send = prepare_posttroll_message(input_msg) + dataset = [] + for filepath in sdr_files: + sdrfile = {} + sdrfile['uri'] = '{path}'.format(path=filepath) + sdrfile['uid'] = os.path.basename(filepath) + dataset.append(sdrfile) + + to_send['type'] = 'HDF5' + to_send['format'] = 'SDR' + to_send['data_processing_level'] = '1B' + to_send['dataset'] = dataset + to_send['orig_orbit_number'] = to_send.get('orbit_number') + to_send['orbit_number'] = orbit_number + + pubmsg = Message(topic, 'dataset', to_send) + out_messages.append(pubmsg) + + return out_messages + + def close(self): + """Shutdown the ATMS SDR processing.""" + logger.info('Terminating ATMS RDR to SDR processing.') + self.loop = False + try: + self.listener.stop() + except Exception: + logger.exception("Couldn't stop listener.") + if self.publisher: + try: + self.publisher.stop() + except Exception: + logger.exception("Couldn't stop publisher.") + + +def _fix_orbit_number(sdr_files, sdr_file_patterns): + """Get the orbit number from the SDR files produced with CSPP.""" + s_pattern = get_tb_files_pattern(sdr_file_patterns) + + p__ = Parser(s_pattern) + orbit_numbers = [] + for filename in sdr_files: + bname = os.path.basename(str(filename)) + logger.debug("SDR filename: %s", str(bname)) + try: + result = p__.parse(bname) + except ValueError: + continue + + orbit = result.get('orbit', 0) + logger.debug("Orbit number = %s", orbit) + orbit_numbers.append(orbit) + + # Test if there are more orbit numbers and at least log an info. + # FIXME! + return orbit_numbers[0] + + +def prepare_posttroll_message(input_msg): + """Create the basic posttroll-message fields and return.""" + to_send = input_msg.data.copy() + to_send.pop('dataset', None) + to_send.pop('collection', None) + to_send.pop('uri', None) + to_send.pop('uid', None) + to_send.pop('format', None) + to_send.pop('type', None) + + return to_send + + +def move_files_to_destination(sdr_filepaths, sdr_file_patterns, sdr_home): + """Move the SDR files from tmp-directory to a final destination.""" + dirpath = create_subdir_from_filepaths(sdr_filepaths, sdr_file_patterns, sdr_home) + for filename in sdr_filepaths: + shutil.move(filename, dirpath) + + return glob(str(dirpath / "*")) + + +def get_tb_files_pattern(sdr_file_patterns): + """Get the file name pattern for the TB SDR files (SATMS).""" + for pattern in sdr_file_patterns: + if pattern.startswith('S'): + return pattern + + +def create_subdir_from_filepaths(sdr_filepaths, sdr_file_patterns, sdr_home): + """From the list of SDR files create a sub-directory where files should be moved.""" + s_pattern = get_tb_files_pattern(sdr_file_patterns) + + start_time = datetime.now() + p__ = Parser(s_pattern) + + orbit = 0 + platform = 'unknown' + result = {} + for filename in sdr_filepaths: + bname = os.path.basename(str(filename)) + logger.debug("SDR filename: %s", str(bname)) + try: + result = p__.parse(bname) + except ValueError: + continue + + stime = result.get('start_time') + if stime and stime < start_time: + start_time = stime + + orbit = result.get('orbit', 0) + platform = PLATFORM_LONGNAMES.get(result.get('platform_shortname', 'unknown')) + + subdirname = "{platform}_{dtime:%Y%m%d_%H%M}_{orbit:05d}".format(platform=platform.lower().replace('-', ''), + dtime=start_time, orbit=orbit) + if isinstance(sdr_home, str): + sdr_home = pathlib.Path(sdr_home) + + dirpath = sdr_home / subdirname + dirpath.mkdir() + return dirpath + + +def get_filepaths(directory, msg_data, file_patterns): + """Identify the ATMS files output from CSPP and return filepaths.""" + files = [] + for pattern in file_patterns: + # p__ = Parser(pattern) + # mda = {'orbit': msg_data['orbit_number'], + # 'platform_shortname': PLATFORM_SHORTNAMES.get(msg_data['platform_name'])} + mda = {'platform_shortname': PLATFORM_SHORTNAMES.get(msg_data['platform_name'])} + + # 'start_time': msg_data['start_time']} Here check for times, if + # start/end times in the file names are sufficiently close to the + # actual ones from the messages + + glbstr = globify(pattern, mda) + logger.debug("Glob-string = %s", str(glbstr)) + logger.debug("Directory = %s", str(directory)) + flist = glob(os.path.join(directory, glbstr)) + files = files + flist + + logger.debug("Files: %s", str(files)) + return files + + +def run_atms_from_message(posttroll_msg, sdr_call, sdr_options): + """Trigger ATMS processing on ATMS scene, from Posttroll message.""" + # platform_name = posttroll_msg.data.get('platform_name') + # sensor = posttroll_msg.data.get('sensor') + collection = posttroll_msg.data.get('collection') + if collection: + atms_rdr_files = get_filelist_from_collection(collection) + else: + logger.warning("ATMS processing so far only supports running on collection of files.") + return + + # Process ATMS files in a sub process: + check_environment("CSPP_WORKDIR") + cspp_workdir = os.environ.get("CSPP_WORKDIR", '') + pathlib.Path(cspp_workdir).mkdir(parents=True, exist_ok=True) + + try: + working_dir = tempfile.mkdtemp(dir=cspp_workdir) + except OSError: + working_dir = tempfile.mkdtemp() + + os.environ["CSPP_WORKDIR"] = working_dir + + my_env = os.environ.copy() + + # Run the command: + cmdlist = [sdr_call] + cmdlist.extend(sdr_options) + cmdlist.extend(atms_rdr_files) + + t0_clock = time.process_time() + t0_wall = time.time() + logger.debug("Popen call arguments: " + str(cmdlist)) + + sdr_proc = subprocess.Popen(cmdlist, cwd=working_dir, + env=my_env, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE) + while True: + line = sdr_proc.stdout.readline() + if not line: + break + logger.debug(line.decode("utf-8").strip('\n')) + + while True: + errline = sdr_proc.stderr.readline() + if not errline: + break + logger.debug(errline.decode("utf-8").strip('\n')) + + logger.info("Seconds process time: " + (str(time.process_time() - t0_clock))) + logger.info("Seconds wall clock time: " + (str(time.time() - t0_wall))) + + sdr_proc.poll() + + return working_dir + + +def get_filelist_from_collection(atms_collection): + """From a posttroll message extract the ATMS files from collection.""" + filelist = [] + for obj in atms_collection: + urlobj = urlparse(obj['uri']) + filelist.append(urlobj.path) + + return filelist diff --git a/cspp_runner/config.py b/cspp_runner/config.py new file mode 100644 index 0000000..1e8aff4 --- /dev/null +++ b/cspp_runner/config.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2023 Pytroll Developers + +# Author(s): + +# Adam Dybbroe + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Handling the yaml configurations.""" + +import yaml +from yaml import UnsafeLoader + + +def read_config(config_filepath): + """Read and extract config information.""" + with open(config_filepath, 'r') as fp_: + config = yaml.load(fp_, Loader=UnsafeLoader) + + return config diff --git a/cspp_runner/constants.py b/cspp_runner/constants.py new file mode 100644 index 0000000..f1388e6 --- /dev/null +++ b/cspp_runner/constants.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2023 Adam.Dybbroe + +# Author(s): + +# Adam.Dybbroe + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Constants and utilities for name conversions etc.""" + + +PLATFORM_SHORTNAMES = {'NOAA-20': 'j01', + 'NOAA-21': 'j02', + 'Suomi-NPP': 'npp'} +PLATFORM_LONGNAMES = {'j01': 'NOAA-20', + 'j02': 'NOAA-21', + 'npp': 'Suomi-NPP'} diff --git a/cspp_runner/logger.py b/cspp_runner/logger.py new file mode 100644 index 0000000..ff520ab --- /dev/null +++ b/cspp_runner/logger.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2023 Pytroll developers + +# Author(s): + +# Adam.Dybbroe + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""The log handling.""" + +import logging +import logging.config +import logging.handlers +import yaml + +LOG_FORMAT = "[%(asctime)s %(levelname)-8s] %(message)s" + +log_levels = { + 0: logging.WARN, + 1: logging.INFO, + 2: logging.DEBUG, +} + + +def setup_logging(cmd_args): + """Set up logging.""" + if cmd_args.log_config is not None: + with open(cmd_args.log_config) as fd: + log_dict = yaml.safe_load(fd.read()) + logging.config.dictConfig(log_dict) + return + + root = logging.getLogger('') + root.setLevel(log_levels[cmd_args.verbosity]) + + fh_ = logging.StreamHandler() + + formatter = logging.Formatter(LOG_FORMAT) + fh_.setFormatter(formatter) + + root.addHandler(fh_) diff --git a/cspp_runner/runner.py b/cspp_runner/runner.py index 055315f..4b51ea5 100644 --- a/cspp_runner/runner.py +++ b/cspp_runner/runner.py @@ -163,7 +163,8 @@ def update_files(url_jpss_remote_dir, update_stampfile_prefix, mirror_jpss, the JPSS script in a separat shell. """ - _check_environment("CSPP_WORKDIR") + check_environment("CSPP_WORKDIR") + cspp_workdir = os.environ.get("CSPP_WORKDIR", '') pathlib.Path(cspp_workdir).mkdir(parents=True, exist_ok=True) my_env = os.environ.copy() @@ -208,7 +209,7 @@ def update_files(url_jpss_remote_dir, update_stampfile_prefix, mirror_jpss, LOG.info(f"{what:s} downloaded. {what:s}-update timestamp file = " + filename) -def _check_environment(*args): +def check_environment(*args): """Check that requested environment variables are set. Raise EnvironmentError if they are not. diff --git a/cspp_runner/tests/conftest.py b/cspp_runner/tests/conftest.py new file mode 100644 index 0000000..85da853 --- /dev/null +++ b/cspp_runner/tests/conftest.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2023 Adam.Dybbroe + +# Author(s): + +# Adam.Dybbroe + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Fixtures for unittests.""" + +import pytest +from posttroll.message import Message +import stat + + +TEST_YAML_CONFIG_CONTENT = """# Location to store Sensor Data Record (SDR) files after CSPP SDR processing +# is completed. +level1_home: /path/to/where/the/atms/sdr/files/will/be/stored + +# Examples: +# TATMS_npp_d20230209_t1047306_e1236183_b58482_c20230209123846465182_cspp_dev.h5 +# SATMS_npp_d20230209_t1047306_e1236183_b58482_c20230209123844714309_cspp_dev.h5 +# GATMO_npp_d20230209_t1047306_e1236183_b58482_c20230209123847932256_cspp_dev.h5 + +sdr_file_patterns: + - 'SATMS_{platform_shortname}_d{start_time:%Y%m%d_t%H%M%S%f}_e{end_time:%H%M%S%f}_b{orbit:5d}_c{creation_time:%Y%m%d%H%M%S%f}_{source}.h5' + - 'GATMO_{platform_shortname}_d{start_time:%Y%m%d_t%H%M%S%f}_e{end_time:%H%M%S%f}_b{orbit:5d}_c{creation_time:%Y%m%d%H%M%S%f}_{source}.h5' + - 'TATMS_{platform_shortname}_d{start_time:%Y%m%d_t%H%M%S%f}_e{end_time:%H%M%S%f}_b{orbit:5d}_c{creation_time:%Y%m%d%H%M%S%f}_{source}.h5' + + +working_dir: /san1/cspp/work + +# CSPP-atms batch script and parameters: +atms_sdr_call: atms_sdr.sh + +# Options to pass to the viirs_sdr_call +# see viirs_sdr.sh --help for explanation +atms_sdr_options: + - '-a' + - '-d' + +# Topic to use for publishing posttroll messages +publish_topics: + - /file/atms/sdr + +# Posttroll topics to listen to (comma separated) +subscribe_topics: + - /file/atms/rdr +""" # noqa + +TEST_ATMS_COLLECTION_MESSAGE = """pytroll://atms/rdr/0/gatherer collection safusr.u@lxserv1043.smhi.se 2023-02-08T12:06:01.560943 v1.01 application/json {"start_time": "2023-02-08T11:54:17.200000", "end_time": "2023-02-08T12:04:57.100000", "orbit_number": 27071, "platform_name": "NOAA-20", "sensor": "atms", "format": "RDR", "type": "HDF5", "data_processing_level": "0", "variant": "DR", "collection_area_id": "euron1", "collection": [{"start_time": "2023-02-08T11:54:17.200000", "end_time": "2023-02-08T11:54:49.200000", "uri": "ssh://172.29.1.52/path/to/jpss/atms/rdr/RATMS-RNSCA_j01_d20230208_t1154172_e1154492_b00001_c20230208115457829000_drlu_ops.h5", "uid": "RATMS-RNSCA_j01_d20230208_t1154172_e1154492_b00001_c20230208115457829000_drlu_ops.h5"}, {"start_time": "2023-02-08T11:54:49.200000", "end_time": "2023-02-08T11:55:21.200000", "uri": "ssh://172.29.1.52/path/to/jpss/atms/rdr/RATMS-RNSCA_j01_d20230208_t1154492_e1155212_b00001_c20230208115538023000_drlu_ops.h5", "uid": "RATMS-RNSCA_j01_d20230208_t1154492_e1155212_b00001_c20230208115538023000_drlu_ops.h5"}, {"start_time": "2023-02-08T11:55:21.200000", "end_time": "2023-02-08T11:55:53.200000", "uri": "ssh://172.29.1.52/path/to/jpss/atms/rdr/RATMS-RNSCA_j01_d20230208_t1155212_e1155532_b00001_c20230208115558220000_drlu_ops.h5", "uid": "RATMS-RNSCA_j01_d20230208_t1155212_e1155532_b00001_c20230208115558220000_drlu_ops.h5"}, {"start_time": "2023-02-08T11:55:53.200000", "end_time": "2023-02-08T11:56:25.200000", "uri": "ssh://172.29.1.52/path/to/jpss/atms/rdr/RATMS-RNSCA_j01_d20230208_t1155532_e1156252_b00001_c20230208115638170000_drlu_ops.h5", "uid": "RATMS-RNSCA_j01_d20230208_t1155532_e1156252_b00001_c20230208115638170000_drlu_ops.h5"}, {"start_time": "2023-02-08T11:56:25.200000", "end_time": "2023-02-08T11:56:57.200000", "uri": "ssh://172.29.1.52/path/to/jpss/atms/rdr/RATMS-RNSCA_j01_d20230208_t1156252_e1156572_b00001_c20230208115718069000_drlu_ops.h5", "uid": "RATMS-RNSCA_j01_d20230208_t1156252_e1156572_b00001_c20230208115718069000_drlu_ops.h5"}, {"start_time": "2023-02-08T11:56:57.200000", "end_time": "2023-02-08T11:57:29.200000", "uri": "ssh://172.29.1.52/path/to/jpss/atms/rdr/RATMS-RNSCA_j01_d20230208_t1156572_e1157292_b00001_c20230208115738216000_drlu_ops.h5", "uid": "RATMS-RNSCA_j01_d20230208_t1156572_e1157292_b00001_c20230208115738216000_drlu_ops.h5"}, {"start_time": "2023-02-08T11:57:29.200000", "end_time": "2023-02-08T11:58:01.200000", "uri": "ssh://172.29.1.52/path/to/jpss/atms/rdr/RATMS-RNSCA_j01_d20230208_t1157292_e1158012_b00001_c20230208115818194000_drlu_ops.h5", "uid": "RATMS-RNSCA_j01_d20230208_t1157292_e1158012_b00001_c20230208115818194000_drlu_ops.h5"}, {"start_time": "2023-02-08T11:58:01.200000", "end_time": "2023-02-08T11:58:33.200000", "uri": "ssh://172.29.1.52/path/to/jpss/atms/rdr/RATMS-RNSCA_j01_d20230208_t1158012_e1158332_b00001_c20230208115837985000_drlu_ops.h5", "uid": "RATMS-RNSCA_j01_d20230208_t1158012_e1158332_b00001_c20230208115837985000_drlu_ops.h5"}, {"start_time": "2023-02-08T11:58:33.200000", "end_time": "2023-02-08T11:59:05.200000", "uri": "ssh://172.29.1.52/path/to/jpss/atms/rdr/RATMS-RNSCA_j01_d20230208_t1158332_e1159052_b00001_c20230208115918341000_drlu_ops.h5", "uid": "RATMS-RNSCA_j01_d20230208_t1158332_e1159052_b00001_c20230208115918341000_drlu_ops.h5"}, {"start_time": "2023-02-08T11:59:05.200000", "end_time": "2023-02-08T11:59:37.200000", "uri": "ssh://172.29.1.52/path/to/jpss/atms/rdr/RATMS-RNSCA_j01_d20230208_t1159052_e1159372_b00001_c20230208115957983000_drlu_ops.h5", "uid": "RATMS-RNSCA_j01_d20230208_t1159052_e1159372_b00001_c20230208115957983000_drlu_ops.h5"}, {"start_time": "2023-02-08T11:59:37.200000", "end_time": "2023-02-08T12:00:09.200000", "uri": "ssh://172.29.1.52/path/to/jpss/atms/rdr/RATMS-RNSCA_j01_d20230208_t1159372_e1200092_b00001_c20230208120017590000_drlu_ops.h5", "uid": "RATMS-RNSCA_j01_d20230208_t1159372_e1200092_b00001_c20230208120017590000_drlu_ops.h5"}, {"start_time": "2023-02-08T12:00:09.200000", "end_time": "2023-02-08T12:00:41.200000", "uri": "ssh://172.29.1.52/path/to/jpss/atms/rdr/RATMS-RNSCA_j01_d20230208_t1200092_e1200412_b00001_c20230208120058642000_drlu_ops.h5", "uid": "RATMS-RNSCA_j01_d20230208_t1200092_e1200412_b00001_c20230208120058642000_drlu_ops.h5"}, {"start_time": "2023-02-08T12:00:41.200000", "end_time": "2023-02-08T12:01:13.200000", "uri": "ssh://172.29.1.52/path/to/jpss/atms/rdr/RATMS-RNSCA_j01_d20230208_t1200412_e1201132_b00001_c20230208120118131000_drlu_ops.h5", "uid": "RATMS-RNSCA_j01_d20230208_t1200412_e1201132_b00001_c20230208120118131000_drlu_ops.h5"}, {"start_time": "2023-02-08T12:01:13.200000", "end_time": "2023-02-08T12:01:45.200000", "uri": "ssh://172.29.1.52/path/to/jpss/atms/rdr/RATMS-RNSCA_j01_d20230208_t1201132_e1201452_b00001_c20230208120157708000_drlu_ops.h5", "uid": "RATMS-RNSCA_j01_d20230208_t1201132_e1201452_b00001_c20230208120157708000_drlu_ops.h5"}, {"start_time": "2023-02-08T12:01:45.200000", "end_time": "2023-02-08T12:02:17.200000", "uri": "ssh://172.29.1.52/path/to/jpss/atms/rdr/RATMS-RNSCA_j01_d20230208_t1201452_e1202172_b00001_c20230208120238505000_drlu_ops.h5", "uid": "RATMS-RNSCA_j01_d20230208_t1201452_e1202172_b00001_c20230208120238505000_drlu_ops.h5"}, {"start_time": "2023-02-08T12:02:17.200000", "end_time": "2023-02-08T12:02:49.200000", "uri": "ssh://172.29.1.52/path/to/jpss/atms/rdr/RATMS-RNSCA_j01_d20230208_t1202172_e1202492_b00001_c20230208120258137000_drlu_ops.h5", "uid": "RATMS-RNSCA_j01_d20230208_t1202172_e1202492_b00001_c20230208120258137000_drlu_ops.h5"}, {"start_time": "2023-02-08T12:02:49.200000", "end_time": "2023-02-08T12:03:21.100000", "uri": "ssh://172.29.1.52/path/to/jpss/atms/rdr/RATMS-RNSCA_j01_d20230208_t1202492_e1203211_b00001_c20230208120337761000_drlu_ops.h5", "uid": "RATMS-RNSCA_j01_d20230208_t1202492_e1203211_b00001_c20230208120337761000_drlu_ops.h5"}, {"start_time": "2023-02-08T12:03:21.100000", "end_time": "2023-02-08T12:03:53.100000", "uri": "ssh://172.29.1.52/path/to/jpss/atms/rdr/RATMS-RNSCA_j01_d20230208_t1203211_e1203531_b00001_c20230208120358165000_drlu_ops.h5", "uid": "RATMS-RNSCA_j01_d20230208_t1203211_e1203531_b00001_c20230208120358165000_drlu_ops.h5"}, {"start_time": "2023-02-08T12:03:53.100000", "end_time": "2023-02-08T12:04:25.100000", "uri": "ssh://172.29.1.52/path/to/jpss/atms/rdr/RATMS-RNSCA_j01_d20230208_t1203531_e1204251_b00001_c20230208120437760000_drlu_ops.h5", "uid": "RATMS-RNSCA_j01_d20230208_t1203531_e1204251_b00001_c20230208120437760000_drlu_ops.h5"}, {"start_time": "2023-02-08T12:04:25.100000", "end_time": "2023-02-08T12:04:57.100000", "uri": "ssh://172.29.1.52/path/to/jpss/atms/rdr/RATMS-RNSCA_j01_d20230208_t1204251_e1204571_b00001_c20230208120518174000_drlu_ops.h5", "uid": "RATMS-RNSCA_j01_d20230208_t1204251_e1204571_b00001_c20230208120518174000_drlu_ops.h5"}]}""" # noqa + + +TEST_ATMS_FILE_MESSAGE = """pytroll://atms/rdr/0/file file safusr.u@lxserv1043.smhi.se 2023-02-08T12:06:01.560943 v1.01 application/json {'start_time': datetime.datetime(2023, 2, 9, 9, 6, 10), 'end_time': datetime.datetime(2023, 2, 9, 9, 6, 42), 'orbit_number': 58481, 'platform_name': 'Suomi-NPP', 'sensor': 'atms', 'format': 'RDR', 'type': 'HDF5', 'data_processing_level': '0', 'uid': 'RATMS-RNSCA_npp_d20230209_t0906100_e0906420_b00001_c20230209090700529000_drlu_ops.h5', 'uri': 'ssh://172.29.1.52/san1/polar_in/direct_readout/npp/lvl0/RATMS-RNSCA_npp_d20230209_t0906100_e0906420_b00001_c20230209090700529000_drlu_ops.h5', 'variant': 'DR'}""" # noqa + + +@pytest.fixture(scope="session") +def fake_cspp_workdir(tmp_path_factory): + """Create a fake CSPP working dir.""" + return tmp_path_factory.mktemp("work") + + +@pytest.fixture +def fake_atms_posttroll_message(): + """Create and return a Posttroll message for ATMS.""" + yield Message.decode(rawstr=str(TEST_ATMS_COLLECTION_MESSAGE)) + + +@pytest.fixture +def fake_yamlconfig_file(tmp_path): + """Write fake yaml config file.""" + file_path = tmp_path / 'test_atms_dr_config_config.yaml' + with open(file_path, 'w') as fpt: + fpt.write(TEST_YAML_CONFIG_CONTENT) + + yield file_path + + +# Create the fake CSPP/ATMS bash script +FAKE_ATMS_BASH_SCRIPT = """#!/bin/bash + +if [ -z "$CSPP_SDR_HOME" ]; then + echo "CSPP_SDR_HOME must be set to the path where the CSPP software was installed." + echo "export CSPP_SDR_HOME=/home/me/SDR_x_x" + exit 1 +fi + +python $CSPP_SDR_HOME/atms/fake_adl_atms_script.py -vv "$@" || echo "ATMS SDR did not complete nominally" +""" + +FAKE_ATMS_PYTHON_MAIN_SCRIPT = """ +import argparse + +if __name__ == "__main__": + desc = "Dummy/fake program to mimic launching the CSPP/ATMS rdr-to-sdr script." + parser = argparse.ArgumentParser(description=desc) + + parser.add_argument('-a', '--aggregate', + action="store_true", default=False, help="aggregate products with nagg") + parser.add_argument('-d', '--debug', action="store_true", default=False, + help='enable debug mode on ADL and avoid cleaning workspace') + parser.add_argument('-v', '--verbosity', action="count", default=0, + help='each occurrence increases verbosity 1 level through ERROR-WARNING-INFO-DEBUG') + parser.add_argument('filenames', metavar='filename', type=str, nargs='+', + help='HDF5 ATMS RDR file/s to process') + + args = parser.parse_args() + + print(args.filenames) +""" + + +@pytest.fixture(scope="session") +def fake_adl_atms_scripts(tmp_path_factory): + """Create a fake cspp/atms bash and python main script.""" + cspp_home_dir = tmp_path_factory.mktemp('CSPP') + path = cspp_home_dir / 'atms' + path.mkdir() + filename = path / 'atms_sdr.sh' + with open(filename, 'w') as fpt: + fpt.write(FAKE_ATMS_BASH_SCRIPT) + + # Make executable: + filename.chmod(stat.S_IRWXU) + + filename = cspp_home_dir / 'atms' / 'fake_adl_atms_script.py' + with open(filename, 'w') as fpt: + fpt.write(FAKE_ATMS_PYTHON_MAIN_SCRIPT) + + yield cspp_home_dir + + +SINGLE_ATMS_FILESET = [ + "TATMS_j01_d20230209_t1317546_e1325223_b27088_c20230209132641834076_cspp_dev.h5", + "SATMS_j01_d20230209_t1317546_e1325223_b27088_c20230209132641621751_cspp_dev.h5", + "GATMO_j01_d20230209_t1317546_e1325223_b27088_c20230209132642207625_cspp_dev.h5" +] + + +FAKE_ATMS_SDR_FILENAMES = [ + "TATMS_npp_d20230209_t1047306_e1236183_b58482_c20230209123846465182_cspp_dev.h5", + "SATMS_npp_d20230209_t1047306_e1236183_b58482_c20230209123844714309_cspp_dev.h5", + "GATMO_npp_d20230209_t1047306_e1236183_b58482_c20230209123847932256_cspp_dev.h5", +] +FAKE_ATMS_SDR_FILENAMES = FAKE_ATMS_SDR_FILENAMES + SINGLE_ATMS_FILESET + + +@pytest.fixture +def fake_atms_sdr_files_several_passes(tmp_path): + """Make fake ATMS SDR files.""" + for fname in FAKE_ATMS_SDR_FILENAMES: + filepath = tmp_path / fname + filepath.touch() + + yield tmp_path + + +@pytest.fixture +def fake_atms_sdr_files_one_pass(tmp_path_factory): + """Make fake ATMS SDR files.""" + dirname = tmp_path_factory.mktemp('data') + for fname in SINGLE_ATMS_FILESET: + filepath = dirname / fname + filepath.touch() + + yield dirname + + +@pytest.fixture +def fake_sdr_homedir(tmp_path_factory): + """Make a fake home directory for the SDR files.""" + dirname = tmp_path_factory.mktemp('sdr_home') + yield dirname diff --git a/cspp_runner/tests/test_config.py b/cspp_runner/tests/test_config.py new file mode 100644 index 0000000..16108c9 --- /dev/null +++ b/cspp_runner/tests/test_config.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2023 Adam.Dybbroe + +# Author(s): + +# Adam.Dybbroe + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +"""Test getting the yaml configurations from file.""" + +from cspp_runner.config import read_config + + +def test_get_yaml_configuration(fake_yamlconfig_file): + """Test read and get the yaml configuration from file.""" + config = read_config(fake_yamlconfig_file) + + assert len(config['subscribe_topics']) == 1 + assert config['subscribe_topics'][0] == '/file/atms/rdr' + assert len(config['publish_topics']) == 1 + assert config['publish_topics'][0] == '/file/atms/sdr' + + assert config['level1_home'] == '/path/to/where/the/atms/sdr/files/will/be/stored' + assert config['working_dir'] == '/san1/cspp/work' + assert config['atms_sdr_call'] == 'atms_sdr.sh' + assert config['atms_sdr_options'] == ['-a', '-d'] + + assert len(config['sdr_file_patterns']) == 3 diff --git a/cspp_runner/tests/test_run_atms.py b/cspp_runner/tests/test_run_atms.py new file mode 100644 index 0000000..19660c8 --- /dev/null +++ b/cspp_runner/tests/test_run_atms.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2023 Pytroll developers + +# Author(s): + +# Adam Dybbroe + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Testing the ATMS processing.""" + +import os +import logging +from glob import glob +from datetime import datetime + +from trollsift import Parser + +from cspp_runner.config import read_config + +from cspp_runner.atms_rdr2sdr_runner import get_filepaths +from cspp_runner.atms_rdr2sdr_runner import run_atms_from_message +from cspp_runner.atms_rdr2sdr_runner import get_filelist_from_collection +from cspp_runner.atms_rdr2sdr_runner import move_files_to_destination +from cspp_runner.atms_rdr2sdr_runner import _fix_orbit_number + + +ATMS_FILENAMES = ['RATMS-RNSCA_j01_d20230208_t1154172_e1154492_b00001_c20230208115457829000_drlu_ops.h5', + 'RATMS-RNSCA_j01_d20230208_t1154492_e1155212_b00001_c20230208115538023000_drlu_ops.h5', + 'RATMS-RNSCA_j01_d20230208_t1155212_e1155532_b00001_c20230208115558220000_drlu_ops.h5', + 'RATMS-RNSCA_j01_d20230208_t1155532_e1156252_b00001_c20230208115638170000_drlu_ops.h5', + 'RATMS-RNSCA_j01_d20230208_t1156252_e1156572_b00001_c20230208115718069000_drlu_ops.h5', + 'RATMS-RNSCA_j01_d20230208_t1156572_e1157292_b00001_c20230208115738216000_drlu_ops.h5', + 'RATMS-RNSCA_j01_d20230208_t1157292_e1158012_b00001_c20230208115818194000_drlu_ops.h5', + 'RATMS-RNSCA_j01_d20230208_t1158012_e1158332_b00001_c20230208115837985000_drlu_ops.h5', + 'RATMS-RNSCA_j01_d20230208_t1158332_e1159052_b00001_c20230208115918341000_drlu_ops.h5', + 'RATMS-RNSCA_j01_d20230208_t1159052_e1159372_b00001_c20230208115957983000_drlu_ops.h5', + 'RATMS-RNSCA_j01_d20230208_t1159372_e1200092_b00001_c20230208120017590000_drlu_ops.h5', + 'RATMS-RNSCA_j01_d20230208_t1200092_e1200412_b00001_c20230208120058642000_drlu_ops.h5', + 'RATMS-RNSCA_j01_d20230208_t1200412_e1201132_b00001_c20230208120118131000_drlu_ops.h5', + 'RATMS-RNSCA_j01_d20230208_t1201132_e1201452_b00001_c20230208120157708000_drlu_ops.h5', + 'RATMS-RNSCA_j01_d20230208_t1201452_e1202172_b00001_c20230208120238505000_drlu_ops.h5', + 'RATMS-RNSCA_j01_d20230208_t1202172_e1202492_b00001_c20230208120258137000_drlu_ops.h5', + 'RATMS-RNSCA_j01_d20230208_t1202492_e1203211_b00001_c20230208120337761000_drlu_ops.h5', + 'RATMS-RNSCA_j01_d20230208_t1203211_e1203531_b00001_c20230208120358165000_drlu_ops.h5', + 'RATMS-RNSCA_j01_d20230208_t1203531_e1204251_b00001_c20230208120437760000_drlu_ops.h5', + 'RATMS-RNSCA_j01_d20230208_t1204251_e1204571_b00001_c20230208120518174000_drlu_ops.h5'] + + +def test_run_atms_from_message(caplog, monkeypatch, fake_cspp_workdir, + fake_atms_posttroll_message, fake_adl_atms_scripts): + """Test launch and run the ATMS processing from a Posttroll message.""" + monkeypatch.setenv("CSPP_WORKDIR", str(fake_cspp_workdir)) + + cspp_homedir = fake_adl_atms_scripts + monkeypatch.setenv("CSPP_SDR_HOME", str(cspp_homedir)) + + mypath = cspp_homedir / 'atms' + path_env = os.environ.get('PATH') + monkeypatch.setenv("PATH", path_env + ":" + str(mypath)) + + sdr_call = 'atms_sdr.sh' + sdr_options = ['-d', '-a'] + with caplog.at_level(logging.DEBUG): + dirpath = run_atms_from_message(fake_atms_posttroll_message, sdr_call, sdr_options) + + assert os.path.dirname(dirpath) == str(fake_cspp_workdir) + res = caplog.text.strip().split('\n') + assert len(res) == 4 + + for atmsfile in ATMS_FILENAMES: + assert atmsfile in res[0] + assert atmsfile in res[1] + + assert "Seconds process time:" in res[2] + assert "Seconds wall clock time:" in res[3] + + +def test_get_filelist_from_collection(fake_atms_posttroll_message): + """Test launch and run the ATMS processing from a Posttroll message.""" + collection = fake_atms_posttroll_message.data.get('collection') + + files = get_filelist_from_collection(collection) + + assert len(files) == 20 + bnames = [os.path.basename(item) for item in files] + assert bnames == ATMS_FILENAMES + + +def test_get_filepaths(fake_yamlconfig_file, fake_atms_sdr_files_several_passes): + """Test get the filepaths of the ATMS SDR files produced from CSPP.""" + config = read_config(fake_yamlconfig_file) + + patterns = config['sdr_file_patterns'] + fake_message_data = {} + fake_message_data['orbit_number'] = 27088 + fake_message_data['platform_name'] = 'NOAA-20' + fake_message_data['start_time'] = datetime.strptime("2023-02-09T13:17:20.600000", "%Y-%m-%dT%H:%M:%S.%f") + fake_message_data['end_time'] = datetime.strptime("2023-02-09T13:25:52.600000", "%Y-%m-%dT%H:%M:%S.%f") + files = get_filepaths(str(fake_atms_sdr_files_several_passes), fake_message_data, patterns) + + assert len(files) == 3 + p__ = Parser(patterns[0]) + result = p__.parse(os.path.basename(files[0])) + assert result['platform_shortname'] == 'j01' + assert result['orbit'] == 27088 + assert result['start_time'] == datetime(2023, 2, 9, 13, 17, 54, 600000) + + +def test_move_files_to_destination_dir_is_str(fake_yamlconfig_file, fake_sdr_homedir, fake_atms_sdr_files_one_pass): + """Test move the ATMS SDR files to a destination dir.""" + config = read_config(fake_yamlconfig_file) + patterns = config['sdr_file_patterns'] + + sdr_file_paths = glob(str(fake_atms_sdr_files_one_pass / '*h5')) + expected = [os.path.basename(f) for f in sdr_file_paths] + expected.sort() + + filelist = move_files_to_destination(sdr_file_paths, patterns, str(fake_sdr_homedir)) + + assert len(filelist) == 3 + assert os.path.basename(os.path.normpath(os.path.dirname(filelist[0]))) == "noaa20_20230209_1317_27088" + bnames = [os.path.basename(f) for f in filelist] + bnames.sort() + + assert bnames == expected + + +def test_move_files_to_destination_pathlib(fake_yamlconfig_file, fake_sdr_homedir, fake_atms_sdr_files_one_pass): + """Test move the ATMS SDR files to a destination dir.""" + config = read_config(fake_yamlconfig_file) + patterns = config['sdr_file_patterns'] + + sdr_file_paths = glob(str(fake_atms_sdr_files_one_pass / '*h5')) + expected = [os.path.basename(f) for f in sdr_file_paths] + expected.sort() + + filelist = move_files_to_destination(sdr_file_paths, patterns, fake_sdr_homedir) + + assert len(filelist) == 3 + assert os.path.basename(os.path.normpath(os.path.dirname(filelist[0]))) == "noaa20_20230209_1317_27088" + bnames = [os.path.basename(f) for f in filelist] + bnames.sort() + + assert bnames == expected + + +def test_fix_orbit_number(fake_yamlconfig_file, fake_atms_sdr_files_one_pass): + """Test fixing the orbit number from the SDR filenames.""" + config = read_config(fake_yamlconfig_file) + patterns = config['sdr_file_patterns'] + + sdr_files = glob(str(fake_atms_sdr_files_one_pass / '*h5')) + + result = _fix_orbit_number(sdr_files, patterns) + assert result == 27088 diff --git a/cspp_runner/viirs_dr_runner.py b/cspp_runner/viirs_dr_runner.py index 5882b87..d9fafe1 100644 --- a/cspp_runner/viirs_dr_runner.py +++ b/cspp_runner/viirs_dr_runner.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# Copyright (c) 2013 - 2021, 2023 cspp-runner developers +# Copyright (c) 2013 - 2023 Pytroll developers # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,6 +15,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . + """Pytroll processing converting VIIRS RDR to SDR using CSPP. Level-1 processing for VIIRS Suomi NPP Direct Readout data. Using the CSPP diff --git a/examples/atms_dr_config.cfg_template b/examples/atms_dr_config.cfg_template new file mode 100644 index 0000000..8c86d9c --- /dev/null +++ b/examples/atms_dr_config.cfg_template @@ -0,0 +1,19 @@ +[DEFAULT] +# Location to store Sensor Data Record (SDR) files after CSPP SDR processing +# is completed. +level1_home = /san1/polar_in/direct_readout/npp/atms/sdr/ + +working_dir = /san1/cspp/work + +# CSPP-atms batch script and parameters: +atms_sdr_call = atms_sdr.sh -a -d + +# Options to pass to the viirs_sdr_call +# see viirs_sdr.sh --help for explanation +atms_sdr_options = ['-a', '-d'] + +# Topic to use for publishing posttroll messages +publish_topic = /file/atms/sdr + +# Posttroll topics to listen to (comma separated) +subscribe_topics = /file/atms/rdr diff --git a/examples/atms_dr_config.yaml_template b/examples/atms_dr_config.yaml_template new file mode 100644 index 0000000..9ade552 --- /dev/null +++ b/examples/atms_dr_config.yaml_template @@ -0,0 +1,33 @@ +# Location to store Sensor Data Record (SDR) files after CSPP SDR processing +# is completed. +level1_home: /path/to/where/the/atms/sdr/files/will/be/stored + +# Examples: +# TATMS_npp_d20230209_t1047306_e1236183_b58482_c20230209123846465182_cspp_dev.h5 +# SATMS_npp_d20230209_t1047306_e1236183_b58482_c20230209123844714309_cspp_dev.h5 +# GATMO_npp_d20230209_t1047306_e1236183_b58482_c20230209123847932256_cspp_dev.h5 + +sdr_file_patterns: + - 'SATMS_{platform_shortname}_d{start_time:%Y%m%d_t%H%M%S%f}_e{end_time:%H%M%S%f}_b{orbit:5d}_c{creation_time:%Y%m%d%H%M%S%f}_{source}.h5' + - 'GATMO_{platform_shortname}_d{start_time:%Y%m%d_t%H%M%S%f}_e{end_time:%H%M%S%f}_b{orbit:5d}_c{creation_time:%Y%m%d%H%M%S%f}_{source}.h5' + - 'TATMS_{platform_shortname}_d{start_time:%Y%m%d_t%H%M%S%f}_e{end_time:%H%M%S%f}_b{orbit:5d}_c{creation_time:%Y%m%d%H%M%S%f}_{source}.h5' + + +working_dir: /san1/cspp/work + +# CSPP-atms batch script and parameters: +atms_sdr_call: atms_sdr.sh + +# Options to pass to the viirs_sdr_call +# see viirs_sdr.sh --help for explanation +atms_sdr_options: + - '-a' + - '-d' + +# Topic to use for publishing posttroll messages +publish_topics: + - /file/atms/sdr + +# Posttroll topics to listen to (comma separated) +subscribe_topics: + - /file/atms/rdr diff --git a/examples/log_config.yaml_template b/examples/log_config.yaml_template new file mode 100644 index 0000000..c134fce --- /dev/null +++ b/examples/log_config.yaml_template @@ -0,0 +1,19 @@ +version: 1 +disable_existing_loggers: false +formatters: + pytroll: + format: '[%(asctime)s %(levelname)-8s %(name)s] %(message)s' +handlers: + console: + class: logging.StreamHandler + level: DEBUG + formatter: pytroll + stream: ext://sys.stdout +loggers: + posttroll: + level: ERROR + propagate: false + handlers: [console,] +root: + level: DEBUG + handlers: [console,] \ No newline at end of file diff --git a/setup.py b/setup.py index 972052b..a096f0f 100644 --- a/setup.py +++ b/setup.py @@ -42,8 +42,8 @@ setup(name=NAME, description=DESCRIPTION, - author='Adam Dybroe', - author_email='adam.dybroe@smhi.se', + author='The Pytroll Team', + author_email='pytroll@googlegroups.com', classifiers=["Development Status :: 3 - Alpha", "Intended Audience :: Science/Research", "License :: OSI Approved :: GNU General Public License v3 " + @@ -54,10 +54,9 @@ url="https://github.com/pytroll/pytroll-cspp-runner", long_description=long_description, packages=['cspp_runner', ], + scripts=['bin/atms_dr_runner.py'], data_files=[], install_requires=['posttroll>1.7', 'trollsift'], - # test_requires=['mock'], - # test_suite='cspp_runner.tests.suite', python_requires='>=3.8', zip_safe=False, use_scm_version=True