diff --git a/gwsumm/__main__.py b/gwsumm/__main__.py index 274f6868..946df33f 100644 --- a/gwsumm/__main__.py +++ b/gwsumm/__main__.py @@ -1,5 +1,6 @@ # coding=utf-8 # Copyright (C) Duncan Macleod (2013) +# Evan Goetz (2026) # # This file is part of GWSumm. # @@ -29,9 +30,7 @@ options available for each mode. """ -import argparse import calendar -import datetime import getpass import os import re @@ -43,7 +42,7 @@ from collections import OrderedDict from configparser import (DEFAULTSECT, NoOptionError, NoSectionError) -from dateutil.relativedelta import relativedelta +from matplotlib import font_manager as fm from urllib.parse import urlparse from glue.lal import (Cache, LIGOTimeGPS) @@ -51,12 +50,11 @@ from gwpy import time from gwpy.segments import (Segment, SegmentList) from gwpy.signal.spectral import _lal as fft_lal -from gwpy.time import (Time, _tconvert, tconvert, to_gps) +from gwpy.time import (Time, _tconvert, tconvert) from gwdetchar.utils.cli import logger from . import ( - __version__, archive, globalv, mode, @@ -73,8 +71,8 @@ get_tab, ) from .utils import ( - get_default_ifo, re_flagdiv, + create_parser, ) from .data import get_timeseries_dict @@ -88,440 +86,17 @@ _tconvert.LIGOTimeGPS = LIGOTimeGPS __author__ = 'Duncan Macleod ' +__credits__ = "Evan Goetz " # set defaults VERBOSE = False PROFILE = False -try: - DEFAULT_IFO = get_default_ifo() -except ValueError: - DEFAULT_IFO = None # initialize logger PROG = ('python -m gwsumm' if sys.argv[0].endswith('.py') else os.path.basename(sys.argv[0])) LOGGER = logger(name=PROG.split('python -m ').pop()) -# find today's date -TODAY = datetime.datetime.utcnow().strftime('%Y%m%d') - - -# -- argparse utilities ------------------------------------------------------- - -class GWArgumentParser(argparse.ArgumentParser): - def __init__(self, *args, **kwargs): - super(GWArgumentParser, self).__init__(*args, **kwargs) - self._positionals.title = 'Positional arguments' - self._optionals.title = 'Optional arguments' - - -class GWHelpFormatter(argparse.ArgumentDefaultsHelpFormatter): - def __init__(self, *args, **kwargs): - kwargs.setdefault('indent_increment', 4) - super(GWHelpFormatter, self).__init__(*args, **kwargs) - - -class DateAction(argparse.Action): - TIMESCALE = {'days': 1} - - @staticmethod - def set_gps_times(namespace, startdate, enddate): - setattr(namespace, 'gpsstart', to_gps(startdate)) - setattr(namespace, 'gpsend', to_gps(enddate)) - - def __call__(self, parser, namespace, values, option_string=None): - try: - date = datetime.datetime.strptime(values, self.DATEFORMAT) - except ValueError: - raise parser.error("%s malformed: %r. Please format as %s" - % (self.dest.title(), values, self.METAVAR)) - else: - self.set_gps_times(namespace, date, - date + relativedelta(**self.TIMESCALE)) - setattr(namespace, self.dest, date) - return date - - -class DayAction(DateAction): - TIMESCALE = {'days': 1} - DATEFORMAT = '%Y%m%d' - METAVAR = 'YYYYMMDD' - - -class WeekAction(DayAction): - TIMESCALE = {'days': 7} - - -class MonthAction(DateAction): - TIMESCALE = {'months': 1} - DATEFORMAT = '%Y%m' - METAVAR = 'YYYYMM' - - -class YearAction(DateAction): - TIMESCALE = {'years': 1} - DATEFORMAT = '%Y' - METAVAR = 'YYYY' - - -class GPSAction(argparse.Action): - def __call__(self, parser, namespace, values, option_string=False): - try: - values = float(values) - except (TypeError, ValueError): - pass - setattr(namespace, self.dest, to_gps(values)) - - -# -- parse command-line ------------------------------------------------------- - -def add_output_options(parser_): - """Add outuput options to the subparser - - This is only needed because argparse can't handle mutually exclusive - groups in a parent parser handed to a subparser for some reason. - """ - outopts = parser_.add_argument_group("Output options") - outopts.add_argument( - '-o', - '--output-dir', - action='store', - type=str, - metavar='DIR', - default=os.curdir, - help="Output directory for summary information", - ) - htmlopts = outopts.add_mutually_exclusive_group() - htmlopts.add_argument( - '-m', - '--html-only', - action='store_true', - default=False, - help="Generate container HTML and navigation only", - ) - htmlopts.add_argument( - '-n', - '--no-html', - action='store_true', - default=False, - help="Generate inner HTML and contents only, not supporting HTML", - ) - outopts.add_argument( - '-N', - '--no-htaccess', - action='store_true', - default=False, - help="don't create a .htaccess file to customise 404 errors", - ) - - -def add_archive_options(parser_): - """Add archiving options to the subparser - - This is only needed because argparse can't handle mutually exclusive - groups in a parent parser handed to a subparser for some reason. - """ - hierarchopts = parser_.add_argument_group('Archive options') - hierarchchoice = hierarchopts.add_mutually_exclusive_group() - hierarchchoice.add_argument( - '-a', - '--archive', - metavar='FILE_TAG', - default=False, - const='GW_SUMMARY_ARCHIVE', - nargs='?', - help="Read archived data from, and write processed data to " - "an HDF archive file written with the FILE_TAG. If not " - "given, no archive will be used, if given with no file " - "tag, a default of '%(const)s' will be used.", - ) - hierarchchoice.add_argument( - '-d', - '--daily-archive', - metavar='FILE_TAG', - default=False, - const='GW_SUMMARY_ARCHIVE', - nargs='?', - help="Read data from the daily archives, with the given FILE_TAG." - "If not given, daily archives will be used, if given with no " - "file tag, a default of '%(const)s' will be used.", - ) - - -def create_parser(): - """Create a command-line parser for this entry point - """ - # initialize top-level argument parser - parser = GWArgumentParser( - formatter_class=GWHelpFormatter, - prog=PROG, - description=__doc__, - fromfile_prefix_chars="@", - epilog="Arguments and options may be written into files and passed as " - "positional arguments prepended with '@', e.g. '%(prog)s " - "@args.txt'. In this format, options must be give as " - "'--argument=value', and not '--argument value'.", - ) - - # global arguments - parser.add_argument( - '-V', - '--version', - action='version', - version=__version__, - help="show program's version number and exit", - ) - - # shared arguments - sharedopts = GWArgumentParser(add_help=False) - sharedopts.title = 'Progress arguments' - sharedopts.add_argument( - '-v', - '--verbose', - action='store_true', - default=False, - help="show verbose logging output", - ) - sharedopts.add_argument( - '-D', - '--debug', - action='store_true', - default=False, - help="show information that could be useful in debugging", - ) - - # configuration arguments - copts = sharedopts.add_argument_group( - "Configuration options", - "Provide a number of INI-format configuration files", - ) - copts.add_argument( - '-i', - '--ifo', - default=DEFAULT_IFO, - metavar='IFO', - help="IFO prefix for interferometer to process. " - "If this option is set in the [DEFAULT] of any of " - "the INI files, giving it here is redundant.", - ) - copts.add_argument( - '-f', - '--config-file', - action='append', - type=str, - metavar='FILE', - default=[], - help="INI file for analysis, may be given multiple times", - ) - copts.add_argument( - '-t', - '--process-tab', - action='append', - type=str, - help="process only this tab, can be given multiple times", - ) - - # process options - popts = sharedopts.add_argument_group( - "Process options", - "Configure how this summary will be processed.", - ) - popts.add_argument( - '--nds', - action='store_true', - default=None, - help="use NDS as the data source, default: 'guess'", - ) - popts.add_argument( - '-j', - '--multi-process', - action='store', - type=int, - default=1, - dest='multiprocess', - metavar='N', - help="use a maximum of N parallel processes at any time", - ) - popts.add_argument( - '-b', - '--bulk-read', - action='store_true', - default=False, - help="read all data up-front at the start of the job, " - "rather than when it is needed for a tab", - ) - popts.add_argument( - '-S', - '--on-segdb-error', - action='store', - type=str, - default='raise', - choices=['raise', 'ignore', 'warn'], - help="action upon error fetching segments from SegDB", - ) - popts.add_argument( - '-G', - '--on-datafind-error', - action='store', - type=str, - default='raise', - choices=['raise', 'ignore', 'warn'], - help="action upon error querying for frames from the " - "datafind server, default: %(default)s", - ) - popts.add_argument( - '--data-cache', - action='append', - default=[], - help='path to LAL-format cache of TimeSeries data files', - ) - popts.add_argument( - '--event-cache', - action='append', - default=[], - help='path to LAL-format cache of event trigger files', - ) - popts.add_argument( - '--segment-cache', - action='append', - default=[], - help='path to LAL-format cache of state ' - 'or data-quality segment files', - ) - - # define sub-parser handler - subparsers = parser.add_subparsers( - dest='mode', - title='Modes', - description='Note: all dates are defined with UTC boundaries.\n' - 'The valid modes are:', - ) - subparser = dict() - - # DAY mode - daydoc = """ - Run %s over a full UTC day, and link this day to others with a calendar - built into the HTML navigation bar. In this mode you can also archive data - in HDF-format files to allow progressive processing of live data without - restarting from scratch every time.""" % parser.prog - subparser['day'] = subparsers.add_parser( - 'day', - description=daydoc, - epilog=parser.epilog, - parents=[sharedopts], - formatter_class=GWHelpFormatter, - help="Process one day of data", - ) - subparser['day'].add_argument( - 'day', - action=DayAction, - type=str, - nargs='?', - metavar=DayAction.METAVAR, - default=TODAY, - help="Day to process", - ) - add_output_options(subparser['day']) - - darchopts = subparser['day'].add_argument_group( - 'Archive options', - 'Choose if, and how, to archive data from this run', - ) - darchopts.add_argument( - '-a', - '--archive', - metavar='FILE_TAG', - default=False, - const='GW_SUMMARY_ARCHIVE', - nargs='?', - help="Read archived data from, and write processed " - "data to, an HDF archive file written with the " - "FILE_TAG. If not given, no archive will be used, " - "if given with no file tag, a default of " - "'%(const)s' will be used.", - ) - - # WEEK mode - subparser['week'] = subparsers.add_parser( - 'week', - parents=[sharedopts], - epilog=parser.epilog, - formatter_class=GWHelpFormatter, - help="Process one week of data", - ) - subparser['week'].add_argument( - 'week', - action=WeekAction, - type=str, - metavar=WeekAction.METAVAR, - help="Week to process (given as starting day)", - ) - add_output_options(subparser['week']) - add_archive_options(subparser['week']) - - # MONTH mode - subparser['month'] = subparsers.add_parser( - 'month', - parents=[sharedopts], - epilog=parser.epilog, - formatter_class=GWHelpFormatter, - help="Process one month of data", - ) - subparser['month'].add_argument( - 'month', - action=MonthAction, - type=str, - metavar=MonthAction.METAVAR, - help="Month to process", - ) - add_output_options(subparser['month']) - add_archive_options(subparser['month']) - - # GPS mode - subparser['gps'] = subparsers.add_parser( - 'gps', - parents=[sharedopts], - epilog=parser.epilog, - formatter_class=GWHelpFormatter, - help="Process GPS interval", - ) - subparser['gps'].add_argument( - 'gpsstart', - action=GPSAction, - type=str, - metavar='GPSSTART', - help='GPS start time', - ) - subparser['gps'].add_argument( - 'gpsend', - action=GPSAction, - type=str, - metavar='GPSEND', - help='GPS end time.', - ) - - garchopts = subparser['gps'].add_argument_group( - 'Archive options', - 'Choose if, and how, to archive data from this run', - ) - garchopts.add_argument( - '-a', - '--archive', - metavar='FILE_TAG', - default=False, - const='GW_SUMMARY_ARCHIVE', - nargs='?', - help="Read archived data from, and write processed " - "data to, an HDF archive file written with the " - "FILE_TAG. If not given, no archive will be used, " - "if given with no file tag, a default of " - "'%(const)s' will be used.") - - add_output_options(subparser['gps']) - - # return the argument parser - return parser - # -- main code block ---------------------------------------------------------- @@ -541,11 +116,6 @@ def main(args=None): args.config_file = [os.path.expanduser(fp) for csv in args.config_file for fp in csv.split(',')] - # check segdb option - if args.on_segdb_error not in ['raise', 'warn', 'ignore']: - parser.error("Invalid option --on-segdb-error='%s'" % - args.on_segdb_error) - # read configuration file config = GWSummConfigParser() config.optionxform = str @@ -598,6 +168,10 @@ def main(args=None): path = mode.get_base(utc) except ValueError: path = os.path.join('gps', f'{args.gpsstart}-{args.gpsend}') + if args.archive_write_dir is None: + args.archive_write_dir = f'{path}/archive' + if args.archive_read_dir is None: + args.archive_read_dir = f'{path}/archive' # set LAL FFT plan wisdom level duration = min(globalv.NOW, args.gpsend) - args.gpsstart @@ -641,6 +215,16 @@ def main(args=None): LOGGER.debug("Output directory: {}".format( os.path.abspath(os.path.join(args.output_dir, path)))) + # Set fonts + if ('XDG_DATA_HOME' in os.environ and + os.path.exists( + p := os.path.join(os.environ['XDG_DATA_HOME'], 'fonts') + )): + LOGGER.debug(f"Adding fonts for matplotlib from {p}") + font_files = fm.findSystemFonts(fontpaths=p) + for f in font_files: + fm.fontManager.addfont(f) + # -- Finalise configuration LOGGER.info("Loading configuration") plugins = config.load_plugins() @@ -681,8 +265,9 @@ def main(args=None): # -- read archive ------------------------------- - if not hasattr(args, 'archive'): - args.archive = False + # EG: I don't know why this would be needed. Commenting out for now + # if not hasattr(args, 'archive'): + # args.archive = False if args.html_only: args.archive = False @@ -690,31 +275,43 @@ def main(args=None): elif args.archive is True: args.archive = 'GW_SUMMARY_ARCHIVE' - archives = [] + archives_read = [] + archives_write = [] if args.archive: - archivedir = os.path.join(path, 'archive') - os.makedirs(archivedir, exist_ok=True) - args.archive = os.path.join(archivedir, '%s-%s-%d-%d.h5' - % (ifo, args.archive, args.gpsstart, - args.gpsend - args.gpsstart)) - if os.path.isfile(args.archive): - archives.append(args.archive) + os.makedirs(args.archive_write_dir, exist_ok=True) + archive_file_read = os.path.join( + args.archive_read_dir, + (f'{ifo}-{args.archive}-{args.gpsstart}-' + f'{args.gpsend - args.gpsstart}.h5') + ) + archive_file_write = os.path.join( + args.archive_write_dir, + (f'{ifo}-{args.archive}-{args.gpsstart}-' + f'{args.gpsend - args.gpsstart}.h5') + ) + if not os.path.isfile(archive_file_read): + LOGGER.debug(f"No archive found in {archive_file_read}, one will" + f" be created at {archive_file_write}") else: - LOGGER.debug( - "No archive found in %s, one will be created at the end" - % args.archive) + archives_read.append(archive_file_read) + archives_write.append(archive_file_write) # read daily archive for week/month/... mode if hasattr(args, 'daily_archive') and args.daily_archive: # find daily archive files - archives.extend(archive.find_daily_archives( - args.gpsstart, args.gpsend, ifo, args.daily_archive, archivedir)) + archives_read.extend(archive.find_daily_archives( + args.gpsstart, + args.gpsend, + ifo, + args.daily_archive, + args.archive_read_dir, + )) # then don't read any actual data cache['datacache'] = Cache() - for arch in archives: - LOGGER.info("Reading archived data from %s" % arch) + for arch in archives_read: + LOGGER.info(f"Reading archived data from {arch}") archive.read_data_archive(arch) LOGGER.debug("Archive data loaded") @@ -871,9 +468,9 @@ def main(args=None): if args.archive: LOGGER.info("Writing data to archive") try: - archive.write_data_archive(args.archive) + archive.write_data_archive(archives_write[0]) LOGGER.debug( - f"Archive written to {os.path.abspath(args.archive)}") + f"Archive written to {os.path.abspath(archives_write[0])}") except Exception: LOGGER.warning( "New data archiving failed. Previous archive preserved.") diff --git a/gwsumm/batch.py b/gwsumm/batch.py index 4dba5a18..3d18c14d 100644 --- a/gwsumm/batch.py +++ b/gwsumm/batch.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (C) Duncan Macleod (2013) +# Evan Goetz (2026) # # This file is part of GWSumm. # @@ -23,16 +24,18 @@ a workflow to be submitted via the HTCondor scheduler. """ -import argparse import os +from pathlib import Path import shutil import sys from glue import pipeline from gwdetchar.utils import cli +from gwpy.time import tconvert -from . import __version__ +from . import mode +from .utils import create_parser __author__ = 'Duncan Macleod ' __credits__ = ('Alex Urban , ' @@ -52,12 +55,12 @@ class GWSummaryJob(pipeline.CondorDAGJob): """ logtag = '$(cluster)-$(process)' - def __init__(self, universe, tag='gw_summary', + def __init__(self, universe, executable='python3', tag='gw_summary', subdir=None, logdir=None, **cmds): - pipeline.CondorDAGJob.__init__(self, universe, sys.executable) + pipeline.CondorDAGJob.__init__(self, universe, executable) if subdir: subdir = os.path.abspath(subdir) - self.set_sub_file(os.path.join(subdir, '%s.sub' % (tag))) + self.set_sub_file(os.path.join(subdir, '%s.sub' % tag)) if logdir: logdir = os.path.abspath(logdir) self.set_log_file(os.path.join( @@ -72,7 +75,10 @@ def __init__(self, universe, tag='gw_summary', else: self.add_condor_cmd(key, val) # add python module sub-command - self._command = ' '.join(['-m', __package__]) + if executable != 'python3': + self._command = 'exec' + else: + self._command = ' '.join(['-m', __package__]) def add_opt(self, opt, value=''): pipeline.CondorDAGJob.add_opt(self, opt, str(value)) @@ -109,315 +115,12 @@ def get_cmd_line(self): ]) -# -- parse command-line ------------------------------------------------------- - -class GWHelpFormatter(argparse.HelpFormatter): - def __init__(self, *args, **kwargs): - kwargs.setdefault('indent_increment', 4) - super(GWHelpFormatter, self).__init__(*args, **kwargs) - - -def create_parser(): - """Create a command-line parser for this entry point - """ - # initialize argument parser - usage = ('%(prog)s --global-config defaults.ini --config-file ' - 'myconfig.ini [--config-file myconfig2.ini] [options]') - parser = argparse.ArgumentParser( - prog=PROG, - usage=usage, - description=__doc__, - formatter_class=GWHelpFormatter, - ) - bopts = parser.add_argument_group("Basic options") - htcopts = parser.add_argument_group("Condor options") - copts = parser.add_argument_group( - "Configuration options", - "Each --global-config file will be used in all nodes of the workflow, " - "while a single node will be created for each other --config-file", - ) - popts = parser.add_argument_group( - "Process options", - "Configure how this summary will be processed.", - ) - outopts = parser.add_argument_group("Output options") - topts = parser.add_argument_group( - "Time mode options", - "Choose a stadard time mode, or a GPS [start, stop) interval", - ) - - # general arguments - parser.add_argument( - "-v", - "--verbose", - action="store_true", - default=False, - help="show verbose output, default: %(default)s", - ) - parser.add_argument( - "-V", - "--version", - action="version", - help="show program's version number and exit", - ) - parser.version = __version__ - - # basic options - bopts.add_argument( - '-i', - '--ifo', - action='store', - type=str, - metavar='IFO', - help="Instrument to process. If this option is set in " - "the [DEFAULT] of any of the INI files, giving it " - "here is redundant.", - ) - wrapgroup = bopts.add_mutually_exclusive_group() - wrapgroup.add_argument( - '-w', - '--skip-html-wrapper', - action='store_true', - default=False, - help="Do not configure first job for HTML htmlnode, default: " - "%(default)s. Useful for separating large summary pipeline " - "across multiple DAGs", - ) - wrapgroup.add_argument( - '-W', - '--html-wrapper-only', - action='store_true', - help="Only run first job for HTML htmlnode.", - ) - bopts.add_argument( - '-t', - '--file-tag', - action='store', - type=str, - default='gw_summary_pipe', - help="file tag for pipeline files, default: %(default)s", - ) - - # HTCondor options - htcopts.add_argument( - '-u', - '--universe', - action='store', - type=str, - default='vanilla', - help="Universe for condor jobs, default: %(default)s", - ) - htcopts.add_argument( - '-l', - '--log-dir', - action='store', - type=str, - default=os.environ.get('LOCALDIR', None), - help="Directory path for condor log files, default: %(default)s", - ) - htcopts.add_argument( - '-m', - '--maxjobs', - action='store', - type=int, - default=None, - metavar='N', - help="Restrict the DAG to submit only N jobs at any one " - "time, default: %(default)s", - ) - htcopts.add_argument( - '-T', - '--condor-timeout', - action='store', - type=float, - default=None, - metavar='T', - help='Configure condor to terminate jobs after T hours ' - 'to prevent idling, default: %(default)s', - ) - htcopts.add_argument( - '-c', - '--condor-command', - action='append', - type=str, - default=[], - help="Extra condor submit commands to add to gw_summary submit file. " - "Can be given multiple times in the form \"key=value\"", - ) - - # configuration options - copts.add_argument( - '-f', - '--config-file', - action='append', - type=str, - metavar='FILE', - default=[], - help="INI file for analysis, may be given multiple times", - ) - copts.add_argument( - '-g', - '--global-config', - action='append', - type=str, - metavar='FILE', - default=[], - help="INI file for use in all workflow jobs, may be given " - "multiple times", - ) - copts.add_argument( - '-p', - '--priority', - action='append', - type=str, - default=[], - help="priority for DAG node, should be given once " - "for each --config-file in the same order", - ) - - # process options - popts.add_argument( - '--nds', - action='store_true', - default='guess', - help='use NDS as the data source, default: %(default)s', - ) - popts.add_argument( - '--single-process', - action='store_true', - default=False, - help="restrict gw_summary to a single process, mainly for " - "debugging purposes, default: %(default)s", - ) - popts.add_argument( - '--multi-process', - action='store', - type=int, - default=None, - help="maximum number of concurrent sub-processes for each " - "gw_summary job, {number of CPUs} / {min(number of jobs, 4)}", - ) - popts.add_argument( - '-a', - '--archive', - action='store_true', - default=False, - help="Read archived data from the FILE, and " - "write back to it at the end", - ) - popts.add_argument( - '-S', - '--on-segdb-error', - action='store', - type=str, - default='raise', - choices=['raise', 'ignore', 'warn'], - help="action upon error fetching segments from SegDB", - ) - popts.add_argument( - '-G', - '--on-datafind-error', - action='store', - type=str, - default='raise', - choices=['raise', 'ignore', 'warn'], - help="action upon error querying for frames from the " - "datafind server: default: %(default)s", - ) - popts.add_argument( - '--data-cache', - action='append', - default=[], - help='path to LAL-format cache of TimeSeries data files', - ) - popts.add_argument( - '--event-cache', - action='append', - default=[], - help='path to LAL-format cache of event trigger files', - ) - popts.add_argument( - '--segment-cache', - action='append', - default=[], - help='path to LAL-format cache of state ' - 'or data-quality segment files', - ) - popts.add_argument( - '--no-htaccess', - action='store_true', - default=False, - help='tell gw_summary to not write .htaccess files', - ) - - # output options - outopts.add_argument( - '-o', - '--output-dir', - action='store', - type=str, - metavar='OUTDIR', - default=os.curdir, - help="Output directory for summary information, " - "default: '%(default)s'", - ) - - # time mode options - topts.add_argument( - "--day", - action="store", - type=str, - metavar='YYYYMMDD', - help="UTC date to process", - ) - topts.add_argument( - "--week", - action="store", - type=str, - metavar="YYYYMMDD", - help="week to process (by UTC starting date)", - ) - topts.add_argument( - "--month", - action="store", - type=str, - metavar="YYYYMM", - help="calendar month to process", - ) - topts.add_argument( - "--year", - action="store", - type=str, - metavar="YYYY", - help="calendar year to process", - ) - topts.add_argument( - "-s", - "--gps-start-time", - action="store", - type=int, - metavar="GPSSTART", - help="GPS start time", - ) - topts.add_argument( - "-e", - "--gps-end-time", - action="store", - type=int, - metavar="GPSEND", - help="GPS end time", - ) - - # return the argument parser - return parser - - # -- main code block ---------------------------------------------------------- def main(args=None): """Run the command-line Omega scan tool in batch mode """ - parser = create_parser() + parser = create_parser(include_condor_opts=True) args = parser.parse_args(args=args) # initialize logger @@ -426,12 +129,16 @@ def main(args=None): level='DEBUG' if args.verbose else 'INFO', ) - # check time options - N = sum([args.day is not None, args.month is not None, - args.gps_start_time is not None, args.gps_end_time is not None]) - if N > 1 and not (args.gps_start_time and args.gps_end_time): - raise parser.error("Please give only one of --day, --month, or " - "--gps-start-time and --gps-end-time.") + # set mode and output directory + # This will set the output directory to be day/YYYYMMDD, week/YYYYMMDD, + # month/YYYYMM, or gps/-. + mode.set_mode(args.mode) + try: + utc = tconvert(args.gpsstart) + outpath = mode.get_base(utc) + outpath = str(Path(outpath).parent) + except ValueError: + outpath = 'gps' for (i, cf) in enumerate(args.config_file): args.config_file[i] = ','.join(map(os.path.abspath, cf.split(','))) @@ -478,8 +185,6 @@ def main(args=None): dag = pipeline.CondorDAG(os.path.join(htclogdir, f'{args.file_tag}.log')) dag.set_dag_file(os.path.join(outdir, args.file_tag)) - universe = args.universe - # -- parse condor commands ---------------------- # parse into a dict @@ -551,47 +256,100 @@ def main(args=None): globalconfig = ','.join(args.global_config) jobs = [] - if not args.skip_html_wrapper: + # Define an HTML only job which runs in the local universe. + # This is always just one job, based on the structure we've defined. + if args.html_only or not args.no_html: htmljob = GWSummaryJob( - 'local', subdir=outdir, logdir=logdir, + 'local', executable='apptainer', + subdir=outdir, logdir=logdir, tag=f'{args.file_tag}_local', **condorcmds, getenv=envvars, ) jobs.append(htmljob) - if not args.html_wrapper_only: + # Define data jobs which run either in the local or container universe. We + # need to handle the case where jobs may not have completed previously + # so there may be no archive file. + if not args.html_only or args.no_html: + # HTCondor file transfer commands + transfer_aux_files = [] + if args.universe != 'local': + # set executable + executable = 'python3' + # Set transfer commands + for cmd, val in {'should_transfer_files': 'YES', + 'transfer_input_files': '$(inputfiles)', + 'transfer_output_files': outpath, + 'container_image': args.container_path}.items(): + if cmd not in condorcmds: + condorcmds[cmd] = val + # Change the environment variable definitions when transferring + # files to an execute point. + # List the all of the environment variables set by the submit file + condor_envvars = condorcmds['environment'].split(' ') + # When an environment variable points to a directory or file, + # change the environment variable just to the name and add the + # path to the list of files to be transferred. + for idx, entry in enumerate(condor_envvars): + var, val = entry.split('=', 1) + if (testpath := Path(val)).exists(): + transfer_aux_files.append(str(testpath)) + updated_env = testpath.name + condor_envvars[idx] = (f'{var}=$$(CondorScratchDir)/' + f'{updated_env}') + # Now rejoin the environment variables + condorcmds['environment'] = ' '.join(condor_envvars) + # Join the auxiliary files in a comma separated list + transfer_aux_files = ','.join(transfer_aux_files) + + # container universe jobs condor commands + for cmd_ in args.condor_command_container: + (key, value) = cmd_.split('=', 1) + if key in condorcmds: + logger.debug(f"Appending {value.strip()} to condor '{key}'" + f" value") + condorcmds[key] += f" {value.strip()}" + else: + condorcmds[key.rstrip()] = value.strip() + else: + executable = 'apptainer' + # A data job datajob = GWSummaryJob( - universe, subdir=outdir, logdir=logdir, + args.universe, executable=executable, + subdir=outdir, logdir=logdir, tag=args.file_tag, **condorcmds, getenv=envvars, ) + # A data job with no archive file + datajob_noarchive = GWSummaryJob( + args.universe, executable=executable, + subdir=outdir, logdir=logdir, + tag=f'{args.file_tag}_noarchive', **condorcmds, + getenv=envvars, + ) jobs.append(datajob) + jobs.append(datajob_noarchive) # add common command-line options for job in jobs: - if args.day: - job.set_command('day') - job.add_arg(args.day) - elif args.week: - job.set_command('week') - job.add_arg(args.week) - elif args.month: - job.set_command('month') - job.add_arg(args.month) - elif args.year: - job.set_command('year') - job.add_arg(args.year) - elif args.gps_start_time or args.gps_end_time: - job.set_command('gps') - job.add_arg(str(args.gps_start_time)) - job.add_arg(str(args.gps_end_time)) - else: - job.set_command('day') + if job.get_universe() == 'local': + job.set_command( + f'{os.path.abspath(args.container_path)} python3 -m ' + f'{__package__}' + ) + job.set_command(args.mode) + if args.mode == 'day': + job.add_arg(args.day.strftime("%Y%m%d")) + elif args.mode == 'week': + job.add_arg(args.week.strftime("%Y%m%d")) + elif args.mode == 'month': + job.add_arg(args.month.strftime("%Y%m")) + elif args.mode == 'gps': + job.add_arg(str(args.gpsstart)) + job.add_arg(str(args.gpsend)) if args.nds is True: job.add_opt('nds') - if args.single_process: - job.add_opt('single-process') - elif args.multi_process is not None: - job.add_opt('multi-process', args.multi_process) + if args.multiprocess is not None: + job.add_opt('multi-process', args.multiprocess) if args.verbose: job.add_opt('verbose') if args.ifo: @@ -606,47 +364,111 @@ def main(args=None): job.add_arg('%s %s' % (opt, (' %s ' % opt).join(fplist))) if args.no_htaccess: job.add_opt('no-htaccess') + # If the job is in the container universe, the transferred archive file + # will be in the flat directory so set the archive read directory for + # the job to read from the flat directory + if ('noarchive' not in job.get_sub_file() and + args.archive and + job.get_universe() == 'container'): + job.add_opt('archive-read-dir', '.') + if (args.archive and job.get_universe() == 'container' and + args.archive_write_dir): + job.add_opt('archive-write-dir', args.archive_write_dir) # make surrounding HTML first - if not args.skip_html_wrapper: + if args.html_only or not args.no_html: htmljob.add_opt('html-only', '') htmljob.add_opt('config-file', ','.join( [globalconfig]+args.config_file).strip(',')) htmlnode = GWSummaryDAGNode(htmljob) - for configfile in args.config_file: - htmlnode.add_input_file(args.config_file) + htmlnode.add_input_file(args.config_file) htmlnode.set_category('gw_summary') dag.add_node(htmlnode) logger.debug(" -- Configured HTML htmlnode job") # create node for each config file - if not args.html_wrapper_only: + if not args.html_only or args.no_html: # add html opts datajob.add_opt('no-html', '') + datajob_noarchive.add_opt('no-html', '') if args.archive: datajob.add_condor_cmd('+SummaryNodeType', '"$(macroarchive)"') + datajob_noarchive.add_condor_cmd( + '+SummaryNodeType', '"$(macroarchive)"' + ) # configure each data node for (i, configfile) in enumerate(args.config_file): - node = GWSummaryDAGNode(datajob) - node.add_var_arg('--config-file %s' % ','.join( - [globalconfig, configfile]).strip(',')) + # If we are using archive files, then we need to transfer any + # HDF5 archive files with the tag in args.archive if running in + # the container universe + files = '' # Comma separated list of existing archive files if args.archive: + # First figure out the archive tag jobtag = os.path.splitext(os.path.basename(configfile))[0] archivetag = jobtag.upper().replace('-', '_') - if args.ifo and archivetag.startswith('%s_' % - args.ifo.upper()): + if args.ifo and archivetag.startswith(f'{args.ifo.upper()}_'): archivetag = archivetag[3:] + + # If running in the container universe, check to see if any + # archive file already exists. If it does then we need to + # transfer it there. + # Both datajob types have the same universe, so only check one + if datajob.get_universe() == 'container': + # If we find any matching archive files make a comma + # separated string with those files + if len(archives := sorted(Path(args.archive_read_dir).glob( + f'*-{archivetag}-*.h5'))) > 0: + archives = [str(f) for f in archives] + files = ','.join(archives).strip(',') + + # If there was no archive file found for this config group and + # running in the container universe for the data jobs, then + # this node will be a "no archive". We don't transfer an + # archive file, and we don't give the --archive-read-dir . + # option. + if len(files) == 0 and datajob.get_universe() == 'container': + node = GWSummaryDAGNode(datajob_noarchive) + else: + node = GWSummaryDAGNode(datajob) + + # Finally, we have a node, so add the --archive + # option node.add_var_opt('archive', archivetag) - for cf in configfile.split(','): - node.add_input_file(cf) + else: + node = GWSummaryDAGNode(datajob) + + # Configuration files for gw_summary jobs are going to be + # given as the full path (local universe jobs), or just the + # file name (container universe jobs) + if datajob.get_universe() == 'local': + config_files = ','.join([globalconfig, configfile]).strip(',') + elif datajob.get_universe() == 'container': + # Use just the file name + config_files = ','.join( + [Path(f).name for f in + ','.join( + [globalconfig.strip(','), configfile.strip(',')] + ).split(',')] + ) + # Transferred files are the paths to file relative to the + # submit location (usually ~detchar/public_html/summary). + # The comma separated string is added to the dag as a macro + # variable inputfiles + txfr_files = ','.join( + [globalconfig, configfile, transfer_aux_files, files] + ).strip(',') + node.add_macro('inputfiles', txfr_files) + else: + raise ValueError(f"Unknown universe {node.get_universe()}") + node.add_var_opt('config-file', config_files) node.set_category('gw_summary') try: node.set_priority(args.priority[i]) except IndexError: node.set_priority(0) node.set_retry(1) - if not args.skip_html_wrapper: + if not args.no_html: node.add_parent(htmlnode) dag.add_node(node) logger.debug(" -- Configured job for config %s" % configfile) diff --git a/gwsumm/tabs/guardian.py b/gwsumm/tabs/guardian.py index 8f1a383c..07b80579 100644 --- a/gwsumm/tabs/guardian.py +++ b/gwsumm/tabs/guardian.py @@ -27,7 +27,7 @@ import numpy -from matplotlib.cm import get_cmap +from matplotlib.pyplot import get_cmap from matplotlib.colors import Normalize from astropy.time import Time diff --git a/gwsumm/tests/test_batch.py b/gwsumm/tests/test_batch.py index 653d083d..13edfe43 100644 --- a/gwsumm/tests/test_batch.py +++ b/gwsumm/tests/test_batch.py @@ -20,8 +20,8 @@ """ import os +from pathlib import Path import pytest -import shutil from .. import batch @@ -34,15 +34,14 @@ def _get_inputs(): """Prepare and return paths to input data products """ - indir = os.getcwd() inputs = ( - os.path.join(indir, "global.ini"), - os.path.join(indir, "k1-test.ini"), + "global.ini", + "k1-test.ini", + "K1-TEST-1.h5" ) # write empty input files for filename in inputs: - with open(filename, 'w') as f: - f.write("") + Path(filename).touch() return inputs @@ -50,8 +49,9 @@ def _get_inputs(): def test_main(tmpdir, caplog): outdir = str(tmpdir) - (global_, k1test,) = _get_inputs() + (global_, k1test, archivefile) = _get_inputs() args = [ + 'day', '--verbose', '--ifo', 'K1', '--maxjobs', '5', @@ -64,9 +64,11 @@ def test_main(tmpdir, caplog): '--nds', '--multi-process', '4', '--archive', + '--archive-read-dir', str(Path('.').absolute()), '--event-cache', '/this/cache/is/not/real.cache', '--no-htaccess', '--output-dir', outdir, + '--container-path', str(Path('./fake_container.sif').absolute()), ] # test log output batch.main(args) @@ -92,33 +94,38 @@ def test_main(tmpdir, caplog): } assert set(os.listdir(os.path.join(outdir, "logs"))) == set() # clean up - for filename in (global_, k1test,): - os.remove(filename) - shutil.rmtree(outdir, ignore_errors=True) + for filename in (global_, k1test, archivefile): + Path(filename).unlink() + for filename in Path(outdir, 'etc').iterdir(): + Path(filename).unlink() + Path(outdir, 'etc').rmdir() + Path(outdir, 'logs').rmdir() + for filename in Path(outdir).iterdir(): + Path(filename).unlink() + Path(outdir).rmdir() @pytest.mark.parametrize( 'mode', - (['--day', '20170209'], - ['--week', '20170209'], - ['--month', '201702'], - ['--year', '2017'], - ['--gps-start-time', '1170633618', '--gps-end-time', '1170720018']), + (['day', '20170209'], + ['week', '20170209'], + ['month', '201702'], + ['gps', '1170633618', '1170720018']), ) def test_main_loop_over_modes(tmpdir, caplog, mode): outdir = str(tmpdir) - (global_, k1test,) = _get_inputs() + (global_, k1test, archivefile) = _get_inputs() args = [ '--verbose', '--ifo', 'K1', '--universe', 'local', '--config-file', k1test, '--global-config', global_, - '--single-process', '--output-dir', outdir, + '--container-path', str(Path('./fake_container.sif').absolute()), ] # test log output - batch.main(args + mode) + batch.main(mode + args) assert "Copied all INI configuration files to ./etc" in caplog.text assert " -- Configured HTML htmlnode job" in caplog.text assert " -- Configured job for config {}".format( @@ -126,20 +133,12 @@ def test_main_loop_over_modes(tmpdir, caplog, mode): assert "Setup complete, DAG written to: {}".format( os.path.join(outdir, "gw_summary_pipe.dag")) in caplog.text # clean up - for filename in (global_, k1test,): - os.remove(filename) - shutil.rmtree(outdir, ignore_errors=True) - - -def test_main_invalid_modes(capsys): - args = [ - '--ifo', 'V1', - '--day', '20170209', - '--month', '201702', - ] - # test output - with pytest.raises(SystemExit): - batch.main(args) - (_, err) = capsys.readouterr() - assert err.endswith("Please give only one of --day, --month, or " - "--gps-start-time and --gps-end-time.\n") + for filename in (global_, k1test, archivefile): + Path(filename).unlink() + for filename in Path(outdir, 'etc').iterdir(): + Path(filename).unlink() + Path(outdir, 'etc').rmdir() + Path(outdir, 'logs').rmdir() + for filename in Path(outdir).iterdir(): + Path(filename).unlink() + Path(outdir).rmdir() diff --git a/gwsumm/utils.py b/gwsumm/utils.py index 5d6fa6a6..4d8b7007 100644 --- a/gwsumm/utils.py +++ b/gwsumm/utils.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (C) Duncan Macleod (2013) +# Evan Goetz (2026) # # This file is part of GWSumm. # @@ -19,6 +20,10 @@ """Utilities for GWSumm """ +import argparse +import datetime +from dateutil.relativedelta import relativedelta +import os import re import sys from socket import getfqdn @@ -27,7 +32,9 @@ from math import pi # noqa: F401 import numpy # noqa: F401 -from . import globalv +from gwpy.time import to_gps + +from . import globalv, __version__ re_cchar = re.compile(r"[\W_]+") re_quote = re.compile(r'^[\s\"\']+|[\s\"\']+$') @@ -42,6 +49,9 @@ UNSAFE_EVAL_STRS = [r'os\.(?![$\'\" ])', 'shutil', r'\.rm', r'\.mv'] UNSAFE_EVAL = re.compile(r'(%s)' % '|'.join(UNSAFE_EVAL_STRS)) +PROG = ('python -m gwsumm' if sys.argv[0].endswith('.py') + else os.path.basename(sys.argv[0])) + # -- utilities ---------------------------------------------------------------- @@ -223,3 +233,563 @@ def get_default_ifo(fqdn=getfqdn()): elif '.virgo.' in fqdn or '.ego-gw.' in fqdn: return 'V1' raise ValueError("Cannot determine default IFO for host %r" % fqdn) + + +# -- argparse utilities ------------------------------------------------------- + +class GWArgumentParser(argparse.ArgumentParser): + def __init__(self, *args, **kwargs): + super(GWArgumentParser, self).__init__(*args, **kwargs) + self._positionals.title = 'Positional arguments' + self._optionals.title = 'Optional arguments' + + +class GWHelpFormatter(argparse.ArgumentDefaultsHelpFormatter): + def __init__(self, *args, **kwargs): + kwargs.setdefault('indent_increment', 4) + super(GWHelpFormatter, self).__init__(*args, **kwargs) + + +class DateAction(argparse.Action): + TIMESCALE = {'days': 1} + + @staticmethod + def set_gps_times(namespace, startdate, enddate): + setattr(namespace, 'gpsstart', to_gps(startdate)) + setattr(namespace, 'gpsend', to_gps(enddate)) + + def __call__(self, parser, namespace, values, option_string=None): + try: + date = datetime.datetime.strptime(values, self.DATEFORMAT) + except ValueError: + raise parser.error("%s malformed: %r. Please format as %s" + % (self.dest.title(), values, self.METAVAR)) + else: + self.set_gps_times(namespace, date, + date + relativedelta(**self.TIMESCALE)) + setattr(namespace, self.dest, date) + return date + + +class DayAction(DateAction): + TIMESCALE = {'days': 1} + DATEFORMAT = '%Y%m%d' + METAVAR = 'YYYYMMDD' + + +class WeekAction(DayAction): + TIMESCALE = {'days': 7} + DATEFORMAT = '%Y%m%d' + METAVAR = 'YYYYMMDD' + + +class MonthAction(DateAction): + TIMESCALE = {'months': 1} + DATEFORMAT = '%Y%m' + METAVAR = 'YYYYMM' + + +class YearAction(DateAction): + TIMESCALE = {'years': 1} + DATEFORMAT = '%Y' + METAVAR = 'YYYY' + + +class GPSAction(argparse.Action): + def __call__(self, parser, namespace, values, option_string=False): + try: + values = float(values) + except (TypeError, ValueError): + pass + setattr(namespace, self.dest, to_gps(values)) + + +# -- parse command-line ------------------------------------------------------- + +def add_configuration_options(sharedopts_): + try: + default_ifo = get_default_ifo() + except ValueError: + default_ifo = None + + # configuration arguments + copts = sharedopts_.add_argument_group( + "Configuration options", + "Provide a number of INI-format configuration files", + ) + copts.add_argument( + '-i', + '--ifo', + type=str, + default=default_ifo, + metavar='IFO', + help="IFO prefix for interferometer to process. " + "If this option is set in the [DEFAULT] of any of " + "the INI files, giving it here is redundant.", + ) + copts.add_argument( + '-g', + '--global-config', + action='append', + type=str, + metavar='FILE', + default=[], + help="INI file for use in all workflow jobs, may be given " + "multiple times", + ) + copts.add_argument( + '-f', + '--config-file', + action='append', + type=str, + metavar='FILE', + default=[], + help="INI file for analysis, may be given multiple times", + ) + copts.add_argument( + '-t', + '--process-tab', + action='append', + type=str, + help="process only this tab, can be given multiple times", + ) + + +def add_process_options(sharedopts_): + popts = sharedopts_.add_argument_group( + "Process options", + "Configure how this summary will be processed.", + ) + popts.add_argument( + '--nds', + action='store_true', + help="use NDS as the data source", + ) + popts.add_argument( + '-j', + '--multi-process', + type=int, + default=1, + dest='multiprocess', + metavar='N', + help="use a maximum of N parallel processes at any time", + ) + popts.add_argument( + '-b', + '--bulk-read', + action='store_true', + help="read all data up-front at the start of the job, " + "rather than when it is needed for a tab", + ) + popts.add_argument( + '-S', + '--on-segdb-error', + type=str, + default='raise', + choices=['raise', 'ignore', 'warn'], + help="action upon error fetching segments from SegDB", + ) + popts.add_argument( + '-G', + '--on-datafind-error', + type=str, + default='raise', + choices=['raise', 'ignore', 'warn'], + help="action upon error querying for frames from the " + "datafind server, default: %(default)s", + ) + popts.add_argument( + '--data-cache', + action='append', + default=[], + help='path to LAL-format cache of TimeSeries data files', + ) + popts.add_argument( + '--event-cache', + action='append', + default=[], + help='path to LAL-format cache of event trigger files', + ) + popts.add_argument( + '--segment-cache', + action='append', + default=[], + help='path to LAL-format cache of state ' + 'or data-quality segment files', + ) + + +def add_output_options(parser_): + """Add output options to the subparser + + This is only needed because argparse can't handle mutually exclusive + groups in a parent parser handed to a subparser for some reason. + """ + outopts = parser_.add_argument_group("Output options") + outopts.add_argument( + '-o', + '--output-dir', + type=str, + metavar='DIR', + default=os.curdir, + help="Output directory for summary information", + ) + htmlopts = outopts.add_mutually_exclusive_group() + htmlopts.add_argument( + '-M', + '--html-only', + action='store_true', + help="Generate container HTML and navigation only", + ) + htmlopts.add_argument( + '-n', + '--no-html', + action='store_true', + help="Generate inner HTML and contents only, not supporting HTML", + ) + outopts.add_argument( + '-N', + '--no-htaccess', + action='store_true', + help="don't create a .htaccess file to customise 404 errors", + ) + + +def add_archive_options(parser_, archive_dir_str): + """Add archiving options to the subparser + + This is only needed because argparse can't handle mutually exclusive + groups in a parent parser handed to a subparser for some reason. + """ + hierarchopts = parser_.add_argument_group('Archive options') + hierarchchoice = hierarchopts.add_mutually_exclusive_group() + hierarchchoice.add_argument( + '-a', + '--archive', + metavar='FILE_TAG', + default=False, + const='GW_SUMMARY_ARCHIVE', + nargs='?', + help="Read archived data from, and write processed data to " + "an HDF archive file written with the FILE_TAG. If not " + "given, no archive will be used, if given with no file " + "tag, a default of '%(const)s' will be used.", + ) + hierarchchoice.add_argument( + '-d', + '--daily-archive', + metavar='FILE_TAG', + default=False, + const='GW_SUMMARY_ARCHIVE', + nargs='?', + help="Read data from the daily archives, with the given FILE_TAG." + "If not given, daily archives will be used, if given with no " + "file tag, a default of '%(const)s' will be used.", + ) + hierarchopts.add_argument( + '--archive-read-dir', + metavar='DIR', + type=str, + default=archive_dir_str, + help="Read archived data from this directory", + ) + hierarchopts.add_argument( + '--archive-write-dir', + metavar='DIR', + type=str, + default=archive_dir_str, + help="Write archived data to this directory", + ) + + +def add_condor_options(sharedopts_): + """Add condor options to the subparser""" + htcopts = sharedopts_.add_argument_group('Condor options') + htcopts.add_argument( + '--container-path', + type=str, + required=True, + help="Path to the container image file (.sif)" + ) + htcopts.add_argument( + '-T', + '--file-tag', + type=str, + default='gw_summary_pipe', + help="file tag for pipeline files, default: %(default)s", + ) + htcopts.add_argument( + '-u', + '--universe', + type=str, + default='container', + choices=['container', 'local'], + help="Universe for condor jobs, default: %(default)s", + ) + htcopts.add_argument( + '-l', + '--log-dir', + type=str, + default=os.environ.get('LOCALDIR', None), + help="Directory path for condor log files, default: %(default)s", + ) + htcopts.add_argument( + '-m', + '--maxjobs', + type=int, + metavar='N', + help="Restrict the DAG to submit only N jobs at any one " + "time, default: %(default)s", + ) + htcopts.add_argument( + '-O', + '--condor-timeout', + type=float, + metavar='T', + help='Configure condor to terminate jobs after T hours ' + 'to prevent idling, default: %(default)s', + ) + htcopts.add_argument( + '-c', + '--condor-command', + action='append', + type=str, + default=[], + help="Extra condor submit commands to add to gw_summary submit file. " + "Can be given multiple times in the form \"key=value\"", + ) + htcopts.add_argument( + '--condor-command-container', + action='append', + type=str, + default=[], + help="Extra condor submit commands to add to container universe " + "gw_summary submit file. Can be given multiple times in the " + "form \"key=value\"", + ) + htcopts.add_argument( + '-p', + '--priority', + action='append', + type=str, + default=[], + help="priority for DAG node, should be given once " + "for each --config-file in the same order", + ) + + +def create_parser(include_condor_opts=False): + """Create a command-line parser for this entry point + """ + # initialize top-level argument parser + parser = GWArgumentParser( + formatter_class=GWHelpFormatter, + prog=PROG, + description=__doc__, + fromfile_prefix_chars="@", + epilog="Arguments and options may be written into files and passed as " + "positional arguments prepended with '@', e.g. '%(prog)s " + "@args.txt'. In this format, options must be give as " + "'--argument=value', and not '--argument value'.", + ) + + # global arguments + parser.add_argument( + '-V', + '--version', + action='version', + version=__version__, + help="show program's version number and exit", + ) + + # shared arguments + sharedopts = GWArgumentParser(add_help=False) + sharedopts.title = 'Progress arguments' + sharedopts.add_argument( + '-v', + '--verbose', + action='store_true', + default=False, + help="show verbose logging output", + ) + sharedopts.add_argument( + '-D', + '--debug', + action='store_true', + default=False, + help="show information that could be useful in debugging", + ) + + # configuration options + add_configuration_options(sharedopts) + + # process options + add_process_options(sharedopts) + + # condor options, if needed + if include_condor_opts: + add_condor_options(sharedopts) + + # define sub-parser handler + subparsers = parser.add_subparsers( + dest='mode', + title='Modes', + description='Note: all dates are defined with UTC boundaries.\n' + 'The valid modes are:', + ) + subparser = dict() + + # DAY mode + today = datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%d') + daydoc = """ + Run %s over a full UTC day, and link this day to others with a calendar + built into the HTML navigation bar. In this mode you can also archive data + in HDF-format files to allow progressive processing of live data without + restarting from scratch every time.""" % parser.prog + subparser['day'] = subparsers.add_parser( + 'day', + description=daydoc, + epilog=parser.epilog, + parents=[sharedopts], + formatter_class=GWHelpFormatter, + help="Process one day of data", + ) + subparser['day'].add_argument( + 'day', + action=DayAction, + type=str, + nargs='?', + metavar=DayAction.METAVAR, + default=today, + help="Day to process", + ) + add_output_options(subparser['day']) + + darchopts = subparser['day'].add_argument_group( + 'Archive options', + 'Choose if, and how, to archive data from this run', + ) + darchopts.add_argument( + '-a', + '--archive', + metavar='FILE_TAG', + default=False, + const='GW_SUMMARY_ARCHIVE', + nargs='?', + help="Read archived data from, and write processed " + "data to, an HDF archive file written with the " + "FILE_TAG. If not given, no archive will be used, " + "if given with no file tag, a default of " + "'%(const)s' will be used.", + ) + darchopts.add_argument( + '--archive-read-dir', + metavar='DIR', + type=str, + default=f'day/{today}/archive', + help="Read archived data from this directory", + ) + darchopts.add_argument( + '--archive-write-dir', + metavar='DIR', + type=str, + default=f'day/{today}/archive', + help="Write archived data into this directory", + ) + + # WEEK mode + subparser['week'] = subparsers.add_parser( + 'week', + parents=[sharedopts], + epilog=parser.epilog, + formatter_class=GWHelpFormatter, + help="Process one week of data", + ) + subparser['week'].add_argument( + 'week', + action=WeekAction, + type=str, + metavar=WeekAction.METAVAR, + help="Week to process (given as starting day)", + ) + add_output_options(subparser['week']) + add_archive_options(subparser['week'], f'week/{today}/archive') + + # MONTH mode + today = datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m') + subparser['month'] = subparsers.add_parser( + 'month', + parents=[sharedopts], + epilog=parser.epilog, + formatter_class=GWHelpFormatter, + help="Process one month of data", + ) + subparser['month'].add_argument( + 'month', + action=MonthAction, + type=str, + metavar=MonthAction.METAVAR, + help="Month to process", + ) + add_output_options(subparser['month']) + add_archive_options(subparser['month'], f'month/{today}/archive') + + # GPS mode + subparser['gps'] = subparsers.add_parser( + 'gps', + parents=[sharedopts], + epilog=parser.epilog, + formatter_class=GWHelpFormatter, + help="Process GPS interval", + ) + subparser['gps'].add_argument( + 'gpsstart', + action=GPSAction, + type=str, + metavar='GPSSTART', + help='GPS start time', + ) + subparser['gps'].add_argument( + 'gpsend', + action=GPSAction, + type=str, + metavar='GPSEND', + help='GPS end time.', + ) + add_output_options(subparser['gps']) + + garchopts = subparser['gps'].add_argument_group( + 'Archive options', + 'Choose if, and how, to archive data from this run', + ) + garchopts.add_argument( + '-a', + '--archive', + metavar='FILE_TAG', + default=False, + const='GW_SUMMARY_ARCHIVE', + nargs='?', + help="Read archived data from, and write processed " + "data to, an HDF archive file written with the " + "FILE_TAG. If not given, no archive will be used, " + "if given with no file tag, a default of " + "'%(const)s' will be used.") + garchopts.add_argument( + '--archive-read-dir', + metavar='DIR', + type=str, + default=None, + help="Read archived data from this directory, default:" + "gps/-/archive", + ) + garchopts.add_argument( + '--archive-write-dir', + metavar='DIR', + type=str, + default=None, + help="Write archived data into this directory, default:" + "gps/-/archive", + ) + + # return the argument parser + return parser diff --git a/pyproject.toml b/pyproject.toml index cd294137..c4edaead 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ dependencies = [ "lxml", "markdown", "MarkupPy", - "matplotlib >=3.5", + "matplotlib >=3.6", "numpy >=1.16", "pygments >=2.7.0", "python-dateutil",