From 8cc2b843d60ebf8e8fed945c5784151700901f0a Mon Sep 17 00:00:00 2001
From: Joseph Areeda <joseph.areeda@ligo.org>
Date: Fri, 16 Jun 2023 15:00:29 -0700
Subject: [PATCH 01/40] fix a pytest

---
 hveto/tests/test_html.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/hveto/tests/test_html.py b/hveto/tests/test_html.py
index e0605c7..367bccb 100644
--- a/hveto/tests/test_html.py
+++ b/hveto/tests/test_html.py
@@ -28,10 +28,10 @@
 
 from .. import html
 
-BANNER = '''<div class="page-header" role="banner">
+BANNER = """<div class="page-header" role="banner">
 <h1 class="pb-2 mt-3 mb-2 border-bottom">L1 HierarchicalVeto</h1>
-<h3 class="mt-3">0 - 100 1980-01-06 00:00:00 - 1980-01-06 00:01:40</h3>
-</div>'''
+<h3 class="mt-3">0 - 100 1980-01-06 00:00:00 - 1980-01-06 00:01:40 -- 0:01:40</h3>
+</div>"""
 
 NAVBAR = """<nav class="navbar fixed-top navbar-expand-md navbar-h1 shadow-sm">
 <div class="container-fluid">

From b42cb82ce3b0a943353c5811234f70e5af13401e Mon Sep 17 00:00:00 2001
From: Joseph Areeda <joseph.areeda@ligo.org>
Date: Fri, 7 Jul 2023 20:57:20 -0700
Subject: [PATCH 02/40] add toml

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 101dc1c..911c4c5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -57,7 +57,7 @@ dynamic = [
   "version",
 ]
 
-[project.optional-dependencies]
+[project.optional-depend  encies]
 # test suite
 test = [
   "coverage[toml]",

From 4ef43d7baefc775de0e6e3ceba9ad60589864e75 Mon Sep 17 00:00:00 2001
From: Joseph Areeda <joseph.areeda@ligo.org>
Date: Sat, 8 Jul 2023 14:16:16 -0700
Subject: [PATCH 03/40] fix unintended mod to toml file. (I blame my cat)

---
 pyproject.toml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index cd44734..101dc1c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -57,7 +57,7 @@ dynamic = [
   "version",
 ]
 
-[project.optional-depend  encies]
+[project.optional-dependencies]
 # test suite
 test = [
   "coverage[toml]",
@@ -67,7 +67,7 @@ test = [
 # sphinx documentation
 docs = [
   "numpydoc",
-  "Sphinx >=1.8",
+  "Sphinx",
   "sphinx_bootstrap_theme",
   "sphinxcontrib-programoutput",
 ]

From ff664505c1817085c330fff9043489cc5fe64852 Mon Sep 17 00:00:00 2001
From: Joseph Areeda <joseph.areeda@ligo.org>
Date: Fri, 7 Jul 2023 20:57:20 -0700
Subject: [PATCH 04/40] add toml

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 66d47f3..cd44734 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -57,7 +57,7 @@ dynamic = [
   "version",
 ]
 
-[project.optional-dependencies]
+[project.optional-depend  encies]
 # test suite
 test = [
   "coverage[toml]",

From b80a95b918b3e3dc3f955b90dfff6148145d260e Mon Sep 17 00:00:00 2001
From: Joseph Areeda <joseph.areeda@ligo.org>
Date: Sat, 8 Jul 2023 14:16:16 -0700
Subject: [PATCH 05/40] fix unintended mod to toml file. (I blame my cat)

---
 pyproject.toml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index cd44734..101dc1c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -57,7 +57,7 @@ dynamic = [
   "version",
 ]
 
-[project.optional-depend  encies]
+[project.optional-dependencies]
 # test suite
 test = [
   "coverage[toml]",
@@ -67,7 +67,7 @@ test = [
 # sphinx documentation
 docs = [
   "numpydoc",
-  "Sphinx >=1.8",
+  "Sphinx",
   "sphinx_bootstrap_theme",
   "sphinxcontrib-programoutput",
 ]

From 7ed22c9ee48646c70c45950b26ed8ad5b0638d23 Mon Sep 17 00:00:00 2001
From: Evan Goetz <evan.goetz@ligo.org>
Date: Tue, 11 Mar 2025 08:24:58 -0700
Subject: [PATCH 06/40] Update github actions

---
 .github/workflows/build.yml | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 090e0a9..8443a4b 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -47,12 +47,12 @@ jobs:
 
     steps:
     - name: Get source code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
 
     - name: Cache conda packages
-      uses: actions/cache@v2
+      uses: actions/cache@v4
       env:
         # increment to reset cache
         CACHE_NUMBER: 0
@@ -62,12 +62,11 @@ jobs:
         restore-keys: ${{ runner.os }}-conda-${{ matrix.python-version }}-
 
     - name: Configure conda
-      uses: conda-incubator/setup-miniconda@v2
+      uses: conda-incubator/setup-miniconda@v3
       with:
-        activate-environment: test
-        miniforge-variant: Mambaforge
+        auto-update-conda: true
+        miniforge-version: latest
         python-version: ${{ matrix.python-version }}
-        use-mamba: true
 
     - name: Conda info
       run: conda info --all
@@ -107,14 +106,14 @@ jobs:
       run: python -m coverage xml
 
     - name: Publish coverage to Codecov
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v5
       with:
         files: coverage.xml
         flags: Conda,${{ runner.os }},python${{ matrix.python-version }}
 
     - name: Upload test results
       if: always()
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
         name: pytest-conda-${{ matrix.os }}-${{ matrix.python-version }}
         path: pytest.xml

From 06c66c42a3eb9b9e50e0117273e912bd5ce661d7 Mon Sep 17 00:00:00 2001
From: Evan Goetz <evan.goetz@ligo.org>
Date: Tue, 11 Mar 2025 09:31:20 -0700
Subject: [PATCH 07/40] Update html tests

---
 hveto/tests/test_html.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/hveto/tests/test_html.py b/hveto/tests/test_html.py
index 367bccb..6837374 100644
--- a/hveto/tests/test_html.py
+++ b/hveto/tests/test_html.py
@@ -36,7 +36,7 @@
 NAVBAR = """<nav class="navbar fixed-top navbar-expand-md navbar-h1 shadow-sm">
 <div class="container-fluid">
 <div class="navbar-brand border border-white rounded">H1 Hveto</div>
-<button class="navbar-toggler navbar-toggler-right" type="button" data-toggle="collapse" data-target=".navbar-collapse">
+<button class="navbar-toggler navbar-toggler-right" type="button" data-bs-toggle="collapse" data-target=".navbar-collapse">
 <span class="navbar-toggler-icon"></span>
 </button>
 <div class="collapse navbar-collapse justify-content-between">
@@ -45,7 +45,7 @@
 <a href="#" class="nav-link">Summary</a>
 </li>
 <li class="nav-item dropdown">
-<a href="#" class="nav-link dropdown-toggle" role="button" data-toggle="dropdown">Rounds</a>
+<a href="#" class="nav-link dropdown-toggle" role="button" data-bs-toggle="dropdown">Rounds</a>
 <div class="dropdown-menu dropdown-1-col shadow">
 <a href="#hveto-round-1" class="dropdown-item">1: H1:TEST-STRAIN</a>
 </div>
@@ -53,8 +53,8 @@
 </ul>
 <ul class="nav navbar-nav">
 <li class="nav-item dropdown">
-<a class="nav-link dropdown-toggle" href="#" role="button" data-toggle="dropdown">Links</a>
-<div class="dropdown-menu dropdown-menu-right shadow">
+<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown">Links</a>
+<div class="dropdown-menu dropdown-menu-end shadow">
 <h6 class="dropdown-header">Internal</h6>
 <a href="about" class="dropdown-item">About this page</a>
 <div class="dropdown-divider"></div>

From b3d06fdeb75aacee5ae331f9761cff2f05bec18d Mon Sep 17 00:00:00 2001
From: Evan Goetz <evan.goetz@ligo.org>
Date: Tue, 11 Mar 2025 08:18:33 -0700
Subject: [PATCH 08/40] Add dependabot configuration file for github actions

Based on https://github.com/gwdetchar/gwdetchar/pull/466
---
 .github/dependabot.yml | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 .github/dependabot.yml

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 0000000..fe36058
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,13 @@
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"
+    commit-message:
+      prefix: "actions"
+    labels:
+      - "build-and-test"

From 43a1d6f25417eb4780626fb4880283ad63eb1cbd Mon Sep 17 00:00:00 2001
From: Derek Davis <derek.davis@ligo.org>
Date: Mon, 28 Apr 2025 16:09:31 -0700
Subject: [PATCH 09/40] Add support for SNAX triggers (#170)

* Add support for SNAX triggers

* Pin gwtrigfind requirement for snax

---------

Co-authored-by: Derek Davis <derek.davis@ldas-pcdev1.ligo-wa.caltech.edu>
---
 hveto/triggers.py | 9 +++++++++
 pyproject.toml    | 2 +-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/hveto/triggers.py b/hveto/triggers.py
index aad5759..7d5b8ce 100644
--- a/hveto/triggers.py
+++ b/hveto/triggers.py
@@ -48,6 +48,7 @@
     "pycbc_live": "hdf5.pycbc_live",
     "kleinewelle": "ligolw",
     "dmt_omega": "ligolw",
+    "snax": "hdf5.snax",
 }
 
 DEFAULT_TRIGFIND_OPTIONS = {
@@ -66,6 +67,9 @@
     },
     ("omicron", "root.omicron"): {
         "ext": "root",
+    },
+    ("snax", "hdf5"): {
+        "ext": "h5",
     }
 }
 
@@ -89,6 +93,9 @@
         "loudest": True,
         "columns": ["end_time", "template_duration", "new_snr"],
     },
+    ("snax", "hdf5.snax"): {
+        "columns": ["time", "frequency", "snr"],
+    },
 }
 
 
@@ -248,6 +255,8 @@ def _format_params(channel, etg, fmt, trigfind_kwargs, read_kwargs):
     # custom params for ETGs
     if etg == "pycbc_live":
         read_kwargs.setdefault("ifo", channel.split(":", 1)[0])
+    if etg == "snax":
+        read_kwargs.setdefault("channels", channel)
 
     return trigfind_kwargs, read_kwargs
 
diff --git a/pyproject.toml b/pyproject.toml
index 101dc1c..a8b70d0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -42,7 +42,7 @@ requires-python = ">=3.9"
 dependencies = [
   "gwdetchar >= 2.0.0",
   "gwpy >=2.0.0",
-  "gwtrigfind",
+  "gwtrigfind >= 0.8.3",
   "h5py",
   "lxml",
   "MarkupPy >=1.14",

From 0945feba07f83a9197eef0baed3b963ec0ab43bf Mon Sep 17 00:00:00 2001
From: "joseph.areeda" <joseph.areeda@ligo.org>
Date: Fri, 8 Aug 2025 10:44:29 -0700
Subject: [PATCH 10/40] weekly_hveto.py ready for testing

---
 hveto/__main__.py         |   4 +-
 hveto/cli/weekly_hveto.py | 364 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 367 insertions(+), 1 deletion(-)
 create mode 100644 hveto/cli/weekly_hveto.py

diff --git a/hveto/__main__.py b/hveto/__main__.py
index b7dca6f..57a1a02 100644
--- a/hveto/__main__.py
+++ b/hveto/__main__.py
@@ -267,7 +267,8 @@ def make_drop_table(oldsignificances, newsignificances, out_file=None, cutoff=1.
                 post.append(float('nan'))
 
     drop_table = EventTable([channels, pre, post], names=['channels', 'pre_significance', 'post_significance'],
-                            dtype=[f'<U{chan_max_chars}', np.float64, np.float64]   )
+
+                            dtype=[f'<U{chan_max_chars}', np.float64, np.float64])
     drop_table.sort('pre_significance', reverse=True)
     col_formats = {'channels': f'{chan_max_chars}s', 'pre_significance': '{:8.2f}', 'post_significance': '{:8.2f}'}
     drop_table.write(out_file, format='ascii.fixed_width', overwrite=True, formats=col_formats)
@@ -647,6 +648,7 @@ def main(args=None):
                 signidir,
                 '%s-HVETO_SIGNIFICANT_CHANNELS_ROUND_%d-%d-%d.txt' % (ifo, rnd.n - 1, start, duration))
             sig_drop_table = make_drop_table(oldsignificances, newsignificances, sigfile)
+            LOGGER.debug(f"Significance drop table written to {sigfile} with {len(sig_drop_table)} rows")
             rounds[-1].files['SIG_TBL'] = sigfile
             LOGGER.info(f"Significance events written to {Path(sigfile).absolute()}")
             svg = (pngname % 'SIG_DROP').replace('.png', '.svg')  # noqa: F821
diff --git a/hveto/cli/weekly_hveto.py b/hveto/cli/weekly_hveto.py
new file mode 100644
index 0000000..0f524bc
--- /dev/null
+++ b/hveto/cli/weekly_hveto.py
@@ -0,0 +1,364 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# vim: nu:ai:ts=4:sw=4
+
+#
+#  Copyright (C) 2025 Joseph Areeda <joseph.areeda@ligo.org>
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+""""""
+import os
+import textwrap
+import time
+from datetime import datetime, timedelta, timezone
+import socket
+
+start_time = time.time()
+
+import argparse
+import logging
+from pathlib import Path
+import re
+import sys
+import traceback
+from gwpy.time import to_gps, from_gps
+
+try:
+    from ._version import __version__
+except ImportError:
+    __version__ = '0.0.0'
+
+__author__ = 'joseph areeda'
+__email__ = 'joseph.areeda@ligo.org'
+__process_name__ = Path(__file__).name
+
+logger = None
+
+hveto_run_sh = textwrap.dedent('''\
+    #!/bin/bash
+    # script used by %program_name% condor DAG to run one analysis
+    # Created by %program_name% version %program_version% on %date%
+
+    configuration="%configuration%"
+    nproc=%nproc%
+    ifo=%ifo%
+    gpsstart="%gpsstart%"
+    gpsend="%gpsend%"
+    outer_dir="%outer_dir%"
+    prefix="%prefix%"
+
+    condaRun="/cvmfs/software.igwn.org/conda/condabin/conda run --prefix $%prefix% --no-capture-output --no-capture-error "
+
+
+    cmd="python -m hveto ${gpsstart} ${gpsend} --ifo ${ifo} --config-file ${configuration} --nproc ${nproc} \
+        --output-directory ${outer_dir}"
+
+    ${condaRun} ${cmd}
+
+    ''')
+
+hveto_job_submit_sh = textwrap.dedent('''
+#!/usr/bin/env condor_submit
+#
+# Condor submit file for %program_name% processing
+# Created by %program_name% version %program_version% on %date%
+#
+
+universe = vanilla
+executable = %exec%
+
+accounting_group = ligo.dev.o4.detchar.dqtriggers.hveto
+accounting_group_user = joseph.areeda
+
+request_memory = %request_memory%M
+request_disk = 10GB
+
+output = %condor_dir%/hveto.out
+error = %condor_dir%/hveto.err
+log = %condor_dir%/hveto.log
+
+batch_name = "hveto %end_day% %out_of% ID: $(ClusterId)"
+
+use_oauth_services = scitokens
+environment = BEARER_TOKEN_FILE=$$(CondorScratchDir)/.condor_creds/scitokens.use
+
+
+queue
+''')
+
+
+def to_day(in_str):
+    in_gps = to_gps(in_str)
+    in_date = from_gps(in_gps)
+    ret = in_date.strftime('%Y-%m-%d')
+    return ret
+
+
+def get_default_ifo():
+    # if at a site we have a default ifo
+    host = socket.getfqdn()
+    if 'ligo-la' in host:
+        ifo = 'L1'
+    elif 'ligo-wa' in host:
+        ifo = 'H1'
+    else:
+        ifo = os.getenv('IFO')
+    if ifo is None:
+        ifo = 'UK'
+    return ifo, host
+
+
+def parser_add_args(parser):
+    """
+    Set up command parser
+    :param argparse.ArgumentParser parser:
+    :return: None but parser object is updated
+    """
+    parser.add_argument('-v', '--verbose', action='count', default=1,
+                        help='increase verbose output')
+    parser.add_argument('-V', '--version', action='version',
+                        version=__version__)
+    parser.add_argument('-q', '--quiet', default=False, action='store_true',
+                        help='show only fatal errors')
+
+    this_script = Path(__file__)
+    user = os.getenv('USER')
+    config_possibles = [
+        this_script.parent.parent / 'configurations' / 'hveto' / 'h1l1-hveto-daily-o4b.ini',
+        Path(f'/home/{user}/etc/ligo-monitors/configurations/hveto/h1l1-hveto-daily-o4b.ini'),
+        Path('/home/detchar/etc/ligo-monitors/configurations/hveto/h1l1-hveto-daily-o4b.ini'),
+    ]
+    default_config_file = None
+    for config in config_possibles:
+        if config.exists():
+            default_config_file = config
+            break
+
+    parser.add_argument('-c', '--config-file', default=default_config_file, help='path to hveto configuration file')
+
+    now_utc = datetime.now(timezone.utc)
+    default_end_date = now_utc.strftime('%Y-%m-%d')
+
+    parser.add_argument('-e', '--end', type=to_day, default=default_end_date,
+                        help='Processng looks back duration days up to but not including this date. (default: %(default)s).')
+    parser.add_argument('-s', '--start', type=to_day, default=default_end_date,
+                        help='We process one day at a time from end daste back to star date each '
+                             'run is duration days long (default: %(default)s).')
+    parser.add_argument('-d', '--duration', type=int, default=7,
+                        help='Number of days analyze for each day (default: %(default)s).')
+    parser.add_argument('--stride', type=int, default=1,
+                        help='Number of days between each hveto run. '
+                             'For example start=7/1, end=7/29, stride=7, duration=7'
+                             'will create 4 hveto reports ending on 7/28, 7/21, 7/14, 7/1 each covering 7 days')
+    parser.add_argument('--no-submit', action='store_true',
+                        help='Create directory, script and submit file but do not submit to Condor')
+    parser.add_argument('-o', '--output-directory', default='/home/detchar/public_html/hveto-weekly',
+                        help='Parent directory for what may be multiple hveto runs')
+
+    prefix = os.getenv('CONDA_PREFIX', '/home/detchar/.conda/envs/ligo-summary-3.10')
+    parser.add_argument('-p', '--prefix', default=prefix, help='Path to conda environment to run hveto in')
+
+    ifo, host = get_default_ifo()
+    parser.add_argument('-i', '--ifo', default=ifo, help='IFO for hveto analysis')
+    parser.add_argument('-n', '--nproc', default=4, type=int, help='Number of parallel tigger readers')
+
+
+def apply_symbols(txt, symbols):
+    """
+    Search text for {<symbol>} entries use <symbol> as key to dicionary then substite its value
+    :param str txt: input string with <symbol> entries
+    :param dict symbols: symbol table
+    :return str: text with substitutions applied
+    """
+    ret = ''
+    for line in txt.splitlines():
+        oline = line
+        while re.match('.*%.+%', oline):
+            for symbol, value in symbols.items():
+                pattern = '%' + rf'{symbol}' + '%'
+                oline = re.sub(pattern, str(value), oline)
+            if oline != line:
+                logger.debug(f'  {line} -> {oline}')
+            else:
+                logger.critical(f'Undefined symbol in {line}')
+            break
+        ret += oline + '\n'
+    return ret
+
+
+def make_job(job_name, job_args):
+    """
+    Creates and prepares the directory and condor files for a single hveto job.
+
+    :param str job_name: The name of the job to create
+    :param Path job_dir: The directory where the job will be generated or executed
+    :param dict job_args: The arguments to use for the job
+    :return Path: Path to the submit file
+    """
+    job_dir = job_args['outer_dir']
+    logger.debug(f'Creating condor file needed for job {job_name} in {job_dir}')
+    job_dir.mkdir(parents=True, exist_ok=True)
+    job_condadir = job_dir / 'condor'
+    job_condadir.mkdir(parents=True, exist_ok=True)
+
+    job_submit_file = job_dir / 'condor' / f'{job_name}.submit'
+    job_run_sh = job_condadir / f'{job_name}.sh'
+    job_script = apply_symbols(hveto_run_sh, job_args)
+    job_run_sh.write_text(job_script)
+    job_run_sh.chmod(0o755)
+    job_args['exec'] = job_run_sh
+
+    job_submit = apply_symbols(hveto_job_submit_sh, job_args)
+    job_submit_file.write_text(job_submit)
+
+    return job_submit_file
+
+
+def main():
+    global logger
+
+    log_file_format = "%(asctime)s - %(levelname)s - %(funcName)s %(lineno)d: %(message)s"
+    log_file_date_format = '%m-%d %H:%M:%S'
+    logging.basicConfig(format=log_file_format, datefmt=log_file_date_format)
+    logger = logging.getLogger(__process_name__)
+    logger.setLevel(logging.DEBUG)
+
+    description = textwrap.dedent("""
+    Create a bash script and condor_submit for a multi-day run of hveto, then submit to vanilla universe
+    By default:
+    - We process 7 days of data ending at 00:00 UTC today, expecting it to run around 02:00Z
+    - Output goes to ~/public_html/hveto/weekly/<YYYYMM>/<YYYYMMDD>/
+    - Uses the configuration file "h1l1-hveto-daily-o4b.ini" found at
+      <path to this script>/../configurations/hveto/ or
+      ~/etc/ligo-monitors/configurations/hveto/ or
+      /home/detchar/etc/ligo-monitors/configurations/hveto/
+    """)
+
+    epilog = textwrap.dedent("""
+    Running with default (no) arguments  will run hveto over 7 days of data ending at 00:00 UTC this morning.
+
+    To run hveto for every day for the month of July, each run covering 7 days:
+
+        python -m weekly_hveto --start 7/1 --end 7/31 --duration 7 --stride 1
+
+    """)
+    parser = argparse.ArgumentParser(description=description, epilog=epilog, prog=__process_name__,
+                                     formatter_class=argparse.RawTextHelpFormatter)
+    parser_add_args(parser)
+    args = parser.parse_args()
+    verbosity = 0 if args.quiet else args.verbose
+
+    if verbosity < 1:
+        logger.setLevel(logging.CRITICAL)
+    elif verbosity < 2:
+        logger.setLevel(logging.INFO)
+    else:
+        logger.setLevel(logging.DEBUG)
+
+    # debugging?
+    logger.debug(f'{__process_name__} version: {__version__} called with arguments:')
+    for k, v in args.__dict__.items():
+        logger.debug('    {} = {}'.format(k, v))
+
+    config = args.config_file
+    if config is None:
+        logger.critical('Configuration file not specified')
+        raise ValueError('Configuration file not specified')
+
+    start_day = args.start
+    start_gps = to_gps(start_day)
+    start_dt = from_gps(start_gps)
+
+    end_day = args.end
+    end_gps = to_gps(end_day)
+    end_dt = from_gps(end_gps)
+    next_dt = end_dt
+
+    duration = args.duration
+    duration_dt = timedelta(days=duration)
+
+    stride = args.stride
+    stride_dt = timedelta(days=stride)
+
+    output_directory = Path(args.output_directory)
+    now_time = datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S')
+    dag_file = output_directory / f'DAG-{now_time}' / 'hveto.dag'
+    logger.info(f'DAG file  will be written to {dag_file}')
+    dag_file.parent.mkdir(parents=True, exist_ok=True)
+
+    with dag_file.open('w') as dag_fh:
+        print(f'# process long duration hveto for {start_day} to {end_day} each {duration} days', file=dag_fh)
+        print(f'# stride = {stride} days', file=dag_fh)
+        print(f'# Created by {__process_name__}, version {__version__}\\n', file=dag_fh)
+        print('max_jobs = 3', file=dag_fh)
+
+        njobs = int((end_dt - start_dt).days / stride)
+        current_job = 1
+
+        while start_dt <= next_dt:
+            begin_dt = to_day(next_dt - duration_dt)
+            begin_day = to_day(begin_dt)
+            next_day = to_day(next_dt)
+            job_name = f'hveto_{current_job:02d}_of_{njobs:02d}'
+            logger.debug(f'Process hveto {begin_day} to {next_day}')
+            job_month = next_dt.strftime('%Y%m')
+            job_day = next_dt.strftime('%Y%m%d')
+
+            #  create the results directory and add the condor submit file and bash script
+            job_dir = output_directory / job_month / job_day
+
+            # rutiime args are sed to create the obs bash script
+            job_args = {
+                "configuration": config,
+                "nproc": args.nproc,
+                "ifo": args.ifo,
+                "gpsstart": begin_day,
+                "gpsend": next_day,
+                "outer_dir": job_dir,
+                "prefix": args.prefix,
+                "program_name": __process_name__,
+                "program_version": __version__,
+                "date": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
+                "condor_dir": job_dir / 'condor',
+                "request_memory": 32768,
+                "end_day": next_day,
+                "out_of": f'{current_job:02d}_of_{njobs:02d}',
+
+            }
+            job_submit_file = make_job(job_name, job_args)
+
+            # add this job to the DAG
+            print(f'JOB {job_name} {job_submit_file}', file=dag_fh)
+            current_job += 1
+
+            next_dt -= stride_dt
+
+    logger.info(f'DAG file with {njobs} jobs written to {dag_file}')
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except (ValueError, TypeError, OSError, NameError, ArithmeticError, RuntimeError) as ex:
+        print(ex, file=sys.stderr)
+        traceback.print_exc(file=sys.stderr)
+
+    if logger is None:
+        logging.basicConfig()
+        logger = logging.getLogger(__process_name__)
+        logger.setLevel(logging.DEBUG)
+    # report our run time
+    logger.info(f'Elapsed time: {time.time() - start_time:.1f}s')

From c3346918bebf58dae628e8365aed3e89f04512a1 Mon Sep 17 00:00:00 2001
From: "joseph.areeda" <joseph.areeda@ligo.org>
Date: Fri, 8 Aug 2025 11:16:05 -0700
Subject: [PATCH 11/40] weekly_hveto.py ready for testing, fix pyproject.toml

---
 pyproject.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pyproject.toml b/pyproject.toml
index a8b70d0..58ce394 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -76,6 +76,7 @@ docs = [
 hveto = "hveto.__main__:main"
 hveto-cache-events = "hveto.cli.cache_events:main"
 hveto-trace = "hveto.cli.trace:main"
+hveto-weekly = "hveto.cli.weekly_hveto:main"
 
 [project.urls]
 "Bug Tracker" = "https://github.com/gwdetchar/hveto/issues"

From 18f0503fdaf45c2f1fab3eb9beebbaf40cc6f08c Mon Sep 17 00:00:00 2001
From: "joseph.areeda" <joseph.areeda@ligo.org>
Date: Sat, 9 Aug 2025 07:34:06 -0700
Subject: [PATCH 12/40] weekly_hveto fix dag max jobs,add copy of log messages
 to file

---
 hveto/cli/weekly_hveto.py | 28 +++++++++++++++++++++++-----
 1 file changed, 23 insertions(+), 5 deletions(-)

diff --git a/hveto/cli/weekly_hveto.py b/hveto/cli/weekly_hveto.py
index 0f524bc..85110a9 100644
--- a/hveto/cli/weekly_hveto.py
+++ b/hveto/cli/weekly_hveto.py
@@ -26,6 +26,8 @@
 from datetime import datetime, timedelta, timezone
 import socket
 
+from pytz import reference
+
 start_time = time.time()
 
 import argparse
@@ -165,7 +167,7 @@ def parser_add_args(parser):
                              'will create 4 hveto reports ending on 7/28, 7/21, 7/14, 7/1 each covering 7 days')
     parser.add_argument('--no-submit', action='store_true',
                         help='Create directory, script and submit file but do not submit to Condor')
-    parser.add_argument('-o', '--output-directory', default='/home/detchar/public_html/hveto-weekly',
+    parser.add_argument('-o', '--output-directory', default=f'/home/{os.getenv("USER")}/public_html/hveto-weekly',
                         help='Parent directory for what may be multiple hveto runs')
 
     prefix = os.getenv('CONDA_PREFIX', '/home/detchar/.conda/envs/ligo-summary-3.10')
@@ -268,6 +270,24 @@ def main():
     else:
         logger.setLevel(logging.DEBUG)
 
+    output_directory = Path(args.output_directory)
+    now_time = datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S')
+    dag_dir = output_directory / f'DAG-{now_time}'
+    dag_dir.mkdir(parents=True, exist_ok=True)
+
+    dag_log = dag_dir / 'weekly-dag.log'
+    dag_file = dag_dir / 'hveto.dag'
+
+    # add logging too a file
+    NOW = datetime.datetime.now()
+    TIMEZONE = reference.LocalTimezone().tzname(NOW)
+    DATEFMT = '%Y-%m-%d %H:%M:%S {}'.format(TIMEZONE)
+    FMT = '%(name)s %(asctime)s %(levelname)+8s: %(filename)s:%(lineno)d:  %(message)s'
+    logf_formatter = logging.Formatter(FMT, datefmt=DATEFMT)
+    logf_handler = logging.FileHandler(dag_log, mode='w')
+    logf_handler.setFormatter(logf_formatter)
+    logger.addHandler(logf_handler)
+
     # debugging?
     logger.debug(f'{__process_name__} version: {__version__} called with arguments:')
     for k, v in args.__dict__.items():
@@ -293,9 +313,6 @@ def main():
     stride = args.stride
     stride_dt = timedelta(days=stride)
 
-    output_directory = Path(args.output_directory)
-    now_time = datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S')
-    dag_file = output_directory / f'DAG-{now_time}' / 'hveto.dag'
     logger.info(f'DAG file  will be written to {dag_file}')
     dag_file.parent.mkdir(parents=True, exist_ok=True)
 
@@ -303,7 +320,8 @@ def main():
         print(f'# process long duration hveto for {start_day} to {end_day} each {duration} days', file=dag_fh)
         print(f'# stride = {stride} days', file=dag_fh)
         print(f'# Created by {__process_name__}, version {__version__}\\n', file=dag_fh)
-        print('max_jobs = 3', file=dag_fh)
+        print('CATEGORY ALL_NODES LIMIT', file=dag_fh)
+        print('', file=dag_fh)
 
         njobs = int((end_dt - start_dt).days / stride)
         current_job = 1

From fd20b22078154d42784680fcc2db93db10d1e10f Mon Sep 17 00:00:00 2001
From: "joseph.areeda" <joseph.areeda@ligo.org>
Date: Sat, 9 Aug 2025 11:58:48 -0700
Subject: [PATCH 13/40] weekly_hveto fix dag max jobs,add copy of log messages
 to file, add more error checking

---
 hveto/cli/weekly_hveto.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/hveto/cli/weekly_hveto.py b/hveto/cli/weekly_hveto.py
index 85110a9..1d54236 100644
--- a/hveto/cli/weekly_hveto.py
+++ b/hveto/cli/weekly_hveto.py
@@ -62,7 +62,7 @@
     outer_dir="%outer_dir%"
     prefix="%prefix%"
 
-    condaRun="/cvmfs/software.igwn.org/conda/condabin/conda run --prefix $%prefix% --no-capture-output --no-capture-error "
+    condaRun="/cvmfs/software.igwn.org/conda/condabin/conda run --prefix ${prefix} --no-capture-output  "
 
 
     cmd="python -m hveto ${gpsstart} ${gpsend} --ifo ${ifo} --config-file ${configuration} --nproc ${nproc} \
@@ -279,7 +279,7 @@ def main():
     dag_file = dag_dir / 'hveto.dag'
 
     # add logging too a file
-    NOW = datetime.datetime.now()
+    NOW = datetime.now()
     TIMEZONE = reference.LocalTimezone().tzname(NOW)
     DATEFMT = '%Y-%m-%d %H:%M:%S {}'.format(TIMEZONE)
     FMT = '%(name)s %(asctime)s %(levelname)+8s: %(filename)s:%(lineno)d:  %(message)s'
@@ -298,6 +298,13 @@ def main():
         logger.critical('Configuration file not specified')
         raise ValueError('Configuration file not specified')
 
+    if args.ifo == 'UK':
+        logger.critical('IFO must be specified on command line or in the environment, if not running at LLO or LHO')
+        raise ValueError('IFO must be specified')
+    elif args.ifo not in ['L1', 'H1']:
+        logger.critical(f'IFO {args.ifo} not in ["L1", "H1"]')
+        raise ValueError(f'IFO {args.ifo} not in ["L1", "H1"]')
+
     start_day = args.start
     start_gps = to_gps(start_day)
     start_dt = from_gps(start_gps)
@@ -319,8 +326,9 @@ def main():
     with dag_file.open('w') as dag_fh:
         print(f'# process long duration hveto for {start_day} to {end_day} each {duration} days', file=dag_fh)
         print(f'# stride = {stride} days', file=dag_fh)
-        print(f'# Created by {__process_name__}, version {__version__}\\n', file=dag_fh)
+        print(f'# Created by {__process_name__}, version {__version__}\n', file=dag_fh)
         print('CATEGORY ALL_NODES LIMIT', file=dag_fh)
+        print('MAXJOBS LIMIT 3', file=dag_fh)
         print('', file=dag_fh)
 
         njobs = int((end_dt - start_dt).days / stride)

From 8b0cf825ad4ede3c8a9fde754cc33613c4c5d17a Mon Sep 17 00:00:00 2001
From: "joseph.areeda" <joseph.areeda@ligo.org>
Date: Sat, 9 Aug 2025 14:00:11 -0700
Subject: [PATCH 14/40] DAG retry added, fixed count of jobs

---
 hveto/cli/weekly_hveto.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/hveto/cli/weekly_hveto.py b/hveto/cli/weekly_hveto.py
index 1d54236..850363e 100644
--- a/hveto/cli/weekly_hveto.py
+++ b/hveto/cli/weekly_hveto.py
@@ -206,7 +206,6 @@ def make_job(job_name, job_args):
     Creates and prepares the directory and condor files for a single hveto job.
 
     :param str job_name: The name of the job to create
-    :param Path job_dir: The directory where the job will be generated or executed
     :param dict job_args: The arguments to use for the job
     :return Path: Path to the submit file
     """
@@ -329,9 +328,10 @@ def main():
         print(f'# Created by {__process_name__}, version {__version__}\n', file=dag_fh)
         print('CATEGORY ALL_NODES LIMIT', file=dag_fh)
         print('MAXJOBS LIMIT 3', file=dag_fh)
+        print('RETRY LIMIT 2\n', file=dag_fh)
         print('', file=dag_fh)
 
-        njobs = int((end_dt - start_dt).days / stride)
+        njobs = int((end_dt - start_dt).days / stride) + 1
         current_job = 1
 
         while start_dt <= next_dt:

From 2d6faf87ae16969d6de2135aee32acf3cbe568db Mon Sep 17 00:00:00 2001
From: "joseph.areeda" <joseph.areeda@ligo.org>
Date: Sat, 9 Aug 2025 14:36:13 -0700
Subject: [PATCH 15/40] DAG retry added, fixed count of jobs, use all_nodes

---
 hveto/cli/weekly_hveto.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hveto/cli/weekly_hveto.py b/hveto/cli/weekly_hveto.py
index 850363e..2602bb7 100644
--- a/hveto/cli/weekly_hveto.py
+++ b/hveto/cli/weekly_hveto.py
@@ -328,7 +328,7 @@ def main():
         print(f'# Created by {__process_name__}, version {__version__}\n', file=dag_fh)
         print('CATEGORY ALL_NODES LIMIT', file=dag_fh)
         print('MAXJOBS LIMIT 3', file=dag_fh)
-        print('RETRY LIMIT 2\n', file=dag_fh)
+        print('RETRY ALL_NODES 2\n', file=dag_fh)
         print('', file=dag_fh)
 
         njobs = int((end_dt - start_dt).days / stride) + 1

From 68466971cc52ff3eba05a6b55b4720d48a78e6be Mon Sep 17 00:00:00 2001
From: "joseph.areeda" <joseph.areeda@ligo.org>
Date: Tue, 12 Aug 2025 05:45:10 -0700
Subject: [PATCH 16/40] Add timeout tojob's run script, submit DAG by defult

---
 hveto/cli/weekly_hveto.py | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/hveto/cli/weekly_hveto.py b/hveto/cli/weekly_hveto.py
index 2602bb7..6a1b81e 100644
--- a/hveto/cli/weekly_hveto.py
+++ b/hveto/cli/weekly_hveto.py
@@ -21,6 +21,7 @@
 
 """"""
 import os
+import subprocess
 import textwrap
 import time
 from datetime import datetime, timedelta, timezone
@@ -54,6 +55,7 @@
     # script used by %program_name% condor DAG to run one analysis
     # Created by %program_name% version %program_version% on %date%
 
+    # Run specific variables
     configuration="%configuration%"
     nproc=%nproc%
     ifo=%ifo%
@@ -61,21 +63,26 @@
     gpsend="%gpsend%"
     outer_dir="%outer_dir%"
     prefix="%prefix%"
+    timeout=%timeout%
 
+    # run proram in conda environment
     condaRun="/cvmfs/software.igwn.org/conda/condabin/conda run --prefix ${prefix} --no-capture-output  "
 
+    # use a 3 hour timeout to prevent hanging jobs
+    timeout_cmd="timeout --kill-after=${timeout}s ${timeout}"
 
     cmd="python -m hveto ${gpsstart} ${gpsend} --ifo ${ifo} --config-file ${configuration} --nproc ${nproc} \
         --output-directory ${outer_dir}"
 
-    ${condaRun} ${cmd}
+    ${condaRun} ${timeout_cmd} ${cmd}
 
     ''')
 
 hveto_job_submit_sh = textwrap.dedent('''
 #!/usr/bin/env condor_submit
 #
-# Condor submit file for %program_name% processing
+# Condor submit file for hveto processing in batch mode
+#
 # Created by %program_name% version %program_version% on %date%
 #
 
@@ -346,7 +353,7 @@ def main():
             #  create the results directory and add the condor submit file and bash script
             job_dir = output_directory / job_month / job_day
 
-            # rutiime args are sed to create the obs bash script
+            # runtime args are used to create the job's bash script
             job_args = {
                 "configuration": config,
                 "nproc": args.nproc,
@@ -359,9 +366,10 @@ def main():
                 "program_version": __version__,
                 "date": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
                 "condor_dir": job_dir / 'condor',
-                "request_memory": 32768,
+                "request_memory": 15384,
                 "end_day": next_day,
                 "out_of": f'{current_job:02d}_of_{njobs:02d}',
+                "timeout": 3 * 3600,
 
             }
             job_submit_file = make_job(job_name, job_args)
@@ -373,6 +381,13 @@ def main():
             next_dt -= stride_dt
 
     logger.info(f'DAG file with {njobs} jobs written to {dag_file}')
+    if not args.no_submit:
+        logger.info(f'Submitting {dag_file} to Condor')
+        cmd = f'condor_submit_dag -import_env {dag_file}'
+        logger.debug(f'Running: {cmd}')
+        subprocess.run(cmd, shell=True, check=True)
+    else:
+        logger.info('DAG file written but not submitted to Condor')
 
 
 if __name__ == "__main__":

From 0c50b91255dcc3e20955680bf87647aff0d26d8d Mon Sep 17 00:00:00 2001
From: "joseph.areeda" <joseph.areeda@ligo.org>
Date: Thu, 14 Aug 2025 06:26:18 -0700
Subject: [PATCH 17/40] Add crondor templates Tweak weeky dag submission

---
 Condor/README.md          |  9 +++++++++
 Condor/hveto-daily.sub    | 36 +++++++++++++++++++++++++++++++++++
 Condor/hveto-weekly.sub   | 40 +++++++++++++++++++++++++++++++++++++++
 hveto/cli/weekly_hveto.py | 10 ++++++----
 4 files changed, 91 insertions(+), 4 deletions(-)
 create mode 100644 Condor/README.md
 create mode 100644 Condor/hveto-daily.sub
 create mode 100644 Condor/hveto-weekly.sub

diff --git a/Condor/README.md b/Condor/README.md
new file mode 100644
index 0000000..1ce9575
--- /dev/null
+++ b/Condor/README.md
@@ -0,0 +1,9 @@
+This directory contains tempplates for Condor cron jobs to run hveto 
+on a daily basis as used by the Detchar account.
+
+These templates are used to create the actual ssubmit files in 
+${HOME}/etc/ligo-monitors/condor
+
+- hveto-daily.sub - runs 3 times per day over 8, 16, and 24 hrs since 00:00 UTC
+- hveto-weekly.su - runs once per day and covers the previous 7 days from
+  00:00 to 00:00 UTc
\ No newline at end of file
diff --git a/Condor/hveto-daily.sub b/Condor/hveto-daily.sub
new file mode 100644
index 0000000..344fae2
--- /dev/null
+++ b/Condor/hveto-daily.sub
@@ -0,0 +1,36 @@
+#!/usr/bin/env condor_submit
+#
+# Condor submit file for Hveto daily processing
+#
+
+universe = local
+executable = $ENV(HOME)/etc/ligo-monitors/scripts/daily-hveto.sh
+accounting_group = ligo.prod.o4.detchar.dqtriggers.hveto
+accounting_group_user = <user name valid for account>
+
+request_memory = 20GB
+request_disk = 5GB
+
+output = $ENV(HOME)/public_html/hveto/logs/daily-hveto.out
+error = $ENV(HOME)/public_html/hveto/logs/daily-hveto.err
+log = $ENV(HOME)/public_html/hveto/logs/daily-hveto.log
+
+batch_name = "daily hveto [cron] ID: $(ClusterId)"
+
+on_exit_remove = false
+periodic_hold = false
+cron_minute = 30
+# we want run to be 8, 16 24 UTC with enough slack for omicron selected by local time at LLO (UTC -5 CDT)
+# so LLO CDT -> 10, 18, 1 UTC. CST -> 11, 19, 2 UTC
+# LHO PDT (UTC - 7) -> 12, 20, 4 UC. PST - 13, 21, 5 TC
+cron_hour = 5, 13, 21
+cron_day_of_month = *
+cron_month = *
+cron_day_of_week = *
+cron_window = 2400
+
+use_oauth_services = scitokens
+environment = BEARER_TOKEN_FILE=$$(CondorScratchDir)/.condor_creds/scitokens.use
+
+getenv = DEFAULT_SEGMENT_SERVER ECP_IDP GWDATAFIND_SERVER HDF5_USE_FILE_LOCKING HOME KRB5CCNAME NDSSERVER KRB5_KTNAME CONDA_PREFIX IFO USER
+queue
diff --git a/Condor/hveto-weekly.sub b/Condor/hveto-weekly.sub
new file mode 100644
index 0000000..16df02a
--- /dev/null
+++ b/Condor/hveto-weekly.sub
@@ -0,0 +1,40 @@
+#!/usr/bin/env condor_submit
+#
+# Condor submit file for Hveto weekly processing
+# it runs inn the local universe creates and submits a DAG to the vbanilla universe
+#
+
+universe = local
+my.conda_exe = /cvmfs/software.igwn.org/conda/condabin/conda
+my.conda_prefix = prefix="/home/detchar.dev/.conda/envs/ligo-summary-3.11-test"
+
+executable = $(my.conda_exe)
+arguments = 'run --prefix $(my.conda_prefix)) timeout 30m hveto-weekly -vv'
+accounting_group = ligo.prod.o4.detchar.dqtriggers.hveto
+accounting_group_user = <user name valid for account>
+
+request_memory = 20GB
+request_disk = 5GB
+
+output = $ENV(HOME)/public_html/hveto-weekly/logs/hveto-weekly.out
+error = $ENV(HOME)/public_html/hveto-weekly/logs/hveto-weekly.err
+log = $ENV(HOME)/public_html/hveto-weekly/logs/hveto-weekly.log
+
+batch_name = "weekly hveto [cron] ID: $(ClusterId)"
+
+on_exit_remove = false
+periodic_hold = false
+
+# We run ths in the Central and Pacific time zones standard and daylght savings
+# We want to give the daily gap filler
+cron_hour = 23
+cron_day_of_month = *
+cron_month = *
+cron_day_of_week = *
+cron_window = 2400
+
+use_oauth_services = scitokens
+environment = BEARER_TOKEN_FILE=$$(CondorScratchDir)/.condor_creds/scitokens.use
+
+getenv = DEFAULT_SEGMENT_SERVER ECP_IDP GWDATAFIND_SERVER HDF5_USE_FILE_LOCKING HOME KRB5CCNAME NDSSERVER KRB5_KTNAME CONDA_PREFIX IFO USER
+queue
diff --git a/hveto/cli/weekly_hveto.py b/hveto/cli/weekly_hveto.py
index 6a1b81e..522a3d9 100644
--- a/hveto/cli/weekly_hveto.py
+++ b/hveto/cli/weekly_hveto.py
@@ -40,7 +40,7 @@
 from gwpy.time import to_gps, from_gps
 
 try:
-    from ._version import __version__
+    from .._version import __version__
 except ImportError:
     __version__ = '0.0.0'
 
@@ -383,12 +383,16 @@ def main():
     logger.info(f'DAG file with {njobs} jobs written to {dag_file}')
     if not args.no_submit:
         logger.info(f'Submitting {dag_file} to Condor')
-        cmd = f'condor_submit_dag -import_env {dag_file}'
+        batch_name = f'hveto {start_dt.strftime("%m/%d")} to {next_dt.strftime("%m/%d")} $(ClusterId)'
+        cmd = f'condor_submit_dag -import_env {dag_file} -batch-name "{batch_name}"'
         logger.debug(f'Running: {cmd}')
         subprocess.run(cmd, shell=True, check=True)
     else:
         logger.info('DAG file written but not submitted to Condor')
 
+    # report our run time
+    logger.info(f'Elapsed time: {time.time() - start_time:.1f}s')
+
 
 if __name__ == "__main__":
     try:
@@ -401,5 +405,3 @@ def main():
         logging.basicConfig()
         logger = logging.getLogger(__process_name__)
         logger.setLevel(logging.DEBUG)
-    # report our run time
-    logger.info(f'Elapsed time: {time.time() - start_time:.1f}s')

From a3bd19771c5ce46d7db5920f73cd5756553eaab8 Mon Sep 17 00:00:00 2001
From: "joseph.areeda" <joseph.areeda@ligo.org>
Date: Thu, 14 Aug 2025 07:05:27 -0700
Subject: [PATCH 18/40] update pyproject.toml or depecation

---
 pyproject.toml | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 58ce394..d13b0c7 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [build-system]
 requires = [
-  "setuptools",
+  "setuptools>=77.0.0",
   "setuptools_scm[toml]>=3.4.3",
   "wheel",
 ]
@@ -17,6 +17,8 @@ authors = [
 maintainers = [
   { name = "Joe Areeda", email = "joseph.areeda@ligo.org" },
 ]
+license-files = [ "LICENSE" ]
+
 classifiers = [
   "Development Status :: 5 - Production/Stable",
   "Intended Audience :: Developers",
@@ -83,9 +85,6 @@ hveto-weekly = "hveto.cli.weekly_hveto:main"
 "Documentation" = "https://hveto.readthedocs.io"
 "Source Code" = "https://github.com/gwdetchar/hveto"
 
-[tool.setuptools]
-license-files = [ "LICENSE" ]
-
 [tool.setuptools_scm]
 write_to = "hveto/_version.py"
 

From 15950db04ae8bf316a3b20f1d39201ff0e4faf62 Mon Sep 17 00:00:00 2001
From: "joseph.areeda" <joseph.areeda@ligo.org>
Date: Thu, 14 Aug 2025 12:43:52 -0700
Subject: [PATCH 19/40] update pyproject.toml for new license specification
 rules

---
 pyproject.toml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index d13b0c7..47f9120 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -24,7 +24,6 @@ classifiers = [
   "Intended Audience :: Developers",
   "Intended Audience :: End Users/Desktop",
   "Intended Audience :: Science/Research",
-  "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
   "Natural Language :: English",
   "Operating System :: POSIX",
   "Operating System :: Unix",

From 6657a9d3b9f1465d7d618405d3638829d9ef6027 Mon Sep 17 00:00:00 2001
From: "joseph.areeda" <joseph.areeda@ligo.org>
Date: Thu, 14 Aug 2025 14:06:45 -0700
Subject: [PATCH 20/40] move condor examples to docs

---
 {Condor => docs/Condor}/README.md        | 0
 {Condor => docs/Condor}/hveto-daily.sub  | 0
 {Condor => docs/Condor}/hveto-weekly.sub | 0
 pyproject.toml                           | 2 +-
 4 files changed, 1 insertion(+), 1 deletion(-)
 rename {Condor => docs/Condor}/README.md (100%)
 rename {Condor => docs/Condor}/hveto-daily.sub (100%)
 rename {Condor => docs/Condor}/hveto-weekly.sub (100%)

diff --git a/Condor/README.md b/docs/Condor/README.md
similarity index 100%
rename from Condor/README.md
rename to docs/Condor/README.md
diff --git a/Condor/hveto-daily.sub b/docs/Condor/hveto-daily.sub
similarity index 100%
rename from Condor/hveto-daily.sub
rename to docs/Condor/hveto-daily.sub
diff --git a/Condor/hveto-weekly.sub b/docs/Condor/hveto-weekly.sub
similarity index 100%
rename from Condor/hveto-weekly.sub
rename to docs/Condor/hveto-weekly.sub
diff --git a/pyproject.toml b/pyproject.toml
index 47f9120..9c1d877 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -40,7 +40,7 @@ classifiers = [
 
 # requirements
 requires-python = ">=3.9"
-dependencies = [
+dependencies = [0
   "gwdetchar >= 2.0.0",
   "gwpy >=2.0.0",
   "gwtrigfind >= 0.8.3",

From a15b623e2705d577a7813633e99cc981727e5a31 Mon Sep 17 00:00:00 2001
From: "joseph.areeda" <joseph.areeda@ligo.org>
Date: Thu, 14 Aug 2025 14:09:08 -0700
Subject: [PATCH 21/40] move condor examples to docs

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 9c1d877..47f9120 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -40,7 +40,7 @@ classifiers = [
 
 # requirements
 requires-python = ">=3.9"
-dependencies = [0
+dependencies = [
   "gwdetchar >= 2.0.0",
   "gwpy >=2.0.0",
   "gwtrigfind >= 0.8.3",

From 60ffad8a8a5f727e9a62886f0059d90c98f198ca Mon Sep 17 00:00:00 2001
From: "joseph.areeda" <joseph.areeda@ligo.org>
Date: Fri, 15 Aug 2025 04:59:55 -0700
Subject: [PATCH 22/40] condor_submit_dag had arguments out of order

---
 hveto/cli/weekly_hveto.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hveto/cli/weekly_hveto.py b/hveto/cli/weekly_hveto.py
index 522a3d9..ea3da77 100644
--- a/hveto/cli/weekly_hveto.py
+++ b/hveto/cli/weekly_hveto.py
@@ -384,7 +384,7 @@ def main():
     if not args.no_submit:
         logger.info(f'Submitting {dag_file} to Condor')
         batch_name = f'hveto {start_dt.strftime("%m/%d")} to {next_dt.strftime("%m/%d")} $(ClusterId)'
-        cmd = f'condor_submit_dag -import_env {dag_file} -batch-name "{batch_name}"'
+        cmd = f'condor_submit_dag -import_env -batch-name "{batch_name}"  {dag_file}'
         logger.debug(f'Running: {cmd}')
         subprocess.run(cmd, shell=True, check=True)
     else:

From 5682e9b03802090c23b9488549d29e0b2c784377 Mon Sep 17 00:00:00 2001
From: "joseph.areeda" <joseph.areeda@ligo.org>
Date: Sun, 21 Sep 2025 15:50:27 -0700
Subject: [PATCH 23/40] add daily runs and allowance for omega batch

---
 README.rst                |   2 +
 hveto/cli/hveto_run.py    | 589 ++++++++++++++++++++++++++++++++++++++
 hveto/cli/weekly_hveto.py | 407 --------------------------
 pyproject.toml            |   2 +-
 4 files changed, 592 insertions(+), 408 deletions(-)
 create mode 100644 hveto/cli/hveto_run.py
 delete mode 100644 hveto/cli/weekly_hveto.py

diff --git a/README.rst b/README.rst
index 1655f31..13da099 100644
--- a/README.rst
+++ b/README.rst
@@ -67,3 +67,5 @@ proposing additions/changes.
    :target: https://codecov.io/gh/gwdetchar/hveto
 .. |Documentation Status| image:: https://readthedocs.org/projects/hveto/badge/?version=stable
    :target: https://hveto.readthedocs.io/en/stable/?badge=stable
+
+.. inlcude README_hveto_run.rst
\ No newline at end of file
diff --git a/hveto/cli/hveto_run.py b/hveto/cli/hveto_run.py
new file mode 100644
index 0000000..b60a7f0
--- /dev/null
+++ b/hveto/cli/hveto_run.py
@@ -0,0 +1,589 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# vim: nu:ai:ts=4:sw=4
+
+#
+#  Copyright (C) 2025 Joseph Areeda <joseph.areeda@ligo.org>
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+""""""
+import os
+import subprocess
+import textwrap
+import time
+from datetime import datetime, timedelta, timezone
+import socket
+
+from pytz import reference
+
+start_time = time.time()
+
+import argparse
+import logging
+from pathlib import Path
+import re
+import sys
+import traceback
+from gwpy.time import to_gps, from_gps
+
+try:
+    from .._version import __version__
+except ImportError:
+    __version__ = '0.0.0'
+
+__author__ = 'joseph areeda'
+__email__ = 'joseph.areeda@ligo.org'
+__process_name__ = re.sub(Path(__file__).suffix, '', Path(__file__).name)
+
+logger = None
+TOO_MUCH = 5        # logging level more verbose than DEBUG
+
+hveto_run_sh = textwrap.dedent('''\
+    #!/bin/bash
+    # script used by %program_name% condor DAG to run one analysis
+    # Created by %program_name% version %program_version% on %date%
+
+    # Run specific variables
+    configuration="%configuration%"
+    nproc=%nproc%
+    ifo=%ifo%
+    gpsstart="%gpsstart%"
+    gpsend="%gpsend%"
+    outer_dir="%outer_dir%"
+    prefix="%prefix%"
+    timeout=%timeout%
+    omega_scans=%omega_scans%
+    omega_arg=""
+    if [ "${omega_scans}" -gt 0 ]
+    then
+        omega_arg="--omega-scans ${omega_scans}"
+    fi
+
+    # run proram in conda environment
+    condaRun="/cvmfs/software.igwn.org/conda/condabin/conda run --prefix ${prefix} --no-capture-output  "
+
+    # use a 3 hour timeout to prevent hanging jobs
+    timeout_cmd="timeout --kill-after=${timeout}s ${timeout}"
+
+    cmd="python -m hveto ${gpsstart} ${gpsend} --ifo ${ifo} --config-file ${configuration} --nproc ${nproc} \
+        --output-directory ${outer_dir} ${omega_arg}"
+
+    ${condaRun} ${timeout_cmd} ${cmd}
+
+    ''')
+
+hveto_job_submit = textwrap.dedent('''\
+    #!/usr/bin/env condor_submit
+    #
+    # Condor submit file for hveto processing in batch mode
+    #
+    # Created by %program_name% version %program_version% on %date%
+    #
+
+    universe = vanilla
+    executable = %exec%
+
+    accounting_group = ligo.dev.o4.detchar.dqtriggers.hveto
+    accounting_group_user = joseph.areeda
+
+    request_memory = %request_memory%M
+    request_disk = 10GB
+
+    output = %condor_dir%/hveto.out
+    error = %condor_dir%/hveto.err
+    log = %condor_dir%/hveto.log
+
+    batch_name = "%job_name% ID: $(ClusterId)"
+
+    use_oauth_services = scitokens
+    environment = BEARER_TOKEN_FILE=$$(CondorScratchDir)/.condor_creds/scitokens.use
+
+    queue
+''')
+
+check_dag_txt = textwrap.dedent('''\
+    #!/bin/bash
+    # check whether the omega DAG file exists
+    # Created by %program_name% version %program_version% on %date%
+
+    if [ -f %omega_dag_file% ]
+    then
+        echo "Omega DAG file %omega_dag_file% exists" >> %log_file%
+        exit 0
+    else
+        echo "Omega DAG file %omega_dag_file% does not exist" >> %log_file%
+        exit 12
+    fi
+    ''')
+
+
+def to_day(in_str):
+    in_gps = to_gps(in_str)
+    in_date = from_gps(in_gps)
+    ret = in_date.strftime('%Y-%m-%d')
+    return ret
+
+
+def get_default_ifo():
+    # if at a site we have a default ifo
+    host = socket.getfqdn()
+    if 'ligo-la' in host:
+        ifo = 'L1'
+    elif 'ligo-wa' in host:
+        ifo = 'H1'
+    else:
+        ifo = os.getenv('IFO')
+    if ifo is None:
+        ifo = 'UK'
+    return ifo, host
+
+
+def parser_add_args(parser):
+    """
+    Set up command parser
+    :param argparse.ArgumentParser parser:
+    :return: None but parser object is updated
+    """
+    parser.add_argument('-v', '--verbose', action='count', default=1,
+                        help='increase verbose output')
+    parser.add_argument('-V', '--version', action='version',
+                        version=__version__)
+    parser.add_argument('-q', '--quiet', default=False, action='store_true',
+                        help='show only fatal errors')
+
+    this_script = Path(__file__)
+    user = os.getenv('USER')
+    config_possibles = [
+        this_script.parent.parent / 'configurations' / 'hveto' / 'h1l1-hveto-daily-o4b.ini',
+        Path(f'/home/{user}/etc/ligo-monitors/configurations/hveto/h1l1-hveto-daily-o4b.ini'),
+        Path('/home/detchar/etc/ligo-monitors/configurations/hveto/h1l1-hveto-daily-o4b.ini'),
+    ]
+    default_config_file = None
+    for config in config_possibles:
+        if config.exists():
+            default_config_file = config
+            break
+
+    parser.add_argument('-c', '--config-file', default=default_config_file, help='path to hveto configuration file')
+
+    now_utc = datetime.now(timezone.utc)
+    default_end_date = now_utc.strftime('%Y-%m-%d')
+    default_start_date = (now_utc - timedelta(days=7)).strftime('%Y-%m-%d')
+
+    parser.add_argument('-e', '--end', type=to_day, default=default_end_date,
+                        help='Processng looks back duration days up to but not including this date. (default: %(default)s).')
+    parser.add_argument('-s', '--start', type=to_day, default=default_end_date,
+                        help='We process one day at a time from end daste back to star date each '
+                             'run is duration days long (default: %(default)s).')
+    parser.add_argument('-d', '--duration', type=int, default=7,
+                        help='Number of days analyze for each day (default: %(default)s).')
+    parser.add_argument('--today', action='store_true',
+                        help='Process today\'s data it will be for 8, 16 or 24 hours dependng on current time. '
+                             'Start, end and duration are ignored')
+
+    parser.add_argument('--yesterday', action='store_true',
+                        help='Process yesterday\'s data. '
+                             'This is equivalent to --start yesterday --end yesterday --duration 1')
+
+    parser.add_argument('--this-week', action='store_true',
+                        help='Process this week\'s data. '
+                             f'This is equivalent to --start {default_start_date} --end {default_end_date} --duration 7')
+    parser.add_argument('--stride', type=int, default=1,
+                        help='Number of days between each hveto run. '
+                             'For example start=7/1, end=7/29, stride=7, duration=7'
+                             'will create 4 hveto reports ending on 7/29, 7/22, 7/15, 7/8, 7/1 each covering 7 days')
+    parser.add_argument('--no-submit', action='store_true',
+                        help='Create directory, script and submit file but do not submit to Condor')
+
+    output_group = parser.add_mutually_exclusive_group()
+    output_group.add_argument('-o', '--output-directory', type=Path,
+                              default=f'/home/{os.getenv("USER")}/public_html/hveto',
+                              help='Parent directory for what may be multiple hveto runs. '
+                                   'A subdirectory based on duration will be used for each run. ')
+    output_group.add_argument('-O', '--output-directory-exact', type=Path,
+                              help='Parent directory for what may be multiple hveto runs. '
+                                   'A subdirectory based on duration will NOT be used for each run. ')
+
+    prefix = os.getenv('CONDA_PREFIX', '/home/detchar/.conda/envs/ligo-summary-3.10')
+    parser.add_argument('-p', '--prefix', default=prefix, help='Path to conda environment to run hveto in')
+
+    ifo, host = get_default_ifo()
+    parser.add_argument('-i', '--ifo', default=ifo, help='IFO for hveto analysis')
+    parser.add_argument('-n', '--nproc', default=4, type=int, help='Number of parallel tigger readers')
+    parser.add_argument('--omega-scans', type=int, default=0,
+                        help='Number of omega scans per round.'
+                             ' Small numbers arerecommended for weekly runs because of the high number of rounds')
+
+
+def apply_symbols(txt, symbols):
+    """
+    Search text for {<symbol>} entries use <symbol> as key to dicionary then substite its value
+    :param str txt: input string with <symbol> entries
+    :param dict symbols: symbol table
+    :return str: text with substitutions applied
+    """
+    ret = ''
+    for line in txt.splitlines():
+        oline = line
+        while re.match('.*%.+%', oline):
+            for symbol, value in symbols.items():
+                pattern = '%' + rf'{symbol}' + '%'
+                oline = re.sub(pattern, str(value), oline)
+            if oline != line:
+                logger.log(TOO_MUCH, f'  {line} -> {oline}')
+            else:
+                logger.critical(f'Undefined symbol in {line}')
+            break
+        ret += oline + '\n'
+    return ret
+
+
+def make_job(job_name, job_args):
+    """
+    Creates and prepares the directory and condor files for a single hveto job.
+
+    :param str job_name: The name of the job to create
+    :param dict job_args: The arguments to use for the job
+    :return Path: Path to the submit file
+    """
+    job_dir = job_args['outer_dir']
+    job_dir.mkdir(parents=True, exist_ok=True)
+    job_condordir = job_args['condor_dir']
+    job_condordir.mkdir(parents=True, exist_ok=True)
+
+    job_submit_file = job_condordir / f'{job_name}.submit'
+    job_run_sh = job_condordir / f'{job_name}.sh'
+    job_args['job_name'] = job_name
+    job_script = apply_symbols(hveto_run_sh, job_args)
+    job_run_sh.write_text(job_script)
+    job_run_sh.chmod(0o755)
+    job_args['exec'] = job_run_sh
+
+    job_submit = apply_symbols(hveto_job_submit, job_args)
+    job_submit_file.write_text(job_submit)
+    logger.info(f'Job name {job_name} created in {job_condordir}')
+    logger.info(f'Created condor submit file {job_submit_file} ')
+    logger.info(f' script {job_run_sh}')
+    return job_submit_file
+
+
+def get_today_start():
+    """
+    create a single job for today that processes 8, 16 or 24 hours of triggers based on current time
+
+    :return str, datetime, int: date string (YYMMDD), as datetime object and duration in hours
+    """
+
+    now_dt = datetime.now(timezone.utc)
+    run_start = now_dt.replace(hour=0, minute=0, second=0, microsecond=0)
+    now_hour = now_dt.hour
+    if now_hour < 10:
+        # process full day yesterday
+        run_end = run_start
+        run_start -= timedelta(hours=24)
+        duration = 24
+    elif now_hour < 18:
+        run_end = run_start + timedelta(hours=8)
+        duration = 8
+    else:
+        run_end = run_start + timedelta(hours=16)
+        duration = 16
+    job_day = run_start.strftime('%Y%m%d')
+    logger.info(f'Running today {job_day} from {run_start} to {run_end}')
+
+    return job_day, run_start, duration
+
+
+def process_omega_scans(job_args_in):
+    if job_args_in['omega_scans'] > 0:
+        duration_label = job_args_in['duration_label']
+        job_name = f'omega_{job_args_in["job_day"]}_{duration_label}'
+        # Create a script to check whether omega scans DAG ha been made by hveto
+        omega_dag_file = job_args_in["omega_dag_file"]
+        omega_dag_file.parent.mkdir(parents=True, exist_ok=True)
+        dag_log = job_args_in["dag_log"]
+        job_args_omega = {'omega_dag_file': omega_dag_file, 'log_file': dag_log, 'job_name': job_name}
+        dag_dir = job_args_in['dag_dir']
+        check_dag_file = dag_dir / f'check_{job_name}.sh'
+        job_args = {**job_args_in, **job_args_omega}
+        check_dag_str = apply_symbols(check_dag_txt, job_args)
+        check_dag_file.write_text(check_dag_str)
+
+        dag_fh = job_args_in['dag_fh']
+        job_day = job_args_in['job_day']
+        print(f'SUBDAG EXTERNAL {job_name} "{omega_dag_file}"\n', file=dag_fh)
+        print(f'PRE SCRIPT {job_name} "{check_dag_file}" "{omega_dag_file}"', file=dag_fh)
+        print(f'PRE_SKIP {job_name} 12', file=dag_fh)
+        print(f'PARENT {job_name} CHILD hveto_{job_day}_{duration_label}', file=dag_fh)
+
+
+def main():
+    global logger
+
+    log_file_format = "%(asctime)s - %(levelname)s - %(funcName)s %(lineno)d: %(message)s"
+    log_file_date_format = '%m-%d %H:%M:%S'
+    logging.basicConfig(format=log_file_format, datefmt=log_file_date_format)
+    logger = logging.getLogger(__process_name__)
+    logger.setLevel(logging.DEBUG)
+
+    description = textwrap.dedent("""
+    Create a bash script and condor_submit for a multi-day run of hveto, then submit to vanilla universe
+    By default:
+    - We process 7 days of data ending at 00:00 UTC today, expecting it to run around 02:00Z
+    - Output goes to ~/public_html/hveto/weekly/<YYYYMM>/<YYYYMMDD>/
+    - Uses the configuration file "h1l1-hveto-daily-o4b.ini" found at
+      <path to this script>/../configurations/hveto/ or
+      ~/etc/ligo-monitors/configurations/hveto/ or
+      /home/detchar/etc/ligo-monitors/configurations/hveto/
+    """)
+
+    epilog = textwrap.dedent("""
+    Running with default (no) arguments  will run hveto over 7 days of data ending at 00:00 UTC this morning.
+
+    To run hveto for every day for the month of July, each run covering 7 days:
+
+        python -m weekly_hveto --start 7/1 --end 7/31 --duration 7 --stride 1
+
+    """)
+    parser = argparse.ArgumentParser(description=description, epilog=epilog, prog=__process_name__,
+                                     formatter_class=argparse.RawTextHelpFormatter)
+    parser_add_args(parser)
+    args = parser.parse_args()
+    verbosity = 0 if args.quiet else args.verbose
+
+    if verbosity < 1:
+        logger.setLevel(logging.CRITICAL)
+    elif verbosity < 2:
+        logger.setLevel(logging.WARNING)
+    elif verbosity < 3:
+        logger.setLevel(logging.INFO)
+    elif verbosity < 4:
+        logger.setLevel(logging.DEBUG)
+    else:
+        logger.setLevel(TOO_MUCH)
+
+    duration = args.duration
+    duration_dt = timedelta(days=duration)
+
+    dur_str = ''
+    if args.output_directory_exact:
+        output_directory = Path(args.output_directory_exact)
+    elif args.output_directory:
+        output_directory = Path(args.output_directory)
+        if duration == 1 or args.today:
+            dur_str = 'day'
+        elif duration == 7:
+            dur_str = 'week'
+        else:
+            dur_str = f'{duration}days'
+
+        output_directory = output_directory / dur_str
+
+    now_dt = datetime.now(timezone.utc)
+    now_time = now_dt.strftime('%Y%m%dT%H%M0%S')
+    now_month = now_dt.strftime('%Y%m')
+
+    if args.today:
+        job_day, job_dt, duration = get_today_start()
+        dag_dir = output_directory / job_day / 'condor'
+    elif args.start == args.end:
+        run_dt = datetime.strptime(args.start, '%Y-%m-%d')
+        # if we are only doing one run then we use the condor directory for the DAG file
+        dag_dir = output_directory / run_dt.strftime('%Y%m%d') / 'condor'
+    else:
+        dag_dir = output_directory / f'{__process_name__}-DAG' / now_month / f'DAG-{now_time}'
+
+    dag_dir.mkdir(parents=True, exist_ok=True)
+    dag_log = dag_dir / 'hveto-run.log'
+    dag_file = dag_dir / 'hveto-run.dag'
+    logger.info(f'Top level DAG file  will be written to {dag_file}')
+    dag_file.parent.mkdir(parents=True, exist_ok=True)
+
+    # add a copy of our logging to a file
+    NOW = datetime.now()
+    TIMEZONE = reference.LocalTimezone().tzname(NOW)
+    DATEFMT = '%Y-%m-%d %H:%M:%S {}'.format(TIMEZONE)
+    FMT = '%(name)s %(asctime)s %(levelname)+8s: %(filename)s:%(lineno)d:  %(message)s'
+    logf_formatter = logging.Formatter(FMT, datefmt=DATEFMT)
+    logf_handler = logging.FileHandler(dag_log, mode='w')
+    logf_handler.setFormatter(logf_formatter)
+    logger.addHandler(logf_handler)
+
+    # debugging?
+    logger.debug(f'{__process_name__} version: {__version__} called with arguments:')
+    log_level_names = {logging.CRITICAL: 'CRITICAL', logging.ERROR: 'ERROR', logging.WARNING: 'WARNING',
+                       logging.INFO: 'INFO', logging.DEBUG: 'DEBUG', TOO_MUCH: 'TOO_MUCH'}
+    logger.debug(f'logger level set to {logger.getEffectiveLevel()} - {log_level_names[logger.getEffectiveLevel()]}')
+
+    for k, v in args.__dict__.items():
+        logger.debug('    {} = {}'.format(k, v))
+
+    config = args.config_file
+    if config is None:
+        logger.critical('Configuration file not specified')
+        raise ValueError('Configuration file not specified')
+
+    if args.ifo == 'UK':
+        logger.critical('IFO must be specified on command line or in the environment, if not running at LLO or LHO')
+        raise ValueError('IFO must be specified')
+    elif args.ifo not in ['L1', 'H1']:
+        logger.critical(f'IFO {args.ifo} not in ["L1", "H1"]')
+        raise ValueError(f'IFO {args.ifo} not in ["L1", "H1"]')
+
+    start_day = args.start
+    start_gps = to_gps(start_day)
+    start_dt = from_gps(start_gps)
+
+    end_day = args.end
+    end_gps = to_gps(end_day)
+    end_dt = from_gps(end_gps)
+    next_dt = end_dt
+
+    stride = args.stride
+    stride_dt = timedelta(days=stride)
+
+    # runtime args are used to create the job's bash script for both
+    # the today job and the regular jobs
+    job_args_common = {
+        "configuration": config,
+        "nproc": args.nproc,
+        "ifo": args.ifo,
+        "prefix": args.prefix,
+        "program_name": __process_name__,
+        "program_version": __version__,
+        "date": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
+        "request_disk": 100000,
+        "timeout": 3 * 3600,
+    }
+
+    # create a DAG file for each day in the range
+    with dag_file.open('w') as dag_fh:
+        job_args_common['dag_dir'] = dag_dir
+        job_args_common['dag_log'] = dag_log
+        job_args_common['dag_fh'] = dag_fh
+
+        print(f'# process long duration hveto for {start_day} to {end_day}  \n'
+              f'# run every (stride)  {stride} day(s) analyzing each over {duration} days', file=dag_fh)
+        print(f'# Created by {__process_name__}, version {__version__}\n', file=dag_fh)
+
+        print('CATEGORY ALL_NODES LIMIT', file=dag_fh)
+        print('MAXJOBS LIMIT 3', file=dag_fh)
+        print('RETRY ALL_NODES 2\n', file=dag_fh)
+
+        if args.today:
+            job_day, job_dt, duration = get_today_start()
+            job_name = f'hveto_{job_day}_{duration:02d}hr'
+
+            #  create the results directory and add the condor submit file and bash script
+            job_dir = output_directory / job_day
+            condor_dir = job_dir / 'condor'
+            if duration == 24:
+                job_end_dt = job_dt + timedelta(hours=duration)
+                job_end_str = job_end_dt.strftime('%Y-%m-%d %H:%M:%S')
+            else:
+                job_end_str = f'{job_dt.strftime("%Y-%m-%d")} {duration:02d}:00:00'
+            job_args_today = {
+                "gpsstart": f'{job_dt.strftime("%Y-%m-%d")}',
+                "gpsend": job_end_str,
+                'duration_label': f'{duration:02d}_hr',
+                "outer_dir": job_dir,
+                "condor_dir": condor_dir,
+                "end_day": 'end_day',
+                "out_of": 'out_of_',
+                "omega_scans": args.omega_scans,
+                "omega_dag_file": job_dir / 'scans' / 'condor' / 'gwdetchar-omega-batch.submit',
+                "job_day": job_day,
+                "duration": duration,
+                "request_memory": 9216,
+            }
+            job_args = {**job_args_common, **job_args_today}
+            job_submit_file = make_job(job_name, job_args)
+            print(f'JOB {job_name} {job_submit_file}', file=dag_fh)
+            process_omega_scans(job_args)
+
+            njobs = 1
+        else:
+
+            njobs = int((end_dt - start_dt).days / stride) + 1
+            current_job = 1
+
+            while start_dt <= next_dt:
+                begin_dt = to_day(next_dt - duration_dt)
+                begin_day = to_day(begin_dt)
+                next_day = to_day(next_dt)
+                job_name = 'hveto'
+                job_name += f'_{current_job:02d}_of_{njobs:02d}' if njobs > 1 else ''
+                logger.debug(f'Process hveto {begin_day} to {next_day}')
+                job_month = next_dt.strftime('%Y%m')
+                job_day = next_dt.strftime('%Y%m%d')
+
+                #  create the results directory and add the condor submit file and bash script
+                if duration == 1:
+                    job_dir = output_directory / job_day
+                else:
+                    job_dir = output_directory / job_month / job_day
+
+                job_args_multi_day = {
+                    "gpsstart": begin_day,
+                    "gpsend": next_day,
+                    'job_day': job_day,
+                    'duration_label': f'{duration}_day',
+                    "outer_dir": job_dir,
+                    "condor_dir": job_dir / 'condor',
+                    "end_day": next_day,
+                    "out_of": f'{current_job:02d}_of_{njobs:02d}' if njobs > 1 else '',
+                    "omega_dag_file": job_dir / 'scans' / 'condor' / 'gwdetchar-omega-batch.submit',
+                    "omega_scans": args.omega_scans,
+                    "request_memory": 10240,
+                }
+                job_args = {**job_args_common, **job_args_multi_day}
+                job_submit_file = make_job(job_name, job_args)
+
+                # add this job to the DAG
+                print(f'JOB {job_name} {job_submit_file}', file=dag_fh)
+                current_job += 1
+
+                if job_args['omega_scans'] > 0:
+
+                    process_omega_scans(job_args)
+
+                next_dt -= stride_dt
+
+        logger.info(f'DAG file with {njobs} jobs written to {dag_file}')
+        if not args.no_submit:
+            logger.info(f'Submitting {dag_file} to Condor')
+            batch_name = f'hveto {start_dt.strftime("%m/%d")} to {next_dt.strftime("%m/%d")} $(ClusterId)'
+            cmd = f'condor_submit_dag -import_env -batch-name "{batch_name}"  {dag_file}'
+            logger.debug(f'Running: {cmd}')
+            subprocess.run(cmd, shell=True, check=True)
+        else:
+            logger.info('DAG file written but not submitted to Condor')
+
+    # report our run time
+    logger.info(f'Elapsed time: {time.time() - start_time:.1f}s')
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except (ValueError, TypeError, OSError, NameError, ArithmeticError, RuntimeError) as ex:
+        print(ex, file=sys.stderr)
+        traceback.print_exc(file=sys.stderr)
+
+    if logger is None:
+        logging.basicConfig()
+        logger = logging.getLogger(__process_name__)
+        logger.setLevel(logging.DEBUG)
diff --git a/hveto/cli/weekly_hveto.py b/hveto/cli/weekly_hveto.py
deleted file mode 100644
index ea3da77..0000000
--- a/hveto/cli/weekly_hveto.py
+++ /dev/null
@@ -1,407 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# vim: nu:ai:ts=4:sw=4
-
-#
-#  Copyright (C) 2025 Joseph Areeda <joseph.areeda@ligo.org>
-#
-#  This program is free software: you can redistribute it and/or modify
-#  it under the terms of the GNU General Public License as published by
-#  the Free Software Foundation, either version 3 of the License, or
-#  (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-""""""
-import os
-import subprocess
-import textwrap
-import time
-from datetime import datetime, timedelta, timezone
-import socket
-
-from pytz import reference
-
-start_time = time.time()
-
-import argparse
-import logging
-from pathlib import Path
-import re
-import sys
-import traceback
-from gwpy.time import to_gps, from_gps
-
-try:
-    from .._version import __version__
-except ImportError:
-    __version__ = '0.0.0'
-
-__author__ = 'joseph areeda'
-__email__ = 'joseph.areeda@ligo.org'
-__process_name__ = Path(__file__).name
-
-logger = None
-
-hveto_run_sh = textwrap.dedent('''\
-    #!/bin/bash
-    # script used by %program_name% condor DAG to run one analysis
-    # Created by %program_name% version %program_version% on %date%
-
-    # Run specific variables
-    configuration="%configuration%"
-    nproc=%nproc%
-    ifo=%ifo%
-    gpsstart="%gpsstart%"
-    gpsend="%gpsend%"
-    outer_dir="%outer_dir%"
-    prefix="%prefix%"
-    timeout=%timeout%
-
-    # run proram in conda environment
-    condaRun="/cvmfs/software.igwn.org/conda/condabin/conda run --prefix ${prefix} --no-capture-output  "
-
-    # use a 3 hour timeout to prevent hanging jobs
-    timeout_cmd="timeout --kill-after=${timeout}s ${timeout}"
-
-    cmd="python -m hveto ${gpsstart} ${gpsend} --ifo ${ifo} --config-file ${configuration} --nproc ${nproc} \
-        --output-directory ${outer_dir}"
-
-    ${condaRun} ${timeout_cmd} ${cmd}
-
-    ''')
-
-hveto_job_submit_sh = textwrap.dedent('''
-#!/usr/bin/env condor_submit
-#
-# Condor submit file for hveto processing in batch mode
-#
-# Created by %program_name% version %program_version% on %date%
-#
-
-universe = vanilla
-executable = %exec%
-
-accounting_group = ligo.dev.o4.detchar.dqtriggers.hveto
-accounting_group_user = joseph.areeda
-
-request_memory = %request_memory%M
-request_disk = 10GB
-
-output = %condor_dir%/hveto.out
-error = %condor_dir%/hveto.err
-log = %condor_dir%/hveto.log
-
-batch_name = "hveto %end_day% %out_of% ID: $(ClusterId)"
-
-use_oauth_services = scitokens
-environment = BEARER_TOKEN_FILE=$$(CondorScratchDir)/.condor_creds/scitokens.use
-
-
-queue
-''')
-
-
-def to_day(in_str):
-    in_gps = to_gps(in_str)
-    in_date = from_gps(in_gps)
-    ret = in_date.strftime('%Y-%m-%d')
-    return ret
-
-
-def get_default_ifo():
-    # if at a site we have a default ifo
-    host = socket.getfqdn()
-    if 'ligo-la' in host:
-        ifo = 'L1'
-    elif 'ligo-wa' in host:
-        ifo = 'H1'
-    else:
-        ifo = os.getenv('IFO')
-    if ifo is None:
-        ifo = 'UK'
-    return ifo, host
-
-
-def parser_add_args(parser):
-    """
-    Set up command parser
-    :param argparse.ArgumentParser parser:
-    :return: None but parser object is updated
-    """
-    parser.add_argument('-v', '--verbose', action='count', default=1,
-                        help='increase verbose output')
-    parser.add_argument('-V', '--version', action='version',
-                        version=__version__)
-    parser.add_argument('-q', '--quiet', default=False, action='store_true',
-                        help='show only fatal errors')
-
-    this_script = Path(__file__)
-    user = os.getenv('USER')
-    config_possibles = [
-        this_script.parent.parent / 'configurations' / 'hveto' / 'h1l1-hveto-daily-o4b.ini',
-        Path(f'/home/{user}/etc/ligo-monitors/configurations/hveto/h1l1-hveto-daily-o4b.ini'),
-        Path('/home/detchar/etc/ligo-monitors/configurations/hveto/h1l1-hveto-daily-o4b.ini'),
-    ]
-    default_config_file = None
-    for config in config_possibles:
-        if config.exists():
-            default_config_file = config
-            break
-
-    parser.add_argument('-c', '--config-file', default=default_config_file, help='path to hveto configuration file')
-
-    now_utc = datetime.now(timezone.utc)
-    default_end_date = now_utc.strftime('%Y-%m-%d')
-
-    parser.add_argument('-e', '--end', type=to_day, default=default_end_date,
-                        help='Processng looks back duration days up to but not including this date. (default: %(default)s).')
-    parser.add_argument('-s', '--start', type=to_day, default=default_end_date,
-                        help='We process one day at a time from end daste back to star date each '
-                             'run is duration days long (default: %(default)s).')
-    parser.add_argument('-d', '--duration', type=int, default=7,
-                        help='Number of days analyze for each day (default: %(default)s).')
-    parser.add_argument('--stride', type=int, default=1,
-                        help='Number of days between each hveto run. '
-                             'For example start=7/1, end=7/29, stride=7, duration=7'
-                             'will create 4 hveto reports ending on 7/28, 7/21, 7/14, 7/1 each covering 7 days')
-    parser.add_argument('--no-submit', action='store_true',
-                        help='Create directory, script and submit file but do not submit to Condor')
-    parser.add_argument('-o', '--output-directory', default=f'/home/{os.getenv("USER")}/public_html/hveto-weekly',
-                        help='Parent directory for what may be multiple hveto runs')
-
-    prefix = os.getenv('CONDA_PREFIX', '/home/detchar/.conda/envs/ligo-summary-3.10')
-    parser.add_argument('-p', '--prefix', default=prefix, help='Path to conda environment to run hveto in')
-
-    ifo, host = get_default_ifo()
-    parser.add_argument('-i', '--ifo', default=ifo, help='IFO for hveto analysis')
-    parser.add_argument('-n', '--nproc', default=4, type=int, help='Number of parallel tigger readers')
-
-
-def apply_symbols(txt, symbols):
-    """
-    Search text for {<symbol>} entries use <symbol> as key to dicionary then substite its value
-    :param str txt: input string with <symbol> entries
-    :param dict symbols: symbol table
-    :return str: text with substitutions applied
-    """
-    ret = ''
-    for line in txt.splitlines():
-        oline = line
-        while re.match('.*%.+%', oline):
-            for symbol, value in symbols.items():
-                pattern = '%' + rf'{symbol}' + '%'
-                oline = re.sub(pattern, str(value), oline)
-            if oline != line:
-                logger.debug(f'  {line} -> {oline}')
-            else:
-                logger.critical(f'Undefined symbol in {line}')
-            break
-        ret += oline + '\n'
-    return ret
-
-
-def make_job(job_name, job_args):
-    """
-    Creates and prepares the directory and condor files for a single hveto job.
-
-    :param str job_name: The name of the job to create
-    :param dict job_args: The arguments to use for the job
-    :return Path: Path to the submit file
-    """
-    job_dir = job_args['outer_dir']
-    logger.debug(f'Creating condor file needed for job {job_name} in {job_dir}')
-    job_dir.mkdir(parents=True, exist_ok=True)
-    job_condadir = job_dir / 'condor'
-    job_condadir.mkdir(parents=True, exist_ok=True)
-
-    job_submit_file = job_dir / 'condor' / f'{job_name}.submit'
-    job_run_sh = job_condadir / f'{job_name}.sh'
-    job_script = apply_symbols(hveto_run_sh, job_args)
-    job_run_sh.write_text(job_script)
-    job_run_sh.chmod(0o755)
-    job_args['exec'] = job_run_sh
-
-    job_submit = apply_symbols(hveto_job_submit_sh, job_args)
-    job_submit_file.write_text(job_submit)
-
-    return job_submit_file
-
-
-def main():
-    global logger
-
-    log_file_format = "%(asctime)s - %(levelname)s - %(funcName)s %(lineno)d: %(message)s"
-    log_file_date_format = '%m-%d %H:%M:%S'
-    logging.basicConfig(format=log_file_format, datefmt=log_file_date_format)
-    logger = logging.getLogger(__process_name__)
-    logger.setLevel(logging.DEBUG)
-
-    description = textwrap.dedent("""
-    Create a bash script and condor_submit for a multi-day run of hveto, then submit to vanilla universe
-    By default:
-    - We process 7 days of data ending at 00:00 UTC today, expecting it to run around 02:00Z
-    - Output goes to ~/public_html/hveto/weekly/<YYYYMM>/<YYYYMMDD>/
-    - Uses the configuration file "h1l1-hveto-daily-o4b.ini" found at
-      <path to this script>/../configurations/hveto/ or
-      ~/etc/ligo-monitors/configurations/hveto/ or
-      /home/detchar/etc/ligo-monitors/configurations/hveto/
-    """)
-
-    epilog = textwrap.dedent("""
-    Running with default (no) arguments  will run hveto over 7 days of data ending at 00:00 UTC this morning.
-
-    To run hveto for every day for the month of July, each run covering 7 days:
-
-        python -m weekly_hveto --start 7/1 --end 7/31 --duration 7 --stride 1
-
-    """)
-    parser = argparse.ArgumentParser(description=description, epilog=epilog, prog=__process_name__,
-                                     formatter_class=argparse.RawTextHelpFormatter)
-    parser_add_args(parser)
-    args = parser.parse_args()
-    verbosity = 0 if args.quiet else args.verbose
-
-    if verbosity < 1:
-        logger.setLevel(logging.CRITICAL)
-    elif verbosity < 2:
-        logger.setLevel(logging.INFO)
-    else:
-        logger.setLevel(logging.DEBUG)
-
-    output_directory = Path(args.output_directory)
-    now_time = datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S')
-    dag_dir = output_directory / f'DAG-{now_time}'
-    dag_dir.mkdir(parents=True, exist_ok=True)
-
-    dag_log = dag_dir / 'weekly-dag.log'
-    dag_file = dag_dir / 'hveto.dag'
-
-    # add logging too a file
-    NOW = datetime.now()
-    TIMEZONE = reference.LocalTimezone().tzname(NOW)
-    DATEFMT = '%Y-%m-%d %H:%M:%S {}'.format(TIMEZONE)
-    FMT = '%(name)s %(asctime)s %(levelname)+8s: %(filename)s:%(lineno)d:  %(message)s'
-    logf_formatter = logging.Formatter(FMT, datefmt=DATEFMT)
-    logf_handler = logging.FileHandler(dag_log, mode='w')
-    logf_handler.setFormatter(logf_formatter)
-    logger.addHandler(logf_handler)
-
-    # debugging?
-    logger.debug(f'{__process_name__} version: {__version__} called with arguments:')
-    for k, v in args.__dict__.items():
-        logger.debug('    {} = {}'.format(k, v))
-
-    config = args.config_file
-    if config is None:
-        logger.critical('Configuration file not specified')
-        raise ValueError('Configuration file not specified')
-
-    if args.ifo == 'UK':
-        logger.critical('IFO must be specified on command line or in the environment, if not running at LLO or LHO')
-        raise ValueError('IFO must be specified')
-    elif args.ifo not in ['L1', 'H1']:
-        logger.critical(f'IFO {args.ifo} not in ["L1", "H1"]')
-        raise ValueError(f'IFO {args.ifo} not in ["L1", "H1"]')
-
-    start_day = args.start
-    start_gps = to_gps(start_day)
-    start_dt = from_gps(start_gps)
-
-    end_day = args.end
-    end_gps = to_gps(end_day)
-    end_dt = from_gps(end_gps)
-    next_dt = end_dt
-
-    duration = args.duration
-    duration_dt = timedelta(days=duration)
-
-    stride = args.stride
-    stride_dt = timedelta(days=stride)
-
-    logger.info(f'DAG file  will be written to {dag_file}')
-    dag_file.parent.mkdir(parents=True, exist_ok=True)
-
-    with dag_file.open('w') as dag_fh:
-        print(f'# process long duration hveto for {start_day} to {end_day} each {duration} days', file=dag_fh)
-        print(f'# stride = {stride} days', file=dag_fh)
-        print(f'# Created by {__process_name__}, version {__version__}\n', file=dag_fh)
-        print('CATEGORY ALL_NODES LIMIT', file=dag_fh)
-        print('MAXJOBS LIMIT 3', file=dag_fh)
-        print('RETRY ALL_NODES 2\n', file=dag_fh)
-        print('', file=dag_fh)
-
-        njobs = int((end_dt - start_dt).days / stride) + 1
-        current_job = 1
-
-        while start_dt <= next_dt:
-            begin_dt = to_day(next_dt - duration_dt)
-            begin_day = to_day(begin_dt)
-            next_day = to_day(next_dt)
-            job_name = f'hveto_{current_job:02d}_of_{njobs:02d}'
-            logger.debug(f'Process hveto {begin_day} to {next_day}')
-            job_month = next_dt.strftime('%Y%m')
-            job_day = next_dt.strftime('%Y%m%d')
-
-            #  create the results directory and add the condor submit file and bash script
-            job_dir = output_directory / job_month / job_day
-
-            # runtime args are used to create the job's bash script
-            job_args = {
-                "configuration": config,
-                "nproc": args.nproc,
-                "ifo": args.ifo,
-                "gpsstart": begin_day,
-                "gpsend": next_day,
-                "outer_dir": job_dir,
-                "prefix": args.prefix,
-                "program_name": __process_name__,
-                "program_version": __version__,
-                "date": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
-                "condor_dir": job_dir / 'condor',
-                "request_memory": 15384,
-                "end_day": next_day,
-                "out_of": f'{current_job:02d}_of_{njobs:02d}',
-                "timeout": 3 * 3600,
-
-            }
-            job_submit_file = make_job(job_name, job_args)
-
-            # add this job to the DAG
-            print(f'JOB {job_name} {job_submit_file}', file=dag_fh)
-            current_job += 1
-
-            next_dt -= stride_dt
-
-    logger.info(f'DAG file with {njobs} jobs written to {dag_file}')
-    if not args.no_submit:
-        logger.info(f'Submitting {dag_file} to Condor')
-        batch_name = f'hveto {start_dt.strftime("%m/%d")} to {next_dt.strftime("%m/%d")} $(ClusterId)'
-        cmd = f'condor_submit_dag -import_env -batch-name "{batch_name}"  {dag_file}'
-        logger.debug(f'Running: {cmd}')
-        subprocess.run(cmd, shell=True, check=True)
-    else:
-        logger.info('DAG file written but not submitted to Condor')
-
-    # report our run time
-    logger.info(f'Elapsed time: {time.time() - start_time:.1f}s')
-
-
-if __name__ == "__main__":
-    try:
-        main()
-    except (ValueError, TypeError, OSError, NameError, ArithmeticError, RuntimeError) as ex:
-        print(ex, file=sys.stderr)
-        traceback.print_exc(file=sys.stderr)
-
-    if logger is None:
-        logging.basicConfig()
-        logger = logging.getLogger(__process_name__)
-        logger.setLevel(logging.DEBUG)
diff --git a/pyproject.toml b/pyproject.toml
index 47f9120..6b863b3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -77,7 +77,7 @@ docs = [
 hveto = "hveto.__main__:main"
 hveto-cache-events = "hveto.cli.cache_events:main"
 hveto-trace = "hveto.cli.trace:main"
-hveto-weekly = "hveto.cli.weekly_hveto:main"
+hveto-run = "hveto.cli.hveto_run:main"
 
 [project.urls]
 "Bug Tracker" = "https://github.com/gwdetchar/hveto/issues"

From 5236570fa23fa68eebe8f1476470cf9a13f42a91 Mon Sep 17 00:00:00 2001
From: "joseph.areeda" <joseph.areeda@ligo.org>
Date: Sun, 21 Sep 2025 19:52:33 -0700
Subject: [PATCH 24/40] updeate submit file exmples and dag submit command

---
 docs/Condor/hveto-daily.sub  | 23 ++++++++++++++---------
 docs/Condor/hveto-weekly.sub | 10 +++++-----
 hveto/cli/hveto_run.py       |  2 +-
 3 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/docs/Condor/hveto-daily.sub b/docs/Condor/hveto-daily.sub
index 344fae2..7ce17b3 100644
--- a/docs/Condor/hveto-daily.sub
+++ b/docs/Condor/hveto-daily.sub
@@ -1,29 +1,34 @@
-#!/usr/bin/env condor_submit
 #
-# Condor submit file for Hveto daily processing
+# Condor submit file for Hveto weekly processing
+# it runs inn the local universe creates and submits a DAG to the vbanilla universe
 #
 
 universe = local
-executable = $ENV(HOME)/etc/ligo-monitors/scripts/daily-hveto.sh
+
+executable = /cvmfs/software.igwn.org/conda/condabin/conda
+arguments = 'run --prefix /home/detchar.dev/.conda/envs/ligo-summary-3.11-test timeout 30m hveto-run -vv --today'
+
 accounting_group = ligo.prod.o4.detchar.dqtriggers.hveto
-accounting_group_user = <user name valid for account>
+accounting_group_user = joseph.areeda
 
-request_memory = 20GB
+request_memory = 2GB
 request_disk = 5GB
 
-output = $ENV(HOME)/public_html/hveto/logs/daily-hveto.out
-error = $ENV(HOME)/public_html/hveto/logs/daily-hveto.err
-log = $ENV(HOME)/public_html/hveto/logs/daily-hveto.log
+output = $ENV(HOME)/public_html/hveto-weekly/logs/hveto-weekly.out
+error = $ENV(HOME)/public_html/hveto-weekly/logs/hveto-weekly.err
+log = $ENV(HOME)/public_html/hveto-weekly/logs/hveto-weekly.log
 
 batch_name = "daily hveto [cron] ID: $(ClusterId)"
 
 on_exit_remove = false
 periodic_hold = false
+
 cron_minute = 30
 # we want run to be 8, 16 24 UTC with enough slack for omicron selected by local time at LLO (UTC -5 CDT)
 # so LLO CDT -> 10, 18, 1 UTC. CST -> 11, 19, 2 UTC
 # LHO PDT (UTC - 7) -> 12, 20, 4 UC. PST - 13, 21, 5 TC
 cron_hour = 5, 13, 21
+
 cron_day_of_month = *
 cron_month = *
 cron_day_of_week = *
@@ -33,4 +38,4 @@ use_oauth_services = scitokens
 environment = BEARER_TOKEN_FILE=$$(CondorScratchDir)/.condor_creds/scitokens.use
 
 getenv = DEFAULT_SEGMENT_SERVER ECP_IDP GWDATAFIND_SERVER HDF5_USE_FILE_LOCKING HOME KRB5CCNAME NDSSERVER KRB5_KTNAME CONDA_PREFIX IFO USER
-queue
+queue
\ No newline at end of file
diff --git a/docs/Condor/hveto-weekly.sub b/docs/Condor/hveto-weekly.sub
index 16df02a..be6c141 100644
--- a/docs/Condor/hveto-weekly.sub
+++ b/docs/Condor/hveto-weekly.sub
@@ -1,19 +1,18 @@
 #!/usr/bin/env condor_submit
 #
 # Condor submit file for Hveto weekly processing
-# it runs inn the local universe creates and submits a DAG to the vbanilla universe
+# it runs inn the local universe creates and submits a DAG to the vanilla universe
 #
 
 universe = local
-my.conda_exe = /cvmfs/software.igwn.org/conda/condabin/conda
-my.conda_prefix = prefix="/home/detchar.dev/.conda/envs/ligo-summary-3.11-test"
 
 executable = $(my.conda_exe)
-arguments = 'run --prefix $(my.conda_prefix)) timeout 30m hveto-weekly -vv'
+arguments = 'run --prefix $(my.conda_prefix)) timeout 30m hveto-run -vv'
+
 accounting_group = ligo.prod.o4.detchar.dqtriggers.hveto
 accounting_group_user = <user name valid for account>
 
-request_memory = 20GB
+request_memory = 2GB
 request_disk = 5GB
 
 output = $ENV(HOME)/public_html/hveto-weekly/logs/hveto-weekly.out
@@ -28,6 +27,7 @@ periodic_hold = false
 # We run ths in the Central and Pacific time zones standard and daylght savings
 # We want to give the daily gap filler
 cron_hour = 23
+cron_minute = 17
 cron_day_of_month = *
 cron_month = *
 cron_day_of_week = *
diff --git a/hveto/cli/hveto_run.py b/hveto/cli/hveto_run.py
index b60a7f0..56e947b 100644
--- a/hveto/cli/hveto_run.py
+++ b/hveto/cli/hveto_run.py
@@ -566,7 +566,7 @@ def main():
         if not args.no_submit:
             logger.info(f'Submitting {dag_file} to Condor')
             batch_name = f'hveto {start_dt.strftime("%m/%d")} to {next_dt.strftime("%m/%d")} $(ClusterId)'
-            cmd = f'condor_submit_dag -import_env -batch-name "{batch_name}"  {dag_file}'
+            cmd = f'condor_submit_dag -import_env -batch-name \'{batch_name}\'  {dag_file}'
             logger.debug(f'Running: {cmd}')
             subprocess.run(cmd, shell=True, check=True)
         else:

From 10a6bfa4afb2e156d0828b2dd1cc3dea8fdd7442 Mon Sep 17 00:00:00 2001
From: "joseph.areeda" <joseph.areeda@ligo.org>
Date: Mon, 22 Sep 2025 07:55:09 -0700
Subject: [PATCH 25/40] dea iwth daily subdirectoree and latest symlink

---
  README_hveto_run.rst  | 18 ++++++++++++++++++
 hveto/cli/hveto_run.py | 23 ++++++++++++++++++++---
 2 files changed, 38 insertions(+), 3 deletions(-)
 create mode 100644  README_hveto_run.rst

diff --git a/ README_hveto_run.rst b/ README_hveto_run.rst
new file mode 100644
index 0000000..8926953
--- /dev/null
+++ b/ README_hveto_run.rst	
@@ -0,0 +1,18 @@
+==============================
+Scheduled runs and backfilling
+==============================
+
+At the LIGO Hanford and Livngston interferometer sites
+hveto is scheduled to run 4 times per day:
+
+- Every 8 hours covering from 00:00 UTC to 08:00, 16:00,
+  and 00:00 UTC the next day.
+- Once a day covering the previous 7 (UTC) days
+
+There are other situations where multiple runs are needed
+such as backfilling a time when problems prevented the
+normally scheduled run.
+
+`hveto_run` is a program used to create and [optionally]
+submit condor jobs to hveto.
+
diff --git a/hveto/cli/hveto_run.py b/hveto/cli/hveto_run.py
index 56e947b..026dd2b 100644
--- a/hveto/cli/hveto_run.py
+++ b/hveto/cli/hveto_run.py
@@ -81,7 +81,7 @@
     cmd="python -m hveto ${gpsstart} ${gpsend} --ifo ${ifo} --config-file ${configuration} --nproc ${nproc} \
         --output-directory ${outer_dir} ${omega_arg}"
 
-    ${condaRun} ${timeout_cmd} ${cmd}
+    ${condaRun} ${timeout_cmd} ${cmd} %create_link%
 
     ''')
 
@@ -483,12 +483,20 @@ def main():
         print('MAXJOBS LIMIT 3', file=dag_fh)
         print('RETRY ALL_NODES 2\n', file=dag_fh)
 
+        # For daily hveto or a 24 hr rerun we have day directory that may have 8, 16 and 24 hr subdirectories
+        # Then a symbolic link called latest pointing to the relevant results directory
+        job_day_dir = None
+
         if args.today:
             job_day, job_dt, duration = get_today_start()
             job_name = f'hveto_{job_day}_{duration:02d}hr'
+            job_start_gps = to_gps(job_dt)
+            job_end_gps = to_gps(job_dt + timedelta(hours=duration))
 
             #  create the results directory and add the condor submit file and bash script
-            job_dir = output_directory / job_day
+            job_day_dir = output_directory / job_day
+            job_dir = job_day_dir / f'{int(job_start_gps)}-{int(job_end_gps)}'
+
             condor_dir = job_dir / 'condor'
             if duration == 24:
                 job_end_dt = job_dt + timedelta(hours=duration)
@@ -508,6 +516,7 @@ def main():
                 "job_day": job_day,
                 "duration": duration,
                 "request_memory": 9216,
+                "create_link": f' && ln -s {job_dir} {job_day_dir}/latest'
             }
             job_args = {**job_args_common, **job_args_today}
             job_submit_file = make_job(job_name, job_args)
@@ -532,7 +541,10 @@ def main():
 
                 #  create the results directory and add the condor submit file and bash script
                 if duration == 1:
-                    job_dir = output_directory / job_day
+                    job_start_gps = to_gps(job_dt)
+                    job_end_gps = to_gps(job_dt + timedelta(hours=24))
+                    job_day_dir = output_directory / job_day
+                    job_dir = job_day_dir / f'{int(job_start_gps)}-{int(job_end_gps)}'
                 else:
                     job_dir = output_directory / job_month / job_day
 
@@ -549,6 +561,11 @@ def main():
                     "omega_scans": args.omega_scans,
                     "request_memory": 10240,
                 }
+                if duration == 1:
+                    job_args_multi_day["create_link"] = f' && ln -s {job_dir} {job_day_dir}/latest'
+                else:
+                    job_args_multi_day["create_link"] = ''
+
                 job_args = {**job_args_common, **job_args_multi_day}
                 job_submit_file = make_job(job_name, job_args)
 

From 3442850f9dbf3cc294f153307223680419b63820 Mon Sep 17 00:00:00 2001
From: "joseph.areeda" <joseph.areeda@ligo.org>
Date: Mon, 22 Sep 2025 08:14:49 -0700
Subject: [PATCH 26/40] dea iwth daily subdirectories and latest symlink

---
 hveto/cli/hveto_run.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/hveto/cli/hveto_run.py b/hveto/cli/hveto_run.py
index 026dd2b..ea5d9bf 100644
--- a/hveto/cli/hveto_run.py
+++ b/hveto/cli/hveto_run.py
@@ -541,8 +541,8 @@ def main():
 
                 #  create the results directory and add the condor submit file and bash script
                 if duration == 1:
-                    job_start_gps = to_gps(job_dt)
-                    job_end_gps = to_gps(job_dt + timedelta(hours=24))
+                    job_start_gps = to_gps(next_dt)
+                    job_end_gps = to_gps(next_dt + timedelta(hours=24))
                     job_day_dir = output_directory / job_day
                     job_dir = job_day_dir / f'{int(job_start_gps)}-{int(job_end_gps)}'
                 else:

From 513ec010c27587c958a0e0df0bd2092aee57122e Mon Sep 17 00:00:00 2001
From: "joseph.areeda" <joseph.areeda@ligo.org>
Date: Mon, 22 Sep 2025 08:41:58 -0700
Subject: [PATCH 27/40] fix script pre statement in dag

---
 hveto/cli/hveto_run.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hveto/cli/hveto_run.py b/hveto/cli/hveto_run.py
index ea5d9bf..5ba23c5 100644
--- a/hveto/cli/hveto_run.py
+++ b/hveto/cli/hveto_run.py
@@ -325,7 +325,7 @@ def process_omega_scans(job_args_in):
         dag_fh = job_args_in['dag_fh']
         job_day = job_args_in['job_day']
         print(f'SUBDAG EXTERNAL {job_name} "{omega_dag_file}"\n', file=dag_fh)
-        print(f'PRE SCRIPT {job_name} "{check_dag_file}" "{omega_dag_file}"', file=dag_fh)
+        print(f'SCRIPT PRE {job_name} "{check_dag_file}" "{omega_dag_file}"', file=dag_fh)
         print(f'PRE_SKIP {job_name} 12', file=dag_fh)
         print(f'PARENT {job_name} CHILD hveto_{job_day}_{duration_label}', file=dag_fh)
 

From e7cf6d436c4086e404b0c8452470b903f7b094d1 Mon Sep 17 00:00:00 2001
From: "joseph.areeda" <joseph.areeda@ligo.org>
Date: Mon, 22 Sep 2025 09:22:11 -0700
Subject: [PATCH 28/40] fixing omega scan dag messiness

---
 hveto/cli/hveto_run.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/hveto/cli/hveto_run.py b/hveto/cli/hveto_run.py
index 5ba23c5..08710d4 100644
--- a/hveto/cli/hveto_run.py
+++ b/hveto/cli/hveto_run.py
@@ -27,6 +27,7 @@
 from datetime import datetime, timedelta, timezone
 import socket
 
+from gwpy.detector.io.omega import omega_param
 from pytz import reference
 
 start_time = time.time()
@@ -310,24 +311,25 @@ def get_today_start():
 def process_omega_scans(job_args_in):
     if job_args_in['omega_scans'] > 0:
         duration_label = job_args_in['duration_label']
-        job_name = f'omega_{job_args_in["job_day"]}_{duration_label}'
+        job_name  = job_args_in['job_name']
+        omega_job_name = f'omega_{job_args_in["job_day"]}_{duration_label}'
         # Create a script to check whether omega scans DAG ha been made by hveto
         omega_dag_file = job_args_in["omega_dag_file"]
         omega_dag_file.parent.mkdir(parents=True, exist_ok=True)
         dag_log = job_args_in["dag_log"]
-        job_args_omega = {'omega_dag_file': omega_dag_file, 'log_file': dag_log, 'job_name': job_name}
+        job_args_omega = {'omega_dag_file': omega_dag_file, 'log_file': dag_log, 'job_name': omega_job_name}
         dag_dir = job_args_in['dag_dir']
-        check_dag_file = dag_dir / f'check_{job_name}.sh'
+        check_dag_file = dag_dir / f'check_{omega_job_name}.sh'
         job_args = {**job_args_in, **job_args_omega}
         check_dag_str = apply_symbols(check_dag_txt, job_args)
         check_dag_file.write_text(check_dag_str)
 
         dag_fh = job_args_in['dag_fh']
         job_day = job_args_in['job_day']
-        print(f'SUBDAG EXTERNAL {job_name} "{omega_dag_file}"\n', file=dag_fh)
-        print(f'SCRIPT PRE {job_name} "{check_dag_file}" "{omega_dag_file}"', file=dag_fh)
-        print(f'PRE_SKIP {job_name} 12', file=dag_fh)
-        print(f'PARENT {job_name} CHILD hveto_{job_day}_{duration_label}', file=dag_fh)
+        print(f'SUBDAG EXTERNAL {omega_job_name} "{omega_dag_file}"', file=dag_fh)
+        print(f'SCRIPT PRE {omega_job_name} "{check_dag_file}" "{omega_dag_file}"', file=dag_fh)
+        print(f'PRE_SKIP {omega_job_name} 12', file=dag_fh)
+        print(f'PARENT {job_name} CHILD {omega_job_name}', file=dag_fh)
 
 
 def main():
@@ -574,9 +576,9 @@ def main():
                 current_job += 1
 
                 if job_args['omega_scans'] > 0:
-
                     process_omega_scans(job_args)
 
+                print('---\n', file=dag_fh)
                 next_dt -= stride_dt
 
         logger.info(f'DAG file with {njobs} jobs written to {dag_file}')

From b7e369f983c1fce5fd9a43cf55638c48e62c1ffd Mon Sep 17 00:00:00 2001
From: "joseph.areeda" <joseph.areeda@ligo.org>
Date: Mon, 22 Sep 2025 09:32:53 -0700
Subject: [PATCH 29/40] fixing omega scan dag messiness again

---
 hveto/cli/hveto_run.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hveto/cli/hveto_run.py b/hveto/cli/hveto_run.py
index 08710d4..ff4036d 100644
--- a/hveto/cli/hveto_run.py
+++ b/hveto/cli/hveto_run.py
@@ -578,7 +578,7 @@ def main():
                 if job_args['omega_scans'] > 0:
                     process_omega_scans(job_args)
 
-                print('---\n', file=dag_fh)
+                print('# ---\n', file=dag_fh)
                 next_dt -= stride_dt
 
         logger.info(f'DAG file with {njobs} jobs written to {dag_file}')

From 58b05da4684ba17ac4eca6acdaded501a0915d9f Mon Sep 17 00:00:00 2001
From: "joseph.areeda" <joseph.areeda@ligo.org>
Date: Mon, 22 Sep 2025 10:00:03 -0700
Subject: [PATCH 30/40] fixing omega scan dag messiness again

---
 hveto/cli/hveto_run.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/hveto/cli/hveto_run.py b/hveto/cli/hveto_run.py
index ff4036d..780bcbe 100644
--- a/hveto/cli/hveto_run.py
+++ b/hveto/cli/hveto_run.py
@@ -323,6 +323,7 @@ def process_omega_scans(job_args_in):
         job_args = {**job_args_in, **job_args_omega}
         check_dag_str = apply_symbols(check_dag_txt, job_args)
         check_dag_file.write_text(check_dag_str)
+        check_dag_file.chmod(0o755)
 
         dag_fh = job_args_in['dag_fh']
         job_day = job_args_in['job_day']

From a9b6e7378fa5da646065eb9114a2f18bc1c85e33 Mon Sep 17 00:00:00 2001
From: "joseph.areeda" <joseph.areeda@ligo.org>
Date: Mon, 22 Sep 2025 19:26:45 -0700
Subject: [PATCH 31/40] workin on omega scan prevscript

---
 hveto/cli/hveto_run.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/hveto/cli/hveto_run.py b/hveto/cli/hveto_run.py
index 780bcbe..3fe8ff1 100644
--- a/hveto/cli/hveto_run.py
+++ b/hveto/cli/hveto_run.py
@@ -119,7 +119,8 @@
     #!/bin/bash
     # check whether the omega DAG file exists
     # Created by %program_name% version %program_version% on %date%
-
+    echo -n $(date) >>%log_file%
+    
     if [ -f %omega_dag_file% ]
     then
         echo "Omega DAG file %omega_dag_file% exists" >> %log_file%
@@ -328,7 +329,7 @@ def process_omega_scans(job_args_in):
         dag_fh = job_args_in['dag_fh']
         job_day = job_args_in['job_day']
         print(f'SUBDAG EXTERNAL {omega_job_name} "{omega_dag_file}"', file=dag_fh)
-        print(f'SCRIPT PRE {omega_job_name} "{check_dag_file}" "{omega_dag_file}"', file=dag_fh)
+        print(f'SCRIPT PRE {omega_job_name} {check_dag_file} {omega_dag_file}', file=dag_fh)
         print(f'PRE_SKIP {omega_job_name} 12', file=dag_fh)
         print(f'PARENT {job_name} CHILD {omega_job_name}', file=dag_fh)
 

From f3bccc0d1fba663de34bc2415e7815c216fcdb77 Mon Sep 17 00:00:00 2001
From: "joseph.areeda" <joseph.areeda@ligo.org>
Date: Tue, 23 Sep 2025 11:38:04 -0700
Subject: [PATCH 32/40] omega batch DAG uses relative paths so we set the
 initial working directory for the subdags

---
 hveto/cli/hveto_run.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/hveto/cli/hveto_run.py b/hveto/cli/hveto_run.py
index 3fe8ff1..cdda56b 100644
--- a/hveto/cli/hveto_run.py
+++ b/hveto/cli/hveto_run.py
@@ -328,7 +328,8 @@ def process_omega_scans(job_args_in):
 
         dag_fh = job_args_in['dag_fh']
         job_day = job_args_in['job_day']
-        print(f'SUBDAG EXTERNAL {omega_job_name} "{omega_dag_file}"', file=dag_fh)
+        job_dir: Path = job_args_in['job_dir']
+        print(f'\nSUBDAG EXTERNAL {omega_job_name} DIR "{job_dir.absolute()}" "{omega_dag_file}"\n', file=dag_fh)
         print(f'SCRIPT PRE {omega_job_name} {check_dag_file} {omega_dag_file}', file=dag_fh)
         print(f'PRE_SKIP {omega_job_name} 12', file=dag_fh)
         print(f'PARENT {job_name} CHILD {omega_job_name}', file=dag_fh)
@@ -571,6 +572,7 @@ def main():
                     job_args_multi_day["create_link"] = ''
 
                 job_args = {**job_args_common, **job_args_multi_day}
+                job_args['job_dir'] = job_dir
                 job_submit_file = make_job(job_name, job_args)
 
                 # add this job to the DAG

From 83f42b81c83093219ecbe3e272747ce43e7c65a3 Mon Sep 17 00:00:00 2001
From: "joseph.areeda" <joseph.areeda@ligo.org>
Date: Tue, 23 Sep 2025 12:16:15 -0700
Subject: [PATCH 33/40] omega batch DAG uses relative paths so we set the
 initial working directory for the subdags

---
 hveto/cli/hveto_run.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hveto/cli/hveto_run.py b/hveto/cli/hveto_run.py
index cdda56b..ab55f38 100644
--- a/hveto/cli/hveto_run.py
+++ b/hveto/cli/hveto_run.py
@@ -329,7 +329,7 @@ def process_omega_scans(job_args_in):
         dag_fh = job_args_in['dag_fh']
         job_day = job_args_in['job_day']
         job_dir: Path = job_args_in['job_dir']
-        print(f'\nSUBDAG EXTERNAL {omega_job_name} DIR "{job_dir.absolute()}" "{omega_dag_file}"\n', file=dag_fh)
+        print(f'\nSUBDAG EXTERNAL {omega_job_name} DIR {job_dir.absolute()} {omega_dag_file}\n', file=dag_fh)
         print(f'SCRIPT PRE {omega_job_name} {check_dag_file} {omega_dag_file}', file=dag_fh)
         print(f'PRE_SKIP {omega_job_name} 12', file=dag_fh)
         print(f'PARENT {job_name} CHILD {omega_job_name}', file=dag_fh)

From 856ac2201597d136bbe13609534be600905e4570 Mon Sep 17 00:00:00 2001
From: "joseph.areeda" <joseph.areeda@ligo.org>
Date: Tue, 23 Sep 2025 12:32:51 -0700
Subject: [PATCH 34/40] omega batch DAG uses relative paths so we set the
 initial working directory for the subdags, again

---
 hveto/cli/hveto_run.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hveto/cli/hveto_run.py b/hveto/cli/hveto_run.py
index ab55f38..2042170 100644
--- a/hveto/cli/hveto_run.py
+++ b/hveto/cli/hveto_run.py
@@ -329,7 +329,7 @@ def process_omega_scans(job_args_in):
         dag_fh = job_args_in['dag_fh']
         job_day = job_args_in['job_day']
         job_dir: Path = job_args_in['job_dir']
-        print(f'\nSUBDAG EXTERNAL {omega_job_name} DIR {job_dir.absolute()} {omega_dag_file}\n', file=dag_fh)
+        print(f'\nSUBDAG EXTERNAL {omega_job_name} {omega_dag_file} DIR {job_dir.absolute()}\n', file=dag_fh)
         print(f'SCRIPT PRE {omega_job_name} {check_dag_file} {omega_dag_file}', file=dag_fh)
         print(f'PRE_SKIP {omega_job_name} 12', file=dag_fh)
         print(f'PARENT {job_name} CHILD {omega_job_name}', file=dag_fh)

From f7ba5842dc329e8758a7206bbc06f024829ba834 Mon Sep 17 00:00:00 2001
From: "joseph.areeda" <joseph.areeda@ligo.org>
Date: Tue, 23 Sep 2025 15:13:07 -0700
Subject: [PATCH 35/40] omega batch DAG uses relative paths so we set the
 initial working directory for the subdags, again and again

---
 hveto/cli/hveto_run.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/hveto/cli/hveto_run.py b/hveto/cli/hveto_run.py
index 2042170..272c283 100644
--- a/hveto/cli/hveto_run.py
+++ b/hveto/cli/hveto_run.py
@@ -98,7 +98,7 @@
     executable = %exec%
 
     accounting_group = ligo.dev.o4.detchar.dqtriggers.hveto
-    accounting_group_user = joseph.areeda
+    accounting_group_user = %accounting_group_user%
 
     request_memory = %request_memory%M
     request_disk = 10GB
@@ -111,7 +111,7 @@
 
     use_oauth_services = scitokens
     environment = BEARER_TOKEN_FILE=$$(CondorScratchDir)/.condor_creds/scitokens.use
-
+    environment = ACCOUNTING_GROUP_USER=%accounting_group_user%
     queue
 ''')
 
@@ -229,6 +229,12 @@ def parser_add_args(parser):
                         help='Number of omega scans per round.'
                              ' Small numbers arerecommended for weekly runs because of the high number of rounds')
 
+    default_accounting_group_user = os.getenv('ACCOUNTING_GROUP_USER', os.getenv('USER'))
+    if default_accounting_group_user is None or 'detchar' in default_accounting_group_user:
+        default_accounting_group_user = 'joseph.areeda'
+    parser.add_argument('--accounting-group-user', default=default_accounting_group_user,
+                        help='User to use for accounting group. ')
+
 
 def apply_symbols(txt, symbols):
     """
@@ -472,6 +478,8 @@ def main():
         "date": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
         "request_disk": 100000,
         "timeout": 3 * 3600,
+        'accounting_group_user': args.accounting_group_user,
+
     }
 
     # create a DAG file for each day in the range

From b6786727358ab6a52a275ee91e3d642ba2198eb4 Mon Sep 17 00:00:00 2001
From: "joseph.areeda" <joseph.areeda@ligo.org>
Date: Wed, 24 Sep 2025 11:51:44 -0700
Subject: [PATCH 36/40] Fix dag_dir when all default parameter.

Update submit file templates to include Omega scans
---
 docs/Condor/hveto-daily.sub  |  2 +-
 docs/Condor/hveto-weekly.sub |  2 +-
 hveto/cli/hveto_run.py       | 37 +++++++++++++++++++-----------------
 3 files changed, 22 insertions(+), 19 deletions(-)

diff --git a/docs/Condor/hveto-daily.sub b/docs/Condor/hveto-daily.sub
index 7ce17b3..f331944 100644
--- a/docs/Condor/hveto-daily.sub
+++ b/docs/Condor/hveto-daily.sub
@@ -6,7 +6,7 @@
 universe = local
 
 executable = /cvmfs/software.igwn.org/conda/condabin/conda
-arguments = 'run --prefix /home/detchar.dev/.conda/envs/ligo-summary-3.11-test timeout 30m hveto-run -vv --today'
+arguments = 'run --prefix /home/detchar.dev/.conda/envs/ligo-summary-3.11-test timeout 30m hveto-run -vv --today --omega-scans 5'
 
 accounting_group = ligo.prod.o4.detchar.dqtriggers.hveto
 accounting_group_user = joseph.areeda
diff --git a/docs/Condor/hveto-weekly.sub b/docs/Condor/hveto-weekly.sub
index be6c141..16a91df 100644
--- a/docs/Condor/hveto-weekly.sub
+++ b/docs/Condor/hveto-weekly.sub
@@ -7,7 +7,7 @@
 universe = local
 
 executable = $(my.conda_exe)
-arguments = 'run --prefix $(my.conda_prefix)) timeout 30m hveto-run -vv'
+arguments = 'run --prefix $(my.conda_prefix)) timeout 30m hveto-run -vv --omega-scans 2'
 
 accounting_group = ligo.prod.o4.detchar.dqtriggers.hveto
 accounting_group_user = <user name valid for account>
diff --git a/hveto/cli/hveto_run.py b/hveto/cli/hveto_run.py
index 272c283..3b39a6f 100644
--- a/hveto/cli/hveto_run.py
+++ b/hveto/cli/hveto_run.py
@@ -27,7 +27,6 @@
 from datetime import datetime, timedelta, timezone
 import socket
 
-from gwpy.detector.io.omega import omega_param
 from pytz import reference
 
 start_time = time.time()
@@ -120,7 +119,7 @@
     # check whether the omega DAG file exists
     # Created by %program_name% version %program_version% on %date%
     echo -n $(date) >>%log_file%
-    
+
     if [ -f %omega_dag_file% ]
     then
         echo "Omega DAG file %omega_dag_file% exists" >> %log_file%
@@ -282,9 +281,9 @@ def make_job(job_name, job_args):
 
     job_submit = apply_symbols(hveto_job_submit, job_args)
     job_submit_file.write_text(job_submit)
-    logger.info(f'Job name {job_name} created in {job_condordir}')
-    logger.info(f'Created condor submit file {job_submit_file} ')
-    logger.info(f' script {job_run_sh}')
+    logger.info(f'Job name: "{job_name}" created in {job_condordir}')
+    logger.info(f'Created condor submit file: {job_submit_file} ')
+    logger.info(f' script: {job_run_sh}')
     return job_submit_file
 
 
@@ -318,7 +317,7 @@ def get_today_start():
 def process_omega_scans(job_args_in):
     if job_args_in['omega_scans'] > 0:
         duration_label = job_args_in['duration_label']
-        job_name  = job_args_in['job_name']
+        job_name = job_args_in['job_name']
         omega_job_name = f'omega_{job_args_in["job_day"]}_{duration_label}'
         # Create a script to check whether omega scans DAG ha been made by hveto
         omega_dag_file = job_args_in["omega_dag_file"]
@@ -333,7 +332,6 @@ def process_omega_scans(job_args_in):
         check_dag_file.chmod(0o755)
 
         dag_fh = job_args_in['dag_fh']
-        job_day = job_args_in['job_day']
         job_dir: Path = job_args_in['job_dir']
         print(f'\nSUBDAG EXTERNAL {omega_job_name} {omega_dag_file} DIR {job_dir.absolute()}\n', file=dag_fh)
         print(f'SCRIPT PRE {omega_job_name} {check_dag_file} {omega_dag_file}', file=dag_fh)
@@ -392,7 +390,7 @@ def main():
     dur_str = ''
     if args.output_directory_exact:
         output_directory = Path(args.output_directory_exact)
-    elif args.output_directory:
+    else:
         output_directory = Path(args.output_directory)
         if duration == 1 or args.today:
             dur_str = 'day'
@@ -412,9 +410,14 @@ def main():
         dag_dir = output_directory / job_day / 'condor'
     elif args.start == args.end:
         run_dt = datetime.strptime(args.start, '%Y-%m-%d')
-        # if we are only doing one run then we use the condor directory for the DAG file
-        dag_dir = output_directory / run_dt.strftime('%Y%m%d') / 'condor'
+        if duration == 1:
+            # hveto daily puts all days in a single directory
+            dag_dir = output_directory / run_dt.strftime('%Y%m%d') / 'condor'
+        else:
+            #  All other duration add another level of subdirectories, each containing one month
+            dag_dir = output_directory / run_dt.strftime('%Y%m') / run_dt.strftime('%Y%m%d') / 'condor'
     else:
+        # if we are back filling multiple days, regardless of duration the DAG that controls each day s in a separate directory
         dag_dir = output_directory / f'{__process_name__}-DAG' / now_month / f'DAG-{now_time}'
 
     dag_dir.mkdir(parents=True, exist_ok=True)
@@ -424,11 +427,11 @@ def main():
     dag_file.parent.mkdir(parents=True, exist_ok=True)
 
     # add a copy of our logging to a file
-    NOW = datetime.now()
-    TIMEZONE = reference.LocalTimezone().tzname(NOW)
-    DATEFMT = '%Y-%m-%d %H:%M:%S {}'.format(TIMEZONE)
-    FMT = '%(name)s %(asctime)s %(levelname)+8s: %(filename)s:%(lineno)d:  %(message)s'
-    logf_formatter = logging.Formatter(FMT, datefmt=DATEFMT)
+    now = datetime.now()
+    log_timezone = reference.LocalTimezone().tzname(now)
+    datefmt = '%Y-%m-%d %H:%M:%S {}'.format(log_timezone)
+    fmt = '%(name)s %(asctime)s %(levelname)+8s: %(filename)s:%(lineno)d:  %(message)s'
+    logf_formatter = logging.Formatter(fmt, datefmt=datefmt)
     logf_handler = logging.FileHandler(dag_log, mode='w')
     logf_handler.setFormatter(logf_formatter)
     logger.addHandler(logf_handler)
@@ -592,8 +595,8 @@ def main():
 
                 print('# ---\n', file=dag_fh)
                 next_dt -= stride_dt
-
-        logger.info(f'DAG file with {njobs} jobs written to {dag_file}')
+        job_is_plural = 's' if njobs > 1 else ''
+        logger.info(f'DAG file with {njobs} job{job_is_plural} written to {dag_file}')
         if not args.no_submit:
             logger.info(f'Submitting {dag_file} to Condor')
             batch_name = f'hveto {start_dt.strftime("%m/%d")} to {next_dt.strftime("%m/%d")} $(ClusterId)'

From c7adbe1339642830e815e6935e5b0872f0f5a1e6 Mon Sep 17 00:00:00 2001
From: "joseph.areeda" <joseph.areeda@ligo.org>
Date: Thu, 25 Sep 2025 05:41:26 -0700
Subject: [PATCH 37/40] set accounting user environment variable

---
 hveto/cli/hveto_run.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hveto/cli/hveto_run.py b/hveto/cli/hveto_run.py
index 3b39a6f..fea45d6 100644
--- a/hveto/cli/hveto_run.py
+++ b/hveto/cli/hveto_run.py
@@ -110,7 +110,7 @@
 
     use_oauth_services = scitokens
     environment = BEARER_TOKEN_FILE=$$(CondorScratchDir)/.condor_creds/scitokens.use
-    environment = ACCOUNTING_GROUP_USER=%accounting_group_user%
+    environment = _CONDOR_ACCOUNTING_USER=%accounting_group_user%
     queue
 ''')
 

From 56344ad26daea7f57ec1b9a96be4f1d752b89e2c Mon Sep 17 00:00:00 2001
From: "joseph.areeda" <joseph.areeda@ligo.org>
Date: Thu, 25 Sep 2025 06:28:52 -0700
Subject: [PATCH 38/40] Correct batch name for multiple runs

---
 hveto/cli/hveto_run.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/hveto/cli/hveto_run.py b/hveto/cli/hveto_run.py
index fea45d6..ba4808d 100644
--- a/hveto/cli/hveto_run.py
+++ b/hveto/cli/hveto_run.py
@@ -597,11 +597,11 @@ def main():
                 next_dt -= stride_dt
         job_is_plural = 's' if njobs > 1 else ''
         logger.info(f'DAG file with {njobs} job{job_is_plural} written to {dag_file}')
+        batch_name = f'hveto {start_dt.strftime("%m/%d")} to {end_dt.strftime("%m/%d")} $(ClusterId)'
+        cmd = f'condor_submit_dag -import_env -batch-name \'{batch_name}\'  {dag_file}'
+        logger.info(f'Condor submit command: {cmd}')
         if not args.no_submit:
             logger.info(f'Submitting {dag_file} to Condor')
-            batch_name = f'hveto {start_dt.strftime("%m/%d")} to {next_dt.strftime("%m/%d")} $(ClusterId)'
-            cmd = f'condor_submit_dag -import_env -batch-name \'{batch_name}\'  {dag_file}'
-            logger.debug(f'Running: {cmd}')
             subprocess.run(cmd, shell=True, check=True)
         else:
             logger.info('DAG file written but not submitted to Condor')

From 5d6cdc3eaa673a72ad1aed7130caa6dc58235ec5 Mon Sep 17 00:00:00 2001
From: "joseph.areeda" <joseph.areeda@ligo.org>
Date: Thu, 25 Sep 2025 07:23:03 -0700
Subject: [PATCH 39/40] Add duration to batch name

---
 hveto/cli/hveto_run.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/hveto/cli/hveto_run.py b/hveto/cli/hveto_run.py
index ba4808d..8253917 100644
--- a/hveto/cli/hveto_run.py
+++ b/hveto/cli/hveto_run.py
@@ -532,7 +532,8 @@ def main():
                 "job_day": job_day,
                 "duration": duration,
                 "request_memory": 9216,
-                "create_link": f' && ln -s {job_dir} {job_day_dir}/latest'
+                "create_link": f' && ln -s {job_dir} {job_day_dir}/latest',
+                'job_dir':job_dir,
             }
             job_args = {**job_args_common, **job_args_today}
             job_submit_file = make_job(job_name, job_args)
@@ -597,7 +598,9 @@ def main():
                 next_dt -= stride_dt
         job_is_plural = 's' if njobs > 1 else ''
         logger.info(f'DAG file with {njobs} job{job_is_plural} written to {dag_file}')
-        batch_name = f'hveto {start_dt.strftime("%m/%d")} to {end_dt.strftime("%m/%d")} $(ClusterId)'
+        dur_abv = f'{duration}'
+        dur_abv += 'h' if args.today else 'd'
+        batch_name = f'hveto {start_dt.strftime("%m/%d")} to {end_dt.strftime("%m/%d")} [{dur_abv}] $(ClusterId)'
         cmd = f'condor_submit_dag -import_env -batch-name \'{batch_name}\'  {dag_file}'
         logger.info(f'Condor submit command: {cmd}')
         if not args.no_submit:

From efe50e03b70c3fdbfda8d76c681f7d84c9e9017a Mon Sep 17 00:00:00 2001
From: "joseph.areeda" <joseph.areeda@ligo.org>
Date: Fri, 26 Sep 2025 05:39:01 -0700
Subject: [PATCH 40/40] Change --today's start, end to gps from date time
 becaus I couldn't figure out quoting

---
 hveto/cli/hveto_run.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/hveto/cli/hveto_run.py b/hveto/cli/hveto_run.py
index 8253917..cce55bb 100644
--- a/hveto/cli/hveto_run.py
+++ b/hveto/cli/hveto_run.py
@@ -520,8 +520,8 @@ def main():
             else:
                 job_end_str = f'{job_dt.strftime("%Y-%m-%d")} {duration:02d}:00:00'
             job_args_today = {
-                "gpsstart": f'{job_dt.strftime("%Y-%m-%d")}',
-                "gpsend": job_end_str,
+                "gpsstart": int(to_gps(f'{job_dt.strftime("%Y-%m-%d")}')),
+                "gpsend": int(to_gps(job_end_str)),
                 'duration_label': f'{duration:02d}_hr',
                 "outer_dir": job_dir,
                 "condor_dir": condor_dir,
@@ -533,7 +533,7 @@ def main():
                 "duration": duration,
                 "request_memory": 9216,
                 "create_link": f' && ln -s {job_dir} {job_day_dir}/latest',
-                'job_dir':job_dir,
+                'job_dir': job_dir,
             }
             job_args = {**job_args_common, **job_args_today}
             job_submit_file = make_job(job_name, job_args)