From 7bffbd6377788aba4a1b498da4bf28c2874d272b Mon Sep 17 00:00:00 2001 From: Eric Neilsen Date: Wed, 12 Nov 2025 14:35:19 -0800 Subject: [PATCH 1/4] Get scheduler config from new table in EFD --- schedview/app/scheduler_config_at_time.py | 16 +-- schedview/collect/efd.py | 104 +++++++++++++++++++- schedview/examples/app/combined_timeline.py | 14 ++- 3 files changed, 116 insertions(+), 18 deletions(-) diff --git a/schedview/app/scheduler_config_at_time.py b/schedview/app/scheduler_config_at_time.py index 633ef7ea..06a0e25a 100644 --- a/schedview/app/scheduler_config_at_time.py +++ b/schedview/app/scheduler_config_at_time.py @@ -1,11 +1,8 @@ import argparse -import asyncio from astropy.time import Time -import schedview.collect - -GITHUB_API_URL_BASE = "https://api.github.com/repos" +import schedview.collect.efd def scheduler_config_at_time_cli(): @@ -18,7 +15,8 @@ def scheduler_config_at_time_cli(): # Prepare argument parsing parser = argparse.ArgumentParser( prog="scheduler_config_at_time", - description="Get the scheduler configuration script path the observatory as of a given time.", + description="Get the scheduler configuration git ref and " + "script path the observatory as of a given time.", ) parser.add_argument( "what_scheduled", type=str, help="Which scheduler config (simonyi, maintel, ocs, or auxtel)." @@ -35,14 +33,10 @@ def scheduler_config_at_time_cli(): # Do the call time_cut = Time(args.datetime) - ts_config_ocs_version = schedview.collect.get_version_at_time("ts_config_ocs", time_cut) - loop = asyncio.get_event_loop() - scheduler_config = loop.run_until_complete( - schedview.collect.get_scheduler_config(ts_config_ocs_version, args.what_scheduled, time_cut) - ) + config_ref, config_file = schedview.collect.efd.get_scheduler_config(args.what_scheduled, time_cut) - print(scheduler_config) + print(config_ref, config_file) if __name__ == "__main__": diff --git a/schedview/collect/efd.py b/schedview/collect/efd.py index db8306df..be512f1b 100644 --- a/schedview/collect/efd.py +++ b/schedview/collect/efd.py @@ -1,9 +1,11 @@ import asyncio +import re import threading from collections import defaultdict from collections.abc import Iterable from functools import cache, partial -from typing import Literal +from pathlib import Path +from typing import Literal, Optional import pandas as pd import requests @@ -15,7 +17,7 @@ from schedview.dayobs import DayObs EfdDatabase = Literal["efd", "lsst.obsenv"] -ScheduledThing = Literal["maintel", "ocs", "auxtel"] +ScheduledThing = Literal["simonyi", "lsstcam", "maintel", "latiss", "auxtel"] SAL_INDEX_GUESSES = defaultdict( partial([[]].__getitem__, 0), @@ -314,7 +316,7 @@ def get_version_at_time(item: str, time_cut: Time | None = None, max_age: TimeDe return version -async def get_scheduler_config( +async def old_get_scheduler_config( ts_config_ocs_version: str, what_scheduled: ScheduledThing, time_cut: Time | None = None ) -> str: """Get the relative path of the scheduler configuration script. @@ -391,3 +393,99 @@ async def get_scheduler_config( scheduler_config.find("/ts_config_ocs/Scheduler/feature_scheduler") + 1 : ] return relative_scheduler_config + + +def get_scheduler_config(what_scheduled, time_cut: Optional[Time] = None) -> tuple[str, str]: + """Retrieve the Git reference and path for the scheduler configuration used + at a given time. + + Parameters + ---------- + what_scheduled : str + Identifier of the scheduled component (e.g. ``"maintel"``, + ``"auxtel"``, ``"simonyi"``, ``"latiss"``, ``"lsstcam"``). + time_cut : astropy.time.Time, optional + Timestamp at which to query the configuration. + If ``None`` (default), the current time is used. + + Returns + ------- + git_reference: str + The Git reference (commit hash, tag, or branch name) that + corresponds to the version recorded in the EFD for the requested + component. + config_path: str + A relative path to the scheduler configuration file within the + ``ts_config_scheduler`` repository. + + Raises + ------ + ValueError + If the recorded version cannot be matched to a Git reference, or if the + configuration record is missing required fields. + """ + if time_cut is None: + time_cut = Time.now() + assert isinstance(time_cut, Time) + + # Query the EFD for the version recorded + raw_version, configurations, url = sync_query_latest_in_efd_topic( + topic="lsst.sal.Scheduler.logevent_configurationApplied", + num_records=1, + db_name="efd", + fields="version, configurations, url", + time_cut=time_cut, + sal_indexes=SAL_INDEX_GUESSES[what_scheduled.lower()], + ).iloc[0] + + if not isinstance(raw_version, str): + raise ValueError("Unexpected type for version of ts_config_scheduler in EFD") + + assert isinstance(raw_version, str) + + # Initialize git_reference to None to mean we haven't found a ref + # for the returned version (yet). + git_reference: str | None = None + + # The versions seem to usually be a git describe style string that looks + # something like this: heads/temp-0-gb94a182 + # implying that it's a commit on a branch called "temp" + # This branch isn't available on github, but it looks like commits + # that match the implied hash (the 7 characters at the end of + # the string, after the "g") are sometimes (but not always) there. + + # Begin by testing to see if it is of that form, and if it is, extract + # the hash: + match = re.search(r"^heads/.*-g([0-9a-fA-F]+)$", raw_version) + if match is not None and len(match.group(1)) >= 6: + commit_hash = match.group(1) + + # The hash sometimes seems to be available on github, and if it is, + # we can use it as our git reference. So, check whether it exists: + commit_url = f"https://api.github.com/repos/lsst-ts/ts_config_scheduler/commits/{commit_hash}" + commit_exists = requests.get(commit_url).status_code == 200 + if commit_exists: + git_reference = commit_hash + + if git_reference is None: + # If the version is a git reference (raw hash, tag, or branch) + # available on github, return it "as is": + for ref_type in ("commits", "git/ref", "git/ref/tags", "git/ref/heads"): + ref_url = f"https://api.github.com/repos/lsst-ts/ts_config_scheduler/{ref_type}/{raw_version}" + ref_exists = requests.get(ref_url).status_code == 200 + if ref_exists: + git_reference = raw_version + break + + if git_reference is None: + raise ValueError(f"Recorded ts_config_scheduler version {raw_version} not available on github") + + assert isinstance(git_reference, str) + + # Now construct the path to the configuration + # file within the repository. + config_file = configurations.split(",")[-1] + config_path_parts = url.split("/") + [config_file] + config_path = Path(*config_path_parts[config_path_parts.index("ts_config_scheduler") :]).as_posix() + + return git_reference, config_path diff --git a/schedview/examples/app/combined_timeline.py b/schedview/examples/app/combined_timeline.py index 4d6e25b6..f0a09463 100644 --- a/schedview/examples/app/combined_timeline.py +++ b/schedview/examples/app/combined_timeline.py @@ -118,16 +118,22 @@ def update_events(self): ts_config_ocs_version = schedview.collect.efd.get_version_at_time("ts_config_ocs", obs_start_time) sal_indexes = schedview.collect.efd.SAL_INDEX_GUESSES[visit_origin] - opsim_config_script = self._run_async_io( - schedview.collect.get_scheduler_config(ts_config_ocs_version, telescope, obs_start_time) - ) + try: + config_scheduler_ref, scheduler_config_script = schedview.collect.efd.get_scheduler_config( + telescope, obs_start_time + ) + except ValueError: + config_scheduler_ref = "unknown" + scheduler_config_script = "unknown" + completed_visits["filter"] = completed_visits["band"] completed_visits["sim_date"] = None completed_visits["sim_index"] = 0 completed_visits["label"] = "Completed" + completed_visits["config_scheduler_ref"] = config_scheduler_ref + completed_visits["scheduler_config_script"] = scheduler_config_script completed_visits["opsim_config_branch"] = ts_config_ocs_version completed_visits["opsim_config_repository"] = None - completed_visits["opsim_config_script"] = opsim_config_script completed_visits["scheduler_version"] = schedview.collect.efd.get_version_at_time( "rubin_scheduler", obs_start_time ) From 4f7985de4d7f698031283ef1568964278fe01917 Mon Sep 17 00:00:00 2001 From: Eric Neilsen Date: Thu, 13 Nov 2025 08:28:18 -0800 Subject: [PATCH 2/4] Fix combined_timeline sample app to work with latest prenights --- schedview/examples/app/combined_timeline.py | 32 +++++++++++++++------ 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/schedview/examples/app/combined_timeline.py b/schedview/examples/app/combined_timeline.py index f0a09463..fec5fb17 100644 --- a/schedview/examples/app/combined_timeline.py +++ b/schedview/examples/app/combined_timeline.py @@ -3,6 +3,7 @@ from collections import defaultdict from datetime import date from typing import DefaultDict +from uuid import UUID import astropy.time import astropy.utils.iers @@ -29,7 +30,7 @@ class CombinedTimelineDashboard(param.Parameterized): # Parameters set in the UI evening_date = param.Date( - default=date(2025, 4, 27), + default=date(2025, 11, 8), label="Date", doc="Day obs (local calendar date of sunset for the night)", ) @@ -41,6 +42,8 @@ class CombinedTimelineDashboard(param.Parameterized): day_obs = param.Parameter() events = param.Parameter() + show_prenights = False + def _run_async_io(self, io_coroutine): # Run the async io (needed for the EFD) in a separate thread and # block while waiting for it to finish. @@ -139,13 +142,26 @@ def update_events(self): ) completed_visits["sim_runner_kwargs"] = {} - simulated_visits = schedview.collect.multisim.read_multiple_prenights( - day_obs.date, - day_obs.mjd, - stackers=schedview.collect.visits.NIGHT_STACKERS, - telescope=telescope, - ).query(f'sim_date == "{day_obs}"') - visits = pd.concat([completed_visits, simulated_visits]) + if self.show_prenights: + simulated_visits = schedview.collect.multisim.read_multiple_prenights( + day_obs.date, + day_obs.mjd, + stackers=schedview.collect.visits.NIGHT_STACKERS, + telescope=telescope.lower(), + ) + # Bokeh gets confused by columns of type UUID, so convert + # them to strings on loading. + for column_name in simulated_visits.columns: + if isinstance(simulated_visits[column_name].iloc[0], UUID): + simulated_visits[column_name] = simulated_visits[column_name].astype(str) + + # If there are no prenights, the stacker does not run, so skip + # trying to filter on the right day_obs. + if "day_obs_iso8601" in simulated_visits.columns: + simulated_visits.query(f'day_obs_iso8601 == "{day_obs}"', inplace=True) + visits = pd.concat([completed_visits, simulated_visits]) + else: + visits = completed_visits events["visits"] = visits self.events = events From 70104d12fdaa77c52ee7913f5013cd921312dc0e Mon Sep 17 00:00:00 2001 From: Eric Neilsen Date: Thu, 13 Nov 2025 08:51:19 -0800 Subject: [PATCH 3/4] Remove old broken implementation of get_schedule_config --- schedview/collect/efd.py | 80 ---------------------------------------- 1 file changed, 80 deletions(-) diff --git a/schedview/collect/efd.py b/schedview/collect/efd.py index be512f1b..eab19dd5 100644 --- a/schedview/collect/efd.py +++ b/schedview/collect/efd.py @@ -9,7 +9,6 @@ import pandas as pd import requests -import yaml from astropy.time import Time, TimeDelta from lsst_efd_client import EfdClient @@ -316,85 +315,6 @@ def get_version_at_time(item: str, time_cut: Time | None = None, max_age: TimeDe return version -async def old_get_scheduler_config( - ts_config_ocs_version: str, what_scheduled: ScheduledThing, time_cut: Time | None = None -) -> str: - """Get the relative path of the scheduler configuration script. - - Parameters - ---------- - ts_config_ocs_version : `str` - The version of ts_config_ocs from which to fetch the configuration. - Must be a valid string representing the git branch or tag. - what_scheduled: `str` - What it is being scheduled. - Can be `simonyi`, `maintel`, `ocs`, or `auxtel`. - (`simonyi` and `maintel` are synonymous.) - time_cut : `astropy.time.Time` or `None` - The time cut for the query. Default is None, which - uses the current time. - - Returns - ------- - config_script_path : `str` - The relative path to the scheduler configuration script. - - """ - if what_scheduled.lower() in ("auxtel", "latiss"): - what_scheduled = "auxtel" - else: - what_scheduled = "maintel" - - latest_config_df = await query_latest_in_efd_topic( - topic="lsst.sal.Scheduler.logevent_configurationApplied", - fields=["SchedulerId", "configurations", "salIndex", "schemaVersion", "url", "version"], - sal_indexes=SAL_INDEX_GUESSES[what_scheduled.lower()], - time_cut=time_cut, - num_records=1, - ) - - # Find the name of the yaml configuration file for the FBS - # This is typically saved in the last comma-separated values in the - # "configurations" field from - # lsst.sal.Scheduler.logevent_configurationApplied - config_fname = latest_config_df["configurations"].iloc[0].split(",")[-1] - - # schema_version tracks the directory where the above yaml - # configuration file should live inside of ts_config_ocs/Scheduler - schema_version = latest_config_df["schemaVersion"].iloc[0] - - scheduler_config_ocs_url = "/".join( - [ - "https://raw.githubusercontent.com", - "lsst-ts", - "ts_config_ocs", - ts_config_ocs_version, - "Scheduler", - schema_version, - config_fname, - ] - ) - - response = requests.get(scheduler_config_ocs_url, allow_redirects=True) - scheduler_config_ocs = yaml.safe_load(response.content.decode("utf-8")) - - # Find the path to actual python configuration file - # for the FBS referenced in the yaml file - scheduler_config = scheduler_config_ocs[what_scheduled.lower()]["feature_scheduler_driver_configuration"][ - "scheduler_config" - ] - - # Find the path to the FBS python configuration, - # relative to ts_config_ocs/Scheduler/feature_scheduler - # The "+1" skips the initial "/", resulting in a relative path, - # while still only matching when ts_config_ocs is the full - # name of the higest-level named directory. - relative_scheduler_config = scheduler_config[ - scheduler_config.find("/ts_config_ocs/Scheduler/feature_scheduler") + 1 : - ] - return relative_scheduler_config - - def get_scheduler_config(what_scheduled, time_cut: Optional[Time] = None) -> tuple[str, str]: """Retrieve the Git reference and path for the scheduler configuration used at a given time. From 5c920a55c98a98ce12c8d289951bdc0417b62e84 Mon Sep 17 00:00:00 2001 From: Eric Neilsen Date: Thu, 13 Nov 2025 14:17:41 -0800 Subject: [PATCH 4/4] Better support cut-and-paste from docs --- docs/reports.rst | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/docs/reports.rst b/docs/reports.rst index a08dadb3..b45440f1 100644 --- a/docs/reports.rst +++ b/docs/reports.rst @@ -123,7 +123,7 @@ Replace the symlink to point to your new one:: if test -L /sdf/data/rubin/shared/scheduler/packages/schedview_notebooks ; then rm /sdf/data/rubin/shared/scheduler/packages/schedview_notebooks fi - ln -s /sdf/data/rubin/shared/scheduler/packages/schedview_notebooks-v0.1.0.dev2 /sdf/data/rubin/shared/scheduler/packages/schedview_notebooks + ln -s /sdf/data/rubin/shared/scheduler/packages/schedview_notebooks-${NEWTAG} /sdf/data/rubin/shared/scheduler/packages/schedview_notebooks Updating other software used by the jobs ---------------------------------------- @@ -132,7 +132,9 @@ Begin by determining the next available tag. Get sorted existing tags with:: - curl -s https://api.github.com/repos/lsst/schedview/tags \ + MODULENAME="schedview" + MODULEUSER="lsst" + curl -s https://api.github.com/repos/${MODULEUSER}/${MODULENAME}/tags \ | jq -r '.[].name' \ | egrep '^v[0-9]+.[0-9]+.[0-9]+.*$' \ | sort -V @@ -144,21 +146,26 @@ Make and push a new tag (with the base of the repository as the current working echo "New version is ${NEWVERSION} with tag ${NEWTAG}" echo "" git tag ${NEWTAG} - git push origin tag v${NEWTAG} + git push origin tag ${NEWTAG} Then install it in `/sdf/data/rubin/shared/scheduler/packages`:: + PACKAGEDIR="/sdf/data/rubin/shared/scheduler/packages" + TARGETDIR="${PACKAGEDIR}/${MODULENAME}-${NEWVERSION}" + PIPORIGIN="git+https://github.com/${MODULEUSER}/${MODULENAME}.git@${NEWTAG}" + echo "Installing from ${PIPORIGIN} to ${TARGETDIR}" + echo "" pip install \ --no-deps \ - --target=/sdf/data/rubin/shared/scheduler/packages/schedview-${NEWVERSION} \ - git+https://github.com/lsst/schedview.git@${NEWTAG} + --target=${TARGETDIR} \ + ${PIPORIGIN} Replace the symlink to point to your new version:: - if test -L /sdf/data/rubin/shared/scheduler/packages/schedview ; then - rm /sdf/data/rubin/shared/scheduler/packages/schedview + if test -L ${PACKAGEDIR}/${MODULENAME} ; then + rm ${PACKAGEDIR}/${MODULENAME} fi - ln -s /sdf/data/rubin/shared/scheduler/packages/schedview-${NEWVERSION} \ /sdf/data/rubin/shared/scheduler/packages/schedview + ln -s ${TARGETDIR} ${PACKAGEDIR}/${MODULENAME} Hand generation of reports