tickets/SP-2738: Replace schedview.collect.efd.get_scheduler_config with an implementation that gets it from its new location

docs/reports.rst

@@ -123,7 +123,7 @@ Replace the symlink to point to your new one::
     if test -L /sdf/data/rubin/shared/scheduler/packages/schedview_notebooks ; then
       rm /sdf/data/rubin/shared/scheduler/packages/schedview_notebooks
     fi
-    ln -s /sdf/data/rubin/shared/scheduler/packages/schedview_notebooks-v0.1.0.dev2 /sdf/data/rubin/shared/scheduler/packages/schedview_notebooks
+    ln -s /sdf/data/rubin/shared/scheduler/packages/schedview_notebooks-${NEWTAG} /sdf/data/rubin/shared/scheduler/packages/schedview_notebooks
 
 Updating other software used by the jobs
 ----------------------------------------
@@ -132,7 +132,9 @@ Begin by determining the next available tag.
 
 Get sorted existing tags with::
 
-    curl -s https://api.github.com/repos/lsst/schedview/tags \
+    MODULENAME="schedview"
+    MODULEUSER="lsst"
+    curl -s https://api.github.com/repos/${MODULEUSER}/${MODULENAME}/tags \
         | jq -r '.[].name' \
         | egrep '^v[0-9]+.[0-9]+.[0-9]+.*$' \
         | sort -V
@@ -144,21 +146,26 @@ Make and push a new tag (with the base of the repository as the current working
     echo "New version is ${NEWVERSION} with tag ${NEWTAG}"
     echo ""
     git tag ${NEWTAG}
-    git push origin tag v${NEWTAG}
+    git push origin tag ${NEWTAG}
 
 Then install it in `/sdf/data/rubin/shared/scheduler/packages`::
 
+    PACKAGEDIR="/sdf/data/rubin/shared/scheduler/packages"
+    TARGETDIR="${PACKAGEDIR}/${MODULENAME}-${NEWVERSION}"
+    PIPORIGIN="git+https://github.com/${MODULEUSER}/${MODULENAME}.git@${NEWTAG}"
+    echo "Installing from ${PIPORIGIN} to ${TARGETDIR}"
+    echo ""
     pip install \
         --no-deps \
-        --target=/sdf/data/rubin/shared/scheduler/packages/schedview-${NEWVERSION} \
-        git+https://github.com/lsst/schedview.git@${NEWTAG}
+        --target=${TARGETDIR} \
+        ${PIPORIGIN}
 
 Replace the symlink to point to your new version::
 
-    if test -L /sdf/data/rubin/shared/scheduler/packages/schedview ; then
-      rm /sdf/data/rubin/shared/scheduler/packages/schedview
+    if test -L ${PACKAGEDIR}/${MODULENAME} ; then
+      rm ${PACKAGEDIR}/${MODULENAME}
     fi
-    ln -s /sdf/data/rubin/shared/scheduler/packages/schedview-${NEWVERSION} \ /sdf/data/rubin/shared/scheduler/packages/schedview
+    ln -s ${TARGETDIR} ${PACKAGEDIR}/${MODULENAME}
 
 
 Hand generation of reports

schedview/app/scheduler_config_at_time.py

@@ -1,11 +1,8 @@
 import argparse
-import asyncio
 
 from astropy.time import Time
 
-import schedview.collect
-
-GITHUB_API_URL_BASE = "https://api.github.com/repos"
+import schedview.collect.efd
 
 
 def scheduler_config_at_time_cli():
@@ -18,7 +15,8 @@ def scheduler_config_at_time_cli():
     # Prepare argument parsing
     parser = argparse.ArgumentParser(
         prog="scheduler_config_at_time",
-        description="Get the scheduler configuration script path the observatory as of a given time.",
+        description="Get the scheduler configuration git ref and "
+        "script path the observatory as of a given time.",
     )
     parser.add_argument(
         "what_scheduled", type=str, help="Which scheduler config (simonyi, maintel, ocs, or auxtel)."
@@ -35,14 +33,10 @@ def scheduler_config_at_time_cli():
 
     # Do the call
     time_cut = Time(args.datetime)
-    ts_config_ocs_version = schedview.collect.get_version_at_time("ts_config_ocs", time_cut)
 
-    loop = asyncio.get_event_loop()
-    scheduler_config = loop.run_until_complete(
-        schedview.collect.get_scheduler_config(ts_config_ocs_version, args.what_scheduled, time_cut)
-    )
+    config_ref, config_file = schedview.collect.efd.get_scheduler_config(args.what_scheduled, time_cut)
 
-    print(scheduler_config)
+    print(config_ref, config_file)
 
 
 if __name__ == "__main__":

schedview/collect/efd.py

@@ -1,21 +1,22 @@
 import asyncio
+import re
 import threading
 from collections import defaultdict
 from collections.abc import Iterable
 from functools import cache, partial
-from typing import Literal
+from pathlib import Path
+from typing import Literal, Optional
 
 import pandas as pd
 import requests
-import yaml
 from astropy.time import Time, TimeDelta
 from lsst_efd_client import EfdClient
 
 import schedview.clientsite
 from schedview.dayobs import DayObs
 
 EfdDatabase = Literal["efd", "lsst.obsenv"]
-ScheduledThing = Literal["maintel", "ocs", "auxtel"]
+ScheduledThing = Literal["simonyi", "lsstcam", "maintel", "latiss", "auxtel"]
 
 SAL_INDEX_GUESSES = defaultdict(
     partial([[]].__getitem__, 0),
@@ -314,80 +315,97 @@ def get_version_at_time(item: str, time_cut: Time | None = None, max_age: TimeDe
     return version
 
 
-async def get_scheduler_config(
-    ts_config_ocs_version: str, what_scheduled: ScheduledThing, time_cut: Time | None = None
-) -> str:
-    """Get the relative path of the scheduler configuration script.
+def get_scheduler_config(what_scheduled, time_cut: Optional[Time] = None) -> tuple[str, str]:
+    """Retrieve the Git reference and path for the scheduler configuration used
+    at a given time.
 
     Parameters
     ----------
-    ts_config_ocs_version : `str`
-        The version of ts_config_ocs from which to fetch the configuration.
-        Must be a valid string representing the git branch or tag.
-    what_scheduled: `str`
-        What it is being scheduled.
-        Can be `simonyi`, `maintel`, `ocs`, or `auxtel`.
-        (`simonyi` and `maintel` are synonymous.)
-    time_cut : `astropy.time.Time` or `None`
-        The time cut for the query. Default is None, which
-        uses the current time.
+    what_scheduled : str
+        Identifier of the scheduled component (e.g. ``"maintel"``,
+        ``"auxtel"``, ``"simonyi"``, ``"latiss"``, ``"lsstcam"``).
+    time_cut : astropy.time.Time, optional
+        Timestamp at which to query the configuration.
+        If ``None`` (default), the current time is used.
 
     Returns
     -------
-    config_script_path : `str`
-        The relative path to the scheduler configuration script.
-
+    git_reference: str
+        The Git reference (commit hash, tag, or branch name) that
+        corresponds to the version recorded in the EFD for the requested
+        component.
+    config_path: str
+        A relative path to the scheduler configuration file within the
+        ``ts_config_scheduler`` repository.
+
+    Raises
+    ------
+    ValueError
+        If the recorded version cannot be matched to a Git reference, or if the
+        configuration record is missing required fields.
     """
-    if what_scheduled.lower() in ("auxtel", "latiss"):
-        what_scheduled = "auxtel"
-    else:
-        what_scheduled = "maintel"
+    if time_cut is None:
+        time_cut = Time.now()
+    assert isinstance(time_cut, Time)
 
-    latest_config_df = await query_latest_in_efd_topic(
+    # Query the EFD for the version recorded
+    raw_version, configurations, url = sync_query_latest_in_efd_topic(
         topic="lsst.sal.Scheduler.logevent_configurationApplied",
-        fields=["SchedulerId", "configurations", "salIndex", "schemaVersion", "url", "version"],
-        sal_indexes=SAL_INDEX_GUESSES[what_scheduled.lower()],
-        time_cut=time_cut,
         num_records=1,
-    )
-
-    # Find the name of the yaml configuration file for the FBS
-    # This is typically saved in the last comma-separated values in the
-    # "configurations" field from
-    # lsst.sal.Scheduler.logevent_configurationApplied
-    config_fname = latest_config_df["configurations"].iloc[0].split(",")[-1]
-
-    # schema_version tracks the directory where the above yaml
-    # configuration file should live inside of ts_config_ocs/Scheduler
-    schema_version = latest_config_df["schemaVersion"].iloc[0]
-
-    scheduler_config_ocs_url = "/".join(
-        [
-            "https://raw.githubusercontent.com",
-            "lsst-ts",
-            "ts_config_ocs",
-            ts_config_ocs_version,
-            "Scheduler",
-            schema_version,
-            config_fname,
-        ]
-    )
-
-    response = requests.get(scheduler_config_ocs_url, allow_redirects=True)
-    scheduler_config_ocs = yaml.safe_load(response.content.decode("utf-8"))
-
-    # Find the path to actual python configuration file
-    # for the FBS referenced in the yaml file
-    scheduler_config = scheduler_config_ocs[what_scheduled.lower()]["feature_scheduler_driver_configuration"][
-        "scheduler_config"
-    ]
-
-    # Find the path to the FBS python configuration,
-    # relative to ts_config_ocs/Scheduler/feature_scheduler
-    # The "+1" skips the initial "/", resulting in a relative path,
-    # while still only matching when ts_config_ocs is the full
-    # name of the higest-level named directory.
-    relative_scheduler_config = scheduler_config[
-        scheduler_config.find("/ts_config_ocs/Scheduler/feature_scheduler") + 1 :
-    ]
-    return relative_scheduler_config
+        db_name="efd",
+        fields="version, configurations, url",
+        time_cut=time_cut,
+        sal_indexes=SAL_INDEX_GUESSES[what_scheduled.lower()],
+    ).iloc[0]
+
+    if not isinstance(raw_version, str):
+        raise ValueError("Unexpected type for version of ts_config_scheduler in EFD")
+
+    assert isinstance(raw_version, str)
+
+    # Initialize git_reference to None to mean we haven't found a ref
+    # for the returned version (yet).
+    git_reference: str | None = None
+
+    # The versions seem to usually be a git describe style string that looks
+    # something like this: heads/temp-0-gb94a182
+    # implying that it's a commit on a branch called "temp"
+    # This branch isn't available on github, but it looks like commits
+    # that match the implied hash (the 7 characters at the end of
+    # the string, after the "g") are sometimes (but not always) there.
+
+    # Begin by testing to see if it is of that form, and if it is, extract
+    # the hash:
+    match = re.search(r"^heads/.*-g([0-9a-fA-F]+)$", raw_version)
+    if match is not None and len(match.group(1)) >= 6:
+        commit_hash = match.group(1)
+
+        # The hash sometimes seems to be available on github, and if it is,
+        # we can use it as our git reference. So, check whether it exists:
+        commit_url = f"https://api.github.com/repos/lsst-ts/ts_config_scheduler/commits/{commit_hash}"
+        commit_exists = requests.get(commit_url).status_code == 200
+        if commit_exists:
+            git_reference = commit_hash
+
+    if git_reference is None:
+        # If the version is a git reference (raw hash, tag, or branch)
+        # available on github, return it "as is":
+        for ref_type in ("commits", "git/ref", "git/ref/tags", "git/ref/heads"):
+            ref_url = f"https://api.github.com/repos/lsst-ts/ts_config_scheduler/{ref_type}/{raw_version}"
+            ref_exists = requests.get(ref_url).status_code == 200
+            if ref_exists:
+                git_reference = raw_version
+                break
+
+    if git_reference is None:
+        raise ValueError(f"Recorded ts_config_scheduler version {raw_version} not available on github")
+
+    assert isinstance(git_reference, str)
+
+    # Now construct the path to the configuration
+    # file within the repository.
+    config_file = configurations.split(",")[-1]
+    config_path_parts = url.split("/") + [config_file]
+    config_path = Path(*config_path_parts[config_path_parts.index("ts_config_scheduler") :]).as_posix()
+
+    return git_reference, config_path

schedview/examples/app/combined_timeline.py

@@ -3,6 +3,7 @@
 from collections import defaultdict
 from datetime import date
 from typing import DefaultDict
+from uuid import UUID
 
 import astropy.time
 import astropy.utils.iers
@@ -29,7 +30,7 @@ class CombinedTimelineDashboard(param.Parameterized):
 
     # Parameters set in the UI
     evening_date = param.Date(
-        default=date(2025, 4, 27),
+        default=date(2025, 11, 8),
         label="Date",
         doc="Day obs (local calendar date of sunset for the night)",
     )
@@ -41,6 +42,8 @@ class CombinedTimelineDashboard(param.Parameterized):
     day_obs = param.Parameter()
     events = param.Parameter()
 
+    show_prenights = False
+
     def _run_async_io(self, io_coroutine):
         # Run the async io (needed for the EFD) in a separate thread and
         # block while waiting for it to finish.
@@ -118,28 +121,47 @@ def update_events(self):
 
             ts_config_ocs_version = schedview.collect.efd.get_version_at_time("ts_config_ocs", obs_start_time)
             sal_indexes = schedview.collect.efd.SAL_INDEX_GUESSES[visit_origin]
-            opsim_config_script = self._run_async_io(
-                schedview.collect.get_scheduler_config(ts_config_ocs_version, telescope, obs_start_time)
-            )
+            try:
+                config_scheduler_ref, scheduler_config_script = schedview.collect.efd.get_scheduler_config(
+                    telescope, obs_start_time
+                )
+            except ValueError:
+                config_scheduler_ref = "unknown"
+                scheduler_config_script = "unknown"
+
             completed_visits["filter"] = completed_visits["band"]
             completed_visits["sim_date"] = None
             completed_visits["sim_index"] = 0
             completed_visits["label"] = "Completed"
+            completed_visits["config_scheduler_ref"] = config_scheduler_ref
+            completed_visits["scheduler_config_script"] = scheduler_config_script
             completed_visits["opsim_config_branch"] = ts_config_ocs_version
             completed_visits["opsim_config_repository"] = None
-            completed_visits["opsim_config_script"] = opsim_config_script
             completed_visits["scheduler_version"] = schedview.collect.efd.get_version_at_time(
                 "rubin_scheduler", obs_start_time
             )
             completed_visits["sim_runner_kwargs"] = {}
 
-        simulated_visits = schedview.collect.multisim.read_multiple_prenights(
-            day_obs.date,
-            day_obs.mjd,
-            stackers=schedview.collect.visits.NIGHT_STACKERS,
-            telescope=telescope,
-        ).query(f'sim_date == "{day_obs}"')
-        visits = pd.concat([completed_visits, simulated_visits])
+        if self.show_prenights:
+            simulated_visits = schedview.collect.multisim.read_multiple_prenights(
+                day_obs.date,
+                day_obs.mjd,
+                stackers=schedview.collect.visits.NIGHT_STACKERS,
+                telescope=telescope.lower(),
+            )
+            # Bokeh gets confused by columns of type UUID, so convert
+            # them to strings on loading.
+            for column_name in simulated_visits.columns:
+                if isinstance(simulated_visits[column_name].iloc[0], UUID):
+                    simulated_visits[column_name] = simulated_visits[column_name].astype(str)
+
+            # If there are no prenights, the stacker does not run, so skip
+            # trying to filter on the right day_obs.
+            if "day_obs_iso8601" in simulated_visits.columns:
+                simulated_visits.query(f'day_obs_iso8601 == "{day_obs}"', inplace=True)
+            visits = pd.concat([completed_visits, simulated_visits])
+        else:
+            visits = completed_visits
         events["visits"] = visits
 
         self.events = events