From 95c892310cdfbcc0ad4ca0efc85177a5a0d5239e Mon Sep 17 00:00:00 2001 From: Eric Neilsen Date: Tue, 16 Jun 2026 14:19:27 -0700 Subject: [PATCH 01/23] Add statistics to ToC and RSS feed (mixed AI+human edited and human written code and docs) --- design/SP-3225/linktablestats.rst | 377 ++++++++++++++++ design/SP-3225/smallsum.rst | 534 +++++++++++++++++++++++ design/SP-3225/visitcache.rst | 679 +++++++++++++++++++++++++++++ schedview/collect/__init__.py | 3 +- schedview/collect/visits.py | 158 +++++++ schedview/compute/__init__.py | 3 + schedview/compute/smallsum.py | 280 ++++++++++++ schedview/reports.py | 177 +++++++- tests/test_collect_visits_cache.py | 200 +++++++++ tests/test_compute_smallsum.py | 320 ++++++++++++++ tests/test_reports.py | 71 ++- 11 files changed, 2773 insertions(+), 29 deletions(-) create mode 100644 design/SP-3225/linktablestats.rst create mode 100644 design/SP-3225/smallsum.rst create mode 100644 design/SP-3225/visitcache.rst create mode 100644 schedview/compute/smallsum.py create mode 100644 tests/test_collect_visits_cache.py create mode 100644 tests/test_compute_smallsum.py diff --git a/design/SP-3225/linktablestats.rst b/design/SP-3225/linktablestats.rst new file mode 100644 index 00000000..0daff8e6 --- /dev/null +++ b/design/SP-3225/linktablestats.rst @@ -0,0 +1,377 @@ +============================================================ +Design: Per-night statistics in ``schedview.reports`` +============================================================ + +Overview +-------- + +This design extends the two public functions in ``schedview.reports`` — +``make_report_link_table`` and ``make_report_rss_feed`` — so they can +optionally display per-night observing statistics alongside the links to +static reports. + +When a ``visits`` DataFrame is provided: + +- ``make_report_link_table`` appends summary columns (visit count, science + count, night hours, median FWHM, normalized effective time) to the HTML + table, populated only for ``lsstcam`` rows. +- ``make_report_rss_feed`` populates the ```` element of + ``lsstcam`` ``nightsum`` items with a formatted summary string containing + total visits, science visits, seeing, effective-time metrics, and science + targets. + +Additionally, numeric presentation is improved: float columns are rounded +to 2 decimal places and NaN/None values are rendered as empty strings. + +Module Location +--------------- + +:: + + schedview/reports.py + +No new public functions are introduced. The existing ``find_reports``, +``make_report_link_table``, and ``make_report_rss_feed`` remain the public +API. + + +New Module-Level Constants +-------------------------- + +``RSS_DESC_FORMAT`` + A format string template used to populate RSS item descriptions for + ``lsstcam`` ``nightsum`` entries. Contains placeholders for: ``total``, + ``science``, ``fwhm``, ``mean_norm_teff``, ``visit_rate``, ``teff_rate``, + ``targets``. + +``INT_SUMMARY_COLUMNS`` + A list of column names from ``compute_tinysum`` output that should be + displayed as integers in the link table: ``["Total", "science"]``. + +``FLOAT_SUMMARY_COLUMNS`` + A list of column names from ``compute_tinysum`` output that should be + displayed as rounded floats: ``["night_hours", "median FWHM", + "total eff_time/exp_time"]``. + +``SUMMARY_COLUMNS`` + The concatenation of ``INT_SUMMARY_COLUMNS + FLOAT_SUMMARY_COLUMNS``, + defining the full set of summary columns added to the link table. + + +Function: ``make_report_link_table`` +------------------------------------ + +Signature +~~~~~~~~~ + +.. code-block:: python + + def make_report_link_table( + reports: pd.DataFrame, + report_columns=("prenight", "multiprenight", "nightsum", "compareprenight"), + visits: pd.DataFrame | None = None, + ) -> str: + +Parameters +~~~~~~~~~~ + +``reports`` : ``pd.DataFrame`` + A DataFrame of report metadata, as returned by ``find_reports``. + +``report_columns`` : ``tuple`` + Names of reports to include as columns. + +``visits`` : ``pd.DataFrame`` or ``None`` + A visits DataFrame (as from ``cached_read_visits``). If provided, + per-night summary columns are computed and appended to the table. + Only ``lsstcam`` rows are populated; other instruments receive blank + cells. Defaults to ``None``. + +Returns +~~~~~~~ + +``str`` + An HTML ```` element (parseable as XML) with report links and, + optionally, summary statistics. + +Implementation Steps +~~~~~~~~~~~~~~~~~~~~ + +1. **Pivot** the reports DataFrame to create a table with + ``(night, instrument)`` as the index and report types as columns, + containing HTML links. + +2. **Compute tinysum** (only if ``visits`` is not ``None``): + + a. Instantiate an ``Almanac``. + b. Call ``compute_tinysum(visits, almanac=almanac)`` and select only + ``SUMMARY_COLUMNS``. + c. Convert the ``dayObs`` integer index to ``datetime.date`` objects + to align with the report table's ``night`` index level. + +3. **Join summary onto link table**: + + a. Build a boolean mask identifying ``lsstcam`` rows. + b. Reindex the tinysum DataFrame to match the full report_links index + (using the ``night`` level), so that non-lsstcam rows and nights + without visits receive ``NA``. + c. Blank out non-lsstcam rows by assigning ``None``. + d. Assign each summary column into the report_links DataFrame. + +4. **Format for display**: + + a. Round float columns (``FLOAT_SUMMARY_COLUMNS``) to 2 decimal places. + b. Cast all summary columns to ``object`` dtype so that + ``fillna("")`` works uniformly across ``Int64``, ``Float64``, etc. + c. Call ``fillna("")`` to replace any remaining NA values with empty + strings. + +5. **Render** with ``report_links.to_html(escape=False)``. + + +**Without visits — produces valid HTML table:** + +>>> import datetime +>>> import xml.etree.ElementTree as ET +>>> import pandas as pd +>>> from schedview.reports import make_report_link_table + +>>> reports = pd.DataFrame([ +... {"night": datetime.date(2025, 6, 20), "dayobs": "20250620", +... "report": "nightsum", "instrument": "lsstcam", +... "url": "http://example.com/ns", "link": 'nightsum', +... "report_time": "2025-06-21T00:00:00+00:00", "fname": "ns.html"}, +... ]).set_index(["instrument", "dayobs"]).sort_values("night", ascending=False) +>>> html = make_report_link_table(reports) +>>> root = ET.fromstring(html) +>>> root.tag +'table' + +**With visits — summary columns appear in HTML output:** + +>>> import numpy as np +>>> rng = np.random.default_rng(42) +>>> n = 20 +>>> visits = pd.DataFrame({ +... "dayObs": np.full(n, 20250620), +... "observationId": np.arange(n), +... "seeingFwhmGeom": rng.uniform(0.6, 1.5, size=n), +... "eff_time_median": rng.uniform(20.0, 40.0, size=n), +... "exp_time": rng.uniform(25.0, 35.0, size=n), +... "band": rng.choice(list("ugrizy"), size=n), +... "science_program": rng.choice(["BLOCK-365", "ENG-001"], size=n), +... "target_name": rng.choice(["COSMOS", "XMM-LSS", ""], size=n), +... }) +>>> html = make_report_link_table(reports, visits=visits) +>>> "Total" in html +True +>>> "science" in html +True +>>> root = ET.fromstring(html) +>>> root.tag +'table' + +**Non-lsstcam rows do not receive summary values:** + +>>> reports2 = pd.DataFrame([ +... {"night": datetime.date(2025, 6, 20), "dayobs": "20250620", +... "report": "nightsum", "instrument": "lsstcam", +... "url": "http://x.com/a", "link": 'nightsum', +... "report_time": "2025-06-21T00:00:00+00:00", "fname": "a.html"}, +... {"night": datetime.date(2025, 6, 20), "dayobs": "20250620", +... "report": "nightsum", "instrument": "auxtel", +... "url": "http://x.com/b", "link": 'nightsum', +... "report_time": "2025-06-21T00:00:00+00:00", "fname": "b.html"}, +... ]).set_index(["instrument", "dayobs"]).sort_values("night", ascending=False) +>>> html = make_report_link_table(reports2, visits=visits) +>>> root = ET.fromstring(html) +>>> root.tag +'table' + + +Function: ``make_report_rss_feed`` +---------------------------------- + +Signature +~~~~~~~~~ + +.. code-block:: python + + def make_report_rss_feed( + reports: pd.DataFrame, + fname: str | None = None, + max_days: int = 7, + visits: pd.DataFrame | None = None, + title: str = "schedview reports", + description: str = "Statically generated reports on Rubin Observatory/LSST scheduler status and progress", + ) -> ET.ElementTree: + +Parameters +~~~~~~~~~~ + +``reports`` : ``pd.DataFrame`` + A DataFrame of report metadata, as returned by ``find_reports``. + +``fname`` : ``str`` or ``None`` + File path to write the RSS XML. ``None`` to skip writing. + +``max_days`` : ``int`` + Maximum age of reports to include (in days from today). + +``visits`` : ``pd.DataFrame`` or ``None`` + A visits DataFrame. If provided, lsstcam nightsum items receive a + formatted summary description. + +``title`` : ``str`` + The ```` text for the RSS channel. + +``description`` : ``str`` + The ``<description>`` text for the RSS channel. + +Returns +~~~~~~~ + +``ET.ElementTree`` + The RSS XML tree. + +Implementation Steps +~~~~~~~~~~~~~~~~~~~~ + +1. **Compute tinysum** (only if ``visits`` is not ``None``): instantiate + ``Almanac`` and call ``compute_tinysum``. + +2. **Build RSS skeleton**: Create ``<rss>`` root with ``version="2.0"``, + add ``<channel>`` with ``<title>`` and ``<description>`` elements using + the provided parameter values. + +3. **Iterate** over reports rows. Skip items older than ``max_days``. + +4. **Per item**: + + a. Create ``<item>`` with ``<title>`` = + ``"{report} for {instrument} on {night}"``. + b. **Description logic**: + + - If ``instrument == "lsstcam"`` and ``report == "nightsum"`` and + tinysum is available and the dayobs exists in tinysum's index: + format ``RSS_DESC_FORMAT`` with values from tinysum (rounding + floats to 2 decimal places; handling ``TypeError`` on + ``teff/minute`` by falling back to ``NaN``). + - If the dayobs is missing from tinysum: ``"No visits on this night"``. + - Otherwise: empty string. + + c. Add ``<link>``, ``<guid>``, ``<category>``, and ``<pubDate>`` + elements. + +5. **Indent** the XML tree and optionally write to file. + + +**Basic RSS generation (no visits):** + +>>> import datetime +>>> import xml.etree.ElementTree as ET +>>> import pandas as pd +>>> from schedview.reports import make_report_rss_feed + +>>> reports = pd.DataFrame([ +... {"night": datetime.date.today(), "dayobs": "20250620", +... "report": "nightsum", "instrument": "lsstcam", +... "url": "http://example.com/ns", +... "link": '<a href="#">nightsum</a>', +... "report_time": "2025-06-20T12:00:00+00:00", "fname": "ns.html"}, +... ]).set_index(["instrument", "dayobs"]).sort_values("night", ascending=False) +>>> tree = make_report_rss_feed(reports, fname=None, max_days=99999) +>>> isinstance(tree, ET.ElementTree) +True +>>> root = tree.getroot() +>>> root.tag +'rss' +>>> root.attrib["version"] +'2.0' + +**Title parameter controls channel title:** + +>>> tree = make_report_rss_feed( +... reports, fname=None, max_days=99999, title="My Custom Title" +... ) +>>> tree.getroot().find("channel/title").text +'My Custom Title' + +**Description parameter controls channel description:** + +>>> tree = make_report_rss_feed( +... reports, fname=None, max_days=99999, +... description="Custom description" +... ) +>>> tree.getroot().find("channel/description").text +'Custom description' + + +Display Formatting +------------------ + +The link table applies several formatting rules to ensure clean +presentation: + +1. **Float rounding**: All columns in ``FLOAT_SUMMARY_COLUMNS`` are + rounded to 2 decimal places before rendering. + +2. **Object dtype conversion**: Summary columns are cast to ``object`` + dtype before calling ``fillna("")``. This ensures uniform behavior + across nullable integer (``Int64``) and floating-point columns that + would otherwise reject string fill values. + +3. **NaN/None suppression**: ``fillna("")`` replaces all missing values + with empty strings so that the rendered HTML table shows blank cells + rather than "NaN" or "None". + + +Changes from ``main`` +--------------------- + +1. **New parameter on ``make_report_link_table``**: ``visits`` (optional). +2. **New parameters on ``make_report_rss_feed``**: ``visits``, ``title``, + ``description``. +3. **Removed** ``"preprogress"`` from the default ``report_columns`` tuple + in ``make_report_link_table``. +4. **New constants**: ``RSS_DESC_FORMAT``, ``INT_SUMMARY_COLUMNS``, + ``FLOAT_SUMMARY_COLUMNS``, ``SUMMARY_COLUMNS``. +5. **New imports**: ``numpy``, ``rubin_scheduler.site_models.Almanac``, + ``schedview.compute.smallsum.compute_tinysum``. +6. **RSS item titles** changed from ``"{report} report for ..."`` to + ``"{report} for ..."``. +7. **Variable naming** in ``make_report_rss_feed`` changed to avoid + shadowing the ``title`` parameter (``title`` → ``channel_title_elem``, + etc.). + + +Dependencies +------------ + +- ``pandas`` +- ``numpy`` +- ``rubin_scheduler.site_models.Almanac`` (for night-hours computation + inside ``compute_tinysum``) +- ``schedview.compute.smallsum.compute_tinysum`` (the per-night summary + engine) + + +Test Coverage +------------- + +Tests are in ``tests/test_reports.py``. + +``test_make_report_link_table_with_visits`` + Constructs a synthetic visits DataFrame matching the test fixture's + dayObs values, passes it to ``make_report_link_table``, and verifies: + + - The returned HTML is parseable as XML (well-formed table). + - The ``"Total"`` and ``"science"`` column headers appear in the output. + +``test_make_report_rss_feed_uses_title_parameter`` + Passes a custom ``title`` string and verifies the ``<channel><title>`` + element contains exactly that text. + +Pre-existing tests (``test_find_reports``, ``test_make_report_link_table``, +``test_make_report_rss_feed``) continue to pass and verify backward +compatibility when ``visits`` is not supplied. diff --git a/design/SP-3225/smallsum.rst b/design/SP-3225/smallsum.rst new file mode 100644 index 00000000..55b1558d --- /dev/null +++ b/design/SP-3225/smallsum.rst @@ -0,0 +1,534 @@ +=================================================== +Design: ``schedview.compute.smallsum`` Module +=================================================== + +Overview +-------- + +This module provides two functions that summarize visits data at the +per-night level: + +1. ``compute_tinysum`` – produces a single-row-per-night summary DataFrame + (the "tiny summary"). +2. ``compute_smallsum`` – produces a modest-number-of-rows-per-night summary + DataFrame (the "small summary"), with rows broken out by subsets (band, + science/not-science, observation_reason, target name). + +Both functions take a visits ``pd.DataFrame`` as their primary input and +return a new summary ``pd.DataFrame``. + +Module Location +--------------- + +:: + + schedview/compute/smallsum.py + +Registered in ``schedview/compute/__init__.py`` with: + +.. code-block:: python + + from .smallsum import compute_tinysum, compute_smallsum + + +Architecture +------------ + +The module's two public functions serve different audiences: + +- ``compute_tinysum`` produces a **single row per night** suitable for a + compact dashboard or table-of-contents entry. It answers the question + "how did the night go overall?" with counts, efficiency statistics, and + (optionally) observing-rate metrics. + +- ``compute_smallsum`` produces **multiple rows per night**, one for each + subset category (all visits, each band, science/not-science, each + observation_reason, each target name). It answers "how did specific + categories of observations perform?" + +Both functions operate on the same visits ``DataFrame`` produced upstream by +``schedview.collect``, and both are pure computations — no I/O, no side +effects. + +Helper functions +~~~~~~~~~~~~~~~~ + +Three private helpers factor out reusable logic: + +``_unique_targets(target_name_series)`` + Aggregates a column of target names into a single deduplicated, + comma-separated string. Used by ``compute_tinysum`` to populate the + ``"science targets"`` column. + +``_visits_summary(visits_group)`` + Computes per-group statistics (visit count, time range, effective-time + quartiles, FWHM, airmass, hour angle). Used by ``compute_smallsum`` as + the aggregation function applied to each subset group. + +``_build_night_hours(almanac, dayobs_values)`` + Converts an ``Almanac`` instance into a Series mapping each ``dayObs`` to + its night duration in hours. Used by ``compute_tinysum`` to derive the + rate columns (``visits/hour``, ``teff/minute``). + +The sections below document each helper's contract and verify it with +embedded doctests, followed by the full specifications of the two public +functions. + + +Helper: ``_unique_targets`` +--------------------------- + +.. code-block:: python + + def _unique_targets(target_name_series: pd.Series) -> str + +Used by ``compute_tinysum`` to aggregate target names per night. + +Logic: + +- Iterate over each value in the series +- Strip leading ``ddf_`` or ``DDF`` + space prefix (exactly 4 characters) +- Split on ``', '`` to handle multi-target entries +- Collect all non-empty unique targets (preserving first-seen order) +- Return as a comma-separated string + +**Prefix stripping**: The ``ddf_`` and ``DDF`` + space prefixes are removed so +that DDF targets are identified by their field name alone: + +>>> import numpy as np +>>> import pandas as pd +>>> from schedview.compute.smallsum import _unique_targets +>>> _unique_targets(pd.Series(["ddf_COSMOS", "DDF XMM-LSS"])) +'COSMOS, XMM-LSS' + +**Comma splitting**: Multi-target entries (containing ``', '``) are split +into individual targets: + +>>> _unique_targets(pd.Series(["ELAIS, XMM-LSS"])) +'ELAIS, XMM-LSS' + +**Pass-through**: Plain target names are returned unchanged: + +>>> _unique_targets(pd.Series(["COSMOS", "XMM-LSS", "ELAIS"])) +'COSMOS, XMM-LSS, ELAIS' + +**Deduplication**: Duplicate targets (including those that become duplicates +after prefix stripping) appear only once: + +>>> _unique_targets(pd.Series(["COSMOS", "COSMOS", "ddf_COSMOS"])) +'COSMOS' + +**Empty and NaN handling**: Empty strings, ``None``, and ``NaN`` values are +ignored. If only empty/NaN values are present, the result is an empty +string: + +>>> _unique_targets(pd.Series(["", "", "COSMOS"])) +'COSMOS' +>>> _unique_targets(pd.Series([np.nan, None, "COSMOS"])) +'COSMOS' +>>> _unique_targets(pd.Series(["", ""])) +'' + + +Helper: ``_visits_summary`` +--------------------------- + +.. code-block:: python + + def _visits_summary(visits_group: pd.DataFrame) -> pd.Series + +Computes summary statistics for a group of visits (one subset of one night). +Returns a ``pd.Series`` with the following keys: + +.. list-table:: + :header-rows: 1 + + * - Key + - Computation + * - ``visits`` + - ``len(visits_group)`` + * - ``first`` + - ``visits_group["start_timestamp"].min()`` + * - ``last`` + - ``visits_group["start_timestamp"].max()`` + * - ``teff_total`` + - ``np.nan_to_num(eff_time_median.to_numpy(), nan=0.0).sum()`` + * - ``teff_q1`` + - ``eff_time_median.quantile(0.25)`` + * - ``teff_median`` + - ``eff_time_median.median()`` + * - ``teff_q3`` + - ``eff_time_median.quantile(0.75)`` + * - ``fwhm_median`` + - ``visits_group["seeingFwhmGeom"].median()`` + * - ``airmass_median`` + - ``visits_group["airmass"].median()`` + * - ``HA_median`` + - ``visits_group["HA"].median()`` + +Note: ``teff_total`` uses ``np.nan_to_num`` to treat NaN values as 0 before +summing, rather than ``mean * count``, so that nights with partial NaN data +still produce a meaningful total. + +**Output keys**: The function returns all expected keys: + +>>> from schedview.compute.smallsum import _visits_summary +>>> group = pd.DataFrame({ +... "start_timestamp": [1.0, 2.0, 3.0, 4.0, 5.0], +... "eff_time_median": [30.0, 25.0, 35.0, 28.0, 32.0], +... "seeingFwhmGeom": [0.8, 1.0, 0.9, 1.1, 0.7], +... "airmass": [1.1, 1.2, 1.3, 1.4, 1.5], +... "HA": [-1.0, -0.5, 0.0, 0.5, 1.0], +... }) +>>> result = _visits_summary(group) +>>> sorted(result.index) == sorted([ +... "visits", "first", "last", "teff_total", "teff_q1", +... "teff_median", "teff_q3", "fwhm_median", "airmass_median", +... "HA_median", +... ]) +True +>>> result["visits"] +np.float64(5.0) +>>> result["first"] +np.float64(1.0) +>>> result["last"] +np.float64(5.0) + +**teff_total with NaN**: NaN values in ``eff_time_median`` are treated as 0 +when computing the total, so the sum reflects only valid values: + +>>> group_with_nan = pd.DataFrame({ +... "start_timestamp": [1.0, 2.0, 3.0], +... "eff_time_median": [30.0, np.nan, 20.0], +... "seeingFwhmGeom": [0.8, 1.0, 0.9], +... "airmass": [1.1, 1.2, 1.3], +... "HA": [-1.0, 0.0, 1.0], +... }) +>>> result = _visits_summary(group_with_nan) +>>> result["teff_total"] +np.float64(50.0) +>>> result["visits"] +np.float64(3.0) + +**Single visit**: When only one visit is present, ``first == last``: + +>>> single = pd.DataFrame({ +... "start_timestamp": [42.0], +... "eff_time_median": [30.0], +... "seeingFwhmGeom": [0.9], +... "airmass": [1.2], +... "HA": [0.5], +... }) +>>> result = _visits_summary(single) +>>> result["visits"] +np.float64(1.0) +>>> assert result["first"] == result["last"] + + + +Helper: ``_build_night_hours`` +------------------------------ + +.. code-block:: python + + def _build_night_hours(almanac: Almanac, dayobs_values: pd.Index) -> pd.Series + +Builds a Series mapping ``dayObs`` (YYYYMMDD int) to night duration in hours, +defined as the time between astronomical twilight boundaries (sun at −12° +setting to sun at −12° rising). + +The function uses the Almanac's ``sunsets`` array to look up the +``sun_n12_setting`` and ``sun_n12_rising`` times for each night, computing: + +.. code-block:: python + + night_hours = (sun_n12_rising - sun_n12_setting) * 24.0 + +**Return type and length**: Returns a ``pd.Series`` with the same length as +the input index: + +>>> from schedview.compute.smallsum import _build_night_hours +>>> from rubin_scheduler.site_models import Almanac +>>> almanac = Almanac() +>>> index = pd.Index([20250601, 20250602]) +>>> result = _build_night_hours(almanac, index) +>>> isinstance(result, pd.Series) +True +>>> len(result) == 2 +True + +**Positive hours in valid range**: Night durations at Cerro Pachón should +fall between roughly 4 and 12 hours depending on season: + +>>> all(4.0 < v < 12.0 for v in result.dropna()) +True + + +Function 1: ``compute_tinysum`` +------------------------------- + +Signature +~~~~~~~~~ + +.. code-block:: python + + def compute_tinysum( + visits: pd.DataFrame, + science_programs: tuple[str, ...] = SCIENCE_PROGRAMS, + almanac: Almanac | None = None, + ) -> pd.DataFrame: + +Parameters +~~~~~~~~~~ + +``visits`` : ``pd.DataFrame`` + A DataFrame of visits. Must contain columns: ``dayObs`` (int, YYYYMMDD), + ``observationId``, ``seeingFwhmGeom``, ``eff_time_median``, ``exp_time``, + ``band``, ``science_program``, ``target_name``. + +``science_programs`` : ``tuple[str, ...]`` + Tuple of ``science_program`` values considered "science". Defaults to + ``SCIENCE_PROGRAMS`` from ``rubin_nights.reference_values``. + +``almanac`` : ``Almanac`` or ``None`` + A ``rubin_scheduler.site_models.Almanac`` instance. Pass ``None`` to omit + the ``night_hours``, ``visits/hour``, and ``teff/minute`` columns. + +Returns +~~~~~~~ + +A ``pd.DataFrame`` indexed by ``dayObs`` (int, YYYYMMDD format), with one row +per night. Columns: + +.. list-table:: + :header-rows: 1 + + * - Column + - Type + - Description + * - ``Total`` + - Int64 + - Total number of visits that night (nullable integer) + * - ``median FWHM`` + - float + - Median ``seeingFwhmGeom`` across all visits that night + * - ``total exp_time`` + - float + - Sum of ``exp_time`` values for the night + * - ``total eff_time`` + - float + - Sum of ``eff_time_median`` values for the night + * - ``mean eff_time`` + - float + - Mean of ``eff_time_median`` across all visits that night + * - ``q1 eff_time`` + - float + - 25th percentile of ``eff_time_median`` + * - ``median eff_time`` + - float + - 50th percentile of ``eff_time_median`` + * - ``q3 eff_time`` + - float + - 75th percentile of ``eff_time_median`` + * - ``science`` + - Int64 + - Number of visits with ``science_program`` in ``science_programs`` + * - ``# u`` through ``# y`` + - Int64 + - Number of visits in each band (nullable integer) + * - ``science targets`` + - str + - Comma-separated unique target names from science visits + * - ``total eff_time/exp_time`` + - float + - ``total eff_time / total exp_time`` (normalized effective time) + * - ``night_hours`` + - float + - Duration of night in hours (only if almanac provided) + * - ``visits/hour`` + - float + - ``Total / night_hours`` (only if almanac provided) + * - ``teff/minute`` + - float + - ``(visits/hour * mean eff_time) / 60`` (only if almanac provided) + +Implementation Steps +~~~~~~~~~~~~~~~~~~~~ + +1. **Basic stats**: Group ``visits`` by ``dayObs``, aggregate: + + - Count of ``observationId`` → ``Total`` (cast to ``Int64``) + - Median of ``seeingFwhmGeom`` → ``median FWHM`` + - Sum of ``exp_time`` → ``total exp_time`` + - Sum of ``eff_time_median`` → ``total eff_time`` + +2. **Effective time stats**: Group by ``dayObs``, call ``.describe()`` on + ``eff_time_median``, extract ``mean``, ``25%``, ``50%``, ``75%`` and rename + to ``mean eff_time``, ``q1 eff_time``, ``median eff_time``, + ``q3 eff_time``. + +3. **Band counts**: Group by ``['dayObs', 'band']``, count ``observationId``, + unstack so bands become columns, fill NaN with 0, cast to ``Int64``. + Rename to ``# u``, ``# g``, etc. + +4. **Science counts**: Filter visits where ``science_program`` is in + ``science_programs``, group by ``dayObs``, count → ``science``. + +5. **Science targets**: From science visits, group by ``dayObs``, aggregate + ``target_name`` using ``_unique_targets``. + +6. **Join all** intermediate DataFrames on the ``dayObs`` index. Fill NaN in + ``science`` with 0 (cast to ``Int64``) and in ``science targets`` with + empty string. + +7. **Normalized effective time**: Compute + ``total eff_time/exp_time = total eff_time / total exp_time``. + +8. **Night hours** (only if ``almanac`` is not ``None``): Build a mapping from + ``dayObs`` → night duration via ``_build_night_hours``. + +9. **Derived rates** (only if ``almanac`` is not ``None``): + + - ``visits/hour = Total / night_hours`` + - ``teff/minute = visits/hour * mean eff_time / 60`` + + +Function 2: ``compute_smallsum`` +-------------------------------- + +Signature +~~~~~~~~~ + +.. code-block:: python + + def compute_smallsum( + visits: pd.DataFrame, + science_programs: tuple[str, ...] = SCIENCE_PROGRAMS, + ) -> pd.DataFrame: + +Parameters +~~~~~~~~~~ + +``visits`` : ``pd.DataFrame`` + A DataFrame of visits. Must contain columns: ``dayObs``, + ``observationId``, ``start_timestamp``, ``eff_time_median``, + ``seeingFwhmGeom``, ``airmass``, ``HA``, ``band``, ``science_program``, + ``observation_reason``, ``target_name``. + +``science_programs`` : ``tuple[str, ...]`` + Tuple of ``science_program`` values considered "science". Defaults to + ``SCIENCE_PROGRAMS`` from ``rubin_nights.reference_values``. + +Returns +~~~~~~~ + +A ``pd.DataFrame`` with a two-level index: (``dayObs``, ``subset``). Each +night has multiple rows, one for each subset category. The ``subset`` level +contains values like: + +- ``"all"`` – aggregate of all visits that night +- Band names (``"u"``, ``"g"``, ``"r"``, ``"i"``, ``"z"``, ``"y"``) – split + by band +- ``"science"``, ``"not_science"`` – split by whether ``science_program`` is + in ``science_programs`` +- observation_reason values – split by ``observation_reason`` column +- target name values – split by individual target names (with multi-target + values exploded on ``', '``) + +The subset ordering within each night is: ``"all"`` first, then bands, then +science/not_science, then observation_reason, then target names. + +Columns of the returned DataFrame: + +.. list-table:: + :header-rows: 1 + + * - Column + - Type + - Description + * - ``visits`` + - int + - Number of visits in this subset + * - ``first`` + - float + - Earliest ``start_timestamp`` in this subset + * - ``last`` + - float + - Latest ``start_timestamp`` in this subset + * - ``teff_total`` + - float + - Sum of ``eff_time_median`` with NaN treated as 0 + * - ``teff_q1`` + - float + - 25th percentile of ``eff_time_median`` + * - ``teff_median`` + - float + - 50th percentile of ``eff_time_median`` + * - ``teff_q3`` + - float + - 75th percentile of ``eff_time_median`` + * - ``fwhm_median`` + - float + - Median ``seeingFwhmGeom`` + * - ``airmass_median`` + - float + - Median ``airmass`` + * - ``HA_median`` + - float + - Median hour angle + +Implementation Steps +~~~~~~~~~~~~~~~~~~~~ + +1. **Full night ("all") subset**: Group ``visits`` by ``dayObs``, apply + ``_visits_summary``. Add column ``subset = "all"``. Set index to + ``(dayObs, subset)``. + +2. **By band subset**: Group by ``['dayObs', 'band']``, apply + ``_visits_summary``. Rename ``band`` index level to ``subset``. + +3. **By science subset**: Add temporary column ``_science`` = + ``"science"`` or ``"not_science"``. Group by ``['dayObs', '_science']``, + apply ``_visits_summary``. Rename index level to ``subset``. + +4. **By observation_reason subset**: Group by + ``['dayObs', 'observation_reason']``, apply ``_visits_summary``. Rename + index level to ``subset``. + +5. **By target name subset**: Split ``target_name`` on ``', '`` and explode. + Replace empty strings with ``"no target name"``. Group by + ``['dayObs', '_target_names']``, apply ``_visits_summary``. Rename index + level to ``subset``. + +6. **Concatenate** in order: all, band, science, observation_reason, target. + Sort by ``dayObs`` level (preserving order within each night). + + +Dependencies +------------ + +- ``pandas`` +- ``numpy`` +- ``rubin_scheduler.site_models.Almanac`` (for night duration in + ``compute_tinysum``) +- ``rubin_nights.reference_values.SCIENCE_PROGRAMS`` (default value for + science program filtering) + + +Notes +----- + +- Both functions are **pure computations**: they take a visits DataFrame and + return a summary DataFrame. They do not read data from disk or network. +- The ``visits`` DataFrame is expected to already have the necessary columns + populated (e.g., via stackers applied during collection). +- The ``dayObs`` column is expected to be an integer in YYYYMMDD format (as + produced by ``DayObsStacker``). +- The Almanac is only needed by ``compute_tinysum`` for the ``night_hours`` + and rate columns. If these columns are not needed, the Almanac parameter + can be set to ``None`` and those columns will be omitted. +- Integer count columns (``Total``, ``science``, band counts) use pandas + nullable ``Int64`` dtype to avoid float conversion when NaN values are + present from joins. +- The ``exp_time`` column is required by ``compute_tinysum`` for the + ``total eff_time/exp_time`` normalized efficiency metric. diff --git a/design/SP-3225/visitcache.rst b/design/SP-3225/visitcache.rst new file mode 100644 index 00000000..686dc251 --- /dev/null +++ b/design/SP-3225/visitcache.rst @@ -0,0 +1,679 @@ +============================================================ +Design: Caching for ``schedview.collect.visits`` +============================================================ + +Goal +---- + +Add a local file cache layer for expensive consdb queries so they can be +avoided on repeated calls for the same data. The cache is implemented as a +standalone ``cached_read_visits`` function that wraps +``read_visits``/``read_ddf_visits``. + +Source Context +-------------- + +- **Reference notebook:** ``notebooks/smallsum.ipynb`` — contains the + prototype ``cached_read_visits`` function that this design formalizes. +- **Target module:** ``schedview/collect/visits.py`` — contains the existing + ``read_visits`` and ``read_ddf_visits`` functions, plus the new + ``cached_read_visits`` and ``_is_cache_fresh`` helpers. + +Architecture +------------ + +The caching layer consists of one public function and one private helper: + +``cached_read_visits(day_obs, visit_source, cache_dir, stackers, ddf)`` + The public entry point. On a **cache hit** (the cache file exists, is + fresh, and was built with the same set of stackers), it reads from the + local HDF5 file. On a **cache miss**, it queries the real source via + ``read_visits`` / ``read_ddf_visits``, writes the result to the cache, + and returns the filtered data. + +``_is_cache_fresh(cache_path)`` + Determines whether a cache file is "fresh enough" to use based on its + modification time relative to yesterday's sunrise and today's sunset. + +The flow for a single call: + +1. **Validate** ``visit_source`` — only consdb instruments are supported. +2. **Resolve** default stackers based on the ``ddf`` flag. +3. **Construct** the cache file path from ``cache_dir``, ``visit_source``, + and the ``ddf`` flag. +4. **Check freshness** via ``_is_cache_fresh``. +5. If fresh, **verify stacker match** by comparing stored class names to the + requested set. +6. On a miss, **query** the source for all available history, then **write** + the HDF5 cache with both ``"visits"`` and ``"stackers"`` keys. +7. **Filter** the result to visits on or before the requested ``day_obs``. + +The sections below describe each component in detail, with embedded doctests +that verify the implementation follows the design. + + +Module Location +--------------- + +:: + + schedview/collect/visits.py + +Exported from ``schedview.collect`` via ``__init__.py``: + +.. code-block:: python + + from .visits import NIGHT_STACKERS, cached_read_visits, read_visits + + +Public Function: ``cached_read_visits`` +--------------------------------------- + +Signature +~~~~~~~~~ + +.. code-block:: python + + def cached_read_visits( + day_obs: str | int | DayObs, + visit_source: str, + cache_dir: str | Path, + stackers: list | None = None, + ddf: bool = False, + ) -> pd.DataFrame: + +Parameters +~~~~~~~~~~ + +.. list-table:: + :header-rows: 1 + + * - Parameter + - Type + - Description + * - ``day_obs`` + - ``str | int | DayObs`` + - The night of observing. Visits up to and including this night are + returned. + * - ``visit_source`` + - ``str`` + - A consdb instrument name (e.g. ``"lsstcam"``, ``"latiss"``). Only + sources in ``KNOWN_INSTRUMENTS`` are supported; raises ``ValueError`` + otherwise. + * - ``cache_dir`` + - ``str | Path`` + - Directory where cache files are stored. Created automatically if it + does not exist. + * - ``stackers`` + - ``list | None`` + - Stacker instances to apply. If ``None``, defaults to + ``NIGHT_STACKERS`` (when ``ddf=False``) or + ``DDF_STACKERS + [maf.stackers.DayObsStacker()]`` (when ``ddf=True``). + * - ``ddf`` + - ``bool`` + - If ``True``, use ``read_ddf_visits`` instead of ``read_visits`` and + use DDF-appropriate stackers. + +Returns +~~~~~~~ + +A ``pd.DataFrame`` of visits for nights up to and including ``day_obs``. + +Raises +~~~~~~ + +``ValueError`` + If ``visit_source`` is not a known consdb instrument. + + +Input validation +~~~~~~~~~~~~~~~~ + +Only consdb instrument names are accepted. Other source types (opsim +files, ``"baseline"``) raise ``ValueError``: + +>>> import warnings +>>> warnings.filterwarnings("ignore") +>>> from schedview.collect.visits import cached_read_visits + +>>> try: +... cached_read_visits(20260614, "baseline", cache_dir="/tmp/cache") +... except ValueError as e: +... "only supports consdb instruments" in str(e) +True + +>>> try: +... cached_read_visits(20260614, "/path/to/sim.db", cache_dir="/tmp/c") +... except ValueError as e: +... "only supports consdb instruments" in str(e) +True + + +Cache File Format +----------------- + +The cache file is an **HDF5** file (``.h5`` extension) with two keys: + +- ``"visits"`` — the full visits ``DataFrame`` (all nights up to the query + date). +- ``"stackers"`` — a single-column ``DataFrame`` (column ``"class_name"``) + recording the fully-qualified class name of each stacker used to produce + the cached data. Used to detect stale caches caused by a change in the + requested stacker set. + +File Naming +~~~~~~~~~~~ + +- Non-DDF: ``visits_{visit_source}.h5`` (e.g. ``visits_lsstcam.h5``) +- DDF: ``visits_{visit_source}_ddf.h5`` (e.g. ``visits_lsstcam_ddf.h5``) + +The DDF suffix ensures that DDF and non-DDF caches coexist without +collision: + +>>> import tempfile +>>> from pathlib import Path +>>> from unittest.mock import MagicMock, patch +>>> import pandas as pd +>>> from schedview.collect.visits import cached_read_visits + +>>> fake_visits = pd.DataFrame({ +... "dayObs": [20260610, 20260611, 20260612], +... "observationId": range(3), +... }) +>>> def _day_obs_mock(yyyymmdd): +... m = MagicMock() +... m.yyyymmdd = yyyymmdd +... return m + +Non-DDF creates ``visits_lsstcam.h5``: + +>>> with tempfile.TemporaryDirectory() as tmpdir: +... with ( +... patch("schedview.collect.visits._is_cache_fresh", +... return_value=False), +... patch("schedview.collect.visits.read_visits", +... return_value=fake_visits), +... patch("schedview.collect.visits.DayObs.from_date") as mock_fd, +... ): +... mock_fd.side_effect = ( +... lambda a: MagicMock() if a == "today" +... else _day_obs_mock(20260612) +... ) +... _ = cached_read_visits( +... 20260612, "lsstcam", cache_dir=tmpdir, ddf=False +... ) +... (Path(tmpdir) / "visits_lsstcam.h5").exists() +True + +DDF creates ``visits_lsstcam_ddf.h5``: + +>>> with tempfile.TemporaryDirectory() as tmpdir: +... with ( +... patch("schedview.collect.visits._is_cache_fresh", +... return_value=False), +... patch("schedview.collect.visits.read_ddf_visits", +... return_value=fake_visits), +... patch("schedview.collect.visits.DayObs.from_date") as mock_fd, +... ): +... mock_fd.side_effect = ( +... lambda a: MagicMock() if a == "today" +... else _day_obs_mock(20260612) +... ) +... _ = cached_read_visits( +... 20260612, "lsstcam", cache_dir=tmpdir, ddf=True +... ) +... (Path(tmpdir) / "visits_lsstcam_ddf.h5").exists() +True + +HDF5 Structure +~~~~~~~~~~~~~~ + +On a cache miss, both the ``"visits"`` and ``"stackers"`` keys are written. +The ``"stackers"`` key stores a DataFrame with one row per stacker class: + +>>> from rubin_sim import maf +>>> stackers = [maf.stackers.DayObsStacker()] +>>> def _class_names(stackers): +... return {type(s).__module__ + "." + type(s).__qualname__ +... for s in stackers} + +>>> with tempfile.TemporaryDirectory() as tmpdir: +... with ( +... patch("schedview.collect.visits._is_cache_fresh", +... return_value=False), +... patch("schedview.collect.visits.read_visits", +... return_value=fake_visits), +... patch("schedview.collect.visits.DayObs.from_date") as mock_fd, +... ): +... mock_fd.side_effect = ( +... lambda a: MagicMock() if a == "today" +... else _day_obs_mock(20260612) +... ) +... _ = cached_read_visits( +... 20260612, "lsstcam", cache_dir=tmpdir, stackers=stackers +... ) +... cache_path = Path(tmpdir) / "visits_lsstcam.h5" +... stored_visits = pd.read_hdf(str(cache_path), key="visits") +... stored_stackers = pd.read_hdf(str(cache_path), key="stackers") +... len(stored_visits) +... "class_name" in stored_stackers.columns +... set(stored_stackers["class_name"]) == _class_names(stackers) +3 +True +True + + +Private Helper: ``_is_cache_fresh`` +----------------------------------- + +Signature +~~~~~~~~~ + +.. code-block:: python + + def _is_cache_fresh(cache_path: Path) -> bool + +Purpose +~~~~~~~ + +Determine whether a cache file is "fresh enough" to use. + +Logic +~~~~~ + +- If the file does not exist, return ``False``. +- Get the file's modification time as an ``astropy.time.Time``. +- Compute boundaries: + + - ``yesterday = DayObs.from_date("yesterday")`` + - ``today = DayObs.from_date("today")`` + - Lower bound: ``yesterday.sunrise`` (the cache was written after the + last night ended) + - Upper bound: ``today.sunset`` (the cache was written before tonight + starts) + +- Return ``True`` if ``lower_bound < cache_mtime < upper_bound``. + +Rationale +~~~~~~~~~ + +The cache represents "all visits through last night." If it was written +between yesterday's sunrise (after last night ended) and today's sunset +(before tonight starts), it should be complete and not yet stale. + +**Missing file returns False:** + +>>> from schedview.collect.visits import _is_cache_fresh +>>> _is_cache_fresh(Path("/nonexistent/path/visits_lsstcam.h5")) +False + +**Fresh file returns True** (mtime within the freshness window): + +>>> from astropy.time import Time +>>> with tempfile.TemporaryDirectory() as tmpdir: +... cache_path = Path(tmpdir) / "visits_lsstcam.h5" +... _ = cache_path.write_bytes(b"fake") +... yesterday_dayobs = MagicMock() +... yesterday_dayobs.sunrise = Time( +... "2026-06-16 10:00:00", scale="utc" +... ) +... today_dayobs = MagicMock() +... today_dayobs.sunset = Time( +... "2026-06-18 01:00:00", scale="utc" +... ) +... mock_stat = MagicMock() +... mock_stat.st_mtime = Time( +... "2026-06-17 12:00:00", scale="utc" +... ).unix +... with ( +... patch("pathlib.Path.stat", return_value=mock_stat), +... patch("schedview.collect.visits.DayObs.from_date") as mock_fd, +... ): +... mock_fd.side_effect = ( +... lambda arg: yesterday_dayobs if arg == "yesterday" +... else today_dayobs +... ) +... _is_cache_fresh(cache_path) +True + +**Stale file returns False** (mtime before the freshness window): + +>>> with tempfile.TemporaryDirectory() as tmpdir: +... cache_path = Path(tmpdir) / "visits_lsstcam.h5" +... _ = cache_path.write_bytes(b"fake") +... mock_stat = MagicMock() +... mock_stat.st_mtime = Time( +... "2026-06-10 12:00:00", scale="utc" +... ).unix +... yesterday_dayobs = MagicMock() +... yesterday_dayobs.sunrise = Time( +... "2026-06-16 10:00:00", scale="utc" +... ) +... today_dayobs = MagicMock() +... today_dayobs.sunset = Time( +... "2026-06-18 01:00:00", scale="utc" +... ) +... with ( +... patch("pathlib.Path.stat", return_value=mock_stat), +... patch("schedview.collect.visits.DayObs.from_date") as mock_fd, +... ): +... mock_fd.side_effect = ( +... lambda arg: yesterday_dayobs if arg == "yesterday" +... else today_dayobs +... ) +... _is_cache_fresh(cache_path) +False + + +Cache Hit/Miss Logic +-------------------- + +The cache hit/miss decision follows this sequence: + +1. Validate ``visit_source`` is in ``KNOWN_INSTRUMENTS`` (raise + ``ValueError`` if not). +2. Resolve default stackers based on the ``ddf`` flag. +3. Construct ``cache_path``: + ``cache_dir / f"visits_{visit_source}{suffix}.h5"`` + where ``suffix = "_ddf"`` if ``ddf`` else ``""``. +4. Compute ``requested_class_names`` = set of fully-qualified class names + from stackers. +5. If ``_is_cache_fresh(cache_path)``: + + a. Try to read the ``"stackers"`` key from the HDF5 file. + b. If the key is missing, treat as a stale cache → cache miss. + c. If ``cached_class_names == requested_class_names`` → cache hit: + read ``"visits"`` key from HDF5. + d. If class names don't match → cache miss (stacker mismatch). + +6. On cache miss: + + a. Query source using ``read_visits`` or ``read_ddf_visits`` with: + ``day_obs = DayObs.from_date("today")``, + ``num_nights = 365 * 20`` (fetch all available history), + ``stackers = resolved stackers``. + b. Create ``cache_dir`` if it doesn't exist. + c. Write visits to HDF5 under key ``"visits"`` (``mode="w"``). + d. Write stacker class names to HDF5 under key ``"stackers"`` + (``mode="a"``). + +7. Filter to requested ``day_obs``: + + - If ``"dayObs"`` column exists: return visits where + ``dayObs <= day_obs_obj.yyyymmdd``. + - If ``"dayObs"`` column is absent: warn and return unfiltered data. + + +Cache hit — read_visits is not called +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When the cache is fresh and stackers match, the underlying query function +is not invoked: + +>>> with tempfile.TemporaryDirectory() as tmpdir: +... cache_path = Path(tmpdir) / "visits_lsstcam.h5" +... fake_visits.to_hdf(str(cache_path), key="visits", mode="w") +... pd.DataFrame( +... {"class_name": sorted(_class_names(stackers))} +... ).to_hdf(str(cache_path), key="stackers", mode="a") +... with ( +... patch("schedview.collect.visits._is_cache_fresh", +... return_value=True), +... patch("schedview.collect.visits.read_visits") as mock_rv, +... patch("schedview.collect.visits.DayObs.from_date", +... return_value=_day_obs_mock(20260612)), +... ): +... result = cached_read_visits( +... 20260612, "lsstcam", cache_dir=tmpdir, stackers=stackers +... ) +... mock_rv.assert_not_called() +... len(result) +3 + +Stacker mismatch triggers regeneration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If the cache was built with different stackers than those requested, the +cache is regenerated: + +>>> stackers_in_cache = [maf.stackers.DayObsStacker()] +>>> stackers_requested = [ +... maf.stackers.DayObsStacker(), +... maf.stackers.DayObsISOStacker(), +... ] +>>> with tempfile.TemporaryDirectory() as tmpdir: +... cache_path = Path(tmpdir) / "visits_lsstcam.h5" +... fake_visits.to_hdf(str(cache_path), key="visits", mode="w") +... pd.DataFrame( +... {"class_name": sorted(_class_names(stackers_in_cache))} +... ).to_hdf(str(cache_path), key="stackers", mode="a") +... with ( +... patch("schedview.collect.visits._is_cache_fresh", +... return_value=True), +... patch("schedview.collect.visits.read_visits", +... return_value=fake_visits) as mock_rv, +... patch("schedview.collect.visits.DayObs.from_date") as mock_fd, +... ): +... mock_fd.side_effect = ( +... lambda a: MagicMock() if a == "today" +... else _day_obs_mock(20260612) +... ) +... _ = cached_read_visits( +... 20260612, "lsstcam", cache_dir=tmpdir, +... stackers=stackers_requested, +... ) +... mock_rv.assert_called_once() +... new_names = set( +... pd.read_hdf(str(cache_path), key="stackers")["class_name"] +... ) +... new_names == _class_names(stackers_requested) +True + + +Filtering and Warnings +----------------------- + +dayObs filtering +~~~~~~~~~~~~~~~~ + +Results are filtered to visits on or before the requested ``day_obs``: + +>>> with tempfile.TemporaryDirectory() as tmpdir: +... with ( +... patch("schedview.collect.visits._is_cache_fresh", +... return_value=False), +... patch("schedview.collect.visits.read_visits", +... return_value=fake_visits), +... patch("schedview.collect.visits.DayObs.from_date") as mock_fd, +... ): +... mock_fd.side_effect = ( +... lambda a: MagicMock() if a == "today" +... else _day_obs_mock(20260611) +... ) +... result = cached_read_visits( +... 20260611, "lsstcam", cache_dir=tmpdir +... ) +... list(result["dayObs"]) +[20260610, 20260611] + +Missing dayObs column +~~~~~~~~~~~~~~~~~~~~~~ + +If the visits DataFrame lacks a ``"dayObs"`` column (e.g. because +``DayObsStacker`` was not in the stacker list), a warning is issued and +unfiltered data is returned: + +>>> fake_visits_no_dayobs = pd.DataFrame({"observationId": [1, 2, 3]}) +>>> with tempfile.TemporaryDirectory() as tmpdir: +... with ( +... patch("schedview.collect.visits._is_cache_fresh", +... return_value=False), +... patch("schedview.collect.visits.read_visits", +... return_value=fake_visits_no_dayobs), +... patch("schedview.collect.visits.DayObs.from_date") as mock_fd, +... ): +... mock_fd.side_effect = ( +... lambda a: MagicMock() if a == "today" +... else _day_obs_mock(20260614) +... ) +... with warnings.catch_warnings(record=True) as w: +... warnings.simplefilter("always") +... result = cached_read_visits( +... 20260614, "lsstcam", cache_dir=tmpdir +... ) +... len(result) +... any("dayObs" in str(warning.message) for warning in w) +3 +True + + +Default Stackers +----------------- + +When ``stackers=None``, the function selects appropriate defaults: + +- **Non-DDF**: ``NIGHT_STACKERS`` +- **DDF**: ``DDF_STACKERS + [maf.stackers.DayObsStacker()]`` + +Non-DDF defaults +~~~~~~~~~~~~~~~~~ + +>>> from schedview.collect.visits import NIGHT_STACKERS, DDF_STACKERS +>>> with tempfile.TemporaryDirectory() as tmpdir: +... with ( +... patch("schedview.collect.visits._is_cache_fresh", +... return_value=False), +... patch("schedview.collect.visits.read_visits", +... return_value=fake_visits) as mock_rv, +... patch("schedview.collect.visits.DayObs.from_date") as mock_fd, +... ): +... mock_fd.side_effect = ( +... lambda a: MagicMock() if a == "today" +... else _day_obs_mock(20260612) +... ) +... _ = cached_read_visits( +... 20260612, "lsstcam", cache_dir=tmpdir, stackers=None +... ) +... _, call_kwargs = mock_rv.call_args +... _class_names(call_kwargs["stackers"]) == _class_names(NIGHT_STACKERS) +True + +DDF defaults +~~~~~~~~~~~~~ + +>>> ddf_default_stackers = DDF_STACKERS + [maf.stackers.DayObsStacker()] +>>> with tempfile.TemporaryDirectory() as tmpdir: +... with ( +... patch("schedview.collect.visits._is_cache_fresh", +... return_value=False), +... patch("schedview.collect.visits.read_ddf_visits", +... return_value=fake_visits) as mock_rdv, +... patch("schedview.collect.visits.DayObs.from_date") as mock_fd, +... ): +... mock_fd.side_effect = ( +... lambda a: MagicMock() if a == "today" +... else _day_obs_mock(20260612) +... ) +... _ = cached_read_visits( +... 20260612, "lsstcam", cache_dir=tmpdir, +... stackers=None, ddf=True, +... ) +... _, call_kwargs = mock_rdv.call_args +... _class_names(call_kwargs["stackers"]) == _class_names(ddf_default_stackers) +True + + +Automatic Directory Creation +------------------------------ + +If ``cache_dir`` does not exist, it is created automatically (including +intermediate directories): + +>>> with tempfile.TemporaryDirectory() as tmpdir: +... new_cache_dir = Path(tmpdir) / "new" / "subdir" +... new_cache_dir.exists() +... with ( +... patch("schedview.collect.visits._is_cache_fresh", +... return_value=False), +... patch("schedview.collect.visits.read_visits", +... return_value=fake_visits), +... patch("schedview.collect.visits.DayObs.from_date") as mock_fd, +... ): +... mock_fd.side_effect = ( +... lambda a: MagicMock() if a == "today" +... else _day_obs_mock(20260612) +... ) +... _ = cached_read_visits( +... 20260612, "lsstcam", cache_dir=new_cache_dir +... ) +... new_cache_dir.exists() +False +True + + +Idempotence +----------- + +Two consecutive calls with the same parameters return equivalent results — +the first call populates the cache, the second reads from it: + +>>> with tempfile.TemporaryDirectory() as tmpdir: +... with ( +... patch("schedview.collect.visits._is_cache_fresh", +... return_value=False), +... patch("schedview.collect.visits.read_visits", +... return_value=fake_visits), +... patch("schedview.collect.visits.DayObs.from_date") as mock_fd, +... ): +... mock_fd.side_effect = ( +... lambda a: MagicMock() if a == "today" +... else _day_obs_mock(20260612) +... ) +... result1 = cached_read_visits( +... 20260612, "lsstcam", cache_dir=tmpdir, stackers=stackers +... ) +... with ( +... patch("schedview.collect.visits._is_cache_fresh", +... return_value=True), +... patch("schedview.collect.visits.DayObs.from_date", +... return_value=_day_obs_mock(20260612)), +... ): +... result2 = cached_read_visits( +... 20260612, "lsstcam", cache_dir=tmpdir, stackers=stackers +... ) +... result1.equals(result2) +True + + +Logging +------- + +Uses ``logging.getLogger(__name__)`` at the module level. Logs at ``debug`` +level: + +- ``"Reading visits from cache: {cache_path}"`` +- ``"Cache miss or stale, querying source: {visit_source}"`` +- ``"Cache missing 'stackers' key, treating as stale: {cache_path}"`` +- ``"Cache stacker mismatch, regenerating: {cache_path}"`` +- ``"Writing visits cache: {cache_path}"`` + + +Backward Compatibility +----------------------- + +- ``read_visits`` and ``read_ddf_visits`` signatures are **unchanged**. + Caching is provided only through the new ``cached_read_visits`` function. +- ``cached_read_visits`` is exported from ``schedview.collect`` via + ``__init__.py``. +- No new required dependencies. HDF5 support via ``pytables`` is already + used elsewhere in schedview. + + +Dependencies +------------ + +- ``pandas`` (``pd.read_hdf``, ``pd.DataFrame.to_hdf``) +- ``astropy.time.Time`` (for cache freshness comparison) +- ``schedview.DayObs`` (for date/time boundaries) +- ``rubin_sim.maf`` (for stacker instances and class name introspection) +- ``rubin_scheduler.utils.consdb.KNOWN_INSTRUMENTS`` (for input validation) diff --git a/schedview/collect/__init__.py b/schedview/collect/__init__.py index 84c8aae5..cb72178d 100644 --- a/schedview/collect/__init__.py +++ b/schedview/collect/__init__.py @@ -1,6 +1,7 @@ __all__ = [ "NIGHT_STACKERS", "SAL_INDEX_GUESSES", + "cached_read_visits", "find_file_resources", "get_footprint", "get_from_logdb_with_retries", @@ -72,4 +73,4 @@ def read_multiple_prenights(*args, **kwargs): from .rewards import read_rewards from .scheduler_pickle import read_scheduler, sample_pickle from .stars import load_bright_stars -from .visits import NIGHT_STACKERS, read_visits +from .visits import NIGHT_STACKERS, cached_read_visits, read_visits diff --git a/schedview/collect/visits.py b/schedview/collect/visits.py index cec110bc..3fda7192 100644 --- a/schedview/collect/visits.py +++ b/schedview/collect/visits.py @@ -1,6 +1,10 @@ +import logging import re +import warnings +from pathlib import Path import pandas as pd +from astropy.time import Time from lsst.resources import ResourcePath from rubin_scheduler.utils import ddf_locations from rubin_scheduler.utils.consdb import KNOWN_INSTRUMENTS @@ -12,6 +16,8 @@ from .consdb import read_consdb from .opsim import read_opsim +logger = logging.getLogger(__name__) + # Use old-style format, because f-strings are not reusable OPSIMDB_TEMPLATE = ( "/sdf/group/rubin/web_data/sim-data/sims_featureScheduler_runs{sim_version}/baseline/" @@ -156,3 +162,155 @@ def read_ddf_visits(*args, **kwargs) -> pd.DataFrame: ddf_visits = pd.concat(ddf_visits) return ddf_visits + + +def _is_cache_fresh(cache_path: Path) -> bool: + """Determine whether a visits cache file is fresh enough to use. + + A cache file is considered fresh if it was last modified after yesterday's + sunrise (i.e., after the last completed night ended) and before today's + sunset (i.e., before tonight starts). This window ensures the cached data + is complete through the previous night and has not yet been superseded by + new observations. + + Parameters + ---------- + cache_path : `Path` + Path to the cache file. + + Returns + ------- + fresh : `bool` + ``True`` if the file exists and its modification time falls within the + freshness window, ``False`` otherwise. + """ + if not cache_path.exists(): + return False + + cache_mtime = Time(cache_path.stat().st_mtime, format="unix") + lower_bound = DayObs.from_date("yesterday").sunrise + upper_bound = DayObs.from_date("today").sunset + return lower_bound < cache_mtime < upper_bound + + +def cached_read_visits( + day_obs: str | int | DayObs, + visit_source: str, + cache_dir: str | Path, + stackers: list | None = None, + ddf: bool = False, +) -> pd.DataFrame: + """Read visits from a consdb source, using a local HDF5 cache when possible. + + On a cache hit (the cache file exists, is fresh, and was built with the + same set of stackers), the cached data is read from disk and filtered to + the requested ``day_obs``. On a cache miss (file absent, stale, or built + with different stackers), a full query is issued via + `read_visits` / `read_ddf_visits`, the result is written back to the + cache, and the filtered data is returned. + + The cache file is an HDF5 file with two keys: + + - ``"visits"`` — the full visits `~pandas.DataFrame` (all nights up to + the query date). + - ``"stackers"`` — a single-column `~pandas.DataFrame` (column + ``"class_name"``) recording the fully-qualified class name of each + stacker used to produce the cached data. Used to detect stale caches + caused by a change in the requested stacker set. + + Parameters + ---------- + day_obs : `str` or `int` or `DayObs` + The night of observing as a dayobs. Visits up to and including this + night are returned. + visit_source : `str` + A consdb instrument name (e.g. ``"lsstcam"``, ``"latiss"``). Only + sources in `~rubin_scheduler.utils.consdb.KNOWN_INSTRUMENTS` are + supported; a `ValueError` is raised otherwise. + cache_dir : `str` or `Path` + Directory where cache files are stored. Created automatically if it + does not exist. + stackers : `list` or ``None``, optional + Stacker instances to apply when querying the source. If ``None``, + defaults to `NIGHT_STACKERS` (when ``ddf=False``) or + ``DDF_STACKERS + [maf.stackers.DayObsStacker()]`` (when ``ddf=True``). + ddf : `bool`, optional + If ``True``, use `read_ddf_visits` instead of `read_visits` (filters + results to DDF fields and uses DDF-appropriate stackers). + + Returns + ------- + visits : `~pandas.DataFrame` + Visits for nights up to and including ``day_obs``. + + Raises + ------ + ValueError + If ``visit_source`` is not a known consdb instrument. + """ + if visit_source not in KNOWN_INSTRUMENTS: + raise ValueError( + f"cached_read_visits only supports consdb instruments " + f"({', '.join(sorted(KNOWN_INSTRUMENTS))}), got {visit_source!r}." + ) + + # Resolve default stackers. + if stackers is None: + if ddf: + stackers = DDF_STACKERS + [maf.stackers.DayObsStacker()] + else: + stackers = NIGHT_STACKERS + + cache_dir = Path(cache_dir) + suffix = "_ddf" if ddf else "" + cache_path = cache_dir / f"visits_{visit_source}{suffix}.h5" + + requested_class_names = {type(s).__module__ + "." + type(s).__qualname__ for s in stackers} + day_obs_obj = DayObs.from_date(day_obs) + + # Attempt a cache hit. + if _is_cache_fresh(cache_path): + try: + cached_class_names = set(pd.read_hdf(str(cache_path), key="stackers")["class_name"]) + except KeyError: + logger.debug("Cache missing 'stackers' key, treating as stale: %s", cache_path) + all_visits = None + cached_class_names = None + else: + if cached_class_names == requested_class_names: + logger.debug("Reading visits from cache: %s", cache_path) + all_visits = pd.read_hdf(str(cache_path), key="visits") + else: + logger.debug("Cache stacker mismatch, regenerating: %s", cache_path) + all_visits = None + else: + logger.debug("Cache miss or stale, querying source: %s", visit_source) + all_visits = None + + # Cache miss — query the source for all available history. + if all_visits is None: + fetch_kwargs = dict(stackers=stackers, num_nights=365 * 20) + today = DayObs.from_date("today") + if ddf: + all_visits = read_ddf_visits(today, visit_source, **fetch_kwargs) + else: + all_visits = read_visits(today, visit_source, **fetch_kwargs) + + logger.debug("Writing visits cache: %s", cache_path) + cache_dir.mkdir(parents=True, exist_ok=True) + all_visits.to_hdf(str(cache_path), key="visits", mode="w") + pd.DataFrame({"class_name": sorted(requested_class_names)}).to_hdf( + str(cache_path), key="stackers", mode="a" + ) + + # Filter to the requested day_obs. + if "dayObs" not in all_visits.columns: + warnings.warn( + "The visits DataFrame does not have a 'dayObs' column; " + "returning unfiltered data. Add maf.stackers.DayObsStacker() " + "to the stackers list to enable day_obs filtering.", + UserWarning, + stacklevel=2, + ) + return all_visits + return all_visits.loc[all_visits["dayObs"] <= day_obs_obj.yyyymmdd, :] diff --git a/schedview/compute/__init__.py b/schedview/compute/__init__.py index 15a250eb..dab7767e 100644 --- a/schedview/compute/__init__.py +++ b/schedview/compute/__init__.py @@ -26,6 +26,8 @@ "munge_sim_archive_metadata", "find_nearest_pointing_ids", "combine_completed_with_sims", + "compute_tinysum", + "compute_smallsum", ] from .astro import compute_sun_moon_positions, convert_evening_date_to_night_of_survey, night_events @@ -52,6 +54,7 @@ replay_visits, ) from .sim_archive import munge_sim_archive_metadata +from .smallsum import compute_smallsum, compute_tinysum from .survey import compute_maps, make_survey_reward_df try: diff --git a/schedview/compute/smallsum.py b/schedview/compute/smallsum.py new file mode 100644 index 00000000..83e5edd9 --- /dev/null +++ b/schedview/compute/smallsum.py @@ -0,0 +1,280 @@ +"""Functions for creating nightly summary DataFrames from visits data. + +This module provides two functions: + +- ``compute_tinysum``: one row per night with key statistics. +- ``compute_smallsum``: multiple rows per night, broken out by subset + (band, science/not-science, observation_reason, target name). +""" + +__all__ = ["compute_tinysum", "compute_smallsum"] + +from datetime import timedelta + +import numpy as np +import pandas as pd +from rubin_scheduler.scheduler.model_observatory import ModelObservatory +from rubin_scheduler.site_models import Almanac + +from schedview import DayObs + +try: + from rubin_nights.reference_values import SCIENCE_PROGRAMS +except ImportError: + SCIENCE_PROGRAMS = () + +_BANDS: tuple[str, ...] = tuple(ModelObservatory(no_sky=True).bandlist) + + +def _unique_targets(target_name_series: pd.Series) -> str: + """Aggregate target names into a comma-separated string of unique values. + + Strips ``ddf_`` or ``DDF `` prefixes, splits multi-target entries on + ``', '``, and returns a deduplicated comma-separated string. + + Parameters + ---------- + target_name_series : `pandas.Series` + Series of target_name values (strings) for a single night. + + Returns + ------- + result : `str` + Comma-separated unique target names. + """ + targets: list[str] = [] + for value in target_name_series.dropna(): + if not isinstance(value, str): + continue + value = value.strip() + if len(value) == 0: + continue + if value.startswith("ddf_") or value.startswith("DDF "): + value = value[4:] + for subtarget in value.split(", "): + subtarget = subtarget.strip() + if subtarget and subtarget not in targets: + targets.append(subtarget) + return ", ".join(targets) + + +def _build_night_hours(almanac: Almanac, dayobs_values: pd.Index) -> pd.Series: + """Build a Series mapping dayObs (YYYYMMDD int) to night hours. + + Parameters + ---------- + almanac : `Almanac` + Almanac instance with sunset data. + dayobs_values : `pandas.Index` + The dayObs index values for which to compute night hours. + + Returns + ------- + night_hours : `pandas.Series` + Series indexed by dayObs with night duration in hours. + """ + sunsets = pd.DataFrame(almanac.sunsets) + night0 = sunsets[sunsets["night"] == 0] + if night0.empty: + raise ValueError("Almanac.sunsets has no night 0; cannot compute night hours.") + start_date = DayObs.from_time(night0.iloc[0]["sunset"]).date + + sunsets["dayObs"] = [ + int((start_date + timedelta(days=int(n))).strftime("%Y%m%d")) for n in sunsets["night"] + ] + sunsets = sunsets.set_index("dayObs") + night_hours = (sunsets["sun_n12_rising"] - sunsets["sun_n12_setting"]) * 24.0 + return night_hours.reindex(dayobs_values) + + +def _visits_summary(visits_group: pd.DataFrame) -> pd.Series: + """Compute summary statistics for a group of visits. + + Parameters + ---------- + visits_group : `pandas.DataFrame` + A subset of visits (one group from a groupby operation). + + Returns + ------- + summary : `pandas.Series` + Series with keys: visits, first, last, teff_total, teff_q1, + teff_median, teff_q3, fwhm_median, airmass_median, HA_median. + """ + teff = visits_group["eff_time_median"] + return pd.Series( + { + "visits": len(visits_group), + "first": visits_group["start_timestamp"].min(), + "last": visits_group["start_timestamp"].max(), + "teff_total": np.nan_to_num(teff.to_numpy(), nan=0.0).sum(), + "teff_q1": teff.quantile(0.25), + "teff_median": teff.median(), + "teff_q3": teff.quantile(0.75), + "fwhm_median": visits_group["seeingFwhmGeom"].median(), + "airmass_median": visits_group["airmass"].median(), + "HA_median": visits_group["HA"].median(), + } + ) + + +def compute_tinysum( + visits: pd.DataFrame, + science_programs: tuple[str, ...] = SCIENCE_PROGRAMS, + almanac: Almanac | None = None, +) -> pd.DataFrame: + """Create a one-row-per-night summary DataFrame from visits. + + Parameters + ---------- + visits : `pandas.DataFrame` + DataFrame of visits. Must contain columns: ``dayObs`` (int, + YYYYMMDD), ``observationId``, ``seeingFwhmGeom``, + ``eff_time_median``, ``band``, ``science_program``, + ``target_name``. + science_programs : `tuple` [`str`], optional + Tuple of ``science_program`` values considered science. + Defaults to ``SCIENCE_PROGRAMS`` from + ``rubin_nights.reference_values``. + almanac : `Almanac` or `None`, optional + A ``rubin_scheduler.site_models.Almanac`` instance used to + compute night duration. + Pass ``None`` to omit the ``night_hours``, ``visits/hour``, + and ``teff/minute`` columns. + + Returns + ------- + tinysum : `pandas.DataFrame` + DataFrame indexed by ``dayObs`` with one row per night. + """ + basic_stats = ( + visits.groupby("dayObs") + .agg( + { + "observationId": "count", + "seeingFwhmGeom": "median", + "exp_time": "sum", + "eff_time_median": "sum", + } + ) + .sort_index() + .rename( + columns={ + "observationId": "Total", + "seeingFwhmGeom": "median FWHM", + "exp_time": "total exp_time", + "eff_time_median": "total eff_time", + } + ) + ) + basic_stats["Total"] = basic_stats["Total"].astype("Int64") + + teff_stats = ( + visits.groupby("dayObs")["eff_time_median"] + .describe() + .loc[:, ["mean", "25%", "50%", "75%"]] + .rename( + columns={ + "mean": "mean eff_time", + "25%": "q1 eff_time", + "50%": "median eff_time", + "75%": "q3 eff_time", + } + ) + ) + + band_counts = ( + visits.groupby(["dayObs", "band"])["observationId"] + .count() + .unstack("band", fill_value=0) + .reindex(columns=_BANDS, fill_value=0) + .astype("Int64") + .rename(columns={b: f"# {b}" for b in _BANDS}) + ) + + science_visits = visits.loc[visits["science_program"].isin(science_programs), :] + science_counts = science_visits.groupby("dayObs")["observationId"].count().rename("science").to_frame() + + targets = ( + science_visits.groupby("dayObs")["target_name"] + .apply(_unique_targets) + .rename("science targets") + .to_frame() + ) + + tinysum = basic_stats.join([teff_stats, science_counts, band_counts, targets]) + + tinysum["total eff_time/exp_time"] = tinysum["total eff_time"] / tinysum["total exp_time"] + + tinysum["science"] = tinysum["science"].fillna(0).astype("Int64") + tinysum["science targets"] = tinysum["science targets"].fillna("") + + if almanac is not None: + night_hours = _build_night_hours(almanac, tinysum.index) + tinysum["night_hours"] = night_hours + tinysum["visits/hour"] = tinysum["Total"] / tinysum["night_hours"] + tinysum["teff/minute"] = tinysum["visits/hour"] * tinysum["mean eff_time"] / 60.0 + + return tinysum + + +def compute_smallsum( + visits: pd.DataFrame, + science_programs: tuple[str, ...] = SCIENCE_PROGRAMS, +) -> pd.DataFrame: + """Create a multi-row-per-night summary DataFrame from visits. + + Each night has rows for several subsets: all visits, by band, + science/not-science, by observation_reason, and by target name. + + Parameters + ---------- + visits : `pandas.DataFrame` + DataFrame of visits. Must contain columns: ``dayObs``, + ``observationId``, ``start_timestamp``, ``eff_time_median``, + ``seeingFwhmGeom``, ``airmass``, ``HA``, ``band``, + ``science_program``, ``observation_reason``, ``target_name``. + science_programs : `tuple` [`str`], optional + Tuple of ``science_program`` values considered science. + Defaults to ``SCIENCE_PROGRAMS`` from + ``rubin_nights.reference_values``. + + Returns + ------- + smallsum : `pandas.DataFrame` + DataFrame with a two-level index (``dayObs``, ``subset``). + """ + fullnight = visits.groupby("dayObs").apply(_visits_summary, include_groups=False) + fullnight["subset"] = "all" + fullnight = fullnight.reset_index().set_index(["dayObs", "subset"]) + + byband = visits.groupby(["dayObs", "band"]).apply(_visits_summary, include_groups=False) + byband = byband.rename_axis(index={"band": "subset"}) + + visits_with_science = visits.copy() + visits_with_science["_science"] = np.where( + visits_with_science["science_program"].isin(science_programs), + "science", + "not_science", + ) + byscience = visits_with_science.groupby(["dayObs", "_science"]).apply( + _visits_summary, include_groups=False + ) + byscience = byscience.rename_axis(index={"_science": "subset"}) + + byreason = visits.groupby(["dayObs", "observation_reason"]).apply(_visits_summary, include_groups=False) + byreason = byreason.rename_axis(index={"observation_reason": "subset"}) + + visits_with_targets = visits.copy() + visits_with_targets["_target_names"] = ( + visits_with_targets["target_name"].fillna("").astype(str).str.split(", ") + ) + bytarget = visits_with_targets.explode("_target_names") + bytarget["_target_names"] = bytarget["_target_names"].fillna("").str.strip() + bytarget.loc[bytarget["_target_names"] == "", "_target_names"] = "no target name" + bytarget = bytarget.groupby(["dayObs", "_target_names"]).apply(_visits_summary, include_groups=False) + bytarget = bytarget.rename_axis(index={"_target_names": "subset"}) + + smallsum = pd.concat([fullnight, byband, byscience, byreason, bytarget]) + smallsum = smallsum.sort_index(level="dayObs", sort_remaining=False) + return smallsum diff --git a/schedview/reports.py b/schedview/reports.py index f1a883f4..b96b3ba0 100644 --- a/schedview/reports.py +++ b/schedview/reports.py @@ -4,7 +4,21 @@ import xml.etree.ElementTree as ET from pathlib import Path +import numpy as np import pandas as pd +from rubin_scheduler.site_models import Almanac + +from schedview.compute.smallsum import compute_tinysum + +RSS_DESC_FORMAT = """ +Total visits: {total}; +Science visits: {science}; +Median FWHM: {fwhm}; +Total eff_time/exp_time: {mean_norm_teff}; +Mean visit rate: {visit_rate} visits/hour; +Mean eff_time rate: {teff_rate}/minute; +Science targets: {targets} +""" def find_reports( @@ -81,9 +95,39 @@ def find_reports( return reports +INT_SUMMARY_COLUMNS = [ + "Total", + "science", + # "# u", + # "# g", + # "# r", + # "# i", + # "# z", + # "# y", +] + +FLOAT_SUMMARY_COLUMNS = [ + "night_hours", + # "visits/hour", + # "teff/minute", + "median FWHM", + # "mean eff_time", + # "q1 eff_time", + # "median eff_time", + # "q3 eff_time", + "total eff_time/exp_time", +] + +# SUMMARY_COLUMNS = INT_SUMMARY_COLUMNS + FLOAT_SUMMARY_COLUMNS + [ +# "science targets", +# ] +SUMMARY_COLUMNS = INT_SUMMARY_COLUMNS + FLOAT_SUMMARY_COLUMNS + + def make_report_link_table( reports: pd.DataFrame, - report_columns=("prenight", "multiprenight", "nightsum", "compareprenight", "preprogress"), + report_columns=("prenight", "multiprenight", "nightsum", "compareprenight"), + visits: pd.DataFrame | None = None, ) -> str: """Generate an html table of links to reports. @@ -93,6 +137,14 @@ def make_report_link_table( A DataFrame of report metadata, as returned by `find_reports`. report_columns : `tuple` A list of names of reports. + visits : `pd.DataFrame` or `None`, optional + A DataFrame of visits as returned by + `schedview.collect.visits.cached_read_visits`. If supplied, a short + per-night summary is computed via + `schedview.compute.smallsum.compute_tinysum` and the resulting + columns are joined onto the ``lsstcam`` rows of the table. + Non-``lsstcam`` rows receive ``NA`` for these columns. + Defaults to ``None``, in which case no summary columns are added. Returns ------- @@ -108,56 +160,133 @@ def make_report_link_table( .reindex(columns=list(report_columns)) ) - report_table_html = report_links.to_html(escape=False) + if visits is not None: + almanac = Almanac() + tinysum = compute_tinysum(visits, almanac=almanac)[SUMMARY_COLUMNS] + tinysum.index = pd.to_datetime(tinysum.index.astype(str), format="%Y%m%d").date + tinysum.index.name = "night" + + # Build a mask for lsstcam rows and extract their nights + lsstcam_mask = report_links.index.get_level_values("instrument") == "lsstcam" + _ = report_links.index[lsstcam_mask].get_level_values("night") + + # Map tinysum by night onto the full report_links index, then join. + # Reindex to the full index (non-lsstcam rows and missing nights get + # NA) so that dtypes (including Int64) are preserved through + # the assignment. + summary_full = tinysum.reindex(report_links.index.get_level_values("night")) + summary_full.index = report_links.index + # Only lsstcam rows should receive values; blank out the rest + summary_full.loc[~lsstcam_mask] = None + for col in SUMMARY_COLUMNS: + report_links[col] = summary_full[col] + + # Round float columns to 2 decimal places, then convert all summary + # columns to object dtype so fillna("") works uniformly regardless of + # whether pandas chose Int64, Float64, etc. + summary_cols_present = [c for c in SUMMARY_COLUMNS if c in report_links.columns] + float_cols_present = [c for c in FLOAT_SUMMARY_COLUMNS if c in report_links.columns] + for col in float_cols_present: + report_links[col] = report_links[col].round(2) + for col in summary_cols_present: + report_links[col] = report_links[col].astype(object) + + report_table_html = report_links.fillna("").to_html(escape=False) return report_table_html def make_report_rss_feed( - reports: pd.DataFrame, fname: str | None = None, max_days: int = 7 + reports: pd.DataFrame, + fname: str | None = None, + max_days: int = 7, + visits: pd.DataFrame | None = None, + title: str = "schedview reports", + description: str = "Statically generated reports on Rubin Observatory/LSST scheduler status and progress", ) -> ET.ElementTree: """Generate an rss feed of recent schedview reports. - Parameters - ---------- - reports : `pd.DataFrame` - A DataFrame of report metadata, as returned by `find_reports`. - fname : `str` or `None` - The file in which to write the RSS, if any. `None` to not write - a file at all. Defaults to `None`. - max_days : `int` - How many days worth of reports to include in the feed. + Parameters + ---------- + reports : `pd.DataFrame` + A DataFrame of report metadata, as returned by `find_reports`. + fname : `str` or `None` + The file in which to write the RSS, if any. `None` to not write + a file at all. Defaults to `None`. + max_days : `int` + How many days worth of reports to include in the feed. + visits : `pd.DataFrame` or `None`, optional + A DataFrame of visits as returned by + `schedview.collect.visits.cached_read_visits`. If supplied, a short + per-night summary is computed via + `schedview.compute.smallsum.compute_tinysum` and the resulting + columns are joined onto the ``lsstcam`` rows of the table. + Non-``lsstcam`` rows receive ``NA`` for these columns. + Defaults to ``None``, in which case no summary columns are added. + title: `str`, optional + The channel title, defaults to ``schedview reports`` - Returns - ------- - rss : `ET.ElementTree` - The RSS XML itself. + + Returns + ------- + rss : `ET.ElementTree` + The RSS XML itself. """ + if visits is not None: + almanac = Almanac() + tinysum = compute_tinysum(visits, almanac=almanac) + else: + tinysum = None + rss = ET.Element("rss", attrib={"version": "2.0"}) channel = ET.SubElement(rss, "channel") - title = ET.SubElement(channel, "title") - title.text = "schedview reports" + channel_title_elem = ET.SubElement(channel, "title") + channel_title_elem.text = title desc = ET.SubElement(channel, "description") - desc.text = "Statically generated reports on Rubin Observatory/LSST scheduler status and progress" + desc.text = description for row_index, report_row in reports.iterrows(): if (datetime.date.today() - report_row.night).days > max_days: # To make sure we keep the feed a reasonable size, # don't include older stuff. continue - instrument, dayobs = row_index + instrument, dayobs_str = row_index + dayobs = int(dayobs_str) + item = ET.SubElement(channel, "item") - title = ET.SubElement(item, "title") - title.text = f"{report_row.report} report for {instrument} on {report_row.night}" + item_title_elem = ET.SubElement(item, "title") + item_title_elem.text = f"{report_row.report} for {instrument} on {report_row.night}" # It's traditional to put a summary of the content in "description" # and we could eventually have things like total numbers # of visits in each band, stats on the seeing, etc. here. # We can do this when our activities at night are no # longer secret. desc = ET.SubElement(item, "description") - desc.text = f"{report_row.report} report for {instrument} on {report_row.night}" + if instrument == "lsstcam" and report_row.report == "nightsum" and tinysum is not None: + if dayobs in tinysum.index: + try: + teff_rate = np.round(tinysum.loc[dayobs, "teff/minute"], 2) + except TypeError: + teff_rate = np.nan + desc.text = RSS_DESC_FORMAT.format( + report=report_row.report, + instrument=instrument, + night=report_row.night, + total=tinysum.loc[dayobs, "Total"], + science=tinysum.loc[dayobs, "science"], + fwhm=np.round(tinysum.loc[dayobs, "median FWHM"], 2), + mean_norm_teff=np.round(tinysum.loc[dayobs, "total eff_time/exp_time"], 2), + visit_rate=np.round(tinysum.loc[dayobs, "visits/hour"], 2), + teff_rate=teff_rate, + targets=tinysum.loc[dayobs, "science targets"], + ) + else: + desc.text = "No visits on this night" + + else: + desc.text = "" link = ET.SubElement(item, "link") link.text = report_row.url guid = ET.SubElement(item, "guid", attib={"isPermaLink": "false"}) - guid.text = title.text + f", generated {report_row.report_time}" + guid.text = item_title_elem.text + f", generated {report_row.report_time}" category = ET.SubElement(item, "category") category.text = f"{instrument}_{report_row.report}" pubdate = ET.SubElement(item, "pubDate") diff --git a/tests/test_collect_visits_cache.py b/tests/test_collect_visits_cache.py new file mode 100644 index 00000000..296ff9c4 --- /dev/null +++ b/tests/test_collect_visits_cache.py @@ -0,0 +1,200 @@ +"""Black-box tests for cached_read_visits public API.""" + +import tempfile +import unittest +import warnings +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pandas as pd + +from schedview.collect.visits import cached_read_visits + + +class TestCachedReadVisitsAPI(unittest.TestCase): + """Tests for the public contract of cached_read_visits.""" + + def _make_fake_visits(self, day_obs_values=None): + if day_obs_values is None: + day_obs_values = [ + 20260610, + 20260611, + 20260612, + 20260613, + 20260614, + ] + return pd.DataFrame( + { + "dayObs": day_obs_values, + "observationId": range(len(day_obs_values)), + } + ) + + def _day_obs_mock(self, yyyymmdd): + m = MagicMock() + m.yyyymmdd = yyyymmdd + return m + + def test_raises_for_non_instrument_source(self): + with self.assertRaises(ValueError): + cached_read_visits( + 20260614, "baseline", cache_dir="/tmp/cache" + ) + + def test_raises_for_opsim_file_source(self): + with self.assertRaises(ValueError): + cached_read_visits( + 20260614, "/path/to/sim.db", cache_dir="/tmp/cache" + ) + + def test_filters_to_requested_day_obs(self): + """Results contain only visits up to the requested day_obs.""" + fake_visits = self._make_fake_visits() + + with tempfile.TemporaryDirectory() as tmpdir: + with ( + patch( + "schedview.collect.visits._is_cache_fresh", + return_value=False, + ), + patch( + "schedview.collect.visits.read_visits", + return_value=fake_visits, + ), + patch( + "schedview.collect.visits.DayObs.from_date", + ) as mock_from_date, + ): + mock_from_date.side_effect = lambda arg: ( + MagicMock() + if arg == "today" + else self._day_obs_mock(20260612) + ) + result = cached_read_visits( + 20260612, "lsstcam", cache_dir=tmpdir + ) + + self.assertTrue(all(result["dayObs"] <= 20260612)) + + def test_warns_when_no_dayobs_column(self): + """A warning is issued if dayObs column is absent.""" + fake_visits = pd.DataFrame({"observationId": [1, 2, 3]}) + + with tempfile.TemporaryDirectory() as tmpdir: + with ( + patch( + "schedview.collect.visits._is_cache_fresh", + return_value=False, + ), + patch( + "schedview.collect.visits.read_visits", + return_value=fake_visits, + ), + patch( + "schedview.collect.visits.DayObs.from_date", + ) as mock_from_date, + ): + mock_from_date.side_effect = lambda arg: ( + MagicMock() + if arg == "today" + else self._day_obs_mock(20260614) + ) + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + result = cached_read_visits( + 20260614, "lsstcam", cache_dir=tmpdir + ) + + self.assertEqual(len(result), 3) + self.assertTrue( + any("dayObs" in str(warning.message) for warning in w) + ) + + def test_cache_dir_created_if_absent(self): + """cache_dir is created automatically when absent.""" + fake_visits = self._make_fake_visits() + + with tempfile.TemporaryDirectory() as tmpdir: + new_cache_dir = Path(tmpdir) / "new" / "subdir" + self.assertFalse(new_cache_dir.exists()) + + with ( + patch( + "schedview.collect.visits._is_cache_fresh", + return_value=False, + ), + patch( + "schedview.collect.visits.read_visits", + return_value=fake_visits, + ), + patch( + "schedview.collect.visits.DayObs.from_date", + ) as mock_from_date, + ): + mock_from_date.side_effect = lambda arg: ( + MagicMock() + if arg == "today" + else self._day_obs_mock(20260614) + ) + cached_read_visits( + 20260614, "lsstcam", cache_dir=new_cache_dir + ) + + self.assertTrue(new_cache_dir.exists()) + + def test_second_call_returns_same_data(self): + """Two calls with same params return equivalent results.""" + from rubin_sim import maf + + fake_visits = self._make_fake_visits() + stackers = [maf.stackers.DayObsStacker()] + + with tempfile.TemporaryDirectory() as tmpdir: + with ( + patch( + "schedview.collect.visits._is_cache_fresh", + return_value=False, + ), + patch( + "schedview.collect.visits.read_visits", + return_value=fake_visits, + ), + patch( + "schedview.collect.visits.DayObs.from_date", + ) as mock_from_date, + ): + mock_from_date.side_effect = lambda arg: ( + MagicMock() + if arg == "today" + else self._day_obs_mock(20260614) + ) + result1 = cached_read_visits( + 20260614, + "lsstcam", + cache_dir=tmpdir, + stackers=stackers, + ) + + # Second call with cache now populated + with ( + patch( + "schedview.collect.visits._is_cache_fresh", + return_value=True, + ), + patch( + "schedview.collect.visits.DayObs.from_date", + return_value=self._day_obs_mock(20260614), + ), + ): + result2 = cached_read_visits( + 20260614, + "lsstcam", + cache_dir=tmpdir, + stackers=stackers, + ) + + pd.testing.assert_frame_equal(result1, result2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_compute_smallsum.py b/tests/test_compute_smallsum.py new file mode 100644 index 00000000..bcc4bfce --- /dev/null +++ b/tests/test_compute_smallsum.py @@ -0,0 +1,320 @@ +"""Black-box tests for schedview.compute.smallsum public API.""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest +from rubin_scheduler.scheduler.model_observatory import ModelObservatory +from rubin_scheduler.site_models import Almanac + +from schedview.compute.smallsum import compute_smallsum, compute_tinysum + +SCIENCE_PROGRAMS = ("BLOCK-365", "BLOCK-407") +ALL_BANDS: tuple[str, ...] = tuple(ModelObservatory(no_sky=True).bandlist) + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def sample_visits() -> pd.DataFrame: + """Create a sample visits DataFrame spanning two nights.""" + rng = np.random.default_rng(42) + n_night1 = 50 + n_night2 = 30 + n_total = n_night1 + n_night2 + + dayobs = np.array([20250601] * n_night1 + [20250602] * n_night2) + bands = rng.choice(ALL_BANDS, size=n_total) + science_programs = rng.choice( + [*SCIENCE_PROGRAMS, "ENG-001", "CALIB-002"], + size=n_total, + ) + target_names = rng.choice( + ["COSMOS", "XMM-LSS", "ddf_COSMOS", "ELAIS, XMM-LSS", ""], + size=n_total, + ) + observation_reasons = rng.choice( + ["survey", "calibration", "engineering"], + size=n_total, + ) + + # Timestamps: night 1 starts at MJD 60827, night 2 at MJD 60828 + base_timestamps_n1 = 60827.0 * 86400 + np.linspace(0, 8 * 3600, n_night1) + base_timestamps_n2 = 60828.0 * 86400 + np.linspace(0, 7 * 3600, n_night2) + start_timestamps = np.concatenate([base_timestamps_n1, base_timestamps_n2]) + + return pd.DataFrame( + { + "dayObs": dayobs, + "observationId": np.arange(n_total), + "seeingFwhmGeom": rng.uniform(0.6, 1.8, size=n_total), + "eff_time_median": rng.uniform(20.0, 40.0, size=n_total), + "exp_time": rng.uniform(25.0, 35.0, size=n_total), + "band": bands, + "science_program": science_programs, + "target_name": target_names, + "observation_reason": observation_reasons, + "start_timestamp": start_timestamps, + "airmass": rng.uniform(1.0, 2.5, size=n_total), + "HA": rng.uniform(-4.0, 4.0, size=n_total), + } + ) + + +@pytest.fixture +def almanac() -> Almanac: + """Create a shared Almanac instance.""" + return Almanac() + + +# --------------------------------------------------------------------------- +# Tests for compute_tinysum +# --------------------------------------------------------------------------- + + +class TestComputeTinysum: + def test_one_row_per_night(self, sample_visits): + result = compute_tinysum(sample_visits) + assert len(result) == 2 + assert {20250601, 20250602}.issubset(set(result.index)) + + def test_total_column(self, sample_visits): + result = compute_tinysum(sample_visits) + assert result.loc[20250601, "Total"] == 50 + assert result.loc[20250602, "Total"] == 30 + + def test_band_counts_sum_to_total(self, sample_visits): + result = compute_tinysum(sample_visits) + band_cols = [f"# {b}" for b in ALL_BANDS] + for day in result.index: + assert result.loc[day, band_cols].sum() == result.loc[day, "Total"] + + def test_band_and_science_counts_are_int(self, sample_visits): + result = compute_tinysum(sample_visits) + for b in ALL_BANDS: + assert pd.api.types.is_integer_dtype(result[f"# {b}"]) + assert pd.api.types.is_integer_dtype(result["science"]) + + def test_science_count_correct(self, sample_visits): + result = compute_tinysum( + sample_visits, science_programs=SCIENCE_PROGRAMS + ) + for day in result.index: + expected = len( + sample_visits[ + (sample_visits["dayObs"] == day) + & ( + sample_visits["science_program"].isin( + SCIENCE_PROGRAMS + ) + ) + ] + ) + assert result.loc[day, "science"] == expected + + def test_teff_stats_reasonable(self, sample_visits): + result = compute_tinysum(sample_visits) + for day in result.index: + assert ( + result.loc[day, "q1 eff_time"] + <= result.loc[day, "median eff_time"] + <= result.loc[day, "q3 eff_time"] + ) + + def test_total_eff_time_over_exp_time(self, sample_visits): + result = compute_tinysum(sample_visits) + for day in result.index: + day_visits = sample_visits[sample_visits["dayObs"] == day] + expected = ( + day_visits["eff_time_median"].sum() + / day_visits["exp_time"].sum() + ) + assert np.isclose( + result.loc[day, "total eff_time/exp_time"], expected + ) + + def test_no_almanac_omits_rate_columns(self, sample_visits): + result = compute_tinysum(sample_visits, almanac=None) + assert {"night_hours", "visits/hour", "teff/minute"}.isdisjoint( + result.columns + ) + + def test_with_almanac_adds_rate_columns(self, sample_visits, almanac): + result = compute_tinysum(sample_visits, almanac=almanac) + assert {"night_hours", "visits/hour", "teff/minute"}.issubset( + result.columns + ) + assert (result["night_hours"].dropna() > 0).all() + + def test_no_science_visits_night(self): + visits = pd.DataFrame( + { + "dayObs": [20250601] * 5, + "observationId": range(5), + "seeingFwhmGeom": [1.0] * 5, + "eff_time_median": [30.0] * 5, + "exp_time": [30.0] * 5, + "band": list("grriz"), + "science_program": ["ENG-001"] * 5, + "target_name": ["test"] * 5, + } + ) + result = compute_tinysum(visits, science_programs=("BLOCK-365",)) + assert result.loc[20250601, "science"] == 0 + assert result.loc[20250601, "science targets"] == "" + + def test_missing_bands(self): + visits = pd.DataFrame( + { + "dayObs": [20250601] * 3, + "observationId": range(3), + "seeingFwhmGeom": [1.0] * 3, + "eff_time_median": [30.0] * 3, + "exp_time": [30.0] * 3, + "band": ["g", "r", "i"], + "science_program": ["BLOCK-365"] * 3, + "target_name": ["COSMOS"] * 3, + } + ) + result = compute_tinysum(visits, science_programs=("BLOCK-365",)) + assert result.loc[20250601, "# g"] == 1 + for b in ("u", "z", "y"): + assert result.loc[20250601, f"# {b}"] == 0 + + +# --------------------------------------------------------------------------- +# Tests for compute_smallsum +# --------------------------------------------------------------------------- + + +class TestComputeSmallsum: + def test_index_levels(self, sample_visits): + result = compute_smallsum(sample_visits) + assert result.index.names == ["dayObs", "subset"] + + def test_all_subset_present(self, sample_visits): + result = compute_smallsum(sample_visits) + for day in (20250601, 20250602): + assert "all" in result.loc[day].index.get_level_values("subset") + + def test_all_visits_count(self, sample_visits): + result = compute_smallsum(sample_visits) + assert result.loc[(20250601, "all"), "visits"] == 50 + assert result.loc[(20250602, "all"), "visits"] == 30 + + def test_band_subsets_present_and_sum(self, sample_visits): + result = compute_smallsum(sample_visits) + all_subsets = result.index.get_level_values("subset").unique() + bands_present = [b for b in ALL_BANDS if b in all_subsets] + assert bands_present + for day in (20250601, 20250602): + day_data = result.loc[day] + day_bands = [ + b + for b in ALL_BANDS + if b in day_data.index.get_level_values("subset") + ] + assert ( + sum(day_data.loc[b, "visits"] for b in day_bands) + == day_data.loc["all", "visits"] + ) + + def test_science_subsets_present_and_sum(self, sample_visits): + result = compute_smallsum( + sample_visits, science_programs=SCIENCE_PROGRAMS + ) + all_subsets = result.index.get_level_values("subset").unique() + assert {"science", "not_science"}.issubset(all_subsets) + for day in (20250601, 20250602): + day_data = result.loc[day] + if {"science", "not_science"}.issubset(day_data.index): + assert ( + day_data.loc["science", "visits"] + + day_data.loc["not_science", "visits"] + == day_data.loc["all", "visits"] + ) + + def test_observation_reason_subsets(self, sample_visits): + result = compute_smallsum(sample_visits) + all_subsets = result.index.get_level_values("subset").unique() + assert any( + r in all_subsets + for r in ("survey", "calibration", "engineering") + ) + + def test_target_name_subsets(self, sample_visits): + result = compute_smallsum(sample_visits) + all_subsets = result.index.get_level_values("subset").unique() + assert any( + t in all_subsets for t in ("COSMOS", "XMM-LSS", "ELAIS") + ) + + def test_empty_target_becomes_no_target_name(self): + visits = pd.DataFrame( + { + "dayObs": [20250601] * 3, + "observationId": range(3), + "seeingFwhmGeom": [1.0] * 3, + "eff_time_median": [30.0] * 3, + "band": ["g", "r", "i"], + "science_program": ["ENG-001"] * 3, + "target_name": ["", "", ""], + "observation_reason": ["survey"] * 3, + "start_timestamp": [1.0, 2.0, 3.0], + "airmass": [1.2, 1.3, 1.4], + "HA": [0.1, 0.2, 0.3], + } + ) + result = compute_smallsum(visits, science_programs=("BLOCK-365",)) + assert ( + "no target name" + in result.loc[20250601].index.get_level_values("subset") + ) + + def test_output_columns(self, sample_visits): + result = compute_smallsum(sample_visits) + assert set(result.columns) == { + "visits", + "first", + "last", + "teff_total", + "teff_q1", + "teff_median", + "teff_q3", + "fwhm_median", + "airmass_median", + "HA_median", + } + + def test_teff_quartile_ordering(self, sample_visits): + result = compute_smallsum(sample_visits) + assert (result["teff_q1"] <= result["teff_median"]).all() + assert (result["teff_median"] <= result["teff_q3"]).all() + + def test_first_before_last(self, sample_visits): + result = compute_smallsum(sample_visits) + assert (result["first"] <= result["last"]).all() + + def test_subset_order_matches_contract(self, sample_visits): + result = compute_smallsum( + sample_visits, science_programs=SCIENCE_PROGRAMS + ) + day = result.index.get_level_values("dayObs")[0] + subsets = list(result.loc[day].index) + + assert subsets[0] == "all" + + first_band_pos = min( + subsets.index(b) for b in ALL_BANDS if b in subsets + ) + first_science_pos = min( + subsets.index(s) + for s in ("science", "not_science") + if s in subsets + ) + + assert first_band_pos < first_science_pos diff --git a/tests/test_reports.py b/tests/test_reports.py index c606cbc4..406143f7 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -1,8 +1,13 @@ +"""Black-box tests for schedview.reports public API.""" + import unittest import xml.etree.ElementTree as ET from pathlib import Path from tempfile import TemporaryDirectory +import numpy as np +import pandas as pd + import schedview.reports @@ -20,9 +25,13 @@ def setUpClass(cls): for report in cls.reports: for instrument in cls.instruments: for day in cls.days: - test_dir = Path(cls.temp_dir.name).joinpath(report, instrument, year, month, day) + test_dir = Path(cls.temp_dir.name).joinpath( + report, instrument, year, month, day + ) test_dir.mkdir(parents=True) - test_file = test_dir.joinpath(f"{report}_{year}-{month}-{day}.html") + test_file = test_dir.joinpath( + f"{report}_{year}-{month}-{day}.html" + ) cls.test_report_fnames.append(test_file) open(test_file, "a").close() @@ -32,7 +41,14 @@ def tearDownClass(cls): def test_find_reports(self): reports = schedview.reports.find_reports(self.temp_dir.name) - assert set(reports.columns) == {"report", "fname", "report_time", "night", "link", "url"} + assert set(reports.columns) == { + "report", + "fname", + "report_time", + "night", + "link", + "url", + } assert len(reports) == len(self.test_report_fnames) def test_make_report_link_table(self): @@ -41,10 +57,57 @@ def test_make_report_link_table(self): # Make sure we can parse the result as XML ET.fromstring(html_table) + def test_make_report_link_table_with_visits(self): + rng = np.random.default_rng(0) + n = 40 + # Match dayObs values used for test reports + dayobs = np.array([20250620] * 20 + [20250621] * 20) + visits = pd.DataFrame( + { + "dayObs": dayobs, + "observationId": np.arange(n), + "seeingFwhmGeom": rng.uniform(0.6, 1.8, size=n), + "eff_time_median": rng.uniform(20.0, 40.0, size=n), + "exp_time": rng.uniform(25.0, 35.0, size=n), + "band": rng.choice(list("ugrizy"), size=n), + "science_program": rng.choice( + ["BLOCK-365", "ENG-001"], size=n + ), + "target_name": rng.choice( + ["COSMOS", "XMM-LSS", ""], size=n + ), + } + ) + reports = schedview.reports.find_reports(self.temp_dir.name) + html_table = schedview.reports.make_report_link_table( + reports, visits=visits + ) + # Must be parseable as XML + ET.fromstring(html_table) + # Summary column headers must appear in the output + assert "Total" in html_table + assert "science" in html_table + def test_make_report_rss_feed(self): reports = schedview.reports.find_reports(self.temp_dir.name) test_file = Path(self.temp_dir.name).joinpath("test.rss") - rss_tree = schedview.reports.make_report_rss_feed(reports, str(test_file), 99999) + rss_tree = schedview.reports.make_report_rss_feed( + reports, str(test_file), 99999 + ) assert isinstance(rss_tree, ET.ElementTree) # See if we can parse the result as XML ET.parse(str(test_file)) + + def test_make_report_rss_feed_uses_title_parameter(self): + reports = schedview.reports.find_reports(self.temp_dir.name) + channel_title = "Custom Schedview Reports" + rss_tree = schedview.reports.make_report_rss_feed( + reports, + fname=None, + max_days=99999, + title=channel_title, + ) + root = rss_tree.getroot() + title_elem = root.find("channel/title") + assert title_elem is not None + assert title_elem.text == channel_title From 0ebdc5ccc0058cfd2d0082d45cb411431bd2062f Mon Sep 17 00:00:00 2001 From: Eric Neilsen <neilsen@fnal.gov> Date: Fri, 26 Jun 2026 09:02:24 -0700 Subject: [PATCH 02/23] Design doc testing with pytest --- design/SP-3225/designtest.rst | 252 ++++++++++++++++++++++++++++++++++ pyproject.toml | 3 + tests/conftest.py | 11 ++ tests/test_design_docs.py | 44 ++++++ 4 files changed, 310 insertions(+) create mode 100644 design/SP-3225/designtest.rst create mode 100644 tests/test_design_docs.py diff --git a/design/SP-3225/designtest.rst b/design/SP-3225/designtest.rst new file mode 100644 index 00000000..42002bad --- /dev/null +++ b/design/SP-3225/designtest.rst @@ -0,0 +1,252 @@ +============================================================ +Design: Pytest integration for design documents +============================================================ + +Overview +-------- + +Add automated testing for reStructuredText design documents under the +``design/`` directory. Two checks are performed on each ``.rst`` file: + +1. **RST validity** — verifies the document parses without warnings using + ``rst2html --halt=warning``. +2. **Doctest correctness** — verifies all embedded ``>>>`` code blocks + execute successfully using ``python -m doctest``. + +These tests are excluded from normal test runs (``pytest tests``) via a +custom ``design`` marker, and run only when explicitly requested +(``pytest -m design tests``). + + +Motivation +---------- + +The design documents in ``design/SP-3225/`` contain both prose and +executable code examples (doctests) that serve as white-box verification +of the implementation. Without automated checking: + +- RST syntax errors (unclosed inline literals, bad indentation) go + unnoticed until someone tries to render the document. +- Doctests silently rot as the underlying code evolves. + +Integrating these checks into the pytest suite provides a single command +to verify document quality, while keeping them out of the default test +run (which focuses on the library itself). + + +Design Decisions +---------------- + +**Why a marker rather than a separate test directory?** + +Placing the test module in ``tests/test_design_docs.py`` keeps all tests +discoverable through the existing ``tests/`` directory. The ``design`` +marker plus a ``pytest_collection_modifyitems`` hook gives us skip-by-default +behavior without needing ``addopts`` changes or a separate ``testpaths`` +entry. + +**Why ``pytest_collection_modifyitems`` instead of ``addopts = "-m not design"``?** + +Putting ``-m "not design"`` in ``addopts`` conflicts with explicit +``-m design`` on the command line — pytest does not merge marker +expressions, and the interaction between the two ``-m`` flags is +undefined (in practice, one shadows the other depending on pytest +version). The ``pytest_collection_modifyitems`` hook cleanly detects +whether the user supplied an explicit ``-m`` expression and only skips +design tests when no marker filter was requested. + +**Why ``subprocess.run`` rather than importing docutils/doctest directly?** + +Using subprocess ensures each RST file is validated in isolation with the +same command-line tools a developer would use manually. It also avoids +importing docutils internals or managing doctest state across files. + +**Why parametrize over discovered files?** + +Using ``pytest.mark.parametrize`` with a glob of ``design/**/*.rst`` +means new design documents are automatically picked up without editing +the test file. A named helper function (``_rst_id``) generates short +test IDs (e.g. ``test_rst_valid[SP-3225/smallsum.rst]``), making +failures easy to locate. + + +Changes Required +---------------- + +Three files are modified or created: + +1. ``pyproject.toml`` — register the ``design`` marker. +2. ``tests/conftest.py`` — add ``pytest_collection_modifyitems`` hook. +3. ``tests/test_design_docs.py`` — the new test module (created). + + +Change 1: ``pyproject.toml`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Add the ``markers`` list to ``[tool.pytest.ini_options]``: + +.. code-block:: toml + + [tool.pytest.ini_options] + addopts = "--ignore-glob=*/version.py --ignore-glob=*data_dir/*" + testpaths = "." + asyncio_default_fixture_loop_scope="function" + markers = [ + "design: tests for RST design documents (skipped by default)", + ] + +This suppresses the ``PytestUnknownMarkWarning`` that would otherwise +appear when pytest encounters ``@pytest.mark.design``. + + +Change 2: ``tests/conftest.py`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Add a ``pytest_collection_modifyitems`` hook that skips design-marked +tests when no explicit ``-m`` expression is provided: + +.. code-block:: python + + def pytest_collection_modifyitems(config, items): + """Skip design-doc tests unless an explicit marker expression is given.""" + if not config.getoption("-m"): + skip_design = pytest.mark.skip( + reason="design tests not selected (use -m design)" + ) + for item in items: + if "design" in item.keywords: + item.add_marker(skip_design) + +**Behavior:** + +- ``pytest tests`` → design tests are **skipped** (shown as ``SKIPPED`` + in output with a reason message). +- ``pytest -m design tests`` → **only** design tests run (non-design + tests are deselected by pytest's marker filtering). +- ``pytest -m "design or not design" tests`` → **all** tests run + (both design and non-design). + + +Change 3: ``tests/test_design_docs.py`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +New test module with two parametrized test functions: + +.. code-block:: python + + """Tests for RST design documents under design/.""" + + import subprocess + import sys + from pathlib import Path + + import pytest + + DESIGN_DIR = Path(__file__).parent.parent / "design" + RST_FILES = sorted(DESIGN_DIR.rglob("*.rst")) + + + def _rst_id(rst_path): + """Return a short test ID for a design-doc path.""" + return str(rst_path.relative_to(DESIGN_DIR)) + + + @pytest.mark.design + @pytest.mark.parametrize("rst_file", RST_FILES, ids=_rst_id) + def test_rst_valid(rst_file): + """Verify RST file parses without warnings.""" + result = subprocess.run( + ["rst2html", "--halt=warning", str(rst_file), "/dev/null"], + capture_output=True, + text=True, + ) + assert result.returncode == 0, ( + f"RST validation failed for {rst_file.name}:\n{result.stderr}" + ) + + + @pytest.mark.design + @pytest.mark.parametrize("rst_file", RST_FILES, ids=_rst_id) + def test_rst_doctests(rst_file): + """Verify all doctests in RST file pass.""" + result = subprocess.run( + [sys.executable, "-m", "doctest", str(rst_file)], + capture_output=True, + text=True, + ) + assert result.returncode == 0, ( + f"Doctests failed for {rst_file.name}:\n" + f"{result.stdout}\n{result.stderr}" + ) + + +Usage +----- + +Run design document tests only: + +.. code-block:: bash + + pytest -m design tests + +Run all tests including design: + +.. code-block:: bash + + pytest -m "design or not design" tests + +Normal test run (design tests skipped): + +.. code-block:: bash + + pytest tests + + +Expected Output +--------------- + +When design tests are skipped (default): + +.. code-block:: text + + tests/test_design_docs.py::test_rst_valid[SP-3225/smallsum.rst] SKIPPED + tests/test_design_docs.py::test_rst_valid[SP-3225/visitcache.rst] SKIPPED + tests/test_design_docs.py::test_rst_valid[SP-3225/linktablestats.rst] SKIPPED + ... + +When design tests are selected: + +.. code-block:: text + + tests/test_design_docs.py::test_rst_valid[SP-3225/smallsum.rst] PASSED + tests/test_design_docs.py::test_rst_valid[SP-3225/visitcache.rst] PASSED + tests/test_design_docs.py::test_rst_valid[SP-3225/linktablestats.rst] PASSED + tests/test_design_docs.py::test_rst_doctests[SP-3225/smallsum.rst] PASSED + tests/test_design_docs.py::test_rst_doctests[SP-3225/visitcache.rst] PASSED + tests/test_design_docs.py::test_rst_doctests[SP-3225/linktablestats.rst] PASSED + + +Dependencies +------------ + +- ``docutils`` (provides ``rst2html``; already installed in the venv as a + transitive dependency) +- ``pytest`` (already a test dependency) +- No new package installations required. + + +Edge Cases +---------- + +**No RST files found**: If ``design/`` is empty or absent, +``RST_FILES`` will be an empty list and pytest will report "no tests ran" +for the parametrized functions (``collected 0 items``). This is harmless. + +**RST files without doctests**: Files that contain no ``>>>`` lines will +pass ``python -m doctest`` with exit code 0 (no tests collected, no +failures). + +**Long-running doctests**: Since doctests are run via subprocess, each +file runs in its own process. Slow imports (e.g., ``Almanac``) will +add wall-clock time. The design tests are excluded from default runs +specifically to avoid this overhead in routine development. diff --git a/pyproject.toml b/pyproject.toml index b612c4e0..950ce6a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,6 +76,9 @@ where = [ "" ] addopts = "--ignore-glob=*/version.py --ignore-glob=*data_dir/*" testpaths = "." asyncio_default_fixture_loop_scope="function" +markers = [ + "design: tests for RST design documents (skipped by default)", +] [tool.black] line-length = 110 diff --git a/tests/conftest.py b/tests/conftest.py index 55c0413c..3bc75147 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -64,6 +64,17 @@ def pytest_configure(config: pytest.Config) -> None: _resolve_sample_data_dir(Path(config.rootpath)) +def pytest_collection_modifyitems(config, items): + """Skip design-doc tests unless an explicit marker expression is given.""" + if not config.getoption("-m"): + skip_design = pytest.mark.skip( + reason="design tests not selected (use -m design)" + ) + for item in items: + if "design" in item.keywords: + item.add_marker(skip_design) + + @pytest.fixture(scope="session") def sample_data_dir(pytestconfig: pytest.Config) -> Path: """Return the sample-data directory for the test session. diff --git a/tests/test_design_docs.py b/tests/test_design_docs.py new file mode 100644 index 00000000..7a7a4cc2 --- /dev/null +++ b/tests/test_design_docs.py @@ -0,0 +1,44 @@ +"""Tests for RST design documents under design/.""" + +import subprocess +import sys +from pathlib import Path + +import pytest + +DESIGN_DIR = Path(__file__).parent.parent / "design" +RST_FILES = sorted(DESIGN_DIR.rglob("*.rst")) + + +def _rst_id(rst_path): + """Return a short test ID for a design-doc path.""" + return str(rst_path.relative_to(DESIGN_DIR)) + + +@pytest.mark.design +@pytest.mark.parametrize("rst_file", RST_FILES, ids=_rst_id) +def test_rst_valid(rst_file): + """Verify RST file parses without warnings.""" + result = subprocess.run( + ["rst2html", "--halt=warning", str(rst_file), "/dev/null"], + capture_output=True, + text=True, + ) + assert result.returncode == 0, ( + f"RST validation failed for {rst_file.name}:\n{result.stderr}" + ) + + +@pytest.mark.design +@pytest.mark.parametrize("rst_file", RST_FILES, ids=_rst_id) +def test_rst_doctests(rst_file): + """Verify all doctests in RST file pass.""" + result = subprocess.run( + [sys.executable, "-m", "doctest", str(rst_file)], + capture_output=True, + text=True, + ) + assert result.returncode == 0, ( + f"Doctests failed for {rst_file.name}:\n" + f"{result.stdout}\n{result.stderr}" + ) From a3b0aef51b4fce53dba969924aeb9ba6f36f44d8 Mon Sep 17 00:00:00 2001 From: "Eric H. Neilsen, Jr." <ehneilsen@users.noreply.github.com> Date: Fri, 26 Jun 2026 11:37:33 -0500 Subject: [PATCH 03/23] Make sure dashboard dependencies are included in pip installs with dashboard option --- pyproject.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 950ce6a1..40dd155f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ test = [ "isort", "ruff", "pytest-cov", + "pytest-asyncio", "geckodriver", "playwright", "pre-commit" @@ -58,6 +59,10 @@ test = [ dev = [ "documenteer[guide]", ] +dashboard = [ + "boto3>=1.43.36", + "botocore>=1.43.36", +] [project.scripts] prenight = "schedview.app.prenight.prenight:main" From 2d27c05436b04da8c27cb5de340d041305a9cbbc Mon Sep 17 00:00:00 2001 From: "Eric H. Neilsen, Jr." <ehneilsen@users.noreply.github.com> Date: Fri, 26 Jun 2026 13:23:30 -0500 Subject: [PATCH 04/23] get rid of lambda functions --- schedview/app/scheduler_dashboard/utils.py | 2 +- tests/test_collect_visits_cache.py | 46 ++++++++++++---------- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/schedview/app/scheduler_dashboard/utils.py b/schedview/app/scheduler_dashboard/utils.py index 0c6aac19..74f31fa2 100644 --- a/schedview/app/scheduler_dashboard/utils.py +++ b/schedview/app/scheduler_dashboard/utils.py @@ -68,7 +68,7 @@ def mock_schedulers_df(): df = pd.DataFrame(data, columns=["time", "url"]) df = df.sort_index(ascending=False) - df["url"] = df["url"].apply(lambda x: localize_scheduler_url(x)) + df["url"] = df["url"].apply(localize_scheduler_url) return df["url"] diff --git a/tests/test_collect_visits_cache.py b/tests/test_collect_visits_cache.py index 296ff9c4..2adb527c 100644 --- a/tests/test_collect_visits_cache.py +++ b/tests/test_collect_visits_cache.py @@ -35,6 +35,28 @@ def _day_obs_mock(self, yyyymmdd): m.yyyymmdd = yyyymmdd return m + def _from_date_side_effect(self, day_obs_value): + """Return a side_effect callable for DayObs.from_date mocks. + + Parameters + ---------- + day_obs_value : `int` + The yyyymmdd value to return for any argument that is not + ``"today"``. + + Returns + ------- + side_effect : `callable` + A function that returns a `~unittest.mock.MagicMock` when + called with ``"today"`` and a day-obs mock otherwise. + """ + def side_effect(arg): + if arg == "today": + return MagicMock() + return self._day_obs_mock(day_obs_value) + + return side_effect + def test_raises_for_non_instrument_source(self): with self.assertRaises(ValueError): cached_read_visits( @@ -65,11 +87,7 @@ def test_filters_to_requested_day_obs(self): "schedview.collect.visits.DayObs.from_date", ) as mock_from_date, ): - mock_from_date.side_effect = lambda arg: ( - MagicMock() - if arg == "today" - else self._day_obs_mock(20260612) - ) + mock_from_date.side_effect = self._from_date_side_effect(20260612) result = cached_read_visits( 20260612, "lsstcam", cache_dir=tmpdir ) @@ -94,11 +112,7 @@ def test_warns_when_no_dayobs_column(self): "schedview.collect.visits.DayObs.from_date", ) as mock_from_date, ): - mock_from_date.side_effect = lambda arg: ( - MagicMock() - if arg == "today" - else self._day_obs_mock(20260614) - ) + mock_from_date.side_effect = self._from_date_side_effect(20260614) with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") result = cached_read_visits( @@ -131,11 +145,7 @@ def test_cache_dir_created_if_absent(self): "schedview.collect.visits.DayObs.from_date", ) as mock_from_date, ): - mock_from_date.side_effect = lambda arg: ( - MagicMock() - if arg == "today" - else self._day_obs_mock(20260614) - ) + mock_from_date.side_effect = self._from_date_side_effect(20260614) cached_read_visits( 20260614, "lsstcam", cache_dir=new_cache_dir ) @@ -163,11 +173,7 @@ def test_second_call_returns_same_data(self): "schedview.collect.visits.DayObs.from_date", ) as mock_from_date, ): - mock_from_date.side_effect = lambda arg: ( - MagicMock() - if arg == "today" - else self._day_obs_mock(20260614) - ) + mock_from_date.side_effect = self._from_date_side_effect(20260614) result1 = cached_read_visits( 20260614, "lsstcam", From d56b5bd364087e1e86b028744ee3019574798839 Mon Sep 17 00:00:00 2001 From: "Eric H. Neilsen, Jr." <ehneilsen@users.noreply.github.com> Date: Fri, 26 Jun 2026 14:14:17 -0500 Subject: [PATCH 05/23] fix ruff complaint --- schedview/collect/visits.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schedview/collect/visits.py b/schedview/collect/visits.py index 3fda7192..b78eec96 100644 --- a/schedview/collect/visits.py +++ b/schedview/collect/visits.py @@ -200,7 +200,7 @@ def cached_read_visits( stackers: list | None = None, ddf: bool = False, ) -> pd.DataFrame: - """Read visits from a consdb source, using a local HDF5 cache when possible. + """Read visits from consdb, using a local HDF5 cache when possible. On a cache hit (the cache file exists, is fresh, and was built with the same set of stackers), the cached data is read from disk and filtered to From 99762a4dbdd0cca0c3f94c522a93bb64e3e3df23 Mon Sep 17 00:00:00 2001 From: Eric Neilsen <neilsen@fnal.gov> Date: Mon, 29 Jun 2026 10:29:15 -0500 Subject: [PATCH 06/23] black --- tests/conftest.py | 4 +- tests/test_collect_visits_cache.py | 25 +++------- tests/test_compute_smallsum.py | 73 +++++++----------------------- tests/test_design_docs.py | 7 +-- tests/test_reports.py | 24 +++------- 5 files changed, 32 insertions(+), 101 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3bc75147..1b7577c0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -67,9 +67,7 @@ def pytest_configure(config: pytest.Config) -> None: def pytest_collection_modifyitems(config, items): """Skip design-doc tests unless an explicit marker expression is given.""" if not config.getoption("-m"): - skip_design = pytest.mark.skip( - reason="design tests not selected (use -m design)" - ) + skip_design = pytest.mark.skip(reason="design tests not selected (use -m design)") for item in items: if "design" in item.keywords: item.add_marker(skip_design) diff --git a/tests/test_collect_visits_cache.py b/tests/test_collect_visits_cache.py index 2adb527c..b827fbeb 100644 --- a/tests/test_collect_visits_cache.py +++ b/tests/test_collect_visits_cache.py @@ -50,6 +50,7 @@ def _from_date_side_effect(self, day_obs_value): A function that returns a `~unittest.mock.MagicMock` when called with ``"today"`` and a day-obs mock otherwise. """ + def side_effect(arg): if arg == "today": return MagicMock() @@ -59,15 +60,11 @@ def side_effect(arg): def test_raises_for_non_instrument_source(self): with self.assertRaises(ValueError): - cached_read_visits( - 20260614, "baseline", cache_dir="/tmp/cache" - ) + cached_read_visits(20260614, "baseline", cache_dir="/tmp/cache") def test_raises_for_opsim_file_source(self): with self.assertRaises(ValueError): - cached_read_visits( - 20260614, "/path/to/sim.db", cache_dir="/tmp/cache" - ) + cached_read_visits(20260614, "/path/to/sim.db", cache_dir="/tmp/cache") def test_filters_to_requested_day_obs(self): """Results contain only visits up to the requested day_obs.""" @@ -88,9 +85,7 @@ def test_filters_to_requested_day_obs(self): ) as mock_from_date, ): mock_from_date.side_effect = self._from_date_side_effect(20260612) - result = cached_read_visits( - 20260612, "lsstcam", cache_dir=tmpdir - ) + result = cached_read_visits(20260612, "lsstcam", cache_dir=tmpdir) self.assertTrue(all(result["dayObs"] <= 20260612)) @@ -115,14 +110,10 @@ def test_warns_when_no_dayobs_column(self): mock_from_date.side_effect = self._from_date_side_effect(20260614) with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") - result = cached_read_visits( - 20260614, "lsstcam", cache_dir=tmpdir - ) + result = cached_read_visits(20260614, "lsstcam", cache_dir=tmpdir) self.assertEqual(len(result), 3) - self.assertTrue( - any("dayObs" in str(warning.message) for warning in w) - ) + self.assertTrue(any("dayObs" in str(warning.message) for warning in w)) def test_cache_dir_created_if_absent(self): """cache_dir is created automatically when absent.""" @@ -146,9 +137,7 @@ def test_cache_dir_created_if_absent(self): ) as mock_from_date, ): mock_from_date.side_effect = self._from_date_side_effect(20260614) - cached_read_visits( - 20260614, "lsstcam", cache_dir=new_cache_dir - ) + cached_read_visits(20260614, "lsstcam", cache_dir=new_cache_dir) self.assertTrue(new_cache_dir.exists()) diff --git a/tests/test_compute_smallsum.py b/tests/test_compute_smallsum.py index bcc4bfce..21fefd50 100644 --- a/tests/test_compute_smallsum.py +++ b/tests/test_compute_smallsum.py @@ -100,18 +100,12 @@ def test_band_and_science_counts_are_int(self, sample_visits): assert pd.api.types.is_integer_dtype(result["science"]) def test_science_count_correct(self, sample_visits): - result = compute_tinysum( - sample_visits, science_programs=SCIENCE_PROGRAMS - ) + result = compute_tinysum(sample_visits, science_programs=SCIENCE_PROGRAMS) for day in result.index: expected = len( sample_visits[ (sample_visits["dayObs"] == day) - & ( - sample_visits["science_program"].isin( - SCIENCE_PROGRAMS - ) - ) + & (sample_visits["science_program"].isin(SCIENCE_PROGRAMS)) ] ) assert result.loc[day, "science"] == expected @@ -129,25 +123,16 @@ def test_total_eff_time_over_exp_time(self, sample_visits): result = compute_tinysum(sample_visits) for day in result.index: day_visits = sample_visits[sample_visits["dayObs"] == day] - expected = ( - day_visits["eff_time_median"].sum() - / day_visits["exp_time"].sum() - ) - assert np.isclose( - result.loc[day, "total eff_time/exp_time"], expected - ) + expected = day_visits["eff_time_median"].sum() / day_visits["exp_time"].sum() + assert np.isclose(result.loc[day, "total eff_time/exp_time"], expected) def test_no_almanac_omits_rate_columns(self, sample_visits): result = compute_tinysum(sample_visits, almanac=None) - assert {"night_hours", "visits/hour", "teff/minute"}.isdisjoint( - result.columns - ) + assert {"night_hours", "visits/hour", "teff/minute"}.isdisjoint(result.columns) def test_with_almanac_adds_rate_columns(self, sample_visits, almanac): result = compute_tinysum(sample_visits, almanac=almanac) - assert {"night_hours", "visits/hour", "teff/minute"}.issubset( - result.columns - ) + assert {"night_hours", "visits/hour", "teff/minute"}.issubset(result.columns) assert (result["night_hours"].dropna() > 0).all() def test_no_science_visits_night(self): @@ -213,45 +198,30 @@ def test_band_subsets_present_and_sum(self, sample_visits): assert bands_present for day in (20250601, 20250602): day_data = result.loc[day] - day_bands = [ - b - for b in ALL_BANDS - if b in day_data.index.get_level_values("subset") - ] - assert ( - sum(day_data.loc[b, "visits"] for b in day_bands) - == day_data.loc["all", "visits"] - ) + day_bands = [b for b in ALL_BANDS if b in day_data.index.get_level_values("subset")] + assert sum(day_data.loc[b, "visits"] for b in day_bands) == day_data.loc["all", "visits"] def test_science_subsets_present_and_sum(self, sample_visits): - result = compute_smallsum( - sample_visits, science_programs=SCIENCE_PROGRAMS - ) + result = compute_smallsum(sample_visits, science_programs=SCIENCE_PROGRAMS) all_subsets = result.index.get_level_values("subset").unique() assert {"science", "not_science"}.issubset(all_subsets) for day in (20250601, 20250602): day_data = result.loc[day] if {"science", "not_science"}.issubset(day_data.index): assert ( - day_data.loc["science", "visits"] - + day_data.loc["not_science", "visits"] + day_data.loc["science", "visits"] + day_data.loc["not_science", "visits"] == day_data.loc["all", "visits"] ) def test_observation_reason_subsets(self, sample_visits): result = compute_smallsum(sample_visits) all_subsets = result.index.get_level_values("subset").unique() - assert any( - r in all_subsets - for r in ("survey", "calibration", "engineering") - ) + assert any(r in all_subsets for r in ("survey", "calibration", "engineering")) def test_target_name_subsets(self, sample_visits): result = compute_smallsum(sample_visits) all_subsets = result.index.get_level_values("subset").unique() - assert any( - t in all_subsets for t in ("COSMOS", "XMM-LSS", "ELAIS") - ) + assert any(t in all_subsets for t in ("COSMOS", "XMM-LSS", "ELAIS")) def test_empty_target_becomes_no_target_name(self): visits = pd.DataFrame( @@ -270,10 +240,7 @@ def test_empty_target_becomes_no_target_name(self): } ) result = compute_smallsum(visits, science_programs=("BLOCK-365",)) - assert ( - "no target name" - in result.loc[20250601].index.get_level_values("subset") - ) + assert "no target name" in result.loc[20250601].index.get_level_values("subset") def test_output_columns(self, sample_visits): result = compute_smallsum(sample_visits) @@ -300,21 +267,13 @@ def test_first_before_last(self, sample_visits): assert (result["first"] <= result["last"]).all() def test_subset_order_matches_contract(self, sample_visits): - result = compute_smallsum( - sample_visits, science_programs=SCIENCE_PROGRAMS - ) + result = compute_smallsum(sample_visits, science_programs=SCIENCE_PROGRAMS) day = result.index.get_level_values("dayObs")[0] subsets = list(result.loc[day].index) assert subsets[0] == "all" - first_band_pos = min( - subsets.index(b) for b in ALL_BANDS if b in subsets - ) - first_science_pos = min( - subsets.index(s) - for s in ("science", "not_science") - if s in subsets - ) + first_band_pos = min(subsets.index(b) for b in ALL_BANDS if b in subsets) + first_science_pos = min(subsets.index(s) for s in ("science", "not_science") if s in subsets) assert first_band_pos < first_science_pos diff --git a/tests/test_design_docs.py b/tests/test_design_docs.py index 7a7a4cc2..37118325 100644 --- a/tests/test_design_docs.py +++ b/tests/test_design_docs.py @@ -24,9 +24,7 @@ def test_rst_valid(rst_file): capture_output=True, text=True, ) - assert result.returncode == 0, ( - f"RST validation failed for {rst_file.name}:\n{result.stderr}" - ) + assert result.returncode == 0, f"RST validation failed for {rst_file.name}:\n{result.stderr}" @pytest.mark.design @@ -39,6 +37,5 @@ def test_rst_doctests(rst_file): text=True, ) assert result.returncode == 0, ( - f"Doctests failed for {rst_file.name}:\n" - f"{result.stdout}\n{result.stderr}" + f"Doctests failed for {rst_file.name}:\n" f"{result.stdout}\n{result.stderr}" ) diff --git a/tests/test_reports.py b/tests/test_reports.py index 406143f7..c1be161e 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -25,13 +25,9 @@ def setUpClass(cls): for report in cls.reports: for instrument in cls.instruments: for day in cls.days: - test_dir = Path(cls.temp_dir.name).joinpath( - report, instrument, year, month, day - ) + test_dir = Path(cls.temp_dir.name).joinpath(report, instrument, year, month, day) test_dir.mkdir(parents=True) - test_file = test_dir.joinpath( - f"{report}_{year}-{month}-{day}.html" - ) + test_file = test_dir.joinpath(f"{report}_{year}-{month}-{day}.html") cls.test_report_fnames.append(test_file) open(test_file, "a").close() @@ -70,18 +66,12 @@ def test_make_report_link_table_with_visits(self): "eff_time_median": rng.uniform(20.0, 40.0, size=n), "exp_time": rng.uniform(25.0, 35.0, size=n), "band": rng.choice(list("ugrizy"), size=n), - "science_program": rng.choice( - ["BLOCK-365", "ENG-001"], size=n - ), - "target_name": rng.choice( - ["COSMOS", "XMM-LSS", ""], size=n - ), + "science_program": rng.choice(["BLOCK-365", "ENG-001"], size=n), + "target_name": rng.choice(["COSMOS", "XMM-LSS", ""], size=n), } ) reports = schedview.reports.find_reports(self.temp_dir.name) - html_table = schedview.reports.make_report_link_table( - reports, visits=visits - ) + html_table = schedview.reports.make_report_link_table(reports, visits=visits) # Must be parseable as XML ET.fromstring(html_table) # Summary column headers must appear in the output @@ -91,9 +81,7 @@ def test_make_report_link_table_with_visits(self): def test_make_report_rss_feed(self): reports = schedview.reports.find_reports(self.temp_dir.name) test_file = Path(self.temp_dir.name).joinpath("test.rss") - rss_tree = schedview.reports.make_report_rss_feed( - reports, str(test_file), 99999 - ) + rss_tree = schedview.reports.make_report_rss_feed(reports, str(test_file), 99999) assert isinstance(rss_tree, ET.ElementTree) # See if we can parse the result as XML ET.parse(str(test_file)) From c282a12478e1d1ec924994306036a5a3a91a9fc4 Mon Sep 17 00:00:00 2001 From: Eric Neilsen <neilsen@fnal.gov> Date: Mon, 29 Jun 2026 10:36:23 -0500 Subject: [PATCH 07/23] Fix label of eff_time/exp_time in rss description --- schedview/reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schedview/reports.py b/schedview/reports.py index b96b3ba0..1b18557f 100644 --- a/schedview/reports.py +++ b/schedview/reports.py @@ -14,7 +14,7 @@ Total visits: {total}; Science visits: {science}; Median FWHM: {fwhm}; -Total eff_time/exp_time: {mean_norm_teff}; +Total eff_time/ total exp_time: {mean_norm_teff}; Mean visit rate: {visit_rate} visits/hour; Mean eff_time rate: {teff_rate}/minute; Science targets: {targets} From 3ac6b9cf46e23a096723aea13ad920c091e0caaf Mon Sep 17 00:00:00 2001 From: Eric Neilsen <neilsen@fnal.gov> Date: Mon, 29 Jun 2026 10:37:03 -0500 Subject: [PATCH 08/23] reduce default max days in rss feed to 1 --- schedview/reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schedview/reports.py b/schedview/reports.py index 1b18557f..7d5ae665 100644 --- a/schedview/reports.py +++ b/schedview/reports.py @@ -198,7 +198,7 @@ def make_report_link_table( def make_report_rss_feed( reports: pd.DataFrame, fname: str | None = None, - max_days: int = 7, + max_days: int = 1, visits: pd.DataFrame | None = None, title: str = "schedview reports", description: str = "Statically generated reports on Rubin Observatory/LSST scheduler status and progress", From c5855e42d49f84a0755570806e5aec97b44d2b1a Mon Sep 17 00:00:00 2001 From: "Eric H. Neilsen, Jr." <ehneilsen@users.noreply.github.com> Date: Mon, 29 Jun 2026 11:44:09 -0500 Subject: [PATCH 09/23] use same guid in rss if description is the same --- schedview/reports.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/schedview/reports.py b/schedview/reports.py index 7d5ae665..a6c8c152 100644 --- a/schedview/reports.py +++ b/schedview/reports.py @@ -1,5 +1,6 @@ import datetime import email.utils +import hashlib import os import xml.etree.ElementTree as ET from pathlib import Path @@ -286,7 +287,7 @@ def make_report_rss_feed( link = ET.SubElement(item, "link") link.text = report_row.url guid = ET.SubElement(item, "guid", attib={"isPermaLink": "false"}) - guid.text = item_title_elem.text + f", generated {report_row.report_time}" + guid.text = item_title_elem.text + hashlib.sha1(desc.text.encode("utf-8")).hexdigest()[:6] category = ET.SubElement(item, "category") category.text = f"{instrument}_{report_row.report}" pubdate = ET.SubElement(item, "pubDate") From f82c06f676c68056d8c03a51553f51794d713ba7 Mon Sep 17 00:00:00 2001 From: "Eric H. Neilsen, Jr." <ehneilsen@users.noreply.github.com> Date: Mon, 29 Jun 2026 11:44:32 -0500 Subject: [PATCH 10/23] Attempt to fix CI test failure from EFD API change --- pyproject.toml | 2 +- requirements.txt | 2 +- schedview/collect/efd.py | 11 +++++++++++ tests/test_lfa_scheduler_dashboard.py | 27 ++++++++++++++++++++++++--- 4 files changed, 37 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 40dd155f..8fd70cd1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ "healpy", "holoviews", "hvplot", - "lsst-efd-client", + "lsst-efd-client>=0.12.0", "lsst-resources", "matplotlib < 3.11.0", "numpy", diff --git a/requirements.txt b/requirements.txt index 2148d185..824b5d8d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ colorcet healpy holoviews hvplot -lsst-efd-client +lsst-efd-client>=0.12.0 matplotlib-base<3.11.0 numpy pandas diff --git a/schedview/collect/efd.py b/schedview/collect/efd.py index eab19dd5..cc2886e6 100644 --- a/schedview/collect/efd.py +++ b/schedview/collect/efd.py @@ -1,4 +1,5 @@ import asyncio +import inspect import re import threading from collections import defaultdict @@ -55,6 +56,16 @@ def make_efd_client(efd_name: str | None = None, *args, **kwargs): efd_name = schedview.clientsite.EFD_NAME assert isinstance(efd_name, str) + + # schedview always consumes EFD results as pandas DataFrames, so + # request the "dataframe" output mode explicitly. Older lsst-efd-client + # (0.12.0) hardcodes this and has no ``output_mode`` parameter, while + # newer versions (>=0.14, 1.x) default it to None, which aioinflux + # rejects with "Invalid output format". Only pass the kwarg when the + # installed EfdClient accepts it. + if "output_mode" in inspect.signature(EfdClient.__init__).parameters and "output_mode" not in kwargs: + kwargs["output_mode"] = "dataframe" + return EfdClient(efd_name, *args, **kwargs) diff --git a/tests/test_lfa_scheduler_dashboard.py b/tests/test_lfa_scheduler_dashboard.py index 1e612399..d541d645 100644 --- a/tests/test_lfa_scheduler_dashboard.py +++ b/tests/test_lfa_scheduler_dashboard.py @@ -2,22 +2,43 @@ import unittest from datetime import datetime +import pandas as pd import pytest # Objects to test instances against. from rubin_scheduler.scheduler.features.conditions import Conditions from rubin_scheduler.scheduler.schedulers.core_scheduler import CoreScheduler +import schedview.collect.efd from schedview.app.scheduler_dashboard.scheduler_dashboard_app import LFASchedulerSnapshotDashboard +class _StubEfdClient: + """A minimal stand-in for ``lsst_efd_client.EfdClient``. + + It ducktypes the query methods used by the snapshot-list code path and + returns empty DataFrames, so the test does not depend on a real EFD + connection, network access, or the installed ``lsst-efd-client`` version. + """ + + async def select_top_n(self, *args, **kwargs): + return pd.DataFrame() + + async def select_time_series(self, *args, **kwargs): + return pd.DataFrame() + + @pytest.mark.asyncio -async def test_get_scheduler_list(): +async def test_get_scheduler_list(monkeypatch): + # Avoid constructing a real EfdClient (which reaches out to segwarides for + # credentials and to InfluxDB for a health check). + monkeypatch.setattr(schedview.collect.efd, "make_efd_client", lambda *a, **k: _StubEfdClient()) + scheduler = LFASchedulerSnapshotDashboard() scheduler.telescope = None await scheduler._async_get_scheduler_list() - # No snapshots should be retreived if it isn't an LFA environment - # the dropdown has one empty option + # No snapshots are retrieved from the stub client, so the dropdown keeps + # its single empty default option. assert len(scheduler.param.scheduler_fname.objects) >= 1 From 1314b0a367f4bd2eb3dbf4788ae017fae04abc94 Mon Sep 17 00:00:00 2001 From: "Eric H. Neilsen, Jr." <ehneilsen@users.noreply.github.com> Date: Mon, 29 Jun 2026 11:48:33 -0500 Subject: [PATCH 11/23] black --- schedview/reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schedview/reports.py b/schedview/reports.py index a6c8c152..5ecd4347 100644 --- a/schedview/reports.py +++ b/schedview/reports.py @@ -287,7 +287,7 @@ def make_report_rss_feed( link = ET.SubElement(item, "link") link.text = report_row.url guid = ET.SubElement(item, "guid", attib={"isPermaLink": "false"}) - guid.text = item_title_elem.text + hashlib.sha1(desc.text.encode("utf-8")).hexdigest()[:6] + guid.text = item_title_elem.text + hashlib.sha1(desc.text.encode("utf-8")).hexdigest()[:6] category = ET.SubElement(item, "category") category.text = f"{instrument}_{report_row.report}" pubdate = ET.SubElement(item, "pubDate") From 4c40bab1ee78ea11000f7e3f88bd5793ff45250b Mon Sep 17 00:00:00 2001 From: "Eric H. Neilsen, Jr." <ehneilsen@users.noreply.github.com> Date: Mon, 29 Jun 2026 13:15:01 -0500 Subject: [PATCH 12/23] Attempt to fix FileSelectorWithEmptyOption CI failure --- schedview/param.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/schedview/param.py b/schedview/param.py index 01ea6610..fd05891c 100644 --- a/schedview/param.py +++ b/schedview/param.py @@ -91,6 +91,17 @@ class FileSelectorWithEmptyOption(param.FileSelector): Like param.FileSelector, but allows None to be deliberately selected. """ + def __init__(self, default=Undefined, *, path=Undefined, allow_None=True, **kwargs): + super().__init__(default=default, path=path, allow_None=allow_None, **kwargs) + # param >=2.4 consumes ``allow_None`` in ``FileSelector.__init__`` + # without forwarding it to the base ``Parameter``, leaving it falsy. + # Set it explicitly so a deliberate ``None`` (the "empty option") + # stays a valid default and selection. Without this, param's + # class-creation re-validation of the default (triggered because the + # parent class types this parameter as a ``param.String``) rejects + # ``None``. + self.allow_None = bool(allow_None) + def update(self, path=Undefined): if path is Undefined: path = self.path From b38dbd3d1cc0c4ce80054eeb9c33362063d26748 Mon Sep 17 00:00:00 2001 From: "Eric H. Neilsen, Jr." <ehneilsen@users.noreply.github.com> Date: Mon, 29 Jun 2026 13:44:51 -0500 Subject: [PATCH 13/23] Include band breakdown in RSS description --- design/SP-3225/smallsum.rst | 68 +++++++++++++++++++++++++++++++--- schedview/compute/smallsum.py | 49 +++++++++++++++++++++++- schedview/reports.py | 11 ++++-- tests/test_compute_smallsum.py | 49 +++++++++++++++++++++++- tests/test_reports.py | 32 ++++++++++++++++ 5 files changed, 197 insertions(+), 12 deletions(-) diff --git a/design/SP-3225/smallsum.rst b/design/SP-3225/smallsum.rst index 55b1558d..5e168c44 100644 --- a/design/SP-3225/smallsum.rst +++ b/design/SP-3225/smallsum.rst @@ -70,6 +70,14 @@ Three private helpers factor out reusable logic: its night duration in hours. Used by ``compute_tinysum`` to derive the rate columns (``visits/hour``, ``teff/minute``). +In addition, one **public** helper is exported alongside the two main +functions: + +``format_band_breakdown(row, prefix="# ", suffix="")`` + Renders a per-band visit-count breakdown string (e.g. ``500g, 170r, 6i``) + from a single ``compute_tinysum`` row. Used by + ``schedview.reports.make_report_rss_feed`` for the RSS feed description. + The sections below document each helper's contract and verify it with embedded doctests, followed by the full specifications of the two public functions. @@ -264,6 +272,44 @@ fall between roughly 4 and 12 hours depending on season: True +Helper: ``format_band_breakdown`` +--------------------------------- + +.. code-block:: python + + def format_band_breakdown(row: pd.Series, prefix: str = "# ", suffix: str = "") -> str + +A **public** helper (exported in ``__all__``) that renders a per-band +visit-count breakdown string such as ``500g, 170r, 6i`` from a single +``compute_tinysum`` row. It is used by +``schedview.reports.make_report_rss_feed`` to add the parenthetical band +breakdown to the ``Total visits`` and ``Science visits`` lines of the RSS +feed description. + +Logic: + +- For each band ``b`` in ``_BANDS`` order, read the count from + ``row[f"{prefix}{b}{suffix}"]``. +- Skip bands whose column is absent, ``NA``, or zero. +- Emit ``{count}{band}`` for each remaining band, joined by ``", "``. + +The ``prefix``/``suffix`` parameters select which column family to read: +the defaults (``prefix="# "``, ``suffix=""``) read the total band columns +(``# g`` …); passing ``suffix=" science"`` reads the science band columns +(``# g science`` …). Returns ``""`` when every count is zero (the caller is +responsible for deciding whether to wrap the result in parentheses). + +>>> from schedview.compute.smallsum import format_band_breakdown +>>> row = pd.Series({"# u": 0, "# g": 500, "# r": 170, "# i": 6, "# z": 0, "# y": 0}) +>>> format_band_breakdown(row) +'500g, 170r, 6i' +>>> sci = pd.Series({"# g science": 400, "# r science": 85, "# i science": 5}) +>>> format_band_breakdown(sci, suffix=" science") +'400g, 85r, 5i' +>>> format_band_breakdown(pd.Series({"# u": 0, "# g": 0})) +'' + + Function 1: ``compute_tinysum`` ------------------------------- @@ -336,6 +382,10 @@ per night. Columns: * - ``# u`` through ``# y`` - Int64 - Number of visits in each band (nullable integer) + * - ``# u science`` through ``# y science`` + - Int64 + - Number of science visits in each band (visits with + ``science_program`` in ``science_programs``), zero filled, ``Int64`` * - ``science targets`` - str - Comma-separated unique target names from science visits @@ -374,20 +424,26 @@ Implementation Steps 4. **Science counts**: Filter visits where ``science_program`` is in ``science_programs``, group by ``dayObs``, count → ``science``. -5. **Science targets**: From science visits, group by ``dayObs``, aggregate +5. **Science band counts**: From the same science-visit subset, group by + ``['dayObs', 'band']``, count ``observationId``, unstack, reindex to all + bands, cast to ``Int64``. Rename to ``# u science``, ``# g science``, etc. + (same pattern as the total band counts in step 3). + +6. **Science targets**: From science visits, group by ``dayObs``, aggregate ``target_name`` using ``_unique_targets``. -6. **Join all** intermediate DataFrames on the ``dayObs`` index. Fill NaN in +7. **Join all** intermediate DataFrames on the ``dayObs`` index. Fill NaN in ``science`` with 0 (cast to ``Int64``) and in ``science targets`` with - empty string. + empty string. Fill NaN in the ``# {band} science`` columns (nights with no + science visits) with 0 and cast to ``Int64``. -7. **Normalized effective time**: Compute +8. **Normalized effective time**: Compute ``total eff_time/exp_time = total eff_time / total exp_time``. -8. **Night hours** (only if ``almanac`` is not ``None``): Build a mapping from +9. **Night hours** (only if ``almanac`` is not ``None``): Build a mapping from ``dayObs`` → night duration via ``_build_night_hours``. -9. **Derived rates** (only if ``almanac`` is not ``None``): +10. **Derived rates** (only if ``almanac`` is not ``None``): - ``visits/hour = Total / night_hours`` - ``teff/minute = visits/hour * mean eff_time / 60`` diff --git a/schedview/compute/smallsum.py b/schedview/compute/smallsum.py index 83e5edd9..04b92d83 100644 --- a/schedview/compute/smallsum.py +++ b/schedview/compute/smallsum.py @@ -7,7 +7,7 @@ (band, science/not-science, observation_reason, target name). """ -__all__ = ["compute_tinysum", "compute_smallsum"] +__all__ = ["compute_tinysum", "compute_smallsum", "format_band_breakdown"] from datetime import timedelta @@ -87,6 +87,40 @@ def _build_night_hours(almanac: Almanac, dayobs_values: pd.Index) -> pd.Series: return night_hours.reindex(dayobs_values) +def format_band_breakdown(row: pd.Series, prefix: str = "# ", suffix: str = "") -> str: + """Format a per-band visit-count breakdown like ``500g, 170r, 6i``. + + Parameters + ---------- + row : `pandas.Series` + A single ``compute_tinysum`` row (e.g. ``tinysum.loc[dayobs]``). + prefix : `str`, optional + Text before the band name in the column label. Defaults to ``"# "``. + suffix : `str`, optional + Text after the band name in the column label. The count for band + ``b`` is read from ``row[f"{prefix}{b}{suffix}"]``. The defaults + select the total band columns (``# g`` ...); pass ``suffix=" science"`` + to select the science band columns (``# g science`` ...). + + Returns + ------- + breakdown : `str` + Comma-separated ``{count}{band}`` pairs for bands with a nonzero + count, in ``_BANDS`` order. Empty string if every count is zero or + absent. + """ + parts = [] + for band in _BANDS: + column = f"{prefix}{band}{suffix}" + if column not in row.index: + continue + count = row[column] + if pd.isna(count) or int(count) == 0: + continue + parts.append(f"{int(count)}{band}") + return ", ".join(parts) + + def _visits_summary(visits_group: pd.DataFrame) -> pd.Series: """Compute summary statistics for a group of visits. @@ -195,6 +229,15 @@ def compute_tinysum( science_visits = visits.loc[visits["science_program"].isin(science_programs), :] science_counts = science_visits.groupby("dayObs")["observationId"].count().rename("science").to_frame() + science_band_counts = ( + science_visits.groupby(["dayObs", "band"])["observationId"] + .count() + .unstack("band", fill_value=0) + .reindex(columns=_BANDS, fill_value=0) + .astype("Int64") + .rename(columns={b: f"# {b} science" for b in _BANDS}) + ) + targets = ( science_visits.groupby("dayObs")["target_name"] .apply(_unique_targets) @@ -202,11 +245,13 @@ def compute_tinysum( .to_frame() ) - tinysum = basic_stats.join([teff_stats, science_counts, band_counts, targets]) + tinysum = basic_stats.join([teff_stats, science_counts, band_counts, science_band_counts, targets]) tinysum["total eff_time/exp_time"] = tinysum["total eff_time"] / tinysum["total exp_time"] tinysum["science"] = tinysum["science"].fillna(0).astype("Int64") + science_band_cols = [f"# {b} science" for b in _BANDS] + tinysum[science_band_cols] = tinysum[science_band_cols].fillna(0).astype("Int64") tinysum["science targets"] = tinysum["science targets"].fillna("") if almanac is not None: diff --git a/schedview/reports.py b/schedview/reports.py index 5ecd4347..8bc7f801 100644 --- a/schedview/reports.py +++ b/schedview/reports.py @@ -9,7 +9,7 @@ import pandas as pd from rubin_scheduler.site_models import Almanac -from schedview.compute.smallsum import compute_tinysum +from schedview.compute.smallsum import compute_tinysum, format_band_breakdown RSS_DESC_FORMAT = """ Total visits: {total}; @@ -267,12 +267,17 @@ def make_report_rss_feed( teff_rate = np.round(tinysum.loc[dayobs, "teff/minute"], 2) except TypeError: teff_rate = np.nan + row = tinysum.loc[dayobs] + total_bands = format_band_breakdown(row, suffix="") + science_bands = format_band_breakdown(row, suffix=" science") + total_str = f"{row['Total']}" + (f" ({total_bands})" if total_bands else "") + science_str = f"{row['science']}" + (f" ({science_bands})" if science_bands else "") desc.text = RSS_DESC_FORMAT.format( report=report_row.report, instrument=instrument, night=report_row.night, - total=tinysum.loc[dayobs, "Total"], - science=tinysum.loc[dayobs, "science"], + total=total_str, + science=science_str, fwhm=np.round(tinysum.loc[dayobs, "median FWHM"], 2), mean_norm_teff=np.round(tinysum.loc[dayobs, "total eff_time/exp_time"], 2), visit_rate=np.round(tinysum.loc[dayobs, "visits/hour"], 2), diff --git a/tests/test_compute_smallsum.py b/tests/test_compute_smallsum.py index 21fefd50..4be610f2 100644 --- a/tests/test_compute_smallsum.py +++ b/tests/test_compute_smallsum.py @@ -8,7 +8,7 @@ from rubin_scheduler.scheduler.model_observatory import ModelObservatory from rubin_scheduler.site_models import Almanac -from schedview.compute.smallsum import compute_smallsum, compute_tinysum +from schedview.compute.smallsum import compute_smallsum, compute_tinysum, format_band_breakdown SCIENCE_PROGRAMS = ("BLOCK-365", "BLOCK-407") ALL_BANDS: tuple[str, ...] = tuple(ModelObservatory(no_sky=True).bandlist) @@ -110,6 +110,27 @@ def test_science_count_correct(self, sample_visits): ) assert result.loc[day, "science"] == expected + def test_science_band_columns_present_and_int(self, sample_visits): + result = compute_tinysum(sample_visits, science_programs=SCIENCE_PROGRAMS) + for b in ALL_BANDS: + col = f"# {b} science" + assert col in result.columns + assert pd.api.types.is_integer_dtype(result[col]) + + def test_science_band_counts_sum_to_science(self, sample_visits): + result = compute_tinysum(sample_visits, science_programs=SCIENCE_PROGRAMS) + sci_cols = [f"# {b} science" for b in ALL_BANDS] + for day in result.index: + assert result.loc[day, sci_cols].sum() == result.loc[day, "science"] + + def test_science_band_counts_correct(self, sample_visits): + result = compute_tinysum(sample_visits, science_programs=SCIENCE_PROGRAMS) + sci = sample_visits[sample_visits["science_program"].isin(SCIENCE_PROGRAMS)] + for day in result.index: + for b in ALL_BANDS: + expected = len(sci[(sci["dayObs"] == day) & (sci["band"] == b)]) + assert result.loc[day, f"# {b} science"] == expected + def test_teff_stats_reasonable(self, sample_visits): result = compute_tinysum(sample_visits) for day in result.index: @@ -151,6 +172,8 @@ def test_no_science_visits_night(self): result = compute_tinysum(visits, science_programs=("BLOCK-365",)) assert result.loc[20250601, "science"] == 0 assert result.loc[20250601, "science targets"] == "" + for b in ALL_BANDS: + assert result.loc[20250601, f"# {b} science"] == 0 def test_missing_bands(self): visits = pd.DataFrame( @@ -277,3 +300,27 @@ def test_subset_order_matches_contract(self, sample_visits): first_science_pos = min(subsets.index(s) for s in ("science", "not_science") if s in subsets) assert first_band_pos < first_science_pos + + +# --------------------------------------------------------------------------- +# Tests for format_band_breakdown +# --------------------------------------------------------------------------- + + +class TestFormatBandBreakdown: + def test_total_breakdown(self): + row = pd.Series({"# u": 0, "# g": 500, "# r": 170, "# i": 6, "# z": 0, "# y": 0}) + assert format_band_breakdown(row) == "500g, 170r, 6i" + + def test_science_suffix(self): + row = pd.Series({"# g science": 400, "# r science": 85, "# i science": 5}) + assert format_band_breakdown(row, suffix=" science") == "400g, 85r, 5i" + + def test_all_zero_returns_empty(self): + row = pd.Series({f"# {b}": 0 for b in ALL_BANDS}) + assert format_band_breakdown(row) == "" + + def test_band_order_independent_of_input_order(self): + # Series built out of band order; output must follow _BANDS order. + row = pd.Series({"# i": 6, "# g": 500, "# r": 170}) + assert format_band_breakdown(row) == "500g, 170r, 6i" diff --git a/tests/test_reports.py b/tests/test_reports.py index c1be161e..282ee546 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -86,6 +86,38 @@ def test_make_report_rss_feed(self): # See if we can parse the result as XML ET.parse(str(test_file)) + def test_make_report_rss_feed_has_band_breakdown(self): + import re + + rng = np.random.default_rng(0) + n = 40 + # Match dayObs values used for test reports + dayobs = np.array([20250620] * 20 + [20250621] * 20) + visits = pd.DataFrame( + { + "dayObs": dayobs, + "observationId": np.arange(n), + "seeingFwhmGeom": rng.uniform(0.6, 1.8, size=n), + "eff_time_median": rng.uniform(20.0, 40.0, size=n), + "exp_time": rng.uniform(25.0, 35.0, size=n), + "band": rng.choice(list("ugrizy"), size=n), + "science_program": rng.choice(["BLOCK-365", "ENG-001"], size=n), + "target_name": rng.choice(["COSMOS", "XMM-LSS", ""], size=n), + } + ) + reports = schedview.reports.find_reports(self.temp_dir.name) + rss_tree = schedview.reports.make_report_rss_feed(reports, fname=None, max_days=99999, visits=visits) + descriptions = [d.text or "" for d in rss_tree.getroot().iterfind("channel/item/description")] + joined = "\n".join(descriptions) + # The total visit line must carry a parenthetical band breakdown. + assert re.search(r"Total visits: \d+ \(\d+[ugrizy]", joined) + # The science line carries a band breakdown only when there are science + # visits. Whether a program counts as science depends on the default + # SCIENCE_PROGRAMS, which is empty unless rubin_nights is installed, so + # only assert the breakdown format when science visits are present. + if re.search(r"Science visits: [1-9]", joined): + assert re.search(r"Science visits: \d+ \(\d+[ugrizy]", joined) + def test_make_report_rss_feed_uses_title_parameter(self): reports = schedview.reports.find_reports(self.temp_dir.name) channel_title = "Custom Schedview Reports" From 3d79d43ccd0c27a8425679d182f04c62fc6674bf Mon Sep 17 00:00:00 2001 From: "Eric H. Neilsen, Jr." <ehneilsen@users.noreply.github.com> Date: Mon, 29 Jun 2026 17:36:38 -0500 Subject: [PATCH 14/23] Initial (mostly claude code) addition of description content to lsstcam prenight rss entries --- design/SP-3225/linktablestats.rst | 141 ++++++++++++++++++++++++++---- design/SP-3225/smallsum.rst | 79 +++++++++++++++-- schedview/compute/smallsum.py | 39 +++++++-- schedview/reports.py | 133 +++++++++++++++++++++------- tests/test_compute_smallsum.py | 25 ++++++ tests/test_reports.py | 68 ++++++++++++++ 6 files changed, 422 insertions(+), 63 deletions(-) diff --git a/design/SP-3225/linktablestats.rst b/design/SP-3225/linktablestats.rst index 0daff8e6..4a47363a 100644 --- a/design/SP-3225/linktablestats.rst +++ b/design/SP-3225/linktablestats.rst @@ -20,6 +20,15 @@ When a ``visits`` DataFrame is provided: total visits, science visits, seeing, effective-time metrics, and science targets. +When a ``prenight_visits`` DataFrame is provided: + +- ``make_report_rss_feed`` populates the ``<description>`` element of + ``lsstcam`` ``prenight`` items with the same formatted summary string, but + computed from the **prenight simulation** of the night rather than from + completed visits. A prenight night with no matching simulation visits gets + a completely blank description (unlike ``nightsum``, which falls back to + ``"No visits on this night"``). + Additionally, numeric presentation is improved: float columns are rounded to 2 decimal places and NaN/None values are rendered as empty strings. @@ -204,6 +213,7 @@ Signature visits: pd.DataFrame | None = None, title: str = "schedview reports", description: str = "Statically generated reports on Rubin Observatory/LSST scheduler status and progress", + prenight_visits: pd.DataFrame | None = None, ) -> ET.ElementTree: Parameters @@ -228,6 +238,18 @@ Parameters ``description`` : ``str`` The ``<description>`` text for the RSS channel. +``prenight_visits`` : ``pd.DataFrame`` or ``None`` + A visits DataFrame for the **prenight simulation** of the night. If + provided, lsstcam ``prenight`` items receive a formatted summary + description computed from these visits. These simulation visits carry the + columns ``t_eff`` and ``visitExposureTime`` (rather than + ``eff_time_median``/``exp_time``); those names are passed through to + ``compute_tinysum`` via its ``eff_time_column``/``exp_time_column`` + parameters, so the visits should be supplied **unmodified**. All prenight + visits are counted as science (via ``all_science=True``), since the + simulator only simulates science visits. See "Obtaining prenight visits" + below. + Returns ~~~~~~~ @@ -238,7 +260,12 @@ Implementation Steps ~~~~~~~~~~~~~~~~~~~~ 1. **Compute tinysum** (only if ``visits`` is not ``None``): instantiate - ``Almanac`` and call ``compute_tinysum``. + ``Almanac`` and call ``compute_tinysum`` on ``visits`` (the nightsum + summary). If ``prenight_visits`` is not ``None``, also call + ``compute_tinysum`` on it, passing ``eff_time_column="t_eff"``, + ``exp_time_column="visitExposureTime"``, and ``all_science=True`` (the + prenight summary; the simulator only simulates science visits, so all of + them count as science). A single ``Almanac`` instance is shared. 2. **Build RSS skeleton**: Create ``<rss>`` root with ``version="2.0"``, add ``<channel>`` with ``<title>`` and ``<description>`` elements using @@ -250,14 +277,16 @@ Implementation Steps a. Create ``<item>`` with ``<title>`` = ``"{report} for {instrument} on {night}"``. - b. **Description logic**: - - - If ``instrument == "lsstcam"`` and ``report == "nightsum"`` and - tinysum is available and the dayobs exists in tinysum's index: - format ``RSS_DESC_FORMAT`` with values from tinysum (rounding - floats to 2 decimal places; handling ``TypeError`` on - ``teff/minute`` by falling back to ``NaN``). - - If the dayobs is missing from tinysum: ``"No visits on this night"``. + b. **Description logic** (the formatting of a populated description is + factored into the private helper ``_format_summary_desc``, shared by the + nightsum and prenight branches): + + - If ``instrument == "lsstcam"`` and ``report == "nightsum"`` and the + nightsum tinysum is available: format the summary when the dayobs is + in its index, else ``"No visits on this night"``. + - If ``instrument == "lsstcam"`` and ``report == "prenight"`` and the + prenight tinysum is available: format the summary when the dayobs is + in its index, else an **empty string** (no fallback text). - Otherwise: empty string. c. Add ``<link>``, ``<guid>``, ``<category>``, and ``<pubDate>`` @@ -266,6 +295,42 @@ Implementation Steps 5. **Indent** the XML tree and optionally write to file. +Obtaining prenight visits (caller responsibility) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``make_report_rss_feed`` does **not** fetch prenight-simulation visits itself; +it only formats whatever the caller passes via ``prenight_visits``. schedview +deliberately provides no helper for this step yet — that functionality may +eventually live in ``rubin_sim.sim_archive``. The caller (e.g. the +``schedview_reports_toc`` Times Square notebook) is responsible for selecting +and reading the appropriate simulation for the night. The procedure, using +``rubin_sim.sim_archive``: + +.. code-block:: python + + from rubin_sim.sim_archive.prenightindex import ( + get_prenight_index, select_latest_prenight_sim) + from rubin_sim.sim_archive import vseqarchive + from schedview.collect.visits import NIGHT_STACKERS + from schedview import DayObs + + day_obs = DayObs.from_date("today") + # lsstcam is mounted on simonyi; latiss on auxtel. + sims = get_prenight_index(day_obs.date, telescope="simonyi") + sim = select_latest_prenight_sim(sims) # dict or None + if sim is not None: + prenight_visits = vseqarchive.get_visits( + sim["visitseq_url"], + query=f"floor(observationStartMJD-0.5)=={day_obs.mjd}", + stackers=NIGHT_STACKERS, + ) + +The returned visits carry ``t_eff`` and ``visitExposureTime`` columns and are +passed to ``make_report_rss_feed`` unmodified. +``schedview.collect.multisim.read_multiple_prenights`` is a working example of +this same ``get_prenight_index`` → ``vseqarchive.get_visits`` sequence. + + **Basic RSS generation (no visits):** >>> import datetime @@ -306,6 +371,37 @@ True >>> tree.getroot().find("channel/description").text 'Custom description' +**Prenight visits populate the prenight item description:** the simulation +visits use ``t_eff`` and ``visitExposureTime`` and are passed unmodified. + +>>> import numpy as np +>>> prenight_reports = pd.DataFrame([ +... {"night": datetime.date.today(), "dayobs": "20250620", +... "report": "prenight", "instrument": "lsstcam", +... "url": "http://example.com/pn", +... "link": '<a href="#">prenight</a>', +... "report_time": "2025-06-20T12:00:00+00:00", "fname": "pn.html"}, +... ]).set_index(["instrument", "dayobs"]).sort_values("night", ascending=False) +>>> rng = np.random.default_rng(0) +>>> n = 20 +>>> prenight_visits = pd.DataFrame({ +... "dayObs": np.full(n, 20250620), +... "observationId": np.arange(n), +... "seeingFwhmGeom": rng.uniform(0.6, 1.8, size=n), +... "t_eff": rng.uniform(20.0, 40.0, size=n), +... "visitExposureTime": rng.uniform(25.0, 35.0, size=n), +... "band": rng.choice(list("ugrizy"), size=n), +... "science_program": rng.choice(["BLOCK-365", "ENG-001"], size=n), +... "target_name": rng.choice(["COSMOS", "XMM-LSS", ""], size=n), +... }) +>>> tree = make_report_rss_feed( +... prenight_reports, fname=None, max_days=99999, +... prenight_visits=prenight_visits, +... ) +>>> desc = tree.getroot().find("channel/item/description").text +>>> "Total visits:" in desc +True + Display Formatting ------------------ @@ -331,16 +427,21 @@ Changes from ``main`` 1. **New parameter on ``make_report_link_table``**: ``visits`` (optional). 2. **New parameters on ``make_report_rss_feed``**: ``visits``, ``title``, - ``description``. -3. **Removed** ``"preprogress"`` from the default ``report_columns`` tuple + ``description``, ``prenight_visits``. +3. **New private helper** ``_format_summary_desc`` in ``schedview.reports``, + shared by the nightsum and prenight description branches. +4. **Prenight descriptions**: lsstcam ``prenight`` items get a formatted + summary from ``prenight_visits`` (a prenight simulation), or a blank + description when no simulation visits match the night. +5. **Removed** ``"preprogress"`` from the default ``report_columns`` tuple in ``make_report_link_table``. -4. **New constants**: ``RSS_DESC_FORMAT``, ``INT_SUMMARY_COLUMNS``, +6. **New constants**: ``RSS_DESC_FORMAT``, ``INT_SUMMARY_COLUMNS``, ``FLOAT_SUMMARY_COLUMNS``, ``SUMMARY_COLUMNS``. -5. **New imports**: ``numpy``, ``rubin_scheduler.site_models.Almanac``, +7. **New imports**: ``numpy``, ``rubin_scheduler.site_models.Almanac``, ``schedview.compute.smallsum.compute_tinysum``. -6. **RSS item titles** changed from ``"{report} report for ..."`` to +8. **RSS item titles** changed from ``"{report} report for ..."`` to ``"{report} for ..."``. -7. **Variable naming** in ``make_report_rss_feed`` changed to avoid +9. **Variable naming** in ``make_report_rss_feed`` changed to avoid shadowing the ``title`` parameter (``title`` → ``channel_title_elem``, etc.). @@ -372,6 +473,16 @@ Tests are in ``tests/test_reports.py``. Passes a custom ``title`` string and verifies the ``<channel><title>`` element contains exactly that text. +``test_make_report_rss_feed_prenight_description`` + Passes a synthetic ``prenight_visits`` DataFrame (using the simulator + ``t_eff``/``visitExposureTime`` columns) whose dayObs match the report + fixtures, and verifies the ``lsstcam`` ``prenight`` item descriptions carry + a populated ``Total visits: N (...)`` summary with a band breakdown. + +``test_make_report_rss_feed_prenight_blank_when_no_visits`` + Passes ``prenight_visits`` whose dayObs match no report night, and verifies + every ``prenight`` item description is empty (no fallback text). + Pre-existing tests (``test_find_reports``, ``test_make_report_link_table``, ``test_make_report_rss_feed``) continue to pass and verify backward compatibility when ``visits`` is not supplied. diff --git a/design/SP-3225/smallsum.rst b/design/SP-3225/smallsum.rst index 5e168c44..25591a1b 100644 --- a/design/SP-3225/smallsum.rst +++ b/design/SP-3225/smallsum.rst @@ -322,6 +322,9 @@ Signature visits: pd.DataFrame, science_programs: tuple[str, ...] = SCIENCE_PROGRAMS, almanac: Almanac | None = None, + eff_time_column: str = "eff_time_median", + exp_time_column: str = "exp_time", + all_science: bool = False, ) -> pd.DataFrame: Parameters @@ -329,7 +332,8 @@ Parameters ``visits`` : ``pd.DataFrame`` A DataFrame of visits. Must contain columns: ``dayObs`` (int, YYYYMMDD), - ``observationId``, ``seeingFwhmGeom``, ``eff_time_median``, ``exp_time``, + ``observationId``, ``seeingFwhmGeom``, the effective-time column named by + ``eff_time_column``, the exposure-time column named by ``exp_time_column``, ``band``, ``science_program``, ``target_name``. ``science_programs`` : ``tuple[str, ...]`` @@ -340,6 +344,57 @@ Parameters A ``rubin_scheduler.site_models.Almanac`` instance. Pass ``None`` to omit the ``night_hours``, ``visits/hour``, and ``teff/minute`` columns. +``eff_time_column`` : ``str`` + Name of the per-visit effective-time column in ``visits``. Defaults to + ``"eff_time_median"`` (the consdb/production name). Prenight-simulation + visits carry the same statistic under the name ``"t_eff"``; pass that to + summarize a prenight simulation without renaming its columns. + +``exp_time_column`` : ``str`` + Name of the per-visit exposure-time column in ``visits``. Defaults to + ``"exp_time"`` (the consdb/production name). Prenight-simulation visits + use ``"visitExposureTime"``. + +``all_science`` : ``bool`` + If ``True``, every visit is treated as a science visit regardless of its + ``science_program``, so the ``science`` count and ``# {band} science`` + columns equal the totals. Defaults to ``False``. Pass ``True`` for + prenight-simulation visits, which contain only science visits. + +The output column names are independent of these two parameters: regardless of +the input names, the summed columns are always labelled ``total eff_time`` and +``total exp_time`` (and the quartile columns ``mean eff_time`` etc.). This +lets visits from the production database (``eff_time_median``/``exp_time``) and +from prenight simulations (``t_eff``/``visitExposureTime``) be summarized into +identically-shaped tables. + +**Custom column names**: passing the simulator column names produces the same +result as the production names on equivalent data: + +>>> import numpy as np +>>> import pandas as pd +>>> from schedview.compute.smallsum import compute_tinysum +>>> prod = pd.DataFrame({ +... "dayObs": [20250601] * 4, +... "observationId": [1, 2, 3, 4], +... "seeingFwhmGeom": [0.8, 0.9, 1.0, 1.1], +... "eff_time_median": [30.0, 25.0, 35.0, 28.0], +... "exp_time": [30.0, 30.0, 30.0, 30.0], +... "band": ["g", "g", "r", "i"], +... "science_program": ["", "", "", ""], +... "target_name": ["", "", "", ""], +... }) +>>> sim = prod.rename(columns={ +... "eff_time_median": "t_eff", "exp_time": "visitExposureTime"}) +>>> a = compute_tinysum(prod) +>>> b = compute_tinysum( +... sim, eff_time_column="t_eff", exp_time_column="visitExposureTime") +>>> bool(np.isclose(a.loc[20250601, "total eff_time"], +... b.loc[20250601, "total eff_time"])) +True +>>> bool(a.loc[20250601, "Total"] == b.loc[20250601, "Total"]) +True + Returns ~~~~~~~ @@ -409,11 +464,11 @@ Implementation Steps - Count of ``observationId`` → ``Total`` (cast to ``Int64``) - Median of ``seeingFwhmGeom`` → ``median FWHM`` - - Sum of ``exp_time`` → ``total exp_time`` - - Sum of ``eff_time_median`` → ``total eff_time`` + - Sum of the ``exp_time_column`` → ``total exp_time`` + - Sum of the ``eff_time_column`` → ``total eff_time`` -2. **Effective time stats**: Group by ``dayObs``, call ``.describe()`` on - ``eff_time_median``, extract ``mean``, ``25%``, ``50%``, ``75%`` and rename +2. **Effective time stats**: Group by ``dayObs``, call ``.describe()`` on the + ``eff_time_column``, extract ``mean``, ``25%``, ``50%``, ``75%`` and rename to ``mean eff_time``, ``q1 eff_time``, ``median eff_time``, ``q3 eff_time``. @@ -421,8 +476,9 @@ Implementation Steps unstack so bands become columns, fill NaN with 0, cast to ``Int64``. Rename to ``# u``, ``# g``, etc. -4. **Science counts**: Filter visits where ``science_program`` is in - ``science_programs``, group by ``dayObs``, count → ``science``. +4. **Science counts**: Select the science visits — every visit when + ``all_science`` is ``True``, otherwise those whose ``science_program`` is in + ``science_programs`` — group by ``dayObs``, count → ``science``. 5. **Science band counts**: From the same science-visit subset, group by ``['dayObs', 'band']``, count ``observationId``, unstack, reindex to all @@ -586,5 +642,10 @@ Notes - Integer count columns (``Total``, ``science``, band counts) use pandas nullable ``Int64`` dtype to avoid float conversion when NaN values are present from joins. -- The ``exp_time`` column is required by ``compute_tinysum`` for the - ``total eff_time/exp_time`` normalized efficiency metric. +- The exposure-time column (``exp_time_column``, default ``exp_time``) is + required by ``compute_tinysum`` for the ``total eff_time/exp_time`` + normalized efficiency metric. +- The ``eff_time_column`` and ``exp_time_column`` parameters exist so that + prenight-simulation visits (which name these ``t_eff`` and + ``visitExposureTime``) can be summarized without renaming. They affect only + which input columns are read; the output column names are fixed. diff --git a/schedview/compute/smallsum.py b/schedview/compute/smallsum.py index 04b92d83..2393b57e 100644 --- a/schedview/compute/smallsum.py +++ b/schedview/compute/smallsum.py @@ -156,6 +156,9 @@ def compute_tinysum( visits: pd.DataFrame, science_programs: tuple[str, ...] = SCIENCE_PROGRAMS, almanac: Almanac | None = None, + eff_time_column: str = "eff_time_median", + exp_time_column: str = "exp_time", + all_science: bool = False, ) -> pd.DataFrame: """Create a one-row-per-night summary DataFrame from visits. @@ -163,9 +166,10 @@ def compute_tinysum( ---------- visits : `pandas.DataFrame` DataFrame of visits. Must contain columns: ``dayObs`` (int, - YYYYMMDD), ``observationId``, ``seeingFwhmGeom``, - ``eff_time_median``, ``band``, ``science_program``, - ``target_name``. + YYYYMMDD), ``observationId``, ``seeingFwhmGeom``, the + effective-time column named by ``eff_time_column``, the + exposure-time column named by ``exp_time_column``, ``band``, + ``science_program``, ``target_name``. science_programs : `tuple` [`str`], optional Tuple of ``science_program`` values considered science. Defaults to ``SCIENCE_PROGRAMS`` from @@ -175,6 +179,20 @@ def compute_tinysum( compute night duration. Pass ``None`` to omit the ``night_hours``, ``visits/hour``, and ``teff/minute`` columns. + eff_time_column : `str`, optional + Name of the per-visit effective-time column in ``visits``. + Defaults to ``"eff_time_median"`` (the consdb/production name). + Pass ``"t_eff"`` for prenight-simulation visits, which carry the + same statistic under a different name. + exp_time_column : `str`, optional + Name of the per-visit exposure-time column in ``visits``. + Defaults to ``"exp_time"`` (the consdb/production name). Pass + ``"visitExposureTime"`` for prenight-simulation visits. + all_science : `bool`, optional + If ``True``, treat every visit as a science visit regardless of + its ``science_program``, so the science counts equal the totals. + Defaults to ``False``. Pass ``True`` for prenight-simulation + visits, which contain only science visits. Returns ------- @@ -187,8 +205,8 @@ def compute_tinysum( { "observationId": "count", "seeingFwhmGeom": "median", - "exp_time": "sum", - "eff_time_median": "sum", + exp_time_column: "sum", + eff_time_column: "sum", } ) .sort_index() @@ -196,15 +214,15 @@ def compute_tinysum( columns={ "observationId": "Total", "seeingFwhmGeom": "median FWHM", - "exp_time": "total exp_time", - "eff_time_median": "total eff_time", + exp_time_column: "total exp_time", + eff_time_column: "total eff_time", } ) ) basic_stats["Total"] = basic_stats["Total"].astype("Int64") teff_stats = ( - visits.groupby("dayObs")["eff_time_median"] + visits.groupby("dayObs")[eff_time_column] .describe() .loc[:, ["mean", "25%", "50%", "75%"]] .rename( @@ -226,7 +244,10 @@ def compute_tinysum( .rename(columns={b: f"# {b}" for b in _BANDS}) ) - science_visits = visits.loc[visits["science_program"].isin(science_programs), :] + if all_science: + science_visits = visits + else: + science_visits = visits.loc[visits["science_program"].isin(science_programs), :] science_counts = science_visits.groupby("dayObs")["observationId"].count().rename("science").to_frame() science_band_counts = ( diff --git a/schedview/reports.py b/schedview/reports.py index 8bc7f801..eb7fc4ca 100644 --- a/schedview/reports.py +++ b/schedview/reports.py @@ -196,6 +196,51 @@ def make_report_link_table( return report_table_html +def _format_summary_desc(tinysum: pd.DataFrame, dayobs: int, report: str, instrument: str, night) -> str: + """Format an RSS item description from a ``compute_tinysum`` row. + + Parameters + ---------- + tinysum : `pandas.DataFrame` + Per-night summary as returned by + `schedview.compute.smallsum.compute_tinysum`. + dayobs : `int` + The dayobs (YYYYMMDD) of the night, used to index ``tinysum``. + report : `str` + The report name (e.g. ``"nightsum"`` or ``"prenight"``). + instrument : `str` + The instrument name. + night : `datetime.date` + The local calendar date of the night start. + + Returns + ------- + desc : `str` + The formatted ``RSS_DESC_FORMAT`` text for the night. + """ + try: + teff_rate = np.round(tinysum.loc[dayobs, "teff/minute"], 2) + except TypeError: + teff_rate = np.nan + row = tinysum.loc[dayobs] + total_bands = format_band_breakdown(row, suffix="") + science_bands = format_band_breakdown(row, suffix=" science") + total_str = f"{row['Total']}" + (f" ({total_bands})" if total_bands else "") + science_str = f"{row['science']}" + (f" ({science_bands})" if science_bands else "") + return RSS_DESC_FORMAT.format( + report=report, + instrument=instrument, + night=night, + total=total_str, + science=science_str, + fwhm=np.round(tinysum.loc[dayobs, "median FWHM"], 2), + mean_norm_teff=np.round(tinysum.loc[dayobs, "total eff_time/exp_time"], 2), + visit_rate=np.round(tinysum.loc[dayobs, "visits/hour"], 2), + teff_rate=teff_rate, + targets=tinysum.loc[dayobs, "science targets"], + ) + + def make_report_rss_feed( reports: pd.DataFrame, fname: str | None = None, @@ -203,6 +248,7 @@ def make_report_rss_feed( visits: pd.DataFrame | None = None, title: str = "schedview reports", description: str = "Statically generated reports on Rubin Observatory/LSST scheduler status and progress", + prenight_visits: pd.DataFrame | None = None, ) -> ET.ElementTree: """Generate an rss feed of recent schedview reports. @@ -219,12 +265,41 @@ def make_report_rss_feed( A DataFrame of visits as returned by `schedview.collect.visits.cached_read_visits`. If supplied, a short per-night summary is computed via - `schedview.compute.smallsum.compute_tinysum` and the resulting - columns are joined onto the ``lsstcam`` rows of the table. - Non-``lsstcam`` rows receive ``NA`` for these columns. - Defaults to ``None``, in which case no summary columns are added. + `schedview.compute.smallsum.compute_tinysum` and used to populate the + ``<description>`` of ``lsstcam`` ``nightsum`` items. + Defaults to ``None``, in which case nightsum descriptions are blank. title: `str`, optional The channel title, defaults to ``schedview reports`` + prenight_visits : `pd.DataFrame` or `None`, optional + A DataFrame of visits from the **prenight simulation** of the night, + used to populate the ``<description>`` of ``lsstcam`` ``prenight`` + items. No schedview helper fetches these; the caller is responsible + for selecting and reading the appropriate simulation, e.g. via + ``rubin_sim.sim_archive``:: + + from rubin_sim.sim_archive.prenightindex import ( + get_prenight_index, select_latest_prenight_sim) + from rubin_sim.sim_archive import vseqarchive + from schedview.collect.visits import NIGHT_STACKERS + + sims = get_prenight_index(day_obs, telescope="simonyi") + sim = select_latest_prenight_sim(sims) + prenight_visits = vseqarchive.get_visits( + sim["visitseq_url"], + query=f"floor(observationStartMJD-0.5)=={day_obs_mjd}", + stackers=NIGHT_STACKERS, + ) + + (See ``schedview.collect.multisim.read_multiple_prenights`` for a + working example of this sequence.) These simulation visits carry the + columns ``t_eff`` and ``visitExposureTime`` rather than + ``eff_time_median``/``exp_time``; those names are passed through to + ``compute_tinysum``, so the visits should be supplied **unmodified**. + All prenight visits are counted as science visits (the simulator only + simulates science visits), so the science counts equal the totals. + If the supplied visits contain no visits for a given night, that + night's ``prenight`` description is left completely blank. + Defaults to ``None``, in which case prenight descriptions are blank. Returns @@ -232,11 +307,19 @@ def make_report_rss_feed( rss : `ET.ElementTree` The RSS XML itself. """ - if visits is not None: - almanac = Almanac() - tinysum = compute_tinysum(visits, almanac=almanac) - else: - tinysum = None + almanac = Almanac() if (visits is not None or prenight_visits is not None) else None + tinysum = compute_tinysum(visits, almanac=almanac) if visits is not None else None + prenight_tinysum = ( + compute_tinysum( + prenight_visits, + almanac=almanac, + eff_time_column="t_eff", + exp_time_column="visitExposureTime", + all_science=True, + ) + if prenight_visits is not None + else None + ) rss = ET.Element("rss", attrib={"version": "2.0"}) channel = ET.SubElement(rss, "channel") @@ -263,30 +346,20 @@ def make_report_rss_feed( desc = ET.SubElement(item, "description") if instrument == "lsstcam" and report_row.report == "nightsum" and tinysum is not None: if dayobs in tinysum.index: - try: - teff_rate = np.round(tinysum.loc[dayobs, "teff/minute"], 2) - except TypeError: - teff_rate = np.nan - row = tinysum.loc[dayobs] - total_bands = format_band_breakdown(row, suffix="") - science_bands = format_band_breakdown(row, suffix=" science") - total_str = f"{row['Total']}" + (f" ({total_bands})" if total_bands else "") - science_str = f"{row['science']}" + (f" ({science_bands})" if science_bands else "") - desc.text = RSS_DESC_FORMAT.format( - report=report_row.report, - instrument=instrument, - night=report_row.night, - total=total_str, - science=science_str, - fwhm=np.round(tinysum.loc[dayobs, "median FWHM"], 2), - mean_norm_teff=np.round(tinysum.loc[dayobs, "total eff_time/exp_time"], 2), - visit_rate=np.round(tinysum.loc[dayobs, "visits/hour"], 2), - teff_rate=teff_rate, - targets=tinysum.loc[dayobs, "science targets"], + desc.text = _format_summary_desc( + tinysum, dayobs, report_row.report, instrument, report_row.night ) else: desc.text = "No visits on this night" - + elif instrument == "lsstcam" and report_row.report == "prenight" and prenight_tinysum is not None: + # Unlike nightsum, a prenight night with no matching simulation + # visits gets a completely blank description (no fallback text). + if dayobs in prenight_tinysum.index: + desc.text = _format_summary_desc( + prenight_tinysum, dayobs, report_row.report, instrument, report_row.night + ) + else: + desc.text = "" else: desc.text = "" link = ET.SubElement(item, "link") diff --git a/tests/test_compute_smallsum.py b/tests/test_compute_smallsum.py index 4be610f2..dfb9e50a 100644 --- a/tests/test_compute_smallsum.py +++ b/tests/test_compute_smallsum.py @@ -147,6 +147,31 @@ def test_total_eff_time_over_exp_time(self, sample_visits): expected = day_visits["eff_time_median"].sum() / day_visits["exp_time"].sum() assert np.isclose(result.loc[day, "total eff_time/exp_time"], expected) + def test_custom_column_names(self, sample_visits): + # Prenight-simulation visits carry the same statistics under + # different names: t_eff and visitExposureTime. + renamed = sample_visits.rename(columns={"eff_time_median": "t_eff", "exp_time": "visitExposureTime"}) + result = compute_tinysum( + renamed, + eff_time_column="t_eff", + exp_time_column="visitExposureTime", + ) + # Output column names are unchanged regardless of input names. + baseline = compute_tinysum(sample_visits) + for col in ("total eff_time", "total exp_time", "mean eff_time", "total eff_time/exp_time"): + assert col in result.columns + for day in result.index: + assert np.isclose(result.loc[day, col], baseline.loc[day, col]) + + def test_all_science_counts_equal_totals(self, sample_visits): + # Prenight simulations contain only science visits, so all_science + # makes the science count and science band counts equal the totals. + result = compute_tinysum(sample_visits, all_science=True) + for day in result.index: + assert result.loc[day, "science"] == result.loc[day, "Total"] + for b in ALL_BANDS: + assert result.loc[day, f"# {b} science"] == result.loc[day, f"# {b}"] + def test_no_almanac_omits_rate_columns(self, sample_visits): result = compute_tinysum(sample_visits, almanac=None) assert {"night_hours", "visits/hour", "teff/minute"}.isdisjoint(result.columns) diff --git a/tests/test_reports.py b/tests/test_reports.py index 282ee546..4beeaa55 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -118,6 +118,74 @@ def test_make_report_rss_feed_has_band_breakdown(self): if re.search(r"Science visits: [1-9]", joined): assert re.search(r"Science visits: \d+ \(\d+[ugrizy]", joined) + def test_make_report_rss_feed_prenight_description(self): + import re + + rng = np.random.default_rng(0) + n = 40 + # Match dayObs values used for test reports. Prenight-simulation + # visits carry t_eff and visitExposureTime rather than + # eff_time_median and exp_time. + dayobs = np.array([20250620] * 20 + [20250621] * 20) + prenight_visits = pd.DataFrame( + { + "dayObs": dayobs, + "observationId": np.arange(n), + "seeingFwhmGeom": rng.uniform(0.6, 1.8, size=n), + "t_eff": rng.uniform(20.0, 40.0, size=n), + "visitExposureTime": rng.uniform(25.0, 35.0, size=n), + "band": rng.choice(list("ugrizy"), size=n), + "science_program": rng.choice(["BLOCK-365", "ENG-001"], size=n), + "target_name": rng.choice(["COSMOS", "XMM-LSS", ""], size=n), + } + ) + reports = schedview.reports.find_reports(self.temp_dir.name) + rss_tree = schedview.reports.make_report_rss_feed( + reports, fname=None, max_days=99999, prenight_visits=prenight_visits + ) + root = rss_tree.getroot() + # Collect descriptions for prenight items only (via category). + prenight_descs = [] + for item in root.iterfind("channel/item"): + category = item.findtext("category") or "" + if category == "lsstcam_prenight": + prenight_descs.append(item.findtext("description") or "") + joined = "\n".join(prenight_descs) + # The prenight items must carry a populated summary description. + assert re.search(r"Total visits: \d+ \(\d+[ugrizy]", joined) + # All prenight visits count as science, so the science line is + # populated with its own band breakdown (not zero). + assert re.search(r"Science visits: \d+ \(\d+[ugrizy]", joined) + + def test_make_report_rss_feed_prenight_blank_when_no_visits(self): + rng = np.random.default_rng(0) + n = 40 + # dayObs that do NOT match any test report night (2025-06-20/21). + dayobs = np.array([20250101] * n) + prenight_visits = pd.DataFrame( + { + "dayObs": dayobs, + "observationId": np.arange(n), + "seeingFwhmGeom": rng.uniform(0.6, 1.8, size=n), + "t_eff": rng.uniform(20.0, 40.0, size=n), + "visitExposureTime": rng.uniform(25.0, 35.0, size=n), + "band": rng.choice(list("ugrizy"), size=n), + "science_program": rng.choice(["BLOCK-365", "ENG-001"], size=n), + "target_name": rng.choice(["COSMOS", "XMM-LSS", ""], size=n), + } + ) + reports = schedview.reports.find_reports(self.temp_dir.name) + rss_tree = schedview.reports.make_report_rss_feed( + reports, fname=None, max_days=99999, prenight_visits=prenight_visits + ) + root = rss_tree.getroot() + # Prenight items with no matching simulation visits get a completely + # blank description (no "No visits on this night" fallback). + for item in root.iterfind("channel/item"): + category = item.findtext("category") or "" + if category == "lsstcam_prenight": + assert (item.findtext("description") or "") == "" + def test_make_report_rss_feed_uses_title_parameter(self): reports = schedview.reports.find_reports(self.temp_dir.name) channel_title = "Custom Schedview Reports" From 4b1176574c251abbfdbdfe012bb43e130e2adb11 Mon Sep 17 00:00:00 2001 From: Eric Neilsen <neilsen@fnal.gov> Date: Tue, 30 Jun 2026 14:00:23 -0700 Subject: [PATCH 15/23] express teff night rate as ration of seconds --- schedview/compute/smallsum.py | 6 +++--- schedview/reports.py | 25 +++++-------------------- tests/test_compute_smallsum.py | 8 ++++---- 3 files changed, 12 insertions(+), 27 deletions(-) diff --git a/schedview/compute/smallsum.py b/schedview/compute/smallsum.py index 2393b57e..d1973f13 100644 --- a/schedview/compute/smallsum.py +++ b/schedview/compute/smallsum.py @@ -178,7 +178,7 @@ def compute_tinysum( A ``rubin_scheduler.site_models.Almanac`` instance used to compute night duration. Pass ``None`` to omit the ``night_hours``, ``visits/hour``, - and ``teff/minute`` columns. + and ``teff/night duration`` columns. eff_time_column : `str`, optional Name of the per-visit effective-time column in ``visits``. Defaults to ``"eff_time_median"`` (the consdb/production name). @@ -268,7 +268,7 @@ def compute_tinysum( tinysum = basic_stats.join([teff_stats, science_counts, band_counts, science_band_counts, targets]) - tinysum["total eff_time/exp_time"] = tinysum["total eff_time"] / tinysum["total exp_time"] + tinysum["total eff_time/total exp_time"] = tinysum["total eff_time"] / tinysum["total exp_time"] tinysum["science"] = tinysum["science"].fillna(0).astype("Int64") science_band_cols = [f"# {b} science" for b in _BANDS] @@ -279,7 +279,7 @@ def compute_tinysum( night_hours = _build_night_hours(almanac, tinysum.index) tinysum["night_hours"] = night_hours tinysum["visits/hour"] = tinysum["Total"] / tinysum["night_hours"] - tinysum["teff/minute"] = tinysum["visits/hour"] * tinysum["mean eff_time"] / 60.0 + tinysum["teff/night duration"] = tinysum["total eff_time"] / (tinysum["night_hours"] * 60 * 60) return tinysum diff --git a/schedview/reports.py b/schedview/reports.py index eb7fc4ca..b77ab05d 100644 --- a/schedview/reports.py +++ b/schedview/reports.py @@ -15,9 +15,9 @@ Total visits: {total}; Science visits: {science}; Median FWHM: {fwhm}; -Total eff_time/ total exp_time: {mean_norm_teff}; Mean visit rate: {visit_rate} visits/hour; -Mean eff_time rate: {teff_rate}/minute; +Total eff_time / total exp_time: {mean_norm_teff}; +Total eff_time / total night time: {teff_rate}; Science targets: {targets} """ @@ -99,29 +99,14 @@ def find_reports( INT_SUMMARY_COLUMNS = [ "Total", "science", - # "# u", - # "# g", - # "# r", - # "# i", - # "# z", - # "# y", ] FLOAT_SUMMARY_COLUMNS = [ "night_hours", - # "visits/hour", - # "teff/minute", "median FWHM", - # "mean eff_time", - # "q1 eff_time", - # "median eff_time", - # "q3 eff_time", - "total eff_time/exp_time", + "total eff_time/total exp_time", ] -# SUMMARY_COLUMNS = INT_SUMMARY_COLUMNS + FLOAT_SUMMARY_COLUMNS + [ -# "science targets", -# ] SUMMARY_COLUMNS = INT_SUMMARY_COLUMNS + FLOAT_SUMMARY_COLUMNS @@ -219,7 +204,7 @@ def _format_summary_desc(tinysum: pd.DataFrame, dayobs: int, report: str, instru The formatted ``RSS_DESC_FORMAT`` text for the night. """ try: - teff_rate = np.round(tinysum.loc[dayobs, "teff/minute"], 2) + teff_rate = np.round(tinysum.loc[dayobs, "teff/night duration"], 2) except TypeError: teff_rate = np.nan row = tinysum.loc[dayobs] @@ -234,8 +219,8 @@ def _format_summary_desc(tinysum: pd.DataFrame, dayobs: int, report: str, instru total=total_str, science=science_str, fwhm=np.round(tinysum.loc[dayobs, "median FWHM"], 2), - mean_norm_teff=np.round(tinysum.loc[dayobs, "total eff_time/exp_time"], 2), visit_rate=np.round(tinysum.loc[dayobs, "visits/hour"], 2), + mean_norm_teff=np.round(tinysum.loc[dayobs, "total eff_time/total exp_time"], 2), teff_rate=teff_rate, targets=tinysum.loc[dayobs, "science targets"], ) diff --git a/tests/test_compute_smallsum.py b/tests/test_compute_smallsum.py index dfb9e50a..fa580d84 100644 --- a/tests/test_compute_smallsum.py +++ b/tests/test_compute_smallsum.py @@ -145,7 +145,7 @@ def test_total_eff_time_over_exp_time(self, sample_visits): for day in result.index: day_visits = sample_visits[sample_visits["dayObs"] == day] expected = day_visits["eff_time_median"].sum() / day_visits["exp_time"].sum() - assert np.isclose(result.loc[day, "total eff_time/exp_time"], expected) + assert np.isclose(result.loc[day, "total eff_time/total exp_time"], expected) def test_custom_column_names(self, sample_visits): # Prenight-simulation visits carry the same statistics under @@ -158,7 +158,7 @@ def test_custom_column_names(self, sample_visits): ) # Output column names are unchanged regardless of input names. baseline = compute_tinysum(sample_visits) - for col in ("total eff_time", "total exp_time", "mean eff_time", "total eff_time/exp_time"): + for col in ("total eff_time", "total exp_time", "mean eff_time", "total eff_time/total exp_time"): assert col in result.columns for day in result.index: assert np.isclose(result.loc[day, col], baseline.loc[day, col]) @@ -174,11 +174,11 @@ def test_all_science_counts_equal_totals(self, sample_visits): def test_no_almanac_omits_rate_columns(self, sample_visits): result = compute_tinysum(sample_visits, almanac=None) - assert {"night_hours", "visits/hour", "teff/minute"}.isdisjoint(result.columns) + assert {"night_hours", "visits/hour", "teff/night duration"}.isdisjoint(result.columns) def test_with_almanac_adds_rate_columns(self, sample_visits, almanac): result = compute_tinysum(sample_visits, almanac=almanac) - assert {"night_hours", "visits/hour", "teff/minute"}.issubset(result.columns) + assert {"night_hours", "visits/hour", "teff/night duration"}.issubset(result.columns) assert (result["night_hours"].dropna() > 0).all() def test_no_science_visits_night(self): From 438bd7a624d0920f00160afbea20f97c3005ee8e Mon Sep 17 00:00:00 2001 From: Eric Neilsen <neilsen@fnal.gov> Date: Wed, 1 Jul 2026 10:37:57 -0700 Subject: [PATCH 16/23] Update design to match recent changes in code --- design/SP-3225/designtest.rst | 9 ++++++--- design/SP-3225/linktablestats.rst | 14 ++++++++++---- design/SP-3225/smallsum.rst | 16 ++++++++-------- 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/design/SP-3225/designtest.rst b/design/SP-3225/designtest.rst index 42002bad..cafd1534 100644 --- a/design/SP-3225/designtest.rst +++ b/design/SP-3225/designtest.rst @@ -209,21 +209,24 @@ When design tests are skipped (default): .. code-block:: text + tests/test_design_docs.py::test_rst_valid[SP-3225/designtest.rst] SKIPPED + tests/test_design_docs.py::test_rst_valid[SP-3225/linktablestats.rst] SKIPPED tests/test_design_docs.py::test_rst_valid[SP-3225/smallsum.rst] SKIPPED tests/test_design_docs.py::test_rst_valid[SP-3225/visitcache.rst] SKIPPED - tests/test_design_docs.py::test_rst_valid[SP-3225/linktablestats.rst] SKIPPED ... When design tests are selected: .. code-block:: text + tests/test_design_docs.py::test_rst_valid[SP-3225/designtest.rst] PASSED + tests/test_design_docs.py::test_rst_valid[SP-3225/linktablestats.rst] PASSED tests/test_design_docs.py::test_rst_valid[SP-3225/smallsum.rst] PASSED tests/test_design_docs.py::test_rst_valid[SP-3225/visitcache.rst] PASSED - tests/test_design_docs.py::test_rst_valid[SP-3225/linktablestats.rst] PASSED + tests/test_design_docs.py::test_rst_doctests[SP-3225/designtest.rst] PASSED + tests/test_design_docs.py::test_rst_doctests[SP-3225/linktablestats.rst] PASSED tests/test_design_docs.py::test_rst_doctests[SP-3225/smallsum.rst] PASSED tests/test_design_docs.py::test_rst_doctests[SP-3225/visitcache.rst] PASSED - tests/test_design_docs.py::test_rst_doctests[SP-3225/linktablestats.rst] PASSED Dependencies diff --git a/design/SP-3225/linktablestats.rst b/design/SP-3225/linktablestats.rst index 4a47363a..e719fcad 100644 --- a/design/SP-3225/linktablestats.rst +++ b/design/SP-3225/linktablestats.rst @@ -60,7 +60,7 @@ New Module-Level Constants ``FLOAT_SUMMARY_COLUMNS`` A list of column names from ``compute_tinysum`` output that should be displayed as rounded floats: ``["night_hours", "median FWHM", - "total eff_time/exp_time"]``. + "total eff_time/total exp_time"]``. ``SUMMARY_COLUMNS`` The concatenation of ``INT_SUMMARY_COLUMNS + FLOAT_SUMMARY_COLUMNS``, @@ -209,7 +209,7 @@ Signature def make_report_rss_feed( reports: pd.DataFrame, fname: str | None = None, - max_days: int = 7, + max_days: int = 1, visits: pd.DataFrame | None = None, title: str = "schedview reports", description: str = "Statically generated reports on Rubin Observatory/LSST scheduler status and progress", @@ -437,13 +437,19 @@ Changes from ``main`` in ``make_report_link_table``. 6. **New constants**: ``RSS_DESC_FORMAT``, ``INT_SUMMARY_COLUMNS``, ``FLOAT_SUMMARY_COLUMNS``, ``SUMMARY_COLUMNS``. -7. **New imports**: ``numpy``, ``rubin_scheduler.site_models.Almanac``, - ``schedview.compute.smallsum.compute_tinysum``. +7. **New imports**: ``hashlib``, ``numpy``, + ``rubin_scheduler.site_models.Almanac``, + ``schedview.compute.smallsum.compute_tinysum``, + ``schedview.compute.smallsum.format_band_breakdown``. 8. **RSS item titles** changed from ``"{report} report for ..."`` to ``"{report} for ..."``. 9. **Variable naming** in ``make_report_rss_feed`` changed to avoid shadowing the ``title`` parameter (``title`` → ``channel_title_elem``, etc.). +10. **GUID generation** now uses a SHA-1 hash of the description text + (truncated to 6 hex characters) appended to the title, rather than the + ``report_time``. This ensures the GUID remains stable when the + description content hasn't changed. Dependencies diff --git a/design/SP-3225/smallsum.rst b/design/SP-3225/smallsum.rst index 25591a1b..38d77d21 100644 --- a/design/SP-3225/smallsum.rst +++ b/design/SP-3225/smallsum.rst @@ -68,7 +68,7 @@ Three private helpers factor out reusable logic: ``_build_night_hours(almanac, dayobs_values)`` Converts an ``Almanac`` instance into a Series mapping each ``dayObs`` to its night duration in hours. Used by ``compute_tinysum`` to derive the - rate columns (``visits/hour``, ``teff/minute``). + rate columns (``visits/hour``, ``teff/night duration``). In addition, one **public** helper is exported alongside the two main functions: @@ -342,7 +342,7 @@ Parameters ``almanac`` : ``Almanac`` or ``None`` A ``rubin_scheduler.site_models.Almanac`` instance. Pass ``None`` to omit - the ``night_hours``, ``visits/hour``, and ``teff/minute`` columns. + the ``night_hours``, ``visits/hour``, and ``teff/night duration`` columns. ``eff_time_column`` : ``str`` Name of the per-visit effective-time column in ``visits``. Defaults to @@ -444,7 +444,7 @@ per night. Columns: * - ``science targets`` - str - Comma-separated unique target names from science visits - * - ``total eff_time/exp_time`` + * - ``total eff_time/total exp_time`` - float - ``total eff_time / total exp_time`` (normalized effective time) * - ``night_hours`` @@ -453,9 +453,9 @@ per night. Columns: * - ``visits/hour`` - float - ``Total / night_hours`` (only if almanac provided) - * - ``teff/minute`` + * - ``teff/night duration`` - float - - ``(visits/hour * mean eff_time) / 60`` (only if almanac provided) + - ``total eff_time / (night_hours * 60 * 60)`` (only if almanac provided) Implementation Steps ~~~~~~~~~~~~~~~~~~~~ @@ -494,7 +494,7 @@ Implementation Steps science visits) with 0 and cast to ``Int64``. 8. **Normalized effective time**: Compute - ``total eff_time/exp_time = total eff_time / total exp_time``. + ``total eff_time/total exp_time = total eff_time / total exp_time``. 9. **Night hours** (only if ``almanac`` is not ``None``): Build a mapping from ``dayObs`` → night duration via ``_build_night_hours``. @@ -502,7 +502,7 @@ Implementation Steps 10. **Derived rates** (only if ``almanac`` is not ``None``): - ``visits/hour = Total / night_hours`` - - ``teff/minute = visits/hour * mean eff_time / 60`` + - ``teff/night duration = total eff_time / (night_hours * 60 * 60)`` Function 2: ``compute_smallsum`` @@ -643,7 +643,7 @@ Notes nullable ``Int64`` dtype to avoid float conversion when NaN values are present from joins. - The exposure-time column (``exp_time_column``, default ``exp_time``) is - required by ``compute_tinysum`` for the ``total eff_time/exp_time`` + required by ``compute_tinysum`` for the ``total eff_time/total exp_time`` normalized efficiency metric. - The ``eff_time_column`` and ``exp_time_column`` parameters exist so that prenight-simulation visits (which name these ``t_eff`` and From 529669467696f043870f4bf689ae8375ff477d41 Mon Sep 17 00:00:00 2001 From: Eric Neilsen <neilsen@fnal.gov> Date: Wed, 1 Jul 2026 11:31:10 -0700 Subject: [PATCH 17/23] Avoid instantiating ModelObservatory just to get band list --- schedview/compute/smallsum.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/schedview/compute/smallsum.py b/schedview/compute/smallsum.py index d1973f13..b66c56f7 100644 --- a/schedview/compute/smallsum.py +++ b/schedview/compute/smallsum.py @@ -13,7 +13,6 @@ import numpy as np import pandas as pd -from rubin_scheduler.scheduler.model_observatory import ModelObservatory from rubin_scheduler.site_models import Almanac from schedview import DayObs @@ -23,7 +22,7 @@ except ImportError: SCIENCE_PROGRAMS = () -_BANDS: tuple[str, ...] = tuple(ModelObservatory(no_sky=True).bandlist) +_BANDS: tuple[str, ...] = tuple("ugrizy") def _unique_targets(target_name_series: pd.Series) -> str: From 1e78ee524b0b97f4399055033c52e9916f8509e0 Mon Sep 17 00:00:00 2001 From: Eric Neilsen <neilsen@fnal.gov> Date: Wed, 1 Jul 2026 15:07:42 -0700 Subject: [PATCH 18/23] Initial draft adding eff_time breakdown to rss descriptions --- design/SP-3225/linktablestats.rst | 49 ++++++++++++++++++++++++++++- design/SP-3225/smallsum.rst | 40 ++++++++++++++++++++++-- schedview/compute/smallsum.py | 12 ++++++++ schedview/reports.py | 15 ++++++++- tests/test_reports.py | 51 +++++++++++++++++++++++++++++++ 5 files changed, 162 insertions(+), 5 deletions(-) diff --git a/design/SP-3225/linktablestats.rst b/design/SP-3225/linktablestats.rst index e719fcad..6d01cf3a 100644 --- a/design/SP-3225/linktablestats.rst +++ b/design/SP-3225/linktablestats.rst @@ -53,6 +53,13 @@ New Module-Level Constants ``science``, ``fwhm``, ``mean_norm_teff``, ``visit_rate``, ``teff_rate``, ``targets``. +``EFF_TIME_BREAKDOWN_COLS`` + A tuple of the three ``compute_tinysum`` output column names that carry + the effective-time breakdown factors: + ``("eff_time_psf_scale", "eff_time_zp_scale", "eff_time_skybg_scale")``. + Used by ``_format_summary_desc`` to detect whether breakdown data is + available. + ``INT_SUMMARY_COLUMNS`` A list of column names from ``compute_tinysum`` output that should be displayed as integers in the link table: ``["Total", "science"]``. @@ -295,6 +302,24 @@ Implementation Steps 5. **Indent** the XML tree and optionally write to file. +Helper: ``_format_summary_desc`` – effective-time breakdown +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Within ``_format_summary_desc``, the ``{mean_norm_teff}`` placeholder is +populated conditionally: + +- If all three columns in ``EFF_TIME_BREAKDOWN_COLS`` are present in the + tinysum row **and** are not NaN, the value is formatted as:: + + 0.29 [0.5 PSF, 0.8 transparency, 0.7 sky background] + + where each number is the rounded (2 decimal places) per-night + exposure-time-weighted mean of the corresponding factor column. + +- If any of the three columns is absent or NaN, the value is formatted as + before (just the ratio, e.g. ``0.29``). + + Obtaining prenight visits (caller responsibility) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -436,7 +461,8 @@ Changes from ``main`` 5. **Removed** ``"preprogress"`` from the default ``report_columns`` tuple in ``make_report_link_table``. 6. **New constants**: ``RSS_DESC_FORMAT``, ``INT_SUMMARY_COLUMNS``, - ``FLOAT_SUMMARY_COLUMNS``, ``SUMMARY_COLUMNS``. + ``FLOAT_SUMMARY_COLUMNS``, ``SUMMARY_COLUMNS``, + ``EFF_TIME_BREAKDOWN_COLS``. 7. **New imports**: ``hashlib``, ``numpy``, ``rubin_scheduler.site_models.Almanac``, ``schedview.compute.smallsum.compute_tinysum``, @@ -450,6 +476,13 @@ Changes from ``main`` (truncated to 6 hex characters) appended to the title, rather than the ``report_time``. This ensures the GUID remains stable when the description content hasn't changed. +11. **Effective-time breakdown** in RSS descriptions: when the visits + DataFrame contains ``eff_time_psf_sigma_scale_median``, + ``eff_time_zero_point_scale_median``, and + ``eff_time_sky_bg_scale_median``, the ``Total eff_time / total exp_time`` + line includes a bracketed breakdown of PSF, transparency, and sky + background factors. When any column is absent, the line retains its + original format. Dependencies @@ -489,6 +522,20 @@ Tests are in ``tests/test_reports.py``. Passes ``prenight_visits`` whose dayObs match no report night, and verifies every ``prenight`` item description is empty (no fallback text). +``test_make_report_rss_feed_eff_time_breakdown`` + Passes a synthetic visits DataFrame that includes the three breakdown + columns (``eff_time_psf_sigma_scale_median``, + ``eff_time_zero_point_scale_median``, ``eff_time_sky_bg_scale_median``), + and verifies that the ``lsstcam`` ``nightsum`` item description contains + the bracketed breakdown (``[... PSF, ... transparency, ... sky + background]``). + +``test_make_report_rss_feed_no_breakdown_without_columns`` + Passes a synthetic visits DataFrame that does **not** include the + breakdown columns, and verifies that the ``Total eff_time / total + exp_time`` line does **not** contain a bracketed breakdown (i.e. the + existing format is preserved). + Pre-existing tests (``test_find_reports``, ``test_make_report_link_table``, ``test_make_report_rss_feed``) continue to pass and verify backward compatibility when ``visits`` is not supplied. diff --git a/design/SP-3225/smallsum.rst b/design/SP-3225/smallsum.rst index 38d77d21..56dcb975 100644 --- a/design/SP-3225/smallsum.rst +++ b/design/SP-3225/smallsum.rst @@ -447,6 +447,18 @@ per night. Columns: * - ``total eff_time/total exp_time`` - float - ``total eff_time / total exp_time`` (normalized effective time) + * - ``eff_time_psf_scale`` + - float + - Exposure-time-weighted mean of ``eff_time_psf_sigma_scale_median`` + (only if column present in ``visits``) + * - ``eff_time_zp_scale`` + - float + - Exposure-time-weighted mean of ``eff_time_zero_point_scale_median`` + (only if column present in ``visits``) + * - ``eff_time_skybg_scale`` + - float + - Exposure-time-weighted mean of ``eff_time_sky_bg_scale_median`` + (only if column present in ``visits``) * - ``night_hours`` - float - Duration of night in hours (only if almanac provided) @@ -496,10 +508,32 @@ Implementation Steps 8. **Normalized effective time**: Compute ``total eff_time/total exp_time = total eff_time / total exp_time``. -9. **Night hours** (only if ``almanac`` is not ``None``): Build a mapping from - ``dayObs`` → night duration via ``_build_night_hours``. +9. **Effective-time breakdown factors** (only if all three columns + ``eff_time_psf_sigma_scale_median``, ``eff_time_zero_point_scale_median``, + and ``eff_time_sky_bg_scale_median`` are present in ``visits``): + + For each factor column, compute the per-night exposure-time-weighted mean: + + .. code-block:: python + + weighted = (visits[factor_col] * visits[exp_time_column]) + numerator = weighted.groupby(visits["dayObs"]).sum() + denominator = visits[exp_time_column].groupby(visits["dayObs"]).sum() + tinysum[output_col] = numerator / denominator + + The mapping from input column to output column is: + + - ``eff_time_psf_sigma_scale_median`` → ``eff_time_psf_scale`` + - ``eff_time_zero_point_scale_median`` → ``eff_time_zp_scale`` + - ``eff_time_sky_bg_scale_median`` → ``eff_time_skybg_scale`` + + If any of the three input columns is absent, all three output columns are + omitted. + +10. **Night hours** (only if ``almanac`` is not ``None``): Build a mapping + from ``dayObs`` → night duration via ``_build_night_hours``. -10. **Derived rates** (only if ``almanac`` is not ``None``): +11. **Derived rates** (only if ``almanac`` is not ``None``): - ``visits/hour = Total / night_hours`` - ``teff/night duration = total eff_time / (night_hours * 60 * 60)`` diff --git a/schedview/compute/smallsum.py b/schedview/compute/smallsum.py index b66c56f7..6e2181a5 100644 --- a/schedview/compute/smallsum.py +++ b/schedview/compute/smallsum.py @@ -269,6 +269,18 @@ def compute_tinysum( tinysum["total eff_time/total exp_time"] = tinysum["total eff_time"] / tinysum["total exp_time"] + # Effective-time breakdown factors (only if all three columns present) + _EFF_TIME_FACTOR_MAP = { + "eff_time_psf_sigma_scale_median": "eff_time_psf_scale", + "eff_time_zero_point_scale_median": "eff_time_zp_scale", + "eff_time_sky_bg_scale_median": "eff_time_skybg_scale", + } + if all(col in visits.columns for col in _EFF_TIME_FACTOR_MAP): + exp_by_night = visits[exp_time_column].groupby(visits["dayObs"]).sum() + for input_col, output_col in _EFF_TIME_FACTOR_MAP.items(): + weighted = (visits[input_col] * visits[exp_time_column]).groupby(visits["dayObs"]).sum() + tinysum[output_col] = weighted / exp_by_night + tinysum["science"] = tinysum["science"].fillna(0).astype("Int64") science_band_cols = [f"# {b} science" for b in _BANDS] tinysum[science_band_cols] = tinysum[science_band_cols].fillna(0).astype("Int64") diff --git a/schedview/reports.py b/schedview/reports.py index b77ab05d..49194917 100644 --- a/schedview/reports.py +++ b/schedview/reports.py @@ -11,6 +11,8 @@ from schedview.compute.smallsum import compute_tinysum, format_band_breakdown +EFF_TIME_BREAKDOWN_COLS = ("eff_time_psf_scale", "eff_time_zp_scale", "eff_time_skybg_scale") + RSS_DESC_FORMAT = """ Total visits: {total}; Science visits: {science}; @@ -212,6 +214,17 @@ def _format_summary_desc(tinysum: pd.DataFrame, dayobs: int, report: str, instru science_bands = format_band_breakdown(row, suffix=" science") total_str = f"{row['Total']}" + (f" ({total_bands})" if total_bands else "") science_str = f"{row['science']}" + (f" ({science_bands})" if science_bands else "") + + # Format normalized effective time, with optional breakdown + norm_teff = np.round(tinysum.loc[dayobs, "total eff_time/total exp_time"], 2) + if all(col in row.index and not pd.isna(row[col]) for col in EFF_TIME_BREAKDOWN_COLS): + psf = np.round(row["eff_time_psf_scale"], 2) + zp = np.round(row["eff_time_zp_scale"], 2) + skybg = np.round(row["eff_time_skybg_scale"], 2) + mean_norm_teff_str = f"{norm_teff} [{psf} PSF, {zp} transparency, {skybg} sky background]" + else: + mean_norm_teff_str = f"{norm_teff}" + return RSS_DESC_FORMAT.format( report=report, instrument=instrument, @@ -220,7 +233,7 @@ def _format_summary_desc(tinysum: pd.DataFrame, dayobs: int, report: str, instru science=science_str, fwhm=np.round(tinysum.loc[dayobs, "median FWHM"], 2), visit_rate=np.round(tinysum.loc[dayobs, "visits/hour"], 2), - mean_norm_teff=np.round(tinysum.loc[dayobs, "total eff_time/total exp_time"], 2), + mean_norm_teff=mean_norm_teff_str, teff_rate=teff_rate, targets=tinysum.loc[dayobs, "science targets"], ) diff --git a/tests/test_reports.py b/tests/test_reports.py index 4beeaa55..daaf4c14 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -186,6 +186,57 @@ def test_make_report_rss_feed_prenight_blank_when_no_visits(self): if category == "lsstcam_prenight": assert (item.findtext("description") or "") == "" + def test_make_report_rss_feed_eff_time_breakdown(self): + import re + + rng = np.random.default_rng(0) + n = 40 + dayobs = np.array([20250620] * 20 + [20250621] * 20) + visits = pd.DataFrame( + { + "dayObs": dayobs, + "observationId": np.arange(n), + "seeingFwhmGeom": rng.uniform(0.6, 1.8, size=n), + "eff_time_median": rng.uniform(20.0, 40.0, size=n), + "exp_time": rng.uniform(25.0, 35.0, size=n), + "band": rng.choice(list("ugrizy"), size=n), + "science_program": rng.choice(["BLOCK-365", "ENG-001"], size=n), + "target_name": rng.choice(["COSMOS", "XMM-LSS", ""], size=n), + "eff_time_psf_sigma_scale_median": rng.uniform(0.3, 1.0, size=n), + "eff_time_zero_point_scale_median": rng.uniform(0.5, 1.0, size=n), + "eff_time_sky_bg_scale_median": rng.uniform(0.4, 1.0, size=n), + } + ) + reports = schedview.reports.find_reports(self.temp_dir.name) + rss_tree = schedview.reports.make_report_rss_feed(reports, fname=None, max_days=99999, visits=visits) + descriptions = [d.text or "" for d in rss_tree.getroot().iterfind("channel/item/description")] + joined = "\n".join(descriptions) + # The breakdown must appear in the format [X PSF, Y transparency, Z sky background] + assert re.search(r"\[\d+\.\d+ PSF, \d+\.\d+ transparency, \d+\.\d+ sky background\]", joined) + + def test_make_report_rss_feed_no_breakdown_without_columns(self): + rng = np.random.default_rng(0) + n = 40 + dayobs = np.array([20250620] * 20 + [20250621] * 20) + visits = pd.DataFrame( + { + "dayObs": dayobs, + "observationId": np.arange(n), + "seeingFwhmGeom": rng.uniform(0.6, 1.8, size=n), + "eff_time_median": rng.uniform(20.0, 40.0, size=n), + "exp_time": rng.uniform(25.0, 35.0, size=n), + "band": rng.choice(list("ugrizy"), size=n), + "science_program": rng.choice(["BLOCK-365", "ENG-001"], size=n), + "target_name": rng.choice(["COSMOS", "XMM-LSS", ""], size=n), + } + ) + reports = schedview.reports.find_reports(self.temp_dir.name) + rss_tree = schedview.reports.make_report_rss_feed(reports, fname=None, max_days=99999, visits=visits) + descriptions = [d.text or "" for d in rss_tree.getroot().iterfind("channel/item/description")] + joined = "\n".join(descriptions) + # The breakdown must NOT appear when the columns are absent + assert "[" not in joined or "PSF" not in joined + def test_make_report_rss_feed_uses_title_parameter(self): reports = schedview.reports.find_reports(self.temp_dir.name) channel_title = "Custom Schedview Reports" From 7886bef450666c5bdf678ad043c8e66a6e9940ad Mon Sep 17 00:00:00 2001 From: Eric Neilsen <neilsen@fnal.gov> Date: Thu, 2 Jul 2026 12:51:15 -0700 Subject: [PATCH 19/23] Limit stats in RSS description to science visits --- design/SP-3225/linktablestats.rst | 6 ++ design/SP-3225/smallsum.rst | 115 ++++++++++++++++-------------- schedview/compute/smallsum.py | 72 ++++++++++--------- schedview/reports.py | 106 +++++++++++++-------------- tests/test_compute_smallsum.py | 39 +++++++++- 5 files changed, 196 insertions(+), 142 deletions(-) diff --git a/design/SP-3225/linktablestats.rst b/design/SP-3225/linktablestats.rst index 6d01cf3a..be81cb71 100644 --- a/design/SP-3225/linktablestats.rst +++ b/design/SP-3225/linktablestats.rst @@ -20,6 +20,12 @@ When a ``visits`` DataFrame is provided: total visits, science visits, seeing, effective-time metrics, and science targets. +Note that the statistics displayed (median FWHM, effective-time metrics, +exposure-time totals) are computed from **science visits only** — see +``compute_tinysum`` in ``schedview.compute.smallsum`` for details. The +``Total`` visit count and per-band counts (``# g``, etc.) still reflect all +visits. + When a ``prenight_visits`` DataFrame is provided: - ``make_report_rss_feed`` populates the ``<description>`` element of diff --git a/design/SP-3225/smallsum.rst b/design/SP-3225/smallsum.rst index 56dcb975..34f622db 100644 --- a/design/SP-3225/smallsum.rst +++ b/design/SP-3225/smallsum.rst @@ -386,9 +386,10 @@ result as the production names on equivalent data: ... }) >>> sim = prod.rename(columns={ ... "eff_time_median": "t_eff", "exp_time": "visitExposureTime"}) ->>> a = compute_tinysum(prod) +>>> a = compute_tinysum(prod, all_science=True) >>> b = compute_tinysum( -... sim, eff_time_column="t_eff", exp_time_column="visitExposureTime") +... sim, eff_time_column="t_eff", exp_time_column="visitExposureTime", +... all_science=True) >>> bool(np.isclose(a.loc[20250601, "total eff_time"], ... b.loc[20250601, "total eff_time"])) True @@ -412,25 +413,25 @@ per night. Columns: - Total number of visits that night (nullable integer) * - ``median FWHM`` - float - - Median ``seeingFwhmGeom`` across all visits that night + - Median ``seeingFwhmGeom`` across science visits that night * - ``total exp_time`` - float - - Sum of ``exp_time`` values for the night + - Sum of ``exp_time`` values for science visits that night * - ``total eff_time`` - float - - Sum of ``eff_time_median`` values for the night + - Sum of ``eff_time_median`` values for science visits that night * - ``mean eff_time`` - float - - Mean of ``eff_time_median`` across all visits that night + - Mean of ``eff_time_median`` across science visits that night * - ``q1 eff_time`` - float - - 25th percentile of ``eff_time_median`` + - 25th percentile of ``eff_time_median`` (science visits) * - ``median eff_time`` - float - - 50th percentile of ``eff_time_median`` + - 50th percentile of ``eff_time_median`` (science visits) * - ``q3 eff_time`` - float - - 75th percentile of ``eff_time_median`` + - 75th percentile of ``eff_time_median`` (science visits) * - ``science`` - Int64 - Number of visits with ``science_program`` in ``science_programs`` @@ -446,19 +447,20 @@ per night. Columns: - Comma-separated unique target names from science visits * - ``total eff_time/total exp_time`` - float - - ``total eff_time / total exp_time`` (normalized effective time) + - ``total eff_time / total exp_time`` (normalized effective time, + computed from science visits) * - ``eff_time_psf_scale`` - float - Exposure-time-weighted mean of ``eff_time_psf_sigma_scale_median`` - (only if column present in ``visits``) + over science visits (only if column present in ``visits``) * - ``eff_time_zp_scale`` - float - Exposure-time-weighted mean of ``eff_time_zero_point_scale_median`` - (only if column present in ``visits``) + over science visits (only if column present in ``visits``) * - ``eff_time_skybg_scale`` - float - Exposure-time-weighted mean of ``eff_time_sky_bg_scale_median`` - (only if column present in ``visits``) + over science visits (only if column present in ``visits``) * - ``night_hours`` - float - Duration of night in hours (only if almanac provided) @@ -467,76 +469,81 @@ per night. Columns: - ``Total / night_hours`` (only if almanac provided) * - ``teff/night duration`` - float - - ``total eff_time / (night_hours * 60 * 60)`` (only if almanac provided) + - ``total eff_time / (night_hours * 60 * 60)`` where ``total eff_time`` + is from science visits (only if almanac provided) Implementation Steps ~~~~~~~~~~~~~~~~~~~~ -1. **Basic stats**: Group ``visits`` by ``dayObs``, aggregate: +1. **Basic stats**: Group ``visits`` by ``dayObs``, aggregate count of + ``observationId`` → ``Total`` (cast to ``Int64``). - - Count of ``observationId`` → ``Total`` (cast to ``Int64``) - - Median of ``seeingFwhmGeom`` → ``median FWHM`` - - Sum of the ``exp_time_column`` → ``total exp_time`` - - Sum of the ``eff_time_column`` → ``total eff_time`` +2. **Band counts**: Group all visits by ``['dayObs', 'band']``, count + ``observationId``, unstack so bands become columns, fill NaN with 0, + cast to ``Int64``. Rename to ``# u``, ``# g``, etc. -2. **Effective time stats**: Group by ``dayObs``, call ``.describe()`` on the - ``eff_time_column``, extract ``mean``, ``25%``, ``50%``, ``75%`` and rename - to ``mean eff_time``, ``q1 eff_time``, ``median eff_time``, - ``q3 eff_time``. +3. **Define science visits**: Select the science-visit subset — every visit + when ``all_science`` is ``True``, otherwise those whose + ``science_program`` is in ``science_programs``. -3. **Band counts**: Group by ``['dayObs', 'band']``, count ``observationId``, - unstack so bands become columns, fill NaN with 0, cast to ``Int64``. - Rename to ``# u``, ``# g``, etc. +4. **Science stats**: From science visits, group by ``dayObs``, aggregate: -4. **Science counts**: Select the science visits — every visit when - ``all_science`` is ``True``, otherwise those whose ``science_program`` is in - ``science_programs`` — group by ``dayObs``, count → ``science``. + - Count of ``observationId`` → ``science`` + - Median of ``seeingFwhmGeom`` → ``median FWHM`` + - Sum of the ``exp_time_column`` → ``total exp_time`` + - Sum of the ``eff_time_column`` → ``total eff_time`` -5. **Science band counts**: From the same science-visit subset, group by +5. **Science band counts**: From the science-visit subset, group by ``['dayObs', 'band']``, count ``observationId``, unstack, reindex to all bands, cast to ``Int64``. Rename to ``# u science``, ``# g science``, etc. - (same pattern as the total band counts in step 3). -6. **Science targets**: From science visits, group by ``dayObs``, aggregate +6. **Effective time stats**: From science visits, group by ``dayObs``, call + ``.describe()`` on the ``eff_time_column``, extract ``mean``, ``25%``, + ``50%``, ``75%`` and rename to ``mean eff_time``, ``q1 eff_time``, + ``median eff_time``, ``q3 eff_time``. + +7. **Science targets**: From science visits, group by ``dayObs``, aggregate ``target_name`` using ``_unique_targets``. -7. **Join all** intermediate DataFrames on the ``dayObs`` index. Fill NaN in +8. **Join all** intermediate DataFrames on the ``dayObs`` index. Fill NaN in ``science`` with 0 (cast to ``Int64``) and in ``science targets`` with empty string. Fill NaN in the ``# {band} science`` columns (nights with no science visits) with 0 and cast to ``Int64``. -8. **Normalized effective time**: Compute - ``total eff_time/total exp_time = total eff_time / total exp_time``. +9. **Normalized effective time**: Compute + ``total eff_time/total exp_time = total eff_time / total exp_time`` + (both numerator and denominator are from science visits). -9. **Effective-time breakdown factors** (only if all three columns - ``eff_time_psf_sigma_scale_median``, ``eff_time_zero_point_scale_median``, - and ``eff_time_sky_bg_scale_median`` are present in ``visits``): +10. **Effective-time breakdown factors** (only if all three columns + ``eff_time_psf_sigma_scale_median``, ``eff_time_zero_point_scale_median``, + and ``eff_time_sky_bg_scale_median`` are present in the science visits): - For each factor column, compute the per-night exposure-time-weighted mean: + For each factor column, compute the per-night exposure-time-weighted mean + over **science visits**: - .. code-block:: python + .. code-block:: python - weighted = (visits[factor_col] * visits[exp_time_column]) - numerator = weighted.groupby(visits["dayObs"]).sum() - denominator = visits[exp_time_column].groupby(visits["dayObs"]).sum() - tinysum[output_col] = numerator / denominator + weighted = (science_visits[factor_col] * science_visits[exp_time_column]) + numerator = weighted.groupby(science_visits["dayObs"]).sum() + denominator = science_visits[exp_time_column].groupby(science_visits["dayObs"]).sum() + tinysum[output_col] = numerator / denominator - The mapping from input column to output column is: + The mapping from input column to output column is: - - ``eff_time_psf_sigma_scale_median`` → ``eff_time_psf_scale`` - - ``eff_time_zero_point_scale_median`` → ``eff_time_zp_scale`` - - ``eff_time_sky_bg_scale_median`` → ``eff_time_skybg_scale`` + - ``eff_time_psf_sigma_scale_median`` → ``eff_time_psf_scale`` + - ``eff_time_zero_point_scale_median`` → ``eff_time_zp_scale`` + - ``eff_time_sky_bg_scale_median`` → ``eff_time_skybg_scale`` - If any of the three input columns is absent, all three output columns are - omitted. + If any of the three input columns is absent, all three output columns are + omitted. -10. **Night hours** (only if ``almanac`` is not ``None``): Build a mapping +11. **Night hours** (only if ``almanac`` is not ``None``): Build a mapping from ``dayObs`` → night duration via ``_build_night_hours``. -11. **Derived rates** (only if ``almanac`` is not ``None``): +12. **Derived rates** (only if ``almanac`` is not ``None``): - - ``visits/hour = Total / night_hours`` - - ``teff/night duration = total eff_time / (night_hours * 60 * 60)`` + - ``visits/hour = Total / night_hours`` + - ``teff/night duration = total eff_time / (night_hours * 60 * 60)`` Function 2: ``compute_smallsum`` diff --git a/schedview/compute/smallsum.py b/schedview/compute/smallsum.py index 6e2181a5..d350b867 100644 --- a/schedview/compute/smallsum.py +++ b/schedview/compute/smallsum.py @@ -199,7 +199,27 @@ def compute_tinysum( DataFrame indexed by ``dayObs`` with one row per night. """ basic_stats = ( - visits.groupby("dayObs") + visits.groupby("dayObs", as_index=True) + .agg(Total=("observationId", "count")) + .sort_index() + .astype({"Total": "Int64"}) + ) + + band_counts = ( + visits.groupby(["dayObs", "band"])["observationId"] + .count() + .unstack("band", fill_value=0) + .reindex(columns=_BANDS, fill_value=0) + .astype("Int64") + .rename(columns={b: f"# {b}" for b in _BANDS}) + ) + + science_visits = ( + visits if all_science else visits.loc[visits["science_program"].isin(science_programs), :] + ) + + science_stats = ( + science_visits.groupby("dayObs") .agg( { "observationId": "count", @@ -211,17 +231,25 @@ def compute_tinysum( .sort_index() .rename( columns={ - "observationId": "Total", + "observationId": "science", "seeingFwhmGeom": "median FWHM", exp_time_column: "total exp_time", eff_time_column: "total eff_time", } ) ) - basic_stats["Total"] = basic_stats["Total"].astype("Int64") + + science_band_counts = ( + science_visits.groupby(["dayObs", "band"])["observationId"] + .count() + .unstack("band", fill_value=0) + .reindex(columns=_BANDS, fill_value=0) + .astype("Int64") + .rename(columns={b: f"# {b} science" for b in _BANDS}) + ) teff_stats = ( - visits.groupby("dayObs")[eff_time_column] + science_visits.groupby("dayObs")[eff_time_column] .describe() .loc[:, ["mean", "25%", "50%", "75%"]] .rename( @@ -234,30 +262,6 @@ def compute_tinysum( ) ) - band_counts = ( - visits.groupby(["dayObs", "band"])["observationId"] - .count() - .unstack("band", fill_value=0) - .reindex(columns=_BANDS, fill_value=0) - .astype("Int64") - .rename(columns={b: f"# {b}" for b in _BANDS}) - ) - - if all_science: - science_visits = visits - else: - science_visits = visits.loc[visits["science_program"].isin(science_programs), :] - science_counts = science_visits.groupby("dayObs")["observationId"].count().rename("science").to_frame() - - science_band_counts = ( - science_visits.groupby(["dayObs", "band"])["observationId"] - .count() - .unstack("band", fill_value=0) - .reindex(columns=_BANDS, fill_value=0) - .astype("Int64") - .rename(columns={b: f"# {b} science" for b in _BANDS}) - ) - targets = ( science_visits.groupby("dayObs")["target_name"] .apply(_unique_targets) @@ -265,7 +269,7 @@ def compute_tinysum( .to_frame() ) - tinysum = basic_stats.join([teff_stats, science_counts, band_counts, science_band_counts, targets]) + tinysum = basic_stats.join([science_stats, teff_stats, band_counts, science_band_counts, targets]) tinysum["total eff_time/total exp_time"] = tinysum["total eff_time"] / tinysum["total exp_time"] @@ -275,10 +279,14 @@ def compute_tinysum( "eff_time_zero_point_scale_median": "eff_time_zp_scale", "eff_time_sky_bg_scale_median": "eff_time_skybg_scale", } - if all(col in visits.columns for col in _EFF_TIME_FACTOR_MAP): - exp_by_night = visits[exp_time_column].groupby(visits["dayObs"]).sum() + if all(col in science_visits.columns for col in _EFF_TIME_FACTOR_MAP): + exp_by_night = science_visits[exp_time_column].groupby(visits["dayObs"]).sum() for input_col, output_col in _EFF_TIME_FACTOR_MAP.items(): - weighted = (visits[input_col] * visits[exp_time_column]).groupby(visits["dayObs"]).sum() + weighted = ( + (science_visits[input_col] * science_visits[exp_time_column]) + .groupby(science_visits["dayObs"]) + .sum() + ) tinysum[output_col] = weighted / exp_by_night tinysum["science"] = tinysum["science"].fillna(0).astype("Int64") diff --git a/schedview/reports.py b/schedview/reports.py index 49194917..ffeec52a 100644 --- a/schedview/reports.py +++ b/schedview/reports.py @@ -250,60 +250,60 @@ def make_report_rss_feed( ) -> ET.ElementTree: """Generate an rss feed of recent schedview reports. - Parameters - ---------- - reports : `pd.DataFrame` - A DataFrame of report metadata, as returned by `find_reports`. - fname : `str` or `None` - The file in which to write the RSS, if any. `None` to not write - a file at all. Defaults to `None`. - max_days : `int` - How many days worth of reports to include in the feed. - visits : `pd.DataFrame` or `None`, optional - A DataFrame of visits as returned by - `schedview.collect.visits.cached_read_visits`. If supplied, a short - per-night summary is computed via - `schedview.compute.smallsum.compute_tinysum` and used to populate the - ``<description>`` of ``lsstcam`` ``nightsum`` items. - Defaults to ``None``, in which case nightsum descriptions are blank. + Parameters + ---------- + reports : `pd.DataFrame` + A DataFrame of report metadata, as returned by `find_reports`. + fname : `str` or `None` + The file in which to write the RSS, if any. `None` to not write + a file at all. Defaults to `None`. + max_days : `int` + How many days worth of reports to include in the feed. + visits : `pd.DataFrame` or `None`, optional + A DataFrame of visits as returned by + `schedview.collect.visits.cached_read_visits`. If supplied, a short + per-night summary is computed via + `schedview.compute.smallsum.compute_tinysum` and used to populate the + ``<description>`` of ``lsstcam`` ``nightsum`` items. + Defaults to ``None``, in which case nightsum descriptions are blank. title: `str`, optional - The channel title, defaults to ``schedview reports`` - prenight_visits : `pd.DataFrame` or `None`, optional - A DataFrame of visits from the **prenight simulation** of the night, - used to populate the ``<description>`` of ``lsstcam`` ``prenight`` - items. No schedview helper fetches these; the caller is responsible - for selecting and reading the appropriate simulation, e.g. via - ``rubin_sim.sim_archive``:: - - from rubin_sim.sim_archive.prenightindex import ( - get_prenight_index, select_latest_prenight_sim) - from rubin_sim.sim_archive import vseqarchive - from schedview.collect.visits import NIGHT_STACKERS - - sims = get_prenight_index(day_obs, telescope="simonyi") - sim = select_latest_prenight_sim(sims) - prenight_visits = vseqarchive.get_visits( - sim["visitseq_url"], - query=f"floor(observationStartMJD-0.5)=={day_obs_mjd}", - stackers=NIGHT_STACKERS, - ) - - (See ``schedview.collect.multisim.read_multiple_prenights`` for a - working example of this sequence.) These simulation visits carry the - columns ``t_eff`` and ``visitExposureTime`` rather than - ``eff_time_median``/``exp_time``; those names are passed through to - ``compute_tinysum``, so the visits should be supplied **unmodified**. - All prenight visits are counted as science visits (the simulator only - simulates science visits), so the science counts equal the totals. - If the supplied visits contain no visits for a given night, that - night's ``prenight`` description is left completely blank. - Defaults to ``None``, in which case prenight descriptions are blank. - - - Returns - ------- - rss : `ET.ElementTree` - The RSS XML itself. + The channel title, defaults to ``schedview reports`` + prenight_visits : `pd.DataFrame` or `None`, optional + A DataFrame of visits from the **prenight simulation** of the night, + used to populate the ``<description>`` of ``lsstcam`` ``prenight`` + items. No schedview helper fetches these; the caller is responsible + for selecting and reading the appropriate simulation, e.g. via + ``rubin_sim.sim_archive``:: + + from rubin_sim.sim_archive.prenightindex import ( + get_prenight_index, select_latest_prenight_sim) + from rubin_sim.sim_archive import vseqarchive + from schedview.collect.visits import NIGHT_STACKERS + + sims = get_prenight_index(day_obs, telescope="simonyi") + sim = select_latest_prenight_sim(sims) + prenight_visits = vseqarchive.get_visits( + sim["visitseq_url"], + query=f"floor(observationStartMJD-0.5)=={day_obs_mjd}", + stackers=NIGHT_STACKERS, + ) + + (See ``schedview.collect.multisim.read_multiple_prenights`` for a + working example of this sequence.) These simulation visits carry the + columns ``t_eff`` and ``visitExposureTime`` rather than + ``eff_time_median``/``exp_time``; those names are passed through to + ``compute_tinysum``, so the visits should be supplied **unmodified**. + All prenight visits are counted as science visits (the simulator only + simulates science visits), so the science counts equal the totals. + If the supplied visits contain no visits for a given night, that + night's ``prenight`` description is left completely blank. + Defaults to ``None``, in which case prenight descriptions are blank. + + + Returns + ------- + rss : `ET.ElementTree` + The RSS XML itself. """ almanac = Almanac() if (visits is not None or prenight_visits is not None) else None tinysum = compute_tinysum(visits, almanac=almanac) if visits is not None else None diff --git a/tests/test_compute_smallsum.py b/tests/test_compute_smallsum.py index fa580d84..87378bc3 100644 --- a/tests/test_compute_smallsum.py +++ b/tests/test_compute_smallsum.py @@ -141,12 +141,45 @@ def test_teff_stats_reasonable(self, sample_visits): ) def test_total_eff_time_over_exp_time(self, sample_visits): - result = compute_tinysum(sample_visits) + result = compute_tinysum(sample_visits, science_programs=SCIENCE_PROGRAMS) for day in result.index: - day_visits = sample_visits[sample_visits["dayObs"] == day] - expected = day_visits["eff_time_median"].sum() / day_visits["exp_time"].sum() + sci_visits = sample_visits[ + (sample_visits["dayObs"] == day) + & (sample_visits["science_program"].isin(SCIENCE_PROGRAMS)) + ] + expected = sci_visits["eff_time_median"].sum() / sci_visits["exp_time"].sum() assert np.isclose(result.loc[day, "total eff_time/total exp_time"], expected) + def test_science_only_stats(self, sample_visits): + """Verify that median FWHM, total eff_time, total exp_time are computed + from science visits only, not all visits.""" + result = compute_tinysum(sample_visits, science_programs=SCIENCE_PROGRAMS) + for day in result.index: + sci_visits = sample_visits[ + (sample_visits["dayObs"] == day) + & (sample_visits["science_program"].isin(SCIENCE_PROGRAMS)) + ] + assert np.isclose(result.loc[day, "median FWHM"], sci_visits["seeingFwhmGeom"].median()) + assert np.isclose(result.loc[day, "total eff_time"], sci_visits["eff_time_median"].sum()) + assert np.isclose(result.loc[day, "total exp_time"], sci_visits["exp_time"].sum()) + + def test_teff_stats_from_science_only(self, sample_visits): + """Verify mean/q1/median/q3 eff_time are computed from science visits.""" + result = compute_tinysum(sample_visits, science_programs=SCIENCE_PROGRAMS) + for day in result.index: + sci_visits = sample_visits[ + (sample_visits["dayObs"] == day) + & (sample_visits["science_program"].isin(SCIENCE_PROGRAMS)) + ] + expected_mean = sci_visits["eff_time_median"].mean() + expected_median = sci_visits["eff_time_median"].median() + expected_q1 = sci_visits["eff_time_median"].quantile(0.25) + expected_q3 = sci_visits["eff_time_median"].quantile(0.75) + assert np.isclose(result.loc[day, "mean eff_time"], expected_mean) + assert np.isclose(result.loc[day, "median eff_time"], expected_median) + assert np.isclose(result.loc[day, "q1 eff_time"], expected_q1) + assert np.isclose(result.loc[day, "q3 eff_time"], expected_q3) + def test_custom_column_names(self, sample_visits): # Prenight-simulation visits carry the same statistics under # different names: t_eff and visitExposureTime. From 3c120de87e6a5859fa62420cff8ca256fe75422f Mon Sep 17 00:00:00 2001 From: Eric Neilsen <neilsen@fnal.gov> Date: Thu, 2 Jul 2026 13:28:12 -0700 Subject: [PATCH 20/23] clarifying text --- schedview/reports.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/schedview/reports.py b/schedview/reports.py index ffeec52a..e79ec08f 100644 --- a/schedview/reports.py +++ b/schedview/reports.py @@ -16,10 +16,10 @@ RSS_DESC_FORMAT = """ Total visits: {total}; Science visits: {science}; -Median FWHM: {fwhm}; -Mean visit rate: {visit_rate} visits/hour; -Total eff_time / total exp_time: {mean_norm_teff}; -Total eff_time / total night time: {teff_rate}; +Median FWHM (science visits): {fwhm}; +Mean visit rate (all visits on sky): {visit_rate} visits/hour; +Total eff_time / total exp_time (science visits): {mean_norm_teff}; +Total eff_time / total night time (science visits): {teff_rate}; Science targets: {targets} """ From a52510062189174e1dca6f02e9c01b529b7d8787 Mon Sep 17 00:00:00 2001 From: Eric Neilsen <neilsen@fnal.gov> Date: Thu, 2 Jul 2026 13:36:04 -0700 Subject: [PATCH 21/23] black & ruff --- tests/test_compute_smallsum.py | 11 ++++------- tests/test_reports.py | 3 ++- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/test_compute_smallsum.py b/tests/test_compute_smallsum.py index 87378bc3..672cc4f1 100644 --- a/tests/test_compute_smallsum.py +++ b/tests/test_compute_smallsum.py @@ -144,8 +144,7 @@ def test_total_eff_time_over_exp_time(self, sample_visits): result = compute_tinysum(sample_visits, science_programs=SCIENCE_PROGRAMS) for day in result.index: sci_visits = sample_visits[ - (sample_visits["dayObs"] == day) - & (sample_visits["science_program"].isin(SCIENCE_PROGRAMS)) + (sample_visits["dayObs"] == day) & (sample_visits["science_program"].isin(SCIENCE_PROGRAMS)) ] expected = sci_visits["eff_time_median"].sum() / sci_visits["exp_time"].sum() assert np.isclose(result.loc[day, "total eff_time/total exp_time"], expected) @@ -156,20 +155,18 @@ def test_science_only_stats(self, sample_visits): result = compute_tinysum(sample_visits, science_programs=SCIENCE_PROGRAMS) for day in result.index: sci_visits = sample_visits[ - (sample_visits["dayObs"] == day) - & (sample_visits["science_program"].isin(SCIENCE_PROGRAMS)) + (sample_visits["dayObs"] == day) & (sample_visits["science_program"].isin(SCIENCE_PROGRAMS)) ] assert np.isclose(result.loc[day, "median FWHM"], sci_visits["seeingFwhmGeom"].median()) assert np.isclose(result.loc[day, "total eff_time"], sci_visits["eff_time_median"].sum()) assert np.isclose(result.loc[day, "total exp_time"], sci_visits["exp_time"].sum()) def test_teff_stats_from_science_only(self, sample_visits): - """Verify mean/q1/median/q3 eff_time are computed from science visits.""" + """Verify mean/q1/median/q3 eff_time computed from science visits.""" result = compute_tinysum(sample_visits, science_programs=SCIENCE_PROGRAMS) for day in result.index: sci_visits = sample_visits[ - (sample_visits["dayObs"] == day) - & (sample_visits["science_program"].isin(SCIENCE_PROGRAMS)) + (sample_visits["dayObs"] == day) & (sample_visits["science_program"].isin(SCIENCE_PROGRAMS)) ] expected_mean = sci_visits["eff_time_median"].mean() expected_median = sci_visits["eff_time_median"].median() diff --git a/tests/test_reports.py b/tests/test_reports.py index daaf4c14..525e8651 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -211,7 +211,8 @@ def test_make_report_rss_feed_eff_time_breakdown(self): rss_tree = schedview.reports.make_report_rss_feed(reports, fname=None, max_days=99999, visits=visits) descriptions = [d.text or "" for d in rss_tree.getroot().iterfind("channel/item/description")] joined = "\n".join(descriptions) - # The breakdown must appear in the format [X PSF, Y transparency, Z sky background] + # The breakdown must appear in the format + # [X PSF, Y transparency, Z sky background] assert re.search(r"\[\d+\.\d+ PSF, \d+\.\d+ transparency, \d+\.\d+ sky background\]", joined) def test_make_report_rss_feed_no_breakdown_without_columns(self): From ceb103ddda48fe712febbb69b1149d06b94dbc89 Mon Sep 17 00:00:00 2001 From: Eric Neilsen <neilsen@fnal.gov> Date: Thu, 2 Jul 2026 14:18:07 -0700 Subject: [PATCH 22/23] fix tests --- schedview/reports.py | 5 +++-- tests/test_compute_smallsum.py | 5 +++-- tests/test_reports.py | 4 +++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/schedview/reports.py b/schedview/reports.py index e79ec08f..7a5aed1c 100644 --- a/schedview/reports.py +++ b/schedview/reports.py @@ -9,7 +9,7 @@ import pandas as pd from rubin_scheduler.site_models import Almanac -from schedview.compute.smallsum import compute_tinysum, format_band_breakdown +from schedview.compute.smallsum import SCIENCE_PROGRAMS, compute_tinysum, format_band_breakdown EFF_TIME_BREAKDOWN_COLS = ("eff_time_psf_scale", "eff_time_zp_scale", "eff_time_skybg_scale") @@ -247,6 +247,7 @@ def make_report_rss_feed( title: str = "schedview reports", description: str = "Statically generated reports on Rubin Observatory/LSST scheduler status and progress", prenight_visits: pd.DataFrame | None = None, + science_programs: tuple[str, ...] = SCIENCE_PROGRAMS, ) -> ET.ElementTree: """Generate an rss feed of recent schedview reports. @@ -306,7 +307,7 @@ def make_report_rss_feed( The RSS XML itself. """ almanac = Almanac() if (visits is not None or prenight_visits is not None) else None - tinysum = compute_tinysum(visits, almanac=almanac) if visits is not None else None + tinysum = compute_tinysum(visits, almanac=almanac, science_programs=science_programs) if visits is not None else None prenight_tinysum = ( compute_tinysum( prenight_visits, diff --git a/tests/test_compute_smallsum.py b/tests/test_compute_smallsum.py index 672cc4f1..a7324429 100644 --- a/tests/test_compute_smallsum.py +++ b/tests/test_compute_smallsum.py @@ -132,7 +132,7 @@ def test_science_band_counts_correct(self, sample_visits): assert result.loc[day, f"# {b} science"] == expected def test_teff_stats_reasonable(self, sample_visits): - result = compute_tinysum(sample_visits) + result = compute_tinysum(sample_visits, science_programs=SCIENCE_PROGRAMS) for day in result.index: assert ( result.loc[day, "q1 eff_time"] @@ -185,9 +185,10 @@ def test_custom_column_names(self, sample_visits): renamed, eff_time_column="t_eff", exp_time_column="visitExposureTime", + all_science=True, ) # Output column names are unchanged regardless of input names. - baseline = compute_tinysum(sample_visits) + baseline = compute_tinysum(sample_visits, all_science=True) for col in ("total eff_time", "total exp_time", "mean eff_time", "total eff_time/total exp_time"): assert col in result.columns for day in result.index: diff --git a/tests/test_reports.py b/tests/test_reports.py index 525e8651..8f790025 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -208,7 +208,9 @@ def test_make_report_rss_feed_eff_time_breakdown(self): } ) reports = schedview.reports.find_reports(self.temp_dir.name) - rss_tree = schedview.reports.make_report_rss_feed(reports, fname=None, max_days=99999, visits=visits) + rss_tree = schedview.reports.make_report_rss_feed( + reports, fname=None, max_days=99999, visits=visits, science_programs=("BLOCK-365",) + ) descriptions = [d.text or "" for d in rss_tree.getroot().iterfind("channel/item/description")] joined = "\n".join(descriptions) # The breakdown must appear in the format From fc53147da9247a3d19f27e5a11bffbec2a8ebd55 Mon Sep 17 00:00:00 2001 From: Eric Neilsen <neilsen@fnal.gov> Date: Thu, 2 Jul 2026 14:23:31 -0700 Subject: [PATCH 23/23] black --- schedview/reports.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/schedview/reports.py b/schedview/reports.py index 7a5aed1c..fd4ac831 100644 --- a/schedview/reports.py +++ b/schedview/reports.py @@ -307,7 +307,11 @@ def make_report_rss_feed( The RSS XML itself. """ almanac = Almanac() if (visits is not None or prenight_visits is not None) else None - tinysum = compute_tinysum(visits, almanac=almanac, science_programs=science_programs) if visits is not None else None + tinysum = ( + compute_tinysum(visits, almanac=almanac, science_programs=science_programs) + if visits is not None + else None + ) prenight_tinysum = ( compute_tinysum( prenight_visits,