From 7774c3cecf9f825261757b0c7775499fa9f6c9ba Mon Sep 17 00:00:00 2001 From: Robert Lupton the Good Date: Thu, 9 Jul 2026 10:00:49 -0700 Subject: [PATCH] Add observatory status timeline notebook Visualises Rubin Observatory EFD status states (OPERATIONAL, FAULT, WEATHER, DOWNTIME, UNKNOWN, IDLE), filter-band visit data, dome-shutter intervals, and cumulative per-night and running-total statistics. Times Square parameters: ndays (default 7), date_start, date_end. plot_helpers.py inlined; %matplotlib inline for Times Square rendering. Generated with Claude Co-Authored-By: SLAC AI --- nightly/plot_timeline.ipynb | 1757 +++++++++++++++++++++++++++++++++++ nightly/plot_timeline.yaml | 25 + 2 files changed, 1782 insertions(+) create mode 100644 nightly/plot_timeline.ipynb create mode 100644 nightly/plot_timeline.yaml diff --git a/nightly/plot_timeline.ipynb b/nightly/plot_timeline.ipynb new file mode 100644 index 0000000..4e1b14b --- /dev/null +++ b/nightly/plot_timeline.ipynb @@ -0,0 +1,1757 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "0", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "if os.getenv(\"EXTERNAL_INSTANCE_URL\") is not None:\n", + " !pip install --user --upgrade git+https://github.com/lsst-sims/rubin_nights.git --no-deps --no-build-isolation > /dev/null 2>&1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import matplotlib.dates as mdates\n", + "from matplotlib.patches import Patch\n", + "from astropy.time import Time\n", + "import astropy.units as u\n", + "from rubin_nights.observatory_status import get_dome_open_close\n", + "from scipy.stats import median_abs_deviation\n", + "\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "from lsst.ts.xml.enums.Scheduler import ObservatoryStatus\n", + "\n", + "from rubin_nights import connections\n", + "\n", + "_current_location = os.getenv(\"EXTERNAL_INSTANCE_URL\", \"\")\n", + "if _current_location != \"\":\n", + " _tokenfile = None\n", + " _site = None\n", + "else:\n", + " _tokenfile = os.path.join(os.path.expanduser(\"~\"), \".lsst/usdf_rsp\")\n", + " _site = 'usdf'\n", + "_endpoints = connections.get_clients(tokenfile=_tokenfile, site=_site)\n", + "_efd_client = _endpoints['efd']\n", + "_consdb_client = _endpoints['consdb']\n", + "\n", + "DATA_NDAYS = 100\n", + "_t_end = Time.now()\n", + "_data_now = pd.Timestamp.utcnow()\n", + "_data_now_naive = _data_now.tz_localize(None)\n", + "_t_start = _t_end - DATA_NDAYS * u.day\n", + "\n", + "# --- Software version check ---\n", + "_sw_ver = _efd_client.select_top_n(\n", + " 'lsst.sal.Scheduler.logevent_softwareVersions',\n", + " ['xmlVersion', 'cscVersion'], 1)\n", + "def _ver_tuple(s):\n", + " try:\n", + " return tuple(int(x) for x in str(s).split('.')[:2])\n", + " except (ValueError, AttributeError):\n", + " return (0, 0)\n", + "_xml_ver = _ver_tuple(_sw_ver['xmlVersion'].iloc[0]) if not _sw_ver.empty else (0, 0)\n", + "_csc_ver = _ver_tuple(_sw_ver['cscVersion'].iloc[0]) if not _sw_ver.empty else (0, 0)\n", + "_new_status_logic = _xml_ver >= (27, 0) and _csc_ver >= (2, 10)\n", + "\n", + "# --- Observatory status ---\n", + "df = _efd_client.select_time_series(\n", + " 'lsst.sal.Scheduler.logevent_observatoryStatus',\n", + " ['status', 'note', 'statusLabels', 'salIndex'],\n", + " _t_start, _t_end\n", + ")\n", + "df = df[df['salIndex'] == 1].drop(columns=['salIndex'], errors='ignore')\n", + "\n", + "# --- Visits from ConsDB ---\n", + "_py_start = _t_start.to_datetime()\n", + "_py_end = _t_end.to_datetime()\n", + "_day_start = int(_py_start.strftime('%Y%m%d')) - 1\n", + "_day_end = int(_py_end.strftime('%Y%m%d')) + 1\n", + "visits = _consdb_client.query(f'''\n", + " SELECT\n", + " exp_midpt, visit1.obs_start_mjd,\n", + " visit1.day_obs, visit1.seq_num,\n", + " altitude, azimuth, band, img_type, exp_time,\n", + " science_program, target_name\n", + " FROM cdb_lsstcam.visit1\n", + " LEFT JOIN cdb_lsstcam.visit1_quicklook\n", + " ON visit1.visit_id = visit1_quicklook.visit_id\n", + " WHERE visit1.day_obs >= {_day_start} AND visit1.day_obs <= {_day_end}\n", + " ORDER BY COALESCE(exp_midpt, to_timestamp((visit1.obs_start_mjd - 40587.0) * 86400.0))\n", + "''')\n", + "visits['exp_midpt'] = pd.to_datetime(visits['exp_midpt'], format='ISO8601', utc=True, errors='coerce')\n", + "if 'obs_start_mjd' in visits.columns:\n", + " visits['obs_start_mjd'] = pd.to_numeric(visits['obs_start_mjd'], errors='coerce')\n", + " _missing_midpt = visits['exp_midpt'].isna() & visits['obs_start_mjd'].notna()\n", + " if _missing_midpt.any():\n", + " _half_s = pd.to_timedelta(visits.loc[_missing_midpt, 'exp_time'].fillna(0), unit='s') / 2\n", + " _obs_start = pd.to_datetime(\n", + " (visits.loc[_missing_midpt, 'obs_start_mjd'] - 40587.0) * 86400.0, unit='s', utc=True)\n", + " visits.loc[_missing_midpt, 'exp_midpt'] = _obs_start + _half_s\n", + "\n", + "# Camera shutter intervals from exposure midpoints and times\n", + "camera_shutter_intervals = []\n", + "for _, _vrow in visits.iterrows():\n", + " if pd.notna(_vrow['exp_midpt']) and pd.notna(_vrow.get('exp_time')) and _vrow['exp_time'] > 0:\n", + " _half = pd.Timedelta(seconds=float(_vrow['exp_time']) / 2)\n", + " _mp = _vrow['exp_midpt']\n", + " _t0 = (_mp - _half).tz_localize(None) if _mp.tzinfo else _mp - _half\n", + " _t1 = (_mp + _half).tz_localize(None) if _mp.tzinfo else _mp + _half\n", + " camera_shutter_intervals.append((_t0, _t1))\n", + "\n", + "\n", + "# --- Shutter open intervals ---\n", + "_dome_client = _endpoints['efd']\n", + "_dome_df = get_dome_open_close(_t_start, _t_end, _dome_client, with_sunset_sunrise=False)\n", + "shutter_open_intervals = []\n", + "for _, row in _dome_df.iterrows():\n", + " if pd.notna(row['open_time']) and pd.notna(row['close_time']):\n", + " ot = pd.Timestamp(row['open_time'])\n", + " ct = pd.Timestamp(row['close_time'])\n", + " ot = ot.tz_convert('UTC').tz_localize(None) if ot.tzinfo is not None else ot\n", + " ct = ct.tz_convert('UTC').tz_localize(None) if ct.tzinfo is not None else ct\n", + " shutter_open_intervals.append((ot, ct))\n", + "del _dome_df, _dome_client\n", + "\n", + "\n", + "# Colors: fault=vermillion, weather=gray, downtime=amber, operational=green, unknown=sky-blue\n", + "COLORS = {\n", + " 'UNKNOWN': '#56B4E9',\n", + " 'OPERATIONAL': '#009E73',\n", + " 'FAULT': '#D55E00',\n", + " 'WEATHER': '#888888',\n", + " 'DOWNTIME': '#E69F00',\n", + " 'IDLE': '#CC79A7',\n", + "}\n", + "\n", + "Y_POS = {\n", + " 'UNKNOWN': 0.0,\n", + " 'OPERATIONAL': 1.0,\n", + " 'FAULT': 2.0,\n", + " 'WEATHER': 3.0,\n", + " 'DOWNTIME': 4.0,\n", + " 'IDLE': 5.0,\n", + "}\n", + "\n", + "bar_states = ['UNKNOWN', 'IDLE', 'OPERATIONAL', 'DOWNTIME', 'WEATHER', 'FAULT']\n", + "\n", + "# RTN-045 standard band colors and symbols\n", + "BAND_COLORS = {\n", + " 'u': '#1600ea',\n", + " 'g': '#31de1f',\n", + " 'r': '#b52626',\n", + " 'i': '#370201',\n", + " 'z': '#ba52ff',\n", + " 'y': '#61a2b3',\n", + "}\n", + "BAND_SYMBOLS = {\n", + " 'u': 'o',\n", + " 'g': '^',\n", + " 'r': 'v',\n", + " 'i': 's',\n", + " 'z': '*',\n", + " 'y': 'p',\n", + "}\n", + "BAND_ORDER = ['u', 'g', 'r', 'i', 'z', 'y']\n", + "\n", + "\n", + "_STATUS_BITS = [(m.value, m.name) for m in ObservatoryStatus if m.value != 0]\n", + "\n", + "\n", + "def decode_status(status_val):\n", + " try:\n", + " status_int = int(status_val)\n", + " except (TypeError, ValueError):\n", + " return {'UNKNOWN'}\n", + " labels = {name for val, name in _STATUS_BITS if status_int & val}\n", + " return labels if labels else {'UNKNOWN'}\n", + "\n", + "\n", + "df['labels'] = df['status'].apply(decode_status)\n", + "timestamps = df.index.tolist()\n", + "labels_list = df['labels'].tolist()\n", + "notes_list = df['note'].tolist()\n", + "\n", + "# If shutter is currently open (and not daytime), add partial interval to now\n", + "_current_labels = labels_list[-1] if labels_list else set()\n", + "if 'DAYTIME' not in _current_labels:\n", + " _shutter_now = _efd_client.select_top_n(\n", + " 'lsst.sal.MTDome.apertureShutter',\n", + " ['positionActual0', 'positionActual1'], 1)\n", + " if not _shutter_now.empty and _shutter_now['positionActual0'].iloc[0] > 1.0:\n", + " _recent_shutter = _efd_client.select_time_series(\n", + " 'lsst.sal.MTDome.apertureShutter',\n", + " ['positionActual0'], _t_end - 12*u.hour, _t_end)\n", + " _was_closed = _recent_shutter[_recent_shutter['positionActual0'] < 1.0]\n", + " if not _was_closed.empty:\n", + " _open_since = _was_closed.index[-1]\n", + " else:\n", + " _open_since = _recent_shutter.index[0]\n", + " _ot = pd.Timestamp(_open_since)\n", + " _ot = _ot.tz_convert('UTC').tz_localize(None) if _ot.tzinfo else _ot\n", + " _now_naive = _data_now_naive\n", + " shutter_open_intervals.append((_ot, _now_naive))\n", + "\n", + "\n", + "# Build intervals; ignore DAYTIME\n", + "intervals = []\n", + "_downtime_active = False\n", + "for i in range(len(timestamps)):\n", + " start = timestamps[i]\n", + " end = timestamps[i + 1] if i + 1 < len(timestamps) else _data_now\n", + " active = set(labels_list[i]) - {'DAYTIME'}\n", + "\n", + " if not active:\n", + " _downtime_active = False\n", + " continue\n", + "\n", + " if 'UNKNOWN' in active:\n", + " _downtime_active = False # CSC offline/standby; reset carry-forward\n", + " else:\n", + " # Carry DOWNTIME through WEATHER/IDLE-only gaps (status assignment bug)\n", + " if _downtime_active and 'DOWNTIME' not in active and not (active - {'WEATHER', 'IDLE'}):\n", + " active |= {'DOWNTIME'}\n", + " _downtime_active = 'DOWNTIME' in active\n", + "\n", + " # WEATHER alone is an enum artifact; treat as WEATHER|UNKNOWN.\n", + " # Safe unconditionally: new data always pairs WEATHER with IDLE.\n", + " if active == {'WEATHER'}:\n", + " active |= {'UNKNOWN'}\n", + "\n", + " for state in active:\n", + " if state in COLORS:\n", + " intervals.append((start, end, state))\n", + "\n", + "# tz-naive intervals for twilight comparisons\n", + "intervals_naive = []\n", + "for start, end, state in intervals:\n", + " s = start.tz_localize(None) if start.tzinfo is not None else start\n", + " e = end.tz_localize(None) if end.tzinfo is not None else end\n", + " intervals_naive.append((s, e, state))\n", + "\n", + "\n", + "def filter_fault_during_downtime(intervals):\n", + " dt_ivs = [(s, e) for s, e, st in intervals if st == 'DOWNTIME']\n", + " if not dt_ivs:\n", + " return intervals\n", + " return [\n", + " (s, e, st) for s, e, st in intervals\n", + " if st != 'FAULT' or not any(ds < e and de > s for ds, de in dt_ivs)\n", + " ]\n", + "\n", + "\n", + "def filter_weather_during_downtime(intervals):\n", + " dt_ivs = [(s, e) for s, e, st in intervals if st == 'DOWNTIME']\n", + " if not dt_ivs:\n", + " return intervals\n", + " return [\n", + " (s, e, st) for s, e, st in intervals\n", + " if st != 'WEATHER' or not any(ds < e and de > s for ds, de in dt_ivs)\n", + " ]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "from astroplan import Observer\n", + "from astropy.time import Time\n", + "import astropy.units as u\n", + "from astropy.coordinates import EarthLocation\n", + "\n", + "# Compute -10 deg twilight for Rubin Observatory\n", + "rubin = Observer(\n", + " location=EarthLocation.from_geodetic(-70.7494*u.deg, -30.2444*u.deg, 2663*u.m),\n", + " name=\"Rubin\", timezone=\"Chile/Continental\"\n", + ")\n", + "\n", + "# Generate one entry per day in the date range\n", + "start_date = timestamps[0].date()\n", + "end_date = timestamps[-1].date()\n", + "day_range = pd.date_range(start_date, end_date, freq='D')\n", + "\n", + "twilight_spans = [] # (sunset_twi, sunrise_twi) in matplotlib dates — these are NIGHT spans\n", + "for day in day_range:\n", + " t = Time(str(day.date()), scale='utc') + 12*u.hour # noon UTC ~ morning Chile\n", + " try:\n", + " sunset = rubin.sun_set_time(t, which='next',\n", + " horizon=-10*u.deg).to_datetime(timezone=None)\n", + " sunrise = rubin.sun_rise_time(t + 1*u.day, which='nearest',\n", + " horizon=-10*u.deg).to_datetime(timezone=None)\n", + " twilight_spans.append((sunset, sunrise))\n", + " except Exception:\n", + " pass" + ] + }, + { + "cell_type": "markdown", + "id": "4", + "metadata": {}, + "source": [ + "# Configuration" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "date_start = \"\" # YYYY-MM-DD or \"\" to use ndays\n", + "date_end = \"\" # YYYY-MM-DD exclusive, or \"\"\n", + "ndays = 3*4*7 # number of observing nights\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "COMPRESS_DAYS = True # collapse daytime gaps to a fixed narrow width\n", + "DAY_COMPRESSED_WIDTH = 3/24 # display width of each daytime gap in days (3/24 = 3 h)\n", + "NORMALIZE_NIGHTS = False # make every night the same display width; note: distorts fig3 x-axis\n", + "\n", + "SHOW_OFDU = True # include O+F+D+U sum column in the statistics table\n", + "SHOW_NIGHT_LENGTH = False # add a night-length sub-panel below the cumulative-totals plot\n", + "\n", + "DEDUP_NOTES = True # suppress repeated note markers when the note text is unchanged\n", + "# DEDUP_PREFIX: leading characters compared for deduplication; None = exact match.\n", + "# Set to 60 because the EFD auto-appends varying fault-component boilerplate after ~char 69,\n", + "# causing otherwise-identical human notes to look different on every status change.\n", + "DEDUP_PREFIX = 60\n", + "DEDUP_DEBUG = False # print each note shown or suppressed (for diagnosing dedup behaviour)\n", + "\n", + "READOUT_S = 3.4 # camera readout time in seconds (used as overhead between exposures)\n", + "MAX_SLEW_GAP_S = 120 # gaps <= this (s) between exposures counted as slew+readout overhead\n", + "\n", + "import bisect\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.dates as mdates\n", + "\n", + "# ── Date-range resolution ──────────────────────────────────────────────────\n", + "\n", + "_ndays_end_cutoff = None\n", + "plot_start = pd.Timestamp(date_start) if date_start else None\n", + "plot_end = pd.Timestamp(date_end) if date_end else None\n", + "\n", + "if ndays is not None:\n", + " _n = int(ndays)\n", + " if plot_start is not None and plot_end is None:\n", + " _ps = plot_start.tz_localize(None) if plot_start.tzinfo else plot_start\n", + " _nf = [(s, r) for s, r in twilight_spans if s >= _ps]\n", + " if len(_nf) >= _n:\n", + " plot_start = pd.Timestamp(_nf[0][0])\n", + " plot_end = pd.Timestamp(_nf[_n - 1][1])\n", + " _ndays_end_cutoff = plot_end\n", + " else:\n", + " plot_end = plot_start + pd.Timedelta(days=_n)\n", + " elif plot_end is not None and plot_start is None:\n", + " _pe = plot_end.tz_localize(None) if plot_end.tzinfo else plot_end\n", + " _nt = [(s, r) for s, r in twilight_spans if r <= _pe]\n", + " if len(_nt) >= _n:\n", + " plot_start = pd.Timestamp(_nt[-_n][0])\n", + " plot_end = pd.Timestamp(_nt[-1][1])\n", + " _ndays_end_cutoff = plot_end\n", + " else:\n", + " plot_start = plot_end - pd.Timedelta(days=_n)\n", + " elif plot_start is None and plot_end is None:\n", + " _now = _data_now_naive\n", + " _past = [(s, r) for s, r in twilight_spans if r <= _now]\n", + " _last_sr = _past[-1][1]\n", + " _next_ss = next((s for s, r in twilight_spans if s > _last_sr), None)\n", + " if _past:\n", + " plot_start = pd.Timestamp(_past[max(0, len(_past) - _n)][0])\n", + " if _next_ss is not None and _now >= _next_ss:\n", + " plot_end = pd.Timestamp(_now)\n", + " elif _next_ss is not None:\n", + " plot_end = pd.Timestamp(_next_ss)\n", + " else:\n", + " plot_end = _now\n", + " _ndays_end_cutoff = plot_end\n", + " else:\n", + " plot_end = timestamps[-1]\n", + " plot_start = plot_end - pd.Timedelta(days=_n)\n", + "\n", + "if plot_start is None:\n", + " plot_start = timestamps[0]\n", + "if plot_end is None:\n", + " plot_end = timestamps[-1]\n", + "if plot_start.tzinfo is None and timestamps[0].tzinfo is not None:\n", + " plot_start = plot_start.tz_localize(timestamps[0].tzinfo)\n", + "if plot_end.tzinfo is None and timestamps[0].tzinfo is not None:\n", + " plot_end = plot_end.tz_localize(timestamps[0].tzinfo)\n", + "\n", + "# ── Plot x-limits and twilight boundaries ─────────────────────────────────\n", + "\n", + "xlim0 = mdates.date2num(plot_start - pd.Timedelta(hours=2))\n", + "xlim1 = mdates.date2num(plot_end) + 2/24\n", + "\n", + "all_boundaries = []\n", + "plot_twilight_spans = []\n", + "for _ss, _sr in twilight_spans:\n", + " _sn, _rn = mdates.date2num(_ss), mdates.date2num(_sr)\n", + " if _rn >= xlim0 and _sn <= xlim1:\n", + " if _ndays_end_cutoff is not None and _ss >= _ndays_end_cutoff:\n", + " continue\n", + " all_boundaries.append((_sn, _rn))\n", + " plot_twilight_spans.append((_ss, _sr))\n", + "\n", + "# ── Piecewise-linear compression (cx / uncx) ──────────────────────────────\n", + "\n", + "_segments = []\n", + "_prev_end = xlim0\n", + "for _ns, _nr in all_boundaries:\n", + " if _ns > _prev_end:\n", + " _segments.append((_prev_end, _ns, False)) # daytime\n", + " _segments.append((_ns, _nr, True)) # nighttime\n", + " _prev_end = _nr\n", + "if _prev_end < xlim1:\n", + " _segments.append((_prev_end, xlim1, False))\n", + "\n", + "if COMPRESS_DAYS:\n", + " _real_bp = [xlim0]\n", + " _comp_bp = [0.0]\n", + " _cpos = 0.0\n", + " if NORMALIZE_NIGHTS:\n", + " _nws = [re - rs for rs, re, inight in _segments if inight]\n", + " _night_display_w = np.mean(_nws) if _nws else 0.5\n", + " for rs, re, inight in _segments:\n", + " if inight:\n", + " _cpos += _night_display_w if NORMALIZE_NIGHTS else (re - rs)\n", + " else:\n", + " _cpos += DAY_COMPRESSED_WIDTH\n", + " _real_bp.append(re)\n", + " _comp_bp.append(_cpos)\n", + " _real_bp = np.array(_real_bp)\n", + " _comp_bp = np.array(_comp_bp)\n", + "\n", + " def cx(x):\n", + " return np.interp(x, _real_bp, _comp_bp)\n", + "\n", + " def uncx(x):\n", + " return np.interp(x, _comp_bp, _real_bp)\n", + "\n", + " cxlim0, cxlim1 = float(cx(xlim0)), float(cx(xlim1))\n", + "else:\n", + " def cx(x): return x\n", + " def uncx(x): return x\n", + " cxlim0, cxlim1 = xlim0, xlim1\n", + "\n", + "# ── Tick / interval helpers ────────────────────────────────────────────────\n", + "\n", + "tick_start = plot_start.date() if hasattr(plot_start, 'date') else plot_start\n", + "tick_end = plot_end.date() if hasattr(plot_end, 'date') else plot_end\n", + "\n", + "_interval_starts = [s for s, e, state in intervals_naive]\n", + "\n", + "\n", + "def shade_daytime(ax):\n", + " _pe = cxlim0\n", + " for _ns, _nr in all_boundaries:\n", + " _cn = float(cx(_ns))\n", + " _cr = float(cx(_nr))\n", + " if _cn > _pe:\n", + " ax.axvspan(_pe, _cn, color='#fff9c4', alpha=0.6, zorder=0)\n", + " _pe = _cr\n", + " if _pe < cxlim1:\n", + " ax.axvspan(_pe, cxlim1, color='#fff9c4', alpha=0.6, zorder=0)\n", + "\n", + "\n", + "def setup_xaxis(ax, show_labels=False):\n", + " if COMPRESS_DAYS:\n", + " _span_h = (plot_end - plot_start).total_seconds() / 3600\n", + " if _span_h < 48:\n", + " _dt0 = plot_start.tz_localize(None) if plot_start.tzinfo else plot_start\n", + " _dt1 = plot_end.tz_localize(None) if plot_end.tzinfo else plot_end\n", + " if _span_h <= 6:\n", + " _freq, _fmt = '30min', '%H:%M'\n", + " elif _span_h <= 24:\n", + " _freq, _fmt = '1h', '%H:%M'\n", + " else:\n", + " _freq, _fmt = '3h', '%m-%d %H:%M'\n", + " _tds = pd.date_range(_dt0.floor('h') - pd.Timedelta(hours=1),\n", + " _dt1.ceil('h') + pd.Timedelta(hours=1), freq=_freq)\n", + " ax.set_xticks([float(cx(mdates.date2num(d))) for d in _tds])\n", + " ax.set_xticks([], minor=True)\n", + " ax.set_xticklabels([d.strftime(_fmt) for d in _tds] if show_labels else [])\n", + " else:\n", + " _td2 = pd.date_range(tick_start, tick_end, freq='2D')\n", + " ax.set_xticks([float(cx(mdates.date2num(d + pd.Timedelta(hours=12)))) for d in _td2])\n", + " ax.set_xticklabels([d.strftime('%m-%d') for d in _td2] if show_labels else [])\n", + " _td1 = pd.date_range(tick_start, tick_end, freq='D')\n", + " ax.set_xticks([float(cx(mdates.date2num(d + pd.Timedelta(hours=12)))) for d in _td1],\n", + " minor=True)\n", + " else:\n", + " ax.xaxis.set_major_formatter(mdates.DateFormatter('%m-%d'))\n", + " ax.xaxis.set_major_locator(mdates.DayLocator(interval=2))\n", + " ax.xaxis.set_minor_locator(mdates.DayLocator())\n", + " if not show_labels:\n", + " ax.set_xticklabels([])\n", + " ax.set_xlim(cxlim0, cxlim1)\n", + " ax.fmt_xdata = lambda x: mdates.num2date(uncx(x)).strftime('%Y-%m-%d %H:%M:%S')\n", + " ax.grid(axis='x', alpha=0.3)\n", + "\n", + "\n", + "def iter_status_events():\n", + " \"\"\"Yield (ts_naive, ts_cx, active_labels, text, has_note) for each status change.\n", + "\n", + " With DEDUP_NOTES=True a diamond marker is shown only on the first occurrence\n", + " of each note text. The same text never shows twice, even across empty-note events.\n", + " A genuinely new note text always shows a diamond.\n", + " \"\"\"\n", + " _last_shown_note = None\n", + " for i in range(len(timestamps)):\n", + " raw_labels = labels_list[i]\n", + " if 'UNKNOWN' in raw_labels:\n", + " active = {'UNKNOWN'}\n", + " else:\n", + " active = raw_labels - {'DAYTIME'}\n", + " if not active:\n", + " continue\n", + "\n", + " ts = timestamps[i]\n", + " ts_naive = ts.tz_localize(None) if ts.tzinfo is not None else ts\n", + " ts_cx = float(cx(mdates.date2num(ts)))\n", + "\n", + " raw_note = notes_list[i] if pd.notna(notes_list[i]) else ''\n", + " note = ' '.join(str(raw_note).split()) # collapse all whitespace variants\n", + " if DEDUP_NOTES:\n", + " _dp = globals().get('DEDUP_PREFIX', 60)\n", + " if not note:\n", + " has_note = False\n", + " elif _last_shown_note is None:\n", + " has_note = True\n", + " elif _dp is None:\n", + " has_note = note != _last_shown_note\n", + " else:\n", + " _n = min(len(note), len(_last_shown_note), _dp)\n", + " has_note = note[:_n] != _last_shown_note[:_n]\n", + " if globals().get('DEDUP_DEBUG', False):\n", + " if has_note:\n", + " print(f\" NOTE shown @ {ts_naive}: {note[:80]!r}\")\n", + " elif bool(note):\n", + " print(f\" NOTE suppressed @ {ts_naive}: {note[:80]!r}\")\n", + " if has_note:\n", + " _last_shown_note = note\n", + " else:\n", + " has_note = bool(note)\n", + "\n", + " label_str = ', '.join(sorted(active))\n", + " text = f\"{label_str}\\n{note}\" if note else label_str\n", + "\n", + " yield ts_naive, ts_cx, active, text, has_note\n", + "\n", + "\n", + "def make_hover_annot(ax):\n", + " \"\"\"Return (annot, update_fn(visible, xy, text)).\"\"\"\n", + " annot = ax.annotate('', xy=(0, 0), xytext=(10, 15),\n", + " textcoords='offset points',\n", + " bbox=dict(boxstyle='round,pad=0.3', fc='white',\n", + " ec='gray', alpha=0.95),\n", + " fontsize=7, visible=False, zorder=10)\n", + "\n", + " def update(visible, xy=None, text=None):\n", + " if visible:\n", + " annot.xy = xy\n", + " lines = text.split('\\n')\n", + " annot.set_text('\\n'.join(\n", + " l[:100] + '...' if len(l) > 100 else l for l in lines))\n", + " xlim = ax.get_xlim()\n", + " frac = ((xy[0] - xlim[0]) / (xlim[1] - xlim[0])\n", + " if xlim[1] != xlim[0] else 0)\n", + " if frac > 0.75:\n", + " annot.set_position((-10, 15)); annot.set_ha('right')\n", + " else:\n", + " annot.set_position(( 10, 15)); annot.set_ha('left')\n", + " annot.set_visible(visible)\n", + "\n", + " return annot, update\n", + "\n", + "\n", + "def _format_coord_with_states(x, y):\n", + " dt = mdates.num2date(uncx(x)).replace(tzinfo=None)\n", + " ts_str = dt.strftime('%Y-%m-%d %H:%M:%S')\n", + " active = set()\n", + " idx = bisect.bisect_right(_interval_starts, dt)\n", + " for i in range(idx - 1, -1, -1):\n", + " s, e, state = intervals_naive[i]\n", + " if e <= dt:\n", + " break\n", + " if s <= dt < e:\n", + " active.add(state)\n", + " return f\"{ts_str} [{', '.join(sorted(active))}]\" if active else ts_str\n", + "\n", + "\n", + "title_start = plot_start.strftime('%Y-%m-%d')\n", + "title_end = plot_end.strftime('%Y-%m-%d')\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "id": "7", + "metadata": {}, + "source": [ + "# Make timeline plot" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "# Close previous figure if re-running\n", + "try:\n", + " plt.close(fig)\n", + "except NameError:\n", + " pass\n", + "\n", + "# Order: Alt&Az (top), Band (middle), Status (bottom)\n", + "fig, (ax_altaz, ax_band, ax_status) = plt.subplots(\n", + " 3, 1, figsize=(10, 7), sharex=True,\n", + " gridspec_kw={'height_ratios': [0.8, 1.2, 5], 'hspace': 0})\n", + "fig.subplots_adjust(bottom=0.12)\n", + "\n", + "# --- Panel 1 (top): Altitude and Azimuth ---\n", + "shade_daytime(ax_altaz)\n", + "\n", + "visit_x_real = mdates.date2num(visits['exp_midpt'].values.astype('datetime64[ns]'))\n", + "visit_x = cx(visit_x_real)\n", + "\n", + "ax_altaz.scatter(visit_x, visits['altitude'].values, c='red', s=3, alpha=0.5, zorder=3)\n", + "ax_az = ax_altaz.twinx()\n", + "ax_az.scatter(visit_x, visits['azimuth'].values, c='blue', s=3, alpha=0.3, zorder=3)\n", + "ax_az.fmt_xdata = lambda x: mdates.num2date(uncx(x)).strftime('%Y-%m-%d %H:%M:%S')\n", + "\n", + "def _format_coord_altaz(x, y):\n", + " real_x = uncx(x)\n", + " dt = mdates.num2date(real_x).replace(tzinfo=None)\n", + " ts_str = dt.strftime('%Y-%m-%d %H:%M:%S')\n", + " # Find nearest visit\n", + " if len(visit_x) > 0:\n", + " dists = np.abs(visit_x - x)\n", + " nearest = np.argmin(dists)\n", + " alt = visits['altitude'].values[nearest]\n", + " az = visits['azimuth'].values[nearest]\n", + " return f\"{ts_str} [alt={alt:.1f}, az={az:.1f}]\"\n", + " return ts_str\n", + "\n", + "ax_az.format_coord = _format_coord_altaz\n", + "\n", + "ax_altaz.set_ylabel('Alt', color='red', fontsize=7)\n", + "ax_altaz.set_ylim(-5, 95)\n", + "ax_altaz.tick_params(axis='y', labelcolor='red', labelsize=6)\n", + "ax_az.set_ylabel('Az', color='blue', fontsize=7)\n", + "ax_az.set_ylim(-20, 380)\n", + "ax_az.tick_params(axis='y', labelcolor='blue', labelsize=6)\n", + "\n", + "from matplotlib.lines import Line2D\n", + "leg_handles = [\n", + " Line2D([0], [0], marker='o', color='w', markerfacecolor='red', markersize=4, label='Alt'),\n", + " Line2D([0], [0], marker='o', color='w', markerfacecolor='blue', markersize=4, label='Az'),\n", + "]\n", + "ax_altaz.legend(handles=leg_handles, loc='upper right', ncol=2, fontsize=7)\n", + "ax_altaz.set_title(f'Observatory Status Timeline ({title_start} to {title_end} UTC)', fontsize=10)\n", + "setup_xaxis(ax_altaz, show_labels=False)\n", + "ax_altaz.grid(False, axis=\"x\")\n", + "\n", + "# --- Panel 2 (middle): Band ---\n", + "shade_daytime(ax_band)\n", + "\n", + "is_science = (visits['img_type'] == 'science').values\n", + "bands = visits['band'].fillna('').values\n", + "img_types = visits['img_type'].values\n", + "sci_progs = visits['science_program'].fillna('').values\n", + "seq_nums = visits['seq_num'].apply(lambda x: str(int(x)) if pd.notna(x) else '').values\n", + "target_names = visits['target_name'].fillna('').values\n", + "\n", + "# y positions: science=1, other=0\n", + "band_scatter_y = np.where(is_science, 0.7, 0.0)\n", + "\n", + "# Build tooltip texts\n", + "band_scatter_texts = []\n", + "for band, itype, prog, seq, tgt in zip(bands, img_types, sci_progs, seq_nums, target_names):\n", + " parts = [itype]\n", + " if band:\n", + " parts.append(band)\n", + " if prog:\n", + " parts.append(prog)\n", + " if seq != '':\n", + " parts.append(f'seq {seq}')\n", + " if tgt:\n", + " parts.append(tgt)\n", + " band_scatter_texts.append(': '.join(parts))\n", + "\n", + "# Plot each band separately with its RTN-045 color and symbol\n", + "band_scatters = []\n", + "for b in BAND_ORDER:\n", + " mask = bands == b\n", + " if not np.any(mask):\n", + " continue\n", + " sc = ax_band.scatter(\n", + " visit_x[mask], band_scatter_y[mask],\n", + " c=BAND_COLORS[b], marker=BAND_SYMBOLS[b],\n", + " s=np.where(is_science[mask], 20, 8),\n", + " edgecolors='none', alpha=0.7, zorder=3, label=b)\n", + " band_scatters.append((sc, np.where(mask)[0]))\n", + "\n", + "# Handle visits with unknown/missing band\n", + "other_mask = ~np.isin(bands, BAND_ORDER)\n", + "if np.any(other_mask):\n", + " sc = ax_band.scatter(\n", + " visit_x[other_mask], band_scatter_y[other_mask],\n", + " c='#999999', marker='x', s=4,\n", + " alpha=0.5, zorder=3, label='?')\n", + " band_scatters.append((sc, np.where(other_mask)[0]))\n", + "\n", + "ax_band.set_ylim(-0.4, 2.2)\n", + "ax_band.set_autoscaley_on(False)\n", + "ax_band.set_yticks([0, 0.7, 1.4])\n", + "ax_band.set_yticklabels(['other', 'science', 'dome shutter'], fontsize=7)\n", + "\n", + "band_legend = [\n", + " Line2D([0], [0], marker=BAND_SYMBOLS[b], color='w',\n", + " markerfacecolor=BAND_COLORS[b], markersize=5, label=b)\n", + " for b in BAND_ORDER\n", + "]\n", + "band_legend.append(Patch(facecolor='#CC79A7', alpha=0.5, label='dome open'))\n", + "ax_band.legend(handles=band_legend, loc='upper right', ncol=7, fontsize=6)\n", + "setup_xaxis(ax_band, show_labels=False)\n", + "ax_band.grid(False, axis=\"x\")\n", + "# Show shutter-open intervals as green bar at y=1.5\n", + "for s_start, s_end in shutter_open_intervals:\n", + " x0 = float(cx(mdates.date2num(s_start)))\n", + " x1 = float(cx(mdates.date2num(s_end)))\n", + " ax_band.barh(1.4, x1 - x0, left=x0, height=0.3,\n", + " color='#CC79A7', alpha=0.5, edgecolor='none', zorder=2)\n", + "ax_band.format_coord = _format_coord_with_states\n", + "\n", + "# --- Panel 3 (bottom): Observatory Status ---\n", + "y_min, y_max = -0.5, 4.5\n", + "shade_daytime(ax_status)\n", + "\n", + "for state in bar_states:\n", + " for start, end, s in intervals:\n", + " if s != state:\n", + " continue\n", + " x0 = float(cx(mdates.date2num(start)))\n", + " x1 = float(cx(mdates.date2num(end)))\n", + " width = x1 - x0\n", + " if state == 'DOWNTIME':\n", + " ax_status.barh((y_min + y_max)/2, width, left=x0, height=y_max - y_min,\n", + " color='none', edgecolor=COLORS[state], linewidth=0,\n", + " hatch='///', alpha=0.4, zorder=2)\n", + " else:\n", + " ax_status.barh((y_min + y_max)/2, width, left=x0, height=y_max - y_min,\n", + " color=COLORS[state], edgecolor='none', alpha=0.3, zorder=2)\n", + "\n", + "priority = ['FAULT', 'WEATHER', 'DOWNTIME', 'OPERATIONAL', 'UNKNOWN']\n", + "\n", + "# Split into note/no-note for different markers (diamond=note, circle=normal)\n", + "note_x, note_y, note_c, note_texts = [], [], [], []\n", + "norm_x, norm_y, norm_c, norm_texts = [], [], [], []\n", + "scatter_x, scatter_y, scatter_texts = [], [], [] # combined for hover lookup\n", + "for ts_naive, ts_cx, active, text, has_note in iter_status_events():\n", + " for state in active:\n", + " if state in Y_POS:\n", + " c = COLORS.get(state, '#999')\n", + " scatter_x.append(ts_cx)\n", + " scatter_y.append(Y_POS[state])\n", + " scatter_texts.append(text)\n", + " if has_note:\n", + " _ns = next((s for s in priority if s in active and s in Y_POS),\n", + " next((s for s in active if s in Y_POS), None))\n", + " if state == _ns:\n", + " note_x.append(ts_cx)\n", + " note_y.append(Y_POS[state])\n", + " note_c.append(c)\n", + " note_texts.append(text)\n", + " else:\n", + " norm_x.append(ts_cx)\n", + " norm_y.append(Y_POS[state])\n", + " norm_c.append(c)\n", + " norm_texts.append(text)\n", + " else:\n", + " norm_x.append(ts_cx)\n", + " norm_y.append(Y_POS[state])\n", + " norm_c.append(c)\n", + " norm_texts.append(text)\n", + "\n", + "sc1_norm = ax_status.scatter(norm_x, norm_y, c=norm_c, s=40,\n", + " marker='o', zorder=5, edgecolors='black', linewidths=0.3)\n", + "sc1_note = ax_status.scatter(note_x, note_y, c=note_c, s=50,\n", + " marker='D', zorder=6, edgecolors='black', linewidths=0.5)\n", + "\n", + "ax_status.set_ylim(y_min, y_max)\n", + "ax_status.set_yticks([Y_POS[s] for s in ['UNKNOWN', 'OPERATIONAL', 'FAULT', 'WEATHER', 'DOWNTIME']])\n", + "ax_status.set_yticklabels(['UNKNOWN', 'OPERATIONAL', 'FAULT', 'WEATHER', 'DOWNTIME'])\n", + "\n", + "legend_items = [\n", + " Patch(facecolor='#fff9c4', label='DAYTIME'),\n", + " Patch(facecolor=COLORS['OPERATIONAL'], alpha=0.3, label='OPERATIONAL'),\n", + " Patch(facecolor=COLORS['FAULT'], alpha=0.3, label='FAULT'),\n", + " Patch(facecolor=COLORS['WEATHER'], alpha=0.3, label='WEATHER'),\n", + " Patch(facecolor='white', edgecolor=COLORS['DOWNTIME'], hatch='///', label='DOWNTIME'),\n", + " Patch(facecolor=COLORS['UNKNOWN'], alpha=0.3, label='UNKNOWN'),\n", + "]\n", + "ax_status.legend(handles=legend_items, loc='upper right', ncol=6, fontsize=8)\n", + "setup_xaxis(ax_status, show_labels=True)\n", + "ax_status.set_xlabel('Date (UTC)')\n", + "plt.setp(ax_status.xaxis.get_majorticklabels(), rotation=45, ha='right')\n", + "ax_status.format_coord = _format_coord_with_states\n", + "\n", + "# Mouseover annotations\n", + "_, update_status = make_hover_annot(ax_status)\n", + "_, update_band = make_hover_annot(ax_band)\n", + "_hover_state = [None, None] # [status_key, band_key]\n", + "\n", + "\n", + "def on_hover(event):\n", + " changed = False\n", + " if event.inaxes == ax_status:\n", + " found = False\n", + " for sc, texts, xs, ys in [(sc1_note, note_texts, note_x, note_y),\n", + " (sc1_norm, norm_texts, norm_x, norm_y)]:\n", + " cont, ind = sc.contains(event)\n", + " if cont:\n", + " i = ind['ind'][0]\n", + " key = (id(sc), i)\n", + " if key != _hover_state[0]:\n", + " update_status(True, (xs[i], ys[i]), texts[i])\n", + " _hover_state[0] = key\n", + " changed = True\n", + " found = True\n", + " break\n", + " if not found and _hover_state[0] is not None:\n", + " update_status(False)\n", + " _hover_state[0] = None\n", + " changed = True\n", + " elif _hover_state[0] is not None:\n", + " update_status(False)\n", + " _hover_state[0] = None\n", + " changed = True\n", + "\n", + " if event.inaxes == ax_band:\n", + " found = False\n", + " for sc, indices in band_scatters:\n", + " cont, ind = sc.contains(event)\n", + " if cont:\n", + " orig_idx = indices[ind['ind'][0]]\n", + " key = ('band', orig_idx)\n", + " if key != _hover_state[1]:\n", + " update_band(True,\n", + " (float(visit_x[orig_idx]), float(band_scatter_y[orig_idx])),\n", + " band_scatter_texts[orig_idx])\n", + " _hover_state[1] = key\n", + " changed = True\n", + " found = True\n", + " break\n", + " if not found and _hover_state[1] is not None:\n", + " update_band(False)\n", + " _hover_state[1] = None\n", + " changed = True\n", + " elif _hover_state[1] is not None:\n", + " update_band(False)\n", + " _hover_state[1] = None\n", + " changed = True\n", + "\n", + " if changed:\n", + " fig.canvas.draw_idle()\n", + "\n", + "fig.canvas.mpl_connect('motion_notify_event', on_hover)\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "9", + "metadata": {}, + "source": [ + "# Make plots which are cumulative over a number of days" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "# Cumulative time in each state per observing night\n", + "# An \"observing night\" runs from one -10deg sunset to the next -10deg sunrise\n", + "\n", + "STACKED_CUMUL = False # True = stacked filled areas, False = independent lines\n", + "SHADE_CURVES = True # Shade under individual curves (applies when STACKED_CUMUL is False)\n", + "CUMUL_STATES = ['OPERATIONAL', 'FAULT', 'WEATHER', 'DOWNTIME', 'UNKNOWN', 'IDLE']\n", + "# States to stack; DOWNTIME drawn as line; WEATHER drawn as independent filled area\n", + "LINE_ONLY_STATES = {} # {'DOWNTIME'}\n", + "FILL_ONLY_STATES = {'WEATHER'} # filled but not stacked\n", + "STACK_STATES = [s for s in CUMUL_STATES if s not in LINE_ONLY_STATES and s not in FILL_ONLY_STATES]\n", + "\n", + "CUMUL_LINESTYLE = {\n", + " 'OPERATIONAL': '-',\n", + " 'FAULT': '-',\n", + " 'WEATHER': '-',\n", + " 'DOWNTIME': '--',\n", + " 'UNKNOWN': '-',\n", + " 'IDLE': '-',\n", + "}\n", + "\n", + "CUMUL_COLORS = dict(COLORS)\n", + "\n", + "# Close previous figure if re-running\n", + "try:\n", + " plt.close(fig2)\n", + "except NameError:\n", + " pass\n", + "\n", + "fig2, ax_cumul = plt.subplots(figsize=(10, 7))\n", + "\n", + "shade_daytime(ax_cumul)\n", + "\n", + "# Store cumulative curve data per night for scatter point lookup\n", + "night_curves = []\n", + "\n", + "for sunset, sunrise in plot_twilight_spans:\n", + " sunset_pd = pd.Timestamp(sunset)\n", + " sunrise_pd = min(pd.Timestamp(sunrise), _data_now_naive)\n", + " night_hours = (sunrise_pd - sunset_pd).total_seconds() / 3600.0\n", + "\n", + " night_intervals = []\n", + " for start, end, state in intervals_naive:\n", + " if state not in CUMUL_STATES:\n", + " continue\n", + " s = max(start, sunset_pd)\n", + " e = min(end, sunrise_pd)\n", + " if s < e:\n", + " night_intervals.append((s, e, state))\n", + "\n", + " night_intervals = filter_fault_during_downtime(night_intervals)\n", + " night_intervals = filter_weather_during_downtime(night_intervals)\n", + "\n", + " if not night_intervals:\n", + " continue\n", + "\n", + " night_intervals.sort(key=lambda x: x[0])\n", + "\n", + " cumul = {st: 0.0 for st in CUMUL_STATES}\n", + " line_data = {st: [] for st in CUMUL_STATES}\n", + "\n", + " sunset_cx = float(cx(mdates.date2num(sunset_pd)))\n", + " for st in CUMUL_STATES:\n", + " line_data[st].append((sunset_cx, 0.0))\n", + "\n", + " for s, e, state in night_intervals:\n", + " dt_hours = (e - s).total_seconds() / 3600.0\n", + " t0 = float(cx(mdates.date2num(s)))\n", + " t1 = float(cx(mdates.date2num(e)))\n", + " old_val = cumul[state]\n", + " new_val = old_val + dt_hours\n", + " line_data[state].append((t0, old_val))\n", + " line_data[state].append((t1, new_val))\n", + " cumul[state] = new_val\n", + "\n", + " sunrise_cx = float(cx(mdates.date2num(sunrise_pd)))\n", + " for st in CUMUL_STATES:\n", + " if cumul[st] > 0:\n", + " line_data[st].append((sunrise_cx, cumul[st]))\n", + "\n", + " # Build per-state curves (xs, ys arrays)\n", + " curves = {}\n", + " for st in CUMUL_STATES:\n", + " if cumul[st] > 0:\n", + " pts = line_data[st]\n", + " xs = np.array([p[0] for p in pts])\n", + " ys = np.array([p[1] for p in pts])\n", + " curves[st] = (xs, ys)\n", + "\n", + " if STACKED_CUMUL:\n", + " # Merge all x-breakpoints across stackable states for this night\n", + " all_xs = set([sunset_cx, sunrise_cx])\n", + " for st in STACK_STATES:\n", + " if st in curves:\n", + " all_xs.update(curves[st][0].tolist())\n", + " all_xs = np.array(sorted(all_xs))\n", + "\n", + " # Interpolate each stackable state at all breakpoints\n", + " interp_vals = {}\n", + " for st in STACK_STATES:\n", + " if st in curves:\n", + " interp_vals[st] = np.interp(all_xs, curves[st][0], curves[st][1])\n", + " else:\n", + " interp_vals[st] = np.zeros_like(all_xs)\n", + "\n", + " # Draw stacked fill_between (bottom to top per STACK_STATES order)\n", + " bottom = np.zeros_like(all_xs)\n", + " stacked_tops = {}\n", + " for st in STACK_STATES:\n", + " top = bottom + interp_vals[st]\n", + " stacked_tops[st] = (all_xs, top.copy())\n", + " if np.any(interp_vals[st] > 0):\n", + " ax_cumul.fill_between(all_xs, bottom, top, color=CUMUL_COLORS[st],\n", + " alpha=0.7, step=None, zorder=2,\n", + " linewidth=0.5, edgecolor=CUMUL_COLORS[st])\n", + " bottom = top\n", + "\n", + " # Show stacked total line when there's significant unaccounted time\n", + " stacked_total = bottom[-1]\n", + " if night_hours - stacked_total > 0.1:\n", + " ax_cumul.plot([sunset_cx, sunrise_cx], [stacked_total, stacked_total],\n", + " color='gray', linewidth=1.0, linestyle='-', alpha=0.7, zorder=4)\n", + "\n", + " # Draw fill-only states as independent filled areas (not stacked)\n", + " for st in FILL_ONLY_STATES:\n", + " if st in curves:\n", + " xs, ys = curves[st]\n", + " ax_cumul.fill_between(xs, 0, ys, color=CUMUL_COLORS[st],\n", + " alpha=0.5, zorder=2,\n", + " linewidth=0.5, edgecolor=CUMUL_COLORS[st])\n", + "\n", + " # Draw line-only states as lines\n", + " for st in LINE_ONLY_STATES:\n", + " if st in curves:\n", + " xs, ys = curves[st]\n", + " ax_cumul.plot(xs, ys, color=CUMUL_COLORS[st], linewidth=1.5,\n", + " linestyle=CUMUL_LINESTYLE[st], alpha=0.9, zorder=3)\n", + "\n", + " night_curves.append((sunset_pd, sunrise_pd, sunset_cx, sunrise_cx,\n", + " curves, stacked_tops))\n", + " else:\n", + " for st in CUMUL_STATES:\n", + " if st in curves:\n", + " xs, ys = curves[st]\n", + " ax_cumul.plot(xs, ys, color=CUMUL_COLORS[st], linewidth=1.5,\n", + " linestyle=CUMUL_LINESTYLE[st], alpha=0.9, zorder=3)\n", + " if SHADE_CURVES:\n", + " if st in LINE_ONLY_STATES:\n", + " ax_cumul.fill_between(xs, 0, ys, facecolor='none',\n", + " edgecolor=CUMUL_COLORS[st],\n", + " hatch='///', alpha=0.3, zorder=1)\n", + " else:\n", + " ax_cumul.fill_between(xs, 0, ys, color=CUMUL_COLORS[st],\n", + " alpha=0.2, zorder=1)\n", + "\n", + " night_curves.append((sunset_pd, sunrise_pd, sunset_cx, sunrise_cx,\n", + " curves, None))\n", + "\n", + " # Night-length line\n", + " ax_cumul.plot([sunset_cx, sunrise_cx], [night_hours, night_hours],\n", + " color='black', linewidth=1.5, solid_capstyle='butt', zorder=4)\n", + "\n", + " # Top-down band for DOWNTIME total\n", + " xs_band = [sunset_cx, sunrise_cx]\n", + " if cumul.get('DOWNTIME', 0) > 0:\n", + " ax_cumul.fill_between(xs_band, night_hours - cumul['DOWNTIME'], night_hours,\n", + " facecolor='none', edgecolor=CUMUL_COLORS['DOWNTIME'],\n", + " hatch='///', alpha=0.4, zorder=3.5)\n", + "\n", + " ax_cumul.axvline(sunset_cx, color='orange', linewidth=0.3, alpha=0.5, zorder=1)\n", + " ax_cumul.axvline(sunrise_cx, color='orange', linewidth=0.3, alpha=0.5, zorder=1)\n", + "\n", + " # Cumulative shutter-open time for this night\n", + " shutter_night = []\n", + " for s_start, s_end in shutter_open_intervals:\n", + " s = max(s_start, sunset_pd)\n", + " e = min(s_end, sunrise_pd)\n", + " if s < e:\n", + " shutter_night.append((s, e))\n", + " if shutter_night:\n", + " shutter_night.sort()\n", + " sh_cumul = 0.0\n", + " sh_xs = [sunset_cx]\n", + " sh_ys = [0.0]\n", + " for s, e in shutter_night:\n", + " t0 = float(cx(mdates.date2num(s)))\n", + " t1 = float(cx(mdates.date2num(e)))\n", + " dt_h = (e - s).total_seconds() / 3600.0\n", + " sh_xs.extend([t0, t1])\n", + " sh_ys.extend([sh_cumul, sh_cumul + dt_h])\n", + " sh_cumul += dt_h\n", + " sh_xs.append(sunrise_cx)\n", + " sh_ys.append(sh_cumul)\n", + " ax_cumul.plot(sh_xs, sh_ys, color='#CC79A7', linewidth=1.5,\n", + " linestyle='-', alpha=0.9, zorder=3)\n", + "\n", + "# Overlay status scatter points ON the cumulative curves\n", + "note_x2, note_y2, note_c2, note_texts2 = [], [], [], []\n", + "norm_x2, norm_y2, norm_c2, norm_texts2 = [], [], [], []\n", + "scatter_x2, scatter_y2, scatter_texts2 = [], [], []\n", + "for ts_naive, ts_cx, active, text, has_note in iter_status_events():\n", + " for entry in night_curves:\n", + " sunset_pd, sunrise_pd, sunset_cx, sunrise_cx, curves, stacked_tops = entry\n", + " if sunset_pd <= ts_naive <= sunrise_pd:\n", + " for state in active:\n", + " if state in curves:\n", + " if STACKED_CUMUL and stacked_tops is not None and state in stacked_tops:\n", + " xs, ys = stacked_tops[state]\n", + " y_val = float(np.interp(ts_cx, xs, ys))\n", + " else:\n", + " xs, ys = curves[state]\n", + " y_val = float(np.interp(ts_cx, xs, ys))\n", + " c = CUMUL_COLORS.get(state, '#999')\n", + " scatter_x2.append(ts_cx)\n", + " scatter_y2.append(y_val)\n", + " scatter_texts2.append(text)\n", + " if has_note:\n", + " _ns2 = next((s for s in priority if s in active and s in curves),\n", + " next((s for s in active if s in curves), None))\n", + " if state == _ns2:\n", + " note_x2.append(ts_cx)\n", + " note_y2.append(y_val)\n", + " note_c2.append(c)\n", + " note_texts2.append(text)\n", + " else:\n", + " norm_x2.append(ts_cx)\n", + " norm_y2.append(y_val)\n", + " norm_c2.append(c)\n", + " norm_texts2.append(text)\n", + " else:\n", + " norm_x2.append(ts_cx)\n", + " norm_y2.append(y_val)\n", + " norm_c2.append(c)\n", + " norm_texts2.append(text)\n", + " break\n", + "\n", + "sc2_norm = ax_cumul.scatter(norm_x2, norm_y2, c=norm_c2, s=40,\n", + " marker='o', zorder=5, edgecolors='black', linewidths=0.2)\n", + "sc2_note = ax_cumul.scatter(note_x2, note_y2, c=note_c2, s=50,\n", + " marker='D', zorder=6, edgecolors='black', linewidths=0.4)\n", + "\n", + "max_night_hours = max((sunrise - sunset).total_seconds() / 3600\n", + " for sunset, sunrise in plot_twilight_spans)\n", + "ax_cumul.set_ylim(0, max_night_hours + 1.5)\n", + "ax_cumul.set_ylabel('Cumulative hours', fontsize=8)\n", + "_cumul_title = 'Cumulative Time in State per Night'\n", + "if STACKED_CUMUL:\n", + " _cumul_title += ' (stacked)'\n", + "ax_cumul.set_title(_cumul_title, fontsize=10, loc='left')\n", + "\n", + "setup_xaxis(ax_cumul, show_labels=True)\n", + "ax_cumul.set_xlabel('Date (UTC)')\n", + "plt.setp(ax_cumul.xaxis.get_majorticklabels(), rotation=45, ha='right')\n", + "fig2.subplots_adjust(bottom=0.15)\n", + "\n", + "# Mouseover annotation\n", + "_, update_cumul = make_hover_annot(ax_cumul)\n", + "_hover_state2 = [None]\n", + "\n", + "\n", + "def on_hover2(event):\n", + " changed = False\n", + " if event.inaxes == ax_cumul:\n", + " found = False\n", + " for sc, texts, xs, ys in [(sc2_note, note_texts2, note_x2, note_y2),\n", + " (sc2_norm, norm_texts2, norm_x2, norm_y2)]:\n", + " cont, ind = sc.contains(event)\n", + " if cont:\n", + " i = ind['ind'][0]\n", + " key = (id(sc), i)\n", + " if key != _hover_state2[0]:\n", + " update_cumul(True, (xs[i], ys[i]), texts[i])\n", + " _hover_state2[0] = key\n", + " changed = True\n", + " found = True\n", + " break\n", + " if not found and _hover_state2[0] is not None:\n", + " update_cumul(False)\n", + " _hover_state2[0] = None\n", + " changed = True\n", + " elif _hover_state2[0] is not None:\n", + " update_cumul(False)\n", + " _hover_state2[0] = None\n", + " changed = True\n", + " if changed:\n", + " fig2.canvas.draw_idle()\n", + "\n", + "\n", + "fig2.canvas.mpl_connect('motion_notify_event', on_hover2)\n", + "\n", + "from matplotlib.lines import Line2D\n", + "if STACKED_CUMUL:\n", + " cumul_legend = [\n", + " Patch(facecolor=CUMUL_COLORS[st], alpha=0.7, label=st) for st in STACK_STATES\n", + " ] + [\n", + " Patch(facecolor=CUMUL_COLORS[st], alpha=0.5, label=st) for st in FILL_ONLY_STATES\n", + " ] + [\n", + " Line2D([0], [0], color=CUMUL_COLORS[st], linewidth=2,\n", + " linestyle=CUMUL_LINESTYLE[st], label=st) for st in LINE_ONLY_STATES\n", + " ] + [Line2D([0], [0], color='black', linewidth=1.5, label='Night length')]\n", + "else:\n", + " cumul_legend = [\n", + " Line2D([0], [0], color=CUMUL_COLORS[st], linewidth=2,\n", + " linestyle=CUMUL_LINESTYLE[st], label=st) for st in CUMUL_STATES\n", + " ] + [Line2D([0], [0], color='black', linewidth=1.5, label='Night length')]\n", + "# Add top-band legend entry\n", + "cumul_legend += [\n", + " Patch(facecolor='none', edgecolor=CUMUL_COLORS['DOWNTIME'], hatch='///',\n", + " label='DOWNTIME (total)'),\n", + "]\n", + "cumul_legend += [\n", + " Line2D([0], [0], color='#CC79A7', linewidth=1.5, label='Dome open'),\n", + "]\n", + "ax_cumul.legend(handles=cumul_legend, loc='upper right', ncol=4, fontsize=7)\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11", + "metadata": {}, + "outputs": [], + "source": [ + "# Sync x-axis zoom/pan between fig (timeline) and fig2 (cumulative)\n", + "_xlim_syncing = False\n", + "\n", + "# Persistent \"UTC\" text at lower-right of each bottom axis\n", + "_utc_label_status = ax_status.text(1.0, -0.01, 'UTC', transform=ax_status.transAxes,\n", + " fontsize=8, ha='right', va='top', color='black')\n", + "_utc_label_cumul = ax_cumul.text(1.0, -0.01, 'UTC', transform=ax_cumul.transAxes,\n", + " fontsize=8, ha='right', va='top', color='black')\n", + "\n", + "def _update_titles(xlim):\n", + " \"\"\"Update both figure titles to reflect the visible date range.\"\"\"\n", + " d0 = mdates.num2date(uncx(xlim[0])).strftime('%Y-%m-%d')\n", + " d1 = mdates.num2date(uncx(xlim[1])).strftime('%Y-%m-%d')\n", + " ax_altaz.set_title(f'Observatory Status Timeline ({d0} to {d1} UTC)', fontsize=10)\n", + " _ct = 'Cumulative Time in State per Night'\n", + " if STACKED_CUMUL:\n", + " _ct += ' (stacked)'\n", + " ax_cumul.set_title(f'{_ct} ({d0} to {d1} UTC)', fontsize=10, loc='left')\n", + "\n", + "def _update_ticks(xlim):\n", + " \"\"\"Recompute x-axis ticks so at least 5 are visible in the current view.\"\"\"\n", + " r0 = uncx(xlim[0])\n", + " r1 = uncx(xlim[1])\n", + " dt0 = mdates.num2date(r0).replace(tzinfo=None)\n", + " dt1 = mdates.num2date(r1).replace(tzinfo=None)\n", + " span_sec = (dt1 - dt0).total_seconds()\n", + " if span_sec <= 0:\n", + " return\n", + "\n", + " # Coarsest to finest; pick the first (coarsest) that gives >= 5 ticks\n", + " freq_options = [\n", + " ('D', 14), ('D', 7), ('D', 5), ('D', 2), ('D', 1),\n", + " ('h', 12), ('h', 6), ('h', 3), ('h', 2), ('h', 1),\n", + " ('min', 30), ('min', 15), ('min', 10), ('min', 5), ('min', 2), ('min', 1),\n", + " ]\n", + " chosen_freq = ('D', 2) # fallback\n", + " fmt = '%m-%d'\n", + " for unit, val in freq_options:\n", + " if unit == 'min':\n", + " step_sec = val * 60\n", + " elif unit == 'h':\n", + " step_sec = val * 3600\n", + " else:\n", + " step_sec = val * 86400\n", + " n_ticks = span_sec / step_sec\n", + " if n_ticks >= 5:\n", + " chosen_freq = (unit, val)\n", + " if unit in ('min', 'h'):\n", + " fmt = '%m-%d %H:%M'\n", + " break\n", + "\n", + " unit, val = chosen_freq\n", + " if unit == 'min':\n", + " freq_str = f'{val}min'\n", + " elif unit == 'h':\n", + " freq_str = f'{val}h'\n", + " else:\n", + " freq_str = f'{val}D'\n", + "\n", + " tick_dts = pd.date_range(dt0.replace(minute=0, second=0, microsecond=0),\n", + " dt1 + pd.Timedelta(hours=1), freq=freq_str)\n", + " tick_dts = tick_dts[(tick_dts >= dt0) & (tick_dts <= dt1)]\n", + "\n", + " if unit == 'D':\n", + " tick_pos = [float(cx(mdates.date2num(d + pd.Timedelta(hours=12)))) for d in tick_dts]\n", + " else:\n", + " tick_pos = [float(cx(mdates.date2num(d))) for d in tick_dts]\n", + " tick_lab = [d.strftime(fmt) for d in tick_dts]\n", + "\n", + " for ax in [ax_status, ax_cumul]:\n", + " ax.set_xticks(tick_pos)\n", + " ax.set_xticklabels(tick_lab, rotation=45, ha='right')\n", + " ax.set_xticks([], minor=True)\n", + "\n", + " # Keep labels only on bottom panel; hide on upper panels without affecting shared state\n", + " ax_altaz.tick_params(axis='x', labelbottom=False)\n", + " ax_band.tick_params(axis='x', labelbottom=False)\n", + " # Disable x-grid on upper panels so tick positions don't draw vertical lines\n", + " ax_altaz.grid(False, axis='x')\n", + " ax_band.grid(False, axis='x')\n", + "\n", + "def _sync_from_timeline(ax):\n", + " global _xlim_syncing\n", + " if _xlim_syncing:\n", + " return\n", + " _xlim_syncing = True\n", + " xlim = ax.get_xlim()\n", + " ax_cumul.set_xlim(xlim)\n", + " _update_titles(xlim)\n", + " _update_ticks(xlim)\n", + " fig2.canvas.draw_idle()\n", + " fig.canvas.draw_idle()\n", + " _xlim_syncing = False\n", + "\n", + "def _sync_from_cumul(ax):\n", + " global _xlim_syncing\n", + " if _xlim_syncing:\n", + " return\n", + " _xlim_syncing = True\n", + " xlim = ax.get_xlim()\n", + " ax_status.set_xlim(xlim) # sharex propagates to altaz and band\n", + " _update_titles(xlim)\n", + " _update_ticks(xlim)\n", + " fig.canvas.draw_idle()\n", + " fig2.canvas.draw_idle()\n", + " _xlim_syncing = False\n", + "\n", + "ax_status.callbacks.connect('xlim_changed', _sync_from_timeline)\n", + "ax_cumul.callbacks.connect('xlim_changed', _sync_from_cumul)\n", + "\n", + "# Set initial ticks for the current view\n", + "_update_ticks((cxlim0, cxlim1))\n", + "fig.canvas.draw_idle()\n", + "fig2.canvas.draw_idle()" + ] + }, + { + "cell_type": "markdown", + "id": "12", + "metadata": {}, + "source": [ + "# Cumulative state totals" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "13", + "metadata": {}, + "outputs": [], + "source": [ + "# Cumulative total hours in each state over the date range\n", + "# Independent of the timeline/per-night plots above\n", + "\n", + "TOTAL_STATES = ['OPERATIONAL', 'FAULT', 'WEATHER', 'DOWNTIME', 'UNKNOWN', 'IDLE']\n", + "TOTAL_LINESTYLE = {\n", + " 'OPERATIONAL': '-',\n", + " 'FAULT': '-',\n", + " 'WEATHER': '-',\n", + " 'DOWNTIME': '--',\n", + " 'UNKNOWN': '-',\n", + " 'IDLE': '-',\n", + "}\n", + "\n", + "# Close previous figure if re-running\n", + "try:\n", + " plt.close(fig3)\n", + "except NameError:\n", + " pass\n", + "\n", + "if SHOW_NIGHT_LENGTH:\n", + " fig3, (ax_total, ax_nightlen) = plt.subplots(\n", + " 2, 1, figsize=(10, 6),\n", + " gridspec_kw={'height_ratios': [4, 1]}, sharex=True)\n", + "else:\n", + " fig3, ax_total = plt.subplots(figsize=(10, 5))\n", + " ax_nightlen = None\n", + "\n", + "# Build continuous cumulative curves with intra-night detail.\n", + "# Each curve is a list of (datetime, cumul_hours) points.\n", + "_total_running = {st: 0.0 for st in TOTAL_STATES}\n", + "_total_curves = {st: [] for st in TOTAL_STATES}\n", + "_shutter_running = 0.0\n", + "_shutter_curve = []\n", + "_camera_running = 0.0\n", + "_camera_curve = []\n", + "_overhead_running = 0.0\n", + "_overhead_curve = []\n", + "\n", + "# Build overhead (readout + slew) intervals from consecutive exposures\n", + "_sorted_cam_ivs = sorted(camera_shutter_intervals)\n", + "_overhead_ivs = []\n", + "for _oi, (_ot0, _ot1) in enumerate(_sorted_cam_ivs):\n", + " if _oi + 1 < len(_sorted_cam_ivs):\n", + " _ot0_next = _sorted_cam_ivs[_oi + 1][0]\n", + " _ogap_s = (_ot0_next - _ot1).total_seconds()\n", + " if 0 < _ogap_s <= MAX_SLEW_GAP_S:\n", + " _overhead_ivs.append((_ot1, _ot0_next))\n", + " elif _ogap_s > 0:\n", + " _overhead_ivs.append((_ot1, _ot1 + pd.Timedelta(seconds=READOUT_S)))\n", + " else:\n", + " _overhead_ivs.append((_ot1, _ot1 + pd.Timedelta(seconds=READOUT_S)))\n", + "\n", + "for sunset, sunrise in plot_twilight_spans:\n", + " sunset_pd = pd.Timestamp(sunset)\n", + " sunrise_pd = min(pd.Timestamp(sunrise), _data_now_naive)\n", + "\n", + " # Start of night: record current totals (flat from previous sunrise)\n", + " for st in TOTAL_STATES:\n", + " _total_curves[st].append((sunset_pd, _total_running[st]))\n", + " _shutter_curve.append((sunset_pd, _shutter_running))\n", + " _camera_curve.append((sunset_pd, _camera_running))\n", + "\n", + " # Collect and sort all state intervals clipped to this night\n", + " night_events = []\n", + " for start, end, state in intervals_naive:\n", + " if state not in TOTAL_STATES:\n", + " continue\n", + " s = max(start, sunset_pd)\n", + " e = min(end, sunrise_pd)\n", + " if s < e:\n", + " night_events.append((s, e, state))\n", + " night_events = filter_fault_during_downtime(night_events)\n", + " night_events = filter_weather_during_downtime(night_events)\n", + " night_events.sort(key=lambda x: x[0])\n", + "\n", + " # Step through state intervals, adding points at each transition\n", + " for s, e, state in night_events:\n", + " dt_hours = (e - s).total_seconds() / 3600.0\n", + " _total_curves[state].append((s, _total_running[state]))\n", + " _total_running[state] += dt_hours\n", + " _total_curves[state].append((e, _total_running[state]))\n", + "\n", + " # Shutter intervals clipped to this night\n", + " sh_events = []\n", + " for s_start, s_end in shutter_open_intervals:\n", + " s = max(s_start, sunset_pd)\n", + " e = min(s_end, sunrise_pd)\n", + " if s < e:\n", + " sh_events.append((s, e))\n", + " sh_events.sort()\n", + "\n", + " for s, e in sh_events:\n", + " dt_hours = (e - s).total_seconds() / 3600.0\n", + " _shutter_curve.append((s, _shutter_running))\n", + " _shutter_running += dt_hours\n", + " _shutter_curve.append((e, _shutter_running))\n", + "\n", + " # Camera shutter intervals clipped to this night\n", + " cam_events = []\n", + " for c_start, c_end in camera_shutter_intervals:\n", + " c_s = max(c_start, sunset_pd)\n", + " c_e = min(c_end, sunrise_pd)\n", + " if c_s < c_e:\n", + " cam_events.append((c_s, c_e))\n", + " cam_events.sort()\n", + " for c_s, c_e in cam_events:\n", + " dt_hours = (c_e - c_s).total_seconds() / 3600.0\n", + " _camera_curve.append((c_s, _camera_running))\n", + " _camera_running += dt_hours\n", + " _camera_curve.append((c_e, _camera_running))\n", + "\n", + " # Overhead (readout + slew) intervals clipped to this night\n", + " _ovh_events = []\n", + " for _os, _oe in _overhead_ivs:\n", + " _s = max(_os, sunset_pd)\n", + " _e = min(_oe, sunrise_pd)\n", + " if _s < _e:\n", + " _ovh_events.append((_s, _e))\n", + " _ovh_events.sort()\n", + " _overhead_curve.append((sunset_pd, _overhead_running))\n", + " for _s, _e in _ovh_events:\n", + " _overhead_curve.append((_s, _overhead_running))\n", + " _overhead_running += (_e - _s).total_seconds() / 3600.0\n", + " _overhead_curve.append((_e, _overhead_running))\n", + "\n", + " # End of night: record totals at sunrise (clipped to now for in-progress night)\n", + " for st in TOTAL_STATES:\n", + " _total_curves[st].append((sunrise_pd, _total_running[st]))\n", + " _shutter_curve.append((sunrise_pd, _shutter_running))\n", + " _camera_curve.append((sunrise_pd, _camera_running))\n", + " _overhead_curve.append((sunrise_pd, _overhead_running))\n", + "\n", + "# Plot each state\n", + "for state in TOTAL_STATES:\n", + " if not _total_curves[state]:\n", + " continue\n", + " xs = [float(cx(mdates.date2num(t))) for t, _ in _total_curves[state]]\n", + " ys = [v for _, v in _total_curves[state]]\n", + " ax_total.plot(xs, ys, color=COLORS[state], linewidth=2,\n", + " linestyle=TOTAL_LINESTYLE[state],\n", + " label=state, alpha=0.9)\n", + "\n", + "# Shutter open\n", + "if _shutter_curve:\n", + " xs = [float(cx(mdates.date2num(t))) for t, _ in _shutter_curve]\n", + " ys = [v for _, v in _shutter_curve]\n", + " ax_total.plot(xs, ys, color='#CC79A7', linewidth=2,\n", + " linestyle='-', label='Dome open', alpha=0.9)\n", + "\n", + "# Camera shutter open + overhead fill\n", + "if _camera_curve and _overhead_curve:\n", + " _xs_cam = [float(cx(mdates.date2num(t))) for t, _ in _camera_curve]\n", + " _ys_cam = [v for _, v in _camera_curve]\n", + " _xs_ovh = [float(cx(mdates.date2num(t))) for t, _ in _overhead_curve]\n", + " _ys_ovh = [v for _, v in _overhead_curve]\n", + " _xs_fill = sorted(set(_xs_cam) | set(_xs_ovh))\n", + " _ys_fill_bot = np.interp(_xs_fill, _xs_cam, _ys_cam)\n", + " _ys_fill_top = _ys_fill_bot + np.interp(_xs_fill, _xs_ovh, _ys_ovh)\n", + " ax_total.fill_between(_xs_fill, _ys_fill_bot, _ys_fill_top,\n", + " alpha=0.25, color='#0072B2')\n", + " ax_total.plot(_xs_cam, _ys_cam, color='#0072B2', linewidth=2,\n", + " linestyle='-', label='Camera exposing', alpha=0.9)\n", + " ax_total.plot(_xs_fill, _ys_fill_top, color='#0072B2', linewidth=1.5,\n", + " linestyle='--', alpha=0.7, label='+ readout + slew')\n", + "elif _camera_curve:\n", + " _xs_cam = [float(cx(mdates.date2num(t))) for t, _ in _camera_curve]\n", + " _ys_cam = [v for _, v in _camera_curve]\n", + " ax_total.plot(_xs_cam, _ys_cam, color='#0072B2', linewidth=2,\n", + " linestyle='-', label='Camera exposing', alpha=0.9)\n", + "\n", + "ax_total.set_ylabel('Cumulative hours', fontsize=9)\n", + "\n", + "# Right axis: fraction of cumulative available twilight-to-twilight time\n", + "_cumul_night_hours = []\n", + "_night_total = 0.0\n", + "for sunset, sunrise in plot_twilight_spans:\n", + " _night_total += (pd.Timestamp(sunrise) - pd.Timestamp(sunset)).total_seconds() / 3600.0\n", + " _cumul_night_hours.append(_night_total)\n", + "\n", + "if _night_total > 0:\n", + " ax_frac = ax_total.twinx()\n", + " ax_frac.set_ylabel('Fraction of available time', fontsize=9)\n", + " # Sync the right axis limits to match the left axis scaled by total night hours\n", + " def _sync_frac_ylim(ax):\n", + " lo, hi = ax_total.get_ylim()\n", + " ax_frac.set_ylim(lo / _night_total, hi / _night_total)\n", + " _sync_frac_ylim(ax_total)\n", + " ax_total.callbacks.connect('ylim_changed', _sync_frac_ylim)\n", + "\n", + " # Also plot cumulative night hours as a reference line\n", + " _ref_xs = []\n", + " _ref_ys = []\n", + " _nh_running = 0.0\n", + " for sunset, sunrise in plot_twilight_spans:\n", + " sunset_pd = pd.Timestamp(sunset)\n", + " sunrise_pd = pd.Timestamp(sunrise)\n", + " _nh = (sunrise_pd - sunset_pd).total_seconds() / 3600.0\n", + " _ref_xs.append(float(cx(mdates.date2num(sunset_pd))))\n", + " _ref_ys.append(_nh_running)\n", + " _ref_xs.append(float(cx(mdates.date2num(sunrise_pd))))\n", + " _nh_running += _nh\n", + " _ref_ys.append(_nh_running)\n", + " ax_total.plot(_ref_xs, _ref_ys, color='black', linewidth=1.5,\n", + " linestyle='-', alpha=0.7, label='Available')\n", + "\n", + "if plot_twilight_spans:\n", + " _d0 = pd.Timestamp(plot_twilight_spans[0][0]).strftime('%Y-%m-%d')\n", + " _d1 = pd.Timestamp(plot_twilight_spans[-1][0]).strftime('%Y-%m-%d')\n", + " ax_total.set_title(f'Cumulative State Totals ({_d0} to {_d1} UTC)', fontsize=10)\n", + "\n", + "if SHOW_NIGHT_LENGTH:\n", + " _nl_xs, _nl_ys = [], []\n", + " for sunset, sunrise in plot_twilight_spans:\n", + " _ss_pd = pd.Timestamp(sunset)\n", + " _sr_pd = pd.Timestamp(sunrise)\n", + " _nh = (_sr_pd - _ss_pd).total_seconds() / 3600.0\n", + " _x0 = float(cx(mdates.date2num(_ss_pd)))\n", + " _x1 = float(cx(mdates.date2num(_sr_pd)))\n", + " _nl_xs.append((_x0 + _x1) / 2)\n", + " _nl_ys.append(_nh)\n", + " ax_nightlen.plot(_nl_xs, _nl_ys, color='black', linewidth=1.5, alpha=0.7)\n", + " ax_nightlen.set_ylabel('Night\\nlength (h)', fontsize=8)\n", + " ax_nightlen.locator_params(axis='y', nbins=3)\n", + " ax_nightlen.grid(axis='y', alpha=0.3)\n", + "\n", + "shade_daytime(ax_total)\n", + "if SHOW_NIGHT_LENGTH:\n", + " shade_daytime(ax_nightlen)\n", + " setup_xaxis(ax_total, show_labels=False)\n", + " setup_xaxis(ax_nightlen, show_labels=True)\n", + " plt.setp(ax_nightlen.xaxis.get_majorticklabels(), rotation=45, ha='right')\n", + " ax_nightlen.set_xlabel('Date (UTC)', fontsize=9)\n", + "else:\n", + " setup_xaxis(ax_total, show_labels=True)\n", + " plt.setp(ax_total.xaxis.get_majorticklabels(), rotation=45, ha='right')\n", + " ax_total.set_xlabel('Date (UTC)', fontsize=9)\n", + "ax_total.grid(axis='y', alpha=0.3)\n", + "\n", + "from matplotlib.lines import Line2D\n", + "_pct = (lambda v: f' ({100*v/_night_total:.1f}%)') if _night_total > 0 else (lambda v: '')\n", + "total_legend = [\n", + " Line2D([0], [0], color=COLORS[st], linewidth=2,\n", + " linestyle=TOTAL_LINESTYLE[st], label=st + _pct(_total_running[st])) for st in TOTAL_STATES\n", + "] + [\n", + " Line2D([0], [0], color='#CC79A7', linewidth=2, label='Dome open' + _pct(_shutter_running)),\n", + " Line2D([0], [0], color='#0072B2', linewidth=2, label='Camera exposing' + _pct(_camera_running)),\n", + " Line2D([0], [0], color='#0072B2', linewidth=1.5, linestyle='--',\n", + " label='+ readout + slew' + _pct(_camera_running + _overhead_running)),\n", + " Line2D([0], [0], color='black', linewidth=1.5, label='Available'),\n", + "]\n", + "ax_total.legend(handles=total_legend, loc='upper left', fontsize=8)\n", + "\n", + "fig3.subplots_adjust(bottom=0.15, hspace=0.05 if SHOW_NIGHT_LENGTH else 0)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "14", + "metadata": {}, + "source": [ + "# Observatory Status Statistics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15", + "metadata": {}, + "outputs": [], + "source": [ + "# Per-night hours in each observatory state\n", + "_stats_states = ['OPERATIONAL', 'FAULT', 'WEATHER', 'DOWNTIME', 'UNKNOWN', 'IDLE']\n", + "_stats_cols = ['Night length'] + _stats_states + ['Dome open', 'Camera exposing']\n", + "_stats_nights = [] # list of dicts, one per night\n", + "_stats_dates = [] # night labels (sunset date)\n", + "\n", + "for _sunset, _sunrise in plot_twilight_spans:\n", + " _sunset_pd = pd.Timestamp(_sunset)\n", + " _sunrise_pd = min(pd.Timestamp(_sunrise), _data_now_naive)\n", + " _night_h = (_sunrise_pd - _sunset_pd).total_seconds() / 3600.0\n", + "\n", + " _nivs = []\n", + " for _s, _e, _st in intervals_naive:\n", + " if _st not in _stats_states:\n", + " continue\n", + " _cs = max(_s, _sunset_pd)\n", + " _ce = min(_e, _sunrise_pd)\n", + " if _cs < _ce:\n", + " _nivs.append((_cs, _ce, _st))\n", + "\n", + " _nivs = filter_fault_during_downtime(_nivs)\n", + " _nivs = filter_weather_during_downtime(_nivs)\n", + "\n", + " _cumul = {st: 0.0 for st in _stats_states}\n", + " for _cs, _ce, _st in _nivs:\n", + " _cumul[_st] += (_ce - _cs).total_seconds() / 3600.0\n", + " _ofdu_nivs = [(cs, ce, st) for cs, ce, st in _nivs\n", + " if st in ('OPERATIONAL', 'FAULT', 'DOWNTIME', 'UNKNOWN', 'IDLE')]\n", + " if _ofdu_nivs:\n", + " _last_ofdu_end = max(_ce for _, _ce, _ in _ofdu_nivs)\n", + " _end_gap_s = (_sunrise_pd - _last_ofdu_end).total_seconds()\n", + " if 0 < _end_gap_s < 120:\n", + " _last_st = max(_ofdu_nivs, key=lambda x: x[1])[2]\n", + " _cumul[_last_st] += _end_gap_s / 3600.0\n", + "\n", + " _sh = 0.0\n", + " for _ss, _se in shutter_open_intervals:\n", + " _cs = max(_ss, _sunset_pd)\n", + " _ce = min(_se, _sunrise_pd)\n", + " if _cs < _ce:\n", + " _sh += (_ce - _cs).total_seconds() / 3600.0\n", + "\n", + " _row = {'Night length': _night_h}\n", + " _row.update(_cumul)\n", + " _row['Dome open'] = _sh\n", + "\n", + " _cam = 0.0\n", + " for _cs2, _ce2 in camera_shutter_intervals:\n", + " _cs = max(_cs2, _sunset_pd)\n", + " _ce = min(_ce2, _sunrise_pd)\n", + " if _cs < _ce:\n", + " _cam += (_ce - _cs).total_seconds() / 3600.0\n", + " _row['Camera exposing'] = _cam\n", + " _stats_nights.append(_row)\n", + " _ofdu = sum(_cumul[st] for st in ('OPERATIONAL', 'FAULT', 'DOWNTIME', 'UNKNOWN', 'IDLE'))\n", + " _date_lbl = _sunset_pd.strftime('%m-%d')\n", + " if _night_h - _ofdu > 1 / 3600:\n", + " _date_lbl += '*'\n", + " _stats_dates.append(_date_lbl)\n", + "\n", + "\n", + "# Build DataFrame: nights as rows, states as columns\n", + "_df = pd.DataFrame(_stats_nights, index=_stats_dates, columns=_stats_cols)\n", + "_df.index.name = 'Night'\n", + "if SHOW_OFDU:\n", + " _df.insert(1, 'O+F+D+U', _df[['OPERATIONAL','FAULT','DOWNTIME','UNKNOWN','IDLE']].sum(axis=1))\n", + "\n", + "# Append summary rows\n", + "_summary = pd.DataFrame({\n", + " c: [_df[c].sum(), _df[c].mean(), _df[c].median(), median_abs_deviation(_df[c].values, scale=\"normal\")]\n", + " for c in _df.columns\n", + "}, index=['Total', 'Mean', 'Median', 'σ_MAD'])\n", + "_df_full = pd.concat([_df, _summary])\n", + "\n", + "_n = len(plot_twilight_spans)\n", + "_title = f\"Hours per night ({_n} nights, {title_start} to {title_end} UTC)\\nFAULT excludes time during DOWNTIME; WEATHER excludes time during DOWNTIME\"\n", + "_table_str = _df_full.round(2).to_string()\n", + "_lines = _table_str.split('\\n')\n", + "_width = max(len(l) for l in _lines)\n", + "_sep = '-' * _width\n", + "_lines.insert(1 + len(_df), _sep)\n", + "print(_title)\n", + "print(_sep)\n", + "print('\\n'.join(_lines))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "16", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "LSST", + "language": "python", + "name": "lsst" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.14" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/nightly/plot_timeline.yaml b/nightly/plot_timeline.yaml new file mode 100644 index 0000000..a9dadd2 --- /dev/null +++ b/nightly/plot_timeline.yaml @@ -0,0 +1,25 @@ +title: Observatory Status Timeline +description: > + Rubin Observatory nightly status and visit data visualization. + Shows telescope status states (OPERATIONAL, FAULT, WEATHER, DOWNTIME, UNKNOWN, IDLE), + filter band usage, dome shutter intervals, and cumulative statistics per night. +authors: + - name: Robert Lupton + slack: rhl +parameters: + ndays: + type: integer + description: "Number of observing nights to display, counted backwards from the most recent data" + default: 7 + date_start: + type: string + description: "Start date (YYYY-MM-DD). Leave empty to use ndays." + default: "" + date_end: + type: string + description: "End date exclusive (YYYY-MM-DD). Leave empty for most recent data." + default: "" +schedule_enabled: true +schedule: + - freq: "daily" + hour: 14