diff --git a/ROADMAP.md b/ROADMAP.md index 165b1974..0f41638e 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -10,6 +10,8 @@ This document outlines the development roadmap for ServerKit. Features are organ ### Recently Completed (v1.7.x) +- **Uptime Monitors** - Monitoring is no longer only about the machine the panel runs on. A monitor watches anything you point it at — a website, an API endpoint, a database port, a mail server or a managed WordPress site — with HTTP, keyword, port, ping, DNS and SMTP checks, per-monitor intervals, expected status ranges, redirect and TLS-verification switches, and a retry threshold before anything is declared down. A scheduler polls them continuously (previously checks only ran when someone pressed a button), outages open and close an incident on their own, and each monitor gets a detail page with a response-time chart, a 90-day uptime strip, a live check log and TLS certificate expiry. Host CPU/memory/disk alerting is unchanged and now shares one Incidents timeline with monitor outages, so "something is wrong" is a single place. Status pages publish monitors you already have rather than defining their own duplicate probes. +- **Monitoring Navigation** - Monitors, Incidents, Events and Jobs are tabs of one Monitoring group instead of scattered top-level pages, and the events stream was rebuilt on the same search-and-filter layout as the rest of the panel. - **Themes & Theme Studio** - A gallery of color themes in Settings → Appearance (Paper, Nord Deep, Gruvbox, Phosphor, High Contrast, and the stock look), applied instantly and previewed live, kept as each user's personal choice on top of the dark/light toggle. Themes are data, not code — a small map of color tokens — so a built-in Theme Studio lets anyone edit colors over the live panel and export a shareable file, admins can set a panel-wide default or import a theme, and a public community registry installs themes with one click. - **Deploy Console & Live Run Logs** - Every install and deploy lands on one full-page live console: streaming build output, a step timeline with per-step durations and a live elapsed timer, and failure cards showing the real error tail with plain-language hints and one-click retry. Backed by a unified batched run-log layer (database persistence + WebSocket push with a polling fallback). - **New Service Wizard & Template Catalog** - Three-step service creation (Source / Connect / Review) backed by a server-side template catalog with one-click compose templates and Git-repo templates. diff --git a/VERSION b/VERSION index 8d47170e..4f4b8896 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.7.74 +1.7.75 diff --git a/backend/app/__init__.py b/backend/app/__init__.py index 880e2bea..c8eea017 100644 --- a/backend/app/__init__.py +++ b/backend/app/__init__.py @@ -302,6 +302,11 @@ def create_app(config_name=None): from app.api.uptime import uptime_bp app.register_blueprint(uptime_bp, url_prefix='/api/v1/uptime') + # Register blueprints - Monitors (synthetic checks + their incidents). + # Core: watching a site must not depend on the status-page extension. + from app.api.monitors import monitors_bp + app.register_blueprint(monitors_bp, url_prefix='/api/v1/monitors') + # Register blueprints - Environment Variables from app.api.env_vars import env_vars_bp app.register_blueprint(env_vars_bp, url_prefix='/api/v1/apps') diff --git a/backend/app/api/monitors.py b/backend/app/api/monitors.py new file mode 100644 index 00000000..80a67dfc --- /dev/null +++ b/backend/app/api/monitors.py @@ -0,0 +1,179 @@ +""" +Monitors API + +Uptime/synthetic checks against URLs, hosts and managed sites. Core, not part of +the serverkit-status extension — watching a site must not require publishing a +status page. See app/services/monitor_service.py. +""" + +from flask import Blueprint, request, jsonify +from flask_jwt_extended import jwt_required + +from app.services.monitor_service import MonitorService + +monitors_bp = Blueprint('monitors', __name__) + + +@monitors_bp.route('', methods=['GET']) +@monitors_bp.route('/', methods=['GET']) +@jwt_required() +def list_monitors(): + """List monitors. + + Query params: status, type, q, page_id. + """ + page_id = request.args.get('page_id', type=int) + monitors = MonitorService.list_monitors( + status=request.args.get('status') or None, + check_type=request.args.get('type') or None, + q=request.args.get('q') or None, + page_id=page_id, + ) + # One extra query for every row's sparkline, not one per row. + sparks = MonitorService.recent_response_times([m.id for m in monitors]) + payload = [] + for monitor in monitors: + item = monitor.to_dict() + item['spark'] = sparks.get(monitor.id, []) + payload.append(item) + return jsonify({'monitors': payload}) + + +@monitors_bp.route('/stats', methods=['GET']) +@jwt_required() +def get_stats(): + """KPI-band counts for the Monitors tab.""" + return jsonify({'stats': MonitorService.stats()}) + + +@monitors_bp.route('', methods=['POST']) +@monitors_bp.route('/', methods=['POST']) +@jwt_required() +def create_monitor(): + data = request.get_json() or {} + try: + monitor = MonitorService.create(data) + except ValueError as e: + return {'error': str(e)}, 400 + return jsonify(monitor.to_dict()), 201 + + +@monitors_bp.route('/', methods=['GET']) +@jwt_required() +def get_monitor(monitor_id): + monitor = MonitorService.get(monitor_id) + if not monitor: + return {'error': 'Monitor not found'}, 404 + return jsonify(monitor.to_dict()) + + +@monitors_bp.route('/', methods=['PATCH', 'PUT']) +@jwt_required() +def update_monitor(monitor_id): + data = request.get_json() or {} + try: + monitor = MonitorService.update(monitor_id, data) + except ValueError as e: + return {'error': str(e)}, 400 + if not monitor: + return {'error': 'Monitor not found'}, 404 + return jsonify(monitor.to_dict()) + + +@monitors_bp.route('/', methods=['DELETE']) +@jwt_required() +def delete_monitor(monitor_id): + if not MonitorService.delete(monitor_id): + return {'error': 'Monitor not found'}, 404 + return jsonify({'deleted': True}) + + +@monitors_bp.route('//check', methods=['POST']) +@jwt_required() +def run_check(monitor_id): + """Probe now, out of band from the scheduler.""" + hc = MonitorService.run_check(monitor_id) + if not hc: + return {'error': 'Monitor not found'}, 404 + monitor = MonitorService.get(monitor_id) + return jsonify({'check': hc.to_dict(), 'monitor': monitor.to_dict()}) + + +@monitors_bp.route('//pause', methods=['POST']) +@jwt_required() +def set_paused(monitor_id): + data = request.get_json() or {} + monitor = MonitorService.set_paused(monitor_id, data.get('paused', True)) + if not monitor: + return {'error': 'Monitor not found'}, 404 + return jsonify(monitor.to_dict()) + + +@monitors_bp.route('//history', methods=['GET']) +@jwt_required() +def get_history(monitor_id): + """Recent check results. Query params: hours (default 24), limit.""" + if not MonitorService.get(monitor_id): + return {'error': 'Monitor not found'}, 404 + hours = request.args.get('hours', 24, type=int) + limit = request.args.get('limit', 200, type=int) + checks = MonitorService.get_check_history(monitor_id, hours=hours, limit=limit) + return jsonify({'checks': [c.to_dict() for c in checks]}) + + +@monitors_bp.route('//uptime', methods=['GET']) +@jwt_required() +def get_uptime(monitor_id): + """Per-day uptime buckets for the 90-day bar strip. Query param: days.""" + monitor = MonitorService.get(monitor_id) + if not monitor: + return {'error': 'Monitor not found'}, 404 + days = min(request.args.get('days', 90, type=int), 365) + return jsonify({ + 'days': MonitorService.uptime_days(monitor_id, days=days), + 'uptime_24h': monitor.uptime_24h, + 'uptime_7d': monitor.uptime_7d, + 'uptime_30d': monitor.uptime_30d, + 'uptime_90d': monitor.uptime_90d, + }) + + +# --- Incidents ------------------------------------------------------------- +# Incidents are core because the scheduler opens them automatically, with or +# without the status-page extension installed. + +@monitors_bp.route('/incidents', methods=['GET']) +@jwt_required() +def list_incidents(): + """Query params: state (active|resolved|all), limit.""" + state = request.args.get('state', 'all') + limit = request.args.get('limit', 100, type=int) + incidents = MonitorService.list_incidents(state=state, limit=limit) + return jsonify({'incidents': [i.to_dict() for i in incidents]}) + + +@monitors_bp.route('/incidents', methods=['POST']) +@jwt_required() +def create_incident(): + data = request.get_json() or {} + if not data.get('title'): + return {'error': 'Incident title is required'}, 400 + incident = MonitorService.create_incident(data.get('page_id'), data) + return jsonify(incident.to_dict()), 201 + + +@monitors_bp.route('/incidents/', methods=['PATCH', 'PUT']) +@jwt_required() +def update_incident(incident_id): + incident = MonitorService.update_incident(incident_id, request.get_json() or {}) + if not incident: + return {'error': 'Incident not found'}, 404 + return jsonify(incident.to_dict()) + + +@monitors_bp.route('/incidents/', methods=['DELETE']) +@jwt_required() +def delete_incident(incident_id): + if not MonitorService.delete_incident(incident_id): + return {'error': 'Incident not found'}, 404 + return jsonify({'deleted': True}) diff --git a/backend/app/jobs/builtin_handlers.py b/backend/app/jobs/builtin_handlers.py index caa7119f..5ea9ebaf 100644 --- a/backend/app/jobs/builtin_handlers.py +++ b/backend/app/jobs/builtin_handlers.py @@ -84,22 +84,19 @@ def check_workflow_schedules(): def run_health_checks(): """Run a health check for every managed (production) WordPress site and sync - any status-page components bound to it. Per-site try/except so one hung site - never stalls the whole sweep.""" + any monitors bound to it. Per-site try/except so one hung site never stalls + the whole sweep.""" from app import db from app.models.wordpress_site import WordPressSite from app.models.status_page import StatusComponent from app.services.environment_health_service import EnvironmentHealthService - from app.services.plugin_service import get_installed_extension_attr + from app.services.monitor_service import MonitorService _prune_old_health_checks() - # Status-page component sync lives in the serverkit-status extension (plan 47); - # reach it only when installed. StatusComponent rows can't exist without it, - # so a lean panel simply runs the health checks without the sync. - StatusPageService = get_installed_extension_attr( - 'serverkit-status', 'status_page_service', 'StatusPageService') - + # The check engine is core now (it used to live in the serverkit-status + # extension and was reached through get_installed_extension_attr), so a lean + # panel with no status pages still keeps its site-bound monitors up to date. sites = WordPressSite.query.filter_by(is_production=True).all() for site in sites: try: @@ -111,10 +108,11 @@ def run_health_checks(): overall = result.get('overall_status') if not overall: continue - if StatusPageService is not None: - components = StatusComponent.query.filter_by(wordpress_site_id=site.id).all() - for comp in components: - StatusPageService.sync_component_from_health(comp, overall) + monitors = StatusComponent.query.filter_by(wordpress_site_id=site.id).all() + for monitor in monitors: + if monitor.is_paused: + continue + MonitorService.sync_component_from_health(monitor, overall) except Exception as e: logger.error(f'Health check failed for site {site.id}: {e}') try: @@ -123,6 +121,34 @@ def run_health_checks(): pass +def run_monitor_checks(): + """Poll every monitor whose interval has elapsed. + + This is what actually makes monitoring happen: before it existed, checks only + ran when someone pressed "Check now", so a configured monitor never noticed an + outage on its own. Per-monitor try/except mirrors run_health_checks — one + unreachable target must not stall the sweep. + """ + from app import db + from app.services.monitor_service import MonitorService + + due = MonitorService.due_monitors() + if not due: + return + checked = 0 + for monitor in due: + try: + MonitorService.run_check(monitor.id) + checked += 1 + except Exception as e: + logger.error(f'Monitor check failed for monitor {monitor.id}: {e}') + try: + db.session.rollback() + except Exception: + pass + logger.debug('Monitor sweep polled %s of %s due monitors', checked, len(due)) + + def _prune_old_health_checks(): """Delete health-check samples older than the retention window, at most once per day, so the continuous poller doesn't grow the health_checks table without @@ -315,6 +341,9 @@ def run_extension_update_check(): ('builtin.snapshot_retention', run_snapshot_retention, 'snapshot-retention', 3600, 120), ('builtin.workflow_schedules', check_workflow_schedules, 'workflow-schedules', 60, 60), ('builtin.health_check', run_health_checks, 'health-check', 300, 30), + # 30s tick, not the monitor interval: the sweep only polls monitors whose own + # interval has elapsed, so this bounds how late a 30s check can fire. + ('builtin.monitor_check', run_monitor_checks, 'monitor-check', 30, 20), ('builtin.wp_update', check_update_schedules, 'wp-update', 60, 105), ('builtin.api_background', run_api_background, 'api-background', 3600, 3600), ('builtin.pairing_prune', run_pairing_prune, 'pairing-prune', 3600, 60), diff --git a/backend/app/models/status_page.py b/backend/app/models/status_page.py index ea3c8f73..1f06cbce 100644 --- a/backend/app/models/status_page.py +++ b/backend/app/models/status_page.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timedelta from app import db import json @@ -48,22 +48,53 @@ def to_dict(self): class StatusComponent(db.Model): - """A service/component shown on the status page.""" + """A monitor: one synthetic check against a URL, host or managed site. + + Despite the table name this is the panel's *monitor* primitive — /monitoring + lists these directly. ``page_id`` is optional: a monitor exists on its own, + and a status page merely publishes a subset of them. + """ __tablename__ = 'status_components' id = db.Column(db.Integer, primary_key=True) - page_id = db.Column(db.Integer, db.ForeignKey('status_pages.id'), nullable=False) + # Nullable: a monitor can exist with no status page attached (migration 081). + page_id = db.Column(db.Integer, db.ForeignKey('status_pages.id'), nullable=True) name = db.Column(db.String(128), nullable=False) description = db.Column(db.Text) group = db.Column(db.String(64)) # e.g., "Web Services", "APIs" sort_order = db.Column(db.Integer, default=0) # Health check config - check_type = db.Column(db.String(16), default='http') # http, tcp, dns, smtp, ping + CHECK_TYPES = ('http', 'keyword', 'tcp', 'dns', 'smtp', 'ping') + check_type = db.Column(db.String(16), default='http') check_target = db.Column(db.String(512)) # URL, host:port, etc. check_interval = db.Column(db.Integer, default=60) # seconds check_timeout = db.Column(db.Integer, default=10) + # Probe options (migration 081). Each backs one control on the monitor's + # Configuration section. + check_method = db.Column(db.String(8), default='GET') + expected_status = db.Column(db.String(32), default='200-299') + keyword = db.Column(db.String(256)) # body substring for check_type='keyword' + follow_redirects = db.Column(db.Boolean, default=True) + verify_tls = db.Column(db.Boolean, default=True) + # Failed checks tolerated before an incident opens, and the running counter + # that drives it. Reset to 0 on any successful check. + retries = db.Column(db.Integer, default=2) + consecutive_failures = db.Column(db.Integer, default=0) + + # Paused monitors keep their history and last known status but are skipped + # by the scheduler. Kept separate from `status` so pausing never looks like + # an outage on a status page. + is_paused = db.Column(db.Boolean, default=False, nullable=False) + + # TLS certificate observed on the last https probe. `cert_checked_at` is not + # the same as `last_check_at`: reading the certificate opens a second + # connection, so it is throttled independently of the probe itself. + cert_issuer = db.Column(db.String(256)) + cert_expires_at = db.Column(db.DateTime) + cert_checked_at = db.Column(db.DateTime) + # Optional binding to a managed WordPress site. When set, the component's # status is driven from the site's health checks (#26) rather than a network # probe, and a real uptime % accrues from the recorded HealthCheck rows. @@ -91,6 +122,15 @@ class StatusComponent(db.Model): checks = db.relationship('HealthCheck', backref='component', lazy='dynamic', order_by='HealthCheck.checked_at.desc()', cascade='all, delete-orphan') + @property + def next_check_at(self): + """When the scheduler will next poll this monitor, or None if paused or + never checked. Derived rather than stored so changing the interval takes + effect immediately.""" + if self.is_paused or not self.last_check_at: + return None + return self.last_check_at + timedelta(seconds=self.check_interval or 60) + def to_dict(self): return { 'id': self.id, @@ -103,10 +143,22 @@ def to_dict(self): 'check_target': self.check_target, 'check_interval': self.check_interval, 'check_timeout': self.check_timeout, + 'check_method': self.check_method, + 'expected_status': self.expected_status, + 'keyword': self.keyword, + 'follow_redirects': self.follow_redirects, + 'verify_tls': self.verify_tls, + 'retries': self.retries, + 'consecutive_failures': self.consecutive_failures, + 'is_paused': bool(self.is_paused), + 'cert_issuer': self.cert_issuer, + 'cert_expires_at': self.cert_expires_at.isoformat() if self.cert_expires_at else None, + 'cert_checked_at': self.cert_checked_at.isoformat() if self.cert_checked_at else None, 'wordpress_site_id': self.wordpress_site_id, 'status': self.status, 'last_check_at': self.last_check_at.isoformat() if self.last_check_at else None, 'last_response_time': self.last_response_time, + 'next_check_at': self.next_check_at.isoformat() if self.next_check_at else None, 'uptime_24h': self.uptime_24h, 'uptime_7d': self.uptime_7d, 'uptime_30d': self.uptime_30d, @@ -143,7 +195,9 @@ class StatusIncident(db.Model): __tablename__ = 'status_incidents' id = db.Column(db.Integer, primary_key=True) - page_id = db.Column(db.Integer, db.ForeignKey('status_pages.id'), nullable=False) + # Nullable: an incident auto-opened for a monitor that belongs to no status + # page has no page either (migration 081). + page_id = db.Column(db.Integer, db.ForeignKey('status_pages.id'), nullable=True) # Optional link to the component this incident is about. Set when an incident # is auto-opened from a health check, so it can be auto-resolved on recovery (#26). component_id = db.Column(db.Integer, db.ForeignKey('status_components.id'), nullable=True) diff --git a/backend/app/services/cron_service.py b/backend/app/services/cron_service.py index fe5b78f8..b4a31c04 100644 --- a/backend/app/services/cron_service.py +++ b/backend/app/services/cron_service.py @@ -162,6 +162,18 @@ def list_jobs(cls) -> Dict: 'source': 'serverkit' }) + # Enrich exactly like jobs_for_application(): a human schedule, the next + # fire time, and the last-run join (plan 34). The admin list page renders + # Last run / Next run / Status columns off these, so both surfaces have + # to agree — leaving it out here is what left /cron showing enabled-only. + for job in jobs: + job_id = str(job.get('id', '')) + job['tracked'] = bool(job.get('tracked', False)) + job['schedule_human'] = (job.get('description') + or cls._describe_schedule(job.get('schedule', ''))) + job['next_run'] = cls._next_run(job.get('schedule', '')) + cls._attach_run_tracking(job, job_id) + return { 'success': True, 'jobs': jobs, diff --git a/backend/app/services/monitor_service.py b/backend/app/services/monitor_service.py new file mode 100644 index 00000000..384b8a4a --- /dev/null +++ b/backend/app/services/monitor_service.py @@ -0,0 +1,718 @@ +"""Monitors — the panel's synthetic-check engine. + +A monitor is a ``StatusComponent`` row: one probe (http / keyword / tcp / dns / +smtp / ping) against a URL, host or managed WordPress site, on an interval, with +its results kept as ``HealthCheck`` rows and its outages surfaced as +``StatusIncident``. + +This lives in core, not in the serverkit-status extension, because monitoring a +site should not depend on publishing a status page. The extension now delegates +its check-running here and keeps only page publishing and branding. + +Everything funnels through :meth:`_record` so that a network probe and a managed +site's health verdict produce identical bookkeeping: a sample, a live status, a +recomputed uptime, and the incident open/resolve edges. +""" +import logging +import os +import socket +import ssl +import time +from datetime import datetime, timedelta +from urllib.parse import urlparse + +from app import db +from app.models.status_page import ( + StatusComponent, HealthCheck, StatusIncident, StatusIncidentUpdate, +) + +logger = logging.getLogger(__name__) + +# A monitor's TLS certificate barely moves, and reading it costs a second +# connection. Refresh at most this often rather than on every probe. +CERT_REFRESH_SECONDS = 6 * 3600 + +# Certificate timestamps come back from OpenSSL in this format. +_CERT_TIME_FORMAT = '%b %d %H:%M:%S %Y %Z' + +UPTIME_WINDOWS = {'uptime_24h': 24, 'uptime_7d': 24 * 7, + 'uptime_30d': 24 * 30, 'uptime_90d': 24 * 90} + + +def _parse_expected_status(spec): + """Parse an expected-status spec into a list of (low, high) ranges. + + Accepts "200-299", "200", "200,204,301-302" and whitespace around any part. + Returns None for an unparseable or empty spec so the caller can fall back to + the default "anything below 400 is fine". + """ + if not spec: + return None + ranges = [] + for part in str(spec).split(','): + part = part.strip() + if not part: + continue + try: + if '-' in part: + low, high = part.split('-', 1) + ranges.append((int(low), int(high))) + else: + code = int(part) + ranges.append((code, code)) + except ValueError: + continue + return ranges or None + + +def _status_matches(code, spec): + ranges = _parse_expected_status(spec) + if ranges is None: + return code < 400 + return any(low <= code <= high for low, high in ranges) + + +class MonitorService: + """Create, run and report on monitors.""" + + # ------------------------------------------------------------------ + # CRUD + # ------------------------------------------------------------------ + + WRITABLE_FIELDS = ( + 'name', 'description', 'group', 'sort_order', 'page_id', + 'check_type', 'check_target', 'check_interval', 'check_timeout', + 'check_method', 'expected_status', 'keyword', 'follow_redirects', + 'verify_tls', 'retries', 'wordpress_site_id', 'status', + ) + + @staticmethod + def list_monitors(status=None, check_type=None, q=None, page_id=None, + include_paused=True): + query = StatusComponent.query + if status == 'paused': + query = query.filter(StatusComponent.is_paused.is_(True)) + elif status: + query = query.filter(StatusComponent.status == status, + StatusComponent.is_paused.is_(False)) + if not include_paused: + query = query.filter(StatusComponent.is_paused.is_(False)) + if check_type: + query = query.filter(StatusComponent.check_type == check_type) + if page_id is not None: + query = query.filter(StatusComponent.page_id == page_id) + if q: + like = f'%{q}%' + query = query.filter(db.or_(StatusComponent.name.ilike(like), + StatusComponent.check_target.ilike(like))) + return query.order_by(StatusComponent.sort_order, StatusComponent.name).all() + + @staticmethod + def get(monitor_id): + return StatusComponent.query.get(monitor_id) + + @staticmethod + def create(data): + if not data.get('name'): + raise ValueError('Monitor name is required') + check_type = data.get('check_type', 'http') + if check_type not in StatusComponent.CHECK_TYPES: + raise ValueError(f'Unknown check type: {check_type}') + if check_type == 'keyword' and not data.get('keyword'): + raise ValueError('A keyword check needs a keyword to look for') + # A monitor needs something to probe unless it is driven by a managed + # site's health verdict instead of the network. + if not data.get('check_target') and not data.get('wordpress_site_id'): + raise ValueError('Monitor needs a check target or a bound site') + + monitor = StatusComponent( + page_id=data.get('page_id'), + name=data['name'], + description=data.get('description', ''), + group=data.get('group', 'Services'), + sort_order=data.get('sort_order', 0), + check_type=check_type, + check_target=data.get('check_target', ''), + check_interval=data.get('check_interval', 60), + check_timeout=data.get('check_timeout', 10), + check_method=data.get('check_method', 'GET'), + expected_status=data.get('expected_status', '200-299'), + keyword=data.get('keyword'), + follow_redirects=data.get('follow_redirects', True), + verify_tls=data.get('verify_tls', True), + retries=data.get('retries', 2), + wordpress_site_id=data.get('wordpress_site_id'), + ) + db.session.add(monitor) + db.session.commit() + return monitor + + @staticmethod + def update(monitor_id, data): + monitor = StatusComponent.query.get(monitor_id) + if not monitor: + return None + if 'check_type' in data and data['check_type'] not in StatusComponent.CHECK_TYPES: + raise ValueError(f"Unknown check type: {data['check_type']}") + for field in MonitorService.WRITABLE_FIELDS: + if field in data: + setattr(monitor, field, data[field]) + db.session.commit() + return monitor + + @staticmethod + def delete(monitor_id): + monitor = StatusComponent.query.get(monitor_id) + if not monitor: + return False + # Resolve + unlink any incidents referencing this monitor first, so we + # never dangle the component_id FK (enforced on PostgreSQL) or leave a + # stale active incident behind after the monitor is gone. + for inc in StatusIncident.query.filter_by(component_id=monitor_id).all(): + if inc.status != 'resolved': + inc.status = 'resolved' + inc.resolved_at = datetime.utcnow() + inc.component_id = None + db.session.delete(monitor) + db.session.commit() + return True + + @staticmethod + def set_paused(monitor_id, paused): + monitor = StatusComponent.query.get(monitor_id) + if not monitor: + return None + monitor.is_paused = bool(paused) + if not paused: + # Resuming clears the failure streak so a monitor that was paused + # mid-outage does not immediately re-open an incident. + monitor.consecutive_failures = 0 + db.session.commit() + return monitor + + # ------------------------------------------------------------------ + # Scheduling + # ------------------------------------------------------------------ + + @staticmethod + def due_monitors(now=None): + """Monitors the scheduler should poll right now: not paused, not driven + by a managed site's health sweep, and either never checked or past their + interval.""" + now = now or datetime.utcnow() + candidates = StatusComponent.query.filter( + StatusComponent.is_paused.is_(False), + StatusComponent.wordpress_site_id.is_(None), + ).all() + due = [] + for monitor in candidates: + if not monitor.check_target: + continue + if monitor.last_check_at is None: + due.append(monitor) + continue + interval = monitor.check_interval or 60 + if monitor.last_check_at + timedelta(seconds=interval) <= now: + due.append(monitor) + return due + + # ------------------------------------------------------------------ + # Running checks + # ------------------------------------------------------------------ + + @staticmethod + def run_check(monitor_id): + """Probe a monitor and record the result. Returns the HealthCheck row.""" + monitor = StatusComponent.query.get(monitor_id) + if not monitor: + return None + result = MonitorService._perform_check(monitor) + return MonitorService._record(monitor, result) + + @staticmethod + def _record(monitor, result): + """Single bookkeeping path for every kind of check result. + + Records the sample, moves the live status, recomputes uptime, and opens + or resolves the monitor's incident. The old code only did the last two + for managed-site health syncs, so network-probed monitors never produced + an incident at all. + """ + check_status = result['status'] + prev_status = monitor.status + + hc = HealthCheck( + component_id=monitor.id, + status=check_status, + response_time=result.get('response_time'), + status_code=result.get('status_code'), + error=result.get('error'), + ) + db.session.add(hc) + + monitor.last_check_at = datetime.utcnow() + if result.get('response_time') is not None: + monitor.last_response_time = result.get('response_time') + if result.get('cert_checked_at'): + monitor.cert_checked_at = result['cert_checked_at'] + if result.get('cert_issuer'): + monitor.cert_issuer = result['cert_issuer'] + if result.get('cert_expires_at'): + monitor.cert_expires_at = result['cert_expires_at'] + + if check_status == 'up': + monitor.consecutive_failures = 0 + monitor.status = StatusComponent.STATUS_OPERATIONAL + elif check_status == 'degraded': + monitor.status = StatusComponent.STATUS_DEGRADED + else: + monitor.consecutive_failures = (monitor.consecutive_failures or 0) + 1 + # Hold at degraded until the failure streak clears `retries`, so a + # single blip does not page anyone. + if monitor.consecutive_failures > (monitor.retries or 0): + monitor.status = StatusComponent.STATUS_MAJOR + else: + monitor.status = StatusComponent.STATUS_DEGRADED + + db.session.commit() + + MonitorService.recompute_uptime(monitor) + + # Open an incident when ENTERING a major outage; resolve it when LEAVING + # major (to operational OR degraded). Resolving on the leaving-edge — not + # only on a clean major->operational hop — ensures a recovery that passes + # through an intermediate degraded poll (a common path) never leaves the + # incident stuck open. Degraded itself never opens a full incident. + if monitor.status == StatusComponent.STATUS_MAJOR and prev_status != StatusComponent.STATUS_MAJOR: + MonitorService._open_incident_for_component(monitor, result.get('error')) + elif monitor.status != StatusComponent.STATUS_MAJOR and prev_status == StatusComponent.STATUS_MAJOR: + MonitorService._resolve_incident_for_component(monitor) + return hc + + @staticmethod + def _perform_check(monitor): + """Execute one probe. Never raises — a failure is a 'down' result.""" + start = time.time() + result = {'status': 'down', 'response_time': None, 'error': None} + elapsed = lambda: int((time.time() - start) * 1000) # noqa: E731 + + try: + check_type = monitor.check_type or 'http' + + if check_type in ('http', 'keyword'): + import requests + method = (monitor.check_method or 'GET').upper() + resp = requests.request( + method, + monitor.check_target, + timeout=monitor.check_timeout, + verify=bool(monitor.verify_tls), + allow_redirects=bool(monitor.follow_redirects), + ) + result['response_time'] = elapsed() + result['status_code'] = resp.status_code + + if not _status_matches(resp.status_code, monitor.expected_status): + result['error'] = (f'HTTP {resp.status_code} is outside the ' + f'expected {monitor.expected_status or "2xx/3xx"}') + # A 4xx is the app answering badly; a 5xx (or anything else + # unexpected) is treated as an outage. + result['status'] = 'degraded' if 400 <= resp.status_code < 500 else 'down' + elif check_type == 'keyword': + # A 200 carrying the wrong page is an outage, not a success — + # that is the entire point of a keyword check. + if monitor.keyword and monitor.keyword in (resp.text or ''): + result['status'] = 'up' + else: + result['status'] = 'down' + result['error'] = f'Keyword not found in response: {monitor.keyword!r}' + else: + result['status'] = 'up' + + MonitorService._maybe_attach_certificate(monitor, result) + + elif check_type == 'tcp': + host, port = MonitorService._split_host_port(monitor.check_target) + sock = socket.create_connection((host, port), timeout=monitor.check_timeout) + result['response_time'] = elapsed() + result['status'] = 'up' + sock.close() + + elif check_type == 'ping': + result.update(MonitorService._ping(monitor)) + result['response_time'] = elapsed() + + elif check_type == 'dns': + socket.getaddrinfo(monitor.check_target, None) + result['response_time'] = elapsed() + result['status'] = 'up' + + elif check_type == 'smtp': + import smtplib + host, port = MonitorService._split_host_port(monitor.check_target, default_port=25) + server = smtplib.SMTP(host, port, timeout=monitor.check_timeout) + try: + code, _msg = server.noop() + finally: + try: + server.quit() + except Exception: + pass + result['response_time'] = elapsed() + result['status_code'] = code + result['status'] = 'up' if code == 250 else 'degraded' + + else: + result['error'] = f'Unknown check type: {check_type}' + + except Exception as e: + result['response_time'] = elapsed() + result['error'] = str(e) + result['status'] = 'down' + + return result + + @staticmethod + def _split_host_port(target, default_port=None): + """Split 'host:port' (or a bare host when *default_port* is given).""" + target = (target or '').strip() + if ':' in target: + host, port = target.rsplit(':', 1) + return host, int(port) + if default_port is not None: + return target, default_port + raise ValueError(f'Target must be host:port, got {target!r}') + + @staticmethod + def _ping(monitor): + """One ICMP echo. The previous implementation ignored the return code + entirely, so a ping check could only ever report 'up'.""" + from app.utils.system import run_command + timeout = monitor.check_timeout or 10 + if os.name == 'nt': + cmd = ['ping', '-n', '1', '-w', str(int(timeout) * 1000), monitor.check_target] + else: + cmd = ['ping', '-c', '1', '-W', str(int(timeout)), monitor.check_target] + res = run_command(cmd, timeout=timeout + 5) + if res.get('returncode') == 0: + return {'status': 'up'} + return { + 'status': 'down', + 'error': (res.get('stderr') or res.get('stdout') or 'Ping failed').strip()[:500], + } + + # ------------------------------------------------------------------ + # TLS certificate + # ------------------------------------------------------------------ + + @staticmethod + def _maybe_attach_certificate(monitor, result): + """Read the peer certificate for https monitors, at most every + CERT_REFRESH_SECONDS — it costs a second connection and barely moves. + + Throttled on ``cert_checked_at``, not ``last_check_at``: an active + monitor is checked every 30s, so gating on the probe clock would read the + certificate once and then never refresh it. + """ + target = monitor.check_target or '' + if not target.startswith('https://'): + return + if monitor.cert_checked_at: + age = (datetime.utcnow() - monitor.cert_checked_at).total_seconds() + if age < CERT_REFRESH_SECONDS: + return + # Stamped even when the read fails, so an unreachable TLS endpoint is + # retried on the same cadence instead of on every single probe. + result['cert_checked_at'] = datetime.utcnow() + try: + cert = MonitorService._probe_certificate( + target, monitor.check_timeout or 10, bool(monitor.verify_tls)) + except Exception as e: + logger.debug('Certificate probe failed for monitor %s: %s', monitor.id, e) + return + if cert: + result.update(cert) + + @staticmethod + def _probe_certificate(url, timeout, verify=True): + parsed = urlparse(url) + host = parsed.hostname + if not host: + return None + port = parsed.port or 443 + + context = ssl.create_default_context() + if not verify: + # Still read the certificate even when the monitor does not require + # it to validate — an expiring self-signed cert is worth surfacing. + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + + with socket.create_connection((host, port), timeout=timeout) as sock: + with context.wrap_socket(sock, server_hostname=host) as tls: + cert = tls.getpeercert() + if not cert: + return None + + out = {} + not_after = cert.get('notAfter') + if not_after: + try: + out['cert_expires_at'] = datetime.strptime(not_after, _CERT_TIME_FORMAT) + except ValueError: + pass + issuer = cert.get('issuer') or () + for rdn in issuer: + for key, value in rdn: + if key == 'organizationName': + out['cert_issuer'] = value + return out or None + + # ------------------------------------------------------------------ + # History and uptime + # ------------------------------------------------------------------ + + @staticmethod + def get_check_history(monitor_id, hours=24, limit=None): + since = datetime.utcnow() - timedelta(hours=hours) + query = HealthCheck.query.filter( + HealthCheck.component_id == monitor_id, + HealthCheck.checked_at >= since, + ).order_by(HealthCheck.checked_at.desc()) + if limit: + query = query.limit(limit) + return query.all() + + @staticmethod + def recent_response_times(monitor_ids, per_monitor=24, hours=6): + """Last N response times per monitor, oldest first — the list sparkline. + + One query for the whole page rather than one per row, bounded by a time + window so a busy monitor's history can't crowd out a quiet one's (which + a single global LIMIT would do). + """ + if not monitor_ids: + return {} + since = datetime.utcnow() - timedelta(hours=hours) + rows = HealthCheck.query.filter( + HealthCheck.component_id.in_(list(monitor_ids)), + HealthCheck.response_time.isnot(None), + HealthCheck.checked_at >= since, + ).order_by(HealthCheck.checked_at.desc()).all() + + out = {} + for row in rows: + bucket = out.setdefault(row.component_id, []) + if len(bucket) < per_monitor: + bucket.append(row.response_time) + return {mid: list(reversed(values)) for mid, values in out.items()} + + @staticmethod + def recompute_uptime(monitor): + """Recompute uptime_24h/7d/30d/90d from HealthCheck rows (fraction of + recorded checks with status 'up'). Only fully-healthy checks count as up + — 'degraded' periods reduce the percentage, matching the status-page + convention where degraded is not "operational". Leaves a window's + existing value untouched when it has no samples yet.""" + now = datetime.utcnow() + for field, hours in UPTIME_WINDOWS.items(): + since = now - timedelta(hours=hours) + base = HealthCheck.query.filter( + HealthCheck.component_id == monitor.id, + HealthCheck.checked_at >= since, + ) + total = base.count() + if total: + up = base.filter(HealthCheck.status == 'up').count() + setattr(monitor, field, round(up / total * 100, 2)) + db.session.commit() + + @staticmethod + def uptime_days(monitor_id, days=90): + """Per-day uptime buckets for the 90-day bar strip. + + A day with no samples reports state 'none' rather than 100% so the strip + can render "we weren't watching" differently from "it was fine". + """ + now = datetime.utcnow() + start = (now - timedelta(days=days - 1)).replace(hour=0, minute=0, second=0, microsecond=0) + rows = HealthCheck.query.filter( + HealthCheck.component_id == monitor_id, + HealthCheck.checked_at >= start, + ).all() + + buckets = {} + for row in rows: + key = row.checked_at.date().isoformat() + bucket = buckets.setdefault(key, {'total': 0, 'up': 0, 'down': 0}) + bucket['total'] += 1 + if row.status == 'up': + bucket['up'] += 1 + elif row.status == 'down': + bucket['down'] += 1 + + out = [] + for offset in range(days): + day = (start + timedelta(days=offset)).date() + bucket = buckets.get(day.isoformat()) + if not bucket or not bucket['total']: + out.append({'date': day.isoformat(), 'state': 'none', + 'uptime': None, 'checks': 0, 'down_checks': 0}) + continue + uptime = round(bucket['up'] / bucket['total'] * 100, 3) + if bucket['down'] and uptime < 50: + state = 'down' + elif uptime < 100: + state = 'partial' + else: + state = 'up' + out.append({'date': day.isoformat(), 'state': state, 'uptime': uptime, + 'checks': bucket['total'], 'down_checks': bucket['down']}) + return out + + @staticmethod + def stats(): + """KPI-band counts for the Monitors tab.""" + monitors = StatusComponent.query.all() + active = [m for m in monitors if not m.is_paused] + by_status = {'operational': 0, 'degraded': 0, 'major_outage': 0, + 'partial_outage': 0, 'maintenance': 0} + for monitor in active: + if monitor.status in by_status: + by_status[monitor.status] += 1 + overall = None + if active: + values = [m.uptime_30d for m in active if m.uptime_30d is not None] + if values: + overall = round(sum(values) / len(values), 2) + return { + 'total': len(monitors), + 'paused': len(monitors) - len(active), + 'by_status': by_status, + 'operational': by_status['operational'], + 'degraded': by_status['degraded'] + by_status['partial_outage'], + 'down': by_status['major_outage'], + 'overall_uptime_30d': overall, + } + + # ------------------------------------------------------------------ + # Managed-site health bridge + # ------------------------------------------------------------------ + + # Map an EnvironmentHealthService overall_status to a HealthCheck status. + # 'unknown' is intentionally absent — indeterminate checks are not recorded + # so they don't pollute the uptime %. + _HEALTH_MAP = {'healthy': 'up', 'degraded': 'degraded', 'unhealthy': 'down'} + + @staticmethod + def sync_component_from_health(monitor, overall_status, error=None): + """Drive a managed-site-bound monitor from an EnvironmentHealthService + verdict instead of a network probe (#26). + + Returns the recorded HealthCheck, or None for an indeterminate + ('unknown') verdict (not recorded). + """ + check_status = MonitorService._HEALTH_MAP.get(overall_status) + if not check_status: + return None + # A health verdict is authoritative, not a flaky network blip, so it + # should reach major outage on the first 'unhealthy' rather than waiting + # out the retry streak. + if check_status == 'down': + monitor.consecutive_failures = max(monitor.consecutive_failures or 0, + monitor.retries or 0) + return MonitorService._record(monitor, {'status': check_status, 'error': error}) + + # ------------------------------------------------------------------ + # Incidents + # ------------------------------------------------------------------ + + @staticmethod + def list_incidents(state=None, limit=100): + query = StatusIncident.query + if state == 'active': + query = query.filter(StatusIncident.status != 'resolved') + elif state == 'resolved': + query = query.filter(StatusIncident.status == 'resolved') + return query.order_by(StatusIncident.created_at.desc()).limit(limit).all() + + @staticmethod + def create_incident(page_id, data): + incident = StatusIncident( + page_id=page_id, + component_id=data.get('component_id'), + title=data['title'], + status=data.get('status', 'investigating'), + impact=data.get('impact', 'minor'), + body=data.get('body', ''), + is_maintenance=data.get('is_maintenance', False), + scheduled_start=data.get('scheduled_start'), + scheduled_end=data.get('scheduled_end'), + ) + db.session.add(incident) + db.session.commit() + return incident + + @staticmethod + def update_incident(incident_id, data): + incident = StatusIncident.query.get(incident_id) + if not incident: + return None + for field in ['title', 'status', 'impact', 'body']: + if field in data: + setattr(incident, field, data[field]) + if data.get('status') == 'resolved': + incident.resolved_at = datetime.utcnow() + + # Add timeline update + if data.get('update_body'): + update = StatusIncidentUpdate( + incident_id=incident_id, + status=data.get('status', incident.status), + body=data['update_body'], + ) + db.session.add(update) + + db.session.commit() + return incident + + @staticmethod + def delete_incident(incident_id): + incident = StatusIncident.query.get(incident_id) + if not incident: + return False + db.session.delete(incident) + db.session.commit() + return True + + @staticmethod + def _open_incident_for_component(monitor, error=None): + """Open a major-impact incident for a monitor if one is not already open.""" + existing = StatusIncident.query.filter( + StatusIncident.component_id == monitor.id, + StatusIncident.status != 'resolved', + ).first() + if existing: + return existing + # page_id may be None — a monitor that belongs to no status page still + # gets an incident, it just isn't published anywhere. + return MonitorService.create_incident(monitor.page_id, { + 'title': f'{monitor.name} is experiencing an outage', + 'status': 'investigating', + 'impact': 'major', + 'body': error or 'Automated health check detected an outage.', + 'component_id': monitor.id, + }) + + @staticmethod + def _resolve_incident_for_component(monitor): + """Resolve the open auto-incident for a monitor, if any.""" + existing = StatusIncident.query.filter( + StatusIncident.component_id == monitor.id, + StatusIncident.status != 'resolved', + ).first() + if existing: + MonitorService.update_incident(existing.id, { + 'status': 'resolved', + 'update_body': 'Automated health check detected recovery.', + }) diff --git a/backend/migrations/versions/081_monitors_first_class.py b/backend/migrations/versions/081_monitors_first_class.py new file mode 100644 index 00000000..22d2aaa1 --- /dev/null +++ b/backend/migrations/versions/081_monitors_first_class.py @@ -0,0 +1,118 @@ +"""Promote status-page components to first-class monitors. + +A ``StatusComponent`` was already a full synthetic check (check_type / target / +interval / timeout, HealthCheck history, uptime windows, auto-incidents) — it +was just *framed* as "a component of a status page" and could not exist without +one. This migration makes it stand on its own so /monitoring can own it: + +- ``status_components.page_id`` -> nullable (a monitor needs no status page) +- ``status_incidents.page_id`` -> nullable (a pageless monitor's auto-incident + has no page either; without this the outage path raises IntegrityError) + +Plus the columns backing the monitor UI, one per surfaced control: + +- ``is_paused`` pause/resume without overloading ``status`` +- ``check_method`` HTTP verb (Configuration section) +- ``expected_status`` accepted status range, e.g. "200-299" +- ``keyword`` body substring for the Keyword check type +- ``follow_redirects`` / ``verify_tls`` probe switches +- ``retries`` failed checks tolerated before an incident opens +- ``consecutive_failures`` running counter that drives ``retries`` +- ``cert_issuer`` / ``cert_expires_at`` TLS certificate KPI + +Idempotent: MigrationService._fix_missing_columns runs db.create_all() on boot +before Alembic, so a fresh database already matches the model. Guard on the live +schema like previous migrations. + +Revision ID: 081_monitors_first_class +Revises: 080_dashboard_boards +Create Date: 2026-07-27 +""" +from alembic import op +import sqlalchemy as sa + +revision = '081_monitors_first_class' +down_revision = '080_dashboard_boards' +branch_labels = None +depends_on = None + + +# Batch mode rebuilds the table on SQLite, which needs every reflected FK to +# carry a name. These tables' FKs are unnamed, so supply the convention. +NAMING_CONVENTION = { + 'fk': 'fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s', + 'pk': 'pk_%(table_name)s', +} + +NEW_COLUMNS = [ + ('is_paused', sa.Boolean(), {'nullable': False, 'server_default': sa.false()}), + ('check_method', sa.String(length=8), {'nullable': True, 'server_default': 'GET'}), + ('expected_status', sa.String(length=32), {'nullable': True, 'server_default': '200-299'}), + ('keyword', sa.String(length=256), {'nullable': True}), + ('follow_redirects', sa.Boolean(), {'nullable': False, 'server_default': sa.true()}), + ('verify_tls', sa.Boolean(), {'nullable': False, 'server_default': sa.true()}), + ('retries', sa.Integer(), {'nullable': False, 'server_default': '2'}), + ('consecutive_failures', sa.Integer(), {'nullable': False, 'server_default': '0'}), + ('cert_issuer', sa.String(length=256), {'nullable': True}), + ('cert_expires_at', sa.DateTime(), {'nullable': True}), + # When the certificate was last read, which is NOT when the monitor was last + # checked: the TLS read is throttled to once every few hours because it costs + # a second connection, so it needs a clock of its own. + ('cert_checked_at', sa.DateTime(), {'nullable': True}), +] + + +def _columns(inspector, table): + if table not in set(inspector.get_table_names()): + return {} + return {c['name']: c for c in inspector.get_columns(table)} + + +def _make_page_id_nullable(inspector, table): + """Drop the NOT NULL on .page_id, if it is still there.""" + col = _columns(inspector, table).get('page_id') + if col is None or col.get('nullable'): + return + with op.batch_alter_table(table, naming_convention=NAMING_CONVENTION) as batch_op: + batch_op.alter_column('page_id', existing_type=sa.Integer(), nullable=True) + + +def upgrade(): + conn = op.get_bind() + inspector = sa.inspect(conn) + + if 'status_components' not in set(inspector.get_table_names()): + return + + existing = _columns(inspector, 'status_components') + missing = [(n, t, kw) for n, t, kw in NEW_COLUMNS if n not in existing] + if missing: + with op.batch_alter_table('status_components') as batch_op: + for name, col_type, kwargs in missing: + batch_op.add_column(sa.Column(name, col_type, **kwargs)) + + _make_page_id_nullable(inspector, 'status_components') + _make_page_id_nullable(inspector, 'status_incidents') + + +def downgrade(): + conn = op.get_bind() + inspector = sa.inspect(conn) + + if 'status_components' not in set(inspector.get_table_names()): + return + + existing = _columns(inspector, 'status_components') + present = [n for n, _t, _kw in NEW_COLUMNS if n in existing] + if present: + with op.batch_alter_table('status_components') as batch_op: + for name in present: + batch_op.drop_column(name) + + # page_id goes back to NOT NULL. Rows created as pageless monitors cannot + # satisfy that, so drop them first rather than failing the downgrade. + conn.execute(sa.text('DELETE FROM status_incidents WHERE page_id IS NULL')) + conn.execute(sa.text('DELETE FROM status_components WHERE page_id IS NULL')) + for table in ('status_components', 'status_incidents'): + with op.batch_alter_table(table, naming_convention=NAMING_CONVENTION) as batch_op: + batch_op.alter_column('page_id', existing_type=sa.Integer(), nullable=False) diff --git a/backend/tests/test_cron_runs.py b/backend/tests/test_cron_runs.py index 58ad91f2..777fc493 100644 --- a/backend/tests/test_cron_runs.py +++ b/backend/tests/test_cron_runs.py @@ -211,3 +211,48 @@ class _Result: runs = CronRunService.recent_runs(jid) assert len(runs) == 1 assert runs[0]['status'] == 'success' + + +# --------------------------------------------------------------------------- # +# list_jobs() enrichment — the admin /cron table renders Last run / Next run / +# Status straight off these keys, so the list has to carry the same joins +# jobs_for_application() does. Before this, only the for-app surface had them. +# --------------------------------------------------------------------------- # + +def test_list_jobs_carries_schedule_and_next_run(app, cron_store): + CronService.add_job('0 0 * * *', '/usr/bin/task.sh', name='Nightly') + + job = CronService.list_jobs()['jobs'][0] + + assert job['schedule_human'] + assert job['next_run'] # croniter resolves a concrete next fire + assert job['tracked'] is False + + +def test_list_jobs_carries_last_run_join(app, cron_store, monkeypatch): + import subprocess + + jid = CronService.add_job('0 0 * * *', '/bin/false', name='Flaky')['job_id'] + + class _Result: + returncode = 1 + stdout = '' + stderr = 'boom' + + monkeypatch.setattr(subprocess, 'run', lambda *a, **k: _Result()) + CronService.run_job_now(jid) + + job = next(j for j in CronService.list_jobs()['jobs'] if str(j['id']) == str(jid)) + + assert job['last_status'] == 'failure' + assert job['last_run'] + assert job['success_rate'] == 0.0 + + +def test_list_jobs_without_history_has_null_run_fields(app, cron_store): + CronService.add_job('*/5 * * * *', '/usr/bin/task.sh', name='Fresh') + + job = CronService.list_jobs()['jobs'][0] + + assert job['last_run'] is None + assert job['last_status'] is None diff --git a/backend/tests/test_jobs.py b/backend/tests/test_jobs.py index 439c30f3..fc39acd3 100644 --- a/backend/tests/test_jobs.py +++ b/backend/tests/test_jobs.py @@ -176,16 +176,17 @@ def test_register_and_seed(self, app): assert 'builtin.backup_scheduler' in kinds assert 'builtin.extension_updates' in kinds assert 'builtin.job_retention' in kinds - assert len([k for k in kinds if k.startswith('builtin.')]) == 11 + assert 'builtin.monitor_check' in kinds + assert len([k for k in kinds if k.startswith('builtin.')]) == 12 builtin_handlers.seed_builtin_schedules() - # 11 builtin.* schedules (incl. job-retention) + login-link/SSO reapers + - # drift/FIM/bandwidth sweeps + the host doctor sweep (plan 26) + the - # setup-health nag (plan 22). - assert ScheduledJob.query.count() == 18 + # 12 builtin.* schedules (incl. job-retention and the monitor sweep) + + # login-link/SSO reapers + drift/FIM/bandwidth sweeps + the host doctor + # sweep (plan 26) + the setup-health nag (plan 22). + assert ScheduledJob.query.count() == 19 # Seeding twice doesn't duplicate. builtin_handlers.seed_builtin_schedules() - assert ScheduledJob.query.count() == 18 + assert ScheduledJob.query.count() == 19 class TestApi: diff --git a/backend/tests/test_monitor_service.py b/backend/tests/test_monitor_service.py new file mode 100644 index 00000000..eb9ee01b --- /dev/null +++ b/backend/tests/test_monitor_service.py @@ -0,0 +1,511 @@ +"""Monitors: the promoted first-class check engine (app/services/monitor_service). + +Covers the behaviours that were broken or missing while the engine lived inside +the serverkit-status extension: + +- monitors exist without a status page (page_id nullable) +- a network-probed monitor actually opens and resolves an incident; previously + only the WordPress health-sync path did, so nothing else ever paged +- ping honours the process return code instead of always reporting 'up' +- expected_status / keyword / redirects / verify_tls are wired to the probe +- the scheduler picks up monitors whose interval has elapsed +""" +from datetime import datetime, timedelta +from unittest.mock import patch + +import pytest + +from app import db +from app.models.status_page import StatusComponent, HealthCheck, StatusIncident +from app.services.monitor_service import ( + MonitorService, _parse_expected_status, _status_matches, +) + + +def _monitor(**kwargs): + data = { + 'name': 'Example', + 'check_type': 'http', + 'check_target': 'https://example.test/health', + 'check_interval': 60, + 'retries': 0, + } + data.update(kwargs) + return MonitorService.create(data) + + +class _Resp: + """Minimal stand-in for a requests.Response.""" + + def __init__(self, status_code=200, text=''): + self.status_code = status_code + self.text = text + + +# --------------------------------------------------------------------------- +# Expected-status parsing +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize('spec,code,expected', [ + ('200-299', 204, True), + ('200-299', 301, False), + ('200', 200, True), + ('200', 201, False), + ('200,204,301-302', 302, True), + ('200,204,301-302', 303, False), + (' 200 - 299 ', 250, True), + # An unparseable or empty spec falls back to "anything below 400". + ('', 302, True), + (None, 500, False), + ('nonsense', 200, True), +]) +def test_status_matches(spec, code, expected): + assert _status_matches(code, spec) is expected + + +def test_parse_expected_status_skips_junk_parts(): + assert _parse_expected_status('200, ,oops,300-399') == [(200, 200), (300, 399)] + + +# --------------------------------------------------------------------------- +# A monitor needs no status page +# --------------------------------------------------------------------------- + +def test_monitor_can_exist_without_a_status_page(app): + monitor = _monitor(name='Pageless') + assert monitor.id is not None + assert monitor.page_id is None + + +def test_create_rejects_unknown_check_type(app): + with pytest.raises(ValueError): + _monitor(check_type='carrier-pigeon') + + +def test_create_rejects_keyword_check_without_keyword(app): + with pytest.raises(ValueError): + _monitor(check_type='keyword', keyword=None) + + +def test_create_requires_a_target_or_a_bound_site(app): + with pytest.raises(ValueError): + MonitorService.create({'name': 'Nothing to probe'}) + + +# --------------------------------------------------------------------------- +# HTTP / keyword probes +# --------------------------------------------------------------------------- + +def test_http_check_up_within_expected_status(app): + monitor = _monitor(expected_status='200-299') + with patch('requests.request', return_value=_Resp(204)), \ + patch.object(MonitorService, '_maybe_attach_certificate'): + result = MonitorService._perform_check(monitor) + assert result['status'] == 'up' + assert result['status_code'] == 204 + + +def test_http_check_outside_expected_status_is_not_up(app): + """A 301 passes the old "< 400 is fine" rule but fails an explicit 200-299.""" + monitor = _monitor(expected_status='200-299') + with patch('requests.request', return_value=_Resp(301)), \ + patch.object(MonitorService, '_maybe_attach_certificate'): + result = MonitorService._perform_check(monitor) + assert result['status'] == 'down' + assert '200-299' in result['error'] + + +def test_http_4xx_is_degraded_5xx_is_down(app): + monitor = _monitor(expected_status='200-299') + with patch.object(MonitorService, '_maybe_attach_certificate'): + with patch('requests.request', return_value=_Resp(404)): + assert MonitorService._perform_check(monitor)['status'] == 'degraded' + with patch('requests.request', return_value=_Resp(503)): + assert MonitorService._perform_check(monitor)['status'] == 'down' + + +def test_http_probe_passes_method_redirects_and_verify(app): + monitor = _monitor(check_method='head', follow_redirects=False, verify_tls=False, + check_timeout=7) + with patch('requests.request', return_value=_Resp(200)) as req, \ + patch.object(MonitorService, '_maybe_attach_certificate'): + MonitorService._perform_check(monitor) + args, kwargs = req.call_args + assert args[0] == 'HEAD' + assert kwargs['allow_redirects'] is False + assert kwargs['verify'] is False + assert kwargs['timeout'] == 7 + + +def test_keyword_check_down_when_keyword_missing(app): + """A 200 carrying the wrong page is an outage — the point of a keyword check.""" + monitor = _monitor(check_type='keyword', keyword='Proceed to checkout') + with patch('requests.request', return_value=_Resp(200, 'Server Error')), \ + patch.object(MonitorService, '_maybe_attach_certificate'): + result = MonitorService._perform_check(monitor) + assert result['status'] == 'down' + assert 'Proceed to checkout' in result['error'] + + +def test_keyword_check_up_when_keyword_present(app): + monitor = _monitor(check_type='keyword', keyword='status":"ok') + with patch('requests.request', return_value=_Resp(200, '{"status":"ok"}')), \ + patch.object(MonitorService, '_maybe_attach_certificate'): + assert MonitorService._perform_check(monitor)['status'] == 'up' + + +def test_probe_never_raises_on_transport_error(app): + monitor = _monitor() + with patch('requests.request', side_effect=OSError('connection refused')): + result = MonitorService._perform_check(monitor) + assert result['status'] == 'down' + assert 'connection refused' in result['error'] + + +# --------------------------------------------------------------------------- +# Ping — the return code used to be ignored entirely +# --------------------------------------------------------------------------- + +def test_ping_failure_is_down_not_up(app): + monitor = _monitor(check_type='ping', check_target='10.255.255.1') + with patch('app.utils.system.run_command', + return_value={'returncode': 1, 'stdout': '', 'stderr': '100% packet loss'}): + result = MonitorService._perform_check(monitor) + assert result['status'] == 'down' + assert 'packet loss' in result['error'] + + +def test_ping_success_is_up(app): + monitor = _monitor(check_type='ping', check_target='127.0.0.1') + with patch('app.utils.system.run_command', + return_value={'returncode': 0, 'stdout': '1 received', 'stderr': ''}): + assert MonitorService._perform_check(monitor)['status'] == 'up' + + +def test_ping_uses_platform_appropriate_flags(app): + monitor = _monitor(check_type='ping', check_target='127.0.0.1', check_timeout=3) + with patch('app.utils.system.run_command', + return_value={'returncode': 0, 'stdout': '', 'stderr': ''}) as run: + with patch('app.services.monitor_service.os.name', 'nt'): + MonitorService._perform_check(monitor) + assert run.call_args[0][0][:3] == ['ping', '-n', '1'] + with patch('app.services.monitor_service.os.name', 'posix'): + MonitorService._perform_check(monitor) + assert run.call_args[0][0][:3] == ['ping', '-c', '1'] + + +# --------------------------------------------------------------------------- +# TLS certificate throttling +# --------------------------------------------------------------------------- + +def test_certificate_is_read_when_never_read_before(app): + monitor = _monitor(check_target='https://example.test/') + expires = datetime.utcnow() + timedelta(days=40) + result = {} + with patch.object(MonitorService, '_probe_certificate', + return_value={'cert_issuer': "Let's Encrypt", 'cert_expires_at': expires}) as probe: + MonitorService._maybe_attach_certificate(monitor, result) + assert probe.called + assert result['cert_issuer'] == "Let's Encrypt" + assert result['cert_checked_at'] is not None + + +def test_certificate_read_is_throttled_by_its_own_clock(app): + """Throttling must key on cert_checked_at, not last_check_at: an active + monitor is probed every 30s, so gating on the probe clock would read the + certificate once and then never refresh it.""" + monitor = _monitor(check_target='https://example.test/') + monitor.cert_checked_at = datetime.utcnow() - timedelta(minutes=5) + # A recent probe must NOT unblock a cert re-read. + monitor.last_check_at = datetime.utcnow() + db.session.commit() + + with patch.object(MonitorService, '_probe_certificate') as probe: + MonitorService._maybe_attach_certificate(monitor, {}) + assert not probe.called + + monitor.cert_checked_at = datetime.utcnow() - timedelta(hours=7) + db.session.commit() + with patch.object(MonitorService, '_probe_certificate', return_value={}) as probe: + MonitorService._maybe_attach_certificate(monitor, {}) + assert probe.called + + +def test_certificate_failure_still_stamps_the_clock(app): + """Otherwise an unreachable TLS endpoint is retried on every single probe.""" + monitor = _monitor(check_target='https://example.test/') + result = {} + with patch.object(MonitorService, '_probe_certificate', side_effect=OSError('refused')): + MonitorService._maybe_attach_certificate(monitor, result) + assert result['cert_checked_at'] is not None + + +def test_certificate_skipped_for_non_https(app): + monitor = _monitor(check_target='http://example.test/') + result = {} + with patch.object(MonitorService, '_probe_certificate') as probe: + MonitorService._maybe_attach_certificate(monitor, result) + assert not probe.called + assert result == {} + + +# --------------------------------------------------------------------------- +# Recording: status, retries, uptime, incidents +# --------------------------------------------------------------------------- + +def test_record_writes_a_sample_and_moves_status(app): + monitor = _monitor() + MonitorService._record(monitor, {'status': 'up', 'response_time': 42, 'status_code': 200}) + assert monitor.status == StatusComponent.STATUS_OPERATIONAL + assert monitor.last_response_time == 42 + assert monitor.last_check_at is not None + assert HealthCheck.query.filter_by(component_id=monitor.id).count() == 1 + + +def test_failure_streak_holds_at_degraded_until_retries_exhausted(app): + monitor = _monitor(retries=2) + for _ in range(2): + MonitorService._record(monitor, {'status': 'down', 'error': 'boom'}) + assert monitor.status == StatusComponent.STATUS_DEGRADED + MonitorService._record(monitor, {'status': 'down', 'error': 'boom'}) + assert monitor.status == StatusComponent.STATUS_MAJOR + + +def test_success_clears_the_failure_streak(app): + monitor = _monitor(retries=2) + MonitorService._record(monitor, {'status': 'down', 'error': 'boom'}) + assert monitor.consecutive_failures == 1 + MonitorService._record(monitor, {'status': 'up', 'response_time': 10}) + assert monitor.consecutive_failures == 0 + + +def test_network_monitor_opens_and_resolves_an_incident(app): + """The gap this round closes: only the WordPress health path used to do this, + so a plain URL monitor never produced an incident.""" + monitor = _monitor(name='Checkout API', retries=0) + + MonitorService._record(monitor, {'status': 'down', 'error': 'connection refused'}) + assert monitor.status == StatusComponent.STATUS_MAJOR + incident = StatusIncident.query.filter_by(component_id=monitor.id).one() + assert incident.status != 'resolved' + assert incident.impact == 'major' + assert 'Checkout API' in incident.title + # A pageless monitor still gets an incident; it just isn't published. + assert incident.page_id is None + + MonitorService._record(monitor, {'status': 'up', 'response_time': 12}) + db.session.refresh(incident) + assert incident.status == 'resolved' + assert incident.resolved_at is not None + + +def test_incident_is_not_duplicated_while_already_open(app): + monitor = _monitor(retries=0) + for _ in range(3): + MonitorService._record(monitor, {'status': 'down', 'error': 'boom'}) + assert StatusIncident.query.filter_by(component_id=monitor.id).count() == 1 + + +def test_recovery_through_degraded_still_resolves(app): + """Resolving on the leaving-major edge, not only on a clean major->up hop.""" + monitor = _monitor(retries=0) + MonitorService._record(monitor, {'status': 'down', 'error': 'boom'}) + MonitorService._record(monitor, {'status': 'degraded'}) + incident = StatusIncident.query.filter_by(component_id=monitor.id).one() + assert incident.status == 'resolved' + + +def test_recompute_uptime_counts_only_up_samples(app): + monitor = _monitor() + for status in ('up', 'up', 'up', 'down'): + db.session.add(HealthCheck(component_id=monitor.id, status=status, + checked_at=datetime.utcnow())) + db.session.commit() + MonitorService.recompute_uptime(monitor) + assert monitor.uptime_24h == 75.0 + + +def test_uptime_days_marks_unwatched_days_as_none(app): + monitor = _monitor() + db.session.add(HealthCheck(component_id=monitor.id, status='up', + checked_at=datetime.utcnow())) + db.session.commit() + days = MonitorService.uptime_days(monitor.id, days=5) + assert len(days) == 5 + assert days[-1]['state'] == 'up' + # Days before the monitor existed report "we weren't watching", not 100%. + assert [d['state'] for d in days[:-1]] == ['none'] * 4 + + +# --------------------------------------------------------------------------- +# Pause + scheduling +# --------------------------------------------------------------------------- + +def test_due_monitors_respects_the_interval(app): + monitor = _monitor(check_interval=60) + # Never checked -> due immediately. + assert monitor.id in [m.id for m in MonitorService.due_monitors()] + + monitor.last_check_at = datetime.utcnow() + db.session.commit() + assert monitor.id not in [m.id for m in MonitorService.due_monitors()] + + monitor.last_check_at = datetime.utcnow() - timedelta(seconds=61) + db.session.commit() + assert monitor.id in [m.id for m in MonitorService.due_monitors()] + + +def test_due_monitors_skips_paused_and_site_bound(app): + paused = _monitor(name='Paused') + MonitorService.set_paused(paused.id, True) + site_bound = MonitorService.create({ + 'name': 'WP site', 'wordpress_site_id': 1, 'check_target': '', + }) + due_ids = [m.id for m in MonitorService.due_monitors()] + assert paused.id not in due_ids + # Site-bound monitors are driven by the health sweep, not the network probe. + assert site_bound.id not in due_ids + + +def test_resume_clears_the_failure_streak(app): + """A monitor paused mid-outage must not instantly re-page on resume.""" + monitor = _monitor(retries=2) + MonitorService._record(monitor, {'status': 'down', 'error': 'boom'}) + MonitorService.set_paused(monitor.id, True) + MonitorService.set_paused(monitor.id, False) + assert monitor.consecutive_failures == 0 + + +def test_next_check_at_is_none_while_paused(app): + monitor = _monitor() + MonitorService._record(monitor, {'status': 'up', 'response_time': 5}) + assert monitor.next_check_at is not None + MonitorService.set_paused(monitor.id, True) + assert monitor.next_check_at is None + + +# --------------------------------------------------------------------------- +# Deletion + stats +# --------------------------------------------------------------------------- + +def test_delete_unlinks_and_resolves_incidents(app): + monitor = _monitor(retries=0) + MonitorService._record(monitor, {'status': 'down', 'error': 'boom'}) + incident = StatusIncident.query.filter_by(component_id=monitor.id).one() + + assert MonitorService.delete(monitor.id) is True + db.session.refresh(incident) + assert incident.component_id is None + assert incident.status == 'resolved' + + +def test_stats_counts_by_state(app): + up = _monitor(name='Up one') + MonitorService._record(up, {'status': 'up', 'response_time': 5}) + down = _monitor(name='Down one', retries=0) + MonitorService._record(down, {'status': 'down', 'error': 'boom'}) + paused = _monitor(name='Paused one') + MonitorService.set_paused(paused.id, True) + + stats = MonitorService.stats() + assert stats['total'] == 3 + assert stats['paused'] == 1 + assert stats['operational'] == 1 + assert stats['down'] == 1 + + +# --------------------------------------------------------------------------- +# Managed-site bridge +# --------------------------------------------------------------------------- + +def test_health_sync_reaches_major_on_the_first_unhealthy_verdict(app): + """A health verdict is authoritative, not a flaky network blip, so it should + not have to wait out the retry streak.""" + monitor = MonitorService.create({ + 'name': 'Bound site', 'wordpress_site_id': 1, 'check_target': '', 'retries': 3, + }) + MonitorService.sync_component_from_health(monitor, 'unhealthy') + assert monitor.status == StatusComponent.STATUS_MAJOR + assert StatusIncident.query.filter_by(component_id=monitor.id).count() == 1 + + +def test_health_sync_ignores_unknown_verdict(app): + monitor = MonitorService.create({ + 'name': 'Bound site', 'wordpress_site_id': 1, 'check_target': '', + }) + assert MonitorService.sync_component_from_health(monitor, 'unknown') is None + assert HealthCheck.query.filter_by(component_id=monitor.id).count() == 0 + + +# --------------------------------------------------------------------------- +# Scheduler handler +# --------------------------------------------------------------------------- + +def test_monitors_do_not_depend_on_the_status_extension(): + """The whole point of the promotion: watching a site must keep working on a + lean panel with serverkit-status uninstalled. + + Walks the AST rather than grepping the source so the modules' own prose about + the extension doesn't trip it — only real imports and calls count. + """ + import ast + import inspect + from app.services import monitor_service + from app.api import monitors as monitors_api + + banned_calls = {'get_installed_extension_attr'} + banned_names = {'StatusPageService'} + + for module in (monitor_service, monitors_api): + tree = ast.parse(inspect.getsource(module)) + imported = set() + called = set() + used = set() + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom) and node.module: + imported.add(node.module) + imported.update(a.name for a in node.names) + elif isinstance(node, ast.Import): + imported.update(a.name for a in node.names) + elif isinstance(node, ast.Call) and isinstance(node.func, ast.Name): + called.add(node.func.id) + elif isinstance(node, ast.Name): + used.add(node.id) + + assert not any('plugin' in name for name in imported), \ + f'{module.__name__} imports the plugin loader: {imported}' + assert not (called & banned_calls), f'{module.__name__} calls {called & banned_calls}' + assert not (used & banned_names), f'{module.__name__} references {used & banned_names}' + + +def test_monitor_routes_are_core(app): + rules = [r.rule for r in app.url_map.iter_rules()] + assert '/api/v1/monitors/' in rules or '/api/v1/monitors' in rules + assert '/api/v1/monitors//check' in rules + assert '/api/v1/monitors/incidents' in rules + + +def test_monitor_sweep_is_registered_as_a_builtin(): + from app.jobs.builtin_handlers import _BUILTINS + kinds = {kind for kind, _fn, _name, _interval, _delay in _BUILTINS} + assert 'builtin.monitor_check' in kinds + + +def test_monitor_sweep_polls_due_monitors(app): + from app.jobs.builtin_handlers import run_monitor_checks + monitor = _monitor() + with patch('requests.request', return_value=_Resp(200)), \ + patch.object(MonitorService, '_maybe_attach_certificate'): + run_monitor_checks() + assert monitor.last_check_at is not None + assert HealthCheck.query.filter_by(component_id=monitor.id).count() == 1 + + +def test_monitor_sweep_survives_one_bad_monitor(app): + from app.jobs.builtin_handlers import run_monitor_checks + _monitor(name='First') + _monitor(name='Second') + with patch.object(MonitorService, 'run_check', + side_effect=[RuntimeError('boom'), None]) as run: + run_monitor_checks() # must not raise + assert run.call_count == 2 diff --git a/builtin-extensions/serverkit-status/backend/status_page_service.py b/builtin-extensions/serverkit-status/backend/status_page_service.py index d293a890..4b81fa28 100644 --- a/builtin-extensions/serverkit-status/backend/status_page_service.py +++ b/builtin-extensions/serverkit-status/backend/status_page_service.py @@ -1,18 +1,25 @@ +"""Public status pages. + +Scope note: this service owns *publishing* — pages, branding, grouping and the +public JSON/badge. It no longer owns the check engine. Monitors (what used to be +called "components") and their incidents are core, in +``app.services.monitor_service``, because watching a site must not depend on +having this extension installed. The component/health/incident methods below are +thin delegates kept so existing callers and the extension's own API blueprint +keep working unchanged. +""" import logging -import socket -import time -from datetime import datetime, timedelta -from app import db -from app.models.status_page import ( - StatusPage, StatusComponent, HealthCheck, StatusIncident, StatusIncidentUpdate -) + +from app.models.status_page import StatusPage, StatusComponent, StatusIncident +from app.services.monitor_service import MonitorService from app.utils.slug import validate_slug +from app import db logger = logging.getLogger(__name__) class StatusPageService: - """Service for public status pages and automated health checks.""" + """Service for public status pages.""" @staticmethod def normalize_slug(value): @@ -83,7 +90,10 @@ def get_public_page(slug): # Internal probe config must not appear on the unauthenticated public page # (a health-driven WP component may carry an internal localhost:port target). - public_hidden = ('check_type', 'check_target', 'check_interval', 'check_timeout') + public_hidden = ('check_type', 'check_target', 'check_interval', 'check_timeout', + 'check_method', 'expected_status', 'keyword', 'follow_redirects', + 'verify_tls', 'retries', 'consecutive_failures', + 'cert_issuer', 'cert_expires_at', 'next_check_at') components = page.components.all() grouped = {} for comp in components: @@ -120,290 +130,68 @@ def get_public_page(slug): 'recent_incidents': [i.to_dict() for i in resolved], } - # --- Components --- + # --- Components (monitors attached to a page) --- + # Delegates to MonitorService; a component is just a monitor with a page_id. @staticmethod def create_component(page_id, data): page = StatusPage.query.get(page_id) if not page: raise ValueError('Status page not found') + return MonitorService.create({**data, 'page_id': page_id}) - comp = StatusComponent( - page_id=page_id, - name=data['name'], - description=data.get('description', ''), - group=data.get('group', 'Services'), - sort_order=data.get('sort_order', 0), - check_type=data.get('check_type', 'http'), - check_target=data.get('check_target', ''), - check_interval=data.get('check_interval', 60), - check_timeout=data.get('check_timeout', 10), - wordpress_site_id=data.get('wordpress_site_id'), - ) - db.session.add(comp) - db.session.commit() - return comp + @staticmethod + def attach_component(page_id, monitor_id): + """Publish an existing monitor on a page. Lets an operator build a status + page out of monitors they already have instead of re-declaring probes.""" + if not StatusPage.query.get(page_id): + raise ValueError('Status page not found') + return MonitorService.update(monitor_id, {'page_id': page_id}) + + @staticmethod + def detach_component(monitor_id): + """Remove a monitor from its page without deleting the monitor.""" + return MonitorService.update(monitor_id, {'page_id': None}) @staticmethod def update_component(comp_id, data): - comp = StatusComponent.query.get(comp_id) - if not comp: - return None - for field in ['name', 'description', 'group', 'sort_order', 'check_type', - 'check_target', 'check_interval', 'check_timeout', 'status']: - if field in data: - setattr(comp, field, data[field]) - db.session.commit() - return comp + return MonitorService.update(comp_id, data) @staticmethod def delete_component(comp_id): - comp = StatusComponent.query.get(comp_id) - if not comp: - return False - # Resolve + unlink any incidents referencing this component first, so we - # never dangle the component_id FK (enforced on PostgreSQL) or leave a - # stale active incident on the public page after the component is gone. - for inc in StatusIncident.query.filter_by(component_id=comp_id).all(): - if inc.status != 'resolved': - inc.status = 'resolved' - inc.resolved_at = datetime.utcnow() - inc.component_id = None - db.session.delete(comp) - db.session.commit() - return True + return MonitorService.delete(comp_id) - # --- Health Checks --- + # --- Health Checks (delegated to the core engine) --- @staticmethod def run_check(component_id): - """Run a health check for a component.""" - comp = StatusComponent.query.get(component_id) - if not comp: - return None - - check_result = StatusPageService._perform_check(comp) - - hc = HealthCheck( - component_id=component_id, - status=check_result['status'], - response_time=check_result.get('response_time'), - status_code=check_result.get('status_code'), - error=check_result.get('error'), - ) - db.session.add(hc) - - # Update component - comp.last_check_at = datetime.utcnow() - comp.last_response_time = check_result.get('response_time') - if check_result['status'] == 'up': - comp.status = StatusComponent.STATUS_OPERATIONAL - elif check_result['status'] == 'degraded': - comp.status = StatusComponent.STATUS_DEGRADED - else: - comp.status = StatusComponent.STATUS_MAJOR - - db.session.commit() - return hc - - @staticmethod - def _perform_check(comp): - """Execute the actual health check.""" - start = time.time() - result = {'status': 'down', 'response_time': None, 'error': None} - - try: - if comp.check_type == 'http': - import requests - resp = requests.get(comp.check_target, timeout=comp.check_timeout, verify=True) - result['response_time'] = int((time.time() - start) * 1000) - result['status_code'] = resp.status_code - if resp.status_code < 400: - result['status'] = 'up' - elif resp.status_code < 500: - result['status'] = 'degraded' - else: - result['status'] = 'down' - - elif comp.check_type == 'tcp': - host, port = comp.check_target.rsplit(':', 1) - sock = socket.create_connection((host, int(port)), timeout=comp.check_timeout) - result['response_time'] = int((time.time() - start) * 1000) - result['status'] = 'up' - sock.close() - - elif comp.check_type == 'ping': - from app.utils.system import run_command - res = run_command(['ping', '-c', '1', '-W', str(comp.check_timeout), comp.check_target]) - result['response_time'] = int((time.time() - start) * 1000) - result['status'] = 'up' - - elif comp.check_type == 'dns': - socket.getaddrinfo(comp.check_target, None) - result['response_time'] = int((time.time() - start) * 1000) - result['status'] = 'up' - - except Exception as e: - result['response_time'] = int((time.time() - start) * 1000) - result['error'] = str(e) - - return result + return MonitorService.run_check(component_id) @staticmethod def get_check_history(component_id, hours=24): - since = datetime.utcnow() - timedelta(hours=hours) - return HealthCheck.query.filter( - HealthCheck.component_id == component_id, - HealthCheck.checked_at >= since - ).order_by(HealthCheck.checked_at.desc()).all() + return MonitorService.get_check_history(component_id, hours=hours) @staticmethod def recompute_uptime(comp): - """Recompute uptime_24h/7d/30d/90d for a component from its HealthCheck - rows (fraction of recorded checks with status 'up'). Only fully-healthy - checks count as up — 'degraded' periods reduce the percentage, matching - the status-page convention where degraded is not "operational". Leaves a - window's existing value untouched when it has no samples yet.""" - windows = {'uptime_24h': 24, 'uptime_7d': 24 * 7, - 'uptime_30d': 24 * 30, 'uptime_90d': 24 * 90} - now = datetime.utcnow() - for field, hours in windows.items(): - since = now - timedelta(hours=hours) - base = HealthCheck.query.filter( - HealthCheck.component_id == comp.id, - HealthCheck.checked_at >= since, - ) - total = base.count() - if total: - up = base.filter(HealthCheck.status == 'up').count() - setattr(comp, field, round(up / total * 100, 2)) - db.session.commit() - - # Map an EnvironmentHealthService overall_status to (HealthCheck status, - # StatusComponent status). 'unknown' is intentionally absent — indeterminate - # checks are not recorded so they don't pollute the uptime %. - _HEALTH_MAP = { - 'healthy': ('up', StatusComponent.STATUS_OPERATIONAL), - 'degraded': ('degraded', StatusComponent.STATUS_DEGRADED), - 'unhealthy': ('down', StatusComponent.STATUS_MAJOR), - } + return MonitorService.recompute_uptime(comp) @staticmethod def sync_component_from_health(comp, overall_status, error=None): - """Drive a managed-site-bound component from an EnvironmentHealthService - verdict instead of a network probe (#26): record a HealthCheck sample, - update the component's live status, recompute uptime, and auto-open / - auto-resolve an incident on the operational<->major_outage edge. - - Returns the recorded HealthCheck, or None for an indeterminate - ('unknown') verdict (not recorded). - """ - mapped = StatusPageService._HEALTH_MAP.get(overall_status) - if not mapped: - return None - check_status, comp_status = mapped - prev_status = comp.status + return MonitorService.sync_component_from_health(comp, overall_status, error) - hc = HealthCheck(component_id=comp.id, status=check_status, error=error) - db.session.add(hc) - comp.last_check_at = datetime.utcnow() - comp.status = comp_status - db.session.commit() - - StatusPageService.recompute_uptime(comp) - - # Open an incident when ENTERING a major outage; resolve it when LEAVING - # major (to operational OR degraded). Resolving on the leaving-edge — not - # only on a clean major->operational hop — ensures a recovery that passes - # through an intermediate degraded poll (a common path) never leaves the - # incident stuck open. Degraded itself never opens a full incident. - if comp_status == StatusComponent.STATUS_MAJOR and prev_status != StatusComponent.STATUS_MAJOR: - StatusPageService._open_incident_for_component(comp, error) - elif comp_status != StatusComponent.STATUS_MAJOR and prev_status == StatusComponent.STATUS_MAJOR: - StatusPageService._resolve_incident_for_component(comp) - return hc - - @staticmethod - def _open_incident_for_component(comp, error=None): - """Open a major-impact incident for a component if one is not already open.""" - existing = StatusIncident.query.filter( - StatusIncident.component_id == comp.id, - StatusIncident.status != 'resolved', - ).first() - if existing: - return existing - incident = StatusPageService.create_incident(comp.page_id, { - 'title': f'{comp.name} is experiencing an outage', - 'status': 'investigating', - 'impact': 'major', - 'body': error or 'Automated health check detected an outage.', - }) - incident.component_id = comp.id - db.session.commit() - return incident - - @staticmethod - def _resolve_incident_for_component(comp): - """Resolve the open auto-incident for a component, if any.""" - existing = StatusIncident.query.filter( - StatusIncident.component_id == comp.id, - StatusIncident.status != 'resolved', - ).first() - if existing: - StatusPageService.update_incident(existing.id, { - 'status': 'resolved', - 'update_body': 'Automated health check detected recovery.', - }) - - # --- Incidents --- + # --- Incidents (delegated) --- @staticmethod def create_incident(page_id, data): - incident = StatusIncident( - page_id=page_id, - title=data['title'], - status=data.get('status', 'investigating'), - impact=data.get('impact', 'minor'), - body=data.get('body', ''), - is_maintenance=data.get('is_maintenance', False), - scheduled_start=data.get('scheduled_start'), - scheduled_end=data.get('scheduled_end'), - ) - db.session.add(incident) - db.session.commit() - return incident + return MonitorService.create_incident(page_id, data) @staticmethod def update_incident(incident_id, data): - incident = StatusIncident.query.get(incident_id) - if not incident: - return None - for field in ['title', 'status', 'impact', 'body']: - if field in data: - setattr(incident, field, data[field]) - if data.get('status') == 'resolved': - incident.resolved_at = datetime.utcnow() - - # Add timeline update - if data.get('update_body'): - update = StatusIncidentUpdate( - incident_id=incident_id, - status=data.get('status', incident.status), - body=data['update_body'], - ) - db.session.add(update) - - db.session.commit() - return incident + return MonitorService.update_incident(incident_id, data) @staticmethod def delete_incident(incident_id): - incident = StatusIncident.query.get(incident_id) - if not incident: - return False - db.session.delete(incident) - db.session.commit() - return True + return MonitorService.delete_incident(incident_id) @staticmethod def get_badge(slug): diff --git a/docs/screenshots/monitoring-events.png b/docs/screenshots/monitoring-events.png new file mode 100644 index 00000000..a578d13c Binary files /dev/null and b/docs/screenshots/monitoring-events.png differ diff --git a/docs/screenshots/monitoring-incidents.png b/docs/screenshots/monitoring-incidents.png new file mode 100644 index 00000000..f5d11b6f Binary files /dev/null and b/docs/screenshots/monitoring-incidents.png differ diff --git a/docs/screenshots/monitoring-jobs.png b/docs/screenshots/monitoring-jobs.png new file mode 100644 index 00000000..d43ca9c8 Binary files /dev/null and b/docs/screenshots/monitoring-jobs.png differ diff --git a/docs/screenshots/monitoring-monitor-detail.png b/docs/screenshots/monitoring-monitor-detail.png new file mode 100644 index 00000000..8769d92d Binary files /dev/null and b/docs/screenshots/monitoring-monitor-detail.png differ diff --git a/docs/screenshots/monitoring-monitor-uptime.png b/docs/screenshots/monitoring-monitor-uptime.png new file mode 100644 index 00000000..f67c2854 Binary files /dev/null and b/docs/screenshots/monitoring-monitor-uptime.png differ diff --git a/docs/screenshots/monitoring-monitors.png b/docs/screenshots/monitoring-monitors.png new file mode 100644 index 00000000..63b753ba Binary files /dev/null and b/docs/screenshots/monitoring-monitors.png differ diff --git a/docs/screenshots/monitoring.png b/docs/screenshots/monitoring.png index 9c442122..68b9b414 100644 Binary files a/docs/screenshots/monitoring.png and b/docs/screenshots/monitoring.png differ diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index c69fa47d..0e00e676 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -44,7 +44,6 @@ import { MARKET_TABS } from './components/marketplace/marketTabs'; import { BACKUP_TABS } from './components/backups/backupTabs'; import { SECURITY_TABS } from './components/security/securityTabs'; import { ORG_TABS } from './components/organization/organizationTabs'; -import { JOBS_TABS } from './components/jobs/jobsTabs'; import Downloads from './pages/Downloads'; import SSLCertificates from './pages/SSLCertificates'; import SSOCallback from './pages/SSOCallback'; @@ -61,7 +60,6 @@ import FleetProxy from './pages/FleetProxy'; import PublicStatusPage from './pages/PublicStatusPage'; import Marketplace from './pages/Marketplace'; import Vaults from './pages/Vaults'; -import Webhooks from './pages/Webhooks'; import StyleGuide from './pages/StyleGuide'; import NotFound from './pages/NotFound'; import AppMap from './pages/AppMap'; @@ -74,7 +72,11 @@ import Notifications from './pages/Notifications'; import DeliveryLog from './pages/DeliveryLog'; import Telemetry from './pages/Telemetry'; import Jobs from './pages/Jobs'; +import Monitors from './pages/Monitors'; +import MonitorDetail from './pages/MonitorDetail'; +import Incidents from './pages/Incidents'; import TestSandbox from './pages/TestSandbox'; +import useDevMode from './hooks/useDevMode'; import useExtensionRoutes from './plugins/ExtensionRoutes'; import { useContributions } from './plugins/contributions'; @@ -129,7 +131,6 @@ const PAGE_TITLES = { '/extensions': 'Extensions', '/extensions/installed': 'Installed Extensions', '/vaults': 'Vaults', - '/webhooks': 'Webhooks', '/style-guide': 'Style Guide', '/app-map': 'App Map', '/documentation': 'Documentation', @@ -138,6 +139,10 @@ const PAGE_TITLES = { '/notifications': 'Notifications', '/admin/notifications': 'Notification Delivery Log', '/telemetry': 'Telemetry', + '/monitoring/monitors': 'Monitors', + '/monitoring/monitors/:monitorId': 'Monitor', + '/monitoring/incidents': 'Incidents', + '/monitoring/jobs': 'Jobs', '/jobs': 'Jobs', '/test-sandbox': 'Test Sandbox', }; @@ -198,6 +203,16 @@ function PageTitleUpdater() { return null; } +// Guard for developer-only pages (Test Sandbox). Reads the same shared flag as +// the sidebar's requiresCondition: 'devMode', so the nav entry and the route can +// never disagree. Off ⇒ the URL behaves as if the page doesn't exist. +function DevOnlyRoute({ children }) { + const { devMode, resolved } = useDevMode({ withStatus: true }); + if (!resolved) return ; + if (!devMode) return ; + return children; +} + function PrivateRoute({ children }) { const { isAuthenticated, loading, needsSetup, needsMigration } = useAuth(); @@ -419,10 +434,28 @@ function AppRoutes() { the public /status/:slug route stays core above. */} }> } /> + {/* Static segments outrank the :tab catch-all below, so these + keep their own pages while /monitoring/rules etc. still + land on . */} + } /> + } /> + {/* Alerts merged into Incidents — a host threshold crossing + and a monitor going down are the same question. */} + } /> + } /> + {/* Jobs moved in from its own top-level page: it is the same + class of thing as Events and read as a rival page next to + it. Activity/Scheduled is now an in-page SegControl + (the group's bar already owns the tab row). */} + } /> + } /> } /> } /> {groupRoutes.monitoring} + {/* Outside the tab group: a single monitor is a drill-down with + its own breadcrumb top bar, like /queue/:group/:queue. */} + } /> } /> {/* Fleet Monitor folded into /monitoring: it asked the same questions at fleet scope, so its heatmap is now the Host @@ -451,22 +484,24 @@ function AppRoutes() { } /> } /> } /> - } /> {/* Retired "Secrets & Webhooks" page: Vaults moved into the - Organization tab group (/vaults), Webhooks got its own page. - Redirect old links/bookmarks to their new homes. */} - } /> + Organization tab group (/vaults); the inbound-webhook console + is now a Settings → Admin tab (it's server configuration, not + a daily ops surface). Redirect old links/bookmarks. */} + } /> + } /> } /> } /> } /> - } /> + {/* Developer tooling, not an operator surface — see the + matching requiresCondition on the sidebar item. */} + } /> } /> } /> } /> - }> - } /> - } /> - + {/* Jobs is a Monitoring tab now; keep old links working. */} + } /> + } /> } /> } /> {dashboardRoutes} diff --git a/frontend/src/components/Sidebar.jsx b/frontend/src/components/Sidebar.jsx index 20769f1f..bc2448a4 100644 --- a/frontend/src/components/Sidebar.jsx +++ b/frontend/src/components/Sidebar.jsx @@ -11,6 +11,7 @@ import { SIDEBAR_CATEGORIES, CATEGORY_LABELS, SIDEBAR_PRESETS, getHiddenItemIds, import { useContributions } from '../plugins/contributions'; import { sanitizeSvgInner } from '../utils/sanitizeSvg'; import useModules from '../hooks/useModules'; +import useDevMode from '../hooks/useDevMode'; const Sidebar = ({ mobileOpen = false, isMobile = false, onMobileClose = () => {} }) => { const { user, logout, updateUser, hasPermission } = useAuth(); @@ -117,7 +118,10 @@ const Sidebar = ({ mobileOpen = false, isMobile = false, onMobileClose = () => { const { isEnabled: isModuleEnabled } = useModules(); const wordpressEnabled = isModuleEnabled('wordpress'); - const conditions = { wpInstalled, gpuAvailable, wordpressEnabled }; + // Developer-only items (Test Sandbox) — same source the route guard reads. + const devMode = useDevMode(); + + const conditions = { wpInstalled, gpuAvailable, wordpressEnabled, devMode }; const currentPreset = user?.sidebar_config?.preset || 'recommended'; const [manualExpanded, setManualExpanded] = useState({}); const [autoExpanded, setAutoExpanded] = useState(null); @@ -156,9 +160,9 @@ const Sidebar = ({ mobileOpen = false, isMobile = false, onMobileClose = () => { category: item.category || 'system', })); // Top-level items can gate on a runtime condition (e.g. GPU Monitor - // only when a GPU is present, or the Email/WordPress modules being - // enabled), mirroring sub-item requiresCondition. - const conds = { wpInstalled, gpuAvailable, wordpressEnabled }; + // only when a GPU is present, the Email/WordPress modules being + // enabled, or dev-only tooling), mirroring sub-item requiresCondition. + const conds = { wpInstalled, gpuAvailable, wordpressEnabled, devMode }; let items = [...core, ...fromPlugins].filter( (item) => !item.requiresCondition || conds[item.requiresCondition] ); @@ -194,7 +198,7 @@ const Sidebar = ({ mobileOpen = false, isMobile = false, onMobileClose = () => { } } return applyWorkspaceNavPermissions(items, activeWorkspace, user); - }, [user?.sidebar_config, pluginNav, pluginTabs, wpInstalled, gpuAvailable, wordpressEnabled, user, hasPermission]); + }, [user?.sidebar_config, pluginNav, pluginTabs, wpInstalled, gpuAvailable, wordpressEnabled, devMode, user, hasPermission]); // Group visible items by category const groupedItems = useMemo(() => { diff --git a/frontend/src/components/backups/BackupDetailDrawer.jsx b/frontend/src/components/backups/BackupDetailDrawer.jsx index af34e4c0..bb3350b6 100644 --- a/frontend/src/components/backups/BackupDetailDrawer.jsx +++ b/frontend/src/components/backups/BackupDetailDrawer.jsx @@ -77,7 +77,7 @@ export default function BackupDetailDrawer({ run, open, onClose, onRestore, onVe {run.job_id && ( - + View job )} diff --git a/frontend/src/components/backups/ProtectionPanel.jsx b/frontend/src/components/backups/ProtectionPanel.jsx index cc659dab..0176ceea 100644 --- a/frontend/src/components/backups/ProtectionPanel.jsx +++ b/frontend/src/components/backups/ProtectionPanel.jsx @@ -193,7 +193,7 @@ export default function ProtectionPanel({ targetType, targetId, targetName, show onBackupNow={handleBackupNow} onRunDrill={handleRunDrill} onViewGlobal={() => navigate('/backups')} - onViewJobs={() => navigate('/jobs')} + onViewJobs={() => navigate('/monitoring/jobs')} busy={saving} backingUp={backingUp} drilling={drilling || !!view?.restore_proof?.is_drilling} diff --git a/frontend/src/components/dashboard/widgets/renderers.jsx b/frontend/src/components/dashboard/widgets/renderers.jsx index 3812b0dc..480ea9f4 100644 --- a/frontend/src/components/dashboard/widgets/renderers.jsx +++ b/frontend/src/components/dashboard/widgets/renderers.jsx @@ -779,7 +779,7 @@ const ACTIONS = { domains: ['Domains', Globe, '/domains'], files: ['File manager', FolderOpen, '/files'], security: ['Security', ShieldCheck, '/security'], - jobs: ['Jobs', ListChecks, '/jobs'], + jobs: ['Jobs', ListChecks, '/monitoring/jobs'], }; export const QUICK_ACTION_KEYS = Object.keys(ACTIONS); diff --git a/frontend/src/components/ds/FilterDrawer.jsx b/frontend/src/components/ds/FilterDrawer.jsx index 76856695..c6a10831 100644 --- a/frontend/src/components/ds/FilterDrawer.jsx +++ b/frontend/src/components/ds/FilterDrawer.jsx @@ -80,6 +80,12 @@ export function FilterDrawer({ // legitimately not count a group whose value is just its default (a sort // order sitting on "featured" is not a filter the user applied). activeCount, + // Optional: extra fields rendered below the chip groups, for filters that + // aren't a pick-from-a-list (free text, date bounds). A host that passes + // these must also pass `onClear`, otherwise "Clear all" would reset only the + // groups and silently leave the extra fields applied. + children, + onClear, }) { const isOn = (group, optValue) => ( group.type === 'multi' @@ -101,7 +107,7 @@ export function FilterDrawer({ }; const active = activeCount ?? countActiveFilters(value); - const clearAll = () => onChange(emptyFilterValue(groups)); + const clearAll = () => (onClear ? onClear() : onChange(emptyFilterValue(groups))); return ( ))} + {children &&
{children}
}
+
+

+ Everything below describes the machines ServerKit runs on. Add a monitor to + watch a website, an API endpoint, a database port or a WordPress site and get + an incident when it stops answering. +

+ + ); + } + + const responseTimes = monitors + .filter((m) => m.last_response_time != null && !m.is_paused) + .map((m) => m.last_response_time); + const avgResponse = responseTimes.length + ? Math.round(responseTimes.reduce((a, b) => a + b, 0) / responseTimes.length) + : null; + // The busiest monitor's recent latency stands in for "how are we doing" — + // averaging series of different lengths across monitors would be a lie. + const leadSpark = monitors.reduce( + (best, m) => ((m.spark?.length || 0) > (best?.spark?.length || 0) ? m : best), + null, + ); + + return ( +
+ + } + value={stats?.overall_uptime_30d != null ? `${stats.overall_uptime_30d}%` : '—'} + onClick={() => navigate('/monitoring/monitors')} + /> + } + value={avgResponse ?? '—'} unit={avgResponse != null ? 'ms' : undefined} + spark={leadSpark?.spark?.length > 1 ? ( + + ) : undefined} + /> + } + value={stats?.down ?? 0} + onClick={() => navigate('/monitoring/monitors')} + /> + } + value={stats?.degraded ?? 0} + onClick={() => navigate('/monitoring/monitors')} + /> + + +
+
+
+
+

Monitors

+ {monitors.length} watched +
+ +
+
+ {monitors.slice(0, 8).map((monitor) => { + const state = monitorStateOf(monitor); + return ( + + + {monitor.name} + + {monitor.check_type} · {monitor.check_target || 'bound site'} + + + + {monitor.last_response_time == null ? '—' : `${monitor.last_response_time} ms`} + + {state.label} + + ); + })} +
+
+ +
+
+
+

Live checks

+ Most recent results +
+
+ {feed.length === 0 ? ( +

+ No checks recorded yet — the scheduler polls on each monitor's interval. +

+ ) : ( +
+ {feed.map((monitor) => { + const state = monitorStateOf(monitor); + const ok = state.key === 'up'; + return ( + + + {ok ? 'ok' : state.label.toLowerCase()} + + + {monitor.name} + + {relativeTime(monitor.last_check_at)} + + + + {monitor.last_response_time == null ? 'fail' : `${monitor.last_response_time}ms`} + + + ); + })} +
+ )} +
+
+
+ ); +} diff --git a/frontend/src/components/monitoring/UptimeBars.jsx b/frontend/src/components/monitoring/UptimeBars.jsx new file mode 100644 index 00000000..468e01e7 --- /dev/null +++ b/frontend/src/components/monitoring/UptimeBars.jsx @@ -0,0 +1,40 @@ +// 90-day uptime strip: one bar per day, from the design mock. +// +// A day with no samples renders as its own "unwatched" state rather than a full +// green bar — "we weren't looking" and "it was fine" are different facts, and +// conflating them is how uptime widgets end up lying about their own history. +import { cn } from '@/lib/utils'; + +function tooltipFor(day) { + if (day.state === 'none') return `${day.date} — not monitored`; + const pct = day.uptime != null ? `${day.uptime.toFixed(3)}%` : '—'; + return `${day.date} — ${pct} uptime, ${day.checks} check${day.checks === 1 ? '' : 's'}` + + (day.down_checks ? `, ${day.down_checks} failed` : ''); +} + +export default function UptimeBars({ days = [], selected, onSelect, className }) { + if (!days.length) return null; + return ( +
+ {days.map((day) => ( + // A bar in a chart is not a
+ ); +} diff --git a/frontend/src/components/monitoring/monitorShared.jsx b/frontend/src/components/monitoring/monitorShared.jsx new file mode 100644 index 00000000..436b2f68 --- /dev/null +++ b/frontend/src/components/monitoring/monitorShared.jsx @@ -0,0 +1,50 @@ +// Shared vocabulary for the monitor surfaces (list, detail, overview, incidents). +// Keeping the status mapping in one place stops the list and the detail page +// disagreeing about what "degraded" looks like. + +export const CHECK_TYPES = [ + { value: 'http', label: 'HTTP', hint: 'status code of a URL' }, + { value: 'keyword', label: 'Keyword', hint: 'URL must contain a phrase' }, + { value: 'tcp', label: 'Port', hint: 'host:port accepts a connection' }, + { value: 'ping', label: 'Ping', hint: 'host answers ICMP' }, + { value: 'dns', label: 'DNS', hint: 'name resolves' }, + { value: 'smtp', label: 'SMTP', hint: 'mail server greets' }, +]; + +export const MONITOR_STATUS = [ + { value: 'operational', label: 'Operational' }, + { value: 'degraded', label: 'Degraded' }, + { value: 'major_outage', label: 'Down' }, + { value: 'maintenance', label: 'Maintenance' }, + { value: 'paused', label: 'Paused' }, +]; + +// Paused is deliberately not a `status` on the model — it is its own flag, so a +// paused monitor keeps the last thing it knew rather than looking like an +// outage. Resolve it here, once. +export function monitorStateOf(monitor) { + if (!monitor) return { key: 'unknown', label: 'Unknown', tone: 'gray' }; + if (monitor.is_paused) return { key: 'paused', label: 'Paused', tone: 'gray' }; + switch (monitor.status) { + case 'operational': + return { key: 'up', label: 'Operational', tone: 'green' }; + case 'degraded': + case 'partial_outage': + return { key: 'degraded', label: 'Degraded', tone: 'amber' }; + case 'major_outage': + return { key: 'down', label: 'Down', tone: 'red' }; + case 'maintenance': + return { key: 'maintenance', label: 'Maintenance', tone: 'cyan' }; + default: + return { key: 'unknown', label: monitor.status || 'Unknown', tone: 'gray' }; + } +} + +export const INCIDENT_STATES = ['investigating', 'identified', 'monitoring', 'resolved']; + +export const IMPACT_TONE = { + critical: 'red', + major: 'red', + minor: 'amber', + none: 'gray', +}; diff --git a/frontend/src/components/monitoring/monitorTabs.jsx b/frontend/src/components/monitoring/monitorTabs.jsx index 05729202..e85978f8 100644 --- a/frontend/src/components/monitoring/monitorTabs.jsx +++ b/frontend/src/components/monitoring/monitorTabs.jsx @@ -1,23 +1,38 @@ -import { Activity, Bell, Gauge, ScrollText, SlidersHorizontal, Stethoscope } from 'lucide-react'; +import { + Activity, AlertTriangle, Gauge, ListChecks, Radar, ScrollText, + SlidersHorizontal, Stethoscope, +} from 'lucide-react'; -// Shared sub-nav for the Observability page group. The Monitoring page's own +// Shared sub-nav for the Monitoring page group. The Monitoring page's own // sections live HERE, in the group's one top bar, rather than in a second tab // strip inside the page — a nav under a nav reads as two competing headers // (and is what the design mock does: its tabs sit in the top bar). // // Which host each section describes is a separate axis, carried by the top-bar -// server picker as `?server=` — so Overview/Alerts/Capacity all follow the same -// scope. Fleet Monitor used to be a page of its own under Servers; its heatmap -// is now the Host health grid on Overview, and its analytics are Capacity. +// server picker as `?server=` — so Overview/Capacity all follow the same scope. +// Fleet Monitor used to be a page of its own under Servers; its heatmap is now +// the Host health grid on Overview, and its analytics are Capacity. +// +// Monitors is the synthetic-check list (watch a URL, a service, a WordPress +// site). Incidents absorbed the old Alerts tab: a CPU threshold crossing on a +// host and a monitor going down are both "something is wrong right now", and +// splitting them was what made alerting look like it only ever described the +// panel's own machine. Host threshold alerting is unchanged, just rendered in +// the same timeline. +// +// Events is the telemetry stream and Jobs the unified job runner — both used to +// be top-level pages that read as rivals to this group. // // The Status Pages tab is contributed by the serverkit-status builtin extension // (tab-group contribution, #43) and merged in by TabGroupLayout -// groupId="monitoring". +// groupId="monitoring". TabGroupLayout overflows the tail into "⋯ More". export const MONITOR_TABS = [ { to: '/monitoring', label: 'Overview', end: true, icon: }, - { to: '/monitoring/alerts', label: 'Alerts', icon: }, + { to: '/monitoring/monitors', label: 'Monitors', icon: }, + { to: '/monitoring/incidents', label: 'Incidents', icon: }, { to: '/monitoring/rules', label: 'Rules', icon: }, { to: '/monitoring/capacity', label: 'Capacity', icon: }, - { to: '/monitoring/doctor', label: 'Doctor', icon: }, { to: '/telemetry', label: 'Events', icon: }, + { to: '/monitoring/jobs', label: 'Jobs', icon: }, + { to: '/monitoring/doctor', label: 'Doctor', icon: }, ]; diff --git a/frontend/src/components/organization/organizationTabs.jsx b/frontend/src/components/organization/organizationTabs.jsx index 4483b1fd..f57adfc9 100644 --- a/frontend/src/components/organization/organizationTabs.jsx +++ b/frontend/src/components/organization/organizationTabs.jsx @@ -9,7 +9,7 @@ import { FolderKanban, Braces, KeyRound, LayoutGrid } from 'lucide-react'; // // Vaults (encrypted secret stores) sits beside Shared Variables here because // both are key/value config surfaces; inbound Webhooks — the other half of the -// retired "Secrets & Webhooks" page — now lives on its own /webhooks page. +// retired "Secrets & Webhooks" page — now lives at Settings → Admin → Webhooks. export const ORG_TABS = [ { to: '/projects', label: 'Projects', end: true, icon: }, { to: '/shared-variables', label: 'Shared Variables', icon: }, diff --git a/frontend/src/pages/Webhooks.jsx b/frontend/src/components/settings/WebhooksTab.jsx similarity index 92% rename from frontend/src/pages/Webhooks.jsx rename to frontend/src/components/settings/WebhooksTab.jsx index 28f672a3..0448b6d3 100644 --- a/frontend/src/pages/Webhooks.jsx +++ b/frontend/src/components/settings/WebhooksTab.jsx @@ -1,6 +1,5 @@ import { useEffect, useMemo, useState } from 'react'; -import api from '../services/api'; -import { PageTopbar } from '@/components/ds'; +import api from '../../services/api'; import { Card, CardContent, CardHeader, CardTitle, CardDescription } from '@/components/ui/card'; import { Button } from '@/components/ui/button'; import { Input } from '@/components/ui/input'; @@ -10,20 +9,24 @@ import { Badge } from '@/components/ui/badge'; import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogDescription, DialogFooter } from '@/components/ui/dialog'; import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@/components/ui/table'; import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuTrigger } from '@/components/ui/dropdown-menu'; -import { useToast } from '../contexts/ToastContext'; -import { Webhook, Plus, MoreVertical, Copy, RefreshCw, ArrowRightLeft } from 'lucide-react'; -import EmptyState from '../components/EmptyState'; +import { useToast } from '../../contexts/ToastContext'; +import { useConfirm } from '../../hooks/useConfirm'; +import { Plus, MoreVertical, Copy, RefreshCw, ArrowRightLeft } from 'lucide-react'; +import EmptyState from '../EmptyState'; const formatDate = (d) => (d ? new Date(d).toLocaleString() : '—'); /** - * Webhooks — receive, verify, and forward inbound webhooks. Split out of the - * old "Secrets & Webhooks" page: secret storage now lives under the - * Organization tab group (/vaults). This is a standalone page with its own - * PageTopbar. + * Webhooks — receive, verify, and forward inbound webhooks. + * + * Lives as a Settings → Admin tab rather than a top-level page: it's server + * configuration, not a daily operations surface, and it sits next to the + * outbound notification subscriptions it complements. /webhooks redirects here. + * (Secret storage moved to the Organization tab group at /vaults earlier.) */ -export default function Webhooks() { +export default function WebhooksTab() { const toast = useToast(); + const { confirm } = useConfirm(); const [endpoints, setEndpoints] = useState([]); const [loading, setLoading] = useState(true); @@ -68,7 +71,11 @@ export default function Webhooks() { } async function deleteEndpoint(id) { - if (!confirm('Delete this webhook endpoint?')) return; + const confirmed = await confirm({ + title: 'Delete Webhook Endpoint', + message: 'Delete this webhook endpoint? Inbound deliveries to its URL will stop being accepted.', + }); + if (!confirmed) return; try { await api.deleteWebhookEndpoint(id); if (selectedEndpoint?.id === id) setSelectedEndpoint(null); @@ -118,18 +125,11 @@ export default function Webhooks() { }, [selectedEndpoint]); if (loading) { - return ( -
- } title="Webhooks" /> - -
- ); + return ; } return ( -
- } title="Webhooks" /> - +
{!selectedEndpoint ? ( diff --git a/frontend/src/components/sidebarItems.js b/frontend/src/components/sidebarItems.js index 348b5700..7028d16c 100644 --- a/frontend/src/components/sidebarItems.js +++ b/frontend/src/components/sidebarItems.js @@ -126,11 +126,12 @@ export const SIDEBAR_ITEMS = [ }, { // Redesign: Monitoring uses the top-bar layout (REDESIGN_MAP §6 dec. 3). - // Its sections (Overview / Alerts / Rules / Capacity / Doctor / Events) - // share the top bar (PageTopbar MONITOR_TABS); the sidebar entry lights - // for any of them via matchPrefixes. Events absorbed the old standalone - // Telemetry. Labelled "Monitoring" to match the route and the page — - // this used to say "Observability", which named the same thing twice. + // Its sections (Overview / Monitors / Incidents / Rules / Capacity / + // Events / Jobs / Doctor) share the top bar (PageTopbar MONITOR_TABS); + // the sidebar entry lights for any of them via matchPrefixes. Events + // absorbed the old standalone Telemetry and Jobs the old top-level Jobs + // page. Labelled "Monitoring" to match the route and the page — this + // used to say "Observability", which named the same thing twice. id: 'monitoring', label: 'Monitoring', route: '/monitoring', @@ -175,32 +176,27 @@ export const SIDEBAR_ITEMS = [ category: 'system', icon: '' }, - { - id: 'jobs', - label: 'Jobs', - route: '/jobs', - category: 'system', - icon: '' - }, + // Jobs is a Monitoring tab now (/monitoring/jobs), not a top-level page: it + // is the same class of thing as Events and the two read as rival pages when + // one sat in the sidebar and the other inside the group. /jobs redirects. // GPU Monitor lives in the standalone serverkit-gpu extension (own repo, // registry-installed); its sidebar item (still gated on gpuAvailable via // requiresCondition) is contributed by the extension manifest. - { - // Inbound webhook console. Secret storage ("Vaults") that used to share - // this page now lives under the Organization tab group (/vaults); only - // the receive/verify/forward half remains here on its own page. - id: 'webhooks', - label: 'Webhooks', - route: '/webhooks', - category: 'system', - icon: '' - }, + // The inbound-webhook console is no longer a sidebar page: it's server + // configuration, so it lives at Settings → Admin → Webhooks alongside the + // outbound notification subscriptions. /webhooks redirects there. { // Admin distro-matrix test console (Docker-backed runs, per-distro // logs). Sits next to Jobs under System. + // + // Developer tooling, not an operator surface: hidden unless this is a + // local `npm run dev` build or an admin has turned on Site Settings → + // Developer mode. The route is gated on the same condition, so a + // production install neither shows nor serves it. id: 'test-sandbox', label: 'Test Sandbox', route: '/test-sandbox', + requiresCondition: 'devMode', category: 'system', icon: '' }, @@ -221,12 +217,13 @@ export const SIDEBAR_ITEMS = [ ]; // "Advanced" items are powerful but not part of the everyday core for a solo -// dev / small team: the internal job-queue console and the inbound-Webhooks -// console. They're hidden by the default ("Recommended") view and every curated -// preset, but stay one click away via the "Full" view or Customize Sidebar — and -// remain fully routable (deep links, command palette). The Marketplace is NOT in -// this list — it's alwaysVisible so extensions are always discoverable. -export const ADVANCED_ITEM_IDS = ['queue', 'webhooks']; +// dev / small team — currently the internal job-queue console. They're hidden by +// the default ("Recommended") view and every curated preset, but stay one click +// away via the "Full" view or Customize Sidebar — and remain fully routable +// (deep links, command palette). The Marketplace is NOT in this list — it's +// alwaysVisible so extensions are always discoverable. ("webhooks" left this +// list when the console moved into Settings → Admin.) +export const ADVANCED_ITEM_IDS = ['queue']; // Preset profiles define which items are hidden (top-level only) export const SIDEBAR_PRESETS = { diff --git a/frontend/src/data/palettePages.js b/frontend/src/data/palettePages.js index 036d7c7b..2bbb47f6 100644 --- a/frontend/src/data/palettePages.js +++ b/frontend/src/data/palettePages.js @@ -38,8 +38,12 @@ export const PALETTE_PAGES = [ { label: 'Workspaces', path: '/workspaces', navId: 'organization', keywords: 'organization team' }, { label: 'Vaults', path: '/vaults', navId: 'organization', keywords: 'secrets tokens credentials vault' }, { label: 'Queue Bus', path: '/queue', navId: 'queue', keywords: 'bus operations tasks' }, - { label: 'Jobs', path: '/jobs', navId: 'jobs', keywords: 'scheduler background work' }, - { label: 'Webhooks', path: '/webhooks', navId: 'webhooks', keywords: 'webhook receive forward delivery' }, + { label: 'Jobs', path: '/monitoring/jobs', navId: 'monitoring', keywords: 'scheduler background work' }, + { label: 'Monitors', path: '/monitoring/monitors', navId: 'monitoring', keywords: 'uptime check probe http ping website' }, + { label: 'Incidents', path: '/monitoring/incidents', navId: 'monitoring', keywords: 'outage alert downtime postmortem' }, + // No navId: the inbound-webhook console is a Settings tab now, not a + // sidebar page, so there's no nav item for the permission gate to consult. + { label: 'Webhooks', path: '/settings/webhooks', keywords: 'webhook receive forward delivery inbound' }, ].map((p) => ({ ...p, id: `page:${p.path}`, category: 'Pages' })); export default PALETTE_PAGES; diff --git a/frontend/src/data/settingsIndex.js b/frontend/src/data/settingsIndex.js index ef5689dd..45ed9d62 100644 --- a/frontend/src/data/settingsIndex.js +++ b/frontend/src/data/settingsIndex.js @@ -90,6 +90,13 @@ export const SETTINGS_INDEX = [ { id: 'api-webhooks', label: 'Webhook subscriptions', description: 'Create and manage webhooks for event notifications', keywords: 'webhook events notifications', tab: 'api', adminOnly: false }, { id: 'api-analytics', label: 'API usage analytics', description: 'Monitor API traffic, response times, and errors', keywords: 'api analytics traffic usage endpoints', tab: 'api', adminOnly: true }, + // Inbound webhooks (the receive/verify/forward console, moved here from the + // retired top-level /webhooks page). Distinct from `api-webhooks` above, + // which is the OUTBOUND event-subscription surface. + { id: 'webhooks-endpoints', label: 'Webhook endpoints', description: 'Create inbound webhook endpoints with a slug, signing secret, and optional forward URL', keywords: 'webhook inbound endpoint receive slug secret forward', tab: 'webhooks', adminOnly: true }, + { id: 'webhooks-deliveries', label: 'Webhook deliveries', description: 'Inspect received webhook deliveries, signature validity, and replay them', keywords: 'webhook delivery replay signature payload history', tab: 'webhooks', adminOnly: true }, + { id: 'webhooks-secret', label: 'Regenerate webhook secret', description: 'Rotate the signing secret used to verify an inbound webhook endpoint', keywords: 'webhook secret rotate regenerate signature hmac', tab: 'webhooks', adminOnly: true }, + { id: 'ai-enable', label: 'Enable AI assistant', description: 'Toggle the in-panel AI assistant on or off', keywords: 'ai assistant enable disable', tab: 'ai', adminOnly: true }, { id: 'ai-provider', label: 'AI provider', description: 'Select the AI service provider (OpenAI, Anthropic, Ollama, etc.)', keywords: 'ai provider openai anthropic ollama', tab: 'ai', adminOnly: true }, { id: 'ai-model', label: 'AI model', description: 'Choose the language model to use', keywords: 'model gpt claude ollama', tab: 'ai', adminOnly: true }, diff --git a/frontend/src/hooks/useDevMode.js b/frontend/src/hooks/useDevMode.js new file mode 100644 index 00000000..a1d98d9c --- /dev/null +++ b/frontend/src/hooks/useDevMode.js @@ -0,0 +1,71 @@ +import { useState, useEffect } from 'react'; +import api from '../services/api'; +import { useAuth } from '../contexts/AuthContext'; + +// Shared, app-wide "developer mode" flag — the gate for developer-only surfaces +// (currently the Test Sandbox). True when either: +// +// * this is a local `npm run dev` build (import.meta.env.DEV), or +// * an admin has turned on Site Settings → Developer mode (dev_mode). +// +// Cached at module scope like useModules() so the sidebar item and the route +// guard read one value and can never disagree — a visible nav entry that +// redirects to the dashboard is worse than no entry at all. +// +// /admin/settings is admin-gated, so only admins are asked; everyone else stays +// on the `false` default rather than eating a 403 per session. + +let cache = null; // last-fetched dev_mode boolean +let inflight = null; // de-dupes concurrent initial fetches +const listeners = new Set(); + +function notify() { + for (const listener of listeners) listener(cache); +} + +async function fetchDevMode(force = false) { + if (cache !== null && !force) return cache; + if (inflight && !force) return inflight; + inflight = api.getSystemSettings() + .then((data) => { + cache = !!data?.dev_mode; + notify(); + return cache; + }) + .catch(() => { + cache = cache ?? false; + return cache; + }) + .finally(() => { inflight = null; }); + return inflight; +} + +// Let Settings broadcast a toggle without a reload. +export function refreshDevMode() { + return fetchDevMode(true); +} + +// Returns the resolved flag. `resolved` is false only while an admin's first +// dev_mode fetch is still in flight — route guards must wait for it, or a +// deep link into a dev-only page would bounce before the answer arrives. +export function useDevMode({ withStatus = false } = {}) { + const { isAdmin } = useAuth(); + const [systemDevMode, setSystemDevMode] = useState(cache ?? false); + const [loaded, setLoaded] = useState(cache !== null); + + useEffect(() => { + if (!isAdmin) return undefined; + const listener = (v) => { setSystemDevMode(!!v); setLoaded(true); }; + listeners.add(listener); + fetchDevMode().then(() => setLoaded(true)); + return () => { listeners.delete(listener); }; + }, [isAdmin]); + + // A local dev build short-circuits: nothing to wait for. + const devMode = import.meta.env.DEV || (isAdmin && systemDevMode); + const resolved = import.meta.env.DEV || !isAdmin || loaded; + + return withStatus ? { devMode, resolved } : devMode; +} + +export default useDevMode; diff --git a/frontend/src/pages/CronJobs.jsx b/frontend/src/pages/CronJobs.jsx index 355dcb48..a9b5f177 100644 --- a/frontend/src/pages/CronJobs.jsx +++ b/frontend/src/pages/CronJobs.jsx @@ -1,22 +1,83 @@ -import { useState, useEffect } from 'react'; +import { useState, useEffect, useCallback } from 'react'; import api from '../services/api'; import { useToast } from '../contexts/ToastContext'; +import { useAuth } from '../contexts/AuthContext'; import { useConfirm } from '../hooks/useConfirm'; import EmptyState from '../components/EmptyState'; import Modal from '@/components/Modal'; import { - Clock, CheckCircle, Monitor, Activity, Plus, RefreshCw, - Play, Pause, Pencil, Trash2, Search, + Clock, Plus, RefreshCw, Play, Pause, Trash2, Pencil, } from 'lucide-react'; import { Button } from '@/components/ui/button'; import { Input } from '@/components/ui/input'; import { Label } from '@/components/ui/label'; import { Textarea } from '@/components/ui/textarea'; import { Checkbox } from '@/components/ui/checkbox'; -import { MetricCard, Pill, SegControl, PageTopbar } from '@/components/ds'; +import { Switch } from '@/components/ui/switch'; +import { Pill, SegControl, SearchField, PageTopbar, DataTable, Drawer } from '@/components/ds'; + +// Fallback humanizer for the handful of schedules the backend can't describe. +const SCHEDULE_LABELS = { + '* * * * *': 'Every minute', + '0 * * * *': 'Every hour', + '0 0 * * *': 'Daily at midnight', + '0 0 * * 0': 'Weekly on Sunday', + '0 0 1 * *': 'Monthly on the 1st', + '0 0 * * 1-5': 'Weekdays at midnight', + '0 */6 * * *': 'Every 6 hours', + '0 */12 * * *': 'Every 12 hours', + '*/5 * * * *': 'Every 5 minutes', + '*/15 * * * *': 'Every 15 minutes', + '*/30 * * * *': 'Every 30 minutes', +}; + +const describeSchedule = (job) => ( + job.schedule_human || SCHEDULE_LABELS[job.schedule] || job.schedule +); + +// The browser's zone, not the host's — times are ISO strings rendered client +// side, so this is what the reader is actually seeing. Say so rather than +// implying the schedule's own timezone. +const VIEWER_TZ = Intl.DateTimeFormat().resolvedOptions().timeZone || 'local time'; + +function formatWhen(iso) { + if (!iso) return '—'; + const d = new Date(iso); + if (Number.isNaN(d.getTime())) return '—'; + const diffMs = Date.now() - d.getTime(); + const mins = Math.round(Math.abs(diffMs) / 60000); + const ago = diffMs >= 0; + let rel; + if (mins < 1) rel = 'just now'; + else if (mins < 60) rel = `${mins}m`; + else if (mins < 1440) rel = `${Math.round(mins / 60)}h`; + else rel = `${Math.round(mins / 1440)}d`; + if (rel === 'just now') return rel; + return ago ? `${rel} ago` : `in ${rel}`; +} + +function formatDuration(seconds) { + if (seconds == null) return '—'; + if (seconds < 10) return `${Number(seconds).toFixed(1)}s`; + if (seconds < 60) return `${Math.round(seconds)}s`; + const m = Math.floor(seconds / 60); + const s = Math.round(seconds % 60); + return `${m}m ${String(s).padStart(2, '0')}s`; +} + +// One status per job, folding "is it on?" and "did it last succeed?" together — +// the two questions the list exists to answer. +function jobState(job) { + if (!job.enabled) return { key: 'paused', kind: 'gray', label: 'Paused' }; + if (job.last_status === 'failure') return { key: 'failed', kind: 'red', label: 'Failed' }; + if (job.last_status === 'success') return { key: 'active', kind: 'green', label: 'Healthy' }; + // Enabled but nothing recorded: either never fired yet, or not tracked. + return { key: 'active', kind: 'cyan', label: job.tracked ? 'No runs yet' : 'Untracked' }; +} const CronJobs = () => { const toast = useToast(); + const { isAdmin } = useAuth(); const { confirm } = useConfirm(); const [status, setStatus] = useState(null); const [jobs, setJobs] = useState([]); @@ -26,9 +87,10 @@ const CronJobs = () => { // List filters (client-side, over already-loaded jobs) const [filter, setFilter] = useState('all'); - const [query, setQuery] = useState(''); + const [search, setSearch] = useState(''); - // Modal states + // Drawer + modal states + const [drawerJob, setDrawerJob] = useState(null); const [showJobModal, setShowJobModal] = useState(false); const [editingJob, setEditingJob] = useState(null); const [runningJobId, setRunningJobId] = useState(null); @@ -41,22 +103,17 @@ const CronJobs = () => { schedule: '', description: '', usePreset: true, - preset: 'daily' + preset: 'daily', }); - useEffect(() => { - loadData(); - }, []); - - const loadData = async () => { + const loadData = useCallback(async () => { try { setLoading(true); const [statusRes, jobsRes, presetsRes] = await Promise.all([ api.getCronStatus(), api.getCronJobs(), - api.getCronPresets() + api.getCronPresets(), ]); - setStatus(statusRes); setJobs(jobsRes.jobs || []); setPresets(presetsRes.presets || {}); @@ -65,6 +122,21 @@ const CronJobs = () => { } finally { setLoading(false); } + }, []); + + useEffect(() => { + loadData(); + }, [loadData]); + + const resetForm = () => { + setJobForm({ + name: '', + command: '', + schedule: '', + description: '', + usePreset: true, + preset: 'daily', + }); }; const openCreateModal = () => { @@ -82,7 +154,7 @@ const CronJobs = () => { schedule: job.schedule || '', description: job.description || '', usePreset: !!presetKey, - preset: presetKey || 'daily' + preset: presetKey || 'daily', }); setShowJobModal(true); }; @@ -96,15 +168,12 @@ const CronJobs = () => { const handleSubmitJob = async (e) => { e.preventDefault(); try { - const schedule = jobForm.usePreset - ? presets[jobForm.preset] - : jobForm.schedule; - + const schedule = jobForm.usePreset ? presets[jobForm.preset] : jobForm.schedule; const payload = { name: jobForm.name, command: jobForm.command, - schedule: schedule, - description: jobForm.description + schedule, + description: jobForm.description, }; if (editingJob) { @@ -116,6 +185,7 @@ const CronJobs = () => { } closeJobModal(); + setDrawerJob(null); loadData(); } catch (err) { toast.error(err.message); @@ -123,11 +193,15 @@ const CronJobs = () => { }; const handleDeleteJob = async (jobId) => { - const confirmed = await confirm({ title: 'Delete Cron Job', message: 'Are you sure you want to delete this cron job?' }); + const confirmed = await confirm({ + title: 'Delete Cron Job', + message: 'Are you sure you want to delete this cron job?', + }); if (!confirmed) return; try { await api.deleteCronJob(jobId); toast.success('Cron job deleted'); + setDrawerJob(null); loadData(); } catch (err) { toast.error(err.message); @@ -154,8 +228,9 @@ const CronJobs = () => { jobName: jobs.find(j => j.id === jobId)?.name || jobId, exitCode: result.exit_code, stdout: result.stdout, - stderr: result.stderr + stderr: result.stderr, }); + loadData(); } else { toast.error(result.error || 'Job execution failed'); } @@ -166,56 +241,35 @@ const CronJobs = () => { } }; - const resetForm = () => { - setJobForm({ - name: '', - command: '', - schedule: '', - description: '', - usePreset: true, - preset: 'daily' - }); - }; - - const getScheduleDescription = (schedule) => { - const descriptions = { - '* * * * *': 'Every minute', - '0 * * * *': 'Every hour', - '0 0 * * *': 'Daily at midnight', - '0 0 * * 0': 'Weekly on Sunday', - '0 0 1 * *': 'Monthly on the 1st', - '0 0 * * 1-5': 'Weekdays at midnight', - '0 */6 * * *': 'Every 6 hours', - '0 */12 * * *': 'Every 12 hours', - '*/5 * * * *': 'Every 5 minutes', - '*/15 * * * *': 'Every 15 minutes', - '*/30 * * * *': 'Every 30 minutes' - }; - return descriptions[schedule] || schedule; - }; - const enabledCount = jobs.filter(j => j.enabled).length; - const disabledCount = jobs.length - enabledCount; - - const serviceSub = status?.type === 'cron' - ? (status?.running ? 'daemon running' : 'daemon stopped') - : (status?.type === 'serverkit_scheduler' ? 'internal scheduler' : null); - - const q = query.trim().toLowerCase(); - const shownJobs = jobs.filter(job => ( - (filter === 'all' || (filter === 'enabled' ? job.enabled : !job.enabled)) - && (!q + const failedCount = jobs.filter(j => j.enabled && j.last_status === 'failure').length; + const pausedCount = jobs.length - enabledCount; + + const q = search.trim().toLowerCase(); + const shown = jobs.filter((job) => { + const state = jobState(job); + const matchesFilter = filter === 'all' + || (filter === 'failed' ? state.key === 'failed' + : filter === 'paused' ? state.key === 'paused' + : state.key === 'active'); + const matchesQuery = !q || (job.name || '').toLowerCase().includes(q) - || (job.command || '').toLowerCase().includes(q)) - )); + || (job.command || '').toLowerCase().includes(q); + return matchesFilter && matchesQuery; + }); - if (loading) { - return ( -
- -
- ); - } + const filterOptions = [ + { value: 'all', label: 'All', count: jobs.length }, + { value: 'active', label: 'Active', count: enabledCount - failedCount }, + { value: 'failed', label: 'Failed', count: failedCount }, + { value: 'paused', label: 'Paused', count: pausedCount }, + ]; + + const serviceNote = status?.available === false + ? 'Cron service unavailable' + : status?.type === 'cron' + ? (status?.running ? null : 'cron daemon stopped') + : (status?.type === 'serverkit_scheduler' ? 'internal scheduler' : null); return (
@@ -224,302 +278,530 @@ const CronJobs = () => { title="Cron Jobs" actions={( <> - - + )} /> {error && ( -
+
{error} - +
)} - {/* KPI strip */} -
- } value={jobs.length} label="Cron jobs"> -
{enabledCount} enabled
- - } value={enabledCount} label="Active jobs"> - {disabledCount > 0 && ( -
{disabledCount} disabled
- )} - - } - value={status?.available ? 'Available' : 'Not Available'} - label="Cron service" - > - {serviceSub && ( -
{serviceSub}
- )} - - } value={status?.platform || 'Unknown'} label="Platform" /> -
- - {/* Jobs list */} - {jobs.length === 0 ? ( + {loading ? ( + + ) : jobs.length === 0 ? ( Create Job} + action={} /> ) : ( - <> +
+ {/* No KPI strip: every number it carried is already on the + segment you'd click to act on it (mirrors /domains). */}
-

Scheduled jobs

-
- - setQuery(e.target.value)} - /> -
- + + {serviceNote && {serviceNote}}
- {shownJobs.length === 0 ? ( -
No jobs match the current filter.
+ {shown.length === 0 ? ( +
+ {q ? `No jobs match “${search.trim()}”.` : 'No jobs match this filter.'} +
) : (
-
- - - - - - - - - {shownJobs.map((job) => { - const readable = getScheduleDescription(job.schedule); - return ( - openEditModal(job)} - > - - - - - - ); - })} - -
JobScheduleStatus -
-
- -
-
{job.name || 'Unnamed Job'}
- {job.description && ( -
{job.description}
- )} -
- {job.command} -
-
+ (job.enabled ? undefined : 'is-disabled')} + columns={[ + { + key: 'name', + header: 'Job', + render: (job) => ( +
+ +
+
+ {job.name || 'Unnamed job'}
-
- {job.schedule} - {readable !== job.schedule && ( -
{readable}
- )} - 		- - {job.enabled ? 'Enabled' : 'Disabled'} - - e.stopPropagation()}> -
- - - - +
+ {job.command}
-
+ + + ), + }, + { + key: 'schedule', + header: 'Schedule', + render: (job) => ( + <> + + {job.schedule} + +
+ {describeSchedule(job)} +
+ + ), + }, + { + key: 'last_run', + header: 'Last run', + cellClassName: 'cron-cell-mono', + render: (job) => formatWhen(job.last_run), + }, + { + key: 'next_run', + header: 'Next run', + cellClassName: 'cron-cell-mono', + render: (job) => ( + job.enabled ? formatWhen(job.next_run) : 'paused' + ), + }, + { + key: 'status', + header: 'Status', + render: (job) => { + const state = jobState(job); + return {state.label}; + }, + }, + { + key: 'enabled', + header: 'On', + render: (job) => ( + e.stopPropagation()} + role="presentation" + > + handleToggleJob(job.id, job.enabled)} + aria-label={job.enabled ? 'Disable job' : 'Enable job'} + /> + + ), + }, + { + key: 'run', + header: '', + cellClassName: 'cron-cell-actions', + render: (job) => ( + e.stopPropagation()} role="presentation"> + + + ), + }, + ]} + /> )} - + +
+ Times shown in your local timezone · {VIEWER_TZ} +
+ )} + setDrawerJob(null)} + onRefresh={loadData} + onRun={handleRunJob} + onEdit={(j) => { setDrawerJob(null); openEditModal(j); }} + onDelete={handleDeleteJob} + onToggle={handleToggleJob} + /> + {/* Create/Edit Job Modal */} -
-
- - setJobForm({...jobForm, name: e.target.value})} - placeholder="My Backup Job" - required - /> -
+ +
+ + setJobForm({ ...jobForm, name: e.target.value })} + placeholder="My Backup Job" + required + /> +
-
- - setJobForm({...jobForm, command: e.target.value})} - placeholder="/usr/bin/backup.sh" - required - /> - The command or script to execute -
+
+ + setJobForm({ ...jobForm, command: e.target.value })} + placeholder="/usr/bin/backup.sh" + required + /> + The command or script to execute +
-
- -
+
+ +
- {jobForm.usePreset ? ( -
- - -
- ) : ( -
- - setJobForm({...jobForm, schedule: e.target.value})} - placeholder="0 0 * * *" - required={!jobForm.usePreset} - /> - - Format: minute hour day month weekday (e.g., "0 0 * * *" for daily at midnight) - -
- )} - -
- -