feat(queue): offline operation queue for all panel integrations

.gitignore

@@ -1 +1,3 @@
 .DS_Store
+__pycache__/
+*.pyc

README.md

@@ -48,14 +48,57 @@ curl -sL https://raw.githubusercontent.com/seconddns/dns_integrations/main/hosti
 
 See the README in each directory for options, AXFR configuration, and troubleshooting.
 
+### Offline operation queue
+
+Every panel integration ships with a local offline queue
+([`hosting-panels/common/seconddns-queue`](hosting-panels/common/seconddns-queue)).
+Panel hooks enqueue zone operations into a SQLite database
+(`/var/lib/seconddns/queue.db`); the `seconddns-queued` systemd worker is the
+single delivery path — after every enqueue the hook pokes the worker through a
+unix datagram socket (`/var/lib/seconddns/queued.sock`), so delivery starts
+instantly with no polling; a rare fallback tick (`poll_interval`, 60 s) only
+guards against lost wakeups. Operations are delivered in strict FIFO order. If the SecondDNS API is unreachable (outage or maintenance), the
+worker backs off exponentially and retries until everything is delivered.
+Nothing your customers do in the panel during a SecondDNS downtime is lost,
+and hooks never block on API timeouts (they only do a local INSERT).
+
+Worker settings (optional `[queue]` section in `/etc/seconddns.conf`):
+
+```ini
+[queue]
+poll_interval = 60     # fallback idle tick (wakeups are socket-driven)
+http_timeout = 15      # seconds per API request
+backoff_min = 30       # first retry delay when the API is down
+backoff_max = 300      # retry delay ceiling
+```
+
+Replay semantics:
+
+- retryable errors (timeout, 5xx, redirects) pause delivery; the worker retries with exponential backoff
+- duplicate delivery is safe: `409` on create and `404` on delete count as success
+- hard errors (`400`/`422`) mark the operation `failed` and the drain continues
+
+Inspect the queue on any panel server:
+
+```bash
+seconddns-queue status              # pending / failed / oldest age (exit 0/1/2)
+seconddns-queue status --json       # same as JSON (for monitoring agents)
+seconddns-queue flush               # manual one-shot drain (diagnostics)
+systemctl status seconddns-queued   # delivery worker
+```
+
+Requires `sqlite3` (present by default on all supported panels).
+
 ---
 
 ## Monitoring
 
 | Tool | Type | What it checks |
 |:-----|:-----|:---------------|
 | [Nagios / Icinga](nagios_plugins/) | Check plugin (bash) | Zone sync status, stale zones, master reachability |
+| [Nagios / Icinga](nagios_plugins/check_seconddns_queue.sh) | Check plugin (bash) | Offline queue backlog on a panel server (`-w`/`-c` age thresholds) |
 | [Zabbix](zabbix_templates/) | HTTP Agent template | Zone counters, triggers, graphs — no agent required |
+| [Zabbix](zabbix_templates/seconddns_queue.yaml) | Agent template | Offline queue backlog on a panel server (needs a `UserParameter`) |
 
 Both integrations use the SecondDNS API key. See the README in each directory for installation and configuration.
 

hosting-panels/common/seconddns-queue

@@ -0,0 +1,91 @@
+#!/bin/bash
+# Copyright © 2025-2026 SecondDNS
+# Licensed under GNU General Public License v3.0 or SecondDNS Commercial License
+# See LICENSE (GPLv3) or LICENSE.COMMERCIAL (commercial) for details
+# SecondDNS offline operation queue — CLI.
+#
+# Panel hooks enqueue zone operations into a local SQLite queue; the
+# seconddns-queued systemd worker delivers them to the SecondDNS API in
+# strict FIFO order (surviving outages and maintenance windows).
+#
+# Usage:
+#   seconddns-queue enqueue <create|delete> <domain> [master_ip]
+#   seconddns-queue flush             # manual one-shot drain (diagnostics)
+#   seconddns-queue status [--json]
+#
+# Exit codes for status: 0 = empty/healthy, 1 = pending backlog, 2 = failed ops
+#
+# Environment overrides (mainly for tests):
+#   SECONDDNS_QUEUE_DB    sqlite db path (default /var/lib/seconddns/queue.db)
+#   SECONDDNS_LOG         log file       (default /var/log/seconddns.log)
+
+set -u
+
+QDB="${SECONDDNS_QUEUE_DB:-/var/lib/seconddns/queue.db}"
+QSOCK="${SECONDDNS_QUEUE_SOCK:-/var/lib/seconddns/queued.sock}"
+LOG="${SECONDDNS_LOG:-/var/log/seconddns.log}"
+WORKER="${SECONDDNS_QUEUED_BIN:-/usr/local/bin/seconddns-queued}"
+
+log() { echo "$(date '+%Y-%m-%d %H:%M:%S') $1" >> "$LOG"; }
+
+sq() {
+    sqlite3 -cmd ".timeout 5000" "$QDB" "$@"
+}
+
+init_db() {
+    mkdir -p "$(dirname "$QDB")"
+    sq "PRAGMA journal_mode=WAL;
+        CREATE TABLE IF NOT EXISTS ops (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          ts TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ','now')),
+          op TEXT NOT NULL,
+          domain TEXT NOT NULL,
+          master_ip TEXT,
+          attempts INTEGER NOT NULL DEFAULT 0,
+          status TEXT NOT NULL DEFAULT 'pending',
+          last_error TEXT
+        );" >/dev/null
+}
+
+cmd_enqueue() {
+    local op="$1" domain="$2" master_ip="${3:-}"
+    init_db
+    sq "INSERT INTO ops (op, domain, master_ip) VALUES ('$op', '$domain', '$master_ip');" >/dev/null
+    log "[>] queue: enqueued $op $domain"
+    # wake the delivery worker instantly (best-effort)
+    python3 -c "
+import socket, os
+os.chdir(os.path.dirname('$QSOCK') or '.')  # AF_UNIX ~104-byte path limit
+s = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
+s.sendto(b'1', os.path.basename('$QSOCK'))
+" 2>/dev/null || true
+}
+
+cmd_status() {
+    init_db
+    local pending failed oldest_age
+    pending=$(sq "SELECT COUNT(*) FROM ops WHERE status='pending';")
+    failed=$(sq "SELECT COUNT(*) FROM ops WHERE status='failed';")
+    oldest_age=$(sq "SELECT IFNULL(CAST(strftime('%s','now') - strftime('%s', MIN(ts)) AS INTEGER), 0) FROM ops WHERE status='pending';")
+    if [ "${1:-}" = "--json" ]; then
+        echo "{\"pending\": $pending, \"failed\": $failed, \"oldest_age_seconds\": $oldest_age}"
+    else
+        echo "pending=$pending failed=$failed oldest_age_seconds=$oldest_age"
+    fi
+    [ "$failed" -gt 0 ] && exit 2
+    [ "$pending" -gt 0 ] && exit 1
+    exit 0
+}
+
+case "${1:-}" in
+  enqueue)
+    [ $# -ge 3 ] || { echo "usage: $0 enqueue <create|delete> <domain> [master_ip]" >&2; exit 1; }
+    cmd_enqueue "$2" "$3" "${4:-}"
+    ;;
+  flush)  exec "$WORKER" --once ;;
+  status) cmd_status "${2:-}" ;;
+  *)
+    echo "usage: $0 enqueue|flush|status" >&2
+    exit 1
+    ;;
+esac