From 1eed6b9b48958f0a9eb050f3f07ca64a34bb4517 Mon Sep 17 00:00:00 2001 From: Ihor Rusyn <0kaba0@gmail.com> Date: Tue, 7 Jul 2026 01:23:58 +0200 Subject: [PATCH 1/3] feat(queue): offline operation queue for all panel integrations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Zone operations from panel hooks are enqueued into a local SQLite queue (/var/lib/seconddns/queue.db) and delivered immediately; when the SecondDNS API is unreachable (outage or maintenance) they stay queued and a per-minute cron replays them in strict FIFO order once the API is back. No customer operation is lost during SecondDNS downtime. - hosting-panels/common/seconddns-queue: enqueue/flush/status tool (WAL, busy_timeout, flock with portable fallback) - all six bash hooks (plesk/cpanel/directadmin) switched to queue-first + async flush; plesk rename enqueues delete+create - cyberpanel: network errors now caught; retryable failures enqueue via the same tool - installers deploy the tool + /etc/cron.d/seconddns-queue; uninstallers clean up - replay semantics: 409 create / 404 delete = success; 400/422 -> failed (does not block the queue); 5xx/timeout stop the drain - monitoring: nagios_plugins/check_seconddns_queue.sh (-w/-c age thresholds) + zabbix_templates/seconddns_queue.yaml - tests: hosting-panels/common/tests/test_queue.sh — 14 checks with a scripted mock API (FIFO, concurrency, all status branches) Ref: seconddns/k8s_web#265 --- README.md | 29 +++ hosting-panels/common/seconddns-queue | 205 ++++++++++++++++++ hosting-panels/common/tests/test_queue.sh | 147 +++++++++++++ hosting-panels/cpanel/domain_create.sh | 17 +- hosting-panels/cpanel/domain_delete.sh | 25 +-- hosting-panels/cpanel/install.sh | 16 ++ hosting-panels/cpanel/uninstall.sh | 4 + hosting-panels/cyberpanel/install.sh | 13 ++ hosting-panels/cyberpanel/seconddns.py | 46 +++- hosting-panels/cyberpanel/uninstall.sh | 4 + hosting-panels/directadmin/dns_create_post.sh | 18 +- hosting-panels/directadmin/dns_delete_post.sh | 26 +-- hosting-panels/directadmin/install.sh | 16 ++ hosting-panels/directadmin/uninstall.sh | 4 + hosting-panels/plesk/domain_create.sh | 18 +- hosting-panels/plesk/domain_delete.sh | 26 +-- hosting-panels/plesk/domain_rename.sh | 41 +--- hosting-panels/plesk/install.sh | 16 ++ hosting-panels/plesk/uninstall.sh | 4 + nagios_plugins/check_seconddns_queue.sh | 72 ++++++ zabbix_templates/seconddns_queue.yaml | 100 +++++++++ 21 files changed, 699 insertions(+), 148 deletions(-) create mode 100755 hosting-panels/common/seconddns-queue create mode 100755 hosting-panels/common/tests/test_queue.sh create mode 100755 nagios_plugins/check_seconddns_queue.sh create mode 100644 zabbix_templates/seconddns_queue.yaml diff --git a/README.md b/README.md index 64b15fd..c4bddb6 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,33 @@ curl -sL https://raw.githubusercontent.com/seconddns/dns_integrations/main/hosti See the README in each directory for options, AXFR configuration, and troubleshooting. +### Offline operation queue + +Every panel integration ships with a local offline queue +([`hosting-panels/common/seconddns-queue`](hosting-panels/common/seconddns-queue)). +Zone operations are enqueued into a SQLite database +(`/var/lib/seconddns/queue.db`) and delivered immediately; if the SecondDNS API +is unreachable (outage or maintenance), the operations stay queued and a cron +job (`/etc/cron.d/seconddns-queue`, every minute) replays them in strict FIFO +order once the API is back. Nothing your customers do in the panel during a +SecondDNS downtime is lost. + +Replay semantics: + +- retryable errors (timeout, 5xx, redirects) stop the drain until the next cron tick +- duplicate delivery is safe: `409` on create and `404` on delete count as success +- hard errors (`400`/`422`) mark the operation `failed` and the drain continues + +Inspect the queue on any panel server: + +```bash +seconddns-queue status # pending / failed / oldest age (exit 0/1/2) +seconddns-queue status --json # same as JSON (for monitoring agents) +seconddns-queue flush # force an immediate drain +``` + +Requires `sqlite3` (present by default on all supported panels). + --- ## Monitoring @@ -55,7 +82,9 @@ See the README in each directory for options, AXFR configuration, and troublesho | Tool | Type | What it checks | |:-----|:-----|:---------------| | [Nagios / Icinga](nagios_plugins/) | Check plugin (bash) | Zone sync status, stale zones, master reachability | +| [Nagios / Icinga](nagios_plugins/check_seconddns_queue.sh) | Check plugin (bash) | Offline queue backlog on a panel server (`-w`/`-c` age thresholds) | | [Zabbix](zabbix_templates/) | HTTP Agent template | Zone counters, triggers, graphs — no agent required | +| [Zabbix](zabbix_templates/seconddns_queue.yaml) | Agent template | Offline queue backlog on a panel server (needs a `UserParameter`) | Both integrations use the SecondDNS API key. See the README in each directory for installation and configuration. diff --git a/hosting-panels/common/seconddns-queue b/hosting-panels/common/seconddns-queue new file mode 100755 index 0000000..ab8df85 --- /dev/null +++ b/hosting-panels/common/seconddns-queue @@ -0,0 +1,205 @@ +#!/bin/bash +# Copyright © 2025-2026 SecondDNS +# Licensed under GNU General Public License v3.0 or SecondDNS Commercial License +# See LICENSE (GPLv3) or LICENSE.COMMERCIAL (commercial) for details +# SecondDNS offline operation queue. +# +# Panel hooks enqueue every zone operation here and immediately trigger a +# flush. If the SecondDNS API is unreachable (outage or maintenance), the +# operation stays in a local SQLite queue and is replayed by cron in strict +# FIFO order once the API is back. +# +# Usage: +# seconddns-queue enqueue [master_ip] +# seconddns-queue flush +# seconddns-queue status [--json] +# +# Exit codes for status: 0 = empty/healthy, 1 = pending backlog, 2 = failed ops +# +# Environment overrides (mainly for tests): +# SECONDDNS_CONF config file (default /etc/seconddns.conf) +# SECONDDNS_QUEUE_DB sqlite db path (default /var/lib/seconddns/queue.db) +# SECONDDNS_QUEUE_LOCK flush lockfile (default /var/lib/seconddns/queue.lock) +# SECONDDNS_LOG log file (default /var/log/seconddns.log) + +set -u + +CONFIG="${SECONDDNS_CONF:-/etc/seconddns.conf}" +QDB="${SECONDDNS_QUEUE_DB:-/var/lib/seconddns/queue.db}" +QLOCK="${SECONDDNS_QUEUE_LOCK:-/var/lib/seconddns/queue.lock}" +LOG="${SECONDDNS_LOG:-/var/log/seconddns.log}" +UA="SecondDNS-Queue/1.0" + +log() { echo "$(date '+%Y-%m-%d %H:%M:%S') $1" >> "$LOG"; } + +sq() { + sqlite3 -cmd ".timeout 5000" "$QDB" "$@" +} + +init_db() { + mkdir -p "$(dirname "$QDB")" + sq "PRAGMA journal_mode=WAL; + CREATE TABLE IF NOT EXISTS ops ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ','now')), + op TEXT NOT NULL, + domain TEXT NOT NULL, + master_ip TEXT, + attempts INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT 'pending', + last_error TEXT + );" >/dev/null +} + +read_conf() { + [ -f "$CONFIG" ] || { log "[!] queue: config $CONFIG missing"; exit 0; } + API_URL=$(sed -nE 's/^api_url[[:space:]]*=[[:space:]]*//p' "$CONFIG") + API_KEY=$(sed -nE 's/^api_key[[:space:]]*=[[:space:]]*//p' "$CONFIG") + [ -z "$API_URL" ] || [ -z "$API_KEY" ] && { log "[!] queue: api_url/api_key not configured"; exit 0; } +} + +# api_call METHOD PATH [JSON_BODY] -> sets HTTP_CODE and HTTP_BODY +api_call() { + local method="$1" path="$2" body="${3:-}" + local out + if [ -n "$body" ]; then + out=$(curl -s --max-time 15 -w '\n%{http_code}' \ + -X "$method" \ + -H "X-API-Key: $API_KEY" \ + -H "Content-Type: application/json" \ + -H "User-Agent: $UA" \ + -d "$body" \ + "$API_URL$path" 2>/dev/null) + else + out=$(curl -s --max-time 15 -w '\n%{http_code}' \ + -X "$method" \ + -H "X-API-Key: $API_KEY" \ + -H "User-Agent: $UA" \ + "$API_URL$path" 2>/dev/null) + fi + HTTP_CODE=$(echo "$out" | tail -n1) + HTTP_BODY=$(echo "$out" | sed '$d') + [ -z "$HTTP_CODE" ] && HTTP_CODE=000 +} + +# Attempt one operation. Return: 0 done (success), 1 retryable failure, 2 hard failure. +send_op() { + local op="$1" domain="$2" master_ip="$3" + case "$op" in + create) + api_call POST "/api/zones" "{\"name\":\"$domain\",\"masterIp\":\"$master_ip\"}" + case "$HTTP_CODE" in + 200|201) return 0 ;; + 409) log "[~] queue: zone $domain already exists (treated as success)"; return 0 ;; + 400|422) return 2 ;; + *) return 1 ;; + esac + ;; + delete) + api_call GET "/api/zones/by-name/$domain" + case "$HTTP_CODE" in + 200) ;; + 404) log "[~] queue: zone $domain already absent (treated as success)"; return 0 ;; + 400|422) return 2 ;; + *) return 1 ;; + esac + local zone_id + zone_id=$(echo "$HTTP_BODY" | python3 -c "import sys,json; print(json.load(sys.stdin).get('id',''))" 2>/dev/null) + [ -z "$zone_id" ] && return 1 + api_call DELETE "/api/zones/$zone_id" + case "$HTTP_CODE" in + 200|204) return 0 ;; + 404) return 0 ;; + 400|422) return 2 ;; + *) return 1 ;; + esac + ;; + *) + return 2 ;; + esac +} + +cmd_enqueue() { + local op="$1" domain="$2" master_ip="${3:-}" + init_db + sq "INSERT INTO ops (op, domain, master_ip) VALUES ('$op', '$domain', '$master_ip');" >/dev/null + log "[>] queue: enqueued $op $domain" +} + +cmd_flush() { + init_db + read_conf + # single flusher at a time; strict FIFO requires it + if command -v flock >/dev/null 2>&1; then + exec 9>"$QLOCK" + flock -n 9 || exit 0 + else + # portable fallback (no flock, e.g. macOS): mkdir lock with stale-PID check + if ! mkdir "$QLOCK.d" 2>/dev/null; then + local lock_pid + lock_pid=$(cat "$QLOCK.d/pid" 2>/dev/null) + if [ -n "$lock_pid" ] && kill -0 "$lock_pid" 2>/dev/null; then + exit 0 + fi + rm -rf "$QLOCK.d" + mkdir "$QLOCK.d" 2>/dev/null || exit 0 + fi + echo $$ > "$QLOCK.d/pid" + trap 'rm -rf "$QLOCK.d"' EXIT + fi + + while :; do + local row + row=$(sq -separator '|' "SELECT id, op, domain, IFNULL(master_ip,'') FROM ops WHERE status='pending' ORDER BY id LIMIT 1;") + [ -z "$row" ] && break + local id op domain master_ip + IFS='|' read -r id op domain master_ip <<< "$row" + + send_op "$op" "$domain" "$master_ip" + case $? in + 0) + sq "DELETE FROM ops WHERE id=$id;" >/dev/null + log "[+] queue: delivered $op $domain" + ;; + 2) + sq "UPDATE ops SET status='failed', attempts=attempts+1, last_error='HTTP $HTTP_CODE' WHERE id=$id;" >/dev/null + log "[!] queue: $op $domain permanently failed (HTTP $HTTP_CODE), marked failed" + ;; + *) + # API unreachable / 5xx / maintenance: stop, keep FIFO order, retry next tick + sq "UPDATE ops SET attempts=attempts+1, last_error='HTTP $HTTP_CODE' WHERE id=$id;" >/dev/null + log "[~] queue: API unavailable (HTTP $HTTP_CODE), $(sq "SELECT COUNT(*) FROM ops WHERE status='pending';") op(s) waiting" + break + ;; + esac + done +} + +cmd_status() { + init_db + local pending failed oldest_age + pending=$(sq "SELECT COUNT(*) FROM ops WHERE status='pending';") + failed=$(sq "SELECT COUNT(*) FROM ops WHERE status='failed';") + oldest_age=$(sq "SELECT IFNULL(CAST(strftime('%s','now') - strftime('%s', MIN(ts)) AS INTEGER), 0) FROM ops WHERE status='pending';") + if [ "${1:-}" = "--json" ]; then + echo "{\"pending\": $pending, \"failed\": $failed, \"oldest_age_seconds\": $oldest_age}" + else + echo "pending=$pending failed=$failed oldest_age_seconds=$oldest_age" + fi + [ "$failed" -gt 0 ] && exit 2 + [ "$pending" -gt 0 ] && exit 1 + exit 0 +} + +case "${1:-}" in + enqueue) + [ $# -ge 3 ] || { echo "usage: $0 enqueue [master_ip]" >&2; exit 1; } + cmd_enqueue "$2" "$3" "${4:-}" + ;; + flush) cmd_flush ;; + status) cmd_status "${2:-}" ;; + *) + echo "usage: $0 enqueue|flush|status" >&2 + exit 1 + ;; +esac diff --git a/hosting-panels/common/tests/test_queue.sh b/hosting-panels/common/tests/test_queue.sh new file mode 100755 index 0000000..43bd6fd --- /dev/null +++ b/hosting-panels/common/tests/test_queue.sh @@ -0,0 +1,147 @@ +#!/bin/bash +# Copyright © 2025-2026 SecondDNS +# Licensed under GNU General Public License v3.0 or SecondDNS Commercial License +# See LICENSE (GPLv3) or LICENSE.COMMERCIAL (commercial) for details +# Self-contained test suite for seconddns-queue. +# Requires: bash, sqlite3, curl, python3. No network access needed. +# +# Usage: ./test_queue.sh + +set -u + +HERE="$(cd "$(dirname "$0")" && pwd)" +QUEUE="$HERE/../seconddns-queue" +TMP="$(mktemp -d)" +PORT=8899 + +export SECONDDNS_CONF="$TMP/seconddns.conf" +export SECONDDNS_QUEUE_DB="$TMP/queue.db" +export SECONDDNS_QUEUE_LOCK="$TMP/queue.lock" +export SECONDDNS_LOG="$TMP/seconddns.log" + +cat > "$SECONDDNS_CONF" < /dev/null 2>&1 <<'PYEOF' & +import http.server, sys, os, json +port, tmp = int(sys.argv[1]), sys.argv[2] +class H(http.server.BaseHTTPRequestHandler): + def mode(self): + try: return open(tmp+'/mode').read().strip() + except Exception: return 'ok' + def log_message(self, *a): pass + def reply(self, code, body=b'{}'): + self.send_response(code) + self.send_header('Content-Type','application/json') + self.end_headers() + self.wfile.write(body) + def record(self): + with open(tmp+'/requests.log','a') as f: + f.write(f"{self.command} {self.path.split('?')[0]}\n") + def do_POST(self): + self.rfile.read(int(self.headers.get('Content-Length',0))) + m=self.mode() + if m in ('ok','record'): + if m=='record': self.record() + self.reply(201, b'{"id":"z1"}') + elif m=='dup': self.reply(409, b'{"error":"exists"}') + elif m=='badreq': self.reply(400, b'{"error":"invalid"}') + else: self.reply(503) + def do_GET(self): + m=self.mode() + if m in ('ok','record'): + if m=='record': self.record() + self.reply(200, b'{"id":"z1"}') + elif m=='dup': self.reply(404) + else: self.reply(503) + def do_DELETE(self): + m=self.mode() + if m in ('ok','record'): + if m=='record': self.record() + self.reply(204, b'') + elif m=='dup': self.reply(404) + else: self.reply(503) +http.server.HTTPServer(('127.0.0.1', port), H).serve_forever() +PYEOF +MOCK_PID=$! +trap 'kill $MOCK_PID 2>/dev/null; rm -rf "$TMP"' EXIT +sleep 1 + +echo "== 1. enqueue writes rows" +"$QUEUE" enqueue create example.com 192.0.2.10 +"$QUEUE" enqueue delete old.example.com +assert_eq "$(pending)" 2 "two ops pending" + +echo "== 2. API down: drain stops, order kept" +echo down > "$TMP/mode" +"$QUEUE" flush +assert_eq "$(pending)" 2 "nothing lost while API is down" +attempts=$(sqlite3 "$SECONDDNS_QUEUE_DB" "SELECT attempts FROM ops ORDER BY id LIMIT 1;") +assert_eq "$attempts" 1 "first op attempt counted" + +echo "== 3. API up: queue drains in FIFO order" +echo record > "$TMP/mode" +: > "$TMP/requests.log" +"$QUEUE" flush +assert_eq "$(pending)" 0 "queue drained" +first_req=$(head -1 "$TMP/requests.log") +assert_eq "$first_req" "POST /api/zones" "create delivered first (FIFO)" + +echo "== 4. duplicate delivery: 409/404 treated as success" +echo dup > "$TMP/mode" +"$QUEUE" enqueue create example.com 192.0.2.10 +"$QUEUE" enqueue delete example.com +"$QUEUE" flush +assert_eq "$(pending)" 0 "409 create and 404 delete drained as success" +assert_eq "$(failed)" 0 "no failed rows" + +echo "== 5. hard error: marked failed, queue continues" +echo badreq > "$TMP/mode" +"$QUEUE" enqueue create "bad domain" 192.0.2.10 +echo ok > "$TMP/mode.next" # placeholder to show intent +"$QUEUE" flush +assert_eq "$(failed)" 1 "400 marked failed" +assert_eq "$(pending)" 0 "failed op does not block the queue" + +echo "== 6. status output and exit codes" +out=$("$QUEUE" status --json); rc=$? +assert_eq "$rc" 2 "exit 2 when failed ops present" +echo "$out" | python3 -c "import sys,json; d=json.load(sys.stdin); assert d['failed']==1" \ + && ok "status --json parses with failed=1" || fail "status --json content" +sqlite3 "$SECONDDNS_QUEUE_DB" "DELETE FROM ops;" +"$QUEUE" status > /dev/null; rc=$? +assert_eq "$rc" 0 "exit 0 when queue empty" + +echo "== 7. concurrency: 20 parallel enqueues, none lost" +enq_pids=() +for i in $(seq 1 20); do + "$QUEUE" enqueue create "conc$i.example.com" 192.0.2.10 & + enq_pids+=($!) +done +wait "${enq_pids[@]}" +assert_eq "$(pending)" 20 "all 20 concurrent enqueues persisted" +echo ok > "$TMP/mode" +"$QUEUE" flush +assert_eq "$(pending)" 0 "all 20 drained" + +echo +echo "passed: $PASS, failed: $FAIL" +[ "$FAIL" -eq 0 ] diff --git a/hosting-panels/cpanel/domain_create.sh b/hosting-panels/cpanel/domain_create.sh index d30a141..d8ab17e 100755 --- a/hosting-panels/cpanel/domain_create.sh +++ b/hosting-panels/cpanel/domain_create.sh @@ -36,18 +36,9 @@ except Exception: log "Zone created: $ZONE_NAME (cpanel hook)" -response=$(curl -sf --max-time 15 \ - -X POST \ - -H "X-API-Key: $API_KEY" \ - -H "Content-Type: application/json" \ - -H "User-Agent: SecondDNS-cPanel/1.0" \ - -d "{\"name\":\"$ZONE_NAME\",\"masterIp\":\"$MASTER_IP\"}" \ - "$API_URL/api/zones" 2>/dev/null) - -if [ $? -eq 0 ]; then - log "[+] Zone $ZONE_NAME added to SecondDNS" -else - log "[!] Failed to add zone $ZONE_NAME to SecondDNS" -fi +QUEUE="/usr/local/bin/seconddns-queue" +"$QUEUE" enqueue create "$ZONE_NAME" "$MASTER_IP" +log "[>] Zone $ZONE_NAME queued for SecondDNS" +( "$QUEUE" flush >/dev/null 2>&1 & ) exit 0 diff --git a/hosting-panels/cpanel/domain_delete.sh b/hosting-panels/cpanel/domain_delete.sh index 1329be3..14207f3 100755 --- a/hosting-panels/cpanel/domain_delete.sh +++ b/hosting-panels/cpanel/domain_delete.sh @@ -34,26 +34,9 @@ except Exception: log "Zone deleted: $ZONE_NAME (cpanel hook)" -zone_id=$(curl -sf --max-time 10 \ - -H "X-API-Key: $API_KEY" \ - -H "User-Agent: SecondDNS-cPanel/1.0" \ - "$API_URL/api/zones/by-name/$ZONE_NAME" 2>/dev/null | \ - python3 -c "import sys,json; print(json.load(sys.stdin).get('id',''))" 2>/dev/null) - -if [ -n "$zone_id" ]; then - curl -sf --max-time 15 \ - -X DELETE \ - -H "X-API-Key: $API_KEY" \ - -H "User-Agent: SecondDNS-cPanel/1.0" \ - "$API_URL/api/zones/$zone_id" 2>/dev/null - - if [ $? -eq 0 ]; then - log "[+] Zone $ZONE_NAME removed from SecondDNS" - else - log "[!] Failed to remove zone $ZONE_NAME from SecondDNS" - fi -else - log "[~] Zone $ZONE_NAME not found in SecondDNS (already removed?)" -fi +QUEUE="/usr/local/bin/seconddns-queue" +"$QUEUE" enqueue delete "$ZONE_NAME" +log "[>] Zone $ZONE_NAME removal queued for SecondDNS" +( "$QUEUE" flush >/dev/null 2>&1 & ) exit 0 diff --git a/hosting-panels/cpanel/install.sh b/hosting-panels/cpanel/install.sh index 114ea5e..dd833d3 100755 --- a/hosting-panels/cpanel/install.sh +++ b/hosting-panels/cpanel/install.sh @@ -171,6 +171,22 @@ for script in domain_create.sh domain_delete.sh; do echo "[+] Installed: $SCRIPT_DIR/seconddns-cpanel-${script}" done +# --- Offline operation queue --- +echo "" +echo "--- Installing offline operation queue ---" +QUEUE_URL="${REPO_URL%/*}/common/seconddns-queue" +curl -sf --max-time 10 -o /usr/local/bin/seconddns-queue "$QUEUE_URL?t=$(date +%s)" +chmod +x /usr/local/bin/seconddns-queue +mkdir -p /var/lib/seconddns +if ! command -v sqlite3 &>/dev/null; then + echo "[!] Warning: sqlite3 not found — offline queue disabled until installed" +fi +cat > /etc/cron.d/seconddns-queue << 'CRONEOF' +* * * * * root /usr/local/bin/seconddns-queue flush >/dev/null 2>&1 +CRONEOF +chmod 644 /etc/cron.d/seconddns-queue +echo "[+] Queue tool: /usr/local/bin/seconddns-queue (cron flush every minute)" + # Register hooks echo "" echo "--- Registering cPanel/WHM hooks ---" diff --git a/hosting-panels/cpanel/uninstall.sh b/hosting-panels/cpanel/uninstall.sh index 48cdf15..002d58c 100755 --- a/hosting-panels/cpanel/uninstall.sh +++ b/hosting-panels/cpanel/uninstall.sh @@ -100,3 +100,7 @@ echo "=== Uninstall complete ===" echo "" echo " Note: AXFR settings in WHM and BIND config must be removed manually." echo " Verify: $HOOKS_BIN list" + +# Remove offline queue +rm -f /usr/local/bin/seconddns-queue /etc/cron.d/seconddns-queue +rm -rf /var/lib/seconddns diff --git a/hosting-panels/cyberpanel/install.sh b/hosting-panels/cyberpanel/install.sh index a9064ff..ad55b1a 100755 --- a/hosting-panels/cyberpanel/install.sh +++ b/hosting-panels/cyberpanel/install.sh @@ -220,6 +220,19 @@ with open(p, 'w') as f: f.write(c) # Systemd hook SYSTEMD_SERVICE="/etc/systemd/system/seconddns-signals.service" cp "$CYBER_SRC/seconddns-signals.service" "$SYSTEMD_SERVICE" + +# --- Offline operation queue --- +cp "$WORK_DIR/dns_integrations/hosting-panels/common/seconddns-queue" /usr/local/bin/seconddns-queue +chmod +x /usr/local/bin/seconddns-queue +mkdir -p /var/lib/seconddns +if ! command -v sqlite3 &>/dev/null; then + echo "[!] Warning: sqlite3 not found — offline queue disabled until installed" +fi +cat > /etc/cron.d/seconddns-queue << 'CRONEOF' +* * * * * root /usr/local/bin/seconddns-queue flush >/dev/null 2>&1 +CRONEOF +chmod 644 /etc/cron.d/seconddns-queue +echo "[+] Queue tool: /usr/local/bin/seconddns-queue (cron flush every minute)" systemctl daemon-reload systemctl enable seconddns-signals.service 2>/dev/null echo "[+] Systemd hook installed (survives CyberPanel updates)" diff --git a/hosting-panels/cyberpanel/seconddns.py b/hosting-panels/cyberpanel/seconddns.py index ebff5b8..1d67636 100755 --- a/hosting-panels/cyberpanel/seconddns.py +++ b/hosting-panels/cyberpanel/seconddns.py @@ -103,6 +103,30 @@ def api_request(config, method, path, data=None): except Exception: error_data = {"error": error_body} return {"_status": e.code, **error_data} + except (urllib.error.URLError, TimeoutError, OSError) as e: + return {"_status": 0, "error": str(e)} + + +QUEUE_BIN = "/usr/local/bin/seconddns-queue" + + +def _retryable(status): + """API unreachable, 5xx or maintenance redirect — worth retrying later.""" + return status == 0 or status >= 500 or 300 <= status < 400 + + +def queue_op(op, domain, master_ip=""): + """Persist the operation in the local offline queue for cron replay.""" + try: + subprocess.run([QUEUE_BIN, "enqueue", op, domain, master_ip], + check=True, timeout=10) + subprocess.Popen([QUEUE_BIN, "flush"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + logger.info(" API unavailable — %s %s queued for later delivery", op, domain) + return True + except Exception as e: + logger.error(" Failed to enqueue %s %s: %s", op, domain, e) + return False def list_zones(config): @@ -127,8 +151,10 @@ def add_zone(config, domain): error = result.get("error", "Unknown error") if status == 409: logger.info(" Already exists, skipping.") - else: - logger.error(" Error (%s): %s", status, error) + return True + if _retryable(status): + return queue_op("create", domain, config["master_ip"]) + logger.error(" Error (%s): %s", status, error) return False logger.info(" Done.") return True @@ -143,13 +169,21 @@ def find_zone_by_name(config, domain): def remove_zone(config, domain): domain = domain.lower().rstrip(".") - zone = find_zone_by_name(config, domain) - if not zone: - logger.info("[-] Zone %s not found on secondary DNS.", domain) + lookup = api_request(config, "GET", f"/api/zones/by-name/{domain}") + status = lookup.get("_status") if isinstance(lookup, dict) else None + if status: + if status == 404: + logger.info("[-] Zone %s not found on secondary DNS.", domain) + return False + if _retryable(status): + return queue_op("delete", domain) + logger.error(" Error (%s): %s", status, lookup.get("error", str(lookup))) return False logger.info("[-] Removing zone: %s", domain) - result = api_request(config, "DELETE", f"/api/zones/{zone['id']}") + result = api_request(config, "DELETE", f"/api/zones/{lookup['id']}") if result and result.get("_status"): + if _retryable(result["_status"]): + return queue_op("delete", domain) logger.error(" Error (%s): %s", result.get("_status"), result.get("error", str(result))) return False logger.info(" Done.") diff --git a/hosting-panels/cyberpanel/uninstall.sh b/hosting-panels/cyberpanel/uninstall.sh index 55306b4..6758144 100755 --- a/hosting-panels/cyberpanel/uninstall.sh +++ b/hosting-panels/cyberpanel/uninstall.sh @@ -87,3 +87,7 @@ echo "=== Uninstall complete ===" echo "" echo "Note: DNS zones on SecondDNS were NOT removed." echo "Delete them manually via Dashboard or API if needed." + +# Remove offline queue +rm -f /usr/local/bin/seconddns-queue /etc/cron.d/seconddns-queue +rm -rf /var/lib/seconddns diff --git a/hosting-panels/directadmin/dns_create_post.sh b/hosting-panels/directadmin/dns_create_post.sh index 11e1037..1fde4be 100644 --- a/hosting-panels/directadmin/dns_create_post.sh +++ b/hosting-panels/directadmin/dns_create_post.sh @@ -28,19 +28,9 @@ esac log "Zone created: $domain (caller=$caller, user=$username)" -# Add zone to SecondDNS -response=$(curl -sf --max-time 15 \ - -X POST \ - -H "X-API-Key: $API_KEY" \ - -H "Content-Type: application/json" \ - -H "User-Agent: SecondDNS-DirectAdmin/1.0" \ - -d "{\"name\":\"$domain\",\"masterIp\":\"$MASTER_IP\"}" \ - "$API_URL/api/zones" 2>/dev/null) - -if [ $? -eq 0 ]; then - log "[+] Zone $domain added to SecondDNS" -else - log "[!] Failed to add zone $domain to SecondDNS" -fi +QUEUE="/usr/local/bin/seconddns-queue" +"$QUEUE" enqueue create "$domain" "$MASTER_IP" +log "[>] Zone $domain queued for SecondDNS" +( "$QUEUE" flush >/dev/null 2>&1 & ) exit 0 diff --git a/hosting-panels/directadmin/dns_delete_post.sh b/hosting-panels/directadmin/dns_delete_post.sh index c850157..0e159c5 100644 --- a/hosting-panels/directadmin/dns_delete_post.sh +++ b/hosting-panels/directadmin/dns_delete_post.sh @@ -21,27 +21,9 @@ API_KEY=$(grep "^api_key" "$CONFIG" | sed 's/^api_key\s*=\s*//') log "Zone deleted: $domain (user=$USERNAME)" -# Find zone ID by name -zone_id=$(curl -sf --max-time 10 \ - -H "X-API-Key: $API_KEY" \ - -H "User-Agent: SecondDNS-DirectAdmin/1.0" \ - "$API_URL/api/zones/by-name/$domain" 2>/dev/null | \ - python3 -c "import sys,json; print(json.load(sys.stdin).get('id',''))" 2>/dev/null) - -if [ -n "$zone_id" ]; then - curl -sf --max-time 15 \ - -X DELETE \ - -H "X-API-Key: $API_KEY" \ - -H "User-Agent: SecondDNS-DirectAdmin/1.0" \ - "$API_URL/api/zones/$zone_id" 2>/dev/null - - if [ $? -eq 0 ]; then - log "[+] Zone $domain removed from SecondDNS" - else - log "[!] Failed to remove zone $domain from SecondDNS" - fi -else - log "[~] Zone $domain not found in SecondDNS (already removed?)" -fi +QUEUE="/usr/local/bin/seconddns-queue" +"$QUEUE" enqueue delete "$domain" +log "[>] Zone $domain removal queued for SecondDNS" +( "$QUEUE" flush >/dev/null 2>&1 & ) exit 0 diff --git a/hosting-panels/directadmin/install.sh b/hosting-panels/directadmin/install.sh index a7b4c31..9fa2bbb 100644 --- a/hosting-panels/directadmin/install.sh +++ b/hosting-panels/directadmin/install.sh @@ -168,6 +168,22 @@ for hook in dns_create_post.sh dns_delete_post.sh; do echo "[+] Installed hook: $HOOKS_DIR/$hook" done +# --- Offline operation queue --- +echo "" +echo "--- Installing offline operation queue ---" +QUEUE_URL="${REPO_URL%/*}/common/seconddns-queue" +curl -sf --max-time 10 -o /usr/local/bin/seconddns-queue "$QUEUE_URL?t=$(date +%s)" +chmod +x /usr/local/bin/seconddns-queue +mkdir -p /var/lib/seconddns +if ! command -v sqlite3 &>/dev/null; then + echo "[!] Warning: sqlite3 not found — offline queue disabled until installed" +fi +cat > /etc/cron.d/seconddns-queue << 'CRONEOF' +* * * * * root /usr/local/bin/seconddns-queue flush >/dev/null 2>&1 +CRONEOF +chmod 644 /etc/cron.d/seconddns-queue +echo "[+] Queue tool: /usr/local/bin/seconddns-queue (cron flush every minute)" + # --- Detect DNS server and configure AXFR --- echo "" echo "--- DNS Server detection & AXFR configuration ---" diff --git a/hosting-panels/directadmin/uninstall.sh b/hosting-panels/directadmin/uninstall.sh index 71c8764..9e83e65 100644 --- a/hosting-panels/directadmin/uninstall.sh +++ b/hosting-panels/directadmin/uninstall.sh @@ -31,3 +31,7 @@ echo "" echo "=== Uninstall complete ===" echo " Log file kept at: $LOG_FILE" echo " Your zones on the secondary DNS are not deleted automatically." + +# Remove offline queue +rm -f /usr/local/bin/seconddns-queue /etc/cron.d/seconddns-queue +rm -rf /var/lib/seconddns diff --git a/hosting-panels/plesk/domain_create.sh b/hosting-panels/plesk/domain_create.sh index 307d37d..31d8b18 100755 --- a/hosting-panels/plesk/domain_create.sh +++ b/hosting-panels/plesk/domain_create.sh @@ -33,19 +33,9 @@ elif command -v idn &>/dev/null; then fi # If neither is available, the domain name is sent as-is (API will handle it) -# Add zone to SecondDNS -response=$(curl -sf --max-time 15 \ - -X POST \ - -H "X-API-Key: $API_KEY" \ - -H "Content-Type: application/json" \ - -H "User-Agent: SecondDNS-Plesk/1.0" \ - -d "{\"name\":\"$ZONE_NAME\",\"masterIp\":\"$MASTER_IP\"}" \ - "$API_URL/api/zones" 2>/dev/null) - -if [ $? -eq 0 ]; then - log "[+] Zone $ZONE_NAME added to SecondDNS" -else - log "[!] Failed to add zone $ZONE_NAME to SecondDNS" -fi +QUEUE="/usr/local/bin/seconddns-queue" +"$QUEUE" enqueue create "$ZONE_NAME" "$MASTER_IP" +log "[>] Zone $ZONE_NAME queued for SecondDNS" +( "$QUEUE" flush >/dev/null 2>&1 & ) exit 0 diff --git a/hosting-panels/plesk/domain_delete.sh b/hosting-panels/plesk/domain_delete.sh index 3326f0b..b21a232 100755 --- a/hosting-panels/plesk/domain_delete.sh +++ b/hosting-panels/plesk/domain_delete.sh @@ -32,27 +32,9 @@ elif command -v idn &>/dev/null; then fi # If neither is available, the domain name is sent as-is (API will handle it) -# Find zone ID by name -zone_id=$(curl -sf --max-time 10 \ - -H "X-API-Key: $API_KEY" \ - -H "User-Agent: SecondDNS-Plesk/1.0" \ - "$API_URL/api/zones/by-name/$ZONE_NAME" 2>/dev/null | \ - python3 -c "import sys,json; print(json.load(sys.stdin).get('id',''))" 2>/dev/null) - -if [ -n "$zone_id" ]; then - curl -sf --max-time 15 \ - -X DELETE \ - -H "X-API-Key: $API_KEY" \ - -H "User-Agent: SecondDNS-Plesk/1.0" \ - "$API_URL/api/zones/$zone_id" 2>/dev/null - - if [ $? -eq 0 ]; then - log "[+] Zone $ZONE_NAME removed from SecondDNS" - else - log "[!] Failed to remove zone $ZONE_NAME from SecondDNS" - fi -else - log "[~] Zone $ZONE_NAME not found in SecondDNS (already removed?)" -fi +QUEUE="/usr/local/bin/seconddns-queue" +"$QUEUE" enqueue delete "$ZONE_NAME" +log "[>] Zone $ZONE_NAME removal queued for SecondDNS" +( "$QUEUE" flush >/dev/null 2>&1 & ) exit 0 diff --git a/hosting-panels/plesk/domain_rename.sh b/hosting-panels/plesk/domain_rename.sh index e63226f..e32b089 100644 --- a/hosting-panels/plesk/domain_rename.sh +++ b/hosting-panels/plesk/domain_rename.sh @@ -43,41 +43,10 @@ idn_encode() { OLD_ZONE=$(idn_encode "$OLD_ZONE") NEW_ZONE=$(idn_encode "$NEW_ZONE") -# Delete old zone -zone_id=$(curl -sf --max-time 10 \ - -H "X-API-Key: $API_KEY" \ - -H "User-Agent: SecondDNS-Plesk/1.0" \ - "$API_URL/api/zones/by-name/$OLD_ZONE" 2>/dev/null | \ - python3 -c "import sys,json; print(json.load(sys.stdin).get('id',''))" 2>/dev/null) - -if [ -n "$zone_id" ]; then - curl -sf --max-time 15 \ - -X DELETE \ - -H "X-API-Key: $API_KEY" \ - -H "User-Agent: SecondDNS-Plesk/1.0" \ - "$API_URL/api/zones/$zone_id" 2>/dev/null - if [ $? -eq 0 ]; then - log "[+] Old zone $OLD_ZONE removed from SecondDNS" - else - log "[!] Failed to remove old zone $OLD_ZONE from SecondDNS" - fi -else - log "[~] Old zone $OLD_ZONE not found in SecondDNS (skipping delete)" -fi - -# Create new zone -response=$(curl -sf --max-time 15 \ - -X POST \ - -H "X-API-Key: $API_KEY" \ - -H "Content-Type: application/json" \ - -H "User-Agent: SecondDNS-Plesk/1.0" \ - -d "{\"name\":\"$NEW_ZONE\",\"masterIp\":\"$MASTER_IP\"}" \ - "$API_URL/api/zones" 2>/dev/null) - -if [ $? -eq 0 ]; then - log "[+] New zone $NEW_ZONE added to SecondDNS" -else - log "[!] Failed to add new zone $NEW_ZONE to SecondDNS" -fi +QUEUE="/usr/local/bin/seconddns-queue" +"$QUEUE" enqueue delete "$OLD_ZONE" +"$QUEUE" enqueue create "$NEW_ZONE" "$MASTER_IP" +log "[>] Zone rename $OLD_ZONE -> $NEW_ZONE queued for SecondDNS" +( "$QUEUE" flush >/dev/null 2>&1 & ) exit 0 diff --git a/hosting-panels/plesk/install.sh b/hosting-panels/plesk/install.sh index c0566b0..a00da77 100755 --- a/hosting-panels/plesk/install.sh +++ b/hosting-panels/plesk/install.sh @@ -232,6 +232,22 @@ for script in domain_create.sh domain_delete.sh domain_rename.sh; do echo "[+] Installed: $SCRIPT_DIR/seconddns-plesk-${script}" done +# --- Offline operation queue --- +echo "" +echo "--- Installing offline operation queue ---" +QUEUE_URL="${REPO_URL%/*}/common/seconddns-queue" +curl -sf --max-time 10 -o /usr/local/bin/seconddns-queue "$QUEUE_URL?t=$(date +%s)" +chmod +x /usr/local/bin/seconddns-queue +mkdir -p /var/lib/seconddns +if ! command -v sqlite3 &>/dev/null; then + echo "[!] Warning: sqlite3 not found — offline queue disabled until installed" +fi +cat > /etc/cron.d/seconddns-queue << 'CRONEOF' +* * * * * root /usr/local/bin/seconddns-queue flush >/dev/null 2>&1 +CRONEOF +chmod 644 /etc/cron.d/seconddns-queue +echo "[+] Queue tool: /usr/local/bin/seconddns-queue (cron flush every minute)" + # --- Register Plesk event handlers --- echo "" echo "--- Registering Plesk event handlers ---" diff --git a/hosting-panels/plesk/uninstall.sh b/hosting-panels/plesk/uninstall.sh index 8227253..5ceea94 100755 --- a/hosting-panels/plesk/uninstall.sh +++ b/hosting-panels/plesk/uninstall.sh @@ -121,3 +121,7 @@ fi echo "" echo "=== Uninstall complete ===" echo " Your zones on the secondary DNS are not deleted automatically." + +# Remove offline queue +rm -f /usr/local/bin/seconddns-queue /etc/cron.d/seconddns-queue +rm -rf /var/lib/seconddns diff --git a/nagios_plugins/check_seconddns_queue.sh b/nagios_plugins/check_seconddns_queue.sh new file mode 100755 index 0000000..33617a6 --- /dev/null +++ b/nagios_plugins/check_seconddns_queue.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# Copyright © 2025-2026 SecondDNS +# Licensed under GNU General Public License v3.0 or SecondDNS Commercial License +# See LICENSE (GPLv3) or LICENSE.COMMERCIAL (commercial) for details +# Nagios/Icinga check for the SecondDNS offline operation queue. +# +# Runs on the panel server where the SecondDNS integration is installed and +# inspects the local queue (/var/lib/seconddns/queue.db). A growing or stale +# queue means zone operations are not reaching the SecondDNS API. +# +# Usage: +# check_seconddns_queue.sh [-w ] [-c ] +# +# Options: +# -w MINUTES WARNING when the oldest pending op is older (default: 10) +# -c MINUTES CRITICAL when the oldest pending op is older (default: 30) +# -h Show this help +# +# failed ops > 0 always raises at least WARNING. +# +# Exit codes: 0 = OK, 1 = WARNING, 2 = CRITICAL, 3 = UNKNOWN + +set -euo pipefail + +QUEUE_BIN="${SECONDDNS_QUEUE_BIN:-/usr/local/bin/seconddns-queue}" +WARN_MIN=10 +CRIT_MIN=30 + +usage() { + sed -n '2,/^$/s/^# \?//p' "$0" + exit 3 +} + +while getopts "w:c:h" opt; do + case $opt in + w) WARN_MIN="$OPTARG" ;; + c) CRIT_MIN="$OPTARG" ;; + h|*) usage ;; + esac +done + +if [ ! -x "$QUEUE_BIN" ]; then + echo "UNKNOWN - $QUEUE_BIN not installed" + exit 3 +fi + +json=$("$QUEUE_BIN" status --json 2>/dev/null) || true +if [ -z "$json" ]; then + echo "UNKNOWN - queue status unavailable" + exit 3 +fi + +pending=$(echo "$json" | python3 -c "import sys,json; print(json.load(sys.stdin)['pending'])") +failed=$(echo "$json" | python3 -c "import sys,json; print(json.load(sys.stdin)['failed'])") +oldest=$(echo "$json" | python3 -c "import sys,json; print(json.load(sys.stdin)['oldest_age_seconds'])") + +perfdata="pending=$pending failed=$failed oldest_age=${oldest}s;$((WARN_MIN*60));$((CRIT_MIN*60))" + +if [ "$oldest" -ge $((CRIT_MIN*60)) ]; then + echo "CRITICAL - oldest queued op is $((oldest/60)) min old ($pending pending, $failed failed) | $perfdata" + exit 2 +fi +if [ "$oldest" -ge $((WARN_MIN*60)) ] || [ "$failed" -gt 0 ]; then + echo "WARNING - $pending pending (oldest $((oldest/60)) min), $failed failed | $perfdata" + exit 1 +fi +if [ "$pending" -gt 0 ]; then + echo "OK - $pending op(s) queued, draining | $perfdata" +else + echo "OK - queue empty | $perfdata" +fi +exit 0 diff --git a/zabbix_templates/seconddns_queue.yaml b/zabbix_templates/seconddns_queue.yaml new file mode 100644 index 0000000..955080d --- /dev/null +++ b/zabbix_templates/seconddns_queue.yaml @@ -0,0 +1,100 @@ +zabbix_export: + version: "7.0" + template_groups: + - uuid: 7df96b18c230490a9a0a9e2307226338 + name: Templates/Applications + + templates: + - uuid: a1b2c3d4e5f60000000000000000dns2 + template: SecondDNS Offline Queue + name: SecondDNS Offline Operation Queue + description: | + Monitors the local SecondDNS offline operation queue on a panel server + (Plesk / cPanel / DirectAdmin / CyberPanel integration). + A growing or stale queue means zone operations are not reaching the + SecondDNS API (outage or maintenance on the SecondDNS side, or a local + delivery problem such as a bad API key or firewall). + + Requires the Zabbix agent UserParameter on the panel server: + UserParameter=seconddns.queue.json,/usr/local/bin/seconddns-queue status --json + groups: + - name: Templates/Applications + + macros: + - macro: "{$SECONDDNS_QUEUE_WARN_AGE}" + value: "600" + description: "WARNING when the oldest pending op is older (seconds)" + - macro: "{$SECONDDNS_QUEUE_CRIT_AGE}" + value: "1800" + description: "CRITICAL when the oldest pending op is older (seconds)" + + items: + - uuid: a1b2c3d4e5f60000000000000000qit1 + name: "SecondDNS queue: status JSON" + type: ZABBIX_ACTIVE + key: seconddns.queue.json + value_type: TEXT + delay: 1m + history: 7d + description: Raw JSON from seconddns-queue status --json + + - uuid: a1b2c3d4e5f60000000000000000qit2 + name: "SecondDNS queue: pending operations" + type: DEPENDENT + key: seconddns.queue.pending + master_item: + key: seconddns.queue.json + value_type: UNSIGNED + preprocessing: + - type: JSONPATH + parameters: + - $.pending + history: 30d + + - uuid: a1b2c3d4e5f60000000000000000qit3 + name: "SecondDNS queue: failed operations" + type: DEPENDENT + key: seconddns.queue.failed + master_item: + key: seconddns.queue.json + value_type: UNSIGNED + preprocessing: + - type: JSONPATH + parameters: + - $.failed + history: 30d + + - uuid: a1b2c3d4e5f60000000000000000qit4 + name: "SecondDNS queue: oldest pending op age" + type: DEPENDENT + key: seconddns.queue.oldest_age + master_item: + key: seconddns.queue.json + value_type: UNSIGNED + units: s + preprocessing: + - type: JSONPATH + parameters: + - $.oldest_age_seconds + history: 30d + + triggers: + - uuid: a1b2c3d4e5f60000000000000000qtr1 + expression: "last(/SecondDNS Offline Queue/seconddns.queue.oldest_age)>{$SECONDDNS_QUEUE_CRIT_AGE}" + name: "SecondDNS: zone operations stuck in queue (>{$SECONDDNS_QUEUE_CRIT_AGE}s)" + priority: HIGH + description: "The SecondDNS API has been unreachable for a long time; queued zone operations are not being delivered." + + - uuid: a1b2c3d4e5f60000000000000000qtr2 + expression: "last(/SecondDNS Offline Queue/seconddns.queue.oldest_age)>{$SECONDDNS_QUEUE_WARN_AGE}" + name: "SecondDNS: zone operations queued (>{$SECONDDNS_QUEUE_WARN_AGE}s)" + priority: WARNING + dependencies: + - name: "SecondDNS: zone operations stuck in queue (>{$SECONDDNS_QUEUE_CRIT_AGE}s)" + expression: "last(/SecondDNS Offline Queue/seconddns.queue.oldest_age)>{$SECONDDNS_QUEUE_CRIT_AGE}" + + - uuid: a1b2c3d4e5f60000000000000000qtr3 + expression: "last(/SecondDNS Offline Queue/seconddns.queue.failed)>0" + name: "SecondDNS: permanently failed zone operations in queue" + priority: WARNING + description: "Operations rejected by the API (4xx) need manual review: seconddns-queue status; see /var/log/seconddns.log." From e8095f25c2847acb3ec9e44f9a03fe5bb0a3b7e9 Mon Sep 17 00:00:00 2001 From: Ihor Rusyn <0kaba0@gmail.com> Date: Tue, 7 Jul 2026 02:11:55 +0200 Subject: [PATCH 2/3] refactor(queue): systemd delivery worker instead of per-minute cron - new seconddns-queued (python3 stdlib): watches the queue, delivers in strict FIFO, exponential backoff while the API is down, configurable via [queue] section in /etc/seconddns.conf; --once mode for manual flush; systemd unit with Restart=always - hooks are now enqueue-only (single INSERT, never block on API timeouts); delivery goes exclusively through the worker - cyberpanel: add_zone/remove_zone enqueue-only (reads stay direct) - seconddns-queue CLI simplified: enqueue/status; flush execs the worker --once - installers deploy worker + unit and enable the service; no cron - tests: 16 checks, incl. daemon-mode pickup and outage recovery Ref: seconddns/k8s_web#265 --- .gitignore | 2 + README.md | 32 ++- hosting-panels/common/seconddns-queue | 136 +---------- hosting-panels/common/seconddns-queued | 226 ++++++++++++++++++ .../common/seconddns-queued.service | 21 ++ hosting-panels/common/tests/test_queue.sh | 23 +- hosting-panels/cpanel/domain_create.sh | 1 - hosting-panels/cpanel/domain_delete.sh | 1 - hosting-panels/cpanel/install.sh | 16 +- hosting-panels/cpanel/uninstall.sh | 5 +- hosting-panels/cyberpanel/install.sh | 15 +- hosting-panels/cyberpanel/seconddns.py | 49 +--- hosting-panels/cyberpanel/uninstall.sh | 5 +- hosting-panels/directadmin/dns_create_post.sh | 1 - hosting-panels/directadmin/dns_delete_post.sh | 1 - hosting-panels/directadmin/install.sh | 16 +- hosting-panels/directadmin/uninstall.sh | 5 +- hosting-panels/plesk/domain_create.sh | 1 - hosting-panels/plesk/domain_delete.sh | 1 - hosting-panels/plesk/domain_rename.sh | 1 - hosting-panels/plesk/install.sh | 16 +- hosting-panels/plesk/uninstall.sh | 5 +- 22 files changed, 354 insertions(+), 225 deletions(-) create mode 100755 hosting-panels/common/seconddns-queued create mode 100644 hosting-panels/common/seconddns-queued.service diff --git a/.gitignore b/.gitignore index e43b0f9..2348c91 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ .DS_Store +__pycache__/ +*.pyc diff --git a/README.md b/README.md index c4bddb6..31fd301 100644 --- a/README.md +++ b/README.md @@ -52,25 +52,37 @@ See the README in each directory for options, AXFR configuration, and troublesho Every panel integration ships with a local offline queue ([`hosting-panels/common/seconddns-queue`](hosting-panels/common/seconddns-queue)). -Zone operations are enqueued into a SQLite database -(`/var/lib/seconddns/queue.db`) and delivered immediately; if the SecondDNS API -is unreachable (outage or maintenance), the operations stay queued and a cron -job (`/etc/cron.d/seconddns-queue`, every minute) replays them in strict FIFO -order once the API is back. Nothing your customers do in the panel during a -SecondDNS downtime is lost. +Panel hooks enqueue zone operations into a SQLite database +(`/var/lib/seconddns/queue.db`); the `seconddns-queued` systemd worker is the +single delivery path — it watches the queue and delivers operations in strict +FIFO order. If the SecondDNS API is unreachable (outage or maintenance), the +worker backs off exponentially and retries until everything is delivered. +Nothing your customers do in the panel during a SecondDNS downtime is lost, +and hooks never block on API timeouts (they only do a local INSERT). + +Worker settings (optional `[queue]` section in `/etc/seconddns.conf`): + +```ini +[queue] +poll_interval = 3 # seconds between queue checks when idle +http_timeout = 15 # seconds per API request +backoff_min = 30 # first retry delay when the API is down +backoff_max = 300 # retry delay ceiling +``` Replay semantics: -- retryable errors (timeout, 5xx, redirects) stop the drain until the next cron tick +- retryable errors (timeout, 5xx, redirects) pause delivery; the worker retries with exponential backoff - duplicate delivery is safe: `409` on create and `404` on delete count as success - hard errors (`400`/`422`) mark the operation `failed` and the drain continues Inspect the queue on any panel server: ```bash -seconddns-queue status # pending / failed / oldest age (exit 0/1/2) -seconddns-queue status --json # same as JSON (for monitoring agents) -seconddns-queue flush # force an immediate drain +seconddns-queue status # pending / failed / oldest age (exit 0/1/2) +seconddns-queue status --json # same as JSON (for monitoring agents) +seconddns-queue flush # manual one-shot drain (diagnostics) +systemctl status seconddns-queued # delivery worker ``` Requires `sqlite3` (present by default on all supported panels). diff --git a/hosting-panels/common/seconddns-queue b/hosting-panels/common/seconddns-queue index ab8df85..b226bd4 100755 --- a/hosting-panels/common/seconddns-queue +++ b/hosting-panels/common/seconddns-queue @@ -2,33 +2,28 @@ # Copyright © 2025-2026 SecondDNS # Licensed under GNU General Public License v3.0 or SecondDNS Commercial License # See LICENSE (GPLv3) or LICENSE.COMMERCIAL (commercial) for details -# SecondDNS offline operation queue. +# SecondDNS offline operation queue — CLI. # -# Panel hooks enqueue every zone operation here and immediately trigger a -# flush. If the SecondDNS API is unreachable (outage or maintenance), the -# operation stays in a local SQLite queue and is replayed by cron in strict -# FIFO order once the API is back. +# Panel hooks enqueue zone operations into a local SQLite queue; the +# seconddns-queued systemd worker delivers them to the SecondDNS API in +# strict FIFO order (surviving outages and maintenance windows). # # Usage: # seconddns-queue enqueue [master_ip] -# seconddns-queue flush +# seconddns-queue flush # manual one-shot drain (diagnostics) # seconddns-queue status [--json] # # Exit codes for status: 0 = empty/healthy, 1 = pending backlog, 2 = failed ops # # Environment overrides (mainly for tests): -# SECONDDNS_CONF config file (default /etc/seconddns.conf) # SECONDDNS_QUEUE_DB sqlite db path (default /var/lib/seconddns/queue.db) -# SECONDDNS_QUEUE_LOCK flush lockfile (default /var/lib/seconddns/queue.lock) # SECONDDNS_LOG log file (default /var/log/seconddns.log) set -u -CONFIG="${SECONDDNS_CONF:-/etc/seconddns.conf}" QDB="${SECONDDNS_QUEUE_DB:-/var/lib/seconddns/queue.db}" -QLOCK="${SECONDDNS_QUEUE_LOCK:-/var/lib/seconddns/queue.lock}" LOG="${SECONDDNS_LOG:-/var/log/seconddns.log}" -UA="SecondDNS-Queue/1.0" +WORKER="${SECONDDNS_QUEUED_BIN:-/usr/local/bin/seconddns-queued}" log() { echo "$(date '+%Y-%m-%d %H:%M:%S') $1" >> "$LOG"; } @@ -51,74 +46,6 @@ init_db() { );" >/dev/null } -read_conf() { - [ -f "$CONFIG" ] || { log "[!] queue: config $CONFIG missing"; exit 0; } - API_URL=$(sed -nE 's/^api_url[[:space:]]*=[[:space:]]*//p' "$CONFIG") - API_KEY=$(sed -nE 's/^api_key[[:space:]]*=[[:space:]]*//p' "$CONFIG") - [ -z "$API_URL" ] || [ -z "$API_KEY" ] && { log "[!] queue: api_url/api_key not configured"; exit 0; } -} - -# api_call METHOD PATH [JSON_BODY] -> sets HTTP_CODE and HTTP_BODY -api_call() { - local method="$1" path="$2" body="${3:-}" - local out - if [ -n "$body" ]; then - out=$(curl -s --max-time 15 -w '\n%{http_code}' \ - -X "$method" \ - -H "X-API-Key: $API_KEY" \ - -H "Content-Type: application/json" \ - -H "User-Agent: $UA" \ - -d "$body" \ - "$API_URL$path" 2>/dev/null) - else - out=$(curl -s --max-time 15 -w '\n%{http_code}' \ - -X "$method" \ - -H "X-API-Key: $API_KEY" \ - -H "User-Agent: $UA" \ - "$API_URL$path" 2>/dev/null) - fi - HTTP_CODE=$(echo "$out" | tail -n1) - HTTP_BODY=$(echo "$out" | sed '$d') - [ -z "$HTTP_CODE" ] && HTTP_CODE=000 -} - -# Attempt one operation. Return: 0 done (success), 1 retryable failure, 2 hard failure. -send_op() { - local op="$1" domain="$2" master_ip="$3" - case "$op" in - create) - api_call POST "/api/zones" "{\"name\":\"$domain\",\"masterIp\":\"$master_ip\"}" - case "$HTTP_CODE" in - 200|201) return 0 ;; - 409) log "[~] queue: zone $domain already exists (treated as success)"; return 0 ;; - 400|422) return 2 ;; - *) return 1 ;; - esac - ;; - delete) - api_call GET "/api/zones/by-name/$domain" - case "$HTTP_CODE" in - 200) ;; - 404) log "[~] queue: zone $domain already absent (treated as success)"; return 0 ;; - 400|422) return 2 ;; - *) return 1 ;; - esac - local zone_id - zone_id=$(echo "$HTTP_BODY" | python3 -c "import sys,json; print(json.load(sys.stdin).get('id',''))" 2>/dev/null) - [ -z "$zone_id" ] && return 1 - api_call DELETE "/api/zones/$zone_id" - case "$HTTP_CODE" in - 200|204) return 0 ;; - 404) return 0 ;; - 400|422) return 2 ;; - *) return 1 ;; - esac - ;; - *) - return 2 ;; - esac -} - cmd_enqueue() { local op="$1" domain="$2" master_ip="${3:-}" init_db @@ -126,55 +53,6 @@ cmd_enqueue() { log "[>] queue: enqueued $op $domain" } -cmd_flush() { - init_db - read_conf - # single flusher at a time; strict FIFO requires it - if command -v flock >/dev/null 2>&1; then - exec 9>"$QLOCK" - flock -n 9 || exit 0 - else - # portable fallback (no flock, e.g. macOS): mkdir lock with stale-PID check - if ! mkdir "$QLOCK.d" 2>/dev/null; then - local lock_pid - lock_pid=$(cat "$QLOCK.d/pid" 2>/dev/null) - if [ -n "$lock_pid" ] && kill -0 "$lock_pid" 2>/dev/null; then - exit 0 - fi - rm -rf "$QLOCK.d" - mkdir "$QLOCK.d" 2>/dev/null || exit 0 - fi - echo $$ > "$QLOCK.d/pid" - trap 'rm -rf "$QLOCK.d"' EXIT - fi - - while :; do - local row - row=$(sq -separator '|' "SELECT id, op, domain, IFNULL(master_ip,'') FROM ops WHERE status='pending' ORDER BY id LIMIT 1;") - [ -z "$row" ] && break - local id op domain master_ip - IFS='|' read -r id op domain master_ip <<< "$row" - - send_op "$op" "$domain" "$master_ip" - case $? in - 0) - sq "DELETE FROM ops WHERE id=$id;" >/dev/null - log "[+] queue: delivered $op $domain" - ;; - 2) - sq "UPDATE ops SET status='failed', attempts=attempts+1, last_error='HTTP $HTTP_CODE' WHERE id=$id;" >/dev/null - log "[!] queue: $op $domain permanently failed (HTTP $HTTP_CODE), marked failed" - ;; - *) - # API unreachable / 5xx / maintenance: stop, keep FIFO order, retry next tick - sq "UPDATE ops SET attempts=attempts+1, last_error='HTTP $HTTP_CODE' WHERE id=$id;" >/dev/null - log "[~] queue: API unavailable (HTTP $HTTP_CODE), $(sq "SELECT COUNT(*) FROM ops WHERE status='pending';") op(s) waiting" - break - ;; - esac - done -} - cmd_status() { init_db local pending failed oldest_age @@ -196,7 +74,7 @@ case "${1:-}" in [ $# -ge 3 ] || { echo "usage: $0 enqueue [master_ip]" >&2; exit 1; } cmd_enqueue "$2" "$3" "${4:-}" ;; - flush) cmd_flush ;; + flush) exec "$WORKER" --once ;; status) cmd_status "${2:-}" ;; *) echo "usage: $0 enqueue|flush|status" >&2 diff --git a/hosting-panels/common/seconddns-queued b/hosting-panels/common/seconddns-queued new file mode 100755 index 0000000..e6b1a6b --- /dev/null +++ b/hosting-panels/common/seconddns-queued @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +# Copyright © 2025-2026 SecondDNS +# Licensed under GNU General Public License v3.0 or SecondDNS Commercial License +# See LICENSE (GPLv3) or LICENSE.COMMERCIAL (commercial) for details +"""SecondDNS offline queue delivery worker. + +Watches the local operation queue (SQLite) and delivers zone operations to +the SecondDNS API in strict FIFO order. When the API is unreachable (outage +or maintenance) it backs off exponentially and retries until the queue is +drained. Panel hooks only INSERT into the queue — this worker is the single +delivery path. + +Usage: + seconddns-queued # daemon mode (systemd service) + seconddns-queued --once # drain once and exit (manual flush); + # exit 0 = drained, 1 = API unavailable + +Configuration (/etc/seconddns.conf): + [seconddns] + api_url = https://seconddns.com + api_key = ... + + [queue] # optional, defaults shown + poll_interval = 3 # seconds between queue checks when idle + http_timeout = 15 # seconds per API request + backoff_min = 30 # first retry delay when the API is down + backoff_max = 300 # retry delay ceiling + +Environment overrides (tests): SECONDDNS_CONF, SECONDDNS_QUEUE_DB, SECONDDNS_LOG. +""" + +import configparser +import json +import os +import signal +import sqlite3 +import sys +import time +import urllib.error +import urllib.request + +CONF = os.environ.get("SECONDDNS_CONF", "/etc/seconddns.conf") +QDB = os.environ.get("SECONDDNS_QUEUE_DB", "/var/lib/seconddns/queue.db") +LOG = os.environ.get("SECONDDNS_LOG", "/var/log/seconddns.log") +UA = "SecondDNS-QueueWorker/1.0" + +SCHEMA = """ +CREATE TABLE IF NOT EXISTS ops ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ','now')), + op TEXT NOT NULL, + domain TEXT NOT NULL, + master_ip TEXT, + attempts INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT 'pending', + last_error TEXT +); +""" + +DONE, RETRY, FAILED = 0, 1, 2 + +running = True + + +def log(msg): + line = f"{time.strftime('%Y-%m-%d %H:%M:%S')} {msg}\n" + try: + with open(LOG, "a") as f: + f.write(line) + except OSError: + sys.stderr.write(line) + + +def load_config(): + cp = configparser.ConfigParser() + cp.read(CONF) + api_url = cp.get("seconddns", "api_url", fallback="").strip() + api_key = cp.get("seconddns", "api_key", fallback="").strip() + if not api_url or not api_key: + log("[!] worker: api_url/api_key not configured, exiting") + sys.exit(0) + return { + "api_url": api_url.rstrip("/"), + "api_key": api_key, + "poll_interval": cp.getint("queue", "poll_interval", fallback=3), + "http_timeout": cp.getint("queue", "http_timeout", fallback=15), + "backoff_min": cp.getint("queue", "backoff_min", fallback=30), + "backoff_max": cp.getint("queue", "backoff_max", fallback=300), + } + + +def db(): + os.makedirs(os.path.dirname(QDB), exist_ok=True) + conn = sqlite3.connect(QDB, timeout=5) + conn.execute("PRAGMA journal_mode=WAL") + conn.executescript(SCHEMA) + return conn + + +def api(cfg, method, path, payload=None): + """Returns (status_code, body_dict). status 0 = network-level failure.""" + url = cfg["api_url"] + path + body = json.dumps(payload).encode() if payload is not None else None + req = urllib.request.Request(url, data=body, method=method, headers={ + "X-API-Key": cfg["api_key"], + "Content-Type": "application/json", + "User-Agent": UA, + }) + try: + with urllib.request.urlopen(req, timeout=cfg["http_timeout"]) as resp: + raw = resp.read() + data = json.loads(raw) if raw else {} + return resp.status, data + except urllib.error.HTTPError as e: + try: + data = json.loads(e.read()) + except Exception: + data = {} + return e.code, data + except (urllib.error.URLError, TimeoutError, OSError) as e: + return 0, {"error": str(e)} + + +def send_op(cfg, op, domain, master_ip): + """Deliver one operation. Returns DONE / RETRY / FAILED (+ last status).""" + if op == "create": + code, _ = api(cfg, "POST", "/api/zones", + {"name": domain, "masterIp": master_ip}) + if code in (200, 201): + return DONE, code + if code == 409: + log(f"[~] worker: zone {domain} already exists (treated as success)") + return DONE, code + if code in (400, 422): + return FAILED, code + return RETRY, code + + if op == "delete": + code, zone = api(cfg, "GET", f"/api/zones/by-name/{domain}") + if code == 404: + log(f"[~] worker: zone {domain} already absent (treated as success)") + return DONE, code + if code in (400, 422): + return FAILED, code + if code != 200 or not zone.get("id"): + return RETRY, code + code, _ = api(cfg, "DELETE", f"/api/zones/{zone['id']}") + if code in (200, 204, 404): + return DONE, code + if code in (400, 422): + return FAILED, code + return RETRY, code + + return FAILED, 0 + + +def drain(cfg, conn, once): + """Process pending ops in FIFO order. + + Returns True when the queue is empty, False when delivery must pause + (API unavailable). + """ + backoff = cfg["backoff_min"] + while running: + row = conn.execute( + "SELECT id, op, domain, IFNULL(master_ip,'') FROM ops" + " WHERE status='pending' ORDER BY id LIMIT 1").fetchone() + if row is None: + return True + rid, op, domain, master_ip = row + + result, code = send_op(cfg, op, domain, master_ip) + if result == DONE: + conn.execute("DELETE FROM ops WHERE id=?", (rid,)) + conn.commit() + log(f"[+] worker: delivered {op} {domain}") + backoff = cfg["backoff_min"] + elif result == FAILED: + conn.execute( + "UPDATE ops SET status='failed', attempts=attempts+1," + " last_error=? WHERE id=?", (f"HTTP {code}", rid)) + conn.commit() + log(f"[!] worker: {op} {domain} permanently failed (HTTP {code})") + else: # RETRY — API unavailable, keep FIFO order and wait + conn.execute( + "UPDATE ops SET attempts=attempts+1, last_error=? WHERE id=?", + (f"HTTP {code}", rid)) + conn.commit() + pending = conn.execute( + "SELECT COUNT(*) FROM ops WHERE status='pending'").fetchone()[0] + log(f"[~] worker: API unavailable (HTTP {code})," + f" {pending} op(s) waiting, retry in {backoff}s") + if once: + return False + deadline = time.time() + backoff + while running and time.time() < deadline: + time.sleep(1) + backoff = min(backoff * 2, cfg["backoff_max"]) + return False + + +def main(): + once = "--once" in sys.argv + cfg = load_config() + conn = db() + + def stop(_sig, _frm): + global running + running = False + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + + if once: + sys.exit(0 if drain(cfg, conn, once=True) else 1) + + log("[*] worker: started") + while running: + drain(cfg, conn, once=False) + deadline = time.time() + cfg["poll_interval"] + while running and time.time() < deadline: + time.sleep(0.5) + log("[*] worker: stopped") + + +if __name__ == "__main__": + main() diff --git a/hosting-panels/common/seconddns-queued.service b/hosting-panels/common/seconddns-queued.service new file mode 100644 index 0000000..3c9f63b --- /dev/null +++ b/hosting-panels/common/seconddns-queued.service @@ -0,0 +1,21 @@ +# Copyright © 2025-2026 SecondDNS +# Licensed under GNU General Public License v3.0 or SecondDNS Commercial License +# SecondDNS offline queue delivery worker +[Unit] +Description=SecondDNS offline queue delivery worker +Documentation=https://github.com/seconddns/dns_integrations +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +ExecStart=/usr/local/bin/seconddns-queued +Restart=always +RestartSec=5 +User=root +NoNewPrivileges=true +ProtectSystem=full +ReadWritePaths=/var/lib/seconddns /var/log + +[Install] +WantedBy=multi-user.target diff --git a/hosting-panels/common/tests/test_queue.sh b/hosting-panels/common/tests/test_queue.sh index 43bd6fd..1c91e8a 100755 --- a/hosting-panels/common/tests/test_queue.sh +++ b/hosting-panels/common/tests/test_queue.sh @@ -11,6 +11,8 @@ set -u HERE="$(cd "$(dirname "$0")" && pwd)" QUEUE="$HERE/../seconddns-queue" +WORKER="$HERE/../seconddns-queued" +export SECONDDNS_QUEUED_BIN="$WORKER" TMP="$(mktemp -d)" PORT=8899 @@ -24,6 +26,12 @@ cat > "$SECONDDNS_CONF" < "$TMP/mode" -"$QUEUE" flush +"$QUEUE" flush || true assert_eq "$(pending)" 2 "nothing lost while API is down" attempts=$(sqlite3 "$SECONDDNS_QUEUE_DB" "SELECT attempts FROM ops ORDER BY id LIMIT 1;") assert_eq "$attempts" 1 "first op attempt counted" @@ -142,6 +150,19 @@ echo ok > "$TMP/mode" "$QUEUE" flush assert_eq "$(pending)" 0 "all 20 drained" +echo "== 8. daemon mode: picks up new ops and survives an outage" +echo down > "$TMP/mode" +"$WORKER" & DPID=$! +sleep 1 +"$QUEUE" enqueue create daemon1.example.com 192.0.2.10 +"$QUEUE" enqueue create daemon2.example.com 192.0.2.10 +sleep 2 +assert_eq "$(pending)" 2 "ops wait while API is down" +echo ok > "$TMP/mode" +for i in $(seq 1 15); do [ "$(pending)" = 0 ] && break; sleep 1; done +assert_eq "$(pending)" 0 "daemon drained the queue after API recovery" +kill $DPID 2>/dev/null; wait $DPID 2>/dev/null + echo echo "passed: $PASS, failed: $FAIL" [ "$FAIL" -eq 0 ] diff --git a/hosting-panels/cpanel/domain_create.sh b/hosting-panels/cpanel/domain_create.sh index d8ab17e..065b04d 100755 --- a/hosting-panels/cpanel/domain_create.sh +++ b/hosting-panels/cpanel/domain_create.sh @@ -39,6 +39,5 @@ log "Zone created: $ZONE_NAME (cpanel hook)" QUEUE="/usr/local/bin/seconddns-queue" "$QUEUE" enqueue create "$ZONE_NAME" "$MASTER_IP" log "[>] Zone $ZONE_NAME queued for SecondDNS" -( "$QUEUE" flush >/dev/null 2>&1 & ) exit 0 diff --git a/hosting-panels/cpanel/domain_delete.sh b/hosting-panels/cpanel/domain_delete.sh index 14207f3..dcd8089 100755 --- a/hosting-panels/cpanel/domain_delete.sh +++ b/hosting-panels/cpanel/domain_delete.sh @@ -37,6 +37,5 @@ log "Zone deleted: $ZONE_NAME (cpanel hook)" QUEUE="/usr/local/bin/seconddns-queue" "$QUEUE" enqueue delete "$ZONE_NAME" log "[>] Zone $ZONE_NAME removal queued for SecondDNS" -( "$QUEUE" flush >/dev/null 2>&1 & ) exit 0 diff --git a/hosting-panels/cpanel/install.sh b/hosting-panels/cpanel/install.sh index dd833d3..dfedb61 100755 --- a/hosting-panels/cpanel/install.sh +++ b/hosting-panels/cpanel/install.sh @@ -174,18 +174,18 @@ done # --- Offline operation queue --- echo "" echo "--- Installing offline operation queue ---" -QUEUE_URL="${REPO_URL%/*}/common/seconddns-queue" -curl -sf --max-time 10 -o /usr/local/bin/seconddns-queue "$QUEUE_URL?t=$(date +%s)" -chmod +x /usr/local/bin/seconddns-queue +COMMON_URL="${REPO_URL%/*}/common" +curl -sf --max-time 10 -o /usr/local/bin/seconddns-queue "$COMMON_URL/seconddns-queue?t=$(date +%s)" +curl -sf --max-time 10 -o /usr/local/bin/seconddns-queued "$COMMON_URL/seconddns-queued?t=$(date +%s)" +curl -sf --max-time 10 -o /etc/systemd/system/seconddns-queued.service "$COMMON_URL/seconddns-queued.service?t=$(date +%s)" +chmod +x /usr/local/bin/seconddns-queue /usr/local/bin/seconddns-queued mkdir -p /var/lib/seconddns if ! command -v sqlite3 &>/dev/null; then echo "[!] Warning: sqlite3 not found — offline queue disabled until installed" fi -cat > /etc/cron.d/seconddns-queue << 'CRONEOF' -* * * * * root /usr/local/bin/seconddns-queue flush >/dev/null 2>&1 -CRONEOF -chmod 644 /etc/cron.d/seconddns-queue -echo "[+] Queue tool: /usr/local/bin/seconddns-queue (cron flush every minute)" +systemctl daemon-reload +systemctl enable --now seconddns-queued.service +echo "[+] Queue worker: seconddns-queued.service (systemd, FIFO delivery with backoff)" # Register hooks echo "" diff --git a/hosting-panels/cpanel/uninstall.sh b/hosting-panels/cpanel/uninstall.sh index 002d58c..69606a6 100755 --- a/hosting-panels/cpanel/uninstall.sh +++ b/hosting-panels/cpanel/uninstall.sh @@ -102,5 +102,8 @@ echo " Note: AXFR settings in WHM and BIND config must be removed manually." echo " Verify: $HOOKS_BIN list" # Remove offline queue -rm -f /usr/local/bin/seconddns-queue /etc/cron.d/seconddns-queue +systemctl disable --now seconddns-queued.service 2>/dev/null +rm -f /etc/systemd/system/seconddns-queued.service +systemctl daemon-reload 2>/dev/null +rm -f /usr/local/bin/seconddns-queue /usr/local/bin/seconddns-queued rm -rf /var/lib/seconddns diff --git a/hosting-panels/cyberpanel/install.sh b/hosting-panels/cyberpanel/install.sh index ad55b1a..d14c66a 100755 --- a/hosting-panels/cyberpanel/install.sh +++ b/hosting-panels/cyberpanel/install.sh @@ -222,17 +222,18 @@ with open(p, 'w') as f: f.write(c) cp "$CYBER_SRC/seconddns-signals.service" "$SYSTEMD_SERVICE" # --- Offline operation queue --- -cp "$WORK_DIR/dns_integrations/hosting-panels/common/seconddns-queue" /usr/local/bin/seconddns-queue -chmod +x /usr/local/bin/seconddns-queue +COMMON_SRC="$WORK_DIR/dns_integrations/hosting-panels/common" +cp "$COMMON_SRC/seconddns-queue" /usr/local/bin/seconddns-queue +cp "$COMMON_SRC/seconddns-queued" /usr/local/bin/seconddns-queued +cp "$COMMON_SRC/seconddns-queued.service" /etc/systemd/system/seconddns-queued.service +chmod +x /usr/local/bin/seconddns-queue /usr/local/bin/seconddns-queued mkdir -p /var/lib/seconddns if ! command -v sqlite3 &>/dev/null; then echo "[!] Warning: sqlite3 not found — offline queue disabled until installed" fi -cat > /etc/cron.d/seconddns-queue << 'CRONEOF' -* * * * * root /usr/local/bin/seconddns-queue flush >/dev/null 2>&1 -CRONEOF -chmod 644 /etc/cron.d/seconddns-queue -echo "[+] Queue tool: /usr/local/bin/seconddns-queue (cron flush every minute)" +systemctl daemon-reload +systemctl enable --now seconddns-queued.service +echo "[+] Queue worker: seconddns-queued.service (systemd, FIFO delivery with backoff)" systemctl daemon-reload systemctl enable seconddns-signals.service 2>/dev/null echo "[+] Systemd hook installed (survives CyberPanel updates)" diff --git a/hosting-panels/cyberpanel/seconddns.py b/hosting-panels/cyberpanel/seconddns.py index 1d67636..32be617 100755 --- a/hosting-panels/cyberpanel/seconddns.py +++ b/hosting-panels/cyberpanel/seconddns.py @@ -110,19 +110,14 @@ def api_request(config, method, path, data=None): QUEUE_BIN = "/usr/local/bin/seconddns-queue" -def _retryable(status): - """API unreachable, 5xx or maintenance redirect — worth retrying later.""" - return status == 0 or status >= 500 or 300 <= status < 400 - - def queue_op(op, domain, master_ip=""): - """Persist the operation in the local offline queue for cron replay.""" + """Persist the operation in the local offline queue. + + The seconddns-queued systemd worker delivers it in FIFO order.""" try: subprocess.run([QUEUE_BIN, "enqueue", op, domain, master_ip], check=True, timeout=10) - subprocess.Popen([QUEUE_BIN, "flush"], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - logger.info(" API unavailable — %s %s queued for later delivery", op, domain) + logger.info(" %s %s queued for delivery", op, domain) return True except Exception as e: logger.error(" Failed to enqueue %s %s: %s", op, domain, e) @@ -142,22 +137,7 @@ def add_zone(config, domain): if not config or not config.get("master_ip"): return False logger.info("[+] Adding zone: %s (master: %s)", domain, config["master_ip"]) - result = api_request(config, "POST", "/api/zones", { - "name": domain, - "masterIp": config["master_ip"], - }) - if result and result.get("_status"): - status = result["_status"] - error = result.get("error", "Unknown error") - if status == 409: - logger.info(" Already exists, skipping.") - return True - if _retryable(status): - return queue_op("create", domain, config["master_ip"]) - logger.error(" Error (%s): %s", status, error) - return False - logger.info(" Done.") - return True + return queue_op("create", domain, config["master_ip"]) def find_zone_by_name(config, domain): @@ -169,25 +149,8 @@ def find_zone_by_name(config, domain): def remove_zone(config, domain): domain = domain.lower().rstrip(".") - lookup = api_request(config, "GET", f"/api/zones/by-name/{domain}") - status = lookup.get("_status") if isinstance(lookup, dict) else None - if status: - if status == 404: - logger.info("[-] Zone %s not found on secondary DNS.", domain) - return False - if _retryable(status): - return queue_op("delete", domain) - logger.error(" Error (%s): %s", status, lookup.get("error", str(lookup))) - return False logger.info("[-] Removing zone: %s", domain) - result = api_request(config, "DELETE", f"/api/zones/{lookup['id']}") - if result and result.get("_status"): - if _retryable(result["_status"]): - return queue_op("delete", domain) - logger.error(" Error (%s): %s", result.get("_status"), result.get("error", str(result))) - return False - logger.info(" Done.") - return True + return queue_op("delete", domain) def get_cyberpanel_domains(): diff --git a/hosting-panels/cyberpanel/uninstall.sh b/hosting-panels/cyberpanel/uninstall.sh index 6758144..59a3226 100755 --- a/hosting-panels/cyberpanel/uninstall.sh +++ b/hosting-panels/cyberpanel/uninstall.sh @@ -89,5 +89,8 @@ echo "Note: DNS zones on SecondDNS were NOT removed." echo "Delete them manually via Dashboard or API if needed." # Remove offline queue -rm -f /usr/local/bin/seconddns-queue /etc/cron.d/seconddns-queue +systemctl disable --now seconddns-queued.service 2>/dev/null +rm -f /etc/systemd/system/seconddns-queued.service +systemctl daemon-reload 2>/dev/null +rm -f /usr/local/bin/seconddns-queue /usr/local/bin/seconddns-queued rm -rf /var/lib/seconddns diff --git a/hosting-panels/directadmin/dns_create_post.sh b/hosting-panels/directadmin/dns_create_post.sh index 1fde4be..773992a 100644 --- a/hosting-panels/directadmin/dns_create_post.sh +++ b/hosting-panels/directadmin/dns_create_post.sh @@ -31,6 +31,5 @@ log "Zone created: $domain (caller=$caller, user=$username)" QUEUE="/usr/local/bin/seconddns-queue" "$QUEUE" enqueue create "$domain" "$MASTER_IP" log "[>] Zone $domain queued for SecondDNS" -( "$QUEUE" flush >/dev/null 2>&1 & ) exit 0 diff --git a/hosting-panels/directadmin/dns_delete_post.sh b/hosting-panels/directadmin/dns_delete_post.sh index 0e159c5..80e57b9 100644 --- a/hosting-panels/directadmin/dns_delete_post.sh +++ b/hosting-panels/directadmin/dns_delete_post.sh @@ -24,6 +24,5 @@ log "Zone deleted: $domain (user=$USERNAME)" QUEUE="/usr/local/bin/seconddns-queue" "$QUEUE" enqueue delete "$domain" log "[>] Zone $domain removal queued for SecondDNS" -( "$QUEUE" flush >/dev/null 2>&1 & ) exit 0 diff --git a/hosting-panels/directadmin/install.sh b/hosting-panels/directadmin/install.sh index 9fa2bbb..142a661 100644 --- a/hosting-panels/directadmin/install.sh +++ b/hosting-panels/directadmin/install.sh @@ -171,18 +171,18 @@ done # --- Offline operation queue --- echo "" echo "--- Installing offline operation queue ---" -QUEUE_URL="${REPO_URL%/*}/common/seconddns-queue" -curl -sf --max-time 10 -o /usr/local/bin/seconddns-queue "$QUEUE_URL?t=$(date +%s)" -chmod +x /usr/local/bin/seconddns-queue +COMMON_URL="${REPO_URL%/*}/common" +curl -sf --max-time 10 -o /usr/local/bin/seconddns-queue "$COMMON_URL/seconddns-queue?t=$(date +%s)" +curl -sf --max-time 10 -o /usr/local/bin/seconddns-queued "$COMMON_URL/seconddns-queued?t=$(date +%s)" +curl -sf --max-time 10 -o /etc/systemd/system/seconddns-queued.service "$COMMON_URL/seconddns-queued.service?t=$(date +%s)" +chmod +x /usr/local/bin/seconddns-queue /usr/local/bin/seconddns-queued mkdir -p /var/lib/seconddns if ! command -v sqlite3 &>/dev/null; then echo "[!] Warning: sqlite3 not found — offline queue disabled until installed" fi -cat > /etc/cron.d/seconddns-queue << 'CRONEOF' -* * * * * root /usr/local/bin/seconddns-queue flush >/dev/null 2>&1 -CRONEOF -chmod 644 /etc/cron.d/seconddns-queue -echo "[+] Queue tool: /usr/local/bin/seconddns-queue (cron flush every minute)" +systemctl daemon-reload +systemctl enable --now seconddns-queued.service +echo "[+] Queue worker: seconddns-queued.service (systemd, FIFO delivery with backoff)" # --- Detect DNS server and configure AXFR --- echo "" diff --git a/hosting-panels/directadmin/uninstall.sh b/hosting-panels/directadmin/uninstall.sh index 9e83e65..a3d3551 100644 --- a/hosting-panels/directadmin/uninstall.sh +++ b/hosting-panels/directadmin/uninstall.sh @@ -33,5 +33,8 @@ echo " Log file kept at: $LOG_FILE" echo " Your zones on the secondary DNS are not deleted automatically." # Remove offline queue -rm -f /usr/local/bin/seconddns-queue /etc/cron.d/seconddns-queue +systemctl disable --now seconddns-queued.service 2>/dev/null +rm -f /etc/systemd/system/seconddns-queued.service +systemctl daemon-reload 2>/dev/null +rm -f /usr/local/bin/seconddns-queue /usr/local/bin/seconddns-queued rm -rf /var/lib/seconddns diff --git a/hosting-panels/plesk/domain_create.sh b/hosting-panels/plesk/domain_create.sh index 31d8b18..8f686ec 100755 --- a/hosting-panels/plesk/domain_create.sh +++ b/hosting-panels/plesk/domain_create.sh @@ -36,6 +36,5 @@ fi QUEUE="/usr/local/bin/seconddns-queue" "$QUEUE" enqueue create "$ZONE_NAME" "$MASTER_IP" log "[>] Zone $ZONE_NAME queued for SecondDNS" -( "$QUEUE" flush >/dev/null 2>&1 & ) exit 0 diff --git a/hosting-panels/plesk/domain_delete.sh b/hosting-panels/plesk/domain_delete.sh index b21a232..0fd50b2 100755 --- a/hosting-panels/plesk/domain_delete.sh +++ b/hosting-panels/plesk/domain_delete.sh @@ -35,6 +35,5 @@ fi QUEUE="/usr/local/bin/seconddns-queue" "$QUEUE" enqueue delete "$ZONE_NAME" log "[>] Zone $ZONE_NAME removal queued for SecondDNS" -( "$QUEUE" flush >/dev/null 2>&1 & ) exit 0 diff --git a/hosting-panels/plesk/domain_rename.sh b/hosting-panels/plesk/domain_rename.sh index e32b089..56126a6 100644 --- a/hosting-panels/plesk/domain_rename.sh +++ b/hosting-panels/plesk/domain_rename.sh @@ -47,6 +47,5 @@ QUEUE="/usr/local/bin/seconddns-queue" "$QUEUE" enqueue delete "$OLD_ZONE" "$QUEUE" enqueue create "$NEW_ZONE" "$MASTER_IP" log "[>] Zone rename $OLD_ZONE -> $NEW_ZONE queued for SecondDNS" -( "$QUEUE" flush >/dev/null 2>&1 & ) exit 0 diff --git a/hosting-panels/plesk/install.sh b/hosting-panels/plesk/install.sh index a00da77..1f3a2ee 100755 --- a/hosting-panels/plesk/install.sh +++ b/hosting-panels/plesk/install.sh @@ -235,18 +235,18 @@ done # --- Offline operation queue --- echo "" echo "--- Installing offline operation queue ---" -QUEUE_URL="${REPO_URL%/*}/common/seconddns-queue" -curl -sf --max-time 10 -o /usr/local/bin/seconddns-queue "$QUEUE_URL?t=$(date +%s)" -chmod +x /usr/local/bin/seconddns-queue +COMMON_URL="${REPO_URL%/*}/common" +curl -sf --max-time 10 -o /usr/local/bin/seconddns-queue "$COMMON_URL/seconddns-queue?t=$(date +%s)" +curl -sf --max-time 10 -o /usr/local/bin/seconddns-queued "$COMMON_URL/seconddns-queued?t=$(date +%s)" +curl -sf --max-time 10 -o /etc/systemd/system/seconddns-queued.service "$COMMON_URL/seconddns-queued.service?t=$(date +%s)" +chmod +x /usr/local/bin/seconddns-queue /usr/local/bin/seconddns-queued mkdir -p /var/lib/seconddns if ! command -v sqlite3 &>/dev/null; then echo "[!] Warning: sqlite3 not found — offline queue disabled until installed" fi -cat > /etc/cron.d/seconddns-queue << 'CRONEOF' -* * * * * root /usr/local/bin/seconddns-queue flush >/dev/null 2>&1 -CRONEOF -chmod 644 /etc/cron.d/seconddns-queue -echo "[+] Queue tool: /usr/local/bin/seconddns-queue (cron flush every minute)" +systemctl daemon-reload +systemctl enable --now seconddns-queued.service +echo "[+] Queue worker: seconddns-queued.service (systemd, FIFO delivery with backoff)" # --- Register Plesk event handlers --- echo "" diff --git a/hosting-panels/plesk/uninstall.sh b/hosting-panels/plesk/uninstall.sh index 5ceea94..00ddf43 100755 --- a/hosting-panels/plesk/uninstall.sh +++ b/hosting-panels/plesk/uninstall.sh @@ -123,5 +123,8 @@ echo "=== Uninstall complete ===" echo " Your zones on the secondary DNS are not deleted automatically." # Remove offline queue -rm -f /usr/local/bin/seconddns-queue /etc/cron.d/seconddns-queue +systemctl disable --now seconddns-queued.service 2>/dev/null +rm -f /etc/systemd/system/seconddns-queued.service +systemctl daemon-reload 2>/dev/null +rm -f /usr/local/bin/seconddns-queue /usr/local/bin/seconddns-queued rm -rf /var/lib/seconddns From 859fef5cb7748a32d05bdbb00f49af202ad67237 Mon Sep 17 00:00:00 2001 From: Ihor Rusyn <0kaba0@gmail.com> Date: Tue, 7 Jul 2026 09:45:19 +0200 Subject: [PATCH 3/3] feat(queue): instant worker wakeup via unix datagram socket - enqueue pokes /var/lib/seconddns/queued.sock after INSERT; the worker wakes immediately instead of polling - poll_interval becomes a rare fallback tick (default 60s) guarding lost wakeups only - AF_UNIX ~104-byte path limit handled by binding/sending via relative paths; socket dir auto-created Ref: seconddns/k8s_web#265 --- README.md | 8 ++-- hosting-panels/common/seconddns-queue | 8 ++++ hosting-panels/common/seconddns-queued | 54 ++++++++++++++++++++--- hosting-panels/common/tests/test_queue.sh | 1 + 4 files changed, 62 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 31fd301..b955249 100644 --- a/README.md +++ b/README.md @@ -54,8 +54,10 @@ Every panel integration ships with a local offline queue ([`hosting-panels/common/seconddns-queue`](hosting-panels/common/seconddns-queue)). Panel hooks enqueue zone operations into a SQLite database (`/var/lib/seconddns/queue.db`); the `seconddns-queued` systemd worker is the -single delivery path — it watches the queue and delivers operations in strict -FIFO order. If the SecondDNS API is unreachable (outage or maintenance), the +single delivery path — after every enqueue the hook pokes the worker through a +unix datagram socket (`/var/lib/seconddns/queued.sock`), so delivery starts +instantly with no polling; a rare fallback tick (`poll_interval`, 60 s) only +guards against lost wakeups. Operations are delivered in strict FIFO order. If the SecondDNS API is unreachable (outage or maintenance), the worker backs off exponentially and retries until everything is delivered. Nothing your customers do in the panel during a SecondDNS downtime is lost, and hooks never block on API timeouts (they only do a local INSERT). @@ -64,7 +66,7 @@ Worker settings (optional `[queue]` section in `/etc/seconddns.conf`): ```ini [queue] -poll_interval = 3 # seconds between queue checks when idle +poll_interval = 60 # fallback idle tick (wakeups are socket-driven) http_timeout = 15 # seconds per API request backoff_min = 30 # first retry delay when the API is down backoff_max = 300 # retry delay ceiling diff --git a/hosting-panels/common/seconddns-queue b/hosting-panels/common/seconddns-queue index b226bd4..ad11460 100755 --- a/hosting-panels/common/seconddns-queue +++ b/hosting-panels/common/seconddns-queue @@ -22,6 +22,7 @@ set -u QDB="${SECONDDNS_QUEUE_DB:-/var/lib/seconddns/queue.db}" +QSOCK="${SECONDDNS_QUEUE_SOCK:-/var/lib/seconddns/queued.sock}" LOG="${SECONDDNS_LOG:-/var/log/seconddns.log}" WORKER="${SECONDDNS_QUEUED_BIN:-/usr/local/bin/seconddns-queued}" @@ -51,6 +52,13 @@ cmd_enqueue() { init_db sq "INSERT INTO ops (op, domain, master_ip) VALUES ('$op', '$domain', '$master_ip');" >/dev/null log "[>] queue: enqueued $op $domain" + # wake the delivery worker instantly (best-effort) + python3 -c " +import socket, os +os.chdir(os.path.dirname('$QSOCK') or '.') # AF_UNIX ~104-byte path limit +s = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) +s.sendto(b'1', os.path.basename('$QSOCK')) +" 2>/dev/null || true } cmd_status() { diff --git a/hosting-panels/common/seconddns-queued b/hosting-panels/common/seconddns-queued index e6b1a6b..f89b965 100755 --- a/hosting-panels/common/seconddns-queued +++ b/hosting-panels/common/seconddns-queued @@ -21,18 +21,24 @@ Configuration (/etc/seconddns.conf): api_key = ... [queue] # optional, defaults shown - poll_interval = 3 # seconds between queue checks when idle + poll_interval = 60 # fallback idle tick, seconds (wakeups are event-driven + # via the unix socket, this only guards lost wakeups) http_timeout = 15 # seconds per API request backoff_min = 30 # first retry delay when the API is down backoff_max = 300 # retry delay ceiling -Environment overrides (tests): SECONDDNS_CONF, SECONDDNS_QUEUE_DB, SECONDDNS_LOG. +The queue CLI wakes the worker instantly after every enqueue by sending a +datagram to the unix socket (/var/lib/seconddns/queued.sock). + +Environment overrides (tests): SECONDDNS_CONF, SECONDDNS_QUEUE_DB, +SECONDDNS_LOG, SECONDDNS_QUEUE_SOCK. """ import configparser import json import os import signal +import socket import sqlite3 import sys import time @@ -41,6 +47,7 @@ import urllib.request CONF = os.environ.get("SECONDDNS_CONF", "/etc/seconddns.conf") QDB = os.environ.get("SECONDDNS_QUEUE_DB", "/var/lib/seconddns/queue.db") +SOCK = os.environ.get("SECONDDNS_QUEUE_SOCK", "/var/lib/seconddns/queued.sock") LOG = os.environ.get("SECONDDNS_LOG", "/var/log/seconddns.log") UA = "SecondDNS-QueueWorker/1.0" @@ -82,13 +89,45 @@ def load_config(): return { "api_url": api_url.rstrip("/"), "api_key": api_key, - "poll_interval": cp.getint("queue", "poll_interval", fallback=3), + "poll_interval": cp.getint("queue", "poll_interval", fallback=60), "http_timeout": cp.getint("queue", "http_timeout", fallback=15), "backoff_min": cp.getint("queue", "backoff_min", fallback=30), "backoff_max": cp.getint("queue", "backoff_max", fallback=300), } +def wake_socket(): + """Bind the wakeup socket; enqueue-side sends a datagram after INSERT.""" + os.makedirs(os.path.dirname(SOCK) or ".", exist_ok=True) + try: + os.unlink(SOCK) + except FileNotFoundError: + pass + sk = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + # bind via a relative path: AF_UNIX paths are limited to ~104 bytes + cwd = os.getcwd() + try: + os.chdir(os.path.dirname(SOCK) or ".") + sk.bind(os.path.basename(SOCK)) + finally: + os.chdir(cwd) + os.chmod(SOCK, 0o666) + return sk + + +def wait_for_wakeup(sk, timeout): + """Sleep until a wakeup datagram arrives or the fallback tick fires.""" + sk.settimeout(timeout) + try: + sk.recv(16) + # absorb any burst of wakeups queued while we were busy + sk.settimeout(0) + while True: + sk.recv(16) + except (socket.timeout, BlockingIOError, InterruptedError, OSError): + pass + + def db(): os.makedirs(os.path.dirname(QDB), exist_ok=True) conn = sqlite3.connect(QDB, timeout=5) @@ -213,12 +252,15 @@ def main(): if once: sys.exit(0 if drain(cfg, conn, once=True) else 1) + sk = wake_socket() log("[*] worker: started") while running: drain(cfg, conn, once=False) - deadline = time.time() + cfg["poll_interval"] - while running and time.time() < deadline: - time.sleep(0.5) + wait_for_wakeup(sk, cfg["poll_interval"]) + try: + os.unlink(SOCK) + except OSError: + pass log("[*] worker: stopped") diff --git a/hosting-panels/common/tests/test_queue.sh b/hosting-panels/common/tests/test_queue.sh index 1c91e8a..2523f90 100755 --- a/hosting-panels/common/tests/test_queue.sh +++ b/hosting-panels/common/tests/test_queue.sh @@ -20,6 +20,7 @@ export SECONDDNS_CONF="$TMP/seconddns.conf" export SECONDDNS_QUEUE_DB="$TMP/queue.db" export SECONDDNS_QUEUE_LOCK="$TMP/queue.lock" export SECONDDNS_LOG="$TMP/seconddns.log" +export SECONDDNS_QUEUE_SOCK="$TMP/w.sock" cat > "$SECONDDNS_CONF" <