From cbd9ce243254c7f660e6ecec5d3de53979e80833 Mon Sep 17 00:00:00 2001 From: Neil Cook Date: Sat, 9 Aug 2025 09:07:40 +0100 Subject: [PATCH 01/10] Implement http connection count metric --- common/prometheus.cc | 2 +- common/prometheus.hh | 4 ++-- common/wforce-webserver.cc | 5 ----- common/wforce-webserver.hh | 11 ++++++++++- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/common/prometheus.cc b/common/prometheus.cc index 99a48302..470dcecc 100644 --- a/common/prometheus.cc +++ b/common/prometheus.cc @@ -116,7 +116,7 @@ void PrometheusMetrics::incWebhookMetric(unsigned int id, bool success, bool cus } } -void setPrometheusActiveConns(int value) +void setPrometheusActiveConns(size_t value) { if (prom_metrics) prom_metrics->setActiveConns(value); diff --git a/common/prometheus.hh b/common/prometheus.hh index bb60e6ae..6e0f9f8b 100644 --- a/common/prometheus.hh +++ b/common/prometheus.hh @@ -110,7 +110,7 @@ public: worker_response_duration->Observe(duration); } - void setActiveConns(int value) + void setActiveConns(size_t value) { active_connections->Set(value); } @@ -164,7 +164,7 @@ protected: void initPrometheusMetrics(std::shared_ptr pmp); -void setPrometheusActiveConns(int value); +void setPrometheusActiveConns(size_t value); void addPrometheusCommandMetric(const std::string& name); void incPrometheusCommandMetric(const std::string& name); diff --git a/common/wforce-webserver.cc b/common/wforce-webserver.cc index 3530cbab..d2e0ff14 100644 --- a/common/wforce-webserver.cc +++ b/common/wforce-webserver.cc @@ -58,11 +58,6 @@ NetmaskGroup WforceWebserver::getACL() return d_ACL.getCopy(); } -size_t WforceWebserver::getNumConns() -{ - return 0; // XXX - investigate if this can be retrieved from drogon -} - bool WforceWebserver::registerFunc(const std::string& command, HTTPVerb verb, const WforceWSFunc& wsf) { return registerFuncInternal(command, verb, wsf, true); diff --git a/common/wforce-webserver.hh b/common/wforce-webserver.hh index 7df72a70..871a335a 100644 --- a/common/wforce-webserver.hh +++ b/common/wforce-webserver.hh @@ -110,7 +110,9 @@ public: NetmaskGroup getACL(); - size_t getNumConns(); + size_t getNumConns() { + return static_cast(drogon::app().getConnectionCount()); + } // set the basic-auth password void setBasicAuthPassword(const std::string& password) @@ -269,6 +271,13 @@ public: }, {drogon::Delete, "ACLFilter", "LoginFilter"}); } + std::thread connCountThread([this] { + for (;;) { + sleep(1); + setPrometheusActiveConns(getNumConns()); + } + }); + connCountThread.detach(); } static void start(WforceWebserver* wws) From ccefe885ed7ce8b51937c8d2a07b93fa69afa76f Mon Sep 17 00:00:00 2001 From: Neil Cook Date: Fri, 15 Aug 2025 09:25:45 +0100 Subject: [PATCH 02/10] docs: Remove .header files and add to .gitignore --- docs/.gitignore | 1 + docs/swagger/report_api.7.yml.tmp | 0 docs/trackalert_api.7.header | 8 -------- docs/wforce_api.7.header | 8 -------- 4 files changed, 1 insertion(+), 16 deletions(-) create mode 100644 docs/swagger/report_api.7.yml.tmp delete mode 100644 docs/trackalert_api.7.header delete mode 100644 docs/wforce_api.7.header diff --git a/docs/.gitignore b/docs/.gitignore index 6f32e2e8..5c33fa2a 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -4,3 +4,4 @@ /*.amd /*.mmd /*.fmd +/*.header diff --git a/docs/swagger/report_api.7.yml.tmp b/docs/swagger/report_api.7.yml.tmp new file mode 100644 index 00000000..e69de29b diff --git a/docs/trackalert_api.7.header b/docs/trackalert_api.7.header deleted file mode 100644 index 98896a3c..00000000 --- a/docs/trackalert_api.7.header +++ /dev/null @@ -1,8 +0,0 @@ -= WFORCE_API(7) -:doctype: manpage -:manmanual: WFORCE_API - -## NAME - -WFORCE_API - OpenAPI Spec - diff --git a/docs/wforce_api.7.header b/docs/wforce_api.7.header deleted file mode 100644 index 98896a3c..00000000 --- a/docs/wforce_api.7.header +++ /dev/null @@ -1,8 +0,0 @@ -= WFORCE_API(7) -:doctype: manpage -:manmanual: WFORCE_API - -## NAME - -WFORCE_API - OpenAPI Spec - From 343444db4b2a7457a56283ea281baa9a27c2b000 Mon Sep 17 00:00:00 2001 From: Neil Cook Date: Fri, 15 Aug 2025 09:26:21 +0100 Subject: [PATCH 03/10] docker: Expose regression wforce port in docker-compose.yml --- docker/docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index a3a4092b..d7be1d32 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -48,6 +48,8 @@ services: dockerfile: docker/regression/Dockerfile args: - MAXMIND_LICENSE_KEY=${MAXMIND_LICENSE_KEY} + ports: + - "8084:8084" volumes: - "/sys/fs/cgroup:/sys/fs/cgroup:ro" - ./tmp:/var/tmp/testlog From cfcf11d371b6bf1652984bba7a8135b6496de1b7 Mon Sep 17 00:00:00 2001 From: Neil Cook Date: Fri, 15 Aug 2025 09:28:28 +0100 Subject: [PATCH 04/10] Give the connection count thread a meaningful name --- common/wforce-webserver.hh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/wforce-webserver.hh b/common/wforce-webserver.hh index 871a335a..92a1c268 100644 --- a/common/wforce-webserver.hh +++ b/common/wforce-webserver.hh @@ -37,6 +37,7 @@ #include "drogon/drogon.h" #include "prometheus.hh" #include "perf-stats.hh" +#include "ext/threadname.hh" using WforceWSFuncPtr = void (*)(const drogon::HttpRequestPtr& req, const std::string& command, @@ -272,6 +273,7 @@ public: {drogon::Delete, "ACLFilter", "LoginFilter"}); } std::thread connCountThread([this] { + setThreadName("wf/conn-count"); for (;;) { sleep(1); setPrometheusActiveConns(getNumConns()); From 262100609313ee4b916d771e031a2222f71d0714 Mon Sep 17 00:00:00 2001 From: Neil Cook Date: Fri, 15 Aug 2025 09:29:08 +0100 Subject: [PATCH 05/10] Add a template grafana dashboard --- grafana/grafana_dashboard.json | 1545 ++++++++++++++++++++++++++++++++ 1 file changed, 1545 insertions(+) create mode 100644 grafana/grafana_dashboard.json diff --git a/grafana/grafana_dashboard.json b/grafana/grafana_dashboard.json new file mode 100644 index 00000000..0f4e4873 --- /dev/null +++ b/grafana/grafana_dashboard.json @@ -0,0 +1,1545 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "12.1.0-pre" + }, + { + "type": "panel", + "id": "heatmap", + "name": "Heatmap", + "version": "" + }, + { + "type": "panel", + "id": "piechart", + "name": "Pie chart", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 12, + "panels": [], + "title": "Responses", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 68, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 15, + "x": 0, + "y": 1 + }, + "id": 40, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "editorMode": "code", + "expr": "sum by(status) (rate(wforce_allow_status_total[$__rate_interval]))", + "legendFormat": "__auto", + "range": true, + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } + } + ], + "title": "Allow Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 9, + "x": 15, + "y": 1 + }, + "id": 6, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by(status) (wforce_allow_status_total)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Allow Response Breakdown", + "type": "piechart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 28, + "panels": [], + "title": "Blocklists & Whitelists", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 14, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg by(type) (wforce_bl_entries)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Blocklist Entries", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 30, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg by(type) (wforce_wl_entries)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Whitelist Entries", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 38, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(type) (wforce_bl_entries)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } + } + ], + "title": "Blocklist Entries over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 39, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(type) (wforce_wl_entries)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } + } + ], + "title": "Whitelist Entries over Time", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 26, + "panels": [], + "repeat": "env", + "title": "Response Duration", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 8, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by(le) (rate(wforce_worker_response_duration_seconds_bucket[$__rate_interval]))", + "format": "heatmap", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Wforce Response duration - Heat map", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 16, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by(le) (rate(wforce_dns_query_response_seconds_bucket[$__rate_interval]))", + "format": "heatmap", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "DNS Query Response", + "type": "heatmap" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 36, + "panels": [], + "title": "DNS Queries", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": true, + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "RBLResolv" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg by(resolver) (rate(wforce_dns_query_response_seconds_sum[$__rate_interval]))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "DNS query response", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "# HELP wforce_dns_queries_total How many DNS queries were performed?\n# TYPE wforce_dns_queries_total counter", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 34, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by(resolver) (rate(wforce_dns_queries_total[$__rate_interval]))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "DNS Queries Count", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 46 + }, + "id": 10, + "panels": [], + "title": "Commands and response duration", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by(cmd) (rate(wforce_commands_total[$__rate_interval]))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Commands", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 32, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(status) (rate(wforce_replication_sent_total[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Sum of replication sent by status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 56 + }, + "id": 37, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(url) (rate(wforce_webhook_events_total[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } + } + ], + "title": "WebHook Events", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 56 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "wforce_web_queue_size", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{__name__}}{pod={{pod}}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "WebHook Queue Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "Num Connections", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 41, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "code", + "expr": "wforce_active_http_connections", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } + } + ], + "title": "Active HTTP Connections", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Abuse Shield Metrics", + "uid": "BoPbuCBNk", + "version": 37, + "weekStart": "" +} From 838996a47776f2474dfbd5856b1552110f327d60 Mon Sep 17 00:00:00 2001 From: Neil Cook Date: Fri, 15 Aug 2025 10:28:33 +0100 Subject: [PATCH 06/10] Add support for grafana dashboard into package building --- Makefile.am | 3 ++- builder-support/debian/wforce.docs | 1 + .../dockerfiles/Dockerfile.target.el-10 | 1 + .../dockerfiles/Dockerfile.target.rl-10 | 20 +++++++++++++++++++ builder-support/dockerfiles/Dockerfile.wforce | 2 +- builder-support/specs/wforce.spec | 1 + 6 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 builder-support/dockerfiles/Dockerfile.target.el-10 create mode 100644 builder-support/dockerfiles/Dockerfile.target.rl-10 diff --git a/Makefile.am b/Makefile.am index e190f73c..ff3747a9 100644 --- a/Makefile.am +++ b/Makefile.am @@ -10,5 +10,6 @@ endif SUBDIRS=ext docs common wforce $(BUILD_TRACKALERT) $(BUILD_DOCKER) LOGSTASH_DIST = elk/logstash/config/logstash.conf elk/logstash/templates/wforce_template.json elk/kibana/kibana_saved_objects.ndjson +GRAFANA_DIST = grafana/grafana_dashboard.json -EXTRA_DIST= README.md LICENSE CHANGELOG.md $(LOGSTASH_DIST) +EXTRA_DIST= README.md LICENSE CHANGELOG.md $(LOGSTASH_DIST) $(GRAFANA_DIST) diff --git a/builder-support/debian/wforce.docs b/builder-support/debian/wforce.docs index d8705667..ae5dc971 100644 --- a/builder-support/debian/wforce.docs +++ b/builder-support/debian/wforce.docs @@ -4,3 +4,4 @@ CHANGELOG.md elk/logstash/config/logstash.conf elk/logstash/templates/wforce_template.json elk/kibana/kibana_saved_objects.ndjson +grafana/grafana_dashboard.json \ No newline at end of file diff --git a/builder-support/dockerfiles/Dockerfile.target.el-10 b/builder-support/dockerfiles/Dockerfile.target.el-10 new file mode 100644 index 00000000..0371bf38 --- /dev/null +++ b/builder-support/dockerfiles/Dockerfile.target.el-10 @@ -0,0 +1 @@ +@INCLUDE Dockerfile.target.rl-10 diff --git a/builder-support/dockerfiles/Dockerfile.target.rl-10 b/builder-support/dockerfiles/Dockerfile.target.rl-10 new file mode 100644 index 00000000..34528cc3 --- /dev/null +++ b/builder-support/dockerfiles/Dockerfile.target.rl-10 @@ -0,0 +1,20 @@ +@EXEC export BUILDER_RHEL_VERSION=10 +@EXEC export BUILDER_RHEL_FLAVOUR=oracle +# First do the source builds +@INCLUDE Dockerfile.target.sdist + +# This defines the distribution base layer +# Put only the bare minimum of common commands here, without dev tools +FROM rockylinux/rockylinux:10 as dist-base + +RUN touch /var/lib/rpm/* && dnf install -y epel-release && \ + crb enable + +RUN dnf -y install clang llvm-devel + +# Do the actual rpm build +@INCLUDE Dockerfile.rpmbuild + +# Do a test install and verify +# Can be skipped with skiptests=1 in the environment +@EXEC [ "$skiptests" = "" ] && include Dockerfile.rpmtest diff --git a/builder-support/dockerfiles/Dockerfile.wforce b/builder-support/dockerfiles/Dockerfile.wforce index caef3baf..2b2f40dc 100644 --- a/builder-support/dockerfiles/Dockerfile.wforce +++ b/builder-support/dockerfiles/Dockerfile.wforce @@ -46,7 +46,7 @@ RUN cd drogon && git checkout tags/v1.9.1 -b v1.9.1 RUN cd drogon && git submodule init && git submodule update && mkdir _build && cd _build && cmake .. -DBUILD_REDIS=OFF -DCMAKE_BUILD_TYPE=Release -DBUILD_ORM=OFF && make && make install ADD CHANGELOG.md configure.ac ext LICENSE Makefile.am README.md NOTICE /wforce/ -@EXEC sdist_dirs=(m4 ext docs regression-tests wforce common trackalert docker elk) +@EXEC sdist_dirs=(m4 ext docs regression-tests wforce common trackalert docker elk grafana) @EXEC for d in ${sdist_dirs[@]} ; do echo "COPY $d/ /wforce/$d/" ; done ADD builder/helpers/set-configure-ac-version.sh /wforce/builder/helpers/ diff --git a/builder-support/specs/wforce.spec b/builder-support/specs/wforce.spec index d943a29e..514e827a 100644 --- a/builder-support/specs/wforce.spec +++ b/builder-support/specs/wforce.spec @@ -110,6 +110,7 @@ mv %{buildroot}/etc/%{name}/%{name}.conf.example %{buildroot}/%{_docdir}/%{name} mv elk/logstash/config/logstash.conf %{buildroot}/%{_docdir}/%{name}-%{version}/ mv elk/logstash/templates/wforce_template.json %{buildroot}/%{_docdir}/%{name}-%{version}/ mv elk/kibana/kibana_saved_objects.ndjson %{buildroot}/%{_docdir}/%{name}-%{version}/ +mv grafana/grafana_dashboard.json %{buildroot}/%{_docdir}/%{name}-%{version}/ %clean rm -rf %{buildroot} From 0983c3cd08fd27a4fe049922575bd76f12e2410b Mon Sep 17 00:00:00 2001 From: Neil Cook Date: Fri, 15 Aug 2025 10:29:28 +0100 Subject: [PATCH 07/10] Document changes in release notes and changelog --- CHANGELOG.md | 4 +++- docs/release_notes/ReleaseNotes-3.0.0.md | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ce99908..78825199 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). -## [Unreleased] +## [3.0.0] ### Added - Support new `fail_type` parameter for determining why a login failed @@ -12,6 +12,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Allow ja3 to be passed to the reset API command - Add support for building amazon-2023 packages - Use asciidoctor to build documentation not pandoc +- Add a sample grafana dashboard for monitoring wforce +- Implemented the (existing but previously always 0) `wforce_active_http_connections` metric for counting active HTTP connections ### Removed - Removed support for Enterprise Linux 7 and Amazon 2 diff --git a/docs/release_notes/ReleaseNotes-3.0.0.md b/docs/release_notes/ReleaseNotes-3.0.0.md index f9f555ae..0fe7848a 100644 --- a/docs/release_notes/ReleaseNotes-3.0.0.md +++ b/docs/release_notes/ReleaseNotes-3.0.0.md @@ -9,6 +9,8 @@ * Allow ja3 to be passed to the reset API command * Add support for building amazon-2023 packages * Use asciidoctor to build documentation not pandoc +* Add a sample grafana dashboard for monitoring wforce +* Implemented the (existing but previously always 0) `wforce_active_http_connections` metric for counting active HTTP connections ## Removed * Removed support for Enterprise Linux 7 and Amazon 2 From b9cbeb3ae751fd108141c51bd367a5d583a4c379 Mon Sep 17 00:00:00 2001 From: Neil Cook Date: Fri, 15 Aug 2025 11:13:20 +0100 Subject: [PATCH 08/10] docker: Expose wforce port on 18084 instead of 8084 due to clash in regression CI --- docker/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index d7be1d32..4c35d5db 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -49,7 +49,7 @@ services: args: - MAXMIND_LICENSE_KEY=${MAXMIND_LICENSE_KEY} ports: - - "8084:8084" + - "18084:8084" volumes: - "/sys/fs/cgroup:/sys/fs/cgroup:ro" - ./tmp:/var/tmp/testlog From c1844b68f8169bb43dd206c42a3a3c61a2f2683a Mon Sep 17 00:00:00 2001 From: Neil Cook Date: Fri, 15 Aug 2025 12:18:21 +0100 Subject: [PATCH 09/10] Update .gitignore files --- .gitignore | 1 + docker/.gitignore | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 52205847..252becbc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .deps .dirstamp .history +.idea /.tags* /aclocal.m4 diff --git a/docker/.gitignore b/docker/.gitignore index 12d9bd97..a3f1e74d 100644 --- a/docker/.gitignore +++ b/docker/.gitignore @@ -1,2 +1,4 @@ # Ignore .docker file /.docker +/redis +/tmp From 5e3f35786a34679f7b3b6232f8b8bae788450c90 Mon Sep 17 00:00:00 2001 From: Neil Cook Date: Fri, 15 Aug 2025 12:18:34 +0100 Subject: [PATCH 10/10] docker: copy grafana directory into all the Dockerfiles --- docker/regression/Dockerfile | 1 + docker/wforce_image/Dockerfile | 1 + docker/wforce_image/Dockerfile.minimal | 1 + 3 files changed, 3 insertions(+) diff --git a/docker/regression/Dockerfile b/docker/regression/Dockerfile index ecf7addb..7bfbd11a 100644 --- a/docker/regression/Dockerfile +++ b/docker/regression/Dockerfile @@ -74,6 +74,7 @@ COPY common/ /wforce/common/ COPY trackalert/ /wforce/trackalert/ COPY docker/ /wforce/docker/ COPY elk/ /wforce/elk/ +COPY grafana/ /wforce/grafana/ RUN rm -rf regression-tests/.venv diff --git a/docker/wforce_image/Dockerfile b/docker/wforce_image/Dockerfile index f80c0855..c9cb2914 100644 --- a/docker/wforce_image/Dockerfile +++ b/docker/wforce_image/Dockerfile @@ -49,6 +49,7 @@ COPY docker/Makefile.am /wforce/docker/ RUN mkdir /wforce/docker/wforce_image COPY docker/wforce_image/Makefile.am /wforce/docker/wforce_image COPY elk/ /wforce/elk/ +COPY grafana/ /wforce/grafana/ RUN autoreconf -ivf RUN ./configure --prefix /usr --enable-trackalert --disable-systemd --disable-sodium --disable-docker --with-luajit --sysconfdir=/etc/wforce CC=clang CXX=clang++ diff --git a/docker/wforce_image/Dockerfile.minimal b/docker/wforce_image/Dockerfile.minimal index ac8ff744..1cb83a2b 100644 --- a/docker/wforce_image/Dockerfile.minimal +++ b/docker/wforce_image/Dockerfile.minimal @@ -77,6 +77,7 @@ COPY docker/Makefile.am /wforce/docker/ RUN mkdir /wforce/docker/wforce_image COPY docker/wforce_image/Makefile.am /wforce/docker/wforce_image COPY elk/ /wforce/elk/ +COPY grafana/ /wforce/grafana/ RUN autoreconf -ivf RUN ./configure --prefix /usr --enable-trackalert --disable-systemd --disable-sodium --disable-docker --with-luajit --sysconfdir=/etc/wforce