Improve metrics and monitoring

.gitignore

@@ -1,6 +1,7 @@
 .deps
 .dirstamp
 .history
+.idea
 
 /.tags*
 /aclocal.m4

CHANGELOG.md

@@ -3,7 +3,7 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
-## [Unreleased]
+## [3.0.0]
 
 ### Added
 - Support new `fail_type` parameter for determining why a login failed
@@ -12,6 +12,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - Allow ja3 to be passed to the reset API command
 - Add support for building amazon-2023 packages
 - Use asciidoctor to build documentation not pandoc
+- Add a sample grafana dashboard for monitoring wforce
+- Implemented the (existing but previously always 0) `wforce_active_http_connections` metric for counting active HTTP connections
 
 ### Removed
 - Removed support for Enterprise Linux 7 and Amazon 2

Makefile.am

@@ -10,5 +10,6 @@ endif
 SUBDIRS=ext docs common wforce $(BUILD_TRACKALERT) $(BUILD_DOCKER)
 
 LOGSTASH_DIST = elk/logstash/config/logstash.conf elk/logstash/templates/wforce_template.json elk/kibana/kibana_saved_objects.ndjson
+GRAFANA_DIST = grafana/grafana_dashboard.json
 
-EXTRA_DIST= README.md LICENSE CHANGELOG.md $(LOGSTASH_DIST)
+EXTRA_DIST= README.md LICENSE CHANGELOG.md $(LOGSTASH_DIST) $(GRAFANA_DIST)

builder-support/debian/wforce.docs

@@ -4,3 +4,4 @@ CHANGELOG.md
 elk/logstash/config/logstash.conf
 elk/logstash/templates/wforce_template.json
 elk/kibana/kibana_saved_objects.ndjson
+grafana/grafana_dashboard.json

builder-support/dockerfiles/Dockerfile.target.el-10

@@ -0,0 +1 @@
+@INCLUDE Dockerfile.target.rl-10

builder-support/dockerfiles/Dockerfile.target.rl-10

@@ -0,0 +1,20 @@
+@EXEC export BUILDER_RHEL_VERSION=10
+@EXEC export BUILDER_RHEL_FLAVOUR=oracle
+# First do the source builds
+@INCLUDE Dockerfile.target.sdist
+
+# This defines the distribution base layer
+# Put only the bare minimum of common commands here, without dev tools
+FROM rockylinux/rockylinux:10 as dist-base
+
+RUN touch /var/lib/rpm/* && dnf install -y epel-release && \
+    crb enable
+
+RUN dnf -y install clang llvm-devel
+
+# Do the actual rpm build
+@INCLUDE Dockerfile.rpmbuild
+
+# Do a test install and verify
+# Can be skipped with skiptests=1 in the environment
+@EXEC [ "$skiptests" = "" ] && include Dockerfile.rpmtest

builder-support/dockerfiles/Dockerfile.wforce

@@ -46,7 +46,7 @@ RUN cd drogon && git checkout tags/v1.9.1 -b v1.9.1
 RUN cd drogon && git submodule init && git submodule update && mkdir _build && cd _build && cmake .. -DBUILD_REDIS=OFF -DCMAKE_BUILD_TYPE=Release -DBUILD_ORM=OFF && make && make install
 
 ADD CHANGELOG.md configure.ac ext LICENSE Makefile.am README.md NOTICE /wforce/
-@EXEC sdist_dirs=(m4 ext docs regression-tests wforce common trackalert docker elk)
+@EXEC sdist_dirs=(m4 ext docs regression-tests wforce common trackalert docker elk grafana)
 @EXEC for d in ${sdist_dirs[@]} ; do echo "COPY $d/ /wforce/$d/" ; done
 ADD builder/helpers/set-configure-ac-version.sh /wforce/builder/helpers/
 

builder-support/specs/wforce.spec

@@ -110,6 +110,7 @@ mv %{buildroot}/etc/%{name}/%{name}.conf.example %{buildroot}/%{_docdir}/%{name}
 mv elk/logstash/config/logstash.conf %{buildroot}/%{_docdir}/%{name}-%{version}/
 mv elk/logstash/templates/wforce_template.json %{buildroot}/%{_docdir}/%{name}-%{version}/
 mv elk/kibana/kibana_saved_objects.ndjson %{buildroot}/%{_docdir}/%{name}-%{version}/
+mv grafana/grafana_dashboard.json %{buildroot}/%{_docdir}/%{name}-%{version}/
 
 %clean
 rm -rf %{buildroot}

common/prometheus.cc

@@ -116,7 +116,7 @@ void PrometheusMetrics::incWebhookMetric(unsigned int id, bool success, bool cus
   }
 }
 
-void setPrometheusActiveConns(int value)
+void setPrometheusActiveConns(size_t value)
 {
   if (prom_metrics)
     prom_metrics->setActiveConns(value);

common/prometheus.hh

@@ -110,7 +110,7 @@ public:
     worker_response_duration->Observe(duration);
   }
 
-  void setActiveConns(int value)
+  void setActiveConns(size_t value)
   {
     active_connections->Set(value);
   }
@@ -164,7 +164,7 @@ protected:
 
 void initPrometheusMetrics(std::shared_ptr<PrometheusMetrics> pmp);
 
-void setPrometheusActiveConns(int value);
+void setPrometheusActiveConns(size_t value);
 
 void addPrometheusCommandMetric(const std::string& name);
 void incPrometheusCommandMetric(const std::string& name);

common/wforce-webserver.cc

@@ -58,11 +58,6 @@ NetmaskGroup WforceWebserver::getACL()
   return d_ACL.getCopy();
 }
 
-size_t WforceWebserver::getNumConns()
-{
-  return 0; // XXX - investigate if this can be retrieved from drogon
-}
-
 bool WforceWebserver::registerFunc(const std::string& command, HTTPVerb verb, const WforceWSFunc& wsf)
 {
   return registerFuncInternal(command, verb, wsf, true);

common/wforce-webserver.hh

@@ -37,6 +37,7 @@
 #include "drogon/drogon.h"
 #include "prometheus.hh"
 #include "perf-stats.hh"
+#include "ext/threadname.hh"
 
 using WforceWSFuncPtr = void (*)(const drogon::HttpRequestPtr& req,
                                  const std::string& command,
@@ -110,7 +111,9 @@ public:
 
   NetmaskGroup getACL();
 
-  size_t getNumConns();
+  size_t getNumConns() {
+    return static_cast<size_t>(drogon::app().getConnectionCount());
+  }
 
   // set the basic-auth password
   void setBasicAuthPassword(const std::string& password)
@@ -269,6 +272,14 @@ public:
                                     },
                                     {drogon::Delete, "ACLFilter", "LoginFilter"});
     }
+    std::thread connCountThread([this] {
+      setThreadName("wf/conn-count");
+      for (;;) {
+        sleep(1);
+        setPrometheusActiveConns(getNumConns());
+      }
+    });
+    connCountThread.detach();
   }
 
   static void start(WforceWebserver* wws)

docker/.gitignore

@@ -1,2 +1,4 @@
 # Ignore .docker file
 /.docker
+/redis
+/tmp

docker/docker-compose.yml

@@ -48,6 +48,8 @@ services:
       dockerfile: docker/regression/Dockerfile
       args:
         - MAXMIND_LICENSE_KEY=${MAXMIND_LICENSE_KEY}
+    ports:
+      - "18084:8084"
     volumes:
       - "/sys/fs/cgroup:/sys/fs/cgroup:ro"
       - ./tmp:/var/tmp/testlog

docker/regression/Dockerfile

@@ -74,6 +74,7 @@ COPY common/ /wforce/common/
 COPY trackalert/ /wforce/trackalert/
 COPY docker/ /wforce/docker/
 COPY elk/ /wforce/elk/
+COPY grafana/ /wforce/grafana/
 
 RUN rm -rf regression-tests/.venv  
 

docker/wforce_image/Dockerfile

@@ -49,6 +49,7 @@ COPY docker/Makefile.am /wforce/docker/
 RUN  mkdir /wforce/docker/wforce_image
 COPY docker/wforce_image/Makefile.am /wforce/docker/wforce_image
 COPY elk/ /wforce/elk/
+COPY grafana/ /wforce/grafana/
 
 RUN autoreconf -ivf
 RUN ./configure --prefix /usr --enable-trackalert --disable-systemd --disable-sodium --disable-docker --with-luajit --sysconfdir=/etc/wforce CC=clang CXX=clang++

docker/wforce_image/Dockerfile.minimal

@@ -77,6 +77,7 @@ COPY docker/Makefile.am /wforce/docker/
 RUN  mkdir /wforce/docker/wforce_image
 COPY docker/wforce_image/Makefile.am /wforce/docker/wforce_image
 COPY elk/ /wforce/elk/
+COPY grafana/ /wforce/grafana/
 
 RUN autoreconf -ivf
 RUN ./configure --prefix /usr --enable-trackalert --disable-systemd --disable-sodium --disable-docker --with-luajit --sysconfdir=/etc/wforce

docs/.gitignore

@@ -4,3 +4,4 @@
 /*.amd
 /*.mmd
 /*.fmd
+/*.header

docs/release_notes/ReleaseNotes-3.0.0.md

@@ -9,6 +9,8 @@
 * Allow ja3 to be passed to the reset API command
 * Add support for building amazon-2023 packages
 * Use asciidoctor to build documentation not pandoc
+* Add a sample grafana dashboard for monitoring wforce
+* Implemented the (existing but previously always 0) `wforce_active_http_connections` metric for counting active HTTP connections
 
 ## Removed
 * Removed support for Enterprise Linux 7 and Amazon 2