[add] Prometheus exporter

README.md

@@ -21,10 +21,11 @@ Welcome to the **`Backhaul`** project! This project provides a high-performance
       - [WSS Multiplexing Configuration](#wss-multiplexing-configuration)
 5. [Generating a Self-Signed TLS Certificate with OpenSSL](#generating-a-self-signed-tls-certificate-with-openssl)
 6. [Running backhaul as a service](#running-backhaul-as-a-service)
-7. [FAQ](#faq)
-8. [Benchmark](#benchmark)
-9. [License](#license)
-10. [Donation](#donation)
+7. [Monitoring with prometheus/grafana](#monitoring)
+8. [FAQ](#faq)
+9. [Benchmark](#benchmark)
+10. [License](#license)
+11. [Donation](#donation)
 
 ---
 
@@ -106,8 +107,8 @@ To start using the solution, you'll need to configure both server and client com
     mss = 1360                    # TCP/TCPMux: Maximum Segment Size in bytes; controls max TCP payload size to avoid fragmentation. (default: system-defined)
     so_rcvbuf = 4194304           # TCP/TCPMux: Socket receive buffer size (bytes); larger buffer allows higher throughput on receive side. (default: system-defined)
     so_sndbuf = 1048576           # TCP/TCPMux: Socket send buffer size (bytes); controls send queue size to manage outgoing data flow. (default: system-defined)
-
-
+  
+    metrics = ["default", "prometheus"] # metrics exposed via web_port. only the legacy json ("default") and prometheus ("prometheus") are supported. to achieve backward compatibility without changing the config file, legacy json is enabled by default.
 
     ports = [
     "443-600",                  # Listen on all ports in the range 443 to 600
@@ -154,6 +155,8 @@ To start using the solution, you'll need to configure both server and client com
    mss = 1360                    # TCP/TCPMux: Maximum Segment Size in bytes; controls max TCP payload size to avoid fragmentation. (default: system-defined)
    so_rcvbuf = 1048576           # TCP/TCPMux: Socket receive buffer size (bytes); larger buffer allows higher throughput on receive side. (default: system-defined)
    so_sndbuf = 4194304           # TCP/TCPMux: Socket send buffer size (bytes); controls send queue size to manage outgoing data flow. (default: system-defined)
+
+   metrics = ["default", "prometheus"] # metrics exposed via web_port. only the legacy json ("default") and prometheus ("prometheus") are supported. to achieve backward compatibility without changing the config file, legacy json is enabled by default.
    ```
 
    To start the `client`:
@@ -571,6 +574,35 @@ sudo systemctl status backhaul.service
 journalctl -u backhaul.service -e -f
 ```
 
+## Monitoring
+
+setting `web_port` to a non-zero value will enable the monitoring interface.
+
+you can choose which monitoring interface you want to use by setting `metrics` to `prometheus`, `default` or both. empty array will fallback to `["default"]`
+
+### Basic Monitoring Setup
+you can set up a basic monitoring setup using prometheus and grafana.
+![grafana dashboard](monitoring/dashboard.jpg)
+
+(while not necessary, it is recommended to use docker)
+1. install docker, docker-compose
+    ```bash
+    sudo apt update && sudo apt install docker.io docker-compose-v2
+    # to run docker without sudo
+    sudo groupadd docker
+    sudo usermod -aG docker $USER
+    # If you're running Linux in a virtual machine, it may be necessary to restart the virtual machine for changes to take effect.
+   ```
+2. create a `docker-compose.yml` like [this example](monitoring/docker-compose.yml)
+3. create a prometheus.yaml like [this example](monitoring/prometheus.yml)
+4. run the docker-compose file
+    ```bash
+      docker compose up -d
+    ```
+5. visit grafana dashboard at `http://SERVER_IP:3000` (default username/password is `admin`)
+6. create a new datasource, choose prometheus, and enter the url `http://prometheus:9090`
+7. via `Dashboards > New > import` import the [example dashboard](monitoring/dashboard.jpg) or [create your own](https://grafana.com/tutorials/)
+
 ## FAQ
 
 **Q: How do I decide which transport protocol to use?**

cmd/cmd.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/musix/backhaul/config"
 	"github.com/musix/backhaul/internal/client"
+	"github.com/musix/backhaul/internal/web/metrics"
 
 	"github.com/musix/backhaul/internal/server"
 	"github.com/musix/backhaul/internal/utils"
@@ -26,6 +27,8 @@ func Run(configPath string, ctx context.Context) {
 	// Apply default values to the configuration
 	applyDefaults(cfg)
 
+	metricHandler := metrics.NewMetricsHandler(ctx, logger, *cfg)
+
 	configType := ""
 	if cfg.Server.BindAddr != "" {
 		configType = "server"
@@ -45,6 +48,7 @@ func Run(configPath string, ctx context.Context) {
 
 		srv := server.NewServer(&cfg.Server, ctx) // server
 		go srv.Start()
+		go metricHandler.Monitor()
 
 		// Wait for shutdown signal
 		<-ctx.Done()
@@ -58,6 +62,7 @@ func Run(configPath string, ctx context.Context) {
 
 		clnt := client.NewClient(&cfg.Client, ctx) // client
 		go clnt.Start()
+		go metricHandler.Monitor()
 
 		// Wait for shutdown signal
 		<-ctx.Done()

cmd/defaults.go

@@ -129,4 +129,13 @@ func applyDefaults(cfg *config.Config) {
 	if cfg.Server.MuxCon < 1 {
 		cfg.Server.MuxCon = defaultMuxCon
 	}
+
+	// keep legacy handler
+	if len(cfg.Client.MetricCollectors) == 0 {
+		cfg.Client.MetricCollectors = append(cfg.Client.MetricCollectors, "default")
+	}
+
+	if len(cfg.Server.MetricCollectors) == 0 {
+		cfg.Server.MetricCollectors = append(cfg.Server.MetricCollectors, "default")
+	}
 }

config/config.go

@@ -41,6 +41,7 @@ type ServerConfig struct {
 	MSS              int           `toml:"mss"`
 	SO_RCVBUF        int           `toml:"so_rcvbuf"`
 	SO_SNDBUF        int           `toml:"so_sndbuf"`
+	MetricCollectors []string      `toml:"metrics"`
 }
 
 // ClientConfig represents the configuration for the client.
@@ -69,10 +70,19 @@ type ClientConfig struct {
 	MSS              int           `toml:"mss"`
 	SO_RCVBUF        int           `toml:"so_rcvbuf"`
 	SO_SNDBUF        int           `toml:"so_sndbuf"`
+	MetricCollectors []string      `toml:"metrics"`
 }
 
 // Config represents the complete configuration, including both server and client settings.
 type Config struct {
 	Server ServerConfig `toml:"server"`
 	Client ClientConfig `toml:"client"`
 }
+
+func (c *Config) IsServerConfig() bool {
+	if c.Client.RemoteAddr != "" {
+		return false
+	}
+
+	return true
+}

go.mod

@@ -5,18 +5,26 @@ go 1.23.1
 require (
 	github.com/BurntSushi/toml v1.4.0
 	github.com/gorilla/websocket v1.5.3
+	github.com/prometheus/client_golang v1.23.0
 	github.com/shirou/gopsutil/v4 v4.24.8
 	github.com/sirupsen/logrus v1.9.3
 	github.com/xtaci/smux v1.5.27
 )
 
 require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
-	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/common v0.65.0 // indirect
+	github.com/prometheus/procfs v0.16.1 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/tklauser/go-sysconf v0.3.14 // indirect
 	github.com/tklauser/numcpus v0.8.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
-	golang.org/x/sys v0.25.0 // indirect
+	golang.org/x/sys v0.33.0 // indirect
+	google.golang.org/protobuf v1.36.6 // indirect
 )

go.sum

@@ -1,21 +1,39 @@
 github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0=
 github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
-github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
 github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
-github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
+github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU=
+github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
+github.com/prometheus/client_golang v1.23.0 h1:ust4zpdl9r4trLY/gSjlm07PuiBq2ynaXXlptpfy8Uc=
+github.com/prometheus/client_golang v1.23.0/go.mod h1:i/o0R9ByOnHX0McrTMTyhYvKE4haaf2mW08I+jGAjEE=
+github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
+github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
+github.com/prometheus/common v0.65.0 h1:QDwzd+G1twt//Kwj/Ww6E9FQq1iVMmODnILtW1t2VzE=
+github.com/prometheus/common v0.65.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8=
+github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
+github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
 github.com/shirou/gopsutil/v4 v4.24.8 h1:pVQjIenQkIhqO81mwTaXjTzOMT7d3TZkf43PlVFHENI=
 github.com/shirou/gopsutil/v4 v4.24.8/go.mod h1:wE0OrJtj4dG+hYkxqDH3QiBICdKSf04/npcvLLc/oRg=
 github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
@@ -26,8 +44,8 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
-github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU=
 github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY=
 github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYgY=
@@ -36,12 +54,16 @@ github.com/xtaci/smux v1.5.27 h1:uIU1dpJQQWUCmGxXBgajLfc8cMMb13hCitj+HC5yC/Q=
 github.com/xtaci/smux v1.5.27/go.mod h1:OMlQbT5vcgl2gb49mFkYo6SMf+zP3rcjcwQz7ZU7IGY=
 github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
 github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
-golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
+golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
+google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

internal/client/transport/accept_udp.go

@@ -7,13 +7,13 @@ import (
 	"net"
 	"time"
 
-	"github.com/musix/backhaul/internal/web"
+	"github.com/musix/backhaul/internal/stats"
 	"github.com/sirupsen/logrus"
 )
 
 const BufferSize = 16 * 1024
 
-func UDPDialer(tcp net.Conn, remoteAddr string, logger *logrus.Logger, usage *web.Usage, remotePort int, sniffer bool) {
+func UDPDialer(tcp net.Conn, remoteAddr string, logger *logrus.Logger, remotePort int) {
 	remoteUDPAddr, err := net.ResolveUDPAddr("udp", remoteAddr)
 	if err != nil {
 		logger.Fatalf("failed to resolve remote address: %v", err)
@@ -30,16 +30,16 @@ func UDPDialer(tcp net.Conn, remoteAddr string, logger *logrus.Logger, usage *we
 	done := make(chan struct{})
 
 	go func() {
-		go tcpToUDP(tcp, remoteConn, logger, usage, remotePort, sniffer)
+		go tcpToUDP(tcp, remoteConn, logger, remotePort)
 		done <- struct{}{}
 	}()
 
-	udpToTCP(tcp, remoteConn, logger, usage, remotePort, sniffer)
+	udpToTCP(tcp, remoteConn, logger, remotePort)
 
 	<-done
 }
 
-func tcpToUDP(tcp net.Conn, udp *net.UDPConn, logger *logrus.Logger, usage *web.Usage, remotePort int, sniffer bool) {
+func tcpToUDP(tcp net.Conn, udp *net.UDPConn, logger *logrus.Logger, remotePort int) {
 	buf := make([]byte, BufferSize)
 	lenBuf := make([]byte, 2) // 2-byte header for packet size
 
@@ -83,13 +83,11 @@ func tcpToUDP(tcp net.Conn, udp *net.UDPConn, logger *logrus.Logger, usage *web.
 
 		logger.Tracef("read %d bytes from TCP, wrote %d bytes to UDP", packetSize, totalWritten)
 
-		if sniffer {
-			usage.AddOrUpdatePort(remotePort, uint64(totalWritten))
-		}
+		stats.RecordPortUsage(remotePort, uint64(totalWritten))
 	}
 }
 
-func udpToTCP(tcp net.Conn, udp *net.UDPConn, logger *logrus.Logger, usage *web.Usage, remotePort int, sniffer bool) {
+func udpToTCP(tcp net.Conn, udp *net.UDPConn, logger *logrus.Logger, remotePort int) {
 	buf := make([]byte, BufferSize-6) // reserved for 5 bytes header
 
 	// Pre-allocate headers
@@ -146,8 +144,6 @@ func udpToTCP(tcp net.Conn, udp *net.UDPConn, logger *logrus.Logger, usage *web.
 
 		logger.Tracef("read %d bytes from UDP, wrote %d bytes to TCP", r, totalWritten)
 
-		if sniffer {
-			usage.AddOrUpdatePort(remotePort, uint64(totalWritten))
-		}
+		stats.RecordPortUsage(remotePort, uint64(totalWritten))
 	}
 }