Add Prometheus metrics and hide sensitive info

Dockerfile

@@ -1,12 +1,12 @@
-FROM golang:1.18.4-alpine3.16
+FROM golang:1.24.3-alpine3.21
 
 ENV GOPATH /go
 
 RUN apk update && apk upgrade && \
     apk add --no-cache git build-base wget
 
 RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_amd64 \
- && chmod +x /usr/local/bin/dumb-init
+    && chmod +x /usr/local/bin/dumb-init
 
 RUN mkdir -p /go/src/github.com/pingcap/go-ycsb
 WORKDIR /go/src/github.com/pingcap/go-ycsb
@@ -20,7 +20,7 @@ COPY . .
 
 RUN GO111MODULE=on go build -o /go-ycsb ./cmd/*
 
-FROM alpine:3.8 
+FROM alpine:3.21
 
 COPY --from=0 /go-ycsb /go-ycsb
 COPY --from=0 /usr/local/bin/dumb-init /usr/local/bin/dumb-init

cmd/go-ycsb/client.go

@@ -16,6 +16,7 @@ package main
 import (
 	"fmt"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/pingcap/go-ycsb/pkg/client"
@@ -49,8 +50,22 @@ func runClientCommandFunc(cmd *cobra.Command, args []string, doTransactions bool
 		}
 	})
 
+	// Prepare to hide sensitive information
+	sensitiveProperties := make([]string, 0)
+	if globalProps.GetString(prop.SensitiveProperties, "") != "" {
+		sensitiveProperties = strings.Split(globalProps.GetString(prop.SensitiveProperties, ""), ",")
+	}
+
 	fmt.Println("***************** properties *****************")
 	for key, value := range globalProps.Map() {
+		if len(sensitiveProperties) > 0 {
+			for _, sensitive := range sensitiveProperties {
+				if key == sensitive {
+					value = strings.Repeat("*", len(value))
+					break
+				}
+			}
+		}
 		fmt.Printf("\"%s\"=\"%s\"\n", key, value)
 	}
 	fmt.Println("**********************************************")

cmd/go-ycsb/main.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/magiconair/properties"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 
 	// Register workload
 
@@ -115,6 +116,13 @@ func initialGlobal(dbName string, onProperties func()) {
 		http.ListenAndServe(addr, nil)
 	}()
 
+	metricsAddr := globalProps.GetString(prop.MetricsAddr, prop.MetricsAddrDefault)
+	go func() {
+		fmt.Printf("Starting metrics server at %s\n", metricsAddr)
+		http.Handle("/metrics", promhttp.Handler())
+		http.ListenAndServe(metricsAddr, nil)
+	}()
+
 	measurement.InitMeasure(globalProps)
 
 	if len(tableName) == 0 {

pkg/measurement/measurement.go

@@ -16,13 +16,16 @@ package measurement
 import (
 	"bufio"
 	"os"
+	"strconv"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/magiconair/properties"
 	"github.com/pingcap/go-ycsb/pkg/prop"
 	"github.com/pingcap/go-ycsb/pkg/ycsb"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 var header = []string{"Operation", "Takes(s)", "Count", "OPS", "Avg(us)", "Min(us)", "Max(us)", "50th(us)", "90th(us)", "95th(us)", "99th(us)", "99.9th(us)", "99.99th(us)"}
@@ -32,12 +35,15 @@ type measurement struct {
 
 	p *properties.Properties
 
+	ph *prometheus.HistogramVec
+
 	measurer ycsb.Measurer
 }
 
 func (m *measurement) measure(op string, start time.Time, lan time.Duration) {
 	m.Lock()
 	m.measurer.Measure(op, start, lan)
+	m.ph.WithLabelValues(op).Observe(float64(lan.Microseconds()))
 	m.Unlock()
 }
 
@@ -75,6 +81,19 @@ func (m *measurement) summary() {
 	m.RUnlock()
 }
 
+func convertHistogramBuckets(buckets string) []float64 {
+	bucketStrs := strings.Split(buckets, ",")
+	bucketFloats := make([]float64, len(bucketStrs))
+	for i, bucketStr := range bucketStrs {
+		bucketFloat, err := strconv.ParseFloat(strings.TrimSpace(bucketStr), 64)
+		if err != nil {
+			panic("failed to parse histogram bucket: " + err.Error())
+		}
+		bucketFloats[i] = bucketFloat
+	}
+	return bucketFloats
+}
+
 // InitMeasure initializes the global measurement.
 func InitMeasure(p *properties.Properties) {
 	globalMeasure = new(measurement)
@@ -88,6 +107,13 @@ func InitMeasure(p *properties.Properties) {
 	default:
 		panic("unsupported measurement type: " + measurementType)
 	}
+	histogramBuckets := p.GetString(prop.MetricsHistBuckets, prop.MetricsHistBucketsDefault)
+	globalMeasure.ph = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "ycsb_cmd_duration_microseconds",
+		Help:    "YCSB duration of operation in microseconds",
+		Buckets: convertHistogramBuckets(histogramBuckets),
+	}, []string{"operation"})
+	prometheus.MustRegister(globalMeasure.ph)
 	EnableWarmUp(p.GetInt64(prop.WarmUpTime, 0) > 0)
 }
 

pkg/prop/prop.go

@@ -124,4 +124,14 @@ const (
 	MeasurementHistogramPercentileExportDefault         = false
 	MeasurementHistogramPercentileExportFilepath        = "histogram.percentiles.export.filepath"
 	MeasurementHistogramPercentileExportFilepathDefault = "./"
+
+	// Prometheus metrics properties
+	MetricsAddr               = "metrics.address"
+	MetricsAddrDefault        = ":8765"
+	MetricsHistBuckets        = "metrics.histogram.buckets"
+	MetricsHistBucketsDefault = "1,10,100,1000,2000,3000,4000,5000,6000,7000,8000,9000,10000,20000,30000,40000,50000,60000,70000,80000,90000,100000,1000000,10000000,100000000"
+
+	// Security
+	SensitiveProperties        = "sensitive_properties"
+	SensitivePropertiesDefault = "" // comma separated list of sensitive properties, for example: mysql.password,mysql.user
 )