diff --git a/Dockerfile b/Dockerfile index 68945151..50e2c2cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.18.4-alpine3.16 +FROM golang:1.24.3-alpine3.21 ENV GOPATH /go @@ -6,7 +6,7 @@ RUN apk update && apk upgrade && \ apk add --no-cache git build-base wget RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_amd64 \ - && chmod +x /usr/local/bin/dumb-init + && chmod +x /usr/local/bin/dumb-init RUN mkdir -p /go/src/github.com/pingcap/go-ycsb WORKDIR /go/src/github.com/pingcap/go-ycsb @@ -20,7 +20,7 @@ COPY . . RUN GO111MODULE=on go build -o /go-ycsb ./cmd/* -FROM alpine:3.8 +FROM alpine:3.21 COPY --from=0 /go-ycsb /go-ycsb COPY --from=0 /usr/local/bin/dumb-init /usr/local/bin/dumb-init diff --git a/cmd/go-ycsb/client.go b/cmd/go-ycsb/client.go index 3e4e0e02..97765de7 100644 --- a/cmd/go-ycsb/client.go +++ b/cmd/go-ycsb/client.go @@ -16,6 +16,7 @@ package main import ( "fmt" "strconv" + "strings" "time" "github.com/pingcap/go-ycsb/pkg/client" @@ -49,8 +50,22 @@ func runClientCommandFunc(cmd *cobra.Command, args []string, doTransactions bool } }) + // Prepare to hide sensitive information + sensitiveProperties := make([]string, 0) + if globalProps.GetString(prop.SensitiveProperties, "") != "" { + sensitiveProperties = strings.Split(globalProps.GetString(prop.SensitiveProperties, ""), ",") + } + fmt.Println("***************** properties *****************") for key, value := range globalProps.Map() { + if len(sensitiveProperties) > 0 { + for _, sensitive := range sensitiveProperties { + if key == sensitive { + value = strings.Repeat("*", len(value)) + break + } + } + } fmt.Printf("\"%s\"=\"%s\"\n", key, value) } fmt.Println("**********************************************") diff --git a/cmd/go-ycsb/main.go b/cmd/go-ycsb/main.go index 7cc126db..436c4ee4 100644 --- a/cmd/go-ycsb/main.go +++ b/cmd/go-ycsb/main.go @@ -26,6 +26,7 @@ import ( "time" "github.com/magiconair/properties" + "github.com/prometheus/client_golang/prometheus/promhttp" // Register workload @@ -115,6 +116,13 @@ func initialGlobal(dbName string, onProperties func()) { http.ListenAndServe(addr, nil) }() + metricsAddr := globalProps.GetString(prop.MetricsAddr, prop.MetricsAddrDefault) + go func() { + fmt.Printf("Starting metrics server at %s\n", metricsAddr) + http.Handle("/metrics", promhttp.Handler()) + http.ListenAndServe(metricsAddr, nil) + }() + measurement.InitMeasure(globalProps) if len(tableName) == 0 { diff --git a/pkg/measurement/measurement.go b/pkg/measurement/measurement.go index 26b4c612..452ce98b 100644 --- a/pkg/measurement/measurement.go +++ b/pkg/measurement/measurement.go @@ -16,6 +16,8 @@ package measurement import ( "bufio" "os" + "strconv" + "strings" "sync" "sync/atomic" "time" @@ -23,6 +25,7 @@ import ( "github.com/magiconair/properties" "github.com/pingcap/go-ycsb/pkg/prop" "github.com/pingcap/go-ycsb/pkg/ycsb" + "github.com/prometheus/client_golang/prometheus" ) var header = []string{"Operation", "Takes(s)", "Count", "OPS", "Avg(us)", "Min(us)", "Max(us)", "50th(us)", "90th(us)", "95th(us)", "99th(us)", "99.9th(us)", "99.99th(us)"} @@ -32,12 +35,15 @@ type measurement struct { p *properties.Properties + ph *prometheus.HistogramVec + measurer ycsb.Measurer } func (m *measurement) measure(op string, start time.Time, lan time.Duration) { m.Lock() m.measurer.Measure(op, start, lan) + m.ph.WithLabelValues(op).Observe(float64(lan.Microseconds())) m.Unlock() } @@ -75,6 +81,19 @@ func (m *measurement) summary() { m.RUnlock() } +func convertHistogramBuckets(buckets string) []float64 { + bucketStrs := strings.Split(buckets, ",") + bucketFloats := make([]float64, len(bucketStrs)) + for i, bucketStr := range bucketStrs { + bucketFloat, err := strconv.ParseFloat(strings.TrimSpace(bucketStr), 64) + if err != nil { + panic("failed to parse histogram bucket: " + err.Error()) + } + bucketFloats[i] = bucketFloat + } + return bucketFloats +} + // InitMeasure initializes the global measurement. func InitMeasure(p *properties.Properties) { globalMeasure = new(measurement) @@ -88,6 +107,13 @@ func InitMeasure(p *properties.Properties) { default: panic("unsupported measurement type: " + measurementType) } + histogramBuckets := p.GetString(prop.MetricsHistBuckets, prop.MetricsHistBucketsDefault) + globalMeasure.ph = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "ycsb_cmd_duration_microseconds", + Help: "YCSB duration of operation in microseconds", + Buckets: convertHistogramBuckets(histogramBuckets), + }, []string{"operation"}) + prometheus.MustRegister(globalMeasure.ph) EnableWarmUp(p.GetInt64(prop.WarmUpTime, 0) > 0) } diff --git a/pkg/prop/prop.go b/pkg/prop/prop.go index 6c6daa7b..6e149603 100644 --- a/pkg/prop/prop.go +++ b/pkg/prop/prop.go @@ -124,4 +124,14 @@ const ( MeasurementHistogramPercentileExportDefault = false MeasurementHistogramPercentileExportFilepath = "histogram.percentiles.export.filepath" MeasurementHistogramPercentileExportFilepathDefault = "./" + + // Prometheus metrics properties + MetricsAddr = "metrics.address" + MetricsAddrDefault = ":8765" + MetricsHistBuckets = "metrics.histogram.buckets" + MetricsHistBucketsDefault = "1,10,100,1000,2000,3000,4000,5000,6000,7000,8000,9000,10000,20000,30000,40000,50000,60000,70000,80000,90000,100000,1000000,10000000,100000000" + + // Security + SensitiveProperties = "sensitive_properties" + SensitivePropertiesDefault = "" // comma separated list of sensitive properties, for example: mysql.password,mysql.user )