From 1b6ec6a21d2e980ae9712093bd0ee8e1f7b547ef Mon Sep 17 00:00:00 2001 From: Edward Tsang Date: Mon, 28 May 2018 14:56:17 -0700 Subject: [PATCH 1/3] add sampling example --- src/sampling/README.md | 27 ++++++++++++++++++++++ src/sampling/main.go | 52 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 src/sampling/README.md create mode 100644 src/sampling/main.go diff --git a/src/sampling/README.md b/src/sampling/README.md new file mode 100644 index 0000000..b86c7f5 --- /dev/null +++ b/src/sampling/README.md @@ -0,0 +1,27 @@ +# Log sampling + +Log sampling tries to reduce CPU and I/O pressure by only recording a subset of entries and +dropping duplicate log entries. A log entry having the same log level and message content +are considered duplicate. + +The logger maintains a separate bucket for each log entry. +At each tick, the sampler will emit the first N initial logs in each bucket and every Mth log +therafter. Sampling loggers are safe for concurrent use. +In this example, we will emit the first 5 messages then one every each 100 messages thereafter. + +```console +go run src/sampling/main.go + +Log sampling to reduce the presure on I/O and CPU by reducing logging +Using the built in sampler. You probably need to wrap the whole zapcore Sampler public methods if you need to write our own custom sampler +We will first emit the first 5 messages then one every each 100 messages thereafter +{"level":"info","ts":1527544371.1515696,"msg":"test at info","n":1} +{"level":"info","ts":1527544371.1515965,"msg":"test at info","n":2} +{"level":"info","ts":1527544371.1516032,"msg":"test at info","n":3} +{"level":"info","ts":1527544371.1516066,"msg":"test at info","n":4} +{"level":"info","ts":1527544371.15161,"msg":"test at info","n":5} +{"level":"info","ts":1527544371.1518133,"msg":"test at info","n":105} + +``` + + diff --git a/src/sampling/main.go b/src/sampling/main.go new file mode 100644 index 0000000..c9c898d --- /dev/null +++ b/src/sampling/main.go @@ -0,0 +1,52 @@ +package main + +import ( + "fmt" + "os" + "time" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +type Logger struct { + *zap.Logger +} + +func main() { + fmt.Printf("Log sampling to reduce the presure on I/O and CPU by reducing logging\n") + fmt.Printf("Using the built in sampler. You probably need to wrap the whole zapcore Sampler public methods if you need to write our own custom sampler\n") + encoderConfig := zap.NewProductionEncoderConfig() + atomLevel := zap.NewAtomicLevelAt(zapcore.InfoLevel) + encoder := zapcore.NewJSONEncoder(encoderConfig) + writer := zapcore.Lock(os.Stdout) + + core := zapcore.NewCore(encoder, writer, &atomLevel) + emitInitialMessages := 5 + thereAfterEachMessages := 100 + logger := &Logger{zap.New(core)} + fmt.Printf("We will first emit the first %v messages then one every each %v messages thereafter \n", emitInitialMessages, thereAfterEachMessages) + logger = logger.WithSamplingConfig(time.Second, emitInitialMessages, thereAfterEachMessages) + + for i := 1; i < 110; i++ { + logger.With(zap.Int("n", i)).Info("test at info") + } + logger.Sync() + // Output: + // {"level":"info","ts":1527544371.1515696,"msg":"test at info","n":1} + // {"level":"info","ts":1527544371.1515965,"msg":"test at info","n":2} + // {"level":"info","ts":1527544371.1516032,"msg":"test at info","n":3} + // {"level":"info","ts":1527544371.1516066,"msg":"test at info","n":4} + // {"level":"info","ts":1527544371.15161,"msg":"test at info","n":5} + // {"level":"info","ts":1527544371.1518133,"msg":"test at info","n":105} +} + +func (l *Logger) WithSamplingConfig(tick time.Duration, initial, thereAfter int) *Logger { + core := l.Core() + newLogger := l.WithOptions( + zap.WrapCore( + func(zapcore.Core) zapcore.Core { + return zapcore.NewSampler(core, tick, initial, thereAfter) + })) + return &Logger{newLogger} +} From 0930465bf3ccfb119572c9c0273ee91132664df0 Mon Sep 17 00:00:00 2001 From: Edward Tsang Date: Mon, 28 May 2018 15:02:57 -0700 Subject: [PATCH 2/3] seems like it does not check for positive integer --- src/sampling/main.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/sampling/main.go b/src/sampling/main.go index c9c898d..c865061 100644 --- a/src/sampling/main.go +++ b/src/sampling/main.go @@ -42,6 +42,10 @@ func main() { } func (l *Logger) WithSamplingConfig(tick time.Duration, initial, thereAfter int) *Logger { + if initial < 1 || thereAfter < 1 { + // fmt.Printf("all arguments must be positive") + return l + } core := l.Core() newLogger := l.WithOptions( zap.WrapCore( From e8a0c9122996129e1f7140acec5ffdaa0513dd33 Mon Sep 17 00:00:00 2001 From: Edward Tsang Date: Wed, 30 May 2018 11:05:41 -0700 Subject: [PATCH 3/3] add comment --- src/sampling/main.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/sampling/main.go b/src/sampling/main.go index c865061..a2aeb74 100644 --- a/src/sampling/main.go +++ b/src/sampling/main.go @@ -24,7 +24,10 @@ func main() { core := zapcore.NewCore(encoder, writer, &atomLevel) emitInitialMessages := 5 thereAfterEachMessages := 100 + // You can embed zap.Logger inside your Logger struct for WithSamplingConfig and preserve zap.Logger interface + // Or if you take zap.Logger as one of the parameters for WithSamplingConfig, you don't need to use embedded struct. logger := &Logger{zap.New(core)} + fmt.Printf("We will first emit the first %v messages then one every each %v messages thereafter \n", emitInitialMessages, thereAfterEachMessages) logger = logger.WithSamplingConfig(time.Second, emitInitialMessages, thereAfterEachMessages)