diff --git a/src/sampling/README.md b/src/sampling/README.md new file mode 100644 index 0000000..b86c7f5 --- /dev/null +++ b/src/sampling/README.md @@ -0,0 +1,27 @@ +# Log sampling + +Log sampling tries to reduce CPU and I/O pressure by only recording a subset of entries and +dropping duplicate log entries. A log entry having the same log level and message content +are considered duplicate. + +The logger maintains a separate bucket for each log entry. +At each tick, the sampler will emit the first N initial logs in each bucket and every Mth log +therafter. Sampling loggers are safe for concurrent use. +In this example, we will emit the first 5 messages then one every each 100 messages thereafter. + +```console +go run src/sampling/main.go + +Log sampling to reduce the presure on I/O and CPU by reducing logging +Using the built in sampler. You probably need to wrap the whole zapcore Sampler public methods if you need to write our own custom sampler +We will first emit the first 5 messages then one every each 100 messages thereafter +{"level":"info","ts":1527544371.1515696,"msg":"test at info","n":1} +{"level":"info","ts":1527544371.1515965,"msg":"test at info","n":2} +{"level":"info","ts":1527544371.1516032,"msg":"test at info","n":3} +{"level":"info","ts":1527544371.1516066,"msg":"test at info","n":4} +{"level":"info","ts":1527544371.15161,"msg":"test at info","n":5} +{"level":"info","ts":1527544371.1518133,"msg":"test at info","n":105} + +``` + + diff --git a/src/sampling/main.go b/src/sampling/main.go new file mode 100644 index 0000000..a2aeb74 --- /dev/null +++ b/src/sampling/main.go @@ -0,0 +1,59 @@ +package main + +import ( + "fmt" + "os" + "time" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +type Logger struct { + *zap.Logger +} + +func main() { + fmt.Printf("Log sampling to reduce the presure on I/O and CPU by reducing logging\n") + fmt.Printf("Using the built in sampler. You probably need to wrap the whole zapcore Sampler public methods if you need to write our own custom sampler\n") + encoderConfig := zap.NewProductionEncoderConfig() + atomLevel := zap.NewAtomicLevelAt(zapcore.InfoLevel) + encoder := zapcore.NewJSONEncoder(encoderConfig) + writer := zapcore.Lock(os.Stdout) + + core := zapcore.NewCore(encoder, writer, &atomLevel) + emitInitialMessages := 5 + thereAfterEachMessages := 100 + // You can embed zap.Logger inside your Logger struct for WithSamplingConfig and preserve zap.Logger interface + // Or if you take zap.Logger as one of the parameters for WithSamplingConfig, you don't need to use embedded struct. + logger := &Logger{zap.New(core)} + + fmt.Printf("We will first emit the first %v messages then one every each %v messages thereafter \n", emitInitialMessages, thereAfterEachMessages) + logger = logger.WithSamplingConfig(time.Second, emitInitialMessages, thereAfterEachMessages) + + for i := 1; i < 110; i++ { + logger.With(zap.Int("n", i)).Info("test at info") + } + logger.Sync() + // Output: + // {"level":"info","ts":1527544371.1515696,"msg":"test at info","n":1} + // {"level":"info","ts":1527544371.1515965,"msg":"test at info","n":2} + // {"level":"info","ts":1527544371.1516032,"msg":"test at info","n":3} + // {"level":"info","ts":1527544371.1516066,"msg":"test at info","n":4} + // {"level":"info","ts":1527544371.15161,"msg":"test at info","n":5} + // {"level":"info","ts":1527544371.1518133,"msg":"test at info","n":105} +} + +func (l *Logger) WithSamplingConfig(tick time.Duration, initial, thereAfter int) *Logger { + if initial < 1 || thereAfter < 1 { + // fmt.Printf("all arguments must be positive") + return l + } + core := l.Core() + newLogger := l.WithOptions( + zap.WrapCore( + func(zapcore.Core) zapcore.Core { + return zapcore.NewSampler(core, tick, initial, thereAfter) + })) + return &Logger{newLogger} +}