diff --git a/.golangci.yml b/.golangci.yml index 5d03a03ca..9f6fd4700 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -23,7 +23,14 @@ linters: - unused - usestdlibvars - whitespace + - depguard settings: + depguard: + rules: + denied-deps: + deny: + - pkg: go.uber.org/atomic + desc: use `sync/atomic` instead dupl: threshold: 120 forbidigo: diff --git a/fd/file.d.go b/fd/file.d.go index 5606eb4c7..141385857 100644 --- a/fd/file.d.go +++ b/fd/file.d.go @@ -9,6 +9,7 @@ import ( "net/http/pprof" "runtime" "runtime/debug" + "sync/atomic" "github.com/bitly/go-simplejson" "github.com/ozontech/file.d/buildinfo" @@ -20,7 +21,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/promhttp" - "go.uber.org/atomic" ) type FileD struct { diff --git a/go.mod b/go.mod index 9c2b89384..1f346d2e7 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,6 @@ require ( github.com/twmb/franz-go/plugin/kzap v1.1.2 github.com/twmb/tlscfg v1.2.1 github.com/valyala/fasthttp v1.48.0 - go.uber.org/atomic v1.11.0 go.uber.org/automaxprocs v1.5.3 go.uber.org/zap v1.27.0 golang.org/x/oauth2 v0.36.0 @@ -136,6 +135,7 @@ require ( go.opentelemetry.io/otel v1.34.0 // indirect go.opentelemetry.io/otel/metric v1.34.0 // indirect go.opentelemetry.io/otel/trace v1.34.0 // indirect + go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect diff --git a/pipeline/action_watcher.go b/pipeline/action_watcher.go index 386753fb4..cf70cb0ab 100644 --- a/pipeline/action_watcher.go +++ b/pipeline/action_watcher.go @@ -6,7 +6,7 @@ import ( "sync" "time" - "go.uber.org/atomic" + "sync/atomic" ) // actionWatcher is the struct for debugging actions. @@ -38,7 +38,7 @@ func newActionWatcher(procID int) *actionWatcher { return &actionWatcher{ procID: procID, samples: make(map[int][]*sample), - samplesLen: atomic.NewInt64(0), + samplesLen: &atomic.Int64{}, samplesMu: sync.Mutex{}, } } @@ -70,7 +70,7 @@ func (aw *actionWatcher) addSample(actionIdx int) *sample { } aw.samplesMu.Lock() aw.samples[actionIdx] = append(aw.samples[actionIdx], s) - aw.samplesLen.Inc() + aw.samplesLen.Add(1) aw.samplesMu.Unlock() return s @@ -94,7 +94,7 @@ func (aw *actionWatcher) deleteSample(actionIdx int, sample *sample) { samples[deleteInd] = samples[len(samples)-1] samples[len(samples)-1] = nil aw.samples[actionIdx] = samples[:len(samples)-1] - aw.samplesLen.Dec() + aw.samplesLen.Add(-1) } var timeoutEvent = []byte("") diff --git a/pipeline/antispam/antispammer.go b/pipeline/antispam/antispammer.go index 229165650..15ee8fe52 100644 --- a/pipeline/antispam/antispammer.go +++ b/pipeline/antispam/antispammer.go @@ -3,12 +3,12 @@ package antispam import ( "fmt" "sync" + "sync/atomic" "time" "github.com/ozontech/file.d/cfg/matchrule" "github.com/ozontech/file.d/logger" "github.com/ozontech/file.d/metric" - "go.uber.org/atomic" "go.uber.org/zap" ) @@ -172,7 +172,7 @@ func (a *Antispammer) IsSpam(id string, name string, isNewSource bool, event []b x := src.counter.Load() diff := timeEventSeconds - src.timestamp.Swap(timeEventSeconds) if diff < a.maintenanceInterval.Nanoseconds() { - x = src.counter.Inc() + x = src.counter.Add(1) } if x == int32(threshold) { src.counter.Swap(int32(a.unbanIterations * threshold)) diff --git a/pipeline/backoff_test.go b/pipeline/backoff_test.go index f9ff0467a..22dd98192 100644 --- a/pipeline/backoff_test.go +++ b/pipeline/backoff_test.go @@ -2,13 +2,13 @@ package pipeline import ( "errors" + "sync/atomic" "testing" "time" "github.com/ozontech/file.d/metric" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" - "go.uber.org/atomic" ) func TestBackoff(t *testing.T) { @@ -19,7 +19,7 @@ func TestBackoff(t *testing.T) { eventCountBefore := eventCount.Load() errorFn := func(err error, events []*Event) { - errorCount.Inc() + errorCount.Add(1) } batcherBackoff := NewRetriableBatcher( @@ -27,7 +27,7 @@ func TestBackoff(t *testing.T) { MetricCtl: metric.NewCtl("", prometheus.NewRegistry(), time.Minute, 0), }, func(workerData *WorkerData, batch *Batch) error { - eventCount.Inc() + eventCount.Add(1) return nil }, BackoffOpts{ @@ -49,7 +49,7 @@ func TestBackoffWithError(t *testing.T) { errorCount := &atomic.Int32{} prevValue := errorCount.Load() errorFn := func(err error, events []*Event) { - errorCount.Inc() + errorCount.Add(1) } batcherBackoff := NewRetriableBatcher( @@ -79,7 +79,7 @@ func TestBackoffWithErrorWithDeadQueue(t *testing.T) { errorCount := &atomic.Int32{} prevValue := errorCount.Load() errorFn := func(err error, events []*Event) { - errorCount.Inc() + errorCount.Add(1) } batcherBackoff := NewRetriableBatcher( diff --git a/pipeline/batch_test.go b/pipeline/batch_test.go index 2284a0537..8ee6bc64c 100644 --- a/pipeline/batch_test.go +++ b/pipeline/batch_test.go @@ -3,6 +3,7 @@ package pipeline import ( "context" "sync" + "sync/atomic" "testing" "time" @@ -10,7 +11,6 @@ import ( "github.com/ozontech/file.d/metric" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" - "go.uber.org/atomic" ) type batcherTail struct { @@ -41,7 +41,7 @@ func TestBatcher(t *testing.T) { *workerData = batchCount } counter := (*workerData).(*atomic.Int32) - counter.Inc() + counter.Add(1) } seqIDs := make(map[SourceID]uint64) @@ -55,7 +55,7 @@ func TestBatcher(t *testing.T) { } seqIDs[event.SourceID] = event.SeqID - commitsCount.Inc() + commitsCount.Add(1) wg.Done() }} @@ -112,7 +112,7 @@ func TestBatcherMaxSize(t *testing.T) { *workerData = batchCount } counter := (*workerData).(*atomic.Int32) - counter.Inc() + counter.Add(1) } seqIDs := make(map[SourceID]uint64) @@ -126,7 +126,7 @@ func TestBatcherMaxSize(t *testing.T) { } seqIDs[event.SourceID] = event.SeqID - commitsCount.Inc() + commitsCount.Add(1) wg.Done() }} diff --git a/pipeline/event.go b/pipeline/event.go index 31e23217d..21ff51ca1 100644 --- a/pipeline/event.go +++ b/pipeline/event.go @@ -8,9 +8,10 @@ import ( "sync" "time" + "sync/atomic" + "github.com/ozontech/file.d/logger" insaneJSON "github.com/ozontech/insane-json" - "go.uber.org/atomic" ) type Event struct { @@ -226,19 +227,23 @@ func newEventPool(capacity, avgEventSize int) *eventPool { avgEventSize: avgEventSize, capacity: capacity, getMu: &sync.Mutex{}, - backCounter: *atomic.NewInt64(int64(capacity)), runHeartbeatOnce: &sync.Once{}, - stopped: atomic.NewBool(false), - slowWaiters: atomic.NewInt64(0), + stopped: &atomic.Bool{}, + slowWaiters: &atomic.Int64{}, wakeupInterval: time.Second * 5, } + eventPool.backCounter.Store(int64(capacity)) eventPool.getCond = sync.NewCond(eventPool.getMu) - for range capacity { - eventPool.free1 = append(eventPool.free1, *atomic.NewBool(true)) - eventPool.free2 = append(eventPool.free2, *atomic.NewBool(true)) - eventPool.events = append(eventPool.events, newEvent()) + eventPool.free1 = make([]atomic.Bool, capacity) + eventPool.free2 = make([]atomic.Bool, capacity) + eventPool.events = make([]*Event, capacity) + + for i := range capacity { + eventPool.free1[i].Store(true) + eventPool.free2[i].Store(true) + eventPool.events[i] = newEvent() } return eventPool @@ -247,7 +252,7 @@ func newEventPool(capacity, avgEventSize int) *eventPool { const maxTries = 3 func (p *eventPool) get(size int) *Event { - x := (p.getCounter.Inc() - 1) % int64(p.capacity) + x := (p.getCounter.Add(1) - 1) % int64(p.capacity) var tries int for { if x < p.backCounter.Load() { @@ -273,18 +278,18 @@ func (p *eventPool) get(size int) *Event { }) // slowest path - p.slowWaiters.Inc() + p.slowWaiters.Add(1) p.getMu.Lock() p.getCond.Wait() p.getMu.Unlock() - p.slowWaiters.Dec() + p.slowWaiters.Add(-1) tries = 0 } } event := p.events[x] p.events[x] = nil p.free2[x].Store(false) - p.inUseEvents.Inc() + p.inUseEvents.Add(1) event.stage = eventStageInput event.Size = size return event @@ -292,7 +297,7 @@ func (p *eventPool) get(size int) *Event { func (p *eventPool) back(event *Event) { event.stage = eventStagePool - x := (p.backCounter.Inc() - 1) % int64(p.capacity) + x := (p.backCounter.Add(1) - 1) % int64(p.capacity) var tries int for { // fast path @@ -318,7 +323,7 @@ func (p *eventPool) back(event *Event) { p.resetEvent(event) p.events[x] = event p.free1[x].Store(true) - p.inUseEvents.Dec() + p.inUseEvents.Add(-1) p.getCond.Broadcast() } @@ -438,7 +443,7 @@ func (p *lowMemoryEventPool) get(size int) *Event { getPool := p.pools[index] again: - inUse := int(p.inUseEvents.Inc()) + inUse := int(p.inUseEvents.Add(1)) // Fast path: we're not over the capacity. if inUse <= p.capacity { e := getPool.Get().(*Event) @@ -447,7 +452,7 @@ again: } // Slow path: wait until we fit in the capacity. - p.inUseEvents.Dec() + p.inUseEvents.Add(-1) // Run heartbeat to periodically wake up goroutines that are waiting. p.runHeartbeatOnce.Do(func() { @@ -455,13 +460,13 @@ again: }) // Wait until we fit in the capacity. - p.slowWaiters.Inc() + p.slowWaiters.Add(1) p.getCond.L.Lock() if !p.eventsAvailable() { p.getCond.Wait() } p.getCond.L.Unlock() - p.slowWaiters.Dec() + p.slowWaiters.Add(-1) goto again } @@ -471,7 +476,7 @@ func (p *lowMemoryEventPool) back(event *Event) { event.reset() backPool.Put(event) - p.inUseEvents.Dec() + p.inUseEvents.Add(-1) p.getCond.Broadcast() } diff --git a/pipeline/pipeline.go b/pipeline/pipeline.go index 1fb03fe86..28e5ec242 100644 --- a/pipeline/pipeline.go +++ b/pipeline/pipeline.go @@ -9,6 +9,7 @@ import ( "runtime" "strconv" "sync" + "sync/atomic" "time" "github.com/ozontech/file.d/decoder" @@ -18,7 +19,6 @@ import ( "github.com/ozontech/file.d/pipeline/metadata" insaneJSON "github.com/ozontech/insane-json" "github.com/prometheus/client_golang/prometheus" - "go.uber.org/atomic" "go.uber.org/zap" "go.uber.org/zap/zapcore" ) @@ -248,7 +248,7 @@ func New(name string, settings *Settings, registry *prometheus.Registry, lg *zap } func (p *Pipeline) IncReadOps() { - p.readOps.Inc() + p.readOps.Add(1) } func (p *Pipeline) IncMaxEventSizeExceeded(lvs ...string) { @@ -482,7 +482,7 @@ func (p *Pipeline) In(sourceID SourceID, sourceName string, offsets Offsets, byt } } - p.inputEvents.Inc() + p.inputEvents.Add(1) p.inputSize.Add(int64(length)) now := time.Now() @@ -628,7 +628,7 @@ func (p *Pipeline) finalize(event *Event, notifyInput bool, backEvent bool) { if notifyInput { p.input.Commit(event) - p.outputEvents.Inc() + p.outputEvents.Add(1) p.outputSize.Add(int64(event.Size)) } @@ -710,7 +710,7 @@ func (p *Pipeline) AddAction(info *ActionPluginStaticInfo) { totalCounter := make(map[string]*atomic.Uint64) for _, st := range allEventStatuses() { - totalCounter[string(st)] = atomic.NewUint64(0) + totalCounter[string(st)] = &atomic.Uint64{} } p.actionMetrics.set(info.MetricName, &actionMetric{ @@ -728,8 +728,9 @@ func (p *Pipeline) initProcs() { } p.logger.Info("starting pipeline", zap.Int("procs", procCount)) - p.procCount = atomic.NewInt32(int32(procCount)) - p.activeProcs = atomic.NewInt32(0) + p.procCount = &atomic.Int32{} + p.procCount.Store(int32(procCount)) + p.activeProcs = &atomic.Int32{} p.Procs = make([]*processor, 0, procCount) for i := 0; i < procCount; i++ { @@ -989,7 +990,7 @@ func (p *Pipeline) serveActionInfo(info *ActionPluginStaticInfo) func(http.Respo } { c := am.totalCounter[string(status)] if c == nil { - c = atomic.NewUint64(0) + c = &atomic.Uint64{} } events = append(events, Event{ Status: string(status), diff --git a/pipeline/pipeline_whitebox_test.go b/pipeline/pipeline_whitebox_test.go index d56ffefb1..4a059032d 100644 --- a/pipeline/pipeline_whitebox_test.go +++ b/pipeline/pipeline_whitebox_test.go @@ -1,6 +1,7 @@ package pipeline import ( + "sync/atomic" "testing" "github.com/ozontech/file.d/decoder" @@ -8,7 +9,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "go.uber.org/atomic" "go.uber.org/zap" ) @@ -35,7 +35,8 @@ func TestPipelineStreamEvent(t *testing.T) { streamID := StreamID(123123) procs := int32(7) - p.procCount = atomic.NewInt32(procs) + p.procCount = &atomic.Int32{} + p.procCount.Store(procs) p.input = &TestInputPlugin{} event := newEvent() event.SourceID = SourceID(streamID) diff --git a/pipeline/processor.go b/pipeline/processor.go index 126b13833..46dc0bd15 100644 --- a/pipeline/processor.go +++ b/pipeline/processor.go @@ -2,11 +2,11 @@ package pipeline import ( "errors" + "sync/atomic" "github.com/ozontech/file.d/logger" "github.com/ozontech/file.d/pipeline/doif" insaneJSON "github.com/ozontech/insane-json" - "go.uber.org/atomic" "go.uber.org/zap" ) @@ -125,9 +125,9 @@ func (p *processor) process() { return } - p.activeCounter.Inc() + p.activeCounter.Add(1) p.dischargeStream(st) - p.activeCounter.Dec() + p.activeCounter.Add(-1) } } @@ -323,7 +323,7 @@ func (p *processor) countEvent(event *Event, actionIndex int, status eventStatus p.metricsValues = append(p.metricsValues, val) } - am.totalCounter[string(status)].Inc() + am.totalCounter[string(status)].Add(1) am.count.WithLabelValues(p.metricsValues...).Inc() am.size.WithLabelValues(p.metricsValues...).Add(float64(event.Size)) } diff --git a/pipeline/router_test.go b/pipeline/router_test.go index f6518b681..cb3553e1a 100644 --- a/pipeline/router_test.go +++ b/pipeline/router_test.go @@ -2,6 +2,7 @@ package pipeline_test import ( "sync" + "sync/atomic" "testing" "time" @@ -10,7 +11,6 @@ import ( "github.com/ozontech/file.d/test" insaneJSON "github.com/ozontech/insane-json" "github.com/stretchr/testify/assert" - "go.uber.org/atomic" ) type fakeOutputPluginController struct { diff --git a/pipeline/stream.go b/pipeline/stream.go index 60481a63b..4b6d00ef6 100644 --- a/pipeline/stream.go +++ b/pipeline/stream.go @@ -2,10 +2,10 @@ package pipeline import ( "sync" + "sync/atomic" "time" "github.com/ozontech/file.d/logger" - "go.uber.org/atomic" ) // stream is a queue of events @@ -179,7 +179,7 @@ func (s *stream) tryUnblock() bool { } if s.awaySeq != s.commitSeq.Load() { - logger.Panicf("why events are different? away event id=%d, commit event id=%d", s.awaySeq, s.commitSeq) + logger.Panicf("why events are different? away event id=%d, commit event id=%d", s.awaySeq, s.commitSeq.Load()) } timeoutEvent := newTimeoutEvent(s) diff --git a/pipeline/streamer.go b/pipeline/streamer.go index 22c3c4a28..81bda5569 100644 --- a/pipeline/streamer.go +++ b/pipeline/streamer.go @@ -3,10 +3,10 @@ package pipeline import ( "fmt" "sync" + "sync/atomic" "time" "github.com/ozontech/file.d/logger" - "go.uber.org/atomic" ) type StreamID uint64 diff --git a/playground/playground.go b/playground/playground.go index 69c2aacbf..d8638d22b 100644 --- a/playground/playground.go +++ b/playground/playground.go @@ -7,6 +7,7 @@ import ( "fmt" "runtime" "strings" + "sync/atomic" "time" "github.com/bitly/go-simplejson" @@ -40,7 +41,6 @@ import ( "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" - "go.uber.org/atomic" "go.uber.org/zap" "go.uber.org/zap/zapcore" ) @@ -75,7 +75,7 @@ func (h *playground) Play(ctx context.Context, req PlayRequest) (PlayResponse, e fatalErr = fmt.Errorf("fatal: %s: %s", entry.Message, fatalPayload.String()) })) - pipelineName := fmt.Sprintf("playground_%d", h.nextPipelineID.Inc()) + pipelineName := fmt.Sprintf("playground_%d", h.nextPipelineID.Add(1)) settings := &pipeline.Settings{ Decoder: "json", DecoderParams: nil, diff --git a/plugin/action/convert_log_level/convert_log_level_test.go b/plugin/action/convert_log_level/convert_log_level_test.go index 78a5f99a6..5d24e1b19 100644 --- a/plugin/action/convert_log_level/convert_log_level_test.go +++ b/plugin/action/convert_log_level/convert_log_level_test.go @@ -1,13 +1,13 @@ package convert_log_level import ( + "sync/atomic" "testing" "time" "github.com/ozontech/file.d/pipeline" "github.com/ozontech/file.d/test" "github.com/stretchr/testify/require" - "go.uber.org/atomic" ) func TestDo(t *testing.T) { @@ -228,12 +228,13 @@ func TestDo(t *testing.T) { config := test.NewConfig(&tc.Config, nil) p, input, output := test.NewPipelineMock(test.NewActionPluginStaticInfo(factory, config, pipeline.MatchModeAnd, nil, false)) - outCounter := atomic.NewInt32(int32(len(tc.In))) + outCounter := &atomic.Int32{} + outCounter.Store(int32(len(tc.In))) var outEvents []string output.SetOutFn(func(e *pipeline.Event) { outEvents = append(outEvents, e.Root.EncodeToString()) - outCounter.Dec() + outCounter.Add(-1) }) for _, log := range tc.In { diff --git a/plugin/action/join/join_test.go b/plugin/action/join/join_test.go index c4c60b618..bcc11791c 100644 --- a/plugin/action/join/join_test.go +++ b/plugin/action/join/join_test.go @@ -3,12 +3,12 @@ package join import ( "fmt" "strings" + "sync/atomic" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "go.uber.org/atomic" "github.com/ozontech/file.d/cfg" "github.com/ozontech/file.d/pipeline" @@ -158,13 +158,13 @@ func TestSimpleJoin(t *testing.T) { inEvents := atomic.Int32{} input.SetInFn(func() { - inEvents.Inc() + inEvents.Add(1) }) outEvents := atomic.Int32{} lastID := atomic.Uint64{} output.SetOutFn(func(e *pipeline.Event) { - outEvents.Inc() + outEvents.Add(1) id := lastID.Swap(e.SeqID) if id != 0 && id >= e.SeqID { panic("wrong id") @@ -259,13 +259,13 @@ func TestJoinAfterNilNode(t *testing.T) { inEvents := atomic.Int32{} input.SetInFn(func() { - inEvents.Inc() + inEvents.Add(1) }) outEvents := atomic.Int32{} lastID := atomic.Uint64{} output.SetOutFn(func(e *pipeline.Event) { - outEvents.Inc() + outEvents.Add(1) id := lastID.Swap(e.SeqID) if id != 0 && id >= e.SeqID { panic("wrong id") diff --git a/plugin/action/join_template/join_template_test.go b/plugin/action/join_template/join_template_test.go index 916e457c4..a68fc6a9a 100644 --- a/plugin/action/join_template/join_template_test.go +++ b/plugin/action/join_template/join_template_test.go @@ -3,6 +3,7 @@ package join_template import ( "fmt" "strings" + "sync/atomic" "testing" "time" @@ -11,7 +12,6 @@ import ( "github.com/ozontech/file.d/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "go.uber.org/atomic" ) func TestSimpleJoin(t *testing.T) { @@ -84,13 +84,13 @@ func TestSimpleJoin(t *testing.T) { inEvents := atomic.Int32{} input.SetInFn(func() { - inEvents.Inc() + inEvents.Add(1) }) outEvents := atomic.Int32{} lastID := atomic.Uint64{} output.SetOutFn(func(e *pipeline.Event) { - outEvents.Inc() + outEvents.Add(1) id := lastID.Swap(e.SeqID) require.False(t, id != 0 && id >= e.SeqID) }) @@ -172,13 +172,13 @@ func TestJoinAfterNilNode(t *testing.T) { inEvents := atomic.Int32{} input.SetInFn(func() { - inEvents.Inc() + inEvents.Add(1) }) outEvents := atomic.Int32{} lastID := atomic.Uint64{} output.SetOutFn(func(e *pipeline.Event) { - outEvents.Inc() + outEvents.Add(1) id := lastID.Swap(e.SeqID) require.False(t, id != 0 && id >= e.SeqID) }) diff --git a/plugin/action/set_time/set_time_test.go b/plugin/action/set_time/set_time_test.go index 182131242..6f0875399 100644 --- a/plugin/action/set_time/set_time_test.go +++ b/plugin/action/set_time/set_time_test.go @@ -2,6 +2,7 @@ package set_time import ( "fmt" + "sync/atomic" "testing" "time" @@ -11,7 +12,6 @@ import ( "github.com/ozontech/file.d/xtime" insaneJSON "github.com/ozontech/insane-json" "github.com/stretchr/testify/require" - "go.uber.org/atomic" ) func TestPlugin_Do(t *testing.T) { @@ -139,7 +139,7 @@ func TestE2E_Plugin(t *testing.T) { counter := atomic.Int32{} output.SetOutFn(func(e *pipeline.Event) { require.NotEqual(t, "", e.Root.Dig("timestamp").AsString(), "wrong out event") - counter.Dec() + counter.Add(-1) }) counter.Add(1) diff --git a/plugin/action/throttle/limiters_map.go b/plugin/action/throttle/limiters_map.go index e7f5bdc97..11fa2a389 100644 --- a/plugin/action/throttle/limiters_map.go +++ b/plugin/action/throttle/limiters_map.go @@ -9,9 +9,9 @@ import ( "os" "strings" "sync" + "sync/atomic" "time" - "go.uber.org/atomic" "go.uber.org/zap" "github.com/ozontech/file.d/logger" @@ -43,10 +43,12 @@ type limiterWithGen struct { } func newLimiterWithGen(lim limiter, gen int64) *limiterWithGen { - return &limiterWithGen{ + lwg := &limiterWithGen{ limiter: lim, - gen: atomic.NewInt64(gen), + gen: &atomic.Int64{}, } + lwg.gen.Store(gen) + return lwg } // limiterConfig configuration for creation of new limiters. @@ -114,7 +116,7 @@ func newLimitersMap(lmCfg limitersMapConfig, redisOpts *xredis.Options) *limiter ctx: lmCfg.ctx, lims: make(map[string]*limiterWithGen), mu: &sync.RWMutex{}, - activeTasks: *atomic.NewUint32(0), + activeTasks: atomic.Uint32{}, curGen: nowTs, limitersExp: lmCfg.limitersExpiration.Microseconds(), logger: lmCfg.logger, diff --git a/plugin/input/file/file_test.go b/plugin/input/file/file_test.go index 9e583a92e..1266acd09 100644 --- a/plugin/input/file/file_test.go +++ b/plugin/input/file/file_test.go @@ -10,6 +10,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "syscall" "testing" "time" @@ -19,7 +20,6 @@ import ( uuid "github.com/satori/go.uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "go.uber.org/atomic" "github.com/ozontech/file.d/logger" "github.com/ozontech/file.d/pipeline" @@ -1049,7 +1049,8 @@ func TestRotationRenameWhileNotWorking(t *testing.T) { func TestTruncation(t *testing.T) { file := "" - x := atomic.NewInt32(3) + x := &atomic.Int32{} + x.Store(3) run(&test.Case{ Prepare: func() {}, Act: func(p *pipeline.Pipeline) { @@ -1070,7 +1071,7 @@ func TestTruncation(t *testing.T) { }, Out: func(event *pipeline.Event) { logger.Errorf("event=%v", event) - x.Dec() + x.Add(-1) }, }, 5) } @@ -1131,7 +1132,7 @@ func TestTruncationSeq(t *testing.T) { if size.Load() > int32(truncationSize) { size.Swap(0) _ = file.Truncate(0) - if int(truncations.Inc()) > truncationCount { + if int(truncations.Add(1)) > truncationCount { break } } diff --git a/plugin/input/file/offset.go b/plugin/input/file/offset.go index 3e9e1a659..381b0da89 100644 --- a/plugin/input/file/offset.go +++ b/plugin/input/file/offset.go @@ -8,10 +8,11 @@ import ( "strings" "sync" + "sync/atomic" + "github.com/ozontech/file.d/logger" "github.com/ozontech/file.d/pipeline" "github.com/ozontech/file.d/xtime" - "go.uber.org/atomic" ) type offsetDB struct { @@ -231,7 +232,7 @@ func safeSubstring(s string, length int) string { } func (o *offsetDB) save(jobs map[pipeline.SourceID]*Job, mu *sync.RWMutex) { - o.savesTotal.Inc() + o.savesTotal.Add(1) o.mu.Lock() defer o.mu.Unlock() diff --git a/plugin/input/file/offset_test.go b/plugin/input/file/offset_test.go index ea5709093..1bc190a6a 100644 --- a/plugin/input/file/offset_test.go +++ b/plugin/input/file/offset_test.go @@ -6,11 +6,12 @@ import ( "sync" "testing" + "sync/atomic" + "github.com/ozontech/file.d/pipeline" "github.com/ozontech/file.d/xtime" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "go.uber.org/atomic" ) func TestParseOffsets(t *testing.T) { @@ -97,7 +98,7 @@ func TestParallelOffsetsSave(t *testing.T) { lastEventSeq: 0, isVirgin: false, isDone: false, - shouldSkip: *atomic.NewBool(false), + shouldSkip: atomic.Bool{}, offsets: offsets, mu: &sync.Mutex{}, } @@ -111,7 +112,7 @@ func TestParallelOffsetsSave(t *testing.T) { lastEventSeq: 0, isVirgin: false, isDone: false, - shouldSkip: *atomic.NewBool(false), + shouldSkip: atomic.Bool{}, offsets: offsets, mu: &sync.Mutex{}, } diff --git a/plugin/input/file/provider.go b/plugin/input/file/provider.go index 61098f23e..34a169fad 100644 --- a/plugin/input/file/provider.go +++ b/plugin/input/file/provider.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "sync" + "sync/atomic" "syscall" "time" @@ -14,7 +15,6 @@ import ( "github.com/ozontech/file.d/pipeline" "github.com/ozontech/file.d/xtime" "github.com/rjeczalik/notify" - "go.uber.org/atomic" "go.uber.org/zap" ) @@ -168,7 +168,7 @@ func NewJobProvider(config *Config, metrics *metricCollection, sugLogger *zap.Su offsetDB: newOffsetDB(config.OffsetsFile, config.OffsetsFileTmp), jobs: make(map[pipeline.SourceID]*Job, config.MaxFiles), - jobsDone: atomic.NewInt32(0), + jobsDone: &atomic.Int32{}, jobsMu: &sync.RWMutex{}, jobsChan: make(chan *Job, config.MaxFiles), jobsLog: make([]string, 0, 16), @@ -303,7 +303,7 @@ func (jp *jobProvider) commit(event *pipeline.Event) { job.mu.Unlock() - jp.offsetsCommitted.Inc() + jp.offsetsCommitted.Add(1) if jp.config.PersistenceMode_ == persistenceModeSync { jp.offsetDB.save(jp.jobs, jp.jobsMu) } @@ -430,7 +430,7 @@ func (jp *jobProvider) addJob(file *os.File, stat os.FileInfo, filename string, isVirgin: true, isDone: true, - shouldSkip: *atomic.NewBool(false), + shouldSkip: atomic.Bool{}, offsets: nil, @@ -460,7 +460,7 @@ func (jp *jobProvider) addJob(file *os.File, stat os.FileInfo, filename string, ) } jp.jobsLog = append(jp.jobsLog, filename) - jp.jobsDone.Inc() + jp.jobsDone.Add(1) if symlink != "" { jp.logger.Infof("job added for a file %d:%s, symlink=%s", sourceID, filename, symlink) @@ -532,8 +532,8 @@ func (jp *jobProvider) initEofInfo(job *Job) { if !has { return } - eofInfoFromOffsets := eofInfo{} - eofInfoFromOffsets.setUnixNanoTimestamp(offsets.lastReadTimestamp) + + job.eofReadInfo.setUnixNanoTimestamp(offsets.lastReadTimestamp) minOffset := int64(math.MaxInt64) for _, offset := range offsets.streams { @@ -541,9 +541,7 @@ func (jp *jobProvider) initEofInfo(job *Job) { minOffset = offset } } - eofInfoFromOffsets.setOffset(minOffset) - - job.eofReadInfo = eofInfoFromOffsets + job.eofReadInfo.setOffset(minOffset) } // tryResumeJob job should be already locked and it'll be unlocked. @@ -558,7 +556,7 @@ func (jp *jobProvider) tryResumeJobAndUnlock(job *Job, filename string) { job.filename = filename job.isDone = false - if jp.jobsDone.Dec() < 0 { + if jp.jobsDone.Add(-1) < 0 { jp.logger.Panicf("done jobs counter is less than zero") } @@ -579,7 +577,7 @@ func (jp *jobProvider) doneJob(job *Job) { job.isVirgin = false jp.jobsMu.Lock() - v := int(jp.jobsDone.Inc()) + v := int(jp.jobsDone.Add(1)) jobsLen := len(jp.jobs) jp.numberOfCurrentJobsMetric.Set(float64(jobsLen)) @@ -822,7 +820,7 @@ func (jp *jobProvider) deleteJobAndUnlock(job *Job) { jp.jobsMu.Lock() delete(jp.jobs, sourceID) - c := jp.jobsDone.Dec() + c := jp.jobsDone.Add(-1) jp.jobsMu.Unlock() jp.logger.Infof("job %d:%s deleted", job.sourceID, filename) diff --git a/plugin/input/file/watcher_test.go b/plugin/input/file/watcher_test.go index 0e1f8e032..fdc562c17 100644 --- a/plugin/input/file/watcher_test.go +++ b/plugin/input/file/watcher_test.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "strings" + "sync/atomic" "testing" "time" @@ -13,7 +14,6 @@ import ( "github.com/rjeczalik/notify" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "go.uber.org/atomic" "go.uber.org/zap" ) @@ -32,7 +32,7 @@ func TestWatcher(t *testing.T) { dir := t.TempDir() shouldCreate := atomic.Int64{} notifyFn := func(_ notify.Event, _ string, _ os.FileInfo) { - shouldCreate.Inc() + shouldCreate.Add(1) } ctl := metric.NewCtl("test", prometheus.NewRegistry(), time.Minute, 0) w := NewWatcher( @@ -101,7 +101,7 @@ func TestWatcherPaths(t *testing.T) { shouldCreate := atomic.Int64{} notifyFn := func(_ notify.Event, _ string, _ os.FileInfo) { - shouldCreate.Inc() + shouldCreate.Add(1) } ctl := metric.NewCtl("test", prometheus.NewRegistry(), time.Minute, 0) w := NewWatcher( diff --git a/plugin/input/file/worker_test.go b/plugin/input/file/worker_test.go index e3defc21a..4bcaf6cbe 100644 --- a/plugin/input/file/worker_test.go +++ b/plugin/input/file/worker_test.go @@ -7,6 +7,7 @@ import ( "path" "strings" "sync" + "sync/atomic" "testing" "time" @@ -17,7 +18,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "go.uber.org/atomic" "go.uber.org/zap" ) @@ -109,7 +109,7 @@ func TestWorkerWork(t *testing.T) { lastEventSeq: 0, isVirgin: false, isDone: false, - shouldSkip: *atomic.NewBool(false), + shouldSkip: atomic.Bool{}, mu: &sync.Mutex{}, } ctl := metric.NewCtl("test", prometheus.NewRegistry(), 0, 0) @@ -288,7 +288,7 @@ func TestWorkerWorkMultiData(t *testing.T) { job := &Job{ file: f, - shouldSkip: *atomic.NewBool(false), + shouldSkip: atomic.Bool{}, offsets: pipeline.SliceMap{}, mu: &sync.Mutex{}, } @@ -518,7 +518,7 @@ func TestWorkerRemoveAfter(t *testing.T) { lastEventSeq: 0, isVirgin: false, isDone: false, - shouldSkip: *atomic.NewBool(false), + shouldSkip: atomic.Bool{}, mu: &sync.Mutex{}, } diff --git a/plugin/input/k8s/k8s.go b/plugin/input/k8s/k8s.go index 5d88a12ad..1287effcc 100644 --- a/plugin/input/k8s/k8s.go +++ b/plugin/input/k8s/k8s.go @@ -2,6 +2,7 @@ package k8s import ( "net/http" + "sync/atomic" "github.com/ozontech/file.d/cfg" "github.com/ozontech/file.d/decoder" @@ -10,7 +11,6 @@ import ( "github.com/ozontech/file.d/plugin/input/file" "github.com/ozontech/file.d/plugin/input/k8s/meta" - "go.uber.org/atomic" "go.uber.org/zap" ) @@ -190,7 +190,7 @@ func (p *Plugin) Start(config pipeline.AnyConfig, params *pipeline.InputPluginPa p.params = params p.config = config.(*Config) - startCounter := startCounter.Inc() + startCounter := startCounter.Add(1) if startCounter == 1 { meta.DeletedPodsCacheSize = p.config.DeletedPodsCacheSize diff --git a/plugin/input/k8s/meta/gatherer.go b/plugin/input/k8s/meta/gatherer.go index ba35ec602..14e32e1f2 100644 --- a/plugin/input/k8s/meta/gatherer.go +++ b/plugin/input/k8s/meta/gatherer.go @@ -9,11 +9,11 @@ import ( "path/filepath" "strings" "sync" + "sync/atomic" "time" lru "github.com/hashicorp/golang-lru/v2" "github.com/ozontech/file.d/logger" - "go.uber.org/atomic" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -71,7 +71,7 @@ var ( controller cache.Controller canUpdateMetaData atomic.Bool - metaLastAvailableTime atomic.Time + metaLastAvailableTime atomic.Value // stores time.Time expiredItems = make([]*MetaItem, 0, 16) // temporary list of expired items informerStop = make(chan struct{}, 1) @@ -206,7 +206,7 @@ func initInformer() { pod := obj.(*corev1.Pod) PutMeta(pod) deletedPodsCache.Add(PodName(pod.Name), true) - deletedPodsCounter.Inc() + deletedPodsCounter.Add(1) }, }) if err != nil { @@ -330,7 +330,7 @@ func cleanUpItems(items []*MetaItem) { defer metaDataMu.Unlock() for _, item := range items { - expiredItemsCounter.Inc() + expiredItemsCounter.Add(1) delete(MetaData.PodMeta[item.Namespace][item.PodName], item.ContainerID) if len(MetaData.PodMeta[item.Namespace][item.PodName]) == 0 { @@ -365,7 +365,7 @@ func GetPodMeta(ns Namespace, pod PodName, cid ContainerID) (bool, *podMeta) { return success, podMeta } - if !canUpdateMetaData.Load() && time.Since(metaLastAvailableTime.Load()) > metaWaitAvailabilityTimeout { + if !canUpdateMetaData.Load() && time.Since(metaLastAvailableTime.Load().(time.Time)) > metaWaitAvailabilityTimeout { return success, podMeta } @@ -431,7 +431,7 @@ func PutMeta(podData *corev1.Pod) { } } - metaAddedCounter.Inc() + metaAddedCounter.Add(1) } // putContainerMeta fullContainerID must be in format XXX://ID, eg docker://4e0301b633eaa2bfdcafdeba59ba0c72a3815911a6a820bf273534b0f32d98e0 diff --git a/plugin/output/devnull/devnull.go b/plugin/output/devnull/devnull.go index 4c2668ff6..d4d5751a6 100644 --- a/plugin/output/devnull/devnull.go +++ b/plugin/output/devnull/devnull.go @@ -1,9 +1,10 @@ package devnull import ( + "sync/atomic" + "github.com/ozontech/file.d/fd" "github.com/ozontech/file.d/pipeline" - "go.uber.org/atomic" ) /*{ introduction diff --git a/plugin/output/s3/s3_test.go b/plugin/output/s3/s3_test.go index dc455f16e..157792dcc 100644 --- a/plugin/output/s3/s3_test.go +++ b/plugin/output/s3/s3_test.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "strings" + "sync/atomic" "testing" "time" @@ -21,7 +22,6 @@ import ( "github.com/ozontech/file.d/test" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" - "go.uber.org/atomic" "go.uber.org/zap" ) @@ -29,7 +29,7 @@ const targetFile = "filetests/log.log" var ( dir, _ = filepath.Split(targetFile) - fileName atomic.String + fileName atomic.Value // stores string ) func testFactory(objStoreF objStoreFactory) (pipeline.AnyPlugin, pipeline.AnyConfig) { @@ -56,7 +56,7 @@ func fPutObjectOk(ctx context.Context, bucketName, objectName, filePath string, } } fileName.Store(fmt.Sprintf("%s/%s", bucketName, "mockLog.txt")) - f, err := os.OpenFile(fileName.Load(), os.O_CREATE|os.O_APPEND|os.O_RDWR, os.FileMode(0o777)) + f, err := os.OpenFile(fileName.Load().(string), os.O_CREATE|os.O_APPEND|os.O_RDWR, os.FileMode(0o777)) if err != nil { logger.Panicf("could not open or create f: %s, error: %s", fileName.Load(), err.Error()) @@ -87,7 +87,7 @@ func (put *putWithErr) fPutObjectErr(ctx context.Context, bucketName, objectName } fileName.Store(fmt.Sprintf("%s/%s", bucketName, "mockLog.txt")) - f, err := os.OpenFile(fileName.Load(), os.O_CREATE|os.O_APPEND|os.O_RDWR, os.FileMode(0o777)) + f, err := os.OpenFile(fileName.Load().(string), os.O_CREATE|os.O_APPEND|os.O_RDWR, os.FileMode(0o777)) if err != nil { logger.Panicf("could not open or create file: %s, error: %s", fileName, err.Error()) } @@ -172,7 +172,7 @@ func TestStart(t *testing.T) { test.SendPack(t, p, tests.firstPack) time.Sleep(time.Second) - size1 := test.CheckNotZero(t, fileName.Load(), "s3 data is missed after first pack") + size1 := test.CheckNotZero(t, fileName.Load().(string), "s3 data is missed after first pack") // check deletion upload log files match := test.GetMatches(t, pattern) @@ -188,7 +188,7 @@ func TestStart(t *testing.T) { assert.Equal(t, 1, len(match)) test.CheckZero(t, match[0], "log file is not empty") - size2 := test.CheckNotZero(t, fileName.Load(), "s3 data missed after second pack") + size2 := test.CheckNotZero(t, fileName.Load().(string), "s3 data missed after second pack") assert.True(t, size2 > size1) // failed during writing @@ -352,7 +352,7 @@ func TestStartWithMultiBuckets(t *testing.T) { test.SendPack(t, p, tests.firstPack) time.Sleep(time.Second) - size1 := test.CheckNotZero(t, fileName.Load(), "s3 data is missed after first pack") + size1 := test.CheckNotZero(t, fileName.Load().(string), "s3 data is missed after first pack") // check deletion upload log files for _, pattern := range patterns { @@ -372,7 +372,7 @@ func TestStartWithMultiBuckets(t *testing.T) { test.CheckZero(t, match[0], "log file is not empty") } - size2 := test.CheckNotZero(t, fileName.Load(), "s3 data missed after second pack") + size2 := test.CheckNotZero(t, fileName.Load().(string), "s3 data missed after second pack") assert.True(t, size2 > size1) // failed during writing @@ -611,8 +611,8 @@ func TestStartWithSendProblems(t *testing.T) { test.CheckZero(t, matches[0], "log file is not empty after restart sending") // check mock file is not empty and contains more than 3 raws - test.CheckNotZero(t, fileName.Load(), "s3 file is empty") - f, err := os.Open(fileName.Load()) + test.CheckNotZero(t, fileName.Load().(string), "s3 file is empty") + f, err := os.Open(fileName.Load().(string)) assert.NoError(t, err) defer f.Close() @@ -626,6 +626,6 @@ func TestStartWithSendProblems(t *testing.T) { } func isNoSentToS3() bool { - _, err := os.Stat(fileName.Load()) + _, err := os.Stat(fileName.Load().(string)) return err != nil && os.IsNotExist(err) } diff --git a/test/test.go b/test/test.go index e3b05e442..abd60ef27 100644 --- a/test/test.go +++ b/test/test.go @@ -4,6 +4,7 @@ import ( "math/rand" "strconv" "strings" + "sync/atomic" "time" "github.com/ozontech/file.d/cfg" @@ -13,7 +14,6 @@ import ( "github.com/ozontech/file.d/plugin/input/fake" "github.com/ozontech/file.d/plugin/output/devnull" "github.com/prometheus/client_golang/prometheus" - "go.uber.org/atomic" "go.uber.org/zap" "go.uber.org/zap/zapcore" ) @@ -40,7 +40,8 @@ func RunCase(testCase *Case, inputInfo *pipeline.InputPluginInfo, eventCount int } func startCasePipeline(act func(pipeline *pipeline.Pipeline), out func(event *pipeline.Event), eventCount int, inputInfo *pipeline.InputPluginInfo, pipelineOpts ...string) *pipeline.Pipeline { - x := atomic.NewInt32(int32(eventCount)) + x := &atomic.Int32{} + x.Store(int32(eventCount)) pipelineOpts = append(pipelineOpts, "passive") p := NewPipeline(nil, pipelineOpts...) @@ -59,7 +60,7 @@ func startCasePipeline(act func(pipeline *pipeline.Pipeline), out func(event *pi }) outputPlugin.SetOutFn(func(event *pipeline.Event) { - x.Dec() + x.Add(-1) if out != nil { out(event) } diff --git a/xoauth/tokensource.go b/xoauth/tokensource.go index 13af5bd68..375dc3b1c 100644 --- a/xoauth/tokensource.go +++ b/xoauth/tokensource.go @@ -3,10 +3,10 @@ package xoauth import ( "context" "fmt" + "sync/atomic" "time" "github.com/ozontech/file.d/logger" - "go.uber.org/atomic" ) // tokenIssuer issues token for specific token grant type flow