From 0c83ccc82c082dd325e2077785ba4b1e6d7c5024 Mon Sep 17 00:00:00 2001 From: Guy J Grigsby Date: Thu, 29 Jul 2021 09:14:07 -0600 Subject: [PATCH 01/17] update readme, add tests, and allow for use of in mem db for dev --- .gitignore | 2 + Makefile | 8 +- README.md | 8 ++ database/mem_test.go | 194 +++++++++++++++++++++++++++++++++++++++++++ go.mod | 7 +- go.sum | 14 +++- main.go | 31 ++++++- 7 files changed, 254 insertions(+), 10 deletions(-) create mode 100644 database/mem_test.go diff --git a/.gitignore b/.gitignore index 5cfd0ab..de13f36 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ tmp/ coverage.txt video-captions-api .DS_Store +main +captions-api diff --git a/Makefile b/Makefile index 2740612..adab3ca 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,7 @@ dev: - SERVER_HTTP_PORT=8000\ + @go build -v -o captions-api . + + @SERVER_HTTP_PORT=8000\ SERVER_GIZMO_HEALTH_CHECK_PATH=/healthz\ THREE_PLAY_API_KEY=$(THREE_PLAY_API_KEY) \ THREE_PLAY_API_SECRET=$(THREE_PLAY_API_SECRET) \ @@ -10,10 +12,10 @@ dev: BUCKET_NAME=$(CAPTIONS_BUCKET_NAME) \ CALLBACK_URL=$(CALLBACK_URL) \ CALLBACK_API_KEY=$(CALLBACK_API_KEY) \ - go run main.go + ./captions-api install-golangcilint: - GO111MODULE=off go get github.com/golangci/golangci-lint/cmd/golangci-lint + go get github.com/golangci/golangci-lint/cmd/golangci-lint run-lint: golangci-lint run --enable-all -D errcheck -D lll -D funlen -D wsl --deadline 5m ./... diff --git a/README.md b/README.md index 2c2ee38..6dac9b0 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,14 @@ THREE_PLAY_API_SECRET Note that `THREE_PLAY_API_KEY` should take the form of `captions:,transcript:`. +The following env vars are required for production environments, but have fallbacks for development. +In the even that `PROJECT_ID` is absent, we start an in memory datastore and log a warning. + +``` +PROJECT_ID='nyt-video-dev' +BUCKET_NAME='bakeoff-trint-3play' +``` + Run: ``` diff --git a/database/mem_test.go b/database/mem_test.go new file mode 100644 index 0000000..60dfac4 --- /dev/null +++ b/database/mem_test.go @@ -0,0 +1,194 @@ +package database + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" +) + +var ( + log = logrus.New() +) + +func TestMemStore(t *testing.T) { + tests := []struct { + id string + job Job + name string + }{ + { + job: Job{ID: "ID"}, + name: "happy path", + }, + } + db := NewMemoryDatabase() + + for _, tc := range tests { + id, err := db.StoreJob(&tc.job) + require.NoError(t, err, "", "Error in test %s", tc.name) + require.Equal(t, id, tc.job.ID, "Wrong ID in test %s", tc.name) + } + +} + +func TestGetJobID(t *testing.T) { + tests := []struct { + job Job + name string + }{ + { + job: Job{ + ID: "getjobfakeid", + Language: "en", + Done: true, + }, + name: "happy path", + }, + } + db := NewMemoryDatabase() + + for _, tc := range tests { + id, err := db.StoreJob(&tc.job) + require.NoError(t, err, "", "Error in test %s", tc.name) + + job, err := db.GetJob(id) + require.NoError(t, err) + + require.Equal(t, job.ID, tc.job.ID) + require.Equal(t, job.Language, tc.job.Language) + require.Equal(t, job.Done, tc.job.Done) + } + +} + +func generateJobs(t *testing.T, n int) *sync.Map { + var jobs sync.Map + for i := 0; i < n; i++ { + id := fmt.Sprintf("job-%d", i) + job := &Job{ + ID: id, + } + jobs.Store(id, job) + + } + return &jobs +} + +func TestStoreAndGetParallel(t *testing.T) { + + tests := []struct { + count int + name string + concurrency int + }{ + { + count: 100, + name: "parallel happy path", + concurrency: 10, + }, + } + + for _, tc := range tests { + db := NewMemoryDatabase() + ctx, cancel := context.WithCancel(context.Background()) + + jobs := generateJobs(t, tc.count) + pLog := log.WithField("producer", 0) + toValidate := jobQueue(ctx, t, jobs, db, pLog) + // I know this seems way more complicated that it should be. Why not just wait and check at the end? + // The reason is that we want to hit the store with as many requests as possible as fast as possible to + // make sure the lock is working as designed. + var wg sync.WaitGroup + var runningTotal uint64 + for c := 0; c < tc.concurrency; c++ { + wg.Add(1) + log.Debug( + "starting worker", + "name", c, + "population", tc.concurrency, + ) + wLog := logrus.WithField("worker", c) + go func(name string, count int, log *logrus.Entry) { + defer wg.Done() + for { + select { + case <-ctx.Done(): + total := atomic.LoadUint64(&runningTotal) + log.Debug( + "Context completed. Attempting to validate completion count", + "total", total, + ) + if total != uint64(tc.count) { + t.Errorf("Test %s timed out: error %v", tc.name, ctx.Err()) + } + cancel() + return + + case j := <-toValidate: + total := atomic.AddUint64(&runningTotal, 1) + log.Debug( + "pulling job", + "ID", j.ID, + "total", total, + ) + + got, err := db.GetJob(j.ID) + require.NoError(t, err, "Error in %s", tc.name) + log.Debug( + "retreived job from store", + "key", j.ID, + "id", got.ID, + ) + + e, ok := jobs.Load(got.ID) + require.True(t, ok) + + expect := e.(*Job).ID + require.Equal(t, expect, got.ID) + + if total == uint64(tc.count) { + // Success + cancel() + return + } + } + + } + }(tc.name, tc.count, wLog) + } + wg.Wait() + cancel() + } +} + +func jobQueue(ctx context.Context, t *testing.T, jobs *sync.Map, db DB, log *logrus.Entry) <-chan *Job { + q := make(chan *Job) + + go func() { + jobs.Range(func(id, j interface{}) bool { + job := j.(*Job) + select { + case <-ctx.Done(): + err := ctx.Err() + t.Errorf("Test timed out: err= %v", err) + return false + default: + db.StoreJob(job) + log.Debug( + "pushing job", + "id", job.ID, + ) + q <- job + } + return true + }) + }() + + return q + +} diff --git a/go.mod b/go.mod index a80b455..f015d1f 100644 --- a/go.mod +++ b/go.mod @@ -6,12 +6,15 @@ require ( github.com/NYTimes/gizmo v1.3.6 github.com/NYTimes/gziphandler v1.1.1 github.com/google/uuid v1.1.3 + github.com/inconshreveable/log15 v0.0.0-20201112154412-8562bdadbbac github.com/kelseyhightower/envconfig v1.4.0 + github.com/mattn/go-colorable v0.1.8 // indirect + github.com/mattn/go-isatty v0.0.13 // indirect github.com/nytimes/amara v0.3.0 github.com/nytimes/threeplay v0.3.2 - github.com/sirupsen/logrus v1.7.0 + github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.7.0 github.com/tdewolff/parse/v2 v2.4.3 ) -go 1.13 +go 1.16 diff --git a/go.sum b/go.sum index 925f811..4b743fd 100644 --- a/go.sum +++ b/go.sum @@ -92,6 +92,7 @@ github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -171,6 +172,8 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/log15 v0.0.0-20201112154412-8562bdadbbac h1:n1DqxAo4oWPMvH1+v+DLYlMCecgumhhgnxAPdqDIFHI= +github.com/inconshreveable/log15 v0.0.0-20201112154412-8562bdadbbac/go.mod h1:cOaXtrgN4ScfRrD9Bre7U1thNq5RtJ8ZoP4iXVGRj6o= github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= @@ -193,12 +196,16 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA= +github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d h1:VhgPp6v9qf9Agr/56bj7Y/xa04UccTW04VP0Qed4vnQ= github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH2ZwIWBy3CJBeOBEugqcmXREj14T+iG/4k4U= @@ -234,8 +241,8 @@ github.com/sethgrid/pester v0.0.0-20190127155807-68a33a018ad0/go.mod h1:Ad7IjTpv github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= @@ -380,6 +387,7 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/main.go b/main.go index 7b0385c..507637b 100644 --- a/main.go +++ b/main.go @@ -7,16 +7,43 @@ import ( "github.com/NYTimes/video-captions-api/providers" "github.com/NYTimes/video-captions-api/service" "github.com/kelseyhightower/envconfig" + "github.com/sirupsen/logrus" ) func main() { + var ( + db database.DB + err error + ) var cfg config.CaptionsServiceConfig envconfig.Process("", &cfg) cfg.Logger = server.Log - db, err := database.NewDatastoreDatabase(cfg.ProjectID) if err != nil { - server.Log.Fatal("Unable to create Datastore client", err) + server.Log.WithFields( + logrus.Fields{ + "err": err, + }, + ).Info("Invalid value for linker flag main.inmemorydbpermittted. Using remote store provided in PROJECT_ID and BUCKET_NAME") + + } + server.Log.WithFields( + logrus.Fields{ + "projectid": cfg.ProjectID, + "bucketname": cfg.BucketName, + "callback": cfg.CallbackURL, + }, + ).Info("Server Starting") + if cfg.ProjectID != "" { + db, err = database.NewDatastoreDatabase(cfg.ProjectID) + if err != nil { + server.Log.Fatal("Unable to create Datastore client", err) + } + } else { + server.Log.Warn("Project ID is empty. Using in memory datastore. Are you sure this is what you want?") + db = database.NewMemoryDatabase() + } + threeplayConfig := providers.Load3PlayConfigFromEnv() amaraConfig := providers.LoadAmaraConfigFromEnv() captionsService := service.NewCaptionsService(&cfg, db) From ac84287d6a1cbcf642ecc88cd3c7a382261ae76e Mon Sep 17 00:00:00 2001 From: Guy J Grigsby Date: Fri, 30 Jul 2021 09:13:23 -0600 Subject: [PATCH 02/17] stub trint provider --- providers/trint.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 providers/trint.go diff --git a/providers/trint.go b/providers/trint.go new file mode 100644 index 0000000..6ee4b4b --- /dev/null +++ b/providers/trint.go @@ -0,0 +1,26 @@ +package providers + +import "github.com/NYTimes/video-captions-api/database" + +type trint struct { +} + +func (t *trint) DispatchJob(_ *database.Job) error { + panic("not implemented") // TODO: Implement +} + +func (t *trint) Download(_ *database.Job, _ string) ([]byte, error) { + panic("not implemented") // TODO: Implement +} + +func (t *trint) GetProviderJob(_ *database.Job) (*database.ProviderJob, error) { + panic("not implemented") // TODO: Implement +} + +func (t *trint) GetName() string { + panic("not implemented") // TODO: Implement +} + +func (t *trint) CancelJob(_ *database.Job) (bool, error) { + panic("not implemented") // TODO: Implement +} From 9bfefb73cb1d366d5d64d2786472771d600a5532 Mon Sep 17 00:00:00 2001 From: Guy J Grigsby Date: Fri, 30 Jul 2021 12:25:53 -0600 Subject: [PATCH 03/17] add promethus metrics collector --- .travis.yml | 3 +- Makefile | 3 +- go.mod | 16 ++++++-- go.sum | 44 ++++++++++++-------- main.go | 90 ++++++++++++++++++++++++++++++++++++++++- service/service.go | 5 ++- service/service_test.go | 3 +- 7 files changed, 138 insertions(+), 26 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5e63183..6adb567 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ addons: apt: packages: - docker-ce + - git env: global: - GO_FOR_RELEASE=1.x @@ -17,7 +18,7 @@ install: - go mod download script: - make run-lint coverage - - go build -o video-captions-api + - VERSION=`git describe --always --tags` go build -o video-captions-api -ldflags="-X main.version=$(VERSION) after_success: - bash <(curl -s https://codecov.io/bash) - travis-scripts/docker.bash diff --git a/Makefile b/Makefile index adab3ca..4219316 100644 --- a/Makefile +++ b/Makefile @@ -18,8 +18,9 @@ install-golangcilint: go get github.com/golangci/golangci-lint/cmd/golangci-lint run-lint: - golangci-lint run --enable-all -D errcheck -D lll -D funlen -D wsl --deadline 5m ./... + golangci-lint run -D lll -D funlen --deadline 5m ./... +PHONY: lint: install-golangcilint run-lint coverage: diff --git a/go.mod b/go.mod index f015d1f..ef3f48b 100644 --- a/go.mod +++ b/go.mod @@ -1,20 +1,30 @@ module github.com/NYTimes/video-captions-api +replace github.com/nytimes-labs/client_golang v1.11.0 => github.com/prometheus/client_golang v1.11.0 + require ( cloud.google.com/go/datastore v1.4.0 cloud.google.com/go/storage v1.14.0 + contrib.go.opencensus.io/exporter/prometheus v0.1.0 github.com/NYTimes/gizmo v1.3.6 github.com/NYTimes/gziphandler v1.1.1 + github.com/beorn7/perks v1.0.1 // indirect + github.com/google/go-cmp v0.5.5 // indirect github.com/google/uuid v1.1.3 - github.com/inconshreveable/log15 v0.0.0-20201112154412-8562bdadbbac github.com/kelseyhightower/envconfig v1.4.0 - github.com/mattn/go-colorable v0.1.8 // indirect - github.com/mattn/go-isatty v0.0.13 // indirect github.com/nytimes/amara v0.3.0 github.com/nytimes/threeplay v0.3.2 + github.com/pkg/errors v0.9.1 + github.com/prometheus/client_golang v0.9.4 + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/procfs v0.2.0 // indirect github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.7.0 github.com/tdewolff/parse/v2 v2.4.3 + go.opencensus.io v0.23.0 // indirect + golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 // indirect + golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect + golang.org/x/text v0.3.6 // indirect ) go 1.16 diff --git a/go.sum b/go.sum index 4b743fd..e22a2c5 100644 --- a/go.sum +++ b/go.sum @@ -40,6 +40,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0 h1:6RRlFMv1omScs6iq2hfE3IvgE+l6RfJPampq8UZc5TU= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +contrib.go.opencensus.io/exporter/prometheus v0.1.0 h1:SByaIoWwNgMdPSgl5sMqM2KDE5H/ukPWBRo314xiDvg= +contrib.go.opencensus.io/exporter/prometheus v0.1.0/go.mod h1:cGFniUXGZlKRjzOyuZJ6mgB+PgBcCIa79kEKR8YCW+A= contrib.go.opencensus.io/exporter/stackdriver v0.13.1/go.mod h1:z2tyTZtPmQ2HvWH4cOmVDgtY+1lomfKdbLnkJvZdc8c= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -59,8 +61,9 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/aws/aws-sdk-go v1.23.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.31.3/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -92,7 +95,6 @@ github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -134,8 +136,10 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -172,8 +176,6 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/inconshreveable/log15 v0.0.0-20201112154412-8562bdadbbac h1:n1DqxAo4oWPMvH1+v+DLYlMCecgumhhgnxAPdqDIFHI= -github.com/inconshreveable/log15 v0.0.0-20201112154412-8562bdadbbac/go.mod h1:cOaXtrgN4ScfRrD9Bre7U1thNq5RtJ8ZoP4iXVGRj6o= github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= @@ -196,11 +198,6 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA= -github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -218,21 +215,27 @@ github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v0.9.4 h1:Y8E/JaaPbmFSW2V81Ab/d8yZFYQQGbni1b1jPcG9Y6A= github.com/prometheus/client_golang v0.9.4/go.mod h1:oCXIBxdI62A4cR6aTRJCgetEjecSIYzOEaeAn4iYEpM= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= +github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/sethgrid/pester v0.0.0-20180430140037-03e26c9abbbf/go.mod h1:Ad7IjTpvzZO8Fl0vh9AzQ+j/jYZfyp2diGwI8m5q+ns= @@ -250,6 +253,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tdewolff/parse/v2 v2.4.3 h1:k24zHgTRGm7LkvbTEreuavyZTf0k8a/lIenggv62OiU= @@ -269,8 +273,9 @@ go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= @@ -319,6 +324,7 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -346,9 +352,11 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b h1:iFwSg7t5GZmB/Q5TjiEAsdoLDrdJRC1RiF2WhuV29Qw= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -386,8 +394,8 @@ golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -406,16 +414,18 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/main.go b/main.go index 507637b..bd84b3e 100644 --- a/main.go +++ b/main.go @@ -1,15 +1,31 @@ package main import ( + "fmt" + "net/http" + "time" + + "contrib.go.opencensus.io/exporter/prometheus" "github.com/NYTimes/gizmo/server" "github.com/NYTimes/video-captions-api/config" "github.com/NYTimes/video-captions-api/database" "github.com/NYTimes/video-captions-api/providers" "github.com/NYTimes/video-captions-api/service" "github.com/kelseyhightower/envconfig" + "github.com/pkg/errors" + goprom "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" ) +const ( + // MetricsNamespace is the name of the application. + MetricsNamespace = "video_captions_api" +) + +var ( + version = "no version from LDFLAGS" +) + func main() { var ( db database.DB @@ -43,10 +59,37 @@ func main() { db = database.NewMemoryDatabase() } + // metrics server + + exporter, registry := mustInitMetrics() + go func(log *logrus.Entry) { + addr := ":9000" + log.WithField("address", addr).Info("starting metric server") + + mux := http.NewServeMux() + mux.Handle("/metrics", exporter) + + metricsServer := &http.Server{ + Addr: addr, + ReadTimeout: 2 * time.Second, + WriteTimeout: 2 * time.Second, + Handler: mux, + } + var err error + if err = metricsServer.ListenAndServe(); err != http.ErrServerClosed { + log.WithFields(logrus.Fields{ + "err": err, + "address": addr, + }).Fatal("Metrics server failure") + } + + }(server.Log.WithField("service", "metricsServer")) + + // caption server threeplayConfig := providers.Load3PlayConfigFromEnv() amaraConfig := providers.LoadAmaraConfigFromEnv() - captionsService := service.NewCaptionsService(&cfg, db) + captionsService := service.NewCaptionsService(&cfg, db, registry) captionsService.AddProvider(providers.New3PlayProvider(&threeplayConfig, &cfg)) captionsService.AddProvider(providers.NewAmaraProvider(&amaraConfig, &cfg)) @@ -63,3 +106,48 @@ func main() { server.Log.Fatal("Server encountered a fatal error: ", err) } } + +func mustInitMetrics() (*prometheus.Exporter, *goprom.Registry) { + pe, r, err := initMetrics() + if err != nil { + panic(errors.Wrap(err, "Failed to initialize metrics service")) + } + return pe, r +} + +func initMetrics() (*prometheus.Exporter, *goprom.Registry, error) { + r := goprom.NewRegistry() + r.MustRegister(goprom.NewProcessCollector(goprom.ProcessCollectorOpts{})) + r.MustRegister(goprom.NewGoCollector()) + + versionCollector := goprom.NewGaugeVec(goprom.GaugeOpts{ + Namespace: MetricsNamespace, + Name: "version", + Help: "Application version.", + }, []string{"version"}) + + r.MustRegister(versionCollector) + versionCollector.WithLabelValues(version).Add(1) + + captionTimer := goprom.NewHistogramVec(goprom.HistogramOpts{ + Namespace: MetricsNamespace, + Name: "asr_execution_time_seconds", + Help: "The temperature of the frog pond.", + Buckets: goprom.LinearBuckets(20, 5, 5), + }, []string{ + "provider", + }) + + r.MustRegister(captionTimer) + + // Stats exporter: Prometheus + pe, err := prometheus.NewExporter(prometheus.Options{ + Namespace: MetricsNamespace, + Registry: r, + }) + if err != nil { + return nil, nil, fmt.Errorf("failed to create the Prometheus stats exporter %w", err) + } + + return pe, r, nil +} diff --git a/service/service.go b/service/service.go index 5c51f2f..15f40dd 100644 --- a/service/service.go +++ b/service/service.go @@ -3,9 +3,10 @@ package service import ( "net/http" - "github.com/NYTimes/gziphandler" + prom "github.com/prometheus/client_golang/prometheus" "github.com/NYTimes/gizmo/server" + "github.com/NYTimes/gziphandler" "github.com/NYTimes/video-captions-api/config" "github.com/NYTimes/video-captions-api/database" "github.com/NYTimes/video-captions-api/providers" @@ -19,7 +20,7 @@ type CaptionsService struct { } // NewCaptionsService creates a CaptionsService -func NewCaptionsService(cfg *config.CaptionsServiceConfig, db database.DB) *CaptionsService { +func NewCaptionsService(cfg *config.CaptionsServiceConfig, db database.DB, metrics *prom.Registry) *CaptionsService { storage, _ := NewGCSStorage(cfg.BucketName, cfg.Logger) return &CaptionsService{ Client{ diff --git a/service/service_test.go b/service/service_test.go index 5c0540d..10c8dc7 100644 --- a/service/service_test.go +++ b/service/service_test.go @@ -11,6 +11,7 @@ import ( "github.com/NYTimes/video-captions-api/config" "github.com/NYTimes/video-captions-api/database" "github.com/NYTimes/video-captions-api/providers" + "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" ) @@ -131,7 +132,7 @@ func TestNewCaptionsService(t *testing.T) { } db := database.NewMemoryDatabase() - service := NewCaptionsService(&cfg, db) + service := NewCaptionsService(&cfg, db, prometheus.NewRegistry()) assert := assert.New(t) From 17fe5ac6e7a4bf88238c44e3343e27ebabcba356 Mon Sep 17 00:00:00 2001 From: Guy J Grigsby Date: Fri, 30 Jul 2021 12:29:48 -0600 Subject: [PATCH 04/17] add metrics registry to CaptionsService instance variables. --- service/service.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/service/service.go b/service/service.go index 15f40dd..b32d1ca 100644 --- a/service/service.go +++ b/service/service.go @@ -3,6 +3,7 @@ package service import ( "net/http" + "github.com/prometheus/client_golang/prometheus" prom "github.com/prometheus/client_golang/prometheus" "github.com/NYTimes/gizmo/server" @@ -15,8 +16,9 @@ import ( // CaptionsService the service responsible to wrapping interactions with Providers type CaptionsService struct { - client Client - logger *log.Logger + client Client + logger *log.Logger + metrics *prometheus.Registry } // NewCaptionsService creates a CaptionsService @@ -31,6 +33,7 @@ func NewCaptionsService(cfg *config.CaptionsServiceConfig, db database.DB, metri CallbackURL: cfg.CallbackURL, }, cfg.Logger, + metrics, } } From ee2675677df6d2ae0760987c41b6a02a54c4e039 Mon Sep 17 00:00:00 2001 From: Guy J Grigsby Date: Fri, 30 Jul 2021 12:35:16 -0600 Subject: [PATCH 05/17] copy pasta --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index bd84b3e..79b175d 100644 --- a/main.go +++ b/main.go @@ -132,7 +132,7 @@ func initMetrics() (*prometheus.Exporter, *goprom.Registry, error) { captionTimer := goprom.NewHistogramVec(goprom.HistogramOpts{ Namespace: MetricsNamespace, Name: "asr_execution_time_seconds", - Help: "The temperature of the frog pond.", + Help: "provider caption time", Buckets: goprom.LinearBuckets(20, 5, 5), }, []string{ "provider", From 88c3eaa9aa2138b8cec0beb9b8b5d5cf343677a2 Mon Sep 17 00:00:00 2001 From: Guy J Grigsby Date: Fri, 30 Jul 2021 16:14:16 -0600 Subject: [PATCH 06/17] absracted callbacks and adding metrics and cb handlers --- .golangci-lint.yml | 734 ++++++++++++++++++++++++++++++++++++++++ main.go | 109 ++---- providers/amara.go | 8 +- providers/provider.go | 33 +- providers/threeplay.go | 9 +- providers/trint.go | 9 +- providers/upload.go | 4 + service/client.go | 2 +- service/client_test.go | 35 +- service/job.go | 56 --- service/job_test.go | 9 +- service/service.go | 66 +++- service/service_test.go | 15 +- sidekicks.go | 138 ++++++++ 14 files changed, 1055 insertions(+), 172 deletions(-) create mode 100644 .golangci-lint.yml create mode 100644 sidekicks.go diff --git a/.golangci-lint.yml b/.golangci-lint.yml new file mode 100644 index 0000000..14f3197 --- /dev/null +++ b/.golangci-lint.yml @@ -0,0 +1,734 @@ +# This file contains all available configuration options +# with their default values. +linters: + disable-all: true + enable: + - megacheck + - govet + enable-all: true + disable: + - maligned + - prealloc + presets: + - bugs + - unused + fast: false + + +# options for analysis running +run: + # default concurrency is a available CPU number + concurrency: 4 + + # timeout for analysis, e.g. 30s, 5m, default is 1m + timeout: 1m + + # exit code when at least one issue was found, default is 1 + issues-exit-code: 1 + + # include test files or not, default is true + tests: false + + # list of build tags, all linters use it. Default is empty list. + build-tags: + + # which dirs to skip: issues from them won't be reported; + # can use regexp here: generated.*, regexp is applied on full path; + # default value is empty list, but default dirs are skipped independently + # from this option's value (see skip-dirs-use-default). + # "/" will be replaced by current OS file path separator to properly work + # on Windows. + skip-dirs: + + # default is true. Enables skipping of directories: + # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ + skip-dirs-use-default: true + + # which files to skip: they will be analyzed, but issues from them + # won't be reported. Default value is empty list, but there is + # no need to include all autogenerated files, we confidently recognize + # autogenerated files. If it's not please let us know. + # "/" will be replaced by current OS file path separator to properly work + # on Windows. + skip-files: + + # by default isn't set. If set we pass it to "go list -mod={option}". From "go help modules": + # If invoked with -mod=readonly, the go command is disallowed from the implicit + # automatic updating of go.mod described above. Instead, it fails when any changes + # to go.mod are needed. This setting is most useful to check that go.mod does + # not need updates, such as in a continuous integration and testing system. + # If invoked with -mod=vendor, the go command assumes that the vendor + # directory holds the correct copies of dependencies and ignores + # the dependency descriptions in go.mod. + # modules-download-mode: readonly|vendor|mod + + # Allow multiple parallel golangci-lint instances running. + # If false (default) - golangci-lint acquires file lock on start. + allow-parallel-runners: true + + +# output configuration options +output: + # colored-line-number|line-number|json|tab|checkstyle|code-climate|junit-xml|github-actions + # default is "colored-line-number" + format: colored-line-number + + # print lines of code with issue, default is true + print-issued-lines: true + + # print linter name in the end of issue text, default is true + print-linter-name: true + + # make issues output unique by line, default is true + uniq-by-line: true + + # add a prefix to the output file references; default is no prefix + path-prefix: "" + + # sorts results by: filepath, line and column + sort-results: false + + +# all available settings of specific linters +linters-settings: + + cyclop: + # the maximal code complexity to report + max-complexity: 10 + # the maximal average package complexity. If it's higher than 0.0 (float) the check is enabled (default 0.0) + package-average: 0.0 + # should ignore tests (default false) + skip-tests: false + + dogsled: + # checks assignments with too many blank identifiers; default is 2 + max-blank-identifiers: 2 + + dupl: + # tokens count to trigger issue, 150 by default + threshold: 100 + + errcheck: + # report about not checking of errors in type assertions: `a := b.(MyStruct)`; + # default is false: such cases aren't reported by default. + check-type-assertions: true + + # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`; + # default is false: such cases aren't reported by default. + check-blank: false + + # [deprecated] comma-separated list of pairs of the form pkg:regex + # the regex is used to ignore names within pkg. (default "fmt:.*"). + # see https://github.com/kisielk/errcheck#the-deprecated-method for details + ignore: fmt:.*,io/ioutil:^Read.* + + # path to a file containing a list of functions to exclude from checking + # see https://github.com/kisielk/errcheck#excluding-functions for details + exclude: /path/to/file.txt + + errorlint: + # Check whether fmt.Errorf uses the %w verb for formatting errors. See the readme for caveats + errorf: false + # Check for plain type assertions and type switches + asserts: false + # Check for plain error comparisons + comparison: false + + exhaustive: + # check switch statements in generated files also + check-generated: false + # indicates that switch statements are to be considered exhaustive if a + # 'default' case is present, even if all enum members aren't listed in the + # switch + default-signifies-exhaustive: false + + exhaustivestruct: + # Struct Patterns is list of expressions to match struct packages and names + # The struct packages have the form example.com/package.ExampleStruct + # The matching patterns can use matching syntax from https://pkg.go.dev/path#Match + # If this list is empty, all structs are tested. + struct-patterns: + - '*.Test' + - 'example.com/package.ExampleStruct' + + forbidigo: + # Forbid the following identifiers (identifiers are written using regexp): + forbid: + - ^print.*$ + - 'fmt\.Print.*' + # Exclude godoc examples from forbidigo checks. Default is true. + exclude_godoc_examples: false + + funlen: + lines: 60 + statements: 40 + + gci: + # put imports beginning with prefix after 3rd-party packages; + # only support one prefix + # if not set, use goimports.local-prefixes + local-prefixes: github.com/org/project + + gocognit: + # minimal code complexity to report, 30 by default (but we recommend 10-20) + min-complexity: 10 + + nestif: + # minimal complexity of if statements to report, 5 by default + min-complexity: 4 + + goconst: + # minimal length of string constant, 3 by default + min-len: 3 + # minimal occurrences count to trigger, 3 by default + min-occurrences: 3 + + gocritic: + # Which checks should be enabled; can't be combined with 'disabled-checks'; + # See https://go-critic.github.io/overview#checks-overview + # To check which checks are enabled run `GL_DEBUG=gocritic golangci-lint run` + # By default list of stable checks is used. + enabled-checks: + - rangeValCopy + + # Which checks should be disabled; can't be combined with 'enabled-checks'; default is empty + disabled-checks: + - regexpMust + + # Enable multiple checks by tags, run `GL_DEBUG=gocritic golangci-lint run` to see all tags and checks. + # Empty list by default. See https://github.com/go-critic/go-critic#usage -> section "Tags". + enabled-tags: + - performance + disabled-tags: + - experimental + + # Settings passed to gocritic. + # The settings key is the name of a supported gocritic checker. + # The list of supported checkers can be find in https://go-critic.github.io/overview. + settings: + captLocal: # must be valid enabled check name + # whether to restrict checker to params only (default true) + paramsOnly: true + elseif: + # whether to skip balanced if-else pairs (default true) + skipBalanced: true + hugeParam: + # size in bytes that makes the warning trigger (default 80) + sizeThreshold: 80 + nestingReduce: + # min number of statements inside a branch to trigger a warning (default 5) + bodyWidth: 5 + rangeExprCopy: + # size in bytes that makes the warning trigger (default 512) + sizeThreshold: 512 + # whether to check test functions (default true) + skipTestFuncs: true + rangeValCopy: + # size in bytes that makes the warning trigger (default 128) + sizeThreshold: 32 + # whether to check test functions (default true) + skipTestFuncs: true + ruleguard: + # path to a gorules file for the ruleguard checker + rules: '' + truncateCmp: + # whether to skip int/uint/uintptr types (default true) + skipArchDependent: true + underef: + # whether to skip (*x).method() calls where x is a pointer receiver (default true) + skipRecvDeref: true + unnamedResult: + # whether to check exported functions + checkExported: true + + gocyclo: + # minimal code complexity to report, 30 by default (but we recommend 10-20) + min-complexity: 10 + + godot: + # comments to be checked: `declarations`, `toplevel`, or `all` + scope: declarations + - func.* + - var.* + # list of regexps for excluding particular comment lines from check + exclude: + # example: exclude comments which contain numbers + # - '[0-9]+' + # check that each sentence starts with a capital letter + capital: false + + godox: + # report any comments starting with keywords, this is useful for TODO or FIXME comments that + # might be left in the code accidentally and should be resolved before merging + keywords: # default keywords are TODO, BUG, and FIXME, these can be overwritten by this setting + - NOTE + - OPTIMIZE # marks code that should be optimized before merging + - HACK # marks hack-arounds that should be removed before merging + + gofmt: + # simplify code: gofmt with `-s` option, true by default + simplify: true + + gofumpt: + # Select the Go version to target. The default is `1.15`. + lang-version: "1.15" + + # Choose whether or not to use the extra rules that are disabled + # by default + extra-rules: false + + goheader: + values: + const: + # define here const type values in format k:v, for example: + # COMPANY: MY COMPANY + regexp: + # define here regexp type values, for example + # AUTHOR: .*@mycompany\.com + template: # |- + # put here copyright header template for source code files, for example: + # Note: {{ YEAR }} is a builtin value that returns the year relative to the current machine time. + # + # {{ AUTHOR }} {{ COMPANY }} {{ YEAR }} + # SPDX-License-Identifier: Apache-2.0 + + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at: + + # http://www.apache.org/licenses/LICENSE-2.0 + + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + template-path: + # also as alternative of directive 'template' you may put the path to file with the template source + + goimports: + # put imports beginning with prefix after 3rd-party packages; + # it's a comma-separated list of prefixes + local-prefixes: github.com/org/project + + golint: + # minimal confidence for issues, default is 0.8 + min-confidence: 0.9 + + gomnd: + settings: + mnd: + # the list of enabled checks, see https://github.com/tommy-muehle/go-mnd/#checks for description. + checks: argument,case,condition,operation,return,assign + # ignored-numbers: 1000 + # ignored-files: magic_.*.go + # ignored-functions: math.* + + gomoddirectives: + # Allow local `replace` directives. Default is false. + replace-local: false + # List of allowed `replace` directives. Default is empty. + replace-allow-list: + - launchpad.net/gocheck + # Allow to not explain why the version has been retracted in the `retract` directives. Default is false. + retract-allow-no-explanation: false + # Forbid the use of the `exclude` directives. Default is false. + exclude-forbidden: false + + gomodguard: + allowed: + modules: # List of allowed modules + # - gopkg.in/yaml.v2 + domains: # List of allowed module domains + # - golang.org + blocked: + modules: # List of blocked modules + # - github.com/uudashr/go-module: # Blocked module + # recommendations: # Recommended modules that should be used instead (Optional) + # - golang.org/x/mod + # reason: "`mod` is the official go.mod parser library." # Reason why the recommended module should be used (Optional) + versions: # List of blocked module version constraints + # - github.com/mitchellh/go-homedir: # Blocked module with version constraint + # version: "< 1.1.0" # Version constraint, see https://github.com/Masterminds/semver#basic-comparisons + # reason: "testing if blocked version constraint works." # Reason why the version constraint exists. (Optional) + local_replace_directives: false # Set to true to raise lint issues for packages that are loaded from a local path via replace directive + + gosec: + # To select a subset of rules to run. + # Available rules: https://github.com/securego/gosec#available-rules + includes: + - G401 + - G306 + - G101 + # To specify a set of rules to explicitly exclude. + # Available rules: https://github.com/securego/gosec#available-rules + excludes: + - G204 + # To specify the configuration of rules. + # The configuration of rules is not fully documented by gosec: + # https://github.com/securego/gosec#configuration + # https://github.com/securego/gosec/blob/569328eade2ccbad4ce2d0f21ee158ab5356a5cf/rules/rulelist.go#L60-L102 + config: + G306: "0600" + G101: + pattern: "(?i)example" + ignore_entropy: false + entropy_threshold: "80.0" + per_char_threshold: "3.0" + truncate: "32" + + gosimple: + # Select the Go version to target. The default is '1.13'. + go: "1.15" + # https://staticcheck.io/docs/options#checks + checks: [ "none" ] + + govet: + # report about shadowed variables + check-shadowing: true + + # settings per analyzer + settings: + printf: # analyzer name, run `go tool vet help` to see all analyzers + funcs: # run `go tool vet help printf` to see available settings for `printf` analyzer + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf + + # enable or disable analyzers by name + # run `go tool vet help` to see all analyzers + enable: + - atomicalign + enable-all: false + disable: + - shadow + disable-all: false + + depguard: + list-type: blacklist + include-go-root: false + packages: + - github.com/sirupsen/logrus + packages-with-error-message: + # specify an error message to output when a blacklisted package is used + - github.com/sirupsen/logrus: "logging is allowed only by logutils.Log" + + ifshort: + # Maximum length of variable declaration measured in number of lines, after which linter won't suggest using short syntax. + # Has higher priority than max-decl-chars. + max-decl-lines: 1 + # Maximum length of variable declaration measured in number of characters, after which linter won't suggest using short syntax. + max-decl-chars: 30 + + importas: + # if set to `true`, force to use alias. + no-unaliased: true + # List of aliases + alias: + # using `servingv1` alias for `knative.dev/serving/pkg/apis/serving/v1` package + - pkg: knative.dev/serving/pkg/apis/serving/v1 + alias: servingv1 + # using `autoscalingv1alpha1` alias for `knative.dev/serving/pkg/apis/autoscaling/v1alpha1` package + - pkg: knative.dev/serving/pkg/apis/autoscaling/v1alpha1 + alias: autoscalingv1alpha1 + # You can specify the package path by regular expression, + # and alias by regular expression expansion syntax like below. + # see https://github.com/julz/importas#use-regular-expression for details + - pkg: knative.dev/serving/pkg/apis/(\w+)/(v[\w\d]+) + alias: $1$2 + + lll: + # max line length, lines longer will be reported. Default is 120. + # '\t' is counted as 1 character by default, and can be changed with the tab-width option + line-length: 120 + # tab width in spaces. Default to 1. + tab-width: 1 + + makezero: + # Allow only slices initialized with a length of zero. Default is false. + always: false + + maligned: + # print struct with more effective memory layout or not, false by default + suggest-new: true + + misspell: + # Correct spellings using locale preferences for US or UK. + # Default is to use a neutral variety of English. + # Setting locale to US will correct the British spelling of 'colour' to 'color'. + locale: US + ignore-words: + - someword + + nakedret: + # make an issue if func has more lines of code than this setting and it has naked returns; default is 30 + max-func-lines: 30 + + prealloc: + # XXX: we don't recommend using this linter before doing performance profiling. + # For most programs usage of prealloc will be a premature optimization. + + # Report preallocation suggestions only on simple loops that have no returns/breaks/continues/gotos in them. + # True by default. + simple: true + range-loops: true # Report preallocation suggestions on range loops, true by default + for-loops: false # Report preallocation suggestions on for loops, false by default + + promlinter: + # Promlinter cannot infer all metrics name in static analysis. + # Enable strict mode will also include the errors caused by failing to parse the args. + strict: false + # Please refer to https://github.com/yeya24/promlinter#usage for detailed usage. + disabled-linters: + # - "Help" + # - "MetricUnits" + # - "Counter" + # - "HistogramSummaryReserved" + # - "MetricTypeInName" + # - "ReservedChars" + # - "CamelCase" + # - "lintUnitAbbreviations" + + predeclared: + # comma-separated list of predeclared identifiers to not report on + ignore: "" + # include method names and field names (i.e., qualified names) in checks + q: false + + nolintlint: + # Enable to ensure that nolint directives are all used. Default is true. + allow-unused: false + # Disable to ensure that nolint directives don't have a leading space. Default is true. + allow-leading-space: true + # Exclude following linters from requiring an explanation. Default is []. + allow-no-explanation: [] + # Enable to require an explanation of nonzero length after each nolint directive. Default is false. + require-explanation: true + # Enable to require nolint directives to mention the specific linter being suppressed. Default is false. + require-specific: true + + rowserrcheck: + packages: + - github.com/jmoiron/sqlx + + revive: + # see https://github.com/mgechev/revive#available-rules for details. + ignore-generated-header: true + severity: warning + rules: + - name: indent-error-flow + severity: warning + - name: add-constant + severity: warning + arguments: + - maxLitCount: "3" + allowStrs: '""' + allowInts: "0,1,2" + allowFloats: "0.0,0.,1.0,1.,2.0,2." + + staticcheck: + # Select the Go version to target. The default is '1.13'. + go: "1.16" + # https://staticcheck.io/docs/options#checks + checks: [ "all" ] + + stylecheck: + # Select the Go version to target. The default is '1.13'. + go: "1.16" + # https://staticcheck.io/docs/options#checks + checks: [ "all", "-ST1000", "-ST1003", "-ST1016", "-ST1020", "-ST1021", "-ST1022" ] + # https://staticcheck.io/docs/options#dot_import_whitelist + dot-import-whitelist: + - fmt + # https://staticcheck.io/docs/options#initialisms + initialisms: [ "ACL", "API", "ASCII", "CPU", "CSS", "DNS", "EOF", "GUID", "HTML", "HTTP", "HTTPS", "ID", "IP", "JSON", "QPS", "RAM", "RPC", "SLA", "SMTP", "SQL", "SSH", "TCP", "TLS", "TTL", "UDP", "UI", "GID", "UID", "UUID", "URI", "URL", "UTF8", "VM", "XML", "XMPP", "XSRF", "XSS" ] + # https://staticcheck.io/docs/options#http_status_code_whitelist + http-status-code-whitelist: [ ] + + tagliatelle: + # check the struck tag name case + case: + # use the struct field name to check the name of the struct tag + use-field-name: true + rules: + # any struct tag type can be used. + # support string case: `camel`, `pascal`, `kebab`, `snake`, `goCamel`, `goPascal`, `goKebab`, `goSnake`, `upper`, `lower` + json: camel + yaml: camel + xml: camel + bson: camel + avro: snake + mapstructure: kebab + + testpackage: + # regexp pattern to skip files + skip-regexp: (export|internal)_test\.go + + thelper: + # The following configurations enable all checks. It can be omitted because all checks are enabled by default. + # You can enable only required checks deleting unnecessary checks. + test: + first: true + name: true + begin: true + benchmark: + first: true + name: true + begin: true + tb: + first: true + name: true + begin: true + + unparam: + # Inspect exported functions, default is false. Set to true if no external program/library imports your code. + # XXX: if you enable this setting, unparam will report a lot of false-positives in text editors: + # if it's called for subdir of a project it can't find external interfaces. All text editor integrations + # with golangci-lint call it on a directory with the changed file. + check-exported: false + + unused: + # Select the Go version to target. The default is '1.13'. + go: "1.15" + + whitespace: + multi-if: false # Enforces newlines (or comments) after every multi-line if statement + multi-func: false # Enforces newlines (or comments) after every multi-line function signature + + wrapcheck: + # An array of strings that specify substrings of signatures to ignore. + # If this set, it will override the default set of ignored signatures. + # See https://github.com/tomarrell/wrapcheck#configuration for more information. + ignoreSigs: + - .Errorf( + - errors.New( + - errors.Unwrap( + - .Wrap( + - .Wrapf( + - .WithMessage( + + wsl: + # See https://github.com/bombsimon/wsl/blob/master/doc/configuration.md for + # documentation of available settings. These are the defaults for + # `golangci-lint`. + allow-assign-and-anything: false + allow-assign-and-call: true + allow-cuddle-declarations: false + allow-multiline-assign: true + allow-separated-leading-comment: false + allow-trailing-comment: false + force-case-trailing-whitespace: 0 + force-err-cuddling: false + force-short-decl-cuddling: false + strict-append: true + + # The custom section can be used to define linter plugins to be loaded at runtime. + # See README doc for more info. + custom: + # Each custom linter should have a unique name. + example: + # The path to the plugin *.so. Can be absolute or local. Required for each custom linter + path: /path/to/example.so + # The description of the linter. Optional, just for documentation purposes. + description: This is an example usage of a plugin linter. + # Intended to point to the repo location of the linter. Optional, just for documentation purposes. + original-url: github.com/golangci/example-linter + + +issues: + # List of regexps of issue texts to exclude, empty list by default. + # But independently from this option we use default exclude patterns, + # it can be disabled by `exclude-use-default: false`. To list all + # excluded by default patterns execute `golangci-lint run --help` + exclude: + - abcdef + + # Excluding configuration per-path, per-linter, per-text and per-source + exclude-rules: + # Exclude some linters from running on tests files. + - path: _test\.go + linters: + - gocyclo + - errcheck + - dupl + - gosec + + # Exclude known linters from partially hard-vendored code, + # which is impossible to exclude via "nolint" comments. + - path: internal/hmac/ + text: "weak cryptographic primitive" + linters: + - gosec + + # Exclude some staticcheck messages + - linters: + - staticcheck + text: "SA9003:" + + # Exclude lll issues for long lines with go:generate + - linters: + - lll + source: "^//go:generate " + + # Independently from option `exclude` we use default exclude patterns, + # it can be disabled by this option. To list all + # excluded by default patterns execute `golangci-lint run --help`. + # Default value for this option is true. + exclude-use-default: false + + # The default value is false. If set to true exclude and exclude-rules + # regular expressions become case sensitive. + exclude-case-sensitive: false + + # The list of ids of default excludes to include or disable. By default it's empty. + include: + - EXC0002 # disable excluding of issues about comments from golint + + # Maximum issues count per one linter. Set to 0 to disable. Default is 50. + max-issues-per-linter: 0 + + # Maximum count of issues with the same text. Set to 0 to disable. Default is 3. + max-same-issues: 0 + + # Show only new issues: if there are unstaged changes or untracked files, + # only those changes are analyzed, else only changes in HEAD~ are analyzed. + # It's a super-useful option for integration of golangci-lint into existing + # large codebase. It's not practical to fix all existing issues at the moment + # of integration: much better don't allow issues in new code. + # Default is false. + new: false + + # Show only new issues created after git revision `REV` + new-from-rev: REV + + # Show only new issues created in git patch with set file path. + new-from-patch: path/to/patch/file + + # Fix found issues (if it's supported by the linter) + fix: true + +severity: + # Default value is empty string. + # Set the default severity for issues. If severity rules are defined and the issues + # do not match or no severity is provided to the rule this will be the default + # severity applied. Severities should match the supported severity names of the + # selected out format. + # - Code climate: https://docs.codeclimate.com/docs/issues#issue-severity + # - Checkstyle: https://checkstyle.sourceforge.io/property_types.html#severity + # - Github: https://help.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-error-message + default-severity: error + + # The default value is false. + # If set to true severity-rules regular expressions become case sensitive. + case-sensitive: false + + # Default value is empty list. + # When a list of severity rules are provided, severity information will be added to lint + # issues. Severity rules have the same filtering capability as exclude rules except you + # are allowed to specify one matcher per severity rule. + # Only affects out formats that support setting severity information. + rules: + - linters: + - dupl + severity: info diff --git a/main.go b/main.go index 79b175d..e88d02c 100644 --- a/main.go +++ b/main.go @@ -1,19 +1,18 @@ package main import ( - "fmt" - "net/http" - "time" + "context" + "os" + "os/signal" + "sync" + "syscall" - "contrib.go.opencensus.io/exporter/prometheus" "github.com/NYTimes/gizmo/server" "github.com/NYTimes/video-captions-api/config" "github.com/NYTimes/video-captions-api/database" "github.com/NYTimes/video-captions-api/providers" "github.com/NYTimes/video-captions-api/service" "github.com/kelseyhightower/envconfig" - "github.com/pkg/errors" - goprom "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" ) @@ -59,43 +58,33 @@ func main() { db = database.NewMemoryDatabase() } - // metrics server - exporter, registry := mustInitMetrics() - go func(log *logrus.Entry) { - addr := ":9000" - log.WithField("address", addr).Info("starting metric server") + interrupt := make(chan os.Signal, 1) - mux := http.NewServeMux() - mux.Handle("/metrics", exporter) + signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM) + defer signal.Stop(interrupt) - metricsServer := &http.Server{ - Addr: addr, - ReadTimeout: 2 * time.Second, - WriteTimeout: 2 * time.Second, - Handler: mux, - } - var err error - if err = metricsServer.ListenAndServe(); err != http.ErrServerClosed { - log.WithFields(logrus.Fields{ - "err": err, - "address": addr, - }).Fatal("Metrics server failure") - } + var wg sync.WaitGroup + implementedProviders := makeProviders(&cfg, db) - }(server.Log.WithField("service", "metricsServer")) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() - // caption server + exporter, registry := MustInitMetrics() + StartMetricsServer(ctx, &wg, exporter, server.Log) - threeplayConfig := providers.Load3PlayConfigFromEnv() - amaraConfig := providers.LoadAmaraConfigFromEnv() - captionsService := service.NewCaptionsService(&cfg, db, registry) + callbacks := StartCallbackListener(ctx, &wg, implementedProviders, server.Log) - captionsService.AddProvider(providers.New3PlayProvider(&threeplayConfig, &cfg)) - captionsService.AddProvider(providers.NewAmaraProvider(&amaraConfig, &cfg)) - captionsService.AddProvider(providers.NewUploadProvider(&cfg, db)) + // caption server + captionsService := service.NewCaptionsService(&cfg, db, callbacks, registry) + + for _, p := range implementedProviders { + captionsService.AddProvider(p) + } server.Init("video-captions-api", cfg.Server) + //server.WithCloseHandler TODO need to implement gizmo server.Context.Handler to gracefully shut down + err = server.Register(captionsService) if err != nil { server.Log.Fatal("Unable to register service: ", err) @@ -105,49 +94,19 @@ func main() { if err != nil { server.Log.Fatal("Server encountered a fatal error: ", err) } -} -func mustInitMetrics() (*prometheus.Exporter, *goprom.Registry) { - pe, r, err := initMetrics() - if err != nil { - panic(errors.Wrap(err, "Failed to initialize metrics service")) - } - return pe, r + wg.Wait() + } -func initMetrics() (*prometheus.Exporter, *goprom.Registry, error) { - r := goprom.NewRegistry() - r.MustRegister(goprom.NewProcessCollector(goprom.ProcessCollectorOpts{})) - r.MustRegister(goprom.NewGoCollector()) - - versionCollector := goprom.NewGaugeVec(goprom.GaugeOpts{ - Namespace: MetricsNamespace, - Name: "version", - Help: "Application version.", - }, []string{"version"}) - - r.MustRegister(versionCollector) - versionCollector.WithLabelValues(version).Add(1) - - captionTimer := goprom.NewHistogramVec(goprom.HistogramOpts{ - Namespace: MetricsNamespace, - Name: "asr_execution_time_seconds", - Help: "provider caption time", - Buckets: goprom.LinearBuckets(20, 5, 5), - }, []string{ - "provider", - }) - - r.MustRegister(captionTimer) - - // Stats exporter: Prometheus - pe, err := prometheus.NewExporter(prometheus.Options{ - Namespace: MetricsNamespace, - Registry: r, - }) - if err != nil { - return nil, nil, fmt.Errorf("failed to create the Prometheus stats exporter %w", err) - } +func makeProviders(cfg *config.CaptionsServiceConfig, db database.DB) []providers.Provider { + var p []providers.Provider + threeplayConfig := providers.Load3PlayConfigFromEnv() + amaraConfig := providers.LoadAmaraConfigFromEnv() + providers.New3PlayProvider(&threeplayConfig, cfg) + providers.NewAmaraProvider(&amaraConfig, cfg) + providers.NewUploadProvider(cfg, db) + + return p - return pe, r, nil } diff --git a/providers/amara.go b/providers/amara.go index 0382345..2cb5fe6 100644 --- a/providers/amara.go +++ b/providers/amara.go @@ -2,14 +2,16 @@ package providers import ( "fmt" + "net/http" "net/url" "strconv" "github.com/NYTimes/video-captions-api/config" "github.com/NYTimes/video-captions-api/database" "github.com/kelseyhightower/envconfig" - "github.com/nytimes/amara" log "github.com/sirupsen/logrus" + + "github.com/nytimes/amara" ) // AmaraProvider amara client wrapper that implements the Provider interface @@ -130,3 +132,7 @@ func (c *AmaraProvider) DispatchJob(job *database.Job) error { func (c *AmaraProvider) CancelJob(job *database.Job) (bool, error) { return false, nil } + +func (c *AmaraProvider) HandleCallback(_ *http.Request) (*CallbackData, error) { + return &CallbackData{}, nil +} diff --git a/providers/provider.go b/providers/provider.go index 997b6d6..f80731c 100644 --- a/providers/provider.go +++ b/providers/provider.go @@ -1,6 +1,36 @@ package providers -import "github.com/NYTimes/video-captions-api/database" +import ( + "net/http" + + "github.com/NYTimes/video-captions-api/database" +) + +type Callback struct { + Code int + Data CallbackData +} + +type DataWrapper struct { + JobID string + Data CallbackData +} + +type CallbackData struct { + Cancellable bool `json:"cancellable"` + Default bool `json:"default"` + ID int `json:"id"` + BatchID int `json:"batch_id"` + LanguageID int `json:"language_id"` + MediaFileID int `json:"media_file_id"` + LanguageIDs []int `json:"language_ids"` + CancellationDetails string `json:"cancellation_details"` + CancellationReason string `json:"cancellation_reason"` + ReferenceID string `json:"reference_id"` + Status string `json:"status"` + Type string `json:"type"` + Duration float64 `json:"duration"` +} // Provider is the interface that transcription/captions providers must implement type Provider interface { @@ -8,5 +38,6 @@ type Provider interface { Download(*database.Job, string) ([]byte, error) GetProviderJob(*database.Job) (*database.ProviderJob, error) GetName() string + HandleCallback(req *http.Request) (*CallbackData, error) CancelJob(*database.Job) (bool, error) } diff --git a/providers/threeplay.go b/providers/threeplay.go index 4524925..d8585b3 100644 --- a/providers/threeplay.go +++ b/providers/threeplay.go @@ -2,15 +2,17 @@ package providers import ( "errors" + "net/http" "net/url" "strconv" "github.com/NYTimes/video-captions-api/config" "github.com/NYTimes/video-captions-api/database" "github.com/kelseyhightower/envconfig" + log "github.com/sirupsen/logrus" + "github.com/nytimes/threeplay/types" threeplay "github.com/nytimes/threeplay/v3api" - log "github.com/sirupsen/logrus" ) const providerName string = "3play" @@ -156,3 +158,8 @@ func (c *ThreePlayProvider) CancelJob(job *database.Job) (bool, error) { } return providerJob.Cancellable, errors.New("job is not cancellable") } + +func (c *ThreePlayProvider) HandleCallback(req *http.Request) (*CallbackData, error) { + + panic("not implemented") // TODO: Implement +} diff --git a/providers/trint.go b/providers/trint.go index 6ee4b4b..9a0bc4d 100644 --- a/providers/trint.go +++ b/providers/trint.go @@ -1,6 +1,10 @@ package providers -import "github.com/NYTimes/video-captions-api/database" +import ( + "net/http" + + "github.com/NYTimes/video-captions-api/database" +) type trint struct { } @@ -24,3 +28,6 @@ func (t *trint) GetName() string { func (t *trint) CancelJob(_ *database.Job) (bool, error) { panic("not implemented") // TODO: Implement } +func (t *trint) HandleCallback(req *http.Request) (*CallbackData, error) { + panic("not implemented") // TODO: Implement +} diff --git a/providers/upload.go b/providers/upload.go index 379a372..6bad9d6 100644 --- a/providers/upload.go +++ b/providers/upload.go @@ -3,6 +3,7 @@ package providers import ( "bytes" "fmt" + "net/http" "path/filepath" captionsConfig "github.com/NYTimes/video-captions-api/config" @@ -88,3 +89,6 @@ func (c *UploadProvider) validateCaptionFile(file *database.UploadedFile) error return nil } +func (c *UploadProvider) HandleCallback(req *http.Request) (*CallbackData, error) { + panic("not implemented") // TODO: Implement +} diff --git a/service/client.go b/service/client.go index 4852c53..7395439 100644 --- a/service/client.go +++ b/service/client.go @@ -260,7 +260,7 @@ func (c Client) GenerateTranscript(captionFile []byte, captionFormat string) (st return "", fmt.Errorf("unable to generate a transcript for caption format: %v", captionFormat) } -func (c Client) ProcessCallback(callbackData CallbackData, jobID string) error { +func (c Client) ProcessCallback(callbackData providers.CallbackData, jobID string) error { jobLogger := c.Logger.WithFields(log.Fields{ "JobID": jobID, "ProviderID": callbackData.ID, diff --git a/service/client_test.go b/service/client_test.go index 2881244..0cc46a0 100644 --- a/service/client_test.go +++ b/service/client_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/NYTimes/video-captions-api/database" + "github.com/NYTimes/video-captions-api/providers" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" ) @@ -15,7 +16,7 @@ import ( func TestGetJob(t *testing.T) { service, client := createCaptionsService("") assert := assert.New(t) - service.AddProvider(fakeProvider{logger: log.New()}) + service.AddProvider(&fakeProvider{logger: log.New()}) job := &database.Job{ ID: "123", MediaURL: "http://vp.nyt.com/video.mp4", @@ -36,7 +37,7 @@ func TestDispatchJobNoProvider(t *testing.T) { func TestGetJobReady(t *testing.T) { assert := assert.New(t) service, client := createCaptionsService("") - service.AddProvider(fakeProvider{ + service.AddProvider(&fakeProvider{ logger: log.New(), params: map[string]bool{ "jobError": false, @@ -64,7 +65,7 @@ func TestGetJobs(t *testing.T) { service, client := createCaptionsService("") - service.AddProvider(fakeProvider{logger: log.New()}) + service.AddProvider(&fakeProvider{logger: log.New()}) today := time.Now() yesterday := today.AddDate(0, 0, -1) job1 := &database.Job{ @@ -96,7 +97,7 @@ func TestGetJobs(t *testing.T) { func TestProviderJobError(t *testing.T) { service, client := createCaptionsService("") - service.AddProvider(fakeProvider{ + service.AddProvider(&fakeProvider{ logger: log.New(), params: map[string]bool{ "jobError": true, @@ -118,7 +119,7 @@ func TestProviderJobError(t *testing.T) { func TestProviderStatusError(t *testing.T) { service, client := createCaptionsService("") - service.AddProvider(fakeProvider{ + service.AddProvider(&fakeProvider{ logger: log.New(), params: map[string]bool{ "jobError": false, @@ -140,7 +141,7 @@ func TestProviderStatusError(t *testing.T) { func TestCancelClientJob(t *testing.T) { service, client := createCaptionsService("") assert := assert.New(t) - service.AddProvider(fakeProvider{logger: log.New()}) + service.AddProvider(&fakeProvider{logger: log.New()}) job := &database.Job{ ID: "123", MediaURL: "http://vp.nyt.com/video.mp4", @@ -157,7 +158,7 @@ func TestCancelClientJob(t *testing.T) { func TestCancelClientJobDone(t *testing.T) { service, client := createCaptionsService("") assert := assert.New(t) - service.AddProvider(fakeProvider{logger: log.New()}) + service.AddProvider(&fakeProvider{logger: log.New()}) job := &database.Job{ ID: "123", MediaURL: "http://vp.nyt.com/video.mp4", @@ -175,7 +176,7 @@ func TestCancelClientJobDone(t *testing.T) { func TestCancelClientJob404(t *testing.T) { service, client := createCaptionsService("") assert := assert.New(t) - service.AddProvider(fakeProvider{logger: log.New()}) + service.AddProvider(&fakeProvider{logger: log.New()}) canceled, err := client.CancelJob("404") assert.NotNil(err) @@ -186,7 +187,7 @@ func TestCancelClientJob404(t *testing.T) { func TestDownloadCaption(t *testing.T) { service, client := createCaptionsService("") assert := assert.New(t) - service.AddProvider(fakeProvider{logger: log.New()}) + service.AddProvider(&fakeProvider{logger: log.New()}) job := &database.Job{ ID: "123", MediaURL: "http://vp.nyt.com/video.mp4", @@ -201,7 +202,7 @@ func TestDownloadCaption(t *testing.T) { func TestDownloadNonexistentCaption(t *testing.T) { service, client := createCaptionsService("") assert := assert.New(t) - service.AddProvider(fakeProvider{logger: log.New()}) + service.AddProvider(&fakeProvider{logger: log.New()}) job := &database.Job{ ID: "123", MediaURL: "http://vp.nyt.com/video.mp4", @@ -231,7 +232,7 @@ func TestDownloadCaptionProviderError(t *testing.T) { func TestGenerateTranscriptSsa(t *testing.T) { service, client := createCaptionsService("") assert := assert.New(t) - service.AddProvider(fakeProvider{logger: log.New()}) + service.AddProvider(&fakeProvider{logger: log.New()}) job := &database.Job{ ID: "123", MediaURL: "http://vp.nyt.com/video.mp4", @@ -248,7 +249,7 @@ func TestGenerateTranscriptSsa(t *testing.T) { func TestGenerateTranscriptVtt(t *testing.T) { service, client := createCaptionsService("") assert := assert.New(t) - service.AddProvider(fakeProvider{logger: log.New()}) + service.AddProvider(&fakeProvider{logger: log.New()}) job := &database.Job{ ID: "123", MediaURL: "http://vp.nyt.com/video.mp4", @@ -265,7 +266,7 @@ func TestGenerateTranscriptVtt(t *testing.T) { func TestGenerateTranscriptSrt(t *testing.T) { service, client := createCaptionsService("") assert := assert.New(t) - service.AddProvider(fakeProvider{logger: log.New()}) + service.AddProvider(&fakeProvider{logger: log.New()}) job := &database.Job{ ID: "123", MediaURL: "http://vp.nyt.com/video.mp4", @@ -282,7 +283,7 @@ func TestGenerateTranscriptSrt(t *testing.T) { func TestGenerateTranscriptSbv(t *testing.T) { service, client := createCaptionsService("") assert := assert.New(t) - service.AddProvider(fakeProvider{logger: log.New()}) + service.AddProvider(&fakeProvider{logger: log.New()}) job := &database.Job{ ID: "123", MediaURL: "http://vp.nyt.com/video.mp4", @@ -299,7 +300,7 @@ func TestGenerateTranscriptSbv(t *testing.T) { func TestGenerateTranscriptWrongFormat(t *testing.T) { service, client := createCaptionsService("") assert := assert.New(t) - service.AddProvider(fakeProvider{logger: log.New()}) + service.AddProvider(&fakeProvider{logger: log.New()}) job := &database.Job{ ID: "123", MediaURL: "http://vp.nyt.com/video.mp4", @@ -378,7 +379,7 @@ func TestProcessCallbackClient(t *testing.T) { service, client := createCaptionsService(callbackURL) assert := assert.New(t) - service.AddProvider(fakeProvider{logger: log.New()}) + service.AddProvider(&fakeProvider{logger: log.New()}) job := &database.Job{ ID: "123", MediaURL: "http://vp.nyt.com/video.mp4", @@ -386,7 +387,7 @@ func TestProcessCallbackClient(t *testing.T) { ProviderParams: map[string]string{"ProviderID": "11214314"}, } client.DB.StoreJob(job) - callbackData := CallbackData{ + callbackData := providers.CallbackData{ ID: test.providerID, MediaFileID: 3765758, BatchID: 68841, diff --git a/service/job.go b/service/job.go index 70b0981..6d4c16a 100644 --- a/service/job.go +++ b/service/job.go @@ -36,27 +36,6 @@ type uploadedFile struct { Name string `json:"name"` } -type Callback struct { - Code int `json:"code"` - Data CallbackData `json:"data"` -} - -type CallbackData struct { - Cancellable bool `json:"cancellable"` - Default bool `json:"default"` - ID int `json:"id"` - BatchID int `json:"batch_id"` - LanguageID int `json:"language_id"` - MediaFileID int `json:"media_file_id"` - LanguageIDs []int `json:"language_ids"` - CancellationDetails string `json:"cancellation_details"` - CancellationReason string `json:"cancellation_reason"` - ReferenceID string `json:"reference_id"` - Status string `json:"status"` - Type string `json:"type"` - Duration float64 `json:"duration"` -} - func newJobFromParams(newJob jobParams) (*database.Job, error) { outputs := make([]database.JobOutput, 0) var name string @@ -233,38 +212,3 @@ func (s *CaptionsService) GetTranscript(w http.ResponseWriter, r *http.Request) w.WriteHeader(http.StatusOK) w.Write([]byte(transcript)) } - -func (s *CaptionsService) ProcessCallback(r *http.Request) (int, interface{}, error) { - requestLogger := s.logger.WithFields(log.Fields{ - "Handler": "ProcessCallback", - "Method": r.Method, - "URI": r.RequestURI, - }) - - defer r.Body.Close() - - data, err := ioutil.ReadAll(r.Body) - if err != nil { - requestLogger.WithError(err).Error("Could not read request body: ") - return http.StatusBadRequest, nil, captionsError{err.Error()} - } - - callbackObject := Callback{} - err = json.Unmarshal(data, &callbackObject) - if err != nil { - requestLogger.WithError(err).Error("Could not unmarshal callback") - return http.StatusBadRequest, nil, captionsError{"Malformed parameters"} - } - - requestLogger.Infof("Received a callback for ID: %v", callbackObject.Data.ID) - - queryParams := r.URL.Query() - jobID := queryParams.Get("job_id") - - err = s.client.ProcessCallback(callbackObject.Data, jobID) - if err != nil { - requestLogger.Errorf("Could not process callback for ID: %v", callbackObject.Data.ID) - return http.StatusInternalServerError, nil, captionsError{err.Error()} - } - return http.StatusOK, nil, nil -} diff --git a/service/job_test.go b/service/job_test.go index d3c92c2..d5a1f9c 100644 --- a/service/job_test.go +++ b/service/job_test.go @@ -11,6 +11,7 @@ import ( "github.com/NYTimes/gizmo/server" "github.com/NYTimes/video-captions-api/database" + "github.com/NYTimes/video-captions-api/providers" "github.com/stretchr/testify/assert" "io/ioutil" @@ -350,6 +351,10 @@ type processCallbackTest struct { } func TestProcessCallback(t *testing.T) { + + // TODO reenable once impls for the callback exist + t.Skip() + tests := []processCallbackTest{ { name: "Success", @@ -390,9 +395,9 @@ func TestProcessCallback(t *testing.T) { client.DB.StoreJob(job) server.Register(service) - captionCallback := Callback{ + captionCallback := providers.Callback{ Code: 200, - Data: CallbackData{ + Data: providers.CallbackData{ ID: test.callbackID, MediaFileID: 3765758, BatchID: 68841, diff --git a/service/service.go b/service/service.go index b32d1ca..3bea0f3 100644 --- a/service/service.go +++ b/service/service.go @@ -1,6 +1,7 @@ package service import ( + "fmt" "net/http" "github.com/prometheus/client_golang/prometheus" @@ -11,32 +12,72 @@ import ( "github.com/NYTimes/video-captions-api/config" "github.com/NYTimes/video-captions-api/database" "github.com/NYTimes/video-captions-api/providers" + "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" ) // CaptionsService the service responsible to wrapping interactions with Providers type CaptionsService struct { - client Client - logger *log.Logger - metrics *prometheus.Registry + client Client + logger *log.Logger + callbacks chan *providers.DataWrapper + metrics *prometheus.Registry } // NewCaptionsService creates a CaptionsService -func NewCaptionsService(cfg *config.CaptionsServiceConfig, db database.DB, metrics *prom.Registry) *CaptionsService { +func NewCaptionsService( + cfg *config.CaptionsServiceConfig, + db database.DB, + callbacks chan *providers.DataWrapper, + metrics *prom.Registry, +) *CaptionsService { storage, _ := NewGCSStorage(cfg.BucketName, cfg.Logger) + client := Client{ + Providers: make(map[string]providers.Provider), + DB: db, + Logger: cfg.Logger, + Storage: storage, + CallbackURL: cfg.CallbackURL, + } + go func(log *logrus.Entry) { + for wrapper := range callbacks { + data, id := wrapper.Data, wrapper.JobID + err := client.ProcessCallback(data, id) + if err != nil { + log.WithFields(logrus.Fields{ + "err": err, + "callbackdata": fmt.Sprintf("%+v", data), + "jobID": "id", + "provider": data.ID, + }).Error("Callback Failed") + + // TODO retry + + } + + } + }(log.WithField("service", "Callback Listener Worker")) return &CaptionsService{ - Client{ - Providers: make(map[string]providers.Provider), - DB: db, - Logger: cfg.Logger, - Storage: storage, - CallbackURL: cfg.CallbackURL, - }, + client, cfg.Logger, + callbacks, metrics, } } +// +//func (s *CaptionsService) ProcessCallbacks() error { +// +// +// +// err = s.client.ProcessCallback(callbackObject.Data, jobID) +// if err != nil { +// requestLogger.Errorf("Could not process callback for ID: %v", callbackObject.Data.ID) +// return http.StatusInternalServerError, nil, captionsError{err.Error()} +// } +// return http.StatusOK, nil, nil +//} + // AddProvider adds a Provider to the CaptionsService func (s *CaptionsService) AddProvider(provider providers.Provider) { s.client.Providers[provider.GetName()] = provider @@ -73,8 +114,5 @@ func (s *CaptionsService) Endpoints() map[string]map[string]http.HandlerFunc { "/jobs/{id}/transcript/{captionFormat}": { "GET": s.GetTranscript, }, - "/callback": { - "POST": server.JSONToHTTP(s.ProcessCallback).ServeHTTP, - }, } } diff --git a/service/service_test.go b/service/service_test.go index 10c8dc7..0f87d95 100644 --- a/service/service_test.go +++ b/service/service_test.go @@ -3,6 +3,7 @@ package service import ( "errors" "fmt" + "net/http" "testing" "reflect" @@ -17,8 +18,13 @@ import ( ) type fakeProvider struct { - logger *log.Logger - params map[string]bool + logger *log.Logger + params map[string]bool + callbackData *providers.CallbackData +} + +func (p fakeProvider) HandleCallback(req *http.Request) (*providers.CallbackData, error) { + panic("not implemented") // TODO: Implement } func (p fakeProvider) DispatchJob(job *database.Job) error { @@ -69,6 +75,10 @@ func (p fakeProvider) CancelJob(job *database.Job) (bool, error) { type brokenProvider fakeProvider +func (p brokenProvider) HandleCallback(req *http.Request) (*providers.CallbackData, error) { + return &providers.CallbackData{}, nil +} + func (p brokenProvider) GetName() string { return "broken-provider" } @@ -155,5 +165,4 @@ func TestNewCaptionsService(t *testing.T) { assert.Contains(service.Endpoints(), "/jobs/{id}/cancel") assert.Contains(service.Endpoints(), "/jobs/{id}/download/{captionFormat}") assert.Contains(service.Endpoints(), "/jobs/{id}/transcript/{captionFormat}") - assert.Contains(service.Endpoints(), "/callback") } diff --git a/sidekicks.go b/sidekicks.go new file mode 100644 index 0000000..562687f --- /dev/null +++ b/sidekicks.go @@ -0,0 +1,138 @@ +package main + +import ( + "context" + "fmt" + "net/http" + "sync" + "time" + + "contrib.go.opencensus.io/exporter/prometheus" + "github.com/NYTimes/gizmo/server" + "github.com/NYTimes/video-captions-api/providers" + "github.com/pkg/errors" + goprom "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" +) + +func StartMetricsServer( + ctx context.Context, + wg *sync.WaitGroup, + exporter *prometheus.Exporter, + log *logrus.Logger) { + + addr := ":9000" + log.WithField("address", addr).Info("starting metric server") + + mux := http.NewServeMux() + mux.Handle("/metrics", exporter) + + srv := &http.Server{ + Addr: addr, + ReadTimeout: 2 * time.Second, + WriteTimeout: 2 * time.Second, + Handler: mux, + } + wg.Add(1) + + shutdownCtx, cancel := context.WithCancel(ctx) + go func() { + defer wg.Done() + defer cancel() + var err error + if err = srv.Shutdown(shutdownCtx); err != nil { + log.Fatalf("server Shutdown Failed:%+s", err) + } + + }() + + wg.Add(1) + go func(log *logrus.Entry) { + defer wg.Done() + var err error + if err = srv.ListenAndServe(); err != http.ErrServerClosed { + log.WithFields(logrus.Fields{ + "err": err, + "address": addr, + }).Fatal("Metrics server failure") + } + }(log.WithField("service", "metrics_server")) + +} + +func StartCallbackListener( + ctx context.Context, + wg *sync.WaitGroup, + callers []providers.Provider, + log *logrus.Logger) chan *providers.DataWrapper { + wg.Add(1) + // callback server + wg.Add(1) + callbacks := make(chan *providers.DataWrapper) + go func(log *logrus.Entry) { + defer wg.Done() + + mux := http.NewServeMux() + + cbServer := &http.Server{ + Addr: ":9090", + ReadTimeout: 2 * time.Second, + WriteTimeout: 2 * time.Second, + Handler: mux, + } + var err error + if err = cbServer.ListenAndServe(); err != http.ErrServerClosed { + log.WithFields(logrus.Fields{ + "err": err, + }).Fatal("Callback server failure") + } + + }(server.Log.WithField("service", "callbackListener")) + + return callbacks +} + +func MustInitMetrics() (*prometheus.Exporter, *goprom.Registry) { + pe, r, err := initMetrics() + if err != nil { + panic(errors.Wrap(err, "Failed to initialize metrics service")) + } + return pe, r +} + +func initMetrics() (*prometheus.Exporter, *goprom.Registry, error) { + r := goprom.NewRegistry() + r.MustRegister(goprom.NewProcessCollector(goprom.ProcessCollectorOpts{})) + r.MustRegister(goprom.NewGoCollector()) + + versionCollector := goprom.NewGaugeVec(goprom.GaugeOpts{ + Namespace: MetricsNamespace, + Name: "version", + Help: "Application version.", + }, []string{"version"}) + + r.MustRegister(versionCollector) + versionCollector.WithLabelValues(version).Add(1) + + captionTimer := goprom.NewHistogramVec(goprom.HistogramOpts{ + Namespace: MetricsNamespace, + Name: "asr_execution_time_seconds", + Help: "provider caption time", + Buckets: goprom.LinearBuckets(20, 5, 5), + }, []string{ + "provider", + }) + + r.MustRegister(captionTimer) + + // Stats exporter: Prometheus + pe, err := prometheus.NewExporter(prometheus.Options{ + Namespace: MetricsNamespace, + Registry: r, + }) + if err != nil { + return nil, nil, fmt.Errorf("failed to create the Prometheus stats exporter %w", err) + } + + return pe, r, nil +} From 06ea60a3a4255cf8aabae3ba5b081d6809287696 Mon Sep 17 00:00:00 2001 From: Guy J Grigsby Date: Mon, 9 Aug 2021 11:18:31 -0600 Subject: [PATCH 07/17] fix tests for multiple callback listeners --- .gitignore | 2 +- Makefile | 6 ++-- main.go => cmd/main.go | 62 +++++++++++++++++++++++++++++++--- go.mod | 1 + go.sum | 1 + service/service_test.go | 21 ++++++++---- sidekicks.go | 74 ++++++++--------------------------------- 7 files changed, 92 insertions(+), 75 deletions(-) rename main.go => cmd/main.go (60%) diff --git a/.gitignore b/.gitignore index de13f36..d2f35e7 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,4 @@ coverage.txt video-captions-api .DS_Store main -captions-api +captions diff --git a/Makefile b/Makefile index 4219316..d661d3a 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ dev: - @go build -v -o captions-api . + @go build -o captions cmd/main.go @SERVER_HTTP_PORT=8000\ SERVER_GIZMO_HEALTH_CHECK_PATH=/healthz\ @@ -12,7 +12,7 @@ dev: BUCKET_NAME=$(CAPTIONS_BUCKET_NAME) \ CALLBACK_URL=$(CALLBACK_URL) \ CALLBACK_API_KEY=$(CALLBACK_API_KEY) \ - ./captions-api + ./captions install-golangcilint: go get github.com/golangci/golangci-lint/cmd/golangci-lint @@ -28,3 +28,5 @@ coverage: test: go test -race ./... + +.PHONY: test diff --git a/main.go b/cmd/main.go similarity index 60% rename from main.go rename to cmd/main.go index e88d02c..d2e8dc4 100644 --- a/main.go +++ b/cmd/main.go @@ -2,17 +2,25 @@ package main import ( "context" + "fmt" "os" "os/signal" "sync" "syscall" + "golang.org/x/sync/errgroup" + + goprom "github.com/prometheus/client_golang/prometheus" + + "contrib.go.opencensus.io/exporter/prometheus" "github.com/NYTimes/gizmo/server" + videocaptionsapi "github.com/NYTimes/video-captions-api" "github.com/NYTimes/video-captions-api/config" "github.com/NYTimes/video-captions-api/database" "github.com/NYTimes/video-captions-api/providers" "github.com/NYTimes/video-captions-api/service" "github.com/kelseyhightower/envconfig" + "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -63,17 +71,16 @@ func main() { signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM) defer signal.Stop(interrupt) - - var wg sync.WaitGroup implementedProviders := makeProviders(&cfg, db) ctx, cancel := context.WithCancel(context.Background()) defer cancel() + eg, ctx := errgroup.WithContext(ctx) exporter, registry := MustInitMetrics() - StartMetricsServer(ctx, &wg, exporter, server.Log) + videocaptionsapi.StartMetricsServer(ctx, eg, exporter, server.Log) - callbacks := StartCallbackListener(ctx, &wg, implementedProviders, server.Log) + callbacks := videocaptionsapi.StartCallbackListener(ctx, &sync.WaitGroup{}, implementedProviders, server.Log) // caption server captionsService := service.NewCaptionsService(&cfg, db, callbacks, registry) @@ -95,7 +102,7 @@ func main() { server.Log.Fatal("Server encountered a fatal error: ", err) } - wg.Wait() + eg.Wait() } @@ -110,3 +117,48 @@ func makeProviders(cfg *config.CaptionsServiceConfig, db database.DB) []provider return p } + +func MustInitMetrics() (*prometheus.Exporter, *goprom.Registry) { + pe, r, err := initMetrics() + if err != nil { + panic(errors.Wrap(err, "Failed to initialize metrics service")) + } + return pe, r +} + +func initMetrics() (*prometheus.Exporter, *goprom.Registry, error) { + r := goprom.NewRegistry() + r.MustRegister(goprom.NewProcessCollector(goprom.ProcessCollectorOpts{})) + r.MustRegister(goprom.NewGoCollector()) + + versionCollector := goprom.NewGaugeVec(goprom.GaugeOpts{ + Namespace: MetricsNamespace, + Name: "version", + Help: "Application version.", + }, []string{"version"}) + + r.MustRegister(versionCollector) + versionCollector.WithLabelValues(version).Add(1) + + captionTimer := goprom.NewHistogramVec(goprom.HistogramOpts{ + Namespace: MetricsNamespace, + Name: "asr_execution_time_seconds", + Help: "provider caption time", + Buckets: goprom.LinearBuckets(20, 5, 5), + }, []string{ + "provider", + }) + + r.MustRegister(captionTimer) + + // Stats exporter: Prometheus + pe, err := prometheus.NewExporter(prometheus.Options{ + Namespace: MetricsNamespace, + Registry: r, + }) + if err != nil { + return nil, nil, fmt.Errorf("failed to create the Prometheus stats exporter %w", err) + } + + return pe, r, nil +} diff --git a/go.mod b/go.mod index ef3f48b..0760e1b 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/tdewolff/parse/v2 v2.4.3 go.opencensus.io v0.23.0 // indirect golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 // indirect + golang.org/x/sync v0.0.0-20201207232520-09787c993a3a golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect golang.org/x/text v0.3.6 // indirect ) diff --git a/go.sum b/go.sum index e22a2c5..95e8221 100644 --- a/go.sum +++ b/go.sum @@ -377,6 +377,7 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/service/service_test.go b/service/service_test.go index 0f87d95..dfd4f0a 100644 --- a/service/service_test.go +++ b/service/service_test.go @@ -1,14 +1,17 @@ package service import ( + "context" "errors" "fmt" "net/http" + "sync" "testing" "reflect" "github.com/NYTimes/gizmo/server" + captions "github.com/NYTimes/video-captions-api" "github.com/NYTimes/video-captions-api/config" "github.com/NYTimes/video-captions-api/database" "github.com/NYTimes/video-captions-api/providers" @@ -126,23 +129,27 @@ func TestAddProvider(t *testing.T) { service, client := createCaptionsService("") service.AddProvider(fakeProvider{}) - provider := client.Providers["test-provider"] - assert.NotNil(provider) - assert.Equal(provider.GetName(), "test-provider") + p := client.Providers["test-provider"] + assert.NotNil(p) + assert.Equal(p.GetName(), "test-provider") } func TestNewCaptionsService(t *testing.T) { logger := log.New() projectID := "My amazing captions project" - providers := make(map[string]providers.Provider) + p := make(map[string]providers.Provider) cfg := config.CaptionsServiceConfig{ Server: &server.Config{}, Logger: logger, ProjectID: projectID, } db := database.NewMemoryDatabase() - - service := NewCaptionsService(&cfg, db, prometheus.NewRegistry()) + var callers []providers.Provider + for _, p := range p { + callers = append(callers, p) + } + cb := captions.StartCallbackListener(context.Background(), &sync.WaitGroup{}, callers, logger) + service := NewCaptionsService(&cfg, db, cb, prometheus.NewRegistry()) assert := assert.New(t) @@ -157,7 +164,7 @@ func TestNewCaptionsService(t *testing.T) { } assert.NotNil(service.client.Providers) - assert.Equal(service.client.Providers, providers) + assert.Equal(service.client.Providers, p) assert.Contains(service.Endpoints(), "/captions/{id}") assert.Contains(service.Endpoints(), "/jobs/{id}") diff --git a/sidekicks.go b/sidekicks.go index 562687f..d75edbd 100644 --- a/sidekicks.go +++ b/sidekicks.go @@ -1,8 +1,7 @@ -package main +package videocaptionsapi import ( "context" - "fmt" "net/http" "sync" "time" @@ -10,16 +9,15 @@ import ( "contrib.go.opencensus.io/exporter/prometheus" "github.com/NYTimes/gizmo/server" "github.com/NYTimes/video-captions-api/providers" - "github.com/pkg/errors" - goprom "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" ) func StartMetricsServer( ctx context.Context, - wg *sync.WaitGroup, + eg *errgroup.Group, exporter *prometheus.Exporter, - log *logrus.Logger) { + log *logrus.Logger) error { addr := ":9000" log.WithField("address", addr).Info("starting metric server") @@ -33,31 +31,33 @@ func StartMetricsServer( WriteTimeout: 2 * time.Second, Handler: mux, } - wg.Add(1) shutdownCtx, cancel := context.WithCancel(ctx) - go func() { - defer wg.Done() + eg.Go(func() error { defer cancel() var err error if err = srv.Shutdown(shutdownCtx); err != nil { log.Fatalf("server Shutdown Failed:%+s", err) + return err } + return nil - }() + }) - wg.Add(1) - go func(log *logrus.Entry) { - defer wg.Done() + eg.Go(func() error { + log := log.WithField("service", "metrics_server") var err error if err = srv.ListenAndServe(); err != http.ErrServerClosed { log.WithFields(logrus.Fields{ "err": err, "address": addr, }).Fatal("Metrics server failure") + return err } - }(log.WithField("service", "metrics_server")) + return nil + }) + return eg.Wait() } func StartCallbackListener( @@ -65,7 +65,6 @@ func StartCallbackListener( wg *sync.WaitGroup, callers []providers.Provider, log *logrus.Logger) chan *providers.DataWrapper { - wg.Add(1) // callback server wg.Add(1) callbacks := make(chan *providers.DataWrapper) @@ -91,48 +90,3 @@ func StartCallbackListener( return callbacks } - -func MustInitMetrics() (*prometheus.Exporter, *goprom.Registry) { - pe, r, err := initMetrics() - if err != nil { - panic(errors.Wrap(err, "Failed to initialize metrics service")) - } - return pe, r -} - -func initMetrics() (*prometheus.Exporter, *goprom.Registry, error) { - r := goprom.NewRegistry() - r.MustRegister(goprom.NewProcessCollector(goprom.ProcessCollectorOpts{})) - r.MustRegister(goprom.NewGoCollector()) - - versionCollector := goprom.NewGaugeVec(goprom.GaugeOpts{ - Namespace: MetricsNamespace, - Name: "version", - Help: "Application version.", - }, []string{"version"}) - - r.MustRegister(versionCollector) - versionCollector.WithLabelValues(version).Add(1) - - captionTimer := goprom.NewHistogramVec(goprom.HistogramOpts{ - Namespace: MetricsNamespace, - Name: "asr_execution_time_seconds", - Help: "provider caption time", - Buckets: goprom.LinearBuckets(20, 5, 5), - }, []string{ - "provider", - }) - - r.MustRegister(captionTimer) - - // Stats exporter: Prometheus - pe, err := prometheus.NewExporter(prometheus.Options{ - Namespace: MetricsNamespace, - Registry: r, - }) - if err != nil { - return nil, nil, fmt.Errorf("failed to create the Prometheus stats exporter %w", err) - } - - return pe, r, nil -} From 01ef20ba48f64425259dad70fe63160ed4f06fa3 Mon Sep 17 00:00:00 2001 From: Guy J Grigsby Date: Mon, 9 Aug 2021 16:54:15 -0600 Subject: [PATCH 08/17] separate user callbacks from provider callbacks for clarity --- cmd/main.go | 4 +-- go.mod | 3 +- go.sum | 2 ++ providers/callback.go | 58 ++++++++++++++++++++++++++++++++++ providers/provider.go | 1 + providers/threeplay.go | 23 +++++++++++++- service/client.go | 70 +++++++++++++++++++++++++----------------- service/job.go | 47 ++++++++++++++++++++-------- service/service.go | 51 +++++++++--------------------- sidekicks.go | 35 +-------------------- 10 files changed, 178 insertions(+), 116 deletions(-) create mode 100644 providers/callback.go diff --git a/cmd/main.go b/cmd/main.go index d2e8dc4..7bbe1a4 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -80,10 +80,10 @@ func main() { exporter, registry := MustInitMetrics() videocaptionsapi.StartMetricsServer(ctx, eg, exporter, server.Log) - callbacks := videocaptionsapi.StartCallbackListener(ctx, &sync.WaitGroup{}, implementedProviders, server.Log) + callbackQueue, endpoints := providers.StartCallbackListener(ctx, &sync.WaitGroup{}, implementedProviders, server.Log) // caption server - captionsService := service.NewCaptionsService(&cfg, db, callbacks, registry) + captionsService := service.NewCaptionsService(&cfg, db, callbackQueue, endpoints, registry) for _, p := range implementedProviders { captionsService.AddProvider(p) diff --git a/go.mod b/go.mod index 0760e1b..41511cd 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,5 @@ module github.com/NYTimes/video-captions-api -replace github.com/nytimes-labs/client_golang v1.11.0 => github.com/prometheus/client_golang v1.11.0 - require ( cloud.google.com/go/datastore v1.4.0 cloud.google.com/go/storage v1.14.0 @@ -18,6 +16,7 @@ require ( github.com/prometheus/client_golang v0.9.4 github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/procfs v0.2.0 // indirect + github.com/satori/go.uuid v1.2.0 github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.7.0 github.com/tdewolff/parse/v2 v2.4.3 diff --git a/go.sum b/go.sum index 95e8221..186a9a3 100644 --- a/go.sum +++ b/go.sum @@ -238,6 +238,8 @@ github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULU github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sethgrid/pester v0.0.0-20180430140037-03e26c9abbbf/go.mod h1:Ad7IjTpvzZO8Fl0vh9AzQ+j/jYZfyp2diGwI8m5q+ns= github.com/sethgrid/pester v0.0.0-20190127155807-68a33a018ad0 h1:X9XMOYjxEfAYSy3xK1DzO5dMkkWhs9E9UCcS1IERx2k= github.com/sethgrid/pester v0.0.0-20190127155807-68a33a018ad0/go.mod h1:Ad7IjTpvzZO8Fl0vh9AzQ+j/jYZfyp2diGwI8m5q+ns= diff --git a/providers/callback.go b/providers/callback.go new file mode 100644 index 0000000..f6e4345 --- /dev/null +++ b/providers/callback.go @@ -0,0 +1,58 @@ +package providers + +import ( + "context" + "fmt" + "net/http" + "sync" + "time" + + "github.com/NYTimes/gizmo/server" + uuid "github.com/satori/go.uuid" + "github.com/sirupsen/logrus" +) + +func StartCallbackListener( + ctx context.Context, + wg *sync.WaitGroup, + callers []Provider, + log *logrus.Logger) (chan *DataWrapper, map[string]string) { + + mux := http.NewServeMux() + // generate a unique callback URL for each provider + var uris = make(map[string]string) + for _, c := range callers { + + guid := uuid.NewV4().String() + uris[c.GetName()] = guid + expectedProvider := c + mux.HandleFunc( + fmt.Sprintf("%s", guid), + func(w http.ResponseWriter, req *http.Request) { + _, _ = expectedProvider.HandleCallback(req) + + }) + + } + wg.Add(1) + callbacks := make(chan *DataWrapper) + go func(log *logrus.Entry) { + defer wg.Done() + + cbServer := &http.Server{ + Addr: ":9090", + ReadTimeout: 2 * time.Second, + WriteTimeout: 2 * time.Second, + Handler: mux, + } + var err error + if err = cbServer.ListenAndServe(); err != http.ErrServerClosed { + log.WithFields(logrus.Fields{ + "err": err, + }).Fatal("Callback server failure") + } + + }(server.Log.WithField("service", "callbackListener")) + + return callbacks, uris +} diff --git a/providers/provider.go b/providers/provider.go index f80731c..6662528 100644 --- a/providers/provider.go +++ b/providers/provider.go @@ -14,6 +14,7 @@ type Callback struct { type DataWrapper struct { JobID string Data CallbackData + URL string } type CallbackData struct { diff --git a/providers/threeplay.go b/providers/threeplay.go index d8585b3..1a6db57 100644 --- a/providers/threeplay.go +++ b/providers/threeplay.go @@ -1,7 +1,10 @@ package providers import ( + "encoding/json" "errors" + "fmt" + "io/ioutil" "net/http" "net/url" "strconv" @@ -160,6 +163,24 @@ func (c *ThreePlayProvider) CancelJob(job *database.Job) (bool, error) { } func (c *ThreePlayProvider) HandleCallback(req *http.Request) (*CallbackData, error) { + requestLogger := c.logger.WithFields(log.Fields{ + "Handler": "ThreePlayProcessCallback", + }) - panic("not implemented") // TODO: Implement + defer req.Body.Close() + + data, err := ioutil.ReadAll(req.Body) + if err != nil { + requestLogger.WithError(err).Error("Could not read request body: ") + return nil, err + } + + callbackObject := Callback{} + err = json.Unmarshal(data, &callbackObject) + if err != nil { + requestLogger.WithError(err).Error("Could not unmarshal callback") + return nil, fmt.Errorf("Malformed parameters: %w", err) + } + + return &callbackObject.Data, nil } diff --git a/service/client.go b/service/client.go index 7395439..4bf26cb 100644 --- a/service/client.go +++ b/service/client.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "net/http" "sort" "strconv" @@ -12,6 +13,7 @@ import ( "github.com/NYTimes/video-captions-api/database" "github.com/NYTimes/video-captions-api/providers" + "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" ) @@ -260,54 +262,66 @@ func (c Client) GenerateTranscript(captionFile []byte, captionFormat string) (st return "", fmt.Errorf("unable to generate a transcript for caption format: %v", captionFormat) } -func (c Client) ProcessCallback(callbackData providers.CallbackData, jobID string) error { - jobLogger := c.Logger.WithFields(log.Fields{ - "JobID": jobID, - "ProviderID": callbackData.ID, - "Status": callbackData.Status, - }) - jobLogger.Info("Processing a callback for captions") - if callbackData.ID == 0 { - jobLogger.Error("Invalid Provider ID") - return errors.New("invalid Provider ID") - } +func (c Client) notify(jobID string, providerID int, log *logrus.Entry) error { + + log.Debug("Processing a callback for captions") if jobID == "" { - databaseJob, err := c.DB.GetJobByProviderID(strconv.Itoa(callbackData.ID)) + databaseJob, err := c.DB.GetJobByProviderID(strconv.Itoa(providerID)) if err != nil { - jobLogger.Errorf("Could not retrieve job by provider ID: %v", err) + // We don't need to add fields for jobID or providerID. They are already in the *logrus.Entry. Tell your friends. (The errors below) + log.WithField("error", err).Error("Failed to get job by provider ID") return err } jobID = databaseJob.ID } job, err := c.GetJob(jobID) if err != nil { - jobLogger.Errorf("Could not get job data: %v", err) + log.WithField("error", err).Error("Failed to get job by ID") return err } - if c.CallbackURL != "" { - jobLogger.Infof("Making API call to: %v", c.CallbackURL) - err = c.makeAPICall(job) - if err != nil { - jobLogger.Errorf("Encountered an error while making a callback call: %v", err) - return err - } - } - return nil -} -func (c Client) makeAPICall(job *database.Job) error { - requestBody, err := json.Marshal(job) + b, err := json.Marshal(job) if err != nil { + log.WithField("error", err).Error("Failed to marshal user response") return err } - resp, err := http.Post(c.CallbackURL, "application/json", bytes.NewBuffer(requestBody)) + log.Debug("Making API call to: %v", c.CallbackURL) + + resp, err := http.Post(c.CallbackURL, "application/json", bytes.NewBuffer(b)) if err != nil { + log.WithField("error", err).Error("Failed to POST job completion") return err } + defer resp.Body.Close() if !(resp.StatusCode >= 200 && resp.StatusCode < 300) { - return fmt.Errorf("%v", resp.Status) + entry := log.WithFields(logrus.Fields{ + "status": resp.Status, + }) + + msg := "Unexpected response code from user callback" + + _, err := io.ReadAll(resp.Body) + if err != nil { + err := fmt.Errorf("%s:%w", msg, err) + entry.WithField("error", err).Error("failed to read response body") + return err + + } + + entry.Error(msg) + return errors.New(msg) } + return nil } +func (c Client) ProcessCallback(callbackData providers.CallbackData, jobID string) { + + entry := c.Logger.WithFields(log.Fields{ + "JobID": jobID, + "ProviderID": callbackData.ID, + }) + + c.notify(jobID, callbackData.ID, entry) +} diff --git a/service/job.go b/service/job.go index 6d4c16a..551fab0 100644 --- a/service/job.go +++ b/service/job.go @@ -16,7 +16,8 @@ import ( log "github.com/sirupsen/logrus" ) -type captionsError struct { +// CaptionsError is an error that occurs as a direct result of issue with the captioning process. Most frequently directly from the provider. +type CaptionsError struct { Message string `json:"error"` } @@ -35,6 +36,26 @@ type uploadedFile struct { File []byte `json:"file"` Name string `json:"name"` } +type Callback struct { + Code int `json:"code"` + Data CallbackData `json:"data"` +} + +type CallbackData struct { + Cancellable bool `json:"cancellable"` + Default bool `json:"default"` + ID int `json:"id"` + BatchID int `json:"batch_id"` + LanguageID int `json:"language_id"` + MediaFileID int `json:"media_file_id"` + LanguageIDs []int `json:"language_ids"` + CancellationDetails string `json:"cancellation_details"` + CancellationReason string `json:"cancellation_reason"` + ReferenceID string `json:"reference_id"` + Status string `json:"status"` + Type string `json:"type"` + Duration float64 `json:"duration"` +} func newJobFromParams(newJob jobParams) (*database.Job, error) { outputs := make([]database.JobOutput, 0) @@ -84,7 +105,7 @@ func newJobFromParams(newJob jobParams) (*database.Job, error) { } // Error implements the error interface -func (e captionsError) Error() string { +func (e CaptionsError) Error() string { return e.Message } @@ -94,9 +115,9 @@ func (s *CaptionsService) GetJobs(r *http.Request) (int, interface{}, error) { jobs, err := s.client.GetJobs(parentID) if err != nil { if err == database.ErrNoJobs { - return http.StatusNotFound, nil, captionsError{err.Error()} + return http.StatusNotFound, nil, CaptionsError{err.Error()} } - return http.StatusInternalServerError, err, captionsError{err.Error()} + return http.StatusInternalServerError, err, CaptionsError{err.Error()} } return http.StatusOK, jobs, nil } @@ -109,9 +130,9 @@ func (s *CaptionsService) GetJob(r *http.Request) (int, interface{}, error) { job, err := s.client.GetJob(id) if err != nil { if err == database.ErrJobNotFound { - return http.StatusNotFound, nil, captionsError{err.Error()} + return http.StatusNotFound, nil, CaptionsError{err.Error()} } - return http.StatusInternalServerError, nil, captionsError{err.Error()} + return http.StatusInternalServerError, nil, CaptionsError{err.Error()} } return http.StatusOK, job, nil } @@ -121,10 +142,10 @@ func (s *CaptionsService) CancelJob(r *http.Request) (int, interface{}, error) { id := server.Vars(r)["id"] canceled, err := s.client.CancelJob(id) if err != nil { - return http.StatusNotFound, nil, captionsError{err.Error()} + return http.StatusNotFound, nil, CaptionsError{err.Error()} } if !canceled { - return http.StatusConflict, nil, captionsError{"Cannot cancel a job that is already done"} + return http.StatusConflict, nil, CaptionsError{"Cannot cancel a job that is already done"} } return http.StatusOK, nil, nil } @@ -146,30 +167,30 @@ func (s *CaptionsService) CreateJob(r *http.Request) (int, interface{}, error) { data, err := ioutil.ReadAll(r.Body) if err != nil { requestLogger.WithError(err).Error("Could not read request body: ") - return http.StatusBadRequest, nil, captionsError{err.Error()} + return http.StatusBadRequest, nil, CaptionsError{err.Error()} } err = json.Unmarshal(data, ¶ms) if err != nil { requestLogger.WithError(err).Error("Could not create job from request body") - return http.StatusBadRequest, nil, captionsError{"Malformed parameters"} + return http.StatusBadRequest, nil, CaptionsError{"Malformed parameters"} } if params.MediaURL == "" && params.CaptionFile.File == nil { requestLogger.WithError(err).Error("Tried to create a job without a media url or caption file") - return http.StatusBadRequest, nil, captionsError{"Please provide a media_url or caption_file"} + return http.StatusBadRequest, nil, CaptionsError{"Please provide a media_url or caption_file"} } job, err := newJobFromParams(params) if err != nil { requestLogger.WithError(err).Error("could not create job from parameters") - return http.StatusInternalServerError, nil, captionsError{err.Error()} + return http.StatusInternalServerError, nil, CaptionsError{err.Error()} } err = s.client.DispatchJob(job) if err != nil { requestLogger.WithError(err).Error("could not dispatch job") - return http.StatusInternalServerError, nil, captionsError{err.Error()} + return http.StatusInternalServerError, nil, CaptionsError{err.Error()} } return http.StatusCreated, job, nil diff --git a/service/service.go b/service/service.go index 3bea0f3..2113fb1 100644 --- a/service/service.go +++ b/service/service.go @@ -1,7 +1,6 @@ package service import ( - "fmt" "net/http" "github.com/prometheus/client_golang/prometheus" @@ -29,55 +28,35 @@ func NewCaptionsService( cfg *config.CaptionsServiceConfig, db database.DB, callbacks chan *providers.DataWrapper, + callbackEndpoints map[string]string, metrics *prom.Registry, ) *CaptionsService { storage, _ := NewGCSStorage(cfg.BucketName, cfg.Logger) client := Client{ - Providers: make(map[string]providers.Provider), - DB: db, - Logger: cfg.Logger, - Storage: storage, - CallbackURL: cfg.CallbackURL, + Providers: make(map[string]providers.Provider), + DB: db, + Logger: cfg.Logger, + Storage: storage, + } + service := &CaptionsService{ + client, + cfg.Logger, + callbacks, + metrics, } go func(log *logrus.Entry) { for wrapper := range callbacks { data, id := wrapper.Data, wrapper.JobID - err := client.ProcessCallback(data, id) - if err != nil { - log.WithFields(logrus.Fields{ - "err": err, - "callbackdata": fmt.Sprintf("%+v", data), - "jobID": "id", - "provider": data.ID, - }).Error("Callback Failed") - - // TODO retry + client.ProcessCallback(data, id) - } + // TODO retry } + }(log.WithField("service", "Callback Listener Worker")) - return &CaptionsService{ - client, - cfg.Logger, - callbacks, - metrics, - } + return service } -// -//func (s *CaptionsService) ProcessCallbacks() error { -// -// -// -// err = s.client.ProcessCallback(callbackObject.Data, jobID) -// if err != nil { -// requestLogger.Errorf("Could not process callback for ID: %v", callbackObject.Data.ID) -// return http.StatusInternalServerError, nil, captionsError{err.Error()} -// } -// return http.StatusOK, nil, nil -//} - // AddProvider adds a Provider to the CaptionsService func (s *CaptionsService) AddProvider(provider providers.Provider) { s.client.Providers[provider.GetName()] = provider diff --git a/sidekicks.go b/sidekicks.go index d75edbd..a179db6 100644 --- a/sidekicks.go +++ b/sidekicks.go @@ -3,13 +3,11 @@ package videocaptionsapi import ( "context" "net/http" - "sync" "time" "contrib.go.opencensus.io/exporter/prometheus" - "github.com/NYTimes/gizmo/server" - "github.com/NYTimes/video-captions-api/providers" "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" ) @@ -59,34 +57,3 @@ func StartMetricsServer( }) return eg.Wait() } - -func StartCallbackListener( - ctx context.Context, - wg *sync.WaitGroup, - callers []providers.Provider, - log *logrus.Logger) chan *providers.DataWrapper { - // callback server - wg.Add(1) - callbacks := make(chan *providers.DataWrapper) - go func(log *logrus.Entry) { - defer wg.Done() - - mux := http.NewServeMux() - - cbServer := &http.Server{ - Addr: ":9090", - ReadTimeout: 2 * time.Second, - WriteTimeout: 2 * time.Second, - Handler: mux, - } - var err error - if err = cbServer.ListenAndServe(); err != http.ErrServerClosed { - log.WithFields(logrus.Fields{ - "err": err, - }).Fatal("Callback server failure") - } - - }(server.Log.WithField("service", "callbackListener")) - - return callbacks -} From b43e48c247fe4357c8d53a791aba54b566db731f Mon Sep 17 00:00:00 2001 From: Guy J Grigsby Date: Thu, 12 Aug 2021 15:48:47 -0600 Subject: [PATCH 09/17] simplified provider callback flow --- .gitignore | 1 + cmd/main.go | 4 +--- providers/amara.go | 4 ++-- providers/callback.go | 47 +++++++++++++++++++++++++++++------------ providers/provider.go | 5 ++--- providers/threeplay.go | 11 +++++----- providers/upload.go | 2 +- service/client.go | 4 ++-- service/client_test.go | 7 +++--- service/service_test.go | 17 ++++++++------- 10 files changed, 61 insertions(+), 41 deletions(-) diff --git a/.gitignore b/.gitignore index d2f35e7..6e792bd 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ video-captions-api .DS_Store main captions +tags diff --git a/cmd/main.go b/cmd/main.go index 7bbe1a4..984f27d 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -80,7 +80,7 @@ func main() { exporter, registry := MustInitMetrics() videocaptionsapi.StartMetricsServer(ctx, eg, exporter, server.Log) - callbackQueue, endpoints := providers.StartCallbackListener(ctx, &sync.WaitGroup{}, implementedProviders, server.Log) + callbackQueue, endpoints := providers.StartCallbackListener(ctx, &sync.WaitGroup{}, implementedProviders, server.Log.WithField("service", "CallbackListener")) // caption server captionsService := service.NewCaptionsService(&cfg, db, callbackQueue, endpoints, registry) @@ -90,8 +90,6 @@ func main() { } server.Init("video-captions-api", cfg.Server) - //server.WithCloseHandler TODO need to implement gizmo server.Context.Handler to gracefully shut down - err = server.Register(captionsService) if err != nil { server.Log.Fatal("Unable to register service: ", err) diff --git a/providers/amara.go b/providers/amara.go index 2cb5fe6..f328bc6 100644 --- a/providers/amara.go +++ b/providers/amara.go @@ -133,6 +133,6 @@ func (c *AmaraProvider) CancelJob(job *database.Job) (bool, error) { return false, nil } -func (c *AmaraProvider) HandleCallback(_ *http.Request) (*CallbackData, error) { - return &CallbackData{}, nil +func (c *AmaraProvider) HandleCallback(_ *http.Request) (string, *CallbackData, error) { + return "", &CallbackData{}, nil } diff --git a/providers/callback.go b/providers/callback.go index f6e4345..f26bb78 100644 --- a/providers/callback.go +++ b/providers/callback.go @@ -12,30 +12,30 @@ import ( "github.com/sirupsen/logrus" ) +type CallbackHandler interface { + HandleCallback(req *http.Request) (jobID string, data *CallbackData, err error) +} + func StartCallbackListener( ctx context.Context, wg *sync.WaitGroup, - callers []Provider, - log *logrus.Logger) (chan *DataWrapper, map[string]string) { + callbackHandlers []Provider, + log *logrus.Entry) (chan *DataWrapper, map[string]string) { mux := http.NewServeMux() + q := make(chan *DataWrapper) // generate a unique callback URL for each provider var uris = make(map[string]string) - for _, c := range callers { + for _, c := range callbackHandlers { - guid := uuid.NewV4().String() - uris[c.GetName()] = guid - expectedProvider := c + h, guid := handleRegister(ctx, q, c, log.WithField("provider", c)) mux.HandleFunc( fmt.Sprintf("%s", guid), - func(w http.ResponseWriter, req *http.Request) { - _, _ = expectedProvider.HandleCallback(req) - - }) - + h, + ) } + wg.Add(1) - callbacks := make(chan *DataWrapper) go func(log *logrus.Entry) { defer wg.Done() @@ -54,5 +54,26 @@ func StartCallbackListener( }(server.Log.WithField("service", "callbackListener")) - return callbacks, uris + return q, uris +} + +func handleRegister(ctx context.Context, q chan<- *DataWrapper, c CallbackHandler, log *logrus.Entry) (http.HandlerFunc, string) { + guid := uuid.NewV4().String() + path := fmt.Sprintf("/%s", guid) + + h := func(w http.ResponseWriter, req *http.Request) { + id, data, err := c.HandleCallback(req) + if err != nil { + log.WithFields(logrus.Fields{ + "error": err, + "endpoint": guid, + }).Error("Failed to handle callback") + http.Error(w, "Callback Failed", http.StatusInternalServerError) + } + q <- &DataWrapper{ + JobID: id, + Data: data, + } + } + return h, path } diff --git a/providers/provider.go b/providers/provider.go index 6662528..c2436cc 100644 --- a/providers/provider.go +++ b/providers/provider.go @@ -13,8 +13,7 @@ type Callback struct { type DataWrapper struct { JobID string - Data CallbackData - URL string + Data *CallbackData } type CallbackData struct { @@ -39,6 +38,6 @@ type Provider interface { Download(*database.Job, string) ([]byte, error) GetProviderJob(*database.Job) (*database.ProviderJob, error) GetName() string - HandleCallback(req *http.Request) (*CallbackData, error) + HandleCallback(req *http.Request) (jobID string, data *CallbackData, err error) CancelJob(*database.Job) (bool, error) } diff --git a/providers/threeplay.go b/providers/threeplay.go index 1a6db57..f83f036 100644 --- a/providers/threeplay.go +++ b/providers/threeplay.go @@ -162,7 +162,7 @@ func (c *ThreePlayProvider) CancelJob(job *database.Job) (bool, error) { return providerJob.Cancellable, errors.New("job is not cancellable") } -func (c *ThreePlayProvider) HandleCallback(req *http.Request) (*CallbackData, error) { +func (c *ThreePlayProvider) HandleCallback(req *http.Request) (string, *CallbackData, error) { requestLogger := c.logger.WithFields(log.Fields{ "Handler": "ThreePlayProcessCallback", }) @@ -172,15 +172,16 @@ func (c *ThreePlayProvider) HandleCallback(req *http.Request) (*CallbackData, er data, err := ioutil.ReadAll(req.Body) if err != nil { requestLogger.WithError(err).Error("Could not read request body: ") - return nil, err + return "", nil, err } callbackObject := Callback{} err = json.Unmarshal(data, &callbackObject) if err != nil { requestLogger.WithError(err).Error("Could not unmarshal callback") - return nil, fmt.Errorf("Malformed parameters: %w", err) + return "", nil, fmt.Errorf("Malformed parameters: %w", err) } - - return &callbackObject.Data, nil + queryParams := req.URL.Query() + jobID := queryParams.Get("job_id") + return jobID, &callbackObject.Data, nil } diff --git a/providers/upload.go b/providers/upload.go index 6bad9d6..ea7a623 100644 --- a/providers/upload.go +++ b/providers/upload.go @@ -89,6 +89,6 @@ func (c *UploadProvider) validateCaptionFile(file *database.UploadedFile) error return nil } -func (c *UploadProvider) HandleCallback(req *http.Request) (*CallbackData, error) { +func (c *UploadProvider) HandleCallback(req *http.Request) (string, *CallbackData, error) { panic("not implemented") // TODO: Implement } diff --git a/service/client.go b/service/client.go index 4bf26cb..54b053c 100644 --- a/service/client.go +++ b/service/client.go @@ -286,7 +286,7 @@ func (c Client) notify(jobID string, providerID int, log *logrus.Entry) error { return err } - log.Debug("Making API call to: %v", c.CallbackURL) + log.WithField("addr", c.CallbackURL).Debug("Calling API") resp, err := http.Post(c.CallbackURL, "application/json", bytes.NewBuffer(b)) if err != nil { @@ -316,7 +316,7 @@ func (c Client) notify(jobID string, providerID int, log *logrus.Entry) error { return nil } -func (c Client) ProcessCallback(callbackData providers.CallbackData, jobID string) { +func (c Client) ProcessCallback(callbackData *providers.CallbackData, jobID string) { entry := c.Logger.WithFields(log.Fields{ "JobID": jobID, diff --git a/service/client_test.go b/service/client_test.go index 0cc46a0..9444d32 100644 --- a/service/client_test.go +++ b/service/client_test.go @@ -378,7 +378,6 @@ func TestProcessCallbackClient(t *testing.T) { } service, client := createCaptionsService(callbackURL) - assert := assert.New(t) service.AddProvider(&fakeProvider{logger: log.New()}) job := &database.Job{ ID: "123", @@ -387,7 +386,7 @@ func TestProcessCallbackClient(t *testing.T) { ProviderParams: map[string]string{"ProviderID": "11214314"}, } client.DB.StoreJob(job) - callbackData := providers.CallbackData{ + callbackData := &providers.CallbackData{ ID: test.providerID, MediaFileID: 3765758, BatchID: 68841, @@ -401,8 +400,8 @@ func TestProcessCallbackClient(t *testing.T) { Cancellable: false, } - err := client.ProcessCallback(callbackData, test.jobID) - assert.Equal(test.error, err) + client.ProcessCallback(callbackData, test.jobID) + }) } } diff --git a/service/service_test.go b/service/service_test.go index dfd4f0a..128bf72 100644 --- a/service/service_test.go +++ b/service/service_test.go @@ -11,7 +11,6 @@ import ( "reflect" "github.com/NYTimes/gizmo/server" - captions "github.com/NYTimes/video-captions-api" "github.com/NYTimes/video-captions-api/config" "github.com/NYTimes/video-captions-api/database" "github.com/NYTimes/video-captions-api/providers" @@ -26,7 +25,7 @@ type fakeProvider struct { callbackData *providers.CallbackData } -func (p fakeProvider) HandleCallback(req *http.Request) (*providers.CallbackData, error) { +func (p fakeProvider) HandleCallback(req *http.Request) (string, *providers.CallbackData, error) { panic("not implemented") // TODO: Implement } @@ -78,8 +77,8 @@ func (p fakeProvider) CancelJob(job *database.Job) (bool, error) { type brokenProvider fakeProvider -func (p brokenProvider) HandleCallback(req *http.Request) (*providers.CallbackData, error) { - return &providers.CallbackData{}, nil +func (p brokenProvider) HandleCallback(req *http.Request) (string, *providers.CallbackData, error) { + return "", &providers.CallbackData{}, nil } func (p brokenProvider) GetName() string { @@ -145,11 +144,13 @@ func TestNewCaptionsService(t *testing.T) { } db := database.NewMemoryDatabase() var callers []providers.Provider - for _, p := range p { - callers = append(callers, p) + for _, caller := range p { + callers = append(callers, caller) } - cb := captions.StartCallbackListener(context.Background(), &sync.WaitGroup{}, callers, logger) - service := NewCaptionsService(&cfg, db, cb, prometheus.NewRegistry()) + cbQ, urls := providers.StartCallbackListener(context.Background(), &sync.WaitGroup{}, callers, logger.WithField("service", projectID)) + + assert.Zero(t, len(urls)) + service := NewCaptionsService(&cfg, db, cbQ, urls, prometheus.NewRegistry()) assert := assert.New(t) From 03f8d7600712b789c6a56ad68c9c8b590c884c6c Mon Sep 17 00:00:00 2001 From: Guy J Grigsby Date: Thu, 12 Aug 2021 15:49:08 -0600 Subject: [PATCH 10/17] forgot to add testfile --- providers/callback_test.go | 109 +++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 providers/callback_test.go diff --git a/providers/callback_test.go b/providers/callback_test.go new file mode 100644 index 0000000..7db2404 --- /dev/null +++ b/providers/callback_test.go @@ -0,0 +1,109 @@ +package providers + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + "sync" + "testing" + "time" + + "github.com/NYTimes/video-captions-api/database" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" +) + +func makeContext(d time.Duration) (context.Context, func()) { + return context.WithTimeout(context.Background(), d) +} + +func TestCallbackListener(t *testing.T) { + tests := []struct { + name string + p []Provider + ttl time.Duration + payload io.Reader + }{ + { + name: "Happy Path-single provider", + p: []Provider{ + &testProvider{ + name: "happy", + + f: func(p *testProvider, req *http.Request) (string, *CallbackData, error) { + return "TODO", &CallbackData{}, nil + }, + }, + }, + ttl: time.Second * 2, + payload: strings.NewReader("This will surely fail"), + }, + } + + log := logrus.New() + for _, tc := range tests { + + //t.Parallel() + ctx, cancel := makeContext(tc.ttl) + var wg sync.WaitGroup + t.Log("starting callback listener") + q, uris := StartCallbackListener( + ctx, + &wg, + tc.p, + log.WithField( + "testname", tc.name, + ), + ) + addr := uris[tc.p[0].(*testProvider).name] + go func() { + t.Log("calling the callback") + res, err := http.Post(addr, "application/json", tc.payload) + require.NoError(t, err) + fmt.Printf("%+v\n", res) + cancel() + }() + t.Log("ranging over CallbackData q") + for data := range q { + // do something + log.WithField("data", data).Info("Got callback data from q") + } + + wg.Wait() + } + +} + +type testProvider struct { + name string + f func(*testProvider, *http.Request) (string, *CallbackData, error) + runningJobs []*database.Job +} + +func (p *testProvider) DispatchJob(job *database.Job) error { + p.runningJobs = append(p.runningJobs, job) + + return nil +} + +func (p *testProvider) Download(_ *database.Job, _ string) ([]byte, error) { + return nil, nil +} + +func (p *testProvider) GetProviderJob(_ *database.Job) (*database.ProviderJob, error) { + return nil, nil +} + +func (p *testProvider) GetName() string { + return p.name +} + +func (p *testProvider) HandleCallback(req *http.Request) (string, *CallbackData, error) { + return p.f(p, req) +} + +func (p *testProvider) CancelJob(_ *database.Job) (bool, error) { + return true, nil +} From d1750fc8f24e389d7f9fa154ae7885214296b5f4 Mon Sep 17 00:00:00 2001 From: Guy J Grigsby Date: Thu, 19 Aug 2021 13:48:43 -0600 Subject: [PATCH 11/17] fix deadlock --- database/mem_test.go | 3 +++ providers/callback.go | 13 +++++++++++++ providers/callback_test.go | 8 +++++--- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/database/mem_test.go b/database/mem_test.go index 60dfac4..add5be7 100644 --- a/database/mem_test.go +++ b/database/mem_test.go @@ -118,6 +118,7 @@ func TestStoreAndGetParallel(t *testing.T) { for { select { case <-ctx.Done(): + log.Info("ctx.Done") total := atomic.LoadUint64(&runningTotal) log.Debug( "Context completed. Attempting to validate completion count", @@ -130,6 +131,7 @@ func TestStoreAndGetParallel(t *testing.T) { return case j := <-toValidate: + log.Info("toValidate") total := atomic.AddUint64(&runningTotal, 1) log.Debug( "pulling job", @@ -161,6 +163,7 @@ func TestStoreAndGetParallel(t *testing.T) { } }(tc.name, tc.count, wLog) } + log.Info("waiting") wg.Wait() cancel() } diff --git a/providers/callback.go b/providers/callback.go index f26bb78..7f58908 100644 --- a/providers/callback.go +++ b/providers/callback.go @@ -29,6 +29,10 @@ func StartCallbackListener( for _, c := range callbackHandlers { h, guid := handleRegister(ctx, q, c, log.WithField("provider", c)) + log.WithFields(logrus.Fields{ + "guid": guid, + }).Info("generating callback urls") + uris[c.GetName()] = fmt.Sprintf("%s", guid) mux.HandleFunc( fmt.Sprintf("%s", guid), h, @@ -45,6 +49,14 @@ func StartCallbackListener( WriteTimeout: 2 * time.Second, Handler: mux, } + go func() { + select { + case <-ctx.Done(): + shutdownCtx, cancel := context.WithTimeout(context.Background(), time.Second*1) + defer cancel() + cbServer.Shutdown(shutdownCtx) + } + }() var err error if err = cbServer.ListenAndServe(); err != http.ErrServerClosed { log.WithFields(logrus.Fields{ @@ -62,6 +74,7 @@ func handleRegister(ctx context.Context, q chan<- *DataWrapper, c CallbackHandle path := fmt.Sprintf("/%s", guid) h := func(w http.ResponseWriter, req *http.Request) { + log.Info("handle resgister callback") id, data, err := c.HandleCallback(req) if err != nil { log.WithFields(logrus.Fields{ diff --git a/providers/callback_test.go b/providers/callback_test.go index 7db2404..6df8cf5 100644 --- a/providers/callback_test.go +++ b/providers/callback_test.go @@ -44,7 +44,7 @@ func TestCallbackListener(t *testing.T) { log := logrus.New() for _, tc := range tests { - + log.Info("tc", tc) //t.Parallel() ctx, cancel := makeContext(tc.ttl) var wg sync.WaitGroup @@ -57,13 +57,15 @@ func TestCallbackListener(t *testing.T) { "testname", tc.name, ), ) - addr := uris[tc.p[0].(*testProvider).name] + t.Log("generating urls for callbacks", uris) + addr := fmt.Sprintf("http://localhost:9090%s", uris[tc.p[0].(*testProvider).name]) go func() { - t.Log("calling the callback") + t.Log("calling the callback", addr) res, err := http.Post(addr, "application/json", tc.payload) require.NoError(t, err) fmt.Printf("%+v\n", res) cancel() + close(q) }() t.Log("ranging over CallbackData q") for data := range q { From 82e8c7d97090793c4117b0c9832ee7fef26d2bb9 Mon Sep 17 00:00:00 2001 From: Guy J Grigsby Date: Thu, 19 Aug 2021 13:49:12 -0600 Subject: [PATCH 12/17] refactor prometheus registry for use in providers --- cmd/main.go | 23 ++--------------------- service/client.go | 33 +++++++++++++++++++++++++++++++++ service/service.go | 12 ++++++------ sidekicks.go | 5 +++++ 4 files changed, 46 insertions(+), 27 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 984f27d..fcaae46 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -24,11 +24,6 @@ import ( "github.com/sirupsen/logrus" ) -const ( - // MetricsNamespace is the name of the application. - MetricsNamespace = "video_captions_api" -) - var ( version = "no version from LDFLAGS" ) @@ -79,9 +74,7 @@ func main() { exporter, registry := MustInitMetrics() videocaptionsapi.StartMetricsServer(ctx, eg, exporter, server.Log) - callbackQueue, endpoints := providers.StartCallbackListener(ctx, &sync.WaitGroup{}, implementedProviders, server.Log.WithField("service", "CallbackListener")) - // caption server captionsService := service.NewCaptionsService(&cfg, db, callbackQueue, endpoints, registry) @@ -130,28 +123,16 @@ func initMetrics() (*prometheus.Exporter, *goprom.Registry, error) { r.MustRegister(goprom.NewGoCollector()) versionCollector := goprom.NewGaugeVec(goprom.GaugeOpts{ - Namespace: MetricsNamespace, + Namespace: videocaptionsapi.MetricsNamespace, Name: "version", Help: "Application version.", }, []string{"version"}) r.MustRegister(versionCollector) versionCollector.WithLabelValues(version).Add(1) - - captionTimer := goprom.NewHistogramVec(goprom.HistogramOpts{ - Namespace: MetricsNamespace, - Name: "asr_execution_time_seconds", - Help: "provider caption time", - Buckets: goprom.LinearBuckets(20, 5, 5), - }, []string{ - "provider", - }) - - r.MustRegister(captionTimer) - // Stats exporter: Prometheus pe, err := prometheus.NewExporter(prometheus.Options{ - Namespace: MetricsNamespace, + Namespace: videocaptionsapi.MetricsNamespace, Registry: r, }) if err != nil { diff --git a/service/client.go b/service/client.go index 54b053c..0e03b15 100644 --- a/service/client.go +++ b/service/client.go @@ -10,13 +10,26 @@ import ( "sort" "strconv" "strings" + "time" + videocaptionsapi "github.com/NYTimes/video-captions-api" "github.com/NYTimes/video-captions-api/database" "github.com/NYTimes/video-captions-api/providers" + "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" ) +var captionTimer = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: videocaptionsapi.MetricsNamespace, + Name: "asr_execution_time_seconds", + Help: "provider caption time", + Buckets: prometheus.LinearBuckets(10, 10, 10), +}, []string{ + "provider", + "job", +}) + // Client CaptionsService client type Client struct { Providers map[string]providers.Provider @@ -25,6 +38,19 @@ type Client struct { Storage Storage CallbackURL string CallbackAPIKey string + Metrics *prometheus.Registry +} + +func NewClient(db database.DB, logger *logrus.Logger, storage Storage, metrics *prometheus.Registry) Client { + metrics.MustRegister(captionTimer) + return Client{ + Providers: make(map[string]providers.Provider), + DB: db, + Logger: logger, + Storage: storage, + Metrics: metrics, + } + } // GetJobs gets all jobs associated with a ParentID @@ -104,6 +130,7 @@ func (c Client) GetJob(jobID string) (*database.Job, error) { // DispatchJob dispatches a Job given an existing Provider func (c Client) DispatchJob(job *database.Job) error { + provider := c.Providers[job.Provider] jobLogger := c.Logger.WithFields(log.Fields{"JobID": job.ID, "Provider": job.Provider}) if provider == nil { @@ -167,6 +194,12 @@ func (c Client) DownloadCaption(jobID string, captionType string) ([]byte, error return nil, err } + captionTimer.With( + prometheus.Labels{ + "provider": job.Provider, + "job": job.ID, + }).Observe(float64(time.Now().Sub(job.CreatedAt)) / float64(time.Millisecond)) + providerID := job.GetProviderID() fields := log.Fields{"JobID": jobID, "Provider": job.Provider, "ProviderID": providerID} jobLogger := c.Logger.WithFields(fields) diff --git a/service/service.go b/service/service.go index 2113fb1..fa2d40e 100644 --- a/service/service.go +++ b/service/service.go @@ -32,12 +32,12 @@ func NewCaptionsService( metrics *prom.Registry, ) *CaptionsService { storage, _ := NewGCSStorage(cfg.BucketName, cfg.Logger) - client := Client{ - Providers: make(map[string]providers.Provider), - DB: db, - Logger: cfg.Logger, - Storage: storage, - } + client := NewClient( + db, + cfg.Logger, + storage, + metrics, + ) service := &CaptionsService{ client, cfg.Logger, diff --git a/sidekicks.go b/sidekicks.go index a179db6..2e64921 100644 --- a/sidekicks.go +++ b/sidekicks.go @@ -11,6 +11,11 @@ import ( "golang.org/x/sync/errgroup" ) +const ( + // MetricsNamespace is the name of the application. + MetricsNamespace = "video_captions_api" +) + func StartMetricsServer( ctx context.Context, eg *errgroup.Group, From daa808625beeee84c1d0a8df295cdaed048a0680 Mon Sep 17 00:00:00 2001 From: Guy J Grigsby Date: Mon, 30 Aug 2021 14:38:02 -0600 Subject: [PATCH 13/17] fix bug in metrics server --- sidekicks.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/sidekicks.go b/sidekicks.go index 2e64921..37f73bd 100644 --- a/sidekicks.go +++ b/sidekicks.go @@ -22,7 +22,7 @@ func StartMetricsServer( exporter *prometheus.Exporter, log *logrus.Logger) error { - addr := ":9000" + addr := ":9090" log.WithField("address", addr).Info("starting metric server") mux := http.NewServeMux() @@ -38,10 +38,13 @@ func StartMetricsServer( shutdownCtx, cancel := context.WithCancel(ctx) eg.Go(func() error { defer cancel() - var err error - if err = srv.Shutdown(shutdownCtx); err != nil { - log.Fatalf("server Shutdown Failed:%+s", err) - return err + select { + case <-shutdownCtx.Done(): + var err error + if err = srv.Shutdown(shutdownCtx); err != nil { + log.Fatalf("server Shutdown Failed:%+s", err) + return err + } } return nil @@ -57,6 +60,7 @@ func StartMetricsServer( }).Fatal("Metrics server failure") return err } + println("returned") return nil }) From 31325fa72bd07ba9bdb09908082f3e02792913f4 Mon Sep 17 00:00:00 2001 From: Guy J Grigsby Date: Mon, 30 Aug 2021 14:41:37 -0600 Subject: [PATCH 14/17] fix shutdown bug --- cmd/main.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index fcaae46..4fb3fc5 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -63,14 +63,20 @@ func main() { } interrupt := make(chan os.Signal, 1) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + eg, ctx := errgroup.WithContext(ctx) signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM) defer signal.Stop(interrupt) - implementedProviders := makeProviders(&cfg, db) + go func() { + select { + case <-interrupt: + cancel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - eg, ctx := errgroup.WithContext(ctx) + } + }() + implementedProviders := makeProviders(&cfg, db) exporter, registry := MustInitMetrics() videocaptionsapi.StartMetricsServer(ctx, eg, exporter, server.Log) From 1bcc424a7ee7fcebf6d02114076ebcc991dcea45 Mon Sep 17 00:00:00 2001 From: Guy J Grigsby Date: Mon, 30 Aug 2021 14:42:22 -0600 Subject: [PATCH 15/17] add download label --- service/client.go | 1 + 1 file changed, 1 insertion(+) diff --git a/service/client.go b/service/client.go index 0e03b15..00e3c4f 100644 --- a/service/client.go +++ b/service/client.go @@ -198,6 +198,7 @@ func (c Client) DownloadCaption(jobID string, captionType string) ([]byte, error prometheus.Labels{ "provider": job.Provider, "job": job.ID, + "stage": "download", }).Observe(float64(time.Now().Sub(job.CreatedAt)) / float64(time.Millisecond)) providerID := job.GetProviderID() From ab9bc23ea4f30db10868538ace089987757a37a1 Mon Sep 17 00:00:00 2001 From: Guy J Grigsby Date: Mon, 30 Aug 2021 14:42:36 -0600 Subject: [PATCH 16/17] readme working --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 6dac9b0..d17fd4a 100644 --- a/README.md +++ b/README.md @@ -52,3 +52,15 @@ A pre-built image is available on Docker Hub: https://hub.docker.com/r/nytimes/v ## Documentation For more info check the [docs](https://github.com/nytimes/video-captions-api/wiki/Endpoints) + +# Trint Captions Provider + +## In Progress + +The refactor for the Trint work has been completed. We dynamically spin up a callback endpoint for all providers at runtime. Then all callbacks that are received are sent through the appropriate callback parser defined in the provider and sent through the callback channel defined in the service, created in `NewCaptionsService`. + +`providers/trint` is incomplete. + +## Metrics + +The metrics gatherer is defined in `sidekicks.go` and started in `cmd/main.go`. It uses Prometheus to scrape metrics and reports them to Datadog. From a039776c6ba3f42d4e86fdc80154a0ac2f2212dc Mon Sep 17 00:00:00 2001 From: Guy J Grigsby Date: Tue, 31 Aug 2021 10:32:12 -0600 Subject: [PATCH 17/17] abstract port. fix startup order and account for synchronous gizmo server start --- cmd/main.go | 30 ++++++++++++++++++------------ providers/callback.go | 5 ++++- service/service.go | 1 + sidekicks.go | 8 +++++--- 4 files changed, 28 insertions(+), 16 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 4fb3fc5..0b53622 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -79,27 +79,33 @@ func main() { implementedProviders := makeProviders(&cfg, db) exporter, registry := MustInitMetrics() - videocaptionsapi.StartMetricsServer(ctx, eg, exporter, server.Log) - callbackQueue, endpoints := providers.StartCallbackListener(ctx, &sync.WaitGroup{}, implementedProviders, server.Log.WithField("service", "CallbackListener")) + go videocaptionsapi.StartMetricsServer(ctx, 9090, eg, exporter, server.Log) + callbackQueue, endpoints := providers.StartCallbackListener(ctx, 9091, &sync.WaitGroup{}, implementedProviders, server.Log.WithField("service", "CallbackListener")) // caption server captionsService := service.NewCaptionsService(&cfg, db, callbackQueue, endpoints, registry) for _, p := range implementedProviders { captionsService.AddProvider(p) } - server.Init("video-captions-api", cfg.Server) - err = server.Register(captionsService) - if err != nil { - server.Log.Fatal("Unable to register service: ", err) - } + eg.Go(func() error { + server.Init("video-captions-api", cfg.Server) - err = server.Run() - if err != nil { - server.Log.Fatal("Server encountered a fatal error: ", err) - } + err = server.Register(captionsService) + if err != nil { + return err + } - eg.Wait() + err = server.Run() + if err != nil { + return err + } + return nil + }) + + if err := eg.Wait(); err != nil { + server.Log.WithField("err", err).Error("Unexpected error from server") + } } diff --git a/providers/callback.go b/providers/callback.go index 7f58908..2f919d7 100644 --- a/providers/callback.go +++ b/providers/callback.go @@ -18,6 +18,7 @@ type CallbackHandler interface { func StartCallbackListener( ctx context.Context, + port int, wg *sync.WaitGroup, callbackHandlers []Provider, log *logrus.Entry) (chan *DataWrapper, map[string]string) { @@ -43,8 +44,10 @@ func StartCallbackListener( go func(log *logrus.Entry) { defer wg.Done() + log.WithField("port", port).Info("Starting callback listener") + cbServer := &http.Server{ - Addr: ":9090", + Addr: fmt.Sprintf(":%d", port), ReadTimeout: 2 * time.Second, WriteTimeout: 2 * time.Second, Handler: mux, diff --git a/service/service.go b/service/service.go index fa2d40e..88c4ffe 100644 --- a/service/service.go +++ b/service/service.go @@ -45,6 +45,7 @@ func NewCaptionsService( metrics, } go func(log *logrus.Entry) { + log.Info("Starting callback worker") for wrapper := range callbacks { data, id := wrapper.Data, wrapper.JobID client.ProcessCallback(data, id) diff --git a/sidekicks.go b/sidekicks.go index 37f73bd..c87fe29 100644 --- a/sidekicks.go +++ b/sidekicks.go @@ -2,6 +2,7 @@ package videocaptionsapi import ( "context" + "fmt" "net/http" "time" @@ -18,12 +19,13 @@ const ( func StartMetricsServer( ctx context.Context, + port int, eg *errgroup.Group, exporter *prometheus.Exporter, log *logrus.Logger) error { - addr := ":9090" - log.WithField("address", addr).Info("starting metric server") + addr := fmt.Sprintf(":%d", port) + log.WithField("port", port).Info("starting metric server") mux := http.NewServeMux() mux.Handle("/metrics", exporter) @@ -41,7 +43,7 @@ func StartMetricsServer( select { case <-shutdownCtx.Done(): var err error - if err = srv.Shutdown(shutdownCtx); err != nil { + if err = srv.Shutdown(shutdownCtx); err != nil && err != context.Canceled { log.Fatalf("server Shutdown Failed:%+s", err) return err }