diff --git a/cmd/onboarding-worker/main.go b/cmd/onboarding-worker/main.go index ba6d5b09..46b3f915 100644 --- a/cmd/onboarding-worker/main.go +++ b/cmd/onboarding-worker/main.go @@ -5,8 +5,8 @@ import ( "log" "github.com/dxta-dev/app/internal/onboarding" - "github.com/dxta-dev/app/internal/onboarding/activities" - "github.com/dxta-dev/app/internal/onboarding/workflows" + "github.com/dxta-dev/app/internal/onboarding/activity" + "github.com/dxta-dev/app/internal/onboarding/workflow" "go.temporal.io/sdk/client" "go.temporal.io/sdk/worker" ) @@ -17,6 +17,12 @@ func main() { log.Fatalln("Failed to load configuration:", err) } + githubConfig, err := onboarding.LoadGithubConfig() + + if err != nil { + log.Fatalln("Failed to load github configuration:", err) + } + temporalClient, err := client.Dial(client.Options{ HostPort: cfg.TemporalHostPort, Namespace: cfg.TemporalOnboardingNamespace, @@ -26,6 +32,12 @@ func main() { } defer temporalClient.Close() + githubAppClient, err := onboarding.NewAppClient(*githubConfig) + + if err != nil { + log.Fatalf("Unable to init app client: %v", err) + } + err = onboarding.RegisterNamespace( context.Background(), cfg.TemporalHostPort, @@ -38,12 +50,15 @@ func main() { w := worker.New(temporalClient, cfg.TemporalOnboardingQueueName, worker.Options{}) - userActivities := activities.NewUserActivites( + userActivities := activity.NewUserActivites( *cfg, ) + githubInstallationActivities := activity.NewGithubInstallationActivities(*githubAppClient) - w.RegisterWorkflow(workflows.CountUsers) + w.RegisterWorkflow(workflow.CountUsers) + w.RegisterWorkflow(workflow.AfterGithubInstallationWorkflow) w.RegisterActivity(userActivities) + w.RegisterActivity(githubInstallationActivities) if err := w.Run(worker.InterruptCh()); err != nil { log.Fatalln("Worker failed to start", err) diff --git a/go.mod b/go.mod index c7ddf584..7511d91d 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( require ( github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9 // indirect + github.com/bradleyfalzon/ghinstallation/v2 v2.16.0 // indirect github.com/cenkalti/backoff/v5 v5.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect @@ -28,8 +29,13 @@ require ( github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/goccy/go-json v0.10.3 // indirect + github.com/gofri/go-github-ratelimit/v2 v2.0.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v4 v4.5.2 // indirect github.com/golang/mock v1.6.0 // indirect + github.com/google/go-github/v72 v72.0.0 // indirect + github.com/google/go-github/v73 v73.0.0 // indirect + github.com/google/go-querystring v1.1.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect diff --git a/go.sum b/go.sum index e6a0866c..2da26c05 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,8 @@ github.com/XSAM/otelsql v0.39.0/go.mod h1:uMOXLUX+wkuAuP0AR3B45NXX7E9lJS2mERa8gq github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9 h1:goHVqTbFX3AIo0tzGr14pgfAW2ZfPChKO21Z9MGf/gk= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/bradleyfalzon/ghinstallation/v2 v2.16.0 h1:B91r9bHtXp/+XRgS5aZm6ZzTdz3ahgJYmkt4xZkgDz8= +github.com/bradleyfalzon/ghinstallation/v2 v2.16.0/go.mod h1:OeVe5ggFzoBnmgitZe/A+BqGOnv1DvU/0uiLQi1wutM= github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7Zss8= github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -54,8 +56,12 @@ github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/gofri/go-github-ratelimit/v2 v2.0.2 h1:gS8wAS1jTmlWGdTjAM7KIpsLjwY1S0S/gKK5hthfSXM= +github.com/gofri/go-github-ratelimit/v2 v2.0.2/go.mod h1:YBQt4gTbdcbMjJFT05YFEaECwH78P5b0IwrnbLiHGdE= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= @@ -68,9 +74,16 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-github/v72 v72.0.0 h1:FcIO37BLoVPBO9igQQ6tStsv2asG4IPcYFi655PPvBM= +github.com/google/go-github/v72 v72.0.0/go.mod h1:WWtw8GMRiL62mvIquf1kO3onRHeWWKmK01qdCY8c5fg= +github.com/google/go-github/v73 v73.0.0 h1:aR+Utnh+Y4mMkS+2qLQwcQ/cF9mOTpdwnzlaw//rG24= +github.com/google/go-github/v73 v73.0.0/go.mod h1:fa6w8+/V+edSU0muqdhCVY7Beh1M8F1IlQPZIANKIYw= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= diff --git a/internal/internal-api/handler/users_count.go b/internal/internal-api/handler/users_count.go index 53841a4e..2dec50bf 100644 --- a/internal/internal-api/handler/users_count.go +++ b/internal/internal-api/handler/users_count.go @@ -8,7 +8,7 @@ import ( "net/http" "github.com/dxta-dev/app/internal/onboarding" - "github.com/dxta-dev/app/internal/onboarding/workflows" + "github.com/dxta-dev/app/internal/onboarding/workflow" "github.com/dxta-dev/app/internal/util" "go.temporal.io/sdk/client" ) @@ -30,7 +30,7 @@ func NewUsers(temporalClient client.Client, config onboarding.Config) *Users { } func (u *Users) UsersCount(w http.ResponseWriter, r *http.Request) { - out, err := workflows.ExecuteCountUsersWorkflow(r.Context(), u.temporalClient, u.config) + out, err := workflow.ExecuteCountUsersWorkflow(r.Context(), u.temporalClient, u.config) if err != nil { log.Fatal(errors.Unwrap(err)) } diff --git a/internal/onboarding/activity/github_installation.go b/internal/onboarding/activity/github_installation.go new file mode 100644 index 00000000..c693d6a0 --- /dev/null +++ b/internal/onboarding/activity/github_installation.go @@ -0,0 +1,32 @@ +package activity + +import ( + "context" + "fmt" + + "github.com/dxta-dev/app/internal/onboarding" +) + +type GithubInstallationActivities struct { + githubAppClient onboarding.GithubAppClient +} + +func NewGithubInstallationActivities(GithubAppClient onboarding.GithubAppClient) *GithubInstallationActivities { + return &GithubInstallationActivities{ + githubAppClient: GithubAppClient, + } +} + +func (gia *GithubInstallationActivities) GetGithubInstallation( + ctx context.Context, + installationId int64, +) (string, error) { + login, err := gia.githubAppClient.GetOrganizationLogin(ctx, installationId) + + if err != nil { + fmt.Printf("Could not retrieve installation. Error: %v", err.Error()) + return "", err + } + + return login, nil +} diff --git a/internal/onboarding/activities/users.go b/internal/onboarding/activity/users.go similarity index 97% rename from internal/onboarding/activities/users.go rename to internal/onboarding/activity/users.go index 6f829bd6..d93ee112 100644 --- a/internal/onboarding/activities/users.go +++ b/internal/onboarding/activity/users.go @@ -1,4 +1,4 @@ -package activities +package activity import ( "context" diff --git a/internal/onboarding/github_config.go b/internal/onboarding/github_config.go new file mode 100644 index 00000000..9b748688 --- /dev/null +++ b/internal/onboarding/github_config.go @@ -0,0 +1,169 @@ +package onboarding + +import ( + "context" + "encoding/base64" + "errors" + "fmt" + "net/http" + "os" + "strconv" + + "github.com/bradleyfalzon/ghinstallation/v2" + "github.com/gofri/go-github-ratelimit/v2/github_ratelimit" + "github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_primary_ratelimit" + "github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_secondary_ratelimit" + "github.com/google/go-github/v73/github" +) + +type GithubConfig struct { + GithubAppId int64 + GithubAppPrivateKey []byte + RoundTripper http.RoundTripper +} + +func LoadGithubConfig() (*GithubConfig, error) { + appIdStr := os.Getenv("GITHUB_APP_ID") + appPrivateKeyStr := os.Getenv("GITHUB_APP_PRIVATE_KEY") + + if appIdStr == "" { + return nil, errors.New("GITHUB_APP_ID not set") + } + + if appPrivateKeyStr == "" { + return nil, errors.New("GITHUB_APP_PRIVATE_KEY not set") + } + + appId, err := strconv.ParseInt(appIdStr, 10, 64) + + if err != nil { + return nil, errors.New("could not parse app id string to int64") + } + + appPrivateKey, err := base64.StdEncoding.DecodeString(appPrivateKeyStr) + + if err != nil { + return nil, errors.New("failed to decode base64 string") + } + + return &GithubConfig{ + GithubAppId: appId, + GithubAppPrivateKey: appPrivateKey, + RoundTripper: http.DefaultTransport, + }, nil +} + +func getInstallationTransport( + tr http.RoundTripper, + installationId int64, + appId int64, + appPrivateKey []byte, +) (http.RoundTripper, error) { + itt, err := ghinstallation.New(tr, appId, installationId, appPrivateKey) + + if err != nil { + return nil, fmt.Errorf("failed to create apps transport: %w", err) + } + + return itt, nil +} + +func getAppTransport( + tr http.RoundTripper, + appId int64, + appPrivateKey []byte, +) (http.RoundTripper, error) { + atr, err := ghinstallation.NewAppsTransport(tr, appId, appPrivateKey) + + if err != nil { + return nil, fmt.Errorf("failed to create apps transport: %w", err) + } + + return atr, nil +} + +func createLimiter(tr http.RoundTripper) http.RoundTripper { + return github_ratelimit.New( + tr, + github_primary_ratelimit.WithLimitDetectedCallback( + func(ctx *github_primary_ratelimit.CallbackContext) { + fmt.Printf( + "Primary rate limit detected: category %s, reset time: %v\n", + ctx.Category, + ctx.ResetTime, + ) + }, + ), + github_secondary_ratelimit.WithLimitDetectedCallback( + func(ctx *github_secondary_ratelimit.CallbackContext) { + fmt.Printf( + "Secondary rate limit detected: reset time: %v, total sleep time: %v\n", + ctx.ResetTime, + ctx.TotalSleepTime, + ) + }, + ), + ) +} + +func NewInstallationClient( + installationId int64, + tr http.RoundTripper, + cfg GithubConfig, +) (*github.Client, error) { + tr, err := getInstallationTransport( + tr, + installationId, + cfg.GithubAppId, + cfg.GithubAppPrivateKey, + ) + + if err != nil { + return nil, err + } + + tr = createLimiter(tr) + + return github.NewClient(&http.Client{Transport: tr}), nil +} + +type GithubAppClient struct { + client *github.Client +} + +func (gac *GithubAppClient) GetOrganizationLogin( + ctx context.Context, + installationID int64, +) (string, error) { + installation, _, error := gac.client.Apps.GetInstallation(ctx, installationID) + if error != nil { + return "", errors.New("failed to get installation: " + error.Error()) + + } + + if installation.Account == nil || installation.Account.Login == nil { + return "", errors.New("installation account or login is nil") + } + + if installation.TargetType == nil || *installation.TargetType != "organization" { + return "", errors.New("installation is not for an organization") + } + + return *installation.Account.Login, nil +} + +func NewAppClient(cfg GithubConfig) (*GithubAppClient, error) { + tr, err := getAppTransport(cfg.RoundTripper, cfg.GithubAppId, cfg.GithubAppPrivateKey) + + if err != nil { + return nil, err + } + + tr = createLimiter(tr) + + client := github.NewClient(&http.Client{Transport: tr}) + + return &GithubAppClient{ + client: client, + }, nil +} diff --git a/internal/onboarding/workflow/after_github_installation.go b/internal/onboarding/workflow/after_github_installation.go new file mode 100644 index 00000000..0bd0e15d --- /dev/null +++ b/internal/onboarding/workflow/after_github_installation.go @@ -0,0 +1,82 @@ +package workflow + +import ( + "context" + "fmt" + "time" + + "github.com/google/go-github/v73/github" + "go.temporal.io/sdk/client" + "go.temporal.io/sdk/temporal" + "go.temporal.io/sdk/workflow" + + "github.com/dxta-dev/app/internal/onboarding/activity" +) + +type AfterGithubInstallationParams struct { + InstallationID int64 + AuthID string + DBURL string +} + +func AfterGithubInstallationWorkflow( + ctx workflow.Context, + params AfterGithubInstallationParams, +) (err error) { + ao := workflow.ActivityOptions{ + StartToCloseTimeout: time.Second * 30, + RetryPolicy: &temporal.RetryPolicy{ + MaximumAttempts: 10, + }, + } + + ctx = workflow.WithActivityOptions(ctx, ao) + + installationId := params.InstallationID + + var installation *github.Installation + err = workflow.ExecuteActivity(ctx, (*activity.GithubInstallationActivities).GetGithubInstallation, installationId). + Get(ctx, &installation) + + if err != nil { + return + } + + return +} + +type ExecuteAfterGithubInstallationParams struct { + TemporalOnboardingQueueName string + InstallationID int64 + AuthID string + DBURL string +} + +func ExecuteAfterGithubInstallationWorkflow( + ctx context.Context, + temporalClient client.Client, + params ExecuteAfterGithubInstallationParams, +) (string, error) { + _, err := temporalClient.ExecuteWorkflow( + ctx, + client.StartWorkflowOptions{ + ID: fmt.Sprintf( + "onboarding-workflow-github-%v-%v", + params.InstallationID, params.AuthID, + ), + TaskQueue: params.TemporalOnboardingQueueName, + }, + AfterGithubInstallationWorkflow, + AfterGithubInstallationParams{ + InstallationID: params.InstallationID, + AuthID: params.AuthID, + DBURL: params.DBURL, + }, + ) + + if err != nil { + return "Unable to execute ", err + } + + return "Success", nil +} diff --git a/internal/onboarding/workflows/count-users.go b/internal/onboarding/workflow/count-users.go similarity index 85% rename from internal/onboarding/workflows/count-users.go rename to internal/onboarding/workflow/count-users.go index c72dfff6..ac34dffc 100644 --- a/internal/onboarding/workflows/count-users.go +++ b/internal/onboarding/workflow/count-users.go @@ -1,11 +1,11 @@ -package workflows +package workflow import ( "context" "fmt" "time" - "github.com/dxta-dev/app/internal/onboarding/activities" + "github.com/dxta-dev/app/internal/onboarding/activity" "go.temporal.io/sdk/workflow" "github.com/dxta-dev/app/internal/onboarding" @@ -20,7 +20,7 @@ func CountUsers(ctx workflow.Context) (int, error) { ctx = workflow.WithActivityOptions(ctx, ao) var count int - err := workflow.ExecuteActivity(ctx, (*activities.UserActivites).CountUsers).Get(ctx, &count) + err := workflow.ExecuteActivity(ctx, (*activity.UserActivites).CountUsers).Get(ctx, &count) if err != nil { return 0, err }