From 9aee9fd42d1fbdee16b6363373c77a27180c9de8 Mon Sep 17 00:00:00 2001 From: stefanskoricdev Date: Thu, 3 Jul 2025 13:10:22 +0200 Subject: [PATCH 1/7] feat: Add get github installation data activity --- cmd/onboarding-worker/main.go | 18 ++- go.mod | 6 + go.sum | 13 +++ .../activities/github_installation.go | 32 +++++ internal/onboarding/activities/users.go | 2 +- internal/onboarding/github_config.go | 110 ++++++++++++++++++ internal/onboarding/workflows/count-users.go | 4 +- .../provision_github_installation_data.go | 77 ++++++++++++ 8 files changed, 257 insertions(+), 5 deletions(-) create mode 100644 internal/onboarding/activities/github_installation.go create mode 100644 internal/onboarding/github_config.go create mode 100644 internal/onboarding/workflows/provision_github_installation_data.go diff --git a/cmd/onboarding-worker/main.go b/cmd/onboarding-worker/main.go index ba6d5b09..15c70af3 100644 --- a/cmd/onboarding-worker/main.go +++ b/cmd/onboarding-worker/main.go @@ -5,7 +5,7 @@ import ( "log" "github.com/dxta-dev/app/internal/onboarding" - "github.com/dxta-dev/app/internal/onboarding/activities" + activity "github.com/dxta-dev/app/internal/onboarding/activities" "github.com/dxta-dev/app/internal/onboarding/workflows" "go.temporal.io/sdk/client" "go.temporal.io/sdk/worker" @@ -17,6 +17,12 @@ func main() { log.Fatalln("Failed to load configuration:", err) } + githubConfig, err := onboarding.LoadGithubConfig() + + if err != nil { + log.Fatalln("Failed to load github configuration:", err) + } + temporalClient, err := client.Dial(client.Options{ HostPort: cfg.TemporalHostPort, Namespace: cfg.TemporalOnboardingNamespace, @@ -26,6 +32,12 @@ func main() { } defer temporalClient.Close() + githubAppClient, err := onboarding.InitAppClient(*githubConfig) + + if err != nil { + log.Fatalf("Unable to init app client: %v", err) + } + err = onboarding.RegisterNamespace( context.Background(), cfg.TemporalHostPort, @@ -38,12 +50,14 @@ func main() { w := worker.New(temporalClient, cfg.TemporalOnboardingQueueName, worker.Options{}) - userActivities := activities.NewUserActivites( + userActivities := activity.NewUserActivites( *cfg, ) + githubInstallationActivities := activity.GithubInstallationActivities(githubAppClient) w.RegisterWorkflow(workflows.CountUsers) w.RegisterActivity(userActivities) + w.RegisterActivity(githubInstallationActivities) if err := w.Run(worker.InterruptCh()); err != nil { log.Fatalln("Worker failed to start", err) diff --git a/go.mod b/go.mod index c7ddf584..7511d91d 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( require ( github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9 // indirect + github.com/bradleyfalzon/ghinstallation/v2 v2.16.0 // indirect github.com/cenkalti/backoff/v5 v5.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect @@ -28,8 +29,13 @@ require ( github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/goccy/go-json v0.10.3 // indirect + github.com/gofri/go-github-ratelimit/v2 v2.0.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v4 v4.5.2 // indirect github.com/golang/mock v1.6.0 // indirect + github.com/google/go-github/v72 v72.0.0 // indirect + github.com/google/go-github/v73 v73.0.0 // indirect + github.com/google/go-querystring v1.1.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect diff --git a/go.sum b/go.sum index e6a0866c..2da26c05 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,8 @@ github.com/XSAM/otelsql v0.39.0/go.mod h1:uMOXLUX+wkuAuP0AR3B45NXX7E9lJS2mERa8gq github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9 h1:goHVqTbFX3AIo0tzGr14pgfAW2ZfPChKO21Z9MGf/gk= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/bradleyfalzon/ghinstallation/v2 v2.16.0 h1:B91r9bHtXp/+XRgS5aZm6ZzTdz3ahgJYmkt4xZkgDz8= +github.com/bradleyfalzon/ghinstallation/v2 v2.16.0/go.mod h1:OeVe5ggFzoBnmgitZe/A+BqGOnv1DvU/0uiLQi1wutM= github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7Zss8= github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -54,8 +56,12 @@ github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/gofri/go-github-ratelimit/v2 v2.0.2 h1:gS8wAS1jTmlWGdTjAM7KIpsLjwY1S0S/gKK5hthfSXM= +github.com/gofri/go-github-ratelimit/v2 v2.0.2/go.mod h1:YBQt4gTbdcbMjJFT05YFEaECwH78P5b0IwrnbLiHGdE= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= @@ -68,9 +74,16 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-github/v72 v72.0.0 h1:FcIO37BLoVPBO9igQQ6tStsv2asG4IPcYFi655PPvBM= +github.com/google/go-github/v72 v72.0.0/go.mod h1:WWtw8GMRiL62mvIquf1kO3onRHeWWKmK01qdCY8c5fg= +github.com/google/go-github/v73 v73.0.0 h1:aR+Utnh+Y4mMkS+2qLQwcQ/cF9mOTpdwnzlaw//rG24= +github.com/google/go-github/v73 v73.0.0/go.mod h1:fa6w8+/V+edSU0muqdhCVY7Beh1M8F1IlQPZIANKIYw= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= diff --git a/internal/onboarding/activities/github_installation.go b/internal/onboarding/activities/github_installation.go new file mode 100644 index 00000000..8e607255 --- /dev/null +++ b/internal/onboarding/activities/github_installation.go @@ -0,0 +1,32 @@ +package activity + +import ( + "context" + "fmt" + + "github.com/google/go-github/v73/github" +) + +type InstallationActivities struct { + GithubAppClient *github.Client +} + +func GithubInstallationActivities(GithubAppClient *github.Client) *InstallationActivities { + return &InstallationActivities{ + GithubAppClient: GithubAppClient, + } +} + +func (a InstallationActivities) GetGithubInstallation( + ctx context.Context, + installationId int64, +) (*github.Installation, error) { + installation, _, err := a.GithubAppClient.Apps.GetInstallation(ctx, installationId) + + if err != nil { + fmt.Printf("Could not retrieve installation. Error: %v", err.Error()) + return nil, err + } + + return installation, nil +} diff --git a/internal/onboarding/activities/users.go b/internal/onboarding/activities/users.go index 6f829bd6..d93ee112 100644 --- a/internal/onboarding/activities/users.go +++ b/internal/onboarding/activities/users.go @@ -1,4 +1,4 @@ -package activities +package activity import ( "context" diff --git a/internal/onboarding/github_config.go b/internal/onboarding/github_config.go new file mode 100644 index 00000000..39b5cc2a --- /dev/null +++ b/internal/onboarding/github_config.go @@ -0,0 +1,110 @@ +package onboarding + +import ( + "encoding/base64" + "errors" + "fmt" + "net/http" + "os" + "strconv" + + "github.com/bradleyfalzon/ghinstallation/v2" + "github.com/gofri/go-github-ratelimit/v2/github_ratelimit" + "github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_primary_ratelimit" + "github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_secondary_ratelimit" + "github.com/google/go-github/v73/github" +) + +type GithubConfig struct { + GithubAppId int64 + GithubAppPrivateKey []byte + RoundTripper http.RoundTripper +} + +func LoadGithubConfig() (*GithubConfig, error) { + appIdStr := os.Getenv("GITHUB_APP_ID") + appPrivateKeyStr := os.Getenv("GITHUB_APP_PRIVATE_KEY") + + if appIdStr == "" { + return nil, errors.New("GITHUB_APP_ID not set") + } + + if appPrivateKeyStr == "" { + return nil, errors.New("GITHUB_APP_PRIVATE_KEY not set") + } + + appId, err := strconv.ParseInt(appIdStr, 10, 64) + + if err != nil { + return nil, errors.New("could not parse app id string to int64") + } + + appPrivateKey, err := base64.StdEncoding.DecodeString(appPrivateKeyStr) + + if err != nil { + return nil, errors.New("failed to decode base64 string") + } + + return &GithubConfig{ + GithubAppId: appId, + GithubAppPrivateKey: appPrivateKey, + RoundTripper: http.DefaultTransport, + }, nil +} + +func getInstallationTransport(tr http.RoundTripper, installationId int64, appId int64, appPrivateKey []byte) (http.RoundTripper, error) { + itt, err := ghinstallation.New(tr, appId, installationId, appPrivateKey) + + if err != nil { + return nil, fmt.Errorf("failed to create apps transport: %w", err) + } + + return itt, nil +} + +func getAppTransport(tr http.RoundTripper, appId int64, appPrivateKey []byte) (http.RoundTripper, error) { + atr, err := ghinstallation.NewAppsTransport(tr, appId, appPrivateKey) + + if err != nil { + return nil, fmt.Errorf("failed to create apps transport: %w", err) + } + + return atr, nil +} + +func createLimiter(tr http.RoundTripper) http.RoundTripper { + return github_ratelimit.New(tr, + github_primary_ratelimit.WithLimitDetectedCallback(func(ctx *github_primary_ratelimit.CallbackContext) { + fmt.Printf("Primary rate limit detected: category %s, reset time: %v\n", ctx.Category, ctx.ResetTime) + }), + github_secondary_ratelimit.WithLimitDetectedCallback(func(ctx *github_secondary_ratelimit.CallbackContext) { + fmt.Printf("Secondary rate limit detected: reset time: %v, total sleep time: %v\n", ctx.ResetTime, ctx.TotalSleepTime) + }), + ) +} + +func NewInstallationClient(installationId int64, tr http.RoundTripper, cfg GithubConfig) (*github.Client, error) { + tr, err := getInstallationTransport(tr, installationId, cfg.GithubAppId, cfg.GithubAppPrivateKey) + + if err != nil { + return nil, err + } + + tr = createLimiter(tr) + + return github.NewClient(&http.Client{Transport: tr}), nil +} + +func InitAppClient(cfg GithubConfig) (*github.Client, error) { + tr, err := getAppTransport(cfg.RoundTripper, cfg.GithubAppId, cfg.GithubAppPrivateKey) + + if err != nil { + return nil, err + } + + tr = createLimiter(tr) + + client := github.NewClient(&http.Client{Transport: tr}) + + return client, nil +} diff --git a/internal/onboarding/workflows/count-users.go b/internal/onboarding/workflows/count-users.go index c72dfff6..8cb89c07 100644 --- a/internal/onboarding/workflows/count-users.go +++ b/internal/onboarding/workflows/count-users.go @@ -5,7 +5,7 @@ import ( "fmt" "time" - "github.com/dxta-dev/app/internal/onboarding/activities" + activity "github.com/dxta-dev/app/internal/onboarding/activities" "go.temporal.io/sdk/workflow" "github.com/dxta-dev/app/internal/onboarding" @@ -20,7 +20,7 @@ func CountUsers(ctx workflow.Context) (int, error) { ctx = workflow.WithActivityOptions(ctx, ao) var count int - err := workflow.ExecuteActivity(ctx, (*activities.UserActivites).CountUsers).Get(ctx, &count) + err := workflow.ExecuteActivity(ctx, (*activity.UserActivites).CountUsers).Get(ctx, &count) if err != nil { return 0, err } diff --git a/internal/onboarding/workflows/provision_github_installation_data.go b/internal/onboarding/workflows/provision_github_installation_data.go new file mode 100644 index 00000000..5faf9033 --- /dev/null +++ b/internal/onboarding/workflows/provision_github_installation_data.go @@ -0,0 +1,77 @@ +package workflows + +import ( + "context" + "fmt" + "time" + + "github.com/google/go-github/v73/github" + "go.temporal.io/sdk/client" + "go.temporal.io/sdk/temporal" + "go.temporal.io/sdk/workflow" + + activity "github.com/dxta-dev/app/internal/onboarding/activities" +) + +type ProvisionGithubInstallationDataParams struct { + InstallationId int64 + AuthId string + DBUrl string +} + +func ProvisionGithubInstallationData(ctx workflow.Context, params ProvisionGithubInstallationDataParams) (count int, err error) { + ao := workflow.ActivityOptions{ + StartToCloseTimeout: time.Second * 30, + RetryPolicy: &temporal.RetryPolicy{ + MaximumAttempts: 10, + }, + } + + ctx = workflow.WithActivityOptions(ctx, ao) + + installationId := params.InstallationId + /* authId := params.AuthId + dbUrl := params.DBUrl */ + + var installation *github.Installation + err = workflow.ExecuteActivity(ctx, (*activity.InstallationActivities).GetGithubInstallation, installationId).Get(ctx, &installation) + + if err != nil { + return + } + + return +} + +type ExecuteGithubInstallationDataProvisionParams struct { + TemporalOnboardingQueueName string + InstallationId int64 + AuthId string + DBUrl string +} + +func ExecuteGithubInstallationDataProvision( + ctx context.Context, + temporalClient client.Client, + params ExecuteGithubInstallationDataProvisionParams, +) (string, error) { + _, err := temporalClient.ExecuteWorkflow( + ctx, + client.StartWorkflowOptions{ + ID: fmt.Sprintf("onboarding-workflow-%v", time.Now().Format("20060102150405")), + TaskQueue: params.TemporalOnboardingQueueName, + }, + ProvisionGithubInstallationData, + ProvisionGithubInstallationDataParams{ + InstallationId: params.InstallationId, + AuthId: params.AuthId, + DBUrl: params.DBUrl, + }, + ) + + if err != nil { + return "Unable to execute ", err + } + + return "Success", nil +} From 9f359cf2be91c4f7eefbf9a6a7618a424ad39421 Mon Sep 17 00:00:00 2001 From: stefanskoricdev Date: Thu, 3 Jul 2025 13:27:49 +0200 Subject: [PATCH 2/7] fix: Register workflow in worker --- cmd/onboarding-worker/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/onboarding-worker/main.go b/cmd/onboarding-worker/main.go index 15c70af3..ea46fa2b 100644 --- a/cmd/onboarding-worker/main.go +++ b/cmd/onboarding-worker/main.go @@ -56,6 +56,7 @@ func main() { githubInstallationActivities := activity.GithubInstallationActivities(githubAppClient) w.RegisterWorkflow(workflows.CountUsers) + w.RegisterWorkflow(workflows.ProvisionGithubInstallationData) w.RegisterActivity(userActivities) w.RegisterActivity(githubInstallationActivities) From 08e909f00d2653f295f4c26798a15d6729ac0cab Mon Sep 17 00:00:00 2001 From: stefanskoricdev Date: Thu, 3 Jul 2025 13:28:33 +0200 Subject: [PATCH 3/7] chore: Remove commented code --- .../workflows/provision_github_installation_data.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/internal/onboarding/workflows/provision_github_installation_data.go b/internal/onboarding/workflows/provision_github_installation_data.go index 5faf9033..de6024e7 100644 --- a/internal/onboarding/workflows/provision_github_installation_data.go +++ b/internal/onboarding/workflows/provision_github_installation_data.go @@ -19,7 +19,7 @@ type ProvisionGithubInstallationDataParams struct { DBUrl string } -func ProvisionGithubInstallationData(ctx workflow.Context, params ProvisionGithubInstallationDataParams) (count int, err error) { +func ProvisionGithubInstallationData(ctx workflow.Context, params ProvisionGithubInstallationDataParams) (err error) { ao := workflow.ActivityOptions{ StartToCloseTimeout: time.Second * 30, RetryPolicy: &temporal.RetryPolicy{ @@ -30,8 +30,6 @@ func ProvisionGithubInstallationData(ctx workflow.Context, params ProvisionGithu ctx = workflow.WithActivityOptions(ctx, ao) installationId := params.InstallationId - /* authId := params.AuthId - dbUrl := params.DBUrl */ var installation *github.Installation err = workflow.ExecuteActivity(ctx, (*activity.InstallationActivities).GetGithubInstallation, installationId).Get(ctx, &installation) From 5683f1e1456002b60891963cdbc495cd54755450 Mon Sep 17 00:00:00 2001 From: stefanskoricdev Date: Thu, 3 Jul 2025 13:45:39 +0200 Subject: [PATCH 4/7] feat: Add facade for github app client interface --- cmd/onboarding-worker/main.go | 2 +- internal/onboarding/activities/github_installation.go | 7 ++++--- internal/onboarding/github_config.go | 11 +++++++++-- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/cmd/onboarding-worker/main.go b/cmd/onboarding-worker/main.go index ea46fa2b..21f38b08 100644 --- a/cmd/onboarding-worker/main.go +++ b/cmd/onboarding-worker/main.go @@ -53,7 +53,7 @@ func main() { userActivities := activity.NewUserActivites( *cfg, ) - githubInstallationActivities := activity.GithubInstallationActivities(githubAppClient) + githubInstallationActivities := activity.GithubInstallationActivities(*githubAppClient) w.RegisterWorkflow(workflows.CountUsers) w.RegisterWorkflow(workflows.ProvisionGithubInstallationData) diff --git a/internal/onboarding/activities/github_installation.go b/internal/onboarding/activities/github_installation.go index 8e607255..35a2cc51 100644 --- a/internal/onboarding/activities/github_installation.go +++ b/internal/onboarding/activities/github_installation.go @@ -4,14 +4,15 @@ import ( "context" "fmt" + "github.com/dxta-dev/app/internal/onboarding" "github.com/google/go-github/v73/github" ) type InstallationActivities struct { - GithubAppClient *github.Client + GithubAppClient onboarding.AppClient } -func GithubInstallationActivities(GithubAppClient *github.Client) *InstallationActivities { +func GithubInstallationActivities(GithubAppClient onboarding.AppClient) *InstallationActivities { return &InstallationActivities{ GithubAppClient: GithubAppClient, } @@ -21,7 +22,7 @@ func (a InstallationActivities) GetGithubInstallation( ctx context.Context, installationId int64, ) (*github.Installation, error) { - installation, _, err := a.GithubAppClient.Apps.GetInstallation(ctx, installationId) + installation, _, err := a.GithubAppClient.GetInstallation(ctx, installationId) if err != nil { fmt.Printf("Could not retrieve installation. Error: %v", err.Error()) diff --git a/internal/onboarding/github_config.go b/internal/onboarding/github_config.go index 39b5cc2a..4a546ecb 100644 --- a/internal/onboarding/github_config.go +++ b/internal/onboarding/github_config.go @@ -1,6 +1,7 @@ package onboarding import ( + "context" "encoding/base64" "errors" "fmt" @@ -95,7 +96,11 @@ func NewInstallationClient(installationId int64, tr http.RoundTripper, cfg Githu return github.NewClient(&http.Client{Transport: tr}), nil } -func InitAppClient(cfg GithubConfig) (*github.Client, error) { +type AppClient struct { + GetInstallation func(ctx context.Context, id int64) (*github.Installation, *github.Response, error) +} + +func InitAppClient(cfg GithubConfig) (*AppClient, error) { tr, err := getAppTransport(cfg.RoundTripper, cfg.GithubAppId, cfg.GithubAppPrivateKey) if err != nil { @@ -106,5 +111,7 @@ func InitAppClient(cfg GithubConfig) (*github.Client, error) { client := github.NewClient(&http.Client{Transport: tr}) - return client, nil + return &AppClient{ + GetInstallation: client.Apps.GetInstallation, + }, nil } From 09399e73751469d712aa21bdf242517b035eba0f Mon Sep 17 00:00:00 2001 From: David Abram Date: Thu, 3 Jul 2025 14:32:11 +0200 Subject: [PATCH 5/7] refactor --- .../activities/github_installation.go | 21 +++-- internal/onboarding/github_config.go | 84 +++++++++++++++---- .../provision_github_installation_data.go | 13 ++- 3 files changed, 88 insertions(+), 30 deletions(-) diff --git a/internal/onboarding/activities/github_installation.go b/internal/onboarding/activities/github_installation.go index 35a2cc51..0583f6a7 100644 --- a/internal/onboarding/activities/github_installation.go +++ b/internal/onboarding/activities/github_installation.go @@ -5,29 +5,28 @@ import ( "fmt" "github.com/dxta-dev/app/internal/onboarding" - "github.com/google/go-github/v73/github" ) -type InstallationActivities struct { - GithubAppClient onboarding.AppClient +type GithubActivities struct { + githubAppClient onboarding.GithubAppClient } -func GithubInstallationActivities(GithubAppClient onboarding.AppClient) *InstallationActivities { - return &InstallationActivities{ - GithubAppClient: GithubAppClient, +func NewGithubInstallationActivities(GithubAppClient onboarding.GithubAppClient) *GithubActivities { + return &GithubActivities{ + githubAppClient: GithubAppClient, } } -func (a InstallationActivities) GetGithubInstallation( +func (ga *GithubActivities) GetGithubInstallation( ctx context.Context, installationId int64, -) (*github.Installation, error) { - installation, _, err := a.GithubAppClient.GetInstallation(ctx, installationId) +) (string, error) { + login, err := ga.githubAppClient.GetOrganizationLogin(ctx, installationId) if err != nil { fmt.Printf("Could not retrieve installation. Error: %v", err.Error()) - return nil, err + return "", err } - return installation, nil + return login, nil } diff --git a/internal/onboarding/github_config.go b/internal/onboarding/github_config.go index 4a546ecb..7c196b49 100644 --- a/internal/onboarding/github_config.go +++ b/internal/onboarding/github_config.go @@ -53,7 +53,12 @@ func LoadGithubConfig() (*GithubConfig, error) { }, nil } -func getInstallationTransport(tr http.RoundTripper, installationId int64, appId int64, appPrivateKey []byte) (http.RoundTripper, error) { +func getInstallationTransport( + tr http.RoundTripper, + installationId int64, + appId int64, + appPrivateKey []byte, +) (http.RoundTripper, error) { itt, err := ghinstallation.New(tr, appId, installationId, appPrivateKey) if err != nil { @@ -63,7 +68,11 @@ func getInstallationTransport(tr http.RoundTripper, installationId int64, appId return itt, nil } -func getAppTransport(tr http.RoundTripper, appId int64, appPrivateKey []byte) (http.RoundTripper, error) { +func getAppTransport( + tr http.RoundTripper, + appId int64, + appPrivateKey []byte, +) (http.RoundTripper, error) { atr, err := ghinstallation.NewAppsTransport(tr, appId, appPrivateKey) if err != nil { @@ -74,18 +83,40 @@ func getAppTransport(tr http.RoundTripper, appId int64, appPrivateKey []byte) (h } func createLimiter(tr http.RoundTripper) http.RoundTripper { - return github_ratelimit.New(tr, - github_primary_ratelimit.WithLimitDetectedCallback(func(ctx *github_primary_ratelimit.CallbackContext) { - fmt.Printf("Primary rate limit detected: category %s, reset time: %v\n", ctx.Category, ctx.ResetTime) - }), - github_secondary_ratelimit.WithLimitDetectedCallback(func(ctx *github_secondary_ratelimit.CallbackContext) { - fmt.Printf("Secondary rate limit detected: reset time: %v, total sleep time: %v\n", ctx.ResetTime, ctx.TotalSleepTime) - }), + return github_ratelimit.New( + tr, + github_primary_ratelimit.WithLimitDetectedCallback( + func(ctx *github_primary_ratelimit.CallbackContext) { + fmt.Printf( + "Primary rate limit detected: category %s, reset time: %v\n", + ctx.Category, + ctx.ResetTime, + ) + }, + ), + github_secondary_ratelimit.WithLimitDetectedCallback( + func(ctx *github_secondary_ratelimit.CallbackContext) { + fmt.Printf( + "Secondary rate limit detected: reset time: %v, total sleep time: %v\n", + ctx.ResetTime, + ctx.TotalSleepTime, + ) + }, + ), ) } -func NewInstallationClient(installationId int64, tr http.RoundTripper, cfg GithubConfig) (*github.Client, error) { - tr, err := getInstallationTransport(tr, installationId, cfg.GithubAppId, cfg.GithubAppPrivateKey) +func NewInstallationClient( + installationId int64, + tr http.RoundTripper, + cfg GithubConfig, +) (*github.Client, error) { + tr, err := getInstallationTransport( + tr, + installationId, + cfg.GithubAppId, + cfg.GithubAppPrivateKey, + ) if err != nil { return nil, err @@ -96,11 +127,32 @@ func NewInstallationClient(installationId int64, tr http.RoundTripper, cfg Githu return github.NewClient(&http.Client{Transport: tr}), nil } -type AppClient struct { - GetInstallation func(ctx context.Context, id int64) (*github.Installation, *github.Response, error) +type GithubAppClient struct { + client *github.Client +} + +func (gac *GithubAppClient) GetOrganizationLogin( + ctx context.Context, + installationID int64, +) (string, error) { + installation, _, error := gac.client.Apps.GetInstallation(ctx, installationID) + if error != nil { + return "", errors.New("failed to get installation: " + error.Error()) + + } + + if installation.Account == nil || installation.Account.Login == nil { + return "", errors.New("installation account or login is nil") + } + + if installation.TargetType == nil || *installation.TargetType != "organization" { + return "", errors.New("installation is not for an organization") + } + + return *installation.Account.Login, nil } -func InitAppClient(cfg GithubConfig) (*AppClient, error) { +func InitAppClient(cfg GithubConfig) (*GithubAppClient, error) { tr, err := getAppTransport(cfg.RoundTripper, cfg.GithubAppId, cfg.GithubAppPrivateKey) if err != nil { @@ -111,7 +163,7 @@ func InitAppClient(cfg GithubConfig) (*AppClient, error) { client := github.NewClient(&http.Client{Transport: tr}) - return &AppClient{ - GetInstallation: client.Apps.GetInstallation, + return &GithubAppClient{ + client: client, }, nil } diff --git a/internal/onboarding/workflows/provision_github_installation_data.go b/internal/onboarding/workflows/provision_github_installation_data.go index de6024e7..8c031af1 100644 --- a/internal/onboarding/workflows/provision_github_installation_data.go +++ b/internal/onboarding/workflows/provision_github_installation_data.go @@ -19,7 +19,10 @@ type ProvisionGithubInstallationDataParams struct { DBUrl string } -func ProvisionGithubInstallationData(ctx workflow.Context, params ProvisionGithubInstallationDataParams) (err error) { +func ProvisionGithubInstallationData( + ctx workflow.Context, + params ProvisionGithubInstallationDataParams, +) (err error) { ao := workflow.ActivityOptions{ StartToCloseTimeout: time.Second * 30, RetryPolicy: &temporal.RetryPolicy{ @@ -32,7 +35,8 @@ func ProvisionGithubInstallationData(ctx workflow.Context, params ProvisionGithu installationId := params.InstallationId var installation *github.Installation - err = workflow.ExecuteActivity(ctx, (*activity.InstallationActivities).GetGithubInstallation, installationId).Get(ctx, &installation) + err = workflow.ExecuteActivity(ctx, (*activity.GithubActivities).GetGithubInstallation, installationId). + Get(ctx, &installation) if err != nil { return @@ -56,7 +60,10 @@ func ExecuteGithubInstallationDataProvision( _, err := temporalClient.ExecuteWorkflow( ctx, client.StartWorkflowOptions{ - ID: fmt.Sprintf("onboarding-workflow-%v", time.Now().Format("20060102150405")), + ID: fmt.Sprintf( + "onboarding-workflow-github-%v", + time.Now().Format("20060102150405"), + ), TaskQueue: params.TemporalOnboardingQueueName, }, ProvisionGithubInstallationData, From d717a987927828423388a77f53735133036f4047 Mon Sep 17 00:00:00 2001 From: David Abram Date: Thu, 3 Jul 2025 14:33:31 +0200 Subject: [PATCH 6/7] fix --- cmd/onboarding-worker/main.go | 8 ++++---- .../{activities => activity}/github_installation.go | 0 internal/onboarding/{activities => activity}/users.go | 0 .../onboarding/{workflows => workflow}/count-users.go | 4 ++-- .../provision_github_installation_data.go | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) rename internal/onboarding/{activities => activity}/github_installation.go (100%) rename internal/onboarding/{activities => activity}/users.go (100%) rename internal/onboarding/{workflows => workflow}/count-users.go (92%) rename internal/onboarding/{workflows => workflow}/provision_github_installation_data.go (95%) diff --git a/cmd/onboarding-worker/main.go b/cmd/onboarding-worker/main.go index 21f38b08..3b60570f 100644 --- a/cmd/onboarding-worker/main.go +++ b/cmd/onboarding-worker/main.go @@ -5,8 +5,8 @@ import ( "log" "github.com/dxta-dev/app/internal/onboarding" - activity "github.com/dxta-dev/app/internal/onboarding/activities" - "github.com/dxta-dev/app/internal/onboarding/workflows" + "github.com/dxta-dev/app/internal/onboarding/activity" + "github.com/dxta-dev/app/internal/onboarding/workflow" "go.temporal.io/sdk/client" "go.temporal.io/sdk/worker" ) @@ -55,8 +55,8 @@ func main() { ) githubInstallationActivities := activity.GithubInstallationActivities(*githubAppClient) - w.RegisterWorkflow(workflows.CountUsers) - w.RegisterWorkflow(workflows.ProvisionGithubInstallationData) + w.RegisterWorkflow(workflow.CountUsers) + w.RegisterWorkflow(workflow.ProvisionGithubInstallationData) w.RegisterActivity(userActivities) w.RegisterActivity(githubInstallationActivities) diff --git a/internal/onboarding/activities/github_installation.go b/internal/onboarding/activity/github_installation.go similarity index 100% rename from internal/onboarding/activities/github_installation.go rename to internal/onboarding/activity/github_installation.go diff --git a/internal/onboarding/activities/users.go b/internal/onboarding/activity/users.go similarity index 100% rename from internal/onboarding/activities/users.go rename to internal/onboarding/activity/users.go diff --git a/internal/onboarding/workflows/count-users.go b/internal/onboarding/workflow/count-users.go similarity index 92% rename from internal/onboarding/workflows/count-users.go rename to internal/onboarding/workflow/count-users.go index 8cb89c07..ac34dffc 100644 --- a/internal/onboarding/workflows/count-users.go +++ b/internal/onboarding/workflow/count-users.go @@ -1,11 +1,11 @@ -package workflows +package workflow import ( "context" "fmt" "time" - activity "github.com/dxta-dev/app/internal/onboarding/activities" + "github.com/dxta-dev/app/internal/onboarding/activity" "go.temporal.io/sdk/workflow" "github.com/dxta-dev/app/internal/onboarding" diff --git a/internal/onboarding/workflows/provision_github_installation_data.go b/internal/onboarding/workflow/provision_github_installation_data.go similarity index 95% rename from internal/onboarding/workflows/provision_github_installation_data.go rename to internal/onboarding/workflow/provision_github_installation_data.go index 8c031af1..f6e557aa 100644 --- a/internal/onboarding/workflows/provision_github_installation_data.go +++ b/internal/onboarding/workflow/provision_github_installation_data.go @@ -1,4 +1,4 @@ -package workflows +package workflow import ( "context" @@ -10,7 +10,7 @@ import ( "go.temporal.io/sdk/temporal" "go.temporal.io/sdk/workflow" - activity "github.com/dxta-dev/app/internal/onboarding/activities" + "github.com/dxta-dev/app/internal/onboarding/activity" ) type ProvisionGithubInstallationDataParams struct { From 7bf0b1f97763dbe3548450f26610174044e6b5a1 Mon Sep 17 00:00:00 2001 From: stefanskoricdev Date: Thu, 3 Jul 2025 15:03:43 +0200 Subject: [PATCH 7/7] fix naming --- cmd/onboarding-worker/main.go | 6 +-- internal/internal-api/handler/users_count.go | 4 +- .../activity/github_installation.go | 10 ++--- internal/onboarding/github_config.go | 2 +- ...n_data.go => after_github_installation.go} | 42 +++++++++---------- 5 files changed, 32 insertions(+), 32 deletions(-) rename internal/onboarding/workflow/{provision_github_installation_data.go => after_github_installation.go} (51%) diff --git a/cmd/onboarding-worker/main.go b/cmd/onboarding-worker/main.go index 3b60570f..46b3f915 100644 --- a/cmd/onboarding-worker/main.go +++ b/cmd/onboarding-worker/main.go @@ -32,7 +32,7 @@ func main() { } defer temporalClient.Close() - githubAppClient, err := onboarding.InitAppClient(*githubConfig) + githubAppClient, err := onboarding.NewAppClient(*githubConfig) if err != nil { log.Fatalf("Unable to init app client: %v", err) @@ -53,10 +53,10 @@ func main() { userActivities := activity.NewUserActivites( *cfg, ) - githubInstallationActivities := activity.GithubInstallationActivities(*githubAppClient) + githubInstallationActivities := activity.NewGithubInstallationActivities(*githubAppClient) w.RegisterWorkflow(workflow.CountUsers) - w.RegisterWorkflow(workflow.ProvisionGithubInstallationData) + w.RegisterWorkflow(workflow.AfterGithubInstallationWorkflow) w.RegisterActivity(userActivities) w.RegisterActivity(githubInstallationActivities) diff --git a/internal/internal-api/handler/users_count.go b/internal/internal-api/handler/users_count.go index 53841a4e..2dec50bf 100644 --- a/internal/internal-api/handler/users_count.go +++ b/internal/internal-api/handler/users_count.go @@ -8,7 +8,7 @@ import ( "net/http" "github.com/dxta-dev/app/internal/onboarding" - "github.com/dxta-dev/app/internal/onboarding/workflows" + "github.com/dxta-dev/app/internal/onboarding/workflow" "github.com/dxta-dev/app/internal/util" "go.temporal.io/sdk/client" ) @@ -30,7 +30,7 @@ func NewUsers(temporalClient client.Client, config onboarding.Config) *Users { } func (u *Users) UsersCount(w http.ResponseWriter, r *http.Request) { - out, err := workflows.ExecuteCountUsersWorkflow(r.Context(), u.temporalClient, u.config) + out, err := workflow.ExecuteCountUsersWorkflow(r.Context(), u.temporalClient, u.config) if err != nil { log.Fatal(errors.Unwrap(err)) } diff --git a/internal/onboarding/activity/github_installation.go b/internal/onboarding/activity/github_installation.go index 0583f6a7..c693d6a0 100644 --- a/internal/onboarding/activity/github_installation.go +++ b/internal/onboarding/activity/github_installation.go @@ -7,21 +7,21 @@ import ( "github.com/dxta-dev/app/internal/onboarding" ) -type GithubActivities struct { +type GithubInstallationActivities struct { githubAppClient onboarding.GithubAppClient } -func NewGithubInstallationActivities(GithubAppClient onboarding.GithubAppClient) *GithubActivities { - return &GithubActivities{ +func NewGithubInstallationActivities(GithubAppClient onboarding.GithubAppClient) *GithubInstallationActivities { + return &GithubInstallationActivities{ githubAppClient: GithubAppClient, } } -func (ga *GithubActivities) GetGithubInstallation( +func (gia *GithubInstallationActivities) GetGithubInstallation( ctx context.Context, installationId int64, ) (string, error) { - login, err := ga.githubAppClient.GetOrganizationLogin(ctx, installationId) + login, err := gia.githubAppClient.GetOrganizationLogin(ctx, installationId) if err != nil { fmt.Printf("Could not retrieve installation. Error: %v", err.Error()) diff --git a/internal/onboarding/github_config.go b/internal/onboarding/github_config.go index 7c196b49..9b748688 100644 --- a/internal/onboarding/github_config.go +++ b/internal/onboarding/github_config.go @@ -152,7 +152,7 @@ func (gac *GithubAppClient) GetOrganizationLogin( return *installation.Account.Login, nil } -func InitAppClient(cfg GithubConfig) (*GithubAppClient, error) { +func NewAppClient(cfg GithubConfig) (*GithubAppClient, error) { tr, err := getAppTransport(cfg.RoundTripper, cfg.GithubAppId, cfg.GithubAppPrivateKey) if err != nil { diff --git a/internal/onboarding/workflow/provision_github_installation_data.go b/internal/onboarding/workflow/after_github_installation.go similarity index 51% rename from internal/onboarding/workflow/provision_github_installation_data.go rename to internal/onboarding/workflow/after_github_installation.go index f6e557aa..0bd0e15d 100644 --- a/internal/onboarding/workflow/provision_github_installation_data.go +++ b/internal/onboarding/workflow/after_github_installation.go @@ -13,15 +13,15 @@ import ( "github.com/dxta-dev/app/internal/onboarding/activity" ) -type ProvisionGithubInstallationDataParams struct { - InstallationId int64 - AuthId string - DBUrl string +type AfterGithubInstallationParams struct { + InstallationID int64 + AuthID string + DBURL string } -func ProvisionGithubInstallationData( +func AfterGithubInstallationWorkflow( ctx workflow.Context, - params ProvisionGithubInstallationDataParams, + params AfterGithubInstallationParams, ) (err error) { ao := workflow.ActivityOptions{ StartToCloseTimeout: time.Second * 30, @@ -32,10 +32,10 @@ func ProvisionGithubInstallationData( ctx = workflow.WithActivityOptions(ctx, ao) - installationId := params.InstallationId + installationId := params.InstallationID var installation *github.Installation - err = workflow.ExecuteActivity(ctx, (*activity.GithubActivities).GetGithubInstallation, installationId). + err = workflow.ExecuteActivity(ctx, (*activity.GithubInstallationActivities).GetGithubInstallation, installationId). Get(ctx, &installation) if err != nil { @@ -45,32 +45,32 @@ func ProvisionGithubInstallationData( return } -type ExecuteGithubInstallationDataProvisionParams struct { +type ExecuteAfterGithubInstallationParams struct { TemporalOnboardingQueueName string - InstallationId int64 - AuthId string - DBUrl string + InstallationID int64 + AuthID string + DBURL string } -func ExecuteGithubInstallationDataProvision( +func ExecuteAfterGithubInstallationWorkflow( ctx context.Context, temporalClient client.Client, - params ExecuteGithubInstallationDataProvisionParams, + params ExecuteAfterGithubInstallationParams, ) (string, error) { _, err := temporalClient.ExecuteWorkflow( ctx, client.StartWorkflowOptions{ ID: fmt.Sprintf( - "onboarding-workflow-github-%v", - time.Now().Format("20060102150405"), + "onboarding-workflow-github-%v-%v", + params.InstallationID, params.AuthID, ), TaskQueue: params.TemporalOnboardingQueueName, }, - ProvisionGithubInstallationData, - ProvisionGithubInstallationDataParams{ - InstallationId: params.InstallationId, - AuthId: params.AuthId, - DBUrl: params.DBUrl, + AfterGithubInstallationWorkflow, + AfterGithubInstallationParams{ + InstallationID: params.InstallationID, + AuthID: params.AuthID, + DBURL: params.DBURL, }, )