diff --git a/cmd/onboarding-worker/main.go b/cmd/onboarding-worker/main.go index 46b3f915..1ef93ac4 100644 --- a/cmd/onboarding-worker/main.go +++ b/cmd/onboarding-worker/main.go @@ -3,6 +3,7 @@ package main import ( "context" "log" + "sync" "github.com/dxta-dev/app/internal/onboarding" "github.com/dxta-dev/app/internal/onboarding/activity" @@ -48,17 +49,21 @@ func main() { log.Fatalln("Failed to register Temporal namespace:", err) } + tenantDBConnections := sync.Map{} + w := worker.New(temporalClient, cfg.TemporalOnboardingQueueName, worker.Options{}) userActivities := activity.NewUserActivites( *cfg, ) githubInstallationActivities := activity.NewGithubInstallationActivities(*githubAppClient) + tenantActivities := activity.NewTenantActivities(&tenantDBConnections) w.RegisterWorkflow(workflow.CountUsers) w.RegisterWorkflow(workflow.AfterGithubInstallationWorkflow) w.RegisterActivity(userActivities) w.RegisterActivity(githubInstallationActivities) + w.RegisterActivity(tenantActivities) if err := w.Run(worker.InterruptCh()); err != nil { log.Fatalln("Worker failed to start", err) diff --git a/db/migrations/tenant/migrations/1750420632_init.up.sql b/db/migrations/tenant/migrations/1750420632_init.up.sql index b307836a..80ddde16 100644 --- a/db/migrations/tenant/migrations/1750420632_init.up.sql +++ b/db/migrations/tenant/migrations/1750420632_init.up.sql @@ -24,7 +24,7 @@ CREATE TABLE "teams" ( "created_at" DATETIME NOT NULL DEFAULT (datetime('now')), "updated_at" DATETIME NOT NULL DEFAULT (datetime('now')), "deleted_at" DATETIME DEFAULT NULL, - FOREIGN KEY ("organization_id") references "organizations" ("id") + FOREIGN KEY ("organization_id") REFERENCES "organizations" ("id") ); CREATE TABLE "members" ( @@ -49,7 +49,7 @@ CREATE TABLE "teams__members" ( CREATE TABLE "github_organizations" ( "id" INTEGER PRIMARY KEY NOT NULL, - "external_id" INTEGER NOT NULL, + "external_id" INTEGER NOT NULL UNIQUE, "name" TEXT NOT NULL, "github_app_installation_id" INTEGER NOT NULL, "created_at" DATETIME NOT NULL DEFAULT (datetime('now')), @@ -69,7 +69,7 @@ CREATE TABLE "organizations__github_organizations" ( CREATE TABLE "github_members" ( "id" INTEGER PRIMARY KEY NOT NULL, - "external_id" INTEGER NOT NULL, + "external_id" INTEGER NOT NULL UNIQUE, "username" TEXT NOT NULL, "email" TEXT DEFAULT NULL, "member_id" INTEGER NULL, @@ -81,7 +81,7 @@ CREATE TABLE "github_members" ( CREATE TABLE "github_teams" ( "id" INTEGER PRIMARY KEY NOT NULL, - "external_id" INTEGER NOT NULL, + "external_id" INTEGER NOT NULL UNIQUE, "name" TEXT NOT NULL, "github_organization_id" INTEGER NOT NULL, "created_at" DATETIME NOT NULL DEFAULT (datetime('now')), @@ -161,4 +161,4 @@ BEGIN UPDATE "github_teams" SET updated_at = datetime('now') WHERE id = OLD.id; -END; +END; \ No newline at end of file diff --git a/db/migrations/tenant/migrations/schema.sql b/db/migrations/tenant/migrations/schema.sql index 3396232a..4c211f2e 100644 --- a/db/migrations/tenant/migrations/schema.sql +++ b/db/migrations/tenant/migrations/schema.sql @@ -9,10 +9,10 @@ CREATE TABLE "organizations" ("id" INTEGER PRIMARY KEY NOT NULL, "name" TEXT NOT CREATE TABLE "teams" ("id" INTEGER PRIMARY KEY NOT NULL, "name" TEXT NOT NULL, "organization_id" INTEGER NOT NULL, "created_at" DATETIME NOT NULL DEFAULT (datetime ('now')), "updated_at" DATETIME NOT NULL DEFAULT (datetime ('now')), "deleted_at" DATETIME DEFAULT NULL, FOREIGN KEY ("organization_id") REFERENCES "organizations" ("id")); CREATE TABLE "members" ("id" INTEGER PRIMARY KEY NOT NULL, "name" TEXT NOT NULL, "email" TEXT DEFAULT NULL, "created_at" DATETIME NOT NULL DEFAULT (datetime ('now')), "updated_at" DATETIME NOT NULL DEFAULT (datetime ('now')), "deleted_at" DATETIME DEFAULT NULL); CREATE TABLE "teams__members" ("team_id" INTEGER NOT NULL, "member_id" INTEGER NOT NULL, "created_at" DATETIME NOT NULL DEFAULT (datetime ('now')), PRIMARY KEY ("team_id", "member_id"), FOREIGN KEY ("team_id") REFERENCES "teams" ("id"), FOREIGN KEY ("member_id") REFERENCES "members" ("id")); -CREATE TABLE "github_organizations" ("id" INTEGER PRIMARY KEY NOT NULL, "external_id" INTEGER NOT NULL, "name" TEXT NOT NULL, "github_app_installation_id" INTEGER NOT NULL, "created_at" DATETIME NOT NULL DEFAULT (datetime ('now')), "updated_at" DATETIME NOT NULL DEFAULT (datetime ('now')), "deleted_at" DATETIME DEFAULT NULL); +CREATE TABLE "github_organizations" ("id" INTEGER PRIMARY KEY NOT NULL, "external_id" INTEGER NOT NULL UNIQUE, "name" TEXT NOT NULL, "github_app_installation_id" INTEGER NOT NULL, "created_at" DATETIME NOT NULL DEFAULT (datetime ('now')), "updated_at" DATETIME NOT NULL DEFAULT (datetime ('now')), "deleted_at" DATETIME DEFAULT NULL); CREATE TABLE "organizations__github_organizations" ("organization_id" INTEGER NOT NULL, "github_organization_id" INTEGER NOT NULL, "created_at" DATETIME NOT NULL DEFAULT (datetime ('now')), PRIMARY KEY ("organization_id", "github_organization_id"), FOREIGN KEY ("organization_id") REFERENCES "organizations" ("id"), FOREIGN KEY ("github_organization_id") REFERENCES "github_organizations" ("id")); -CREATE TABLE "github_members" ("id" INTEGER PRIMARY KEY NOT NULL, "external_id" INTEGER NOT NULL, "username" TEXT NOT NULL, "email" TEXT DEFAULT NULL, "member_id" INTEGER NULL, "created_at" DATETIME NOT NULL DEFAULT (datetime ('now')), "updated_at" DATETIME NOT NULL DEFAULT (datetime ('now')), "deleted_at" DATETIME DEFAULT NULL, FOREIGN KEY ("member_id") REFERENCES "members" ("id")); -CREATE TABLE "github_teams" ("id" INTEGER PRIMARY KEY NOT NULL, "external_id" INTEGER NOT NULL, "name" TEXT NOT NULL, "github_organization_id" INTEGER NOT NULL, "created_at" DATETIME NOT NULL DEFAULT (datetime ('now')), "updated_at" DATETIME NOT NULL DEFAULT (datetime ('now')), "deleted_at" DATETIME DEFAULT NULL, FOREIGN KEY ("github_organization_id") REFERENCES "github_organizations" ("id")); +CREATE TABLE "github_members" ("id" INTEGER PRIMARY KEY NOT NULL, "external_id" INTEGER NOT NULL UNIQUE, "username" TEXT NOT NULL, "email" TEXT DEFAULT NULL, "member_id" INTEGER NULL, "created_at" DATETIME NOT NULL DEFAULT (datetime ('now')), "updated_at" DATETIME NOT NULL DEFAULT (datetime ('now')), "deleted_at" DATETIME DEFAULT NULL, FOREIGN KEY ("member_id") REFERENCES "members" ("id")); +CREATE TABLE "github_teams" ("id" INTEGER PRIMARY KEY NOT NULL, "external_id" INTEGER NOT NULL UNIQUE, "name" TEXT NOT NULL, "github_organization_id" INTEGER NOT NULL, "created_at" DATETIME NOT NULL DEFAULT (datetime ('now')), "updated_at" DATETIME NOT NULL DEFAULT (datetime ('now')), "deleted_at" DATETIME DEFAULT NULL, FOREIGN KEY ("github_organization_id") REFERENCES "github_organizations" ("id")); CREATE TABLE "github_teams__github_members" ("github_team_id" INTEGER NOT NULL, "github_member_id" INTEGER NOT NULL, PRIMARY KEY ("github_team_id", "github_member_id"), FOREIGN KEY ("github_team_id") REFERENCES "github_teams" ("id"), FOREIGN KEY ("github_member_id") REFERENCES "github_members" ("id")); CREATE TRIGGER "settings_set_updated_at" AFTER UPDATE ON "settings" FOR EACH ROW BEGIN UPDATE "settings" SET updated_at = datetime ('now') WHERE id = OLD.id; diff --git a/internal/internal-api/data/tenantDB.go b/internal/internal-api/data/tenantDB.go index 3119ac45..b855bd34 100644 --- a/internal/internal-api/data/tenantDB.go +++ b/internal/internal-api/data/tenantDB.go @@ -1,8 +1,9 @@ package data import ( + "context" "database/sql" - "fmt" + "errors" "os" "github.com/dxta-dev/app/internal/otel" @@ -12,7 +13,7 @@ type TenantDB struct { DB *sql.DB } -func NewTenantDB(dbUrl string) (TenantDB, error) { +func NewTenantDB(dbUrl string, ctx context.Context) (TenantDB, error) { driverName := otel.GetDriverName() devToken := os.Getenv("DXTA_DEV_GROUP_TOKEN") @@ -22,12 +23,11 @@ func NewTenantDB(dbUrl string) (TenantDB, error) { ) if err != nil { - fmt.Printf( - "Issue while opening tenant database connection. DBUrl: %s Error: %s", - dbUrl, - err.Error(), - ) - return TenantDB{}, err + return TenantDB{}, errors.New("failed to open tenant db connection " + err.Error()) + } + + if err := tenantDB.PingContext(ctx); err != nil { + return TenantDB{}, errors.New("failed to verify tenant db connection " + err.Error()) } return TenantDB{ diff --git a/internal/internal-api/internal-api.go b/internal/internal-api/internal-api.go index 9d1bd9be..933bef66 100644 --- a/internal/internal-api/internal-api.go +++ b/internal/internal-api/internal-api.go @@ -59,8 +59,8 @@ func GetTenantDBUrlByAuthId(ctx context.Context, authId string) (TenantDBData, e return tenantData, nil } -func InternalApiState(dbUrl string, r *http.Request) (State, error) { - tenantDB, err := data.NewTenantDB(dbUrl) +func InternalApiState(ctx context.Context, dbUrl string, r *http.Request) (State, error) { + tenantDB, err := data.NewTenantDB(dbUrl, ctx) if err != nil { return State{}, err diff --git a/internal/onboarding/activity/github_installation.go b/internal/onboarding/activity/github_installation.go index c693d6a0..60be4860 100644 --- a/internal/onboarding/activity/github_installation.go +++ b/internal/onboarding/activity/github_installation.go @@ -17,16 +17,24 @@ func NewGithubInstallationActivities(GithubAppClient onboarding.GithubAppClient) } } -func (gia *GithubInstallationActivities) GetGithubInstallation( +type GithubInstallationOrganization struct { + OrganizationID int64 + OrganizationLogin string +} + +func (gia *GithubInstallationActivities) GetInstallationOrganization( ctx context.Context, installationId int64, -) (string, error) { - login, err := gia.githubAppClient.GetOrganizationLogin(ctx, installationId) +) (*GithubInstallationOrganization, error) { + account, err := gia.githubAppClient.GetInstallationAccount(ctx, installationId) if err != nil { fmt.Printf("Could not retrieve installation. Error: %v", err.Error()) - return "", err + return nil, err } - return login, nil + return &GithubInstallationOrganization{ + OrganizationID: account.ID, + OrganizationLogin: account.Login, + }, nil } diff --git a/internal/onboarding/activity/github_organization.go b/internal/onboarding/activity/github_organization.go new file mode 100644 index 00000000..bf7a5824 --- /dev/null +++ b/internal/onboarding/activity/github_organization.go @@ -0,0 +1,70 @@ +package activity + +import ( + "context" + "errors" + + "github.com/dxta-dev/app/internal/onboarding" +) + +func (ta *TenantActivities) UpsertGithubOrganization( + ctx context.Context, + DBURL string, + installationId int64, + installationOrgName string, + installationOrgId int64, + organizationId int64, +) (res int64, err error) { + db, err := onboarding.GetCachedTenantDB(ta.DBConnections, DBURL, ctx) + + if err != nil { + return 0, err + } + + tx, err := db.BeginTx(ctx, nil) + + if err != nil { + return 0, err + } + + defer func() { + if err != nil { + _ = tx.Rollback() + } + }() + + rows := tx.QueryRowContext(ctx, ` + INSERT INTO github_organizations + (github_app_installation_id, name, external_id) + VALUES + (?, ?, ?) + ON CONFLICT (external_id) + DO UPDATE SET + github_app_installation_id = excluded.github_app_installation_id, + name = excluded.name + RETURNING id`, + installationId, installationOrgName, installationOrgId) + + var githubOrganizationId int64 + + if err = rows.Scan(&githubOrganizationId); err != nil { + return 0, errors.New("failed to upsert github_organizations: " + err.Error()) + } + + _, err = tx.Exec(` + INSERT OR IGNORE INTO 'organizations__github_organizations' + ('organization_id', 'github_organization_id') + VALUES + (?, ?)`, + organizationId, githubOrganizationId) + + if err != nil { + return 0, errors.New("failed to upsert organizations__github_organizations:" + err.Error()) + } + + if err := tx.Commit(); err != nil { + return 0, errors.New("failed to commit github organization upsert tx: " + err.Error()) + } + + return githubOrganizationId, nil +} diff --git a/internal/onboarding/activity/organization.go b/internal/onboarding/activity/organization.go new file mode 100644 index 00000000..c98c1dd9 --- /dev/null +++ b/internal/onboarding/activity/organization.go @@ -0,0 +1,40 @@ +package activity + +import ( + "context" + "errors" + "sync" + + "github.com/dxta-dev/app/internal/onboarding" +) + +type TenantActivities struct { + DBConnections *sync.Map +} + +func NewTenantActivities(DBConnections *sync.Map) *TenantActivities { + return &TenantActivities{DBConnections} +} + +func (ta *TenantActivities) GetOrganizationIDByAuthID(ctx context.Context, authID string, DBURL string) (int64, error) { + db, err := onboarding.GetCachedTenantDB(ta.DBConnections, DBURL, ctx) + + if err != nil { + return 0, err + } + + var organizationId int64 + + if err = db.QueryRowContext(ctx, ` + SELECT + id + FROM + organizations + WHERE + auth_id = ?;`, + authID).Scan(&organizationId); err != nil { + return 0, errors.New("failed to retrieve organization: " + err.Error()) + } + + return organizationId, nil +} diff --git a/internal/onboarding/github_config.go b/internal/onboarding/github_config.go index 9b748688..63869db6 100644 --- a/internal/onboarding/github_config.go +++ b/internal/onboarding/github_config.go @@ -131,25 +131,33 @@ type GithubAppClient struct { client *github.Client } -func (gac *GithubAppClient) GetOrganizationLogin( +type Account struct { + ID int64 + Login string +} + +func (gac *GithubAppClient) GetInstallationAccount( ctx context.Context, installationID int64, -) (string, error) { +) (*Account, error) { installation, _, error := gac.client.Apps.GetInstallation(ctx, installationID) if error != nil { - return "", errors.New("failed to get installation: " + error.Error()) + return nil, errors.New("failed to get installation: " + error.Error()) } if installation.Account == nil || installation.Account.Login == nil { - return "", errors.New("installation account or login is nil") + return nil, errors.New("installation account or login is nil") } - if installation.TargetType == nil || *installation.TargetType != "organization" { - return "", errors.New("installation is not for an organization") + if installation.TargetType == nil || *installation.TargetType != "Organization" { + return nil, errors.New("installation is not for an organization") } - return *installation.Account.Login, nil + return &Account{ + ID: *installation.Account.ID, + Login: *installation.Account.Login, + }, nil } func NewAppClient(cfg GithubConfig) (*GithubAppClient, error) { diff --git a/internal/onboarding/tenant.go b/internal/onboarding/tenant.go new file mode 100644 index 00000000..96b7fbd8 --- /dev/null +++ b/internal/onboarding/tenant.go @@ -0,0 +1,27 @@ +package onboarding + +import ( + "context" + "database/sql" + "errors" + "sync" + + internal_api_data "github.com/dxta-dev/app/internal/internal-api/data" +) + +func GetCachedTenantDB(store *sync.Map, dbUrl string, ctx context.Context) (*sql.DB, error) { + db, ok := store.Load(dbUrl) + + if !ok { + tenantDB, err := internal_api_data.NewTenantDB(dbUrl, ctx) + + if err != nil { + return nil, errors.New("failed to create tenant db connection: " + err.Error()) + } + + db = tenantDB.DB + store.Store(dbUrl, db) + } + + return db.(*sql.DB), nil +} diff --git a/internal/onboarding/workflow/after_github_installation.go b/internal/onboarding/workflow/after_github_installation.go index 0bd0e15d..1e131dc1 100644 --- a/internal/onboarding/workflow/after_github_installation.go +++ b/internal/onboarding/workflow/after_github_installation.go @@ -5,7 +5,6 @@ import ( "fmt" "time" - "github.com/google/go-github/v73/github" "go.temporal.io/sdk/client" "go.temporal.io/sdk/temporal" "go.temporal.io/sdk/workflow" @@ -32,11 +31,42 @@ func AfterGithubInstallationWorkflow( ctx = workflow.WithActivityOptions(ctx, ao) - installationId := params.InstallationID + var installation activity.GithubInstallationOrganization - var installation *github.Installation - err = workflow.ExecuteActivity(ctx, (*activity.GithubInstallationActivities).GetGithubInstallation, installationId). - Get(ctx, &installation) + err = workflow.ExecuteActivity( + ctx, + (*activity.GithubInstallationActivities).GetInstallationOrganization, + params.InstallationID, + ).Get(ctx, &installation) + + if err != nil { + return + } + + var organizationId int64 + + err = workflow.ExecuteActivity( + ctx, + (*activity.TenantActivities).GetOrganizationIDByAuthID, + params.AuthID, + params.DBURL, + ).Get(ctx, &organizationId) + + if err != nil { + return + } + + var githubOrganizationId int64 + + err = workflow.ExecuteActivity( + ctx, + (*activity.TenantActivities).UpsertGithubOrganization, + params.DBURL, + params.InstallationID, + installation.OrganizationLogin, + installation.OrganizationID, + organizationId, + ).Get(ctx, &githubOrganizationId) if err != nil { return diff --git a/internal/util/auth.go b/internal/util/auth.go index 9101bafa..edd99489 100644 --- a/internal/util/auth.go +++ b/internal/util/auth.go @@ -9,7 +9,7 @@ import ( "net/http" "os" - "github.com/dxta-dev/app/internal/internal-api" + api "github.com/dxta-dev/app/internal/internal-api" "github.com/go-chi/jwtauth/v5" ) @@ -130,7 +130,7 @@ func Authenticator() func(http.Handler) http.Handler { return } - apiState, err := api.InternalApiState(tenantData.DBUrl, r) + apiState, err := api.InternalApiState(ctx, tenantData.DBUrl, r) if err != nil { JSONError(w, ErrorParam{Error: "Internal Server Error"}, http.StatusInternalServerError)