diff --git a/cmd/sippy/load.go b/cmd/sippy/load.go index 8a75b65223..614b6cbe58 100644 --- a/cmd/sippy/load.go +++ b/cmd/sippy/load.go @@ -37,6 +37,7 @@ import ( "github.com/openshift/sippy/pkg/dataloader/prowloader" "github.com/openshift/sippy/pkg/dataloader/prowloader/gcs" "github.com/openshift/sippy/pkg/dataloader/prowloader/github" + releasedefloader "github.com/openshift/sippy/pkg/dataloader/releasedefloader" "github.com/openshift/sippy/pkg/dataloader/releaseloader" "github.com/openshift/sippy/pkg/dataloader/testownershiploader" "github.com/openshift/sippy/pkg/db" @@ -100,7 +101,7 @@ func (f *LoadFlags) BindFlags(fs *pflag.FlagSet) { f.JiraFlags.BindFlags(fs) fs.BoolVar(&f.InitDatabase, "init-database", false, "Migrate the DB before loading") - fs.StringArrayVar(&f.Loaders, "loader", []string{"prow", "releases", "jira", "github", "bugs", "test-mapping", "feature-gates"}, "Which data sources to use for data loading") + fs.StringArrayVar(&f.Loaders, "loader", []string{"release-definitions", "prow", "releases", "jira", "github", "bugs", "test-mapping", "feature-gates"}, "Which data sources to use for data loading") fs.StringArrayVar(&f.Releases, "release", f.Releases, "Which releases to load (one per arg instance)") fs.StringArrayVar(&f.Architectures, "arch", f.Architectures, "Which architectures to load (one per arg instance)") fs.StringVar(&f.JobVariantsInputFile, "job-variants-input-file", "expected-job-variants.json", "JSON input file for the job-variants loader") @@ -152,8 +153,6 @@ func NewLoadCommand() *cobra.Command { cacheClient = nil // error hygiene, since we pass this down to quite a few functions } - releaseConfigs := []sippyv1.Release{} - // initializing a bigquery client different from the normal one opCtx, ctx := bqcachedclient.OpCtxForCronEnv(ctx, "load") bqc, bigqueryErr := bqcachedclient.New( @@ -164,10 +163,17 @@ func NewLoadCommand() *cobra.Command { if f.CacheFlags.EnablePersistentCaching { bqc = f.CacheFlags.DecorateBiqQueryClientWithPersistentCache(bqc) } - releaseConfigs, err = api.GetReleasesFromBigQuery(context.Background(), bqc) - if err != nil { - return errors.Wrapf(err, "error querying releases from bq") - } + } + + // Read release definitions from PG for downstream loader construction. + // Falls back to BQ if PG is empty (e.g., first run before the + // release-definitions loader has populated the table). + releaseConfigs := []sippyv1.Release{} + if dbErr == nil { + releaseConfigs, _ = api.GetReleasesFromDB(context.Background(), dbc) + } + if len(releaseConfigs) == 0 && bigqueryErr == nil { + releaseConfigs, _ = api.GetReleasesFromBigQuery(context.Background(), bqc) } // Ensure partitions exist for all releases (only when InitDatabase is true) @@ -201,6 +207,17 @@ func NewLoadCommand() *cobra.Command { var regressionCacheAdded bool for _, l := range f.Loaders { + if l == "release-definitions" { + if bigqueryErr != nil { + return errors.Wrap(bigqueryErr, "CRITICAL error getting BigQuery client which prevents release-definitions loading") + } + if dbErr != nil { + return errors.Wrap(dbErr, "CRITICAL error getting postgres client which prevents release-definitions loading") + } + rdl := releasedefloader.NewReleaseDefinitionLoader(ctx, dbc, bqc) + loaders = append(loaders, rdl) + } + // TODO: remove "component-readiness-cache" and "regression-tracker" once the cronjob // manifests are updated to use "regression-cache". if l == "component-readiness-cache" || l == "regression-tracker" || l == "regression-cache" { diff --git a/cmd/sippy/seed_data.go b/cmd/sippy/seed_data.go index adc6817edc..b8736bc5cf 100644 --- a/cmd/sippy/seed_data.go +++ b/cmd/sippy/seed_data.go @@ -411,6 +411,11 @@ func seedSyntheticData(dbc *db.DB) error { return nil } + if err := seedReleaseDefinitions(dbc); err != nil { + return errors.WithMessage(err, "failed to seed release definitions") + } + log.Info("Seeded release definitions") + if err := createTestSuite(dbc, "synthetic"); err != nil { return errors.WithMessage(err, "failed to create test suite") } @@ -447,6 +452,53 @@ func seedSyntheticData(dbc *db.DB) error { return nil } +func seedReleaseDefinitions(dbc *db.DB) error { + now := time.Now().UTC() + allCaps := pq.StringArray{models.CapComponentReadiness, models.CapFeatureGates, models.CapMetrics, models.CapPayloadTags, models.CapSippyClassic} + + type relMeta struct { + previous string + gaDays int // negative = days before now; 0 = no GA (in development) + } + meta := map[string]relMeta{ + "4.19": {previous: "4.18", gaDays: -289}, + "4.20": {previous: "4.19", gaDays: -163}, + "4.21": {previous: "4.20", gaDays: -58}, + "4.22": {previous: "4.21"}, + } + + for _, release := range syntheticReleases { + m := meta[release] + parts := strings.Split(release, ".") + major, minor := 0, 0 + if len(parts) >= 2 { + _, _ = fmt.Sscanf(parts[0], "%d", &major) + _, _ = fmt.Sscanf(parts[1], "%d", &minor) + } + + develStart := now.AddDate(0, 0, m.gaDays-180) + def := models.ReleaseDefinition{ + Release: release, + Major: major, + Minor: minor, + PreviousRelease: m.previous, + DevelopmentStartDate: &develStart, + Product: "OCP", + Status: "Full Support", + Capabilities: allCaps, + } + if m.gaDays != 0 { + ga := now.AddDate(0, 0, m.gaDays) + def.GADate = &ga + } + + if err := dbc.DB.Where("release = ?", release).FirstOrCreate(&def).Error; err != nil { + return fmt.Errorf("failed to create release definition %s: %w", release, err) + } + } + return nil +} + func seedProwJobs(dbc *db.DB) error { for _, release := range syntheticReleases { for _, job := range syntheticJobs { diff --git a/pkg/api/releases.go b/pkg/api/releases.go index c543450568..39857600be 100644 --- a/pkg/api/releases.go +++ b/pkg/api/releases.go @@ -512,6 +512,66 @@ func transformRelease(r sippyv1.ReleaseRow) sippyv1.Release { return release } +// GetReleaseRowsFromBigQuery fetches raw release rows from BigQuery's Releases table. +func GetReleaseRowsFromBigQuery(ctx context.Context, client *bqcachedclient.Client) ([]sippyv1.ReleaseRow, error) { + var rows []sippyv1.ReleaseRow + + queryString := fmt.Sprintf("SELECT * FROM `%s` ORDER BY DevelStartDate DESC", client.ReleasesTable) + + q := client.Query(ctx, bqlabel.ReleaseAllReleases, queryString) + it, err := q.Read(ctx) + if err != nil { + log.WithError(err).Error("error querying releases data from bigquery") + return rows, err + } + + for { + r := sippyv1.ReleaseRow{} + err := it.Next(&r) + if err == iterator.Done { + break + } + if err != nil { + log.WithError(err).Error("error parsing release row from bigquery") + return rows, err + } + rows = append(rows, r) + } + return rows, nil +} + +// GetReleasesFromDB queries release metadata from the release_definitions table +// and converts to []sippyv1.Release for use by existing callers. +func GetReleasesFromDB(ctx context.Context, dbc *db.DB) ([]sippyv1.Release, error) { + var defs []models.ReleaseDefinition + err := dbc.DB.WithContext(ctx).Order("development_start_date DESC").Find(&defs).Error + if err != nil { + return nil, fmt.Errorf("querying release definitions: %w", err) + } + releases := make([]sippyv1.Release, 0, len(defs)) + for _, def := range defs { + releases = append(releases, DefinitionToRelease(def)) + } + return releases, nil +} + +// DefinitionToRelease converts a models.ReleaseDefinition to a sippyv1.Release. +func DefinitionToRelease(def models.ReleaseDefinition) sippyv1.Release { + caps := make(map[sippyv1.ReleaseCapability]bool, len(def.Capabilities)) + for _, cap := range def.Capabilities { + caps[sippyv1.ReleaseCapability(cap)] = true + } + return sippyv1.Release{ + Release: def.Release, + Status: def.Status, + GADate: def.GADate, + DevelopmentStartDate: def.DevelopmentStartDate, + PreviousRelease: def.PreviousRelease, + Capabilities: caps, + Product: def.Product, + } +} + // BuildReleasesResponse creates the API response structure for releases func BuildReleasesResponse(releases []sippyv1.Release, lastUpdated time.Time) apitype.Releases { gaDateMap := make(map[string]time.Time) diff --git a/pkg/dataloader/loaderwithmetrics/loaderwithmetrics.go b/pkg/dataloader/loaderwithmetrics/loaderwithmetrics.go index fc2d5424c5..485ab53664 100644 --- a/pkg/dataloader/loaderwithmetrics/loaderwithmetrics.go +++ b/pkg/dataloader/loaderwithmetrics/loaderwithmetrics.go @@ -46,7 +46,7 @@ func New(wrappedLoaders []dataloader.DataLoader) *LoaderWithMetrics { return loader } -var loaderOrder = []string{"prow", "releases", "jira", "github", "bugs", "test-mapping", "feature-gates", "regression-cache"} +var loaderOrder = []string{"release-definitions", "prow", "releases", "jira", "github", "bugs", "test-mapping", "feature-gates", "regression-cache"} // sortLoaders guarantees that the loaders run in a predictable and proper order func (l *LoaderWithMetrics) sortLoaders() { diff --git a/pkg/dataloader/releasedefloader/releasedefloader.go b/pkg/dataloader/releasedefloader/releasedefloader.go new file mode 100644 index 0000000000..3defa97261 --- /dev/null +++ b/pkg/dataloader/releasedefloader/releasedefloader.go @@ -0,0 +1,103 @@ +package releasedefloader + +import ( + "context" + "fmt" + "sort" + "time" + + "github.com/lib/pq" + log "github.com/sirupsen/logrus" + "gorm.io/gorm/clause" + + "github.com/openshift/sippy/pkg/api" + v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + bqcachedclient "github.com/openshift/sippy/pkg/bigquery" + "github.com/openshift/sippy/pkg/db" + "github.com/openshift/sippy/pkg/db/models" +) + +// ReleaseDefinitionLoader fetches release metadata from BigQuery and syncs it to PostgreSQL. +type ReleaseDefinitionLoader struct { + ctx context.Context + dbc *db.DB + bqClient *bqcachedclient.Client + errs []error +} + +func NewReleaseDefinitionLoader(ctx context.Context, dbc *db.DB, bqClient *bqcachedclient.Client) *ReleaseDefinitionLoader { + return &ReleaseDefinitionLoader{ + ctx: ctx, + dbc: dbc, + bqClient: bqClient, + } +} + +func (l *ReleaseDefinitionLoader) Name() string { + return "release-definitions" +} + +func (l *ReleaseDefinitionLoader) Load() { + releaseRows, err := api.GetReleaseRowsFromBigQuery(l.ctx, l.bqClient) + if err != nil { + l.errs = append(l.errs, fmt.Errorf("fetching releases from bigquery: %w", err)) + return + } + defs := make([]models.ReleaseDefinition, 0, len(releaseRows)) + for _, row := range releaseRows { + defs = append(defs, ReleaseRowToDefinition(row)) + } + if err := syncReleaseDefinitions(l.dbc, defs); err != nil { + l.errs = append(l.errs, fmt.Errorf("syncing release definitions: %w", err)) + } +} + +func (l *ReleaseDefinitionLoader) Errors() []error { + return l.errs +} + +func syncReleaseDefinitions(dbc *db.DB, defs []models.ReleaseDefinition) error { + for _, def := range defs { + err := dbc.DB.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "release"}}, + DoUpdates: clause.AssignmentColumns([]string{"major", "minor", "patch", "previous_release", "ga_date", "development_start_date", "product", "status", "capabilities", "updated_at"}), + }).Create(&def).Error + if err != nil { + return fmt.Errorf("upserting release definition %s: %w", def.Release, err) + } + } + log.WithField("count", len(defs)).Info("synced release definitions to postgres") + return nil +} + +// ReleaseRowToDefinition converts a BigQuery ReleaseRow directly to a ReleaseDefinition DB model. +func ReleaseRowToDefinition(r v1.ReleaseRow) models.ReleaseDefinition { + caps := make(pq.StringArray, 0, len(r.Capabilities)) + for _, cap := range r.Capabilities { + caps = append(caps, string(cap)) + } + sort.Strings(caps) + + def := models.ReleaseDefinition{ + Release: r.Release, + Major: r.Major, + Minor: r.Minor, + PreviousRelease: r.PreviousRelease.StringVal, + Product: r.Product.StringVal, + Status: r.ReleaseStatus.StringVal, + Capabilities: caps, + } + if r.Patch.Valid { + p := int(r.Patch.Int64) + def.Patch = &p + } + if r.GADate.Valid { + ga := r.GADate.Date.In(time.UTC) + def.GADate = &ga + } + if r.DevelStartDate.IsValid() { + ds := r.DevelStartDate.In(time.UTC) + def.DevelopmentStartDate = &ds + } + return def +} diff --git a/pkg/db/db.go b/pkg/db/db.go index 3a76773d93..096d61fc10 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -137,6 +137,7 @@ func (d *DB) UpdateSchema(reportEnd *time.Time) error { // List of all models to migrate modelsToMigrate := []any{ + &models.ReleaseDefinition{}, &models.ReleaseTag{}, &models.ReleasePullRequest{}, &models.ReleaseRepository{}, diff --git a/pkg/db/models/releases.go b/pkg/db/models/releases.go index 01c578e628..a02f533f72 100644 --- a/pkg/db/models/releases.go +++ b/pkg/db/models/releases.go @@ -6,6 +6,41 @@ import ( "github.com/lib/pq" ) +// ReleaseDefinition stores release metadata synced from BigQuery's Releases table. +// This is distinct from ReleaseTag, which tracks individual payload tags. +type ReleaseDefinition struct { + Model + + Release string `json:"release" gorm:"uniqueIndex;column:release"` + Major int `json:"major" gorm:"column:major"` + Minor int `json:"minor" gorm:"column:minor"` + Patch *int `json:"patch,omitempty" gorm:"column:patch"` + PreviousRelease string `json:"previous_release" gorm:"column:previous_release"` + GADate *time.Time `json:"ga_date,omitempty" gorm:"column:ga_date;type:date"` + DevelopmentStartDate *time.Time `json:"development_start_date" gorm:"column:development_start_date;type:date"` + Product string `json:"product" gorm:"column:product"` + Status string `json:"status" gorm:"column:status"` + Capabilities pq.StringArray `json:"capabilities" gorm:"type:text[];column:capabilities"` +} + +const ( + CapComponentReadiness = "componentReadiness" + CapSippyClassic = "sippyClassic" + CapMetrics = "metrics" + CapPullRequests = "pullRequests" + CapFeatureGates = "featureGates" + CapPayloadTags = "payloadTags" +) + +func (rd *ReleaseDefinition) HasCapability(capability string) bool { + for _, c := range rd.Capabilities { + if c == capability { + return true + } + } + return false +} + type ReleaseTag struct { Model