From 39eb557853dd60c9538973af195be8e3940eff87 Mon Sep 17 00:00:00 2001 From: Matthew Staebler Date: Wed, 24 Jun 2026 10:20:19 -0400 Subject: [PATCH] Add release_definitions table and loader Create a release_definitions table to store release metadata (GA dates, development start dates, previous release, capabilities, product, status) synced from BigQuery. This is the first phase of moving release metadata from BigQuery to PostgreSQL. Key changes: - Add ReleaseDefinition model with capability constants and HasCapability method - Add release-definitions loader (--loader release-definitions) that fetches from BQ and syncs to PG via upsert - Add GetReleasesFromDB and DefinitionToRelease for reading from PG - Add GetReleaseRowsFromBigQuery for direct BQ row access - Seed data populates release_definitions for local development No consumers are switched in this phase. The existing getReleases() and QueryReleases() paths continue to use BigQuery / hardcoded metadata. A follow-up PR will switch consumers to read from the new table once it has been populated by the load cycle. Ref: TRT-2734 Co-Authored-By: Claude Opus 4.6 --- cmd/sippy/load.go | 28 +++-- cmd/sippy/seed_data.go | 52 +++++++++ pkg/api/releases.go | 60 ++++++++++ .../loaderwithmetrics/loaderwithmetrics.go | 2 +- .../releasedefloader/releasedefloader.go | 103 ++++++++++++++++++ pkg/db/db.go | 1 + pkg/db/models/releases.go | 35 ++++++ 7 files changed, 273 insertions(+), 8 deletions(-) create mode 100644 pkg/dataloader/releasedefloader/releasedefloader.go diff --git a/cmd/sippy/load.go b/cmd/sippy/load.go index 8a75b65223..a8afa8fc9c 100644 --- a/cmd/sippy/load.go +++ b/cmd/sippy/load.go @@ -37,6 +37,7 @@ import ( "github.com/openshift/sippy/pkg/dataloader/prowloader" "github.com/openshift/sippy/pkg/dataloader/prowloader/gcs" "github.com/openshift/sippy/pkg/dataloader/prowloader/github" + releasedefloader "github.com/openshift/sippy/pkg/dataloader/releasedefloader" "github.com/openshift/sippy/pkg/dataloader/releaseloader" "github.com/openshift/sippy/pkg/dataloader/testownershiploader" "github.com/openshift/sippy/pkg/db" @@ -100,7 +101,7 @@ func (f *LoadFlags) BindFlags(fs *pflag.FlagSet) { f.JiraFlags.BindFlags(fs) fs.BoolVar(&f.InitDatabase, "init-database", false, "Migrate the DB before loading") - fs.StringArrayVar(&f.Loaders, "loader", []string{"prow", "releases", "jira", "github", "bugs", "test-mapping", "feature-gates"}, "Which data sources to use for data loading") + fs.StringArrayVar(&f.Loaders, "loader", []string{"release-definitions", "prow", "releases", "jira", "github", "bugs", "test-mapping", "feature-gates"}, "Which data sources to use for data loading") fs.StringArrayVar(&f.Releases, "release", f.Releases, "Which releases to load (one per arg instance)") fs.StringArrayVar(&f.Architectures, "arch", f.Architectures, "Which architectures to load (one per arg instance)") fs.StringVar(&f.JobVariantsInputFile, "job-variants-input-file", "expected-job-variants.json", "JSON input file for the job-variants loader") @@ -152,8 +153,6 @@ func NewLoadCommand() *cobra.Command { cacheClient = nil // error hygiene, since we pass this down to quite a few functions } - releaseConfigs := []sippyv1.Release{} - // initializing a bigquery client different from the normal one opCtx, ctx := bqcachedclient.OpCtxForCronEnv(ctx, "load") bqc, bigqueryErr := bqcachedclient.New( @@ -164,10 +163,14 @@ func NewLoadCommand() *cobra.Command { if f.CacheFlags.EnablePersistentCaching { bqc = f.CacheFlags.DecorateBiqQueryClientWithPersistentCache(bqc) } - releaseConfigs, err = api.GetReleasesFromBigQuery(context.Background(), bqc) - if err != nil { - return errors.Wrapf(err, "error querying releases from bq") - } + } + + // Read release definitions from PG for downstream loader construction. + // On the first run the table may be empty; the release-definitions + // loader will populate it for subsequent runs. + releaseConfigs := []sippyv1.Release{} + if dbErr == nil { + releaseConfigs, _ = api.GetReleasesFromDB(context.Background(), dbc) } // Ensure partitions exist for all releases (only when InitDatabase is true) @@ -201,6 +204,17 @@ func NewLoadCommand() *cobra.Command { var regressionCacheAdded bool for _, l := range f.Loaders { + if l == "release-definitions" { + if bigqueryErr != nil { + return errors.Wrap(bigqueryErr, "CRITICAL error getting BigQuery client which prevents release-definitions loading") + } + if dbErr != nil { + return errors.Wrap(dbErr, "CRITICAL error getting postgres client which prevents release-definitions loading") + } + rdl := releasedefloader.NewReleaseDefinitionLoader(ctx, dbc, bqc) + loaders = append(loaders, rdl) + } + // TODO: remove "component-readiness-cache" and "regression-tracker" once the cronjob // manifests are updated to use "regression-cache". if l == "component-readiness-cache" || l == "regression-tracker" || l == "regression-cache" { diff --git a/cmd/sippy/seed_data.go b/cmd/sippy/seed_data.go index adc6817edc..b8736bc5cf 100644 --- a/cmd/sippy/seed_data.go +++ b/cmd/sippy/seed_data.go @@ -411,6 +411,11 @@ func seedSyntheticData(dbc *db.DB) error { return nil } + if err := seedReleaseDefinitions(dbc); err != nil { + return errors.WithMessage(err, "failed to seed release definitions") + } + log.Info("Seeded release definitions") + if err := createTestSuite(dbc, "synthetic"); err != nil { return errors.WithMessage(err, "failed to create test suite") } @@ -447,6 +452,53 @@ func seedSyntheticData(dbc *db.DB) error { return nil } +func seedReleaseDefinitions(dbc *db.DB) error { + now := time.Now().UTC() + allCaps := pq.StringArray{models.CapComponentReadiness, models.CapFeatureGates, models.CapMetrics, models.CapPayloadTags, models.CapSippyClassic} + + type relMeta struct { + previous string + gaDays int // negative = days before now; 0 = no GA (in development) + } + meta := map[string]relMeta{ + "4.19": {previous: "4.18", gaDays: -289}, + "4.20": {previous: "4.19", gaDays: -163}, + "4.21": {previous: "4.20", gaDays: -58}, + "4.22": {previous: "4.21"}, + } + + for _, release := range syntheticReleases { + m := meta[release] + parts := strings.Split(release, ".") + major, minor := 0, 0 + if len(parts) >= 2 { + _, _ = fmt.Sscanf(parts[0], "%d", &major) + _, _ = fmt.Sscanf(parts[1], "%d", &minor) + } + + develStart := now.AddDate(0, 0, m.gaDays-180) + def := models.ReleaseDefinition{ + Release: release, + Major: major, + Minor: minor, + PreviousRelease: m.previous, + DevelopmentStartDate: &develStart, + Product: "OCP", + Status: "Full Support", + Capabilities: allCaps, + } + if m.gaDays != 0 { + ga := now.AddDate(0, 0, m.gaDays) + def.GADate = &ga + } + + if err := dbc.DB.Where("release = ?", release).FirstOrCreate(&def).Error; err != nil { + return fmt.Errorf("failed to create release definition %s: %w", release, err) + } + } + return nil +} + func seedProwJobs(dbc *db.DB) error { for _, release := range syntheticReleases { for _, job := range syntheticJobs { diff --git a/pkg/api/releases.go b/pkg/api/releases.go index c543450568..39857600be 100644 --- a/pkg/api/releases.go +++ b/pkg/api/releases.go @@ -512,6 +512,66 @@ func transformRelease(r sippyv1.ReleaseRow) sippyv1.Release { return release } +// GetReleaseRowsFromBigQuery fetches raw release rows from BigQuery's Releases table. +func GetReleaseRowsFromBigQuery(ctx context.Context, client *bqcachedclient.Client) ([]sippyv1.ReleaseRow, error) { + var rows []sippyv1.ReleaseRow + + queryString := fmt.Sprintf("SELECT * FROM `%s` ORDER BY DevelStartDate DESC", client.ReleasesTable) + + q := client.Query(ctx, bqlabel.ReleaseAllReleases, queryString) + it, err := q.Read(ctx) + if err != nil { + log.WithError(err).Error("error querying releases data from bigquery") + return rows, err + } + + for { + r := sippyv1.ReleaseRow{} + err := it.Next(&r) + if err == iterator.Done { + break + } + if err != nil { + log.WithError(err).Error("error parsing release row from bigquery") + return rows, err + } + rows = append(rows, r) + } + return rows, nil +} + +// GetReleasesFromDB queries release metadata from the release_definitions table +// and converts to []sippyv1.Release for use by existing callers. +func GetReleasesFromDB(ctx context.Context, dbc *db.DB) ([]sippyv1.Release, error) { + var defs []models.ReleaseDefinition + err := dbc.DB.WithContext(ctx).Order("development_start_date DESC").Find(&defs).Error + if err != nil { + return nil, fmt.Errorf("querying release definitions: %w", err) + } + releases := make([]sippyv1.Release, 0, len(defs)) + for _, def := range defs { + releases = append(releases, DefinitionToRelease(def)) + } + return releases, nil +} + +// DefinitionToRelease converts a models.ReleaseDefinition to a sippyv1.Release. +func DefinitionToRelease(def models.ReleaseDefinition) sippyv1.Release { + caps := make(map[sippyv1.ReleaseCapability]bool, len(def.Capabilities)) + for _, cap := range def.Capabilities { + caps[sippyv1.ReleaseCapability(cap)] = true + } + return sippyv1.Release{ + Release: def.Release, + Status: def.Status, + GADate: def.GADate, + DevelopmentStartDate: def.DevelopmentStartDate, + PreviousRelease: def.PreviousRelease, + Capabilities: caps, + Product: def.Product, + } +} + // BuildReleasesResponse creates the API response structure for releases func BuildReleasesResponse(releases []sippyv1.Release, lastUpdated time.Time) apitype.Releases { gaDateMap := make(map[string]time.Time) diff --git a/pkg/dataloader/loaderwithmetrics/loaderwithmetrics.go b/pkg/dataloader/loaderwithmetrics/loaderwithmetrics.go index fc2d5424c5..485ab53664 100644 --- a/pkg/dataloader/loaderwithmetrics/loaderwithmetrics.go +++ b/pkg/dataloader/loaderwithmetrics/loaderwithmetrics.go @@ -46,7 +46,7 @@ func New(wrappedLoaders []dataloader.DataLoader) *LoaderWithMetrics { return loader } -var loaderOrder = []string{"prow", "releases", "jira", "github", "bugs", "test-mapping", "feature-gates", "regression-cache"} +var loaderOrder = []string{"release-definitions", "prow", "releases", "jira", "github", "bugs", "test-mapping", "feature-gates", "regression-cache"} // sortLoaders guarantees that the loaders run in a predictable and proper order func (l *LoaderWithMetrics) sortLoaders() { diff --git a/pkg/dataloader/releasedefloader/releasedefloader.go b/pkg/dataloader/releasedefloader/releasedefloader.go new file mode 100644 index 0000000000..3defa97261 --- /dev/null +++ b/pkg/dataloader/releasedefloader/releasedefloader.go @@ -0,0 +1,103 @@ +package releasedefloader + +import ( + "context" + "fmt" + "sort" + "time" + + "github.com/lib/pq" + log "github.com/sirupsen/logrus" + "gorm.io/gorm/clause" + + "github.com/openshift/sippy/pkg/api" + v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + bqcachedclient "github.com/openshift/sippy/pkg/bigquery" + "github.com/openshift/sippy/pkg/db" + "github.com/openshift/sippy/pkg/db/models" +) + +// ReleaseDefinitionLoader fetches release metadata from BigQuery and syncs it to PostgreSQL. +type ReleaseDefinitionLoader struct { + ctx context.Context + dbc *db.DB + bqClient *bqcachedclient.Client + errs []error +} + +func NewReleaseDefinitionLoader(ctx context.Context, dbc *db.DB, bqClient *bqcachedclient.Client) *ReleaseDefinitionLoader { + return &ReleaseDefinitionLoader{ + ctx: ctx, + dbc: dbc, + bqClient: bqClient, + } +} + +func (l *ReleaseDefinitionLoader) Name() string { + return "release-definitions" +} + +func (l *ReleaseDefinitionLoader) Load() { + releaseRows, err := api.GetReleaseRowsFromBigQuery(l.ctx, l.bqClient) + if err != nil { + l.errs = append(l.errs, fmt.Errorf("fetching releases from bigquery: %w", err)) + return + } + defs := make([]models.ReleaseDefinition, 0, len(releaseRows)) + for _, row := range releaseRows { + defs = append(defs, ReleaseRowToDefinition(row)) + } + if err := syncReleaseDefinitions(l.dbc, defs); err != nil { + l.errs = append(l.errs, fmt.Errorf("syncing release definitions: %w", err)) + } +} + +func (l *ReleaseDefinitionLoader) Errors() []error { + return l.errs +} + +func syncReleaseDefinitions(dbc *db.DB, defs []models.ReleaseDefinition) error { + for _, def := range defs { + err := dbc.DB.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "release"}}, + DoUpdates: clause.AssignmentColumns([]string{"major", "minor", "patch", "previous_release", "ga_date", "development_start_date", "product", "status", "capabilities", "updated_at"}), + }).Create(&def).Error + if err != nil { + return fmt.Errorf("upserting release definition %s: %w", def.Release, err) + } + } + log.WithField("count", len(defs)).Info("synced release definitions to postgres") + return nil +} + +// ReleaseRowToDefinition converts a BigQuery ReleaseRow directly to a ReleaseDefinition DB model. +func ReleaseRowToDefinition(r v1.ReleaseRow) models.ReleaseDefinition { + caps := make(pq.StringArray, 0, len(r.Capabilities)) + for _, cap := range r.Capabilities { + caps = append(caps, string(cap)) + } + sort.Strings(caps) + + def := models.ReleaseDefinition{ + Release: r.Release, + Major: r.Major, + Minor: r.Minor, + PreviousRelease: r.PreviousRelease.StringVal, + Product: r.Product.StringVal, + Status: r.ReleaseStatus.StringVal, + Capabilities: caps, + } + if r.Patch.Valid { + p := int(r.Patch.Int64) + def.Patch = &p + } + if r.GADate.Valid { + ga := r.GADate.Date.In(time.UTC) + def.GADate = &ga + } + if r.DevelStartDate.IsValid() { + ds := r.DevelStartDate.In(time.UTC) + def.DevelopmentStartDate = &ds + } + return def +} diff --git a/pkg/db/db.go b/pkg/db/db.go index 3a76773d93..096d61fc10 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -137,6 +137,7 @@ func (d *DB) UpdateSchema(reportEnd *time.Time) error { // List of all models to migrate modelsToMigrate := []any{ + &models.ReleaseDefinition{}, &models.ReleaseTag{}, &models.ReleasePullRequest{}, &models.ReleaseRepository{}, diff --git a/pkg/db/models/releases.go b/pkg/db/models/releases.go index 01c578e628..a02f533f72 100644 --- a/pkg/db/models/releases.go +++ b/pkg/db/models/releases.go @@ -6,6 +6,41 @@ import ( "github.com/lib/pq" ) +// ReleaseDefinition stores release metadata synced from BigQuery's Releases table. +// This is distinct from ReleaseTag, which tracks individual payload tags. +type ReleaseDefinition struct { + Model + + Release string `json:"release" gorm:"uniqueIndex;column:release"` + Major int `json:"major" gorm:"column:major"` + Minor int `json:"minor" gorm:"column:minor"` + Patch *int `json:"patch,omitempty" gorm:"column:patch"` + PreviousRelease string `json:"previous_release" gorm:"column:previous_release"` + GADate *time.Time `json:"ga_date,omitempty" gorm:"column:ga_date;type:date"` + DevelopmentStartDate *time.Time `json:"development_start_date" gorm:"column:development_start_date;type:date"` + Product string `json:"product" gorm:"column:product"` + Status string `json:"status" gorm:"column:status"` + Capabilities pq.StringArray `json:"capabilities" gorm:"type:text[];column:capabilities"` +} + +const ( + CapComponentReadiness = "componentReadiness" + CapSippyClassic = "sippyClassic" + CapMetrics = "metrics" + CapPullRequests = "pullRequests" + CapFeatureGates = "featureGates" + CapPayloadTags = "payloadTags" +) + +func (rd *ReleaseDefinition) HasCapability(capability string) bool { + for _, c := range rd.Capabilities { + if c == capability { + return true + } + } + return false +} + type ReleaseTag struct { Model