From 1dc6e1496a08544c71c5bd566c9d48f4ea1f10da Mon Sep 17 00:00:00 2001 From: Alvar Penning Date: Fri, 13 Feb 2026 10:29:51 +0100 Subject: [PATCH 1/4] CheckSchema: Icinga DB Schema update file in error When Icinga DB detects a schema version mismatch, the expected schema version number is shown in the error message. Unfortunately, the internal schema version number is not reflected in the schema upgrade filenames, matching the Icinga DB release version. For example, starting Icinga DB after upgrading to version 1.4.0 without applying the schema upgrade first, would complain that schema version 7 is expected for MySQL and schema version 5 is expected for PostgreSQL, respectively. However, the shipped schema upgrade files do not reflect these version numbers, but are named 1.4.0.sql, after the Icinga DB release. This output is confusing for end users. Thus, an internal mapping of schema version numbers to Icinga DB releases ware created, allowing to output the expected schema version number together with the Icinga DB release version. > Starting Icinga DB daemon (1.5.1-ga395295-dirty) > Connecting to database at 'mysql://root@localhost:3306/icingadb' > unexpected database schema version: v7 (expected v8), please apply the 1.5.2.sql schema upgrade file to your database after upgrading Icinga DB: https://icinga.com/docs/icinga-db/latest/doc/04-Upgrading/ In order to have a single point to update these versions, the schema version numbers where moved to internal/version.go, where the Icinga DB version is also being defined. --- internal/version.go | 25 +++++++++++++++++++++++++ pkg/icingadb/schema.go | 37 +++++++++++++++++++++++-------------- 2 files changed, 48 insertions(+), 14 deletions(-) diff --git a/internal/version.go b/internal/version.go index 54719dbdc..3094be44b 100644 --- a/internal/version.go +++ b/internal/version.go @@ -8,3 +8,28 @@ import ( // // The placeholders are replaced on `git archive` using the `export-subst` attribute. var Version = version.Version("1.5.1", "$Format:%(describe)$", "$Format:%H$") + +// MySqlSchemaVersions maps MySQL/MariaDB schema versions to Icinga DB release version. +// +// Each schema version implies an available schema upgrade, named after the Icinga DB +// version and stored under ./schema/mysql/upgrades. +// +// The largest key implies the latest and expected schema version. +var MySqlSchemaVersions = map[uint16]string{ + 2: "1.0.0-rc2", + 3: "1.0.0", + 4: "1.1.1", + 5: "1.2.0", + 6: "1.2.1", + 7: "1.4.0", +} + +// PgSqlSchemaVersions maps PostgreSQL schema versions to Icinga DB release version. +// +// Same as MySqlSchemaVersions, but for PostgreSQL instead. +var PgSqlSchemaVersions = map[uint16]string{ + 2: "1.1.1", + 3: "1.2.0", + 4: "1.2.1", + 5: "1.4.0", +} diff --git a/pkg/icingadb/schema.go b/pkg/icingadb/schema.go index d78a8957b..c5dc8b45e 100644 --- a/pkg/icingadb/schema.go +++ b/pkg/icingadb/schema.go @@ -7,15 +7,13 @@ import ( "github.com/icinga/icinga-go-library/backoff" "github.com/icinga/icinga-go-library/database" "github.com/icinga/icinga-go-library/retry" + "github.com/icinga/icingadb/internal" "github.com/jmoiron/sqlx" "github.com/pkg/errors" + "maps" "os" "path" -) - -const ( - expectedMysqlSchemaVersion = 7 - expectedPostgresSchemaVersion = 5 + "slices" ) // ErrSchemaNotExists implies that no Icinga DB schema has been imported. @@ -32,16 +30,18 @@ var ErrSchemaMismatch = stderrors.New("unexpected database schema version") // - If the schema version does not match the expected version, the error returned is ErrSchemaMismatch. // - Otherwise, the original error is returned, for example in case of general database problems. func CheckSchema(ctx context.Context, db *database.DB) error { - var expectedDbSchemaVersion uint16 + var schemaVersions map[uint16]string switch db.DriverName() { case database.MySQL: - expectedDbSchemaVersion = expectedMysqlSchemaVersion + schemaVersions = internal.MySqlSchemaVersions case database.PostgreSQL: - expectedDbSchemaVersion = expectedPostgresSchemaVersion + schemaVersions = internal.PgSqlSchemaVersions default: return errors.Errorf("unsupported database driver %q", db.DriverName()) } + expectedDbSchemaVersion := slices.Max(slices.Sorted(maps.Keys(schemaVersions))) + if hasSchemaTable, err := db.HasTable(ctx, "icingadb_schema"); err != nil { return errors.Wrap(err, "can't verify existence of database schema table") } else if !hasSchemaTable { @@ -75,10 +75,18 @@ func CheckSchema(ctx context.Context, db *database.DB) error { // that each element's successor is the increment of this version, ensuring no gaps in between. for i := 0; i < len(versions)-1; i++ { if versions[i] != versions[i+1]-1 { + missing := versions[i] + 1 + + release := "UNKNOWN" + if releaseVersion, ok := schemaVersions[missing]; ok { + release = releaseVersion + } + return fmt.Errorf( - "%w: incomplete database schema upgrade: intermediate version v%d is missing,"+ - " please make sure you have applied all database migrations after upgrading Icinga DB", - ErrSchemaMismatch, versions[i]+1) + "%w: incomplete database schema upgrade: intermediate version v%d (%s) is missing, "+ + "please inspect the icingadb_schema database table and ensure that all database "+ + "migrations were applied in order after upgrading Icinga DB", + ErrSchemaMismatch, missing, release) } } @@ -86,9 +94,10 @@ func CheckSchema(ctx context.Context, db *database.DB) error { // Since these error messages are trivial and mostly caused by users, we don't need // to print a stack trace here. However, since errors.Errorf() does this automatically, // we need to use fmt instead. - return fmt.Errorf("%w: v%d (expected v%d), please make sure you have applied all database"+ - " migrations after upgrading Icinga DB", ErrSchemaMismatch, latestVersion, expectedDbSchemaVersion, - ) + return fmt.Errorf("%w: v%d (expected v%d), "+ + "please apply the %s.sql schema upgrade file to your database after upgrading Icinga DB: "+ + "https://icinga.com/docs/icinga-db/latest/doc/04-Upgrading/", + ErrSchemaMismatch, latestVersion, expectedDbSchemaVersion, schemaVersions[expectedDbSchemaVersion]) } return nil From c4d7ed314aabae73a2a16a349bcec3100beca5d9 Mon Sep 17 00:00:00 2001 From: Alvar Penning Date: Fri, 13 Feb 2026 10:48:17 +0100 Subject: [PATCH 2/4] Move Redis Schema Version to internal/version.go After moving the database schema version number to the internal/version.go file, the Redis schema version was moved there as well. Now there is only one place left in the codebase to define versions. This reflects the release issue template, listing the internal/version.go to be checked. --- cmd/icingadb/main.go | 9 ++++----- internal/version.go | 5 +++++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/cmd/icingadb/main.go b/cmd/icingadb/main.go index 1cfc64a67..f17fc4a24 100644 --- a/cmd/icingadb/main.go +++ b/cmd/icingadb/main.go @@ -29,9 +29,8 @@ import ( ) const ( - ExitSuccess = 0 - ExitFailure = 1 - expectedRedisSchemaVersion = "6" + ExitSuccess = 0 + ExitFailure = 1 ) func main() { @@ -421,13 +420,13 @@ func checkRedisSchema(logger *logging.Logger, rc *redis.Client, pos string) (new } message := streams[0].Messages[0] - if version := message.Values["version"]; version != expectedRedisSchemaVersion { + if version := message.Values["version"]; version != internal.RedisSchemaVersion { // Since these error messages are trivial and mostly caused by users, we don't need // to print a stack trace here. However, since errors.Errorf() does this automatically, // we need to use fmt instead. return "", fmt.Errorf( "unexpected Redis schema version: %q (expected %q), please make sure you are running compatible"+ - " versions of Icinga 2 and Icinga DB", version, expectedRedisSchemaVersion, + " versions of Icinga 2 and Icinga DB", version, internal.RedisSchemaVersion, ) } diff --git a/internal/version.go b/internal/version.go index 3094be44b..2bdda2f6e 100644 --- a/internal/version.go +++ b/internal/version.go @@ -33,3 +33,8 @@ var PgSqlSchemaVersions = map[uint16]string{ 4: "1.2.1", 5: "1.4.0", } + +// RedisSchemaVersion is the expected Redis schema version. +// +// This version must match between Icinga 2 and Icinga DB. +var RedisSchemaVersion = "6" From 64d1330951d642e35ebc0e524d8e789370558cb9 Mon Sep 17 00:00:00 2001 From: Alvar Penning Date: Wed, 18 Feb 2026 15:57:13 +0100 Subject: [PATCH 3/4] CheckSchema: Loosen schema upgrade check In f49fac079893d79d1ccbb40df67c1ae2e0705c2c, a strict schema upgrade history was enforced. This might result in reports due to misaligned, but fixed schema upgrades. Now, this check was loosened to only enforce that all schema updates between the lowest and latest schema version were applied. If they are not in order, only a warning is being produced. --- cmd/icingadb/main.go | 4 +++ pkg/icingadb/schema.go | 61 +++++++++++++++++++++++++++--------------- 2 files changed, 44 insertions(+), 21 deletions(-) diff --git a/cmd/icingadb/main.go b/cmd/icingadb/main.go index f17fc4a24..ef63cf28a 100644 --- a/cmd/icingadb/main.go +++ b/cmd/icingadb/main.go @@ -88,6 +88,10 @@ func run() int { _ = db.Close() logger.Info("The database schema was successfully imported") + + case errors.Is(err, icingadb.ErrSchemaImperfect): + logger.Warnw("Database schema should be checked", zap.Error(err)) + case err != nil: logger.Fatalf("%+v", err) } diff --git a/pkg/icingadb/schema.go b/pkg/icingadb/schema.go index c5dc8b45e..50920499b 100644 --- a/pkg/icingadb/schema.go +++ b/pkg/icingadb/schema.go @@ -4,16 +4,17 @@ import ( "context" stderrors "errors" "fmt" + "maps" + "os" + "path" + "slices" + "github.com/icinga/icinga-go-library/backoff" "github.com/icinga/icinga-go-library/database" "github.com/icinga/icinga-go-library/retry" "github.com/icinga/icingadb/internal" "github.com/jmoiron/sqlx" "github.com/pkg/errors" - "maps" - "os" - "path" - "slices" ) // ErrSchemaNotExists implies that no Icinga DB schema has been imported. @@ -23,11 +24,16 @@ var ErrSchemaNotExists = stderrors.New("no database schema exists") // missed the schema upgrade. var ErrSchemaMismatch = stderrors.New("unexpected database schema version") +// ErrSchemaImperfect implies some non critical failure condition of the database schema. +var ErrSchemaImperfect = stderrors.New("imperfect database schema") + // CheckSchema verifies the correct database schema is present. // // This function returns the following error types, possibly wrapped: // - If no schema exists, the error returned is ErrSchemaNotExists. // - If the schema version does not match the expected version, the error returned is ErrSchemaMismatch. +// - If there are non fatal database schema conditions, ErrSchemaImperfect is returned. This error must +// be reported back to the user, but should not lead in a program termination. // - Otherwise, the original error is returned, for example in case of general database problems. func CheckSchema(ctx context.Context, db *database.DB) error { var schemaVersions map[uint16]string @@ -53,7 +59,7 @@ func CheckSchema(ctx context.Context, db *database.DB) error { err := retry.WithBackoff( ctx, func(ctx context.Context) error { - query := "SELECT version FROM icingadb_schema ORDER BY version ASC" + query := "SELECT version FROM icingadb_schema ORDER BY id ASC" if err := db.SelectContext(ctx, &versions, query); err != nil { return database.CantPerformQuery(err, query) } @@ -66,19 +72,30 @@ func CheckSchema(ctx context.Context, db *database.DB) error { return errors.Wrap(err, "can't check database schema version") } + // In the following, multiple error conditions are checked. + // + // Since their error messages are trivial and mostly caused by users, we don't need + // to print a stack trace here. However, since errors.Errorf() does this automatically, + // we need to use fmt.Errorf() instead. + + // Check if any schema was imported. if len(versions) == 0 { return fmt.Errorf("%w: no database schema version is stored in the database", ErrSchemaMismatch) } - // Check if each schema update between the initial import and the latest version was applied or, in other words, - // that no schema update was left out. The loop goes over the ascending sorted array of schema versions, verifying - // that each element's successor is the increment of this version, ensuring no gaps in between. - for i := 0; i < len(versions)-1; i++ { - if versions[i] != versions[i+1]-1 { - missing := versions[i] + 1 + // Check if the latest schema version was imported. + if latestVersion := slices.Max(versions); latestVersion != expectedDbSchemaVersion { + return fmt.Errorf("%w: v%d (expected v%d), "+ + "please apply the %s.sql schema upgrade file to your database after upgrading Icinga DB: "+ + "https://icinga.com/docs/icinga-db/latest/doc/04-Upgrading/", + ErrSchemaMismatch, latestVersion, expectedDbSchemaVersion, schemaVersions[expectedDbSchemaVersion]) + } + // Check if all schema updates between the oldest schema version and the expected version were applied. + for version := slices.Min(versions); version < expectedDbSchemaVersion; version++ { + if !slices.Contains(versions, version) { release := "UNKNOWN" - if releaseVersion, ok := schemaVersions[missing]; ok { + if releaseVersion, ok := schemaVersions[version]; ok { release = releaseVersion } @@ -86,18 +103,20 @@ func CheckSchema(ctx context.Context, db *database.DB) error { "%w: incomplete database schema upgrade: intermediate version v%d (%s) is missing, "+ "please inspect the icingadb_schema database table and ensure that all database "+ "migrations were applied in order after upgrading Icinga DB", - ErrSchemaMismatch, missing, release) + ErrSchemaMismatch, version, release) } } - if latestVersion := versions[len(versions)-1]; latestVersion != expectedDbSchemaVersion { - // Since these error messages are trivial and mostly caused by users, we don't need - // to print a stack trace here. However, since errors.Errorf() does this automatically, - // we need to use fmt instead. - return fmt.Errorf("%w: v%d (expected v%d), "+ - "please apply the %s.sql schema upgrade file to your database after upgrading Icinga DB: "+ - "https://icinga.com/docs/icinga-db/latest/doc/04-Upgrading/", - ErrSchemaMismatch, latestVersion, expectedDbSchemaVersion, schemaVersions[expectedDbSchemaVersion]) + // Extend the prior check by checking if the schema updates were applied in a monotonic increasing order. + // However, this returns an ErrSchemaImperfect error instead of an ErrSchemaMismatch. + for i := 0; i < len(versions)-1; i++ { + if versions[i] != versions[i+1]-1 { + return fmt.Errorf( + "%w: unexpected schema upgrade order after schema version %d, "+ + "please inspect the icingadb_schema database table and ensure that all database "+ + "migrations were applied in order after upgrading Icinga DB", + ErrSchemaImperfect, versions[i]) + } } return nil From 14fb0c8a59fee682803f601679a5657d0d4056a6 Mon Sep 17 00:00:00 2001 From: Alvar Penning Date: Mon, 23 Mar 2026 17:44:27 +0100 Subject: [PATCH 4/4] CheckSchema: Mention multiple schema upgrades When skipping versions, all required schema upgrades are needed to be applied. While this is mentioned in our docs, the output might be misleading. Now, each and every intermediate schema version is listed as well. --- pkg/icingadb/schema.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/pkg/icingadb/schema.go b/pkg/icingadb/schema.go index 50920499b..fcdcdacac 100644 --- a/pkg/icingadb/schema.go +++ b/pkg/icingadb/schema.go @@ -8,6 +8,7 @@ import ( "os" "path" "slices" + "strings" "github.com/icinga/icinga-go-library/backoff" "github.com/icinga/icinga-go-library/database" @@ -85,10 +86,19 @@ func CheckSchema(ctx context.Context, db *database.DB) error { // Check if the latest schema version was imported. if latestVersion := slices.Max(versions); latestVersion != expectedDbSchemaVersion { + var missingUpgrades []string + for version := latestVersion + 1; version <= expectedDbSchemaVersion; version++ { + if release, ok := schemaVersions[version]; ok { + missingUpgrades = append(missingUpgrades, release+".sql") + } else { + missingUpgrades = append(missingUpgrades, fmt.Sprintf("UNKNOWN (v%d)", version)) + } + } + return fmt.Errorf("%w: v%d (expected v%d), "+ - "please apply the %s.sql schema upgrade file to your database after upgrading Icinga DB: "+ - "https://icinga.com/docs/icinga-db/latest/doc/04-Upgrading/", - ErrSchemaMismatch, latestVersion, expectedDbSchemaVersion, schemaVersions[expectedDbSchemaVersion]) + "please apply the following schema upgrade(s) to your database in order: %s "+ + "(https://icinga.com/docs/icinga-db/latest/doc/04-Upgrading/)", + ErrSchemaMismatch, latestVersion, expectedDbSchemaVersion, strings.Join(missingUpgrades, ", ")) } // Check if all schema updates between the oldest schema version and the expected version were applied.