Dev

.github/workflows/build-and-test.yml

@@ -2,7 +2,7 @@ name: build-and-test.yml
 on:
   push:
     branches:
-      - main
+      - '**'
   pull_request:
     branches:
       - main

.mockery.yml

@@ -7,3 +7,12 @@ packages:
       Querier:
         config:
           filename: querier_mock.go
+  github.com/walmaa/skemr-api/internal/service:
+    interfaces:
+      ScopeResolver:
+        config:
+          filename: scope_resolver_mock.go
+      RuleStore:
+        config:
+          filename: rule_store_mock.go
+

skemr-api/cmd/server/main.go

@@ -92,21 +92,22 @@ func main() {
 		TLSConfig: nil,
 	})
 
+	if cfg.App.Env == "dev" {
+		runSchema(conn)
+		seedTestData(conn)
+	}
+
 	queries := sqlc.New(conn)
+	scopeResolver := service.NewScopeResolver(queries)
 	projectService := service.NewProjectService(queries)
+	databaseChangeService := service.NewDatabaseChangeService(queries, scopeResolver)
 	databaseService := service.NewDatabaseService(queries, taskClient)
 	webhookService := service.NewWebhookService(queries)
 	projectSecretsService := service.NewAccessTokenService(queries)
-	ruleService := service.NewRuleService(queries)
+	ruleService := service.NewRuleService(queries, scopeResolver)
 	databaseEntityService := service.NewDatabaseEntityService(queries)
-	integrationService := service.NewIntegrationService(ruleService)
-
-	if cfg.App.Env == "dev" {
-		runSchema(conn)
-		seedTestData(conn)
-	}
-
-	worker.StartTaskWorkers(queries, cfg)
+	pipelineRunService := service.NewPipelineRunService(queries, scopeResolver)
+	integrationService := service.NewIntegrationService(ruleService, pipelineRunService)
 
 	// Initialize services
 	services := &routers.Services{
@@ -117,8 +118,12 @@ func main() {
 		RuleService:           ruleService,
 		DatabaseEntityService: databaseEntityService,
 		IntegrationService:    integrationService,
+		DatabaseChangeService: databaseChangeService,
+		PipelineRunService:    pipelineRunService,
 	}
 
+	worker.StartTaskWorkers(queries, cfg)
+
 	// Initialize router
 	router := routers.InitRouter(services)
 

skemr-api/db/migrations/20260225122322_init_schema.sql

@@ -96,20 +96,10 @@ CREATE TABLE databases
     failed_connection_attempts INTEGER     NOT NULL DEFAULT 0,
     created_at                 TIMESTAMPTZ NOT NULL DEFAULT NOW(),
     updated_at                 TIMESTAMPTZ NOT NULL DEFAULT NOW(),
-    CONSTRAINT unique_database_name_per_project UNIQUE (display_name, project_id)
+    CONSTRAINT unique_database_name_per_project UNIQUE (display_name, project_id),
+    CONSTRAINT databases_id_project_id_unique UNIQUE (id, project_id)
 );
 
-CREATE TABLE migration_statements
-(
-    id            UUID PRIMARY KEY                    DEFAULT gen_random_uuid(),
-    raw_statement TEXT                       NOT NULL,
-    action        migration_statement_action NOT NULL,
-    status        migration_status           NOT NULL DEFAULT 'pending',
-    target        TEXT,
-    relation_name TEXT
-);
-
-
 CREATE TABLE tables
 (
     id        UUID PRIMARY KEY DEFAULT gen_random_uuid(),
@@ -120,32 +110,71 @@ CREATE TABLE tables
 CREATE TABLE database_entities
 (
     id            uuid PRIMARY KEY                DEFAULT gen_random_uuid(),
-    fingerprint   text,                                                   -- this is used to track the same entity across syncs even if it is renamed.
-    project_id    uuid                   NOT NULL REFERENCES projects (id) ON DELETE CASCADE,
-    database_id   uuid                   NOT NULL REFERENCES databases (id) ON DELETE CASCADE,
+    fingerprint   text                   NOT NULL,                        -- this is used to track the same entity across syncs even if it is renamed.
+    project_id    uuid                   NOT NULL,
+    database_id   uuid                   NOT NULL,
     status        database_entity_status NOT NULL DEFAULT 'active',
     deleted_at    TIMESTAMPTZ            NULL,                            -- Set when status is 'deleted' to track when it was deleted
     first_seen_at TIMESTAMPTZ            NOT NULL DEFAULT NOW(),          -- Track when we first saw this entity
     entity_type   database_entity_type   NOT NULL,
-    parent_id     uuid                   NULL REFERENCES database_entities (id),
+    parent_id     uuid                   NULL,
 
     -- generic identity at this node
     name          text                   NOT NULL,                        -- e.g. "public", "users", "email", "my_view"
-    attributes    jsonb,                                                  -- Store any additional metadata about the entity here
+    attributes    jsonb                  NOT NULL DEFAULT '{}'::jsonb,    -- Store any additional metadata about the entity here
 
     created_at    TIMESTAMPTZ            NOT NULL DEFAULT NOW(),
 
+    CONSTRAINT database_entities_parent_same_database_fkey
+        FOREIGN KEY (parent_id, database_id)
+        REFERENCES database_entities (id, database_id) ON DELETE CASCADE, -- Ensure parent_id references an entity in the same database
+
+    CONSTRAINT database_entities_database_project_fkey
+        FOREIGN KEY (database_id, project_id)
+            REFERENCES databases (id, project_id)
+            ON DELETE CASCADE, -- Ensure entities are only in the same project
+
+    CONSTRAINT database_entities_id_database_id_unique
+        UNIQUE (id, database_id), -- for rule composite fkey
+
     UNIQUE NULLS NOT DISTINCT (database_id, name, entity_type, parent_id) -- Ensure we do not map the same entity twice, use NULLS NOT DISTINCT so parentless are not duplicated
+
+);
+
+CREATE TABLE database_changes
+(
+    id            UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    database_id   UUID NOT NULL REFERENCES databases (id) ON DELETE CASCADE,
+    entity_id     UUID NOT NULL REFERENCES database_entities (id) ON DELETE CASCADE,
+    action        migration_statement_action NOT NULL,
+    created_at    TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE TABLE pipeline_runs
+(
+    id            UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    database_id   UUID NOT NULL REFERENCES databases (id) ON DELETE CASCADE,
+    status        migration_status NOT NULL DEFAULT 'pending',
+    environment   TEXT,
+    started_at    TIMESTAMPTZ,
+    completed_at  TIMESTAMPTZ,
+    created_at    TIMESTAMPTZ NOT NULL DEFAULT NOW()
 );
 
 
 -- Rules specify the protection mechanisms for databases, schemas, tables, and columns.
 CREATE TABLE rules
 (
     id                 UUID PRIMARY KEY     DEFAULT gen_random_uuid(),
-    name               TEXT        NOT NULL, -- User defined for rule
+    name               TEXT        NOT NULL,                     -- Defined by user
     type               rule_type   NOT NULL,
-    database_entity_id uuid        NOT NULL REFERENCES database_entities (id) ON DELETE CASCADE,
+    attributes         jsonb       NOT NULL DEFAULT '{}'::jsonb, -- Metadata about the rule, removal_date for deprecated types for example
+    database_entity_id uuid        NOT NULL REFERENCES database_entities (id),
     database_id        uuid        NOT NULL REFERENCES databases (id) ON DELETE CASCADE,
-    created_at         TIMESTAMPTZ NOT NULL DEFAULT NOW()
+    created_at         TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    CONSTRAINT unique_rule_name_per_database
+        UNIQUE (name, database_id),
+    CONSTRAINT rules_database_entity_database_fkey
+        FOREIGN KEY (database_entity_id, database_id)
+        REFERENCES database_entities (id, database_id) ON DELETE CASCADE --- ensure rules are only applied to entities in the same database
 );

skemr-api/db/queries/database_changes.sql

@@ -0,0 +1,19 @@
+-- name: CreateDatabaseChange :one
+INSERT INTO database_changes
+    (database_id, entity_id, action)
+VALUES (@database_id, @entity_id, @action)
+RETURNING *;
+
+-- name: GetDatabaseChangeByDatabaseIdAndId :one
+SELECT *
+FROM database_changes c
+WHERE c.id = @id
+  AND c.database_id = @database_id
+LIMIT 1;
+
+-- name: GetDatabaseChangesByDatabaseIdAndId :many
+SELECT *
+FROM database_changes c
+WHERE c.database_id = @database_id
+ORDER BY c.created_at DESC
+LIMIT sqlc.narg('limit')::int OFFSET sqlc.narg('offset')::int;

skemr-api/db/queries/database_entities.sql

@@ -11,6 +11,14 @@ WHERE id = @id
   AND project_id = @project_id
 LIMIT 1;
 
+-- name: GetDatabaseEntityByProjectIdDatabaseIdAndId :one
+SELECT *
+FROM database_entities
+WHERE id = @id
+  AND project_id = @project_id
+  AND database_id = @database_id
+LIMIT 1;
+
 -- name: GetDatabaseEntitiesByProjectId :many
 SELECT *
 FROM database_entities
@@ -60,7 +68,7 @@ LIMIT 1;
 -- name: CreateDatabaseEntity :one
 INSERT INTO database_entities
 (project_id, database_id, entity_type, parent_id, name, attributes, fingerprint)
-VALUES (@project_id, @database_id, @entity_type, @parent_id, @name, @attributes, @fingerprint)
+VALUES (@project_id, @database_id, @entity_type, @parent_id, @name, COALESCE(@attributes, '{}'::jsonb), @fingerprint)
 RETURNING *;
 
 -- name: UpdateDatabaseEntityName :one

skemr-api/db/queries/pipeline_runs.sql

@@ -0,0 +1,18 @@
+-- name: GetPipelineRunByDatabaseIdAndId :one
+SELECT *
+FROM pipeline_runs
+WHERE database_id = @database_id
+  AND id = @id
+LIMIT 1;
+
+-- name: GetPipelineRunsByDatabaseId :many
+SELECT *
+FROM pipeline_runs
+WHERE database_id = @database_id
+ORDER BY created_at DESC;
+
+-- name: CreatePipelineRun :one
+INSERT INTO pipeline_runs
+    (database_id, status, environment, completed_at)
+VALUES (@database_id, @status, @environment, @completed_at)
+RETURNING *;

skemr-api/db/queries/rules.sql

@@ -1,22 +1,32 @@
 -- name: GetRule :one
 SELECT *
+FROM rules r
+WHERE r.database_id = @database_id
+  AND r.id = @rule_id
+LIMIT 1;
+
+-- name: GetRuleByDatabaseAndName :one
+SELECT *
 FROM rules
-WHERE database_id = @database_id AND id = @rule_id
+WHERE database_id = @database_id
+  AND name = @name
 LIMIT 1;
 
 -- name: GetRuleWithEntity :one
-SELECT
-    sqlc.embed(r),
-    sqlc.embed(de)
+SELECT sqlc.embed(r),
+       sqlc.embed(de)
 FROM rules r
-JOIN database_entities de ON r.database_entity_id = de.id
-WHERE r.database_id = @database_id AND r.id = @rule_id
+         JOIN databases d ON r.database_id = d.id
+         JOIN database_entities de ON r.database_entity_id = de.id
+WHERE d.project_id = @project_id
+  AND r.database_id = @database_id
+  AND r.id = @rule_id
 LIMIT 1;
 
 -- name: CreateRule :one
 INSERT INTO rules
-    (name, type, database_entity_id, database_id)
-VALUES (@name, @type, @database_entity_id, @database_id)
+    (name, type, database_entity_id, database_id, attributes)
+VALUES (@name, @type, @database_entity_id, @database_id, COALESCE(@attributes, '{}'::jsonb))
 RETURNING *;
 
 -- name: UpdateRule :exec
@@ -29,7 +39,8 @@ RETURNING *;
 -- name: DeleteRule :exec
 DELETE
 FROM rules
-WHERE database_id = @database_id AND id = @rule_id;
+WHERE database_id = @database_id
+  AND id = @rule_id;
 
 
 -- name: ListRulesByDatabaseId :many
@@ -38,12 +49,13 @@ FROM rules
 WHERE database_id = @database_id;
 
 -- name: GetRulesWithEntities :many
-SELECT
-    sqlc.embed(r),
-    sqlc.embed(de)
-FROM rules r
-JOIN database_entities de ON r.database_entity_id = de.id
-WHERE r.database_id = @database_id;
+SELECT sqlc.embed(rules),
+       sqlc.embed(database_entities)
+FROM rules
+         JOIN databases ON rules.database_id = databases.id
+         JOIN database_entities ON rules.database_entity_id = database_entities.id
+WHERE rules.database_id = @database_id
+  AND databases.project_id = @project_id;
 
 
 -- name: ListRulesByCriteria :many