diff --git a/.agents/skills/designing-db-schemas/SKILL.md b/.agents/skills/designing-db-schemas/SKILL.md new file mode 100644 index 0000000000..913c25a6a4 --- /dev/null +++ b/.agents/skills/designing-db-schemas/SKILL.md @@ -0,0 +1,209 @@ +--- +name: designing-db-schemas +description: | + WSO2 API Platform-specific database schema design, change, and review skill. Use proactively whenever: + - Adding a new table to platform-api or developer-portal schemas + - Modifying an existing table (new column, type change, constraint, index) + - Reviewing schema changes before a PR + - Writing or evaluating a migration plan + - Asking "is this table well designed?" or "what indexes does this table need?" + + Scope: applies to every schema.*.sql file in the repository. Gateway controller schemas (gateway/gateway-controller/) are included for structural rules (R1–R2, R4–R10) but R3 type validation is skipped for them — the gateway controller team owns their type choices. +allowed-tools: Bash, Read, Edit, Write, Glob +--- + +# WSO2 API Platform — Database Schema Design Rules + +This skill governs **all schema work** for the WSO2 API Platform: designing new tables, modifying existing ones, and reviewing DDL changes for correctness. It is not a post-hoc review tool — it is the process to follow when writing DDL. + +The detailed rules live in **`references/api-platform-db-schema-rules.md`** (next to this skill). This file describes the workflow; the reference file is the source of truth for every rule. + +## Usage + +``` +/api-platform-db-schema-design-rules [table-name | path-to-schema-file] +``` + +- **No argument** — review all in-scope schema files in the project. +- **Table name** — apply the relevant workflow for that table (add or modify). +- **Schema file path** — review that specific file only. + +--- + +## Schema File Scope + +Locate all schema files with: + +```bash +find . -name "schema*.sql" | sort +``` + +All `schema*.sql` files in the repository are in scope. The rules that apply depend on which component owns the file: + +| Component | Path pattern | Rules applied | +|---|---|---| +| Platform API | `platform-api/` | R1–R10 (all rules) | +| Developer Portal | `portals/` | R1–R10 (all rules) | +| Gateway Controller | `gateway/gateway-controller/` | R1–R2, R4–R10 — **R3 type rules skipped** | +| Any other component | elsewhere | R1–R10 (all rules) | + +**Gateway controller type exemption** — `gateway/gateway-controller/` schemas are owned by a separate team who manage their own type choices. Apply all structural, constraint, audit, index, alignment, and idempotency rules (R1–R2, R4–R10) as normal, but do **not** raise R3 findings (column types, JSONB, BOOLEAN, TIMESTAMPTZ, VARCHAR widths) against those files. + +--- + +## Workflows + +Choose a workflow based on whether you are making a change or reviewing existing DDL. + +--- + +### Workflow A — Making a Schema Change + +Use this workflow for every ADD TABLE, ADD COLUMN, ALTER COLUMN, ADD INDEX, or ADD CONSTRAINT. + +#### Step A1 — Read the schemas first + +Locate and read all schema files before writing any DDL: + +```bash +find . -name "schema*.sql" | sort +``` + +Read the full content of each file before drafting anything. Apply R3 type rules to all files **except** those under `gateway/gateway-controller/`. + +#### Step A2 — Open the rules reference + +Read `references/api-platform-db-schema-rules.md` in full. The rules you need depend on the change: + +| Change type | Rules to apply | +|---|---| +| New table | R1 (identity), R2 (org-scoping), R3 (types), R4 (constraints), R5 (audit columns), R6 (indexes), R10 (naming) | +| New column | R3 (type), R4 (constraints), R5 (audit), R6 (index if filterable), R10 (lowercase name) | +| Type change | R3 (correct type for target engine), R8 (counterpart schemas if multi-engine) | +| New index | R6 (correct pattern — FK, status, compound, partial), R10 (lowercase index name) | + +Use the quick-reference templates in the rules file as your starting point. + +#### Step A3 — Self-review checklist + +Before writing DDL to disk, confirm each item passes: + +``` +[ ] R1 Entity tables: uuid VARCHAR(40) PRIMARY KEY +[ ] R1 Junction/mapping tables: composite PRIMARY KEY — not UNIQUE-only, not surrogate UUID +[ ] R1 Non-leading FK columns of a composite PK have their own indexes +[ ] R1 Named resource tables carry handle + name + version (all NOT NULL) +[ ] R2 organization_uuid FK present; UNIQUE constraints include it (if org-scoped) +[ ] R3 No bare TEXT in Postgres: use VARCHAR(N), BYTEA, or JSONB (query-only) — SQLite TEXT / SQL Server NVARCHAR(MAX) are intentional (R8), not findings +[ ] R3 Large/variable payloads use BYTEA/BLOB — not wide VARCHAR +[ ] R3 Opaque JSON stored as BYTEA/BLOB — JSONB only when queried with JSON operators inside Postgres +[ ] R3 Boolean flags: SMALLINT (Postgres) or INTEGER (SQLite/SQL Server), DEFAULT 1/0 — no BOOLEAN +[ ] R3 VARCHAR widths match the standard table; nothing above VARCHAR(1023) for plain storage +[ ] R3 Indexed/UNIQUE columns ≤ VARCHAR(255) (safe across all engines with utf8mb4) +[ ] R3 All timestamps: TIMESTAMPTZ (Postgres) / DATETIME (SQLite) / DATETIME2 (SQL Server) +[ ] R4 No CHECK constraints for enum/status values — validation in application code only +[ ] R4 Every FK has an explicit ON DELETE clause +[ ] R5 User-initiated table → all four audit columns present (created_by/at, updated_by/at) +[ ] R5 System-managed table → created_by and updated_by are ABSENT +[ ] R5 Every domain entity table has data_version VARCHAR(20) NOT NULL DEFAULT '1.0' +[ ] R6 FK columns have indexes +[ ] R6 organization_uuid has an index (if org-scoped) +[ ] R6 status column has an index if used as a filter +[ ] R8 Change applied to all schema files (or divergence is intentional and documented) +[ ] R9 All DDL is idempotent (IF NOT EXISTS / OBJECT_ID guards) +[ ] R10 All identifiers (table/column/index/constraint) are lowercase snake_case +[ ] R10 Pure junction/mapping tables are named with a _mappings suffix +``` + +#### Step A4 — Write the DDL + +Write idempotent DDL using the engine-specific guards: + +```sql +-- PostgreSQL / SQLite +CREATE TABLE IF NOT EXISTS (...); +CREATE INDEX IF NOT EXISTS idx_... ON ...; + +-- SQL Server +IF OBJECT_ID(N'dbo.
', N'U') IS NULL +CREATE TABLE dbo.
(...); + +IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = N'idx_...' AND object_id = OBJECT_ID(N'dbo.
')) +CREATE INDEX idx_... ON dbo.
(...); +``` + +Keep `CREATE INDEX` statements in a dedicated block after all `CREATE TABLE` statements. + +#### Step A5 — Apply to all schema files + +Apply the change to every in-scope schema file. For type-level differences between engines, see R8 (intentional divergences). Everything else must be structurally identical. + +--- + +### Workflow B — Reviewing Existing DDL (PR / Audit) + +#### Step B1 — Locate and read all schema files + +```bash +find . -name "schema*.sql" | sort +``` + +Read each file in full before assessing anything. Note which files are under `gateway/gateway-controller/` — those skip R3 type validation. + +#### Step B2 — Open the rules reference + +Read `references/api-platform-db-schema-rules.md`. Evaluate every rule group (R1–R10) in order. + +#### Step B3 — Record findings + +For each violation: + +| Field | Value | +|---|---| +| **Rule** | e.g. `R3-JSONB` | +| **Table · column** | exact location | +| **Severity** | `HIGH` (data safety / correctness) · `MEDIUM` (missing guarantee or index) · `LOW` (style / inconsistency) | +| **Finding** | what is wrong | +| **Fix** | the exact DDL needed | + +#### Step B4 — Cross-check multi-engine alignment + +After the per-table review, verify all schema files are structurally in sync (see R8). Intentional type-level divergences are not findings. + +#### Step B5 — Write findings to JSON + +Write a structured findings file so findings can be consumed by other tools or tracked across reviews. The script path below is **relative to this skill's directory** — run it from the skill folder (where this SKILL.md lives), and pass an absolute `--out` so the report lands in the project rather than the skill folder: + +```bash +# cwd = this skill's directory +node scripts/generate-schema-report.js \ + --findings '' \ + --schema '' \ + --out "$(git rev-parse --show-toplevel)/schema-reports/schema-review.json" +``` + +See the script's `--help` for all flags. The output shape is: + +```json +{ + "meta": { "schema": "", "reviewedAt": "", "rules": ["R1","R2","R3","R4","R5","R6","R7","R8","R9","R10"] }, + "findings": [ + { "id": "r3-001", "severity": "HIGH", "rule": "R3-NO-TEXT", "table": "
", "column": "", "finding": "...", "fix": "..." } + ] +} +``` + +#### Step B6 — Report summary + +Produce a findings table sorted by severity. Include a "No issues" row for any rule group that passed cleanly — reviewers need to know what was checked. + +--- + +## Quick-Reference Templates + +Copy-paste DDL templates and the standard column-type/width cheat sheet live in **`references/api-platform-db-schema-rules.md`**: + +- **New entity table** and **standard column types & widths** — see the *Quick-Reference Templates* section at the end of the rules file. +- **New junction/mapping table** — see rule **R1-COMPOSITE-PK**. + +Use these as the starting point when writing DDL (Step A4). diff --git a/.agents/skills/designing-db-schemas/references/api-platform-db-schema-rules.md b/.agents/skills/designing-db-schemas/references/api-platform-db-schema-rules.md new file mode 100644 index 0000000000..b3b11c54c1 --- /dev/null +++ b/.agents/skills/designing-db-schemas/references/api-platform-db-schema-rules.md @@ -0,0 +1,388 @@ +# API Platform DB Schema Rules (R1–R10) + +Reference for the `designing-db-schemas` skill. These are the WSO2 API Platform conventions for relational database schemas. Step A2 and Step B2 of the skill read this file and evaluate each rule against the schema. + +**Scope:** applies to every `schema*.sql` file in the repository. For `gateway/gateway-controller/` schemas, **skip R3** (type rules) — the gateway controller team owns their type choices. All other rules (R1–R2, R4–R10) apply to every schema file including gateway controller. + +> **Recording blanket-missing findings.** When a rule is violated uniformly across many tables (e.g. a convention that no tables follow), record **one representative finding** that names the pattern and gives a couple of examples, rather than one finding per table. + +--- + +## R1 · Primary Key & Identity + +Every table must have a single UUID primary key and, where it is a named resource, the standard identity triple. + +**R1-UUID** — Primary key must be `uuid VARCHAR(40) PRIMARY KEY`. Do not use `SERIAL`, `BIGINT`, or `INTEGER` as a primary key for domain entities. Junction/mapping tables must use a composite PK (see R1-COMPOSITE-PK). + +**R1-COMPOSITE-PK** — Pure junction/mapping tables (those whose only purpose is to link two or more entities) must use a composite `PRIMARY KEY` over their FK columns — not a surrogate UUID, and not a bare `UNIQUE` constraint. + +Use a composite PK when **all** of the following are true: +- Every query hits the table via the composite key (no query looks up a row by a single generated ID) +- No other table holds a FK reference to a row in this table by a surrogate ID +- The table has no independent lifecycle (rows are inserted or deleted, never updated in place by identity) + +Do **not** use a composite PK (use a UUID PK instead) when: +- Another table references individual rows by ID (e.g. an audit log or event stream that stores a FK to this table's row) +- The table is exposed as a standalone resource in an API with its own URL (e.g. `/associations/{id}`) + +**Why composite PK over UNIQUE-only** — A bare `UNIQUE` constraint without a `PRIMARY KEY` breaks Postgres logical replication for `UPDATE` and `DELETE` operations. Postgres `REPLICA IDENTITY DEFAULT` uses the PK to identify rows in the WAL stream; without a PK it falls back to `REPLICA IDENTITY FULL` (logs entire old row on every write — high WAL volume) or replication fails entirely. CDC tools (Debezium, AWS DMS) have the same requirement. Distributed SQL engines (CockroachDB, YugabyteDB) silently add a hidden PK if you omit one, with unpredictable sharding consequences. + +**Column order** — Put the most common filter/scope column first (typically `organization_uuid`), then the remaining FKs. The leading column is covered by the PK index; add separate indexes only for the non-leading FK columns. + +```sql +-- Correct composite PK pattern +CREATE TABLE IF NOT EXISTS ( + organization_uuid VARCHAR(40) NOT NULL, + entity_a_uuid VARCHAR(40) NOT NULL, + entity_b_uuid VARCHAR(40) NOT NULL, + created_at TIMESTAMPTZ DEFAULT NOW(), + PRIMARY KEY (organization_uuid, entity_a_uuid, entity_b_uuid), + FOREIGN KEY (organization_uuid) REFERENCES organizations(uuid) ON DELETE CASCADE, + FOREIGN KEY (entity_a_uuid) REFERENCES entity_a(uuid) ON DELETE CASCADE, + FOREIGN KEY (entity_b_uuid) REFERENCES entity_b(uuid) ON DELETE CASCADE +); + +-- Indexes for non-leading FK columns only (leading column covered by PK) +CREATE INDEX IF NOT EXISTS idx__entity_a_uuid ON (entity_a_uuid); +CREATE INDEX IF NOT EXISTS idx__entity_b_uuid ON (entity_b_uuid); +``` + +**R1-IDENTITY** — Tables representing named resources (APIs, gateways, providers, applications, subscriptions, or any domain entity with a stable slug and a display name) must carry the full identity triple directly: + +```sql +handle VARCHAR(40) NOT NULL, -- url-safe slug, immutable once set +name VARCHAR(255) NOT NULL, -- human-readable display name +version VARCHAR(30) NOT NULL DEFAULT 'v1.0', -- semver or opaque version string + +-- handle is unique per organisation, never globally: the same handle may +-- exist in different organisations. Enforce org-scoped, not UNIQUE(handle). +UNIQUE(organization_uuid, handle), +``` + +Identity must be denormalised onto the table itself so queries against a single table are self-contained. Do not rely on a parent record for identity fields. + +**R1-HANDLE-NAME** — `handle` and `name` are distinct: +- `handle` is the URL-safe slug used in API paths (e.g. `/resources/{handle}`). It must be unique within its scope (always org-scoped) and treated as immutable after creation. +- `name` is the human-readable display string. It may change and **is not required to be unique** — not even within a project. + +Conflating them (using `name` as the slug, or allowing `handle` to contain spaces) is a finding. Adding a `UNIQUE` constraint on `name` (alone or combined with `project_uuid`) for API-type entities is also a finding — enforce only `UNIQUE(organization_uuid, handle)`. + +--- + +## R2 · Organisation Scoping + +**R2-ORG-FK** — Every domain table that belongs to an organisation must carry `organization_uuid VARCHAR(40) NOT NULL` with: + +```sql +FOREIGN KEY (organization_uuid) REFERENCES organizations(uuid) ON DELETE CASCADE +``` + +**R2-ORG-UNIQUE** — The `handle` of a named resource must be unique within the organisation: + +```sql +UNIQUE(organization_uuid, handle) -- not UNIQUE(handle) alone +``` + +A `UNIQUE(handle)` without the org scope is a critical data-isolation bug. `name` does **not** need a uniqueness constraint — two resources may share a display name within the same org or project. + +--- + +## R3 · Column Types + +**R3-NO-TEXT** — This rule is **engine-scoped to PostgreSQL**. In a Postgres schema, do not use the bare `TEXT` type for any column — use a bounded `VARCHAR(N)`, a binary type (`BYTEA`), or `JSONB` when content is queried. A bare `TEXT` column in a Postgres schema is a finding at MEDIUM severity. + +Do **not** raise R3-NO-TEXT against SQLite or SQL Server schema files: per R8, `TEXT` is the intended type in SQLite (for JSON, large text, and opaque payloads) and `NVARCHAR(MAX)` / `TEXT` is the intended type in SQL Server. These are intentional type-level divergences, not findings. + +**R3-LARGE-PAYLOAD** — Any payload that can grow large or is variable-length must use the engine-appropriate large type, never a wide VARCHAR. Apply: binary payloads → `BYTEA` (Postgres) / `BLOB` (SQLite) / `VARBINARY(MAX)` (SQL Server); large text payloads → `BYTEA` (Postgres) / `TEXT` (SQLite) / `NVARCHAR(MAX)` (SQL Server), per the R8 divergence table. Columns that always need this: `openapi_spec`, `model_list`, `content`, `configuration`, `properties`, `manifest`, `policy_definition`, `metadata`, `api_key_hashes`, or any future column whose value can exceed a few hundred bytes. (The SQLite/SQL Server forms above are the intended divergences — do not flag them as R3-NO-TEXT.) + +**R3-JSONB** — In PostgreSQL, use `JSONB` only when the application queries inside the JSON using Postgres JSON operators. Evidence that a column is actively queried inside: +1. The `DEFAULT` is a JSON literal like `'{}'` +2. The column name implies structure: `settings`, `event_data` +3. The same column in a sibling table already uses `JSONB` + +SQLite and SQL Server equivalents (`TEXT` / `NVARCHAR(MAX)`) are intentional type-level divergences — not findings. + +**R3-JSONB-SCAN-COMPAT** — **PostgreSQL only** (`JSONB` does not exist in SQLite or SQL Server, so this rule never applies to those files). Do not use `JSONB` if the application layer scans it into a plain `string` variable and calls `json.Unmarshal` manually. Postgres drivers return JSONB as binary, which breaks `string` scan targets at runtime. Only use `JSONB` when the scan target implements `sql.Scanner` (e.g. `pgtype.JSONB`, `json.RawMessage`, or a custom struct). + +**R3-BOOLEAN-AS-INT** — Do not use the `BOOLEAN` type. Represent boolean flags as: +- `SMALLINT DEFAULT 0` / `SMALLINT NOT NULL DEFAULT 1` — Postgres +- `INTEGER DEFAULT 0` / `INTEGER NOT NULL DEFAULT 1` — SQLite / SQL Server + +**R3-TIMESTAMPTZ** — Use `TIMESTAMPTZ` for **all** timestamp columns in PostgreSQL. `TIMESTAMP` (without timezone) is a bare clock reading with no timezone attached — a finding at MEDIUM. Use `DATETIME` in SQLite and `DATETIME2(7) DEFAULT SYSUTCDATETIME()` in SQL Server. + +**R3-VARCHAR-SIZES** — Standard widths: + +| Purpose | Width | +|---|---| +| UUID / foreign key | `VARCHAR(40)` | +| User identity (email / sub) | `VARCHAR(200)` | +| Handle / name / display string | `VARCHAR(255)` | +| Version string | `VARCHAR(30)` | +| Lifecycle / status enum | `VARCHAR(20)` | +| Description / reason | `VARCHAR(1023)` | +| Token / hash | `VARCHAR(512)` or `VARCHAR(64)` as appropriate | + +Any width above `VARCHAR(1023)` is a strong signal that the column should be `BYTEA`/`BLOB` instead. + +**R3-VARCHAR-ENGINE-LIMITS** — Key engine limits for indexed/unique columns: + +| Usage | Safe max width | +|---|---| +| Plain storage, no index | `VARCHAR(1023)` | +| Appears in a UNIQUE constraint or any index | `VARCHAR(255)` (safe across all engines with utf8mb4) | +| Oracle target (any index or non-extended) | `VARCHAR(255)` | +| MySQL target (utf8mb4, default prefix limit) | `VARCHAR(191)` | + +When a column must be unique but its value can be large, store the value in `BYTEA`/`BLOB` and put a SHA-256 hash in a separate `VARCHAR(64)` column — index and unique-constrain the hash, not the value. + +--- + +## R4 · Constraints + +**R4-NO-ENUM-CHECK** — **Do NOT add `CHECK (col IN (...))` constraints for enum or status columns.** Enum validation belongs in application code (Go constants, service-layer validation). DB-layer enum checks require a DDL migration just to add a new valid value. + +The only `CHECK` constraints that belong in the schema are structural/cross-column invariants: +```sql +-- Cross-column consistency: both NULL or both non-NULL +CONSTRAINT chk_throttle_pair CHECK ( + (throttle_limit_count IS NULL AND throttle_limit_unit IS NULL) OR + (throttle_limit_count IS NOT NULL AND throttle_limit_unit IS NOT NULL) +) +-- Temporal consistency +CHECK (revoked_at IS NULL OR status = 'revoked') +``` + +**R4-NOT-NULL** — Columns that are always required must be `NOT NULL`. Common offenders: `organization_uuid`, `name`, `handle`, `version`, `status`. + +**R4-FK-BEHAVIOR** — Every foreign key must declare an explicit `ON DELETE` action: + +| Relationship | Rule | +|---|---| +| Child owned by parent (cascade delete is safe) | `ON DELETE CASCADE` | +| Reference that must not dangle but parent cannot be deleted | `ON DELETE RESTRICT` | +| Optional reference — row survives if target is deleted | `ON DELETE SET NULL` | + +Omitting `ON DELETE` means the DB default (`RESTRICT` in most engines) applies silently — flag it. + +--- + +## R5 · Audit Columns + +**R5-AUDIT-SET** — Only tables written as a **direct consequence of a user-initiated action** (e.g. a REST API call) carry `created_by` / `updated_by`. Tables written by background sync, callbacks, or internal system processes must **not** include these columns. + +Rule of thumb: ask "does a human-initiated request cause this row to be inserted or updated?" If yes → include the full audit set. If no → omit `created_by` and `updated_by`. + +```sql +data_version VARCHAR(20) NOT NULL DEFAULT '1.0', +created_by VARCHAR(200), +created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, -- DATETIME on SQLite / DATETIME2 on SQL Server +updated_by VARCHAR(200), +updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, +``` + +**R5-IMMUTABLE-CREATED** — `created_by` and `created_at` must never be updated after insert. + +**R5-REVOKE-PATTERN** — Tables with soft-revocation add: + +```sql +revoked_by VARCHAR(200), +revoked_at TIMESTAMPTZ, +CHECK (revoked_at IS NULL OR status = 'revoked') +``` + +**R5-DATA-VERSION** — Every domain entity table must include `data_version VARCHAR(20) NOT NULL DEFAULT '1.0'`. Place it immediately before `created_by`. Excluded tables: junction/mapping tables and pure system/event tables (e.g. `events`, `gateway_states`, `audit`, `deployment_status`, `gateway_custom_policy_usages`, `application_api_keys`, `application_artifacts`, `gateway_association_mappings`). + +--- + +## R6 · Indexing + +Index every column (or compound) that appears in a `WHERE`, `JOIN`, or `ORDER BY` clause at production query volume. + +**R6-FK-INDEX** — Every foreign key column must have an index unless it is already the leftmost column of the PK or a covering UNIQUE constraint. + +```sql +CREATE INDEX IF NOT EXISTS idx_
_ ON
(); +``` + +**R6-ORG-INDEX** — Every org-scoped table must have an index on `organization_uuid`: + +```sql +CREATE INDEX IF NOT EXISTS idx_
_org ON
(organization_uuid); +``` + +**R6-STATUS-INDEX** — Tables with a `status` column that is filtered in list queries need a status index: + +```sql +CREATE INDEX IF NOT EXISTS idx_
_status ON
(status); +``` + +**R6-COMPOUND-INDEX** — When the common query is `WHERE a = ? AND b = ?`, a single compound index `(a, b)` outperforms two single-column indexes. Most-selective filter first. + +**R6-PARTIAL-INDEX** — Use a partial index when the filter discards most rows: + +```sql +CREATE INDEX IF NOT EXISTS idx_
_expires_at + ON
(expires_at) WHERE expires_at IS NOT NULL; +``` + +**R6-UNIQUE-PARTIAL** — For "at most one default per org" patterns: + +```sql +CREATE UNIQUE INDEX IF NOT EXISTS idx_
_default_per_org + ON
(organization_uuid) WHERE is_default = 1; +``` + +**R6-NO-REDUNDANT-INDEX** — Do not create an index that is a prefix of an existing UNIQUE constraint or PK. + +**R6-GIN-JSONB** — Add a GIN index only when the application concretely queries inside a JSONB column with JSONB operators — do not pre-emptively GIN-index every JSONB column. + +--- + +## R7 · Application Logic Safety + +**R7-NO-SELECT-STAR** — Schema changes break `SELECT *` callers. All queries must select named columns. + +**R7-JSONB-SCAN** — A `JSONB` column must be scanned into a type that implements `sql.Scanner`. Scanning into `string` bypasses Postgres validation. + +**R7-HANDLE-IMMUTABLE** — `handle` must be set on INSERT and never appear in `UPDATE` statements. + +**R7-CREATED-IMMUTABLE** — `created_at` and `created_by` must not appear in `UPDATE` statements. + +**R7-SOFT-DELETE** — Prefer status transitions to terminal states (`REVOKED`, `ARCHIVED`, `RETIRED`) over soft-delete (`deleted_at`) columns. Only add `deleted_at` when the design explicitly requires row-level tombstones. + +**R7-OPTIMISTIC-LOCK** — For resources that are concurrently edited, compare `updated_at` before issuing an UPDATE: + +```sql +UPDATE
+SET ..., updated_at = NOW(), updated_by = $n +WHERE uuid = $1 AND updated_at = $expected_updated_at +``` + +If 0 rows are affected, the caller lost the race and must retry. + +--- + +## R8 · Multi-Engine Schema Alignment + +When the project maintains schema files for multiple database engines (Postgres + SQLite + SQL Server), the files must remain structurally in sync except for the known intentional type-level divergences below. + +**Intentional divergences — not findings:** + +| Feature | SQLite | PostgreSQL | SQL Server | +|---|---|---|---| +| JSON-valued columns (queried with JSON operators) | `TEXT` | `JSONB` | `NVARCHAR(MAX)` | +| JSON-valued columns (opaque storage only) | `TEXT` | `VARCHAR(N)` or `BYTEA` | `VARCHAR(N)` or `NVARCHAR(MAX)` | +| Binary columns | `BLOB` | `BYTEA` | `VARBINARY(MAX)` | +| Large text payloads | `TEXT` | `BYTEA` | `NVARCHAR(MAX)` | +| All timestamps | `DATETIME` | `TIMESTAMPTZ` | `DATETIME2(7) DEFAULT SYSUTCDATETIME()` | +| Boolean flags | `INTEGER` (0/1) | `SMALLINT` (0/1) | `SMALLINT` (0/1) | +| JSON literal defaults | `'{}'` | `'{}'::jsonb` | `'{}'` | + +**R8-SYNC-STRUCTURE** — Table definitions (columns, order, constraints, CHECK values, FK targets) not in the intentional-divergence list must be identical across all schema files. Verify: +- Same columns, same order +- Same `NOT NULL` / nullable on each column +- Same `DEFAULT` values (modulo type syntax) +- Same `CHECK` constraint values +- Same index definitions + +**R8-SQLITE-NO-JSONB** — SQLite does not support `JSONB`. Any `JSONB` in a SQLite schema file is a bug — all JSON columns must be `TEXT`. + +--- + +## R9 · Idempotent DDL + +Every `CREATE TABLE` and `CREATE INDEX` statement must be safe to re-run without errors. + +**R9-TABLE** — Use the engine-specific existence guard: + +```sql +-- PostgreSQL / SQLite +CREATE TABLE IF NOT EXISTS
(...); + +-- SQL Server +IF OBJECT_ID(N'dbo.
', N'U') IS NULL +CREATE TABLE dbo.
(...); +``` + +**R9-INDEX** — Use `IF NOT EXISTS` (Postgres/SQLite) or a `sys.indexes` check (SQL Server): + +```sql +-- PostgreSQL / SQLite +CREATE INDEX IF NOT EXISTS idx_... ON
(...); + +-- SQL Server +IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = N'idx_...' AND object_id = OBJECT_ID(N'dbo.
')) +CREATE INDEX idx_... ON dbo.
(...); +``` + +A `CREATE TABLE` or `CREATE INDEX` without an existence guard is always a finding at MEDIUM severity. + +--- + +## R10 · Naming Conventions + +**R10-LOWERCASE** — Every SQL identifier — table names, column names, index names, and constraint names — must be lowercase `snake_case`. PostgreSQL folds unquoted identifiers to lowercase, so a mixed-case identifier only resolves if **every** reference quotes it (`"MyTable"`), which is brittle and breaks silently across engines and ORMs. Use `organization_uuid`, not `organizationUuid` or `OrganizationUUID`; `idx_apis_org`, not `idx_APIs_Org`. An upper-case or camelCase identifier is a finding at MEDIUM severity. + +**R10-MAPPING-SUFFIX** — Pure junction/mapping tables (those defined under R1-COMPOSITE-PK) must be named with a `_mappings` suffix, e.g. `application_api_mappings`, `gateway_association_mappings`. The suffix distinguishes link tables from entity tables at a glance and keeps the schema self-documenting. A pure mapping table named without the `_mappings` suffix is a finding at LOW severity. (Tables that link two entities but also carry their own identity/lifecycle are entity tables, not mapping tables — they keep a UUID PK and an entity-style name.) + +--- + +## Quick-Reference Templates + +Copy-paste starting points for new DDL. The junction/mapping table template is shown above under **R1-COMPOSITE-PK**. + +### New entity table (Postgres) + +```sql +CREATE TABLE IF NOT EXISTS
( + uuid VARCHAR(40) PRIMARY KEY, + organization_uuid VARCHAR(40) NOT NULL, + handle VARCHAR(40) NOT NULL, + name VARCHAR(255) NOT NULL, + version VARCHAR(30) NOT NULL DEFAULT '1.0', + status VARCHAR(20) NOT NULL DEFAULT 'CREATED', + description VARCHAR(1023), + data_version VARCHAR(20) NOT NULL DEFAULT '1.0', + created_by VARCHAR(200), + created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, + updated_by VARCHAR(200), + updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, + UNIQUE(organization_uuid, handle), + FOREIGN KEY (organization_uuid) REFERENCES organizations(uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_
_org ON
(organization_uuid); +CREATE INDEX IF NOT EXISTS idx_
_status ON
(status); +``` + +### Standard column types & widths + +``` +VARCHAR(20) — status, lifecycle_status, kind, short enums +VARCHAR(30) — version strings (v1.0, v2.3) +VARCHAR(40) — uuid, all FK columns referencing UUIDs +VARCHAR(200) — created_by, updated_by, revoked_by (user email/subject) +VARCHAR(40) — handle (url-safe slug, NOT NULL; unique via UNIQUE(organization_uuid, handle)) +VARCHAR(255) — name, display strings + — hashes (SHA-256 hex) + — SAFE upper bound for indexed/unique columns across all engines +VARCHAR(512) — tokens (encrypted values) +VARCHAR(1023) — description, reason + — UPPER BOUND for plain-storage VARCHAR (above this → BYTEA/BLOB) + +BYTEA (Postgres) / BLOB (SQLite) / VARBINARY(MAX) (SQL Server) + — openapi_spec, model_list, content, configuration, properties, + manifest, policy_definition, metadata, api_key_hashes, + and any payload that can exceed a few hundred bytes + +JSONB — Postgres only; only when queried with JSON operators + — SQLite equivalent: TEXT (intentional, not a finding) + — SQL Server equivalent: NVARCHAR(MAX) (intentional, not a finding) + +TIMESTAMPTZ — all timestamps in Postgres (created_at, updated_at, expires_at, …) +DATETIME — all timestamps in SQLite +DATETIME2(7) DEFAULT SYSUTCDATETIME() — all timestamps in SQL Server + +SMALLINT — boolean flags in Postgres (is_active, is_default …) — use 0/1 +INTEGER — boolean flags in SQLite / SQL Server +``` diff --git a/.agents/skills/designing-db-schemas/scripts/generate-schema-report.js b/.agents/skills/designing-db-schemas/scripts/generate-schema-report.js new file mode 100644 index 0000000000..b2de6975e3 --- /dev/null +++ b/.agents/skills/designing-db-schemas/scripts/generate-schema-report.js @@ -0,0 +1,107 @@ +#!/usr/bin/env node +// generate-schema-report.js +// +// Belongs to the `designing-db-schemas` skill. +// Writes a structured JSON findings report from schema review findings. +// +// Usage: +// node generate-schema-report.js \ +// --findings '' (required) \ +// --schema (required) \ +// [--out ] (default: ./schema-reports/schema-review.json) +// +// Finding shape (each element of --findings array): +// { "rule": "R3-NO-TEXT", "table": "apis", "column": "config", +// "severity": "HIGH"|"MEDIUM"|"LOW", "finding": "...", "fix": "..." } +// +// Output shape: +// { +// "meta": { "schema": "...", "reviewedAt": "...", "rules": [...] }, +// "summary": { "HIGH": N, "MEDIUM": N, "LOW": N }, +// "findings": [ { "id": "r1-001", "severity": "...", "rule": "...", ... } ] +// } + +'use strict'; + +const fs = require('fs'); +const path = require('path'); + +// ---------- arg parsing ---------- +const args = process.argv.slice(2); +function flag(name) { + const i = args.indexOf(name); + return i !== -1 ? args[i + 1] : null; +} + +const findingsRaw = flag('--findings'); +const schemaPath = flag('--schema'); +const outPath = flag('--out') || './schema-reports/schema-review.json'; + +if (!findingsRaw || !schemaPath) { + console.error('Usage: generate-schema-report.js --findings \'[...]\' --schema [--out ]'); + process.exit(1); +} + +// ---------- parse findings ---------- +let findings; +try { + findings = JSON.parse(findingsRaw); +} catch (e) { + console.error('--findings must be a valid JSON array:', e.message); + process.exit(1); +} + +if (!Array.isArray(findings)) { + console.error('--findings must be a JSON array'); + process.exit(1); +} + +// Severity ordering and the set of supported, normalised severity values +const ORDER = { HIGH: 0, MEDIUM: 1, LOW: 2 }; + +// Assign sequential IDs per rule group and normalise +const counters = {}; +const normalised = findings.map(f => { + const rule = f.rule || 'UNKNOWN'; + // Rule-group identifier per the report contract: R3-NO-TEXT -> r3 + const group = rule.split('-')[0].toLowerCase().replace(/[^a-z0-9]/g, '') || 'unknown'; + counters[group] = (counters[group] || 0) + 1; + const seq = String(counters[group]).padStart(3, '0'); + // Normalise severity to a supported uppercase value before sort/summary + const sev = String(f.severity || 'MEDIUM').toUpperCase(); + return { + id: `${group}-${seq}`, + severity: ORDER[sev] !== undefined ? sev : 'MEDIUM', + rule, + table: f.table || null, + column: f.column || null, + finding: f.finding || '', + fix: f.fix || '', + }; +}); + +// Sort: HIGH → MEDIUM → LOW +normalised.sort((a, b) => (ORDER[a.severity] ?? 3) - (ORDER[b.severity] ?? 3)); + +// Summary counts +const summary = { HIGH: 0, MEDIUM: 0, LOW: 0 }; +for (const f of normalised) summary[f.severity] = (summary[f.severity] || 0) + 1; + +// ---------- build output ---------- +const report = { + meta: { + schema: schemaPath, + reviewedAt: new Date().toISOString(), + rules: ['R1','R2','R3','R4','R5','R6','R7','R8','R9','R10'], + }, + summary, + findings: normalised, +}; + +// ---------- write output ---------- +const outDir = path.dirname(outPath); +if (!fs.existsSync(outDir)) fs.mkdirSync(outDir, { recursive: true }); + +fs.writeFileSync(outPath, JSON.stringify(report, null, 2) + '\n'); +console.log(`Schema review report written to: ${outPath}`); +console.log(` HIGH: ${summary.HIGH} MEDIUM: ${summary.MEDIUM} LOW: ${summary.LOW} Total: ${normalised.length}`);