From 906fe93b90725ff65e938a1579bf0dda12d4bbae Mon Sep 17 00:00:00 2001 From: Tomas Valenta <49156497+ValentaTomas@users.noreply.github.com> Date: Fri, 24 Jul 2026 20:59:06 -0700 Subject: [PATCH 1/4] fix(db): drop the unused plain env_builds status index --- ...725040800_drop_env_builds_status_index.sql | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 packages/db/migrations/20260725040800_drop_env_builds_status_index.sql diff --git a/packages/db/migrations/20260725040800_drop_env_builds_status_index.sql b/packages/db/migrations/20260725040800_drop_env_builds_status_index.sql new file mode 100644 index 0000000000..ed28cd5a8b --- /dev/null +++ b/packages/db/migrations/20260725040800_drop_env_builds_status_index.sql @@ -0,0 +1,29 @@ +-- +goose Up +-- +goose NO TRANSACTION + +-- DROP only waits on in-flight lock holders; bound the wait so it neither +-- hangs unbounded nor dies to a short session default. +SET statement_timeout = '1h'; + +-- Plain btree on the raw status column: 10 lifetime scans in +-- pg_stat_user_indexes against ~3.6 GiB of size, while every build +-- insert/supersede pays its maintenance write. Status-filtered access runs +-- on the status_group partial indexes instead. env_builds is the single +-- most write-amplified table (0% HOT updates, 8 indexes), so each dropped +-- index cuts real WAL and dead-tuple volume. +DROP INDEX CONCURRENTLY IF EXISTS public.idx_env_builds_status; + +-- +goose Down +-- +goose NO TRANSACTION + +-- A concurrent rebuild on a ~330M row table runs for hours, and a mid-build +-- timeout would strand an INVALID index. Rolling back is a deliberate manual +-- operation with an operator present — unbounded on purpose. +SET statement_timeout = 0; + +-- An interrupted CONCURRENTLY build leaves an INVALID index that IF NOT +-- EXISTS would then skip forever — clear any leftover before rebuilding. +DROP INDEX CONCURRENTLY IF EXISTS public.idx_env_builds_status; + +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_env_builds_status + ON public.env_builds (status); From d5ec2506a9af25606eb44f1d7954f01a92db3d55 Mon Sep 17 00:00:00 2001 From: Tomas Valenta <49156497+ValentaTomas@users.noreply.github.com> Date: Fri, 24 Jul 2026 22:06:25 -0700 Subject: [PATCH 2/4] review: reset statement_timeout for the reused migrator session --- .../20260725040800_drop_env_builds_status_index.sql | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/db/migrations/20260725040800_drop_env_builds_status_index.sql b/packages/db/migrations/20260725040800_drop_env_builds_status_index.sql index ed28cd5a8b..53c05cde98 100644 --- a/packages/db/migrations/20260725040800_drop_env_builds_status_index.sql +++ b/packages/db/migrations/20260725040800_drop_env_builds_status_index.sql @@ -13,6 +13,10 @@ SET statement_timeout = '1h'; -- index cuts real WAL and dead-tuple volume. DROP INDEX CONCURRENTLY IF EXISTS public.idx_env_builds_status; +-- The migrator session is reused for subsequent migrations — don't leak the +-- widened bound into them. +RESET statement_timeout; + -- +goose Down -- +goose NO TRANSACTION @@ -27,3 +31,5 @@ DROP INDEX CONCURRENTLY IF EXISTS public.idx_env_builds_status; CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_env_builds_status ON public.env_builds (status); + +RESET statement_timeout; From 636c11e31d70c331e7a2f0a0dfb5386e897584c6 Mon Sep 17 00:00:00 2001 From: Tomas Valenta <49156497+ValentaTomas@users.noreply.github.com> Date: Fri, 24 Jul 2026 22:15:43 -0700 Subject: [PATCH 3/4] review: restore the migrator baseline instead of RESET --- .../20260725040800_drop_env_builds_status_index.sql | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/db/migrations/20260725040800_drop_env_builds_status_index.sql b/packages/db/migrations/20260725040800_drop_env_builds_status_index.sql index 53c05cde98..7f235e41c2 100644 --- a/packages/db/migrations/20260725040800_drop_env_builds_status_index.sql +++ b/packages/db/migrations/20260725040800_drop_env_builds_status_index.sql @@ -13,9 +13,10 @@ SET statement_timeout = '1h'; -- index cuts real WAL and dead-tuple volume. DROP INDEX CONCURRENTLY IF EXISTS public.idx_env_builds_status; --- The migrator session is reused for subsequent migrations — don't leak the --- widened bound into them. -RESET statement_timeout; +-- The migrator session is reused for subsequent migrations: restore its +-- baseline (scripts/migrator.go sets 3h per connection; RESET would clear +-- to the unbounded server default instead). +SET statement_timeout = '3h'; -- +goose Down -- +goose NO TRANSACTION @@ -32,4 +33,5 @@ DROP INDEX CONCURRENTLY IF EXISTS public.idx_env_builds_status; CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_env_builds_status ON public.env_builds (status); -RESET statement_timeout; +-- Restore the migrator session baseline (see Up). +SET statement_timeout = '3h'; From 428983f8da0b95184629a44da3118bd5c3a8dc1c Mon Sep 17 00:00:00 2001 From: Tomas Valenta <49156497+ValentaTomas@users.noreply.github.com> Date: Sat, 25 Jul 2026 15:32:30 -0700 Subject: [PATCH 4/4] scrub size figures from migration comment --- .../20260725040800_drop_env_builds_status_index.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/db/migrations/20260725040800_drop_env_builds_status_index.sql b/packages/db/migrations/20260725040800_drop_env_builds_status_index.sql index 7f235e41c2..a78fdc5afe 100644 --- a/packages/db/migrations/20260725040800_drop_env_builds_status_index.sql +++ b/packages/db/migrations/20260725040800_drop_env_builds_status_index.sql @@ -5,8 +5,8 @@ -- hangs unbounded nor dies to a short session default. SET statement_timeout = '1h'; --- Plain btree on the raw status column: 10 lifetime scans in --- pg_stat_user_indexes against ~3.6 GiB of size, while every build +-- Plain btree on the raw status column: a negligible number of lifetime scans in +-- pg_stat_user_indexes, while every build -- insert/supersede pays its maintenance write. Status-filtered access runs -- on the status_group partial indexes instead. env_builds is the single -- most write-amplified table (0% HOT updates, 8 indexes), so each dropped