Add SQL player leaderboard snapshots and weekly-job checkpoints

database/INLINE_BASE_STORAGE.md

@@ -0,0 +1,29 @@
+# Base storage correction (021)
+
+All shared base data now lives on `bases`. Personal ownership remains in `user_saved_bases`.
+
+| Before | After |
+| --- | --- |
+| base_images: base_id, position, image_url | bases.images: ordered text array |
+| base_votes: base_id, user_id, vote, updated_at | bases.votes: JSON object keyed by user ID, containing vote and updatedAt |
+| bases.downloads | Unchanged user ID → first-download timestamp map |
+
+Example row fields:
+
+```json
+{
+  "images": ["https://api.clashk.ing/v2/media/base_example.png"],
+  "votes": {"123456789012345678": {"vote": 1, "updatedAt": "2026-09-17T00:00:00Z"}},
+  "downloads": {"123456789012345678": "2026-09-16T12:00:00Z"}
+}
+```
+
+Migration 021 copies every image position, vote identity/direction/timestamp, and leaves downloads unchanged, then removes the two old tables. Sparse image positions retain null placeholders internally so a partially staged Discord message can resume without replacing another image. API responses omit those placeholders. Existing media objects are not copied, deleted or renamed.
+
+Vote changes use an atomic update of one JSON key; image staging updates only an empty position. Repeated votes remain one vote, switching direction replaces it, and removing a vote removes only that user's key. Voter IDs stay private; Dashboard responses still expose counts and downloader identities only.
+
+## Deployment boundary
+
+This is a coordinated base-storage cutover, not a rolling-compatible migration. Stop/drain old base readers and writers, apply 021 with Goose, deploy the paired API revision, then resume traffic. The migration takes exclusive locks while copying so old writes cannot be silently lost, but old API processes cannot continue using the dropped tables afterward. Do not apply it independently while the old API serves bases. No deployment is performed by this PR.
+
+Down is intentionally blocked rather than silently destroying inline data. An operational rollback needs a separate data-preserving conversion and coordinated API rollback. Earlier applied migrations are unchanged.

database/PLAYER_LEADERBOARD_ROLLOUT.md

@@ -0,0 +1,17 @@
+# Migration 020: player leaderboard snapshots
+
+This is an additive forward Goose migration. Existing history tables and player data are not rewritten or reset by applying it.
+
+| Object | New stored data |
+| --- | --- |
+| player_townhall_leaderboards | Up to 500 players per TH 7–18: tag, name, townhall_level, trophies, league_id, league_group_tag, clan_tag, rank |
+| player_league_leaderboards | Up to 500 players per ranked tier 105000001–105000036, with the same fields |
+| tracking_scheduled_jobs | job text + period date primary key, last_tag text cursor, nullable completed_at timestamp |
+
+Town Hall ordering is league descending, trophies descending, tag ascending. League ordering is trophies descending, tag ascending. Unranked and unknown leagues are excluded; ranked players with zero trophies remain eligible. The existing partial player indexes support the per-board queries. Unique board/rank indexes allow concurrent refresh after initial population.
+
+Example snapshot: `#PLAYER, Example, 18, 2300, 105000034, #GROUP, #CLAN, 1`. Example job: `ranked_trophy_reset, 2026-09-21, #LASTPROCESSED, NULL` while in progress, or a completion timestamp when finished.
+
+Apply 020 first, then deploy Tracking to populate both views and refresh them every six hours. Verify `pg_matviews.ispopulated` for both views before deploying the API reader. They intentionally start WITH NO DATA. If separate database roles are used, the API role needs SELECT on both views and Tracking needs refresh ownership/privileges plus read/write access to the jobs table, following the deployment's existing role policy.
+
+The Tracking PR also uses the jobs table for a bounded Monday trophy reset, gated on successful closeout. This migration does not run that job. Roll back the API reader and Tracking writer before a Goose down, which removes only these three new objects.

database/migrations/bases_test.go

@@ -105,7 +105,7 @@ func TestBaseImportPostgres(t *testing.T) {
 	if _, err := tx.Exec(ctx, `UPDATE bases SET server_id='1',channel_id='2',description='converted' WHERE id=$1`, id); err != nil {
 		t.Fatal(err)
 	}
-	if _, err := tx.Exec(ctx, `INSERT INTO base_votes(base_id,user_id,vote) VALUES($1,'123',1)`, id); err != nil {
+	if _, err := tx.Exec(ctx, `UPDATE bases SET votes=jsonb_build_object('123',jsonb_build_object('vote',1,'updatedAt',now())) WHERE id=$1`, id); err != nil {
 		t.Fatal(err)
 	}
 	row.Downloads["123"] = "2026-09-16T00:00:00Z"
@@ -116,7 +116,7 @@ func TestBaseImportPostgres(t *testing.T) {
 	var preserved bool
 	if err := tx.QueryRow(ctx, `SELECT id=$2 AND downloads->>'123'=$3 AND (SELECT count(*) FROM jsonb_object_keys(downloads))=3
 		AND server_id='1' AND channel_id='2' AND description='converted'
-		AND EXISTS(SELECT 1 FROM base_votes WHERE base_id=$2 AND vote=1)
+		AND votes->'123'->>'vote'='1'
 		FROM bases WHERE message_id=$1`, row.MessageID, id, original).Scan(&preserved); err != nil {
 		t.Fatal(err)
 	}

database/migrations/war_archive_schema_test.go

@@ -32,6 +32,8 @@ func TestTimescaleMigrationsAreExplicitlyNumbered(t *testing.T) {
 		"017_final_operational_contract.sql",
 		"018_personal_base_library.sql",
 		"019_unlimited_personal_bases.sql",
+		"020_player_leaderboard_snapshots.sql",
+		"021_inline_base_images_votes.sql",
 	}
 	if len(files) != len(want) {
 		t.Fatalf("Timescale migrations = %v, want %v", files, want)

database/schema/final_operational_contract_test.go

@@ -97,12 +97,12 @@ func TestBasesUseBigintRelationsAndPrivateVotes(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = tx.Exec(ctx, `INSERT INTO base_images(base_id,position,image_url) VALUES($1,1,'https://api.clashk.ing/v2/media/base.png')`, baseID)
+	_, err = tx.Exec(ctx, `UPDATE bases SET images=ARRAY['https://api.clashk.ing/v2/media/base.png'] WHERE id=$1`, baseID)
 	if err == nil {
 		_, err = tx.Exec(ctx, `UPDATE bases SET downloads=jsonb_build_object('100','2026-09-15T12:00:00Z') WHERE id=$1`, baseID)
 	}
 	if err == nil {
-		_, err = tx.Exec(ctx, `INSERT INTO base_votes(base_id,user_id,vote) VALUES($1,'100',1)`, baseID)
+		_, err = tx.Exec(ctx, `UPDATE bases SET votes=jsonb_build_object('100',jsonb_build_object('vote',1,'updatedAt',now())) WHERE id=$1`, baseID)
 	}
 	if err == nil {
 		_, err = tx.Exec(ctx, `UPDATE bases SET server_id='200',channel_id='300' WHERE id=$1`, baseID)
@@ -119,8 +119,8 @@ func TestBasesUseBigintRelationsAndPrivateVotes(t *testing.T) {
 	}
 	for _, q := range []string{
 		`INSERT INTO bases(message_id,base_link) VALUES('124','https://evil.example/?action=OpenLayout&id=TH17')`,
-		`INSERT INTO base_images(base_id,position,image_url) VALUES(1,1,'https://example.com/base.png')`,
-		`INSERT INTO base_votes(base_id,user_id,vote) VALUES(1,'101',0)`,
+		`INSERT INTO bases(message_id,base_link,images) VALUES('125','https://link.clashofclans.com/en?action=OpenLayout&id=TH17',ARRAY['https://example.com/base.png'])`,
+		`INSERT INTO bases(message_id,base_link,votes) VALUES('126','https://link.clashofclans.com/en?action=OpenLayout&id=TH17','{"101":{"vote":0,"updatedAt":"2026-09-17T00:00:00Z"}}')`,
 	} {
 		if _, err = tx.Exec(ctx, `SAVEPOINT invalid`); err != nil {
 			t.Fatal(err)

database/schema/personal_base_storage_migration_test.go

@@ -120,4 +120,19 @@ func TestPersonalBaseStorageMigration(t *testing.T) {
 	check(`SELECT download_count=3 AND downloads ? '111111111111111111'
 		FROM bases JOIN base_public_counts counts ON counts.base_id=bases.id
 		WHERE message_id='700000000000000001'`)
+
+	run(`INSERT INTO base_images(base_id,position,image_url)
+ SELECT id,3,'https://api.clashk.ing/v2/media/third.png' FROM bases WHERE message_id='700000000000000001';
+ UPDATE base_votes SET updated_at='2026-09-01T02:03:04Z'`)
+	migrate("up-to", "21")
+	check(`SELECT to_regclass('public.base_images') IS NULL AND to_regclass('public.base_votes') IS NULL`)
+	check(`SELECT images=ARRAY[NULL,NULL,'https://api.clashk.ing/v2/media/third.png']::text[]
+ AND votes->'111111111111111111'->>'vote'='1'
+ AND votes->'222222222222222222'->>'vote'='-1'
+ AND (votes->'111111111111111111'->>'updatedAt')::timestamptz='2026-09-01T02:03:04Z'::timestamptz
+ AND download_count=3 AND upvote_count=1 AND downvote_count=1
+ FROM bases JOIN base_public_counts counts ON counts.base_id=bases.id WHERE message_id='700000000000000001'`)
+	check(`SELECT images='{}'::text[] AND votes='{}'::jsonb FROM bases WHERE message_id='700000000000000002'`)
+	reject(`UPDATE bases SET votes='{"bad":{"vote":1,"updatedAt":"2026-09-01T00:00:00Z"}}'`)
+	reject(`UPDATE bases SET images=ARRAY['https://api.clashk.ing/v2/media/a.png','https://api.clashk.ing/v2/media/a.png']`)
 }

database/schema/player_leaderboard_snapshots_test.go

@@ -0,0 +1,54 @@
+package schema
+
+import (
+	"context"
+	"testing"
+)
+
+func TestPlayerLeaderboardSnapshots(t *testing.T) {
+	conn := disposableConn(t)
+	ctx := context.Background()
+	tx, err := conn.Begin(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tx.Rollback(ctx)
+	_, err = tx.Exec(ctx, `
+ INSERT INTO basic_player(tag,name,townhall_level,league_id,trophies)
+ SELECT '#BOARD'||lpad(n::text,4,'0'),'Fixture',17,105000034,1000 FROM generate_series(1,501) n;
+ INSERT INTO basic_player(tag,name,townhall_level,league_id,trophies) VALUES
+ ('#HIGH','Higher tier',17,105000035,0),('#UNRANKED','Unranked',17,105000000,9999),('#UNKNOWN','Unknown',17,NULL,9999);
+ REFRESH MATERIALIZED VIEW player_townhall_leaderboards;
+ REFRESH MATERIALIZED VIEW player_league_leaderboards;`)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var count int
+	var tag string
+	if err = tx.QueryRow(ctx, `SELECT count(*) FROM player_townhall_leaderboards WHERE townhall_level=17`).Scan(&count); err != nil || count != 500 {
+		t.Fatalf("count=%d: %v", count, err)
+	}
+	if err = tx.QueryRow(ctx, `SELECT tag FROM player_townhall_leaderboards WHERE townhall_level=17 AND rank=1`).Scan(&tag); err != nil || tag != "#HIGH" {
+		t.Fatalf("first=%s: %v", tag, err)
+	}
+	if err = tx.QueryRow(ctx, `SELECT tag FROM player_league_leaderboards WHERE league_id=105000034 AND rank=1`).Scan(&tag); err != nil || tag != "#BOARD0001" {
+		t.Fatalf("tie=%s: %v", tag, err)
+	}
+	if err = tx.QueryRow(ctx, `SELECT count(*) FROM player_league_leaderboards WHERE league_id=105000034`).Scan(&count); err != nil || count != 500 {
+		t.Fatalf("league count=%d: %v", count, err)
+	}
+	_, err = tx.Exec(ctx, `UPDATE basic_player SET trophies=2000 WHERE tag='#BOARD0501'`)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err = tx.QueryRow(ctx, `SELECT count(*) FROM player_league_leaderboards WHERE tag='#BOARD0501'`).Scan(&count); err != nil || count != 0 {
+		t.Fatalf("snapshot changed without refresh: %v", err)
+	}
+	_, err = tx.Exec(ctx, `REFRESH MATERIALIZED VIEW CONCURRENTLY player_league_leaderboards`)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err = tx.QueryRow(ctx, `SELECT tag FROM player_league_leaderboards WHERE league_id=105000034 AND rank=1`).Scan(&tag); err != nil || tag != "#BOARD0501" {
+		t.Fatalf("refresh first=%s: %v", tag, err)
+	}
+}

database/timescale/020_player_leaderboard_snapshots.sql

@@ -0,0 +1,38 @@
+-- +goose Up
+-- Populate from Tracking after migration, before deploying the API reader.
+CREATE MATERIALIZED VIEW public.player_townhall_leaderboards AS
+SELECT p.*, row_number() OVER (PARTITION BY townhall_level ORDER BY league_id DESC,trophies DESC,tag)::integer AS rank
+FROM generate_series(7,18) th(level)
+CROSS JOIN LATERAL (
+  SELECT tag,name,townhall_level,trophies,league_id,league_group_tag,clan_tag
+  FROM public.basic_player WHERE townhall_level=th.level AND townhall_level>=7
+    AND league_id BETWEEN 105000001 AND 105000036
+    AND league_id IS NOT NULL AND league_id<>105000000
+  ORDER BY league_id DESC,trophies DESC,tag LIMIT 500
+) p WITH NO DATA;
+CREATE UNIQUE INDEX player_townhall_leaderboards_identity ON public.player_townhall_leaderboards(townhall_level,rank);
+
+CREATE MATERIALIZED VIEW public.player_league_leaderboards AS
+SELECT p.*, row_number() OVER (PARTITION BY league_id ORDER BY trophies DESC,tag)::integer AS rank
+FROM generate_series(105000001,105000036) tiers(id)
+CROSS JOIN LATERAL (
+  SELECT tag,name,townhall_level,trophies,league_id,league_group_tag,clan_tag
+  FROM public.basic_player WHERE league_id=tiers.id AND league_id IS NOT NULL AND league_id<>105000000
+  ORDER BY trophies DESC,tag LIMIT 500
+) p WITH NO DATA;
+CREATE UNIQUE INDEX player_league_leaderboards_identity ON public.player_league_leaderboards(league_id,rank);
+
+-- Small durable control records, not player history. Completion and batch progress
+-- are committed with the work they describe; reset periods are Monday UTC dates.
+CREATE TABLE public.tracking_scheduled_jobs (
+  job text NOT NULL,
+  period date NOT NULL,
+  last_tag text NOT NULL DEFAULT '',
+  completed_at timestamptz,
+  PRIMARY KEY(job,period)
+);
+
+-- +goose Down
+DROP TABLE public.tracking_scheduled_jobs;
+DROP MATERIALIZED VIEW public.player_league_leaderboards;
+DROP MATERIALIZED VIEW public.player_townhall_leaderboards;

database/timescale/021_inline_base_images_votes.sql

@@ -0,0 +1,56 @@
+-- +goose Up
+-- Lock source tables before copying so concurrent old writers cannot be lost.
+LOCK TABLE public.bases, public.base_images, public.base_votes IN ACCESS EXCLUSIVE MODE;
+ALTER TABLE public.bases ADD COLUMN images text[] NOT NULL DEFAULT '{}',
+    ADD COLUMN votes jsonb NOT NULL DEFAULT '{}';
+UPDATE public.bases b SET images=x.images FROM (
+    SELECT base_id,array_agg(i.image_url ORDER BY p.position) images
+    FROM (SELECT base_id,max(position) last_position FROM public.base_images GROUP BY base_id) s
+    CROSS JOIN LATERAL generate_series(1,s.last_position) p(position)
+    LEFT JOIN public.base_images i USING(base_id,position) GROUP BY base_id
+) x WHERE b.id=x.base_id;
+UPDATE public.bases b SET votes=x.votes FROM (
+    SELECT base_id,jsonb_object_agg(user_id,jsonb_build_object('vote',vote,'updatedAt',updated_at)) votes
+    FROM public.base_votes GROUP BY base_id
+) x WHERE b.id=x.base_id;
+
+-- Null array slots preserve positions during partial legacy-image staging.
+-- +goose StatementBegin
+CREATE FUNCTION public.base_images_valid(value text[]) RETURNS boolean
+LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$
+ SELECT cardinality(value)<=4 AND (cardinality(value)=0 OR (array_ndims(value)=1 AND array_lower(value,1)=1))
+ AND NOT EXISTS(SELECT 1 FROM unnest(value) image WHERE image IS NOT NULL AND image !~ '^https://api[.]clashk[.]ing/v2/media/[A-Za-z0-9][A-Za-z0-9._-]*$')
+ AND (SELECT count(image)=count(DISTINCT image) FROM unnest(value) image)
+$$;
+CREATE FUNCTION public.base_votes_valid(value jsonb) RETURNS boolean
+LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE AS $$
+DECLARE entry record; parsed timestamptz;
+BEGIN
+ IF jsonb_typeof(value)<>'object' THEN RETURN false; END IF;
+ FOR entry IN SELECT * FROM jsonb_each(value) LOOP
+  IF entry.key !~ '^[0-9]+$' OR jsonb_typeof(entry.value)<>'object'
+     OR NOT (entry.value ? 'vote' AND entry.value ? 'updatedAt')
+     OR entry.value->'vote' NOT IN ('1'::jsonb,'-1'::jsonb)
+     OR jsonb_typeof(entry.value->'updatedAt')<>'string' THEN RETURN false; END IF;
+  BEGIN parsed:=(entry.value->>'updatedAt')::timestamptz;
+  EXCEPTION WHEN others THEN RETURN false; END;
+ END LOOP;
+ RETURN true;
+END $$;
+-- +goose StatementEnd
+ALTER TABLE public.bases ADD CONSTRAINT bases_images_check CHECK(public.base_images_valid(images)),
+ ADD CONSTRAINT bases_votes_check CHECK(public.base_votes_valid(votes));
+DROP VIEW public.base_public_counts;
+CREATE VIEW public.base_public_counts AS SELECT id base_id,
+ (SELECT count(*) FROM jsonb_object_keys(downloads)) download_count,
+ (SELECT count(*) FROM jsonb_each(votes) v WHERE v.value->>'vote'='1') upvote_count,
+ (SELECT count(*) FROM jsonb_each(votes) v WHERE v.value->>'vote'='-1') downvote_count
+ FROM public.bases;
+DROP TABLE public.base_images;
+DROP TABLE public.base_votes;
+
+-- +goose Down
+-- Require coordinated reader/writer rollback; do not silently discard inline data.
+-- +goose StatementBegin
+DO $$ BEGIN RAISE EXCEPTION '021 requires an explicit data-preserving rollback; do not drop inline images or votes'; END $$;
+-- +goose StatementEnd

docs/cross-repo-contracts.md

@@ -77,3 +77,6 @@ of this cutover.
 - [ ] Obsolete compatibility removed when requested
 - [ ] Stale-contract search returns only intentional history
 - [ ] Targeted tests and `git diff --check` pass
+# Hero mode in army share codes
+
+Canonical hero sections may contain `m<number>` immediately after the hero ID, for example `h2m1p16e5_41`. Tracking and API normalization preserve the mode in the stored/query share code. Mode does not become a separate composition column and is ignored for family matching. Different exact codes can therefore map to the same family.

scripts/retained-api-profile.sh

@@ -1,5 +1,5 @@
 # Canonical migration inventory for retained API tests and local development.
-# This is the same contiguous 001-019 sequence used for production upgrades.
+# This is the same contiguous 001-021 sequence used for production upgrades.
 fixture_image='timescale/timescaledb:2.29.2-pg18@sha256:9508616d5b941ed931198504c5db3fb47e8f53f790732ea1e889591f1062057c'
 fixture_sources=(
   001_initial_stats.sql
@@ -21,4 +21,6 @@ fixture_sources=(
   017_final_operational_contract.sql
   018_personal_base_library.sql
   019_unlimited_personal_bases.sql
+  020_player_leaderboard_snapshots.sql
+  021_inline_base_images_votes.sql
 )

scripts/with-test-timescale.test.mjs

@@ -83,6 +83,8 @@ test('uses only retained authoritative migrations and its own disposable contain
       '017_final_operational_contract.sql',
       '018_personal_base_library.sql',
       '019_unlimited_personal_bases.sql',
+      '020_player_leaderboard_snapshots.sql',
+      '021_inline_base_images_votes.sql',
     ]);
   }
   assert.equal(migrations[1].args.at(-1), 'up');