diff --git a/.github/workflows/schema-upgrade.yml b/.github/workflows/schema-upgrade.yml index 32ed70c..daa7843 100644 --- a/.github/workflows/schema-upgrade.yml +++ b/.github/workflows/schema-upgrade.yml @@ -27,3 +27,6 @@ jobs: - name: Apply 013 then verify the army-family compatibility upgrade working-directory: database run: bash ../scripts/with-test-timescale.sh --profile baseline-013 -- go test ./schema -run '^TestArmyCodeFamilyCompatibilityMigration$' -count=1 -v + - name: Apply 018 then verify the populated personal-base upgrade + working-directory: database + run: bash ../scripts/with-test-timescale.sh --profile baseline-018 -- go test ./schema -run '^TestPersonalBaseStorageMigration$' -count=1 -v diff --git a/RETAINED_API_FIXTURE.md b/RETAINED_API_FIXTURE.md index 4ee8e67..1ebe7c3 100644 --- a/RETAINED_API_FIXTURE.md +++ b/RETAINED_API_FIXTURE.md @@ -1,13 +1,13 @@ # Retained API database fixture -The canonical Goose sequence is `database/timescale/001` through `015`. +The canonical Goose sequence is `database/timescale/001` through `016`. Production was confirmed by the user to be at 006 before consolidation; files 001–006 are unchanged. `007_worker_api.sql` combines the unapplied Worker/API migrations formerly numbered 007, 008, 009, 010, 012, 013, 020, 022, 023, 027 and 028 into one transaction, excluding the user-rejected player-link mutex, global AI-budget mutex, subject mutex, resource ownership ledger and Discord delivery receipt tables. Its Down operation is intentionally irreversible. -`008_ranked_battle_history.sql` adds the one-year farming and two-perspective Ranked/Legend histories, immutable normalized army compositions, and the final `ranked_league_group_members` source-counter shape. `009_league_army_analytics.sql` adds permanent league/Legend rollups and immutable army-family assignments. Both have structural Down paths, but 008 cannot recreate stale duplicate season memberships removed before it enforces one group per player and season. `010_cwl_season_statistics.sql` remains the independent, reversible CWL population summary, and 011 adds active verified-player state. See `docs/ranked-battle-history.md` for every column and lifecycle. +`008_ranked_battle_history.sql` adds the one-year farming and two-perspective Ranked/Legend histories, immutable normalized army compositions, and the final `ranked_league_group_members` source-counter shape. `009_league_army_analytics.sql` adds permanent league/Legend rollups and immutable army-family assignments. Both have structural Down paths, but 008 cannot recreate stale duplicate season memberships removed before it enforces one group per player and season. Migration 010 remains immutable history, 011 adds active verified-player state, and 016 removes the rejected CWL-only population summary. See `docs/ranked-battle-history.md` for every retained analytics column and lifecycle, and `docs/cwl-season-statistics-removal.md` for the consumer cutover. `bash scripts/with-test-timescale.sh --profile retained-api -- COMMAND [ARG ...]` applies the complete canonical sequence to a fresh local tmpfs Timescale diff --git a/contracts/army-family-v1.json b/contracts/army-family-v1.json deleted file mode 100644 index 6c46e32..0000000 --- a/contracts/army-family-v1.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "version": 1, - "exactIdentity": "army_hash-v2", - "anchorImmutable": true, - "assignmentImmutable": true, - "matching": { - "mainTroops": {"metric": "housing-weighted", "minimum": 0.86}, - "spells": {"metric": "capacity-weighted", "includesClanCastle": true, "minimum": 0.80}, - "heroes": {"metric": "exact-set"}, - "equipment": {"minimum": 0.75, "maximumDifferences": 2}, - "ignored": ["clan_castle_troops", "pet_assignments", "siege_machine_id"] - }, - "sources": ["ai", "admin", "fallback"] -} diff --git a/contracts/army-hash-v2.json b/contracts/army-hash-v2.json deleted file mode 100644 index fa73e5d..0000000 --- a/contracts/army-hash-v2.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "version": 2, - "algorithm": "sha256", - "input": "0x02 || utf8(normalized_share_code)", - "normalization": { - "hero_loadouts": "sort by hero id, then pet id; sort each hero's equipment ids", - "section_order": ["h", "i", "d", "u", "s"], - "item_order": "ascending numeric item id within each section", - "preserves": ["hero-to-pet assignment", "hero-to-equipment assignment", "zero-based ids"] - }, - "vectors": [ - { - "normalized_share_code": "h0p4e8_14-1p9e39i3x53d1x70u10x0-2x1s4x35", - "sha256_hex": "381e0786e690608139a21c2a17b131cc4b265ce18353509ded94346e2bc0ff10" - }, - { - "normalized_share_code": "h0p4e8_14-1p9e39", - "sha256_hex": "6628d4ed049e5a5605474ceb2edac33b20978c3499274e8c57207583f52163af" - }, - { - "normalized_share_code": "h0p9e8_14-1p4e39", - "sha256_hex": "cbb0595e3e6bdcf7d00f7b6641532701e7109f5e0c9e35e715badb8983a5f1be" - } - ] -} diff --git a/database/migrations/cleanup_army_compositions.go b/database/migrations/cleanup_army_compositions.go deleted file mode 100644 index 61c5c7b..0000000 --- a/database/migrations/cleanup_army_compositions.go +++ /dev/null @@ -1,123 +0,0 @@ -//go:build ignore - -package main - -import ( - "context" - "fmt" - "strconv" - "time" - - "github.com/ClashKingInc/DevKit/database/migrations/migrateutil" -) - -func main() { migrateutil.Main("cleanup_army_compositions", cleanupArmyCompositions) } - -func cleanupSettings(env map[string]string) (bool, int, error) { - apply := env["ARMY_COMPOSITION_CLEANUP_APPLY"] == "true" - if apply && env["ARMY_COMPOSITION_WRITERS_PAUSED"] != "true" { - return false, 0, fmt.Errorf("apply requires ARMY_COMPOSITION_WRITERS_PAUSED=true; stop battle ingestion and family closeouts first") - } - batch := 1000 - if raw := env["ARMY_COMPOSITION_CLEANUP_BATCH_SIZE"]; raw != "" { - v, err := strconv.Atoi(raw) - if err != nil || v < 1 || v > 5000 { - return false, 0, fmt.Errorf("batch size must be 1..5000") - } - batch = v - } - return apply, batch, nil -} - -func cleanupArmyCompositions(ctx context.Context, cfg migrateutil.Config) error { - apply, batch, err := cleanupSettings(cfg.Env) - if err != nil { - return err - } - pool, err := migrateutil.TimescalePool(ctx, cfg) - if err != nil { - return err - } - defer pool.Close() - conn, err := pool.Acquire(ctx) - if err != nil { - return err - } - defer conn.Release() - if _, err = conn.Exec(ctx, "SET application_name='ck_composition_cleanup'; SET lock_timeout='5s'; SET statement_timeout='120s'"); err != nil { - return err - } - var locked bool - if err = conn.QueryRow(ctx, "SELECT pg_try_advisory_lock(719233812)").Scan(&locked); err != nil { - return err - } - if !locked { - return fmt.Errorf("another cleanup is running") - } - defer conn.Exec(context.Background(), "SELECT pg_advisory_unlock(719233812)") - var constraints int - if err = conn.QueryRow(ctx, `SELECT count(*) FROM pg_constraint WHERE conrelid='public.battles_ranked'::regclass AND confrelid='public.army_compositions'::regclass`).Scan(&constraints); err != nil { - return err - } - if constraints != 0 { - return fmt.Errorf("apply schema 012 first; battle-to-composition foreign keys still exist") - } - // Scan the raw battle history once, not once per delete batch. - if _, err = conn.Exec(ctx, `CREATE TEMP TABLE cleanup_keep ON COMMIT PRESERVE ROWS AS - SELECT army_hash FROM public.battles_ranked WHERE battle_mode='legend' - UNION SELECT anchor_army_hash FROM public.army_families - UNION SELECT army_hash FROM public.army_family_members`); err != nil { - return err - } - if _, err = conn.Exec(ctx, `CREATE UNIQUE INDEX ON cleanup_keep(army_hash); ANALYZE cleanup_keep; - CREATE TEMP TABLE cleanup_candidates ON COMMIT PRESERVE ROWS AS - SELECT c.army_hash FROM public.army_compositions c WHERE NOT EXISTS(SELECT 1 FROM cleanup_keep k WHERE k.army_hash=c.army_hash); - CREATE UNIQUE INDEX ON cleanup_candidates(army_hash); ANALYZE cleanup_candidates`); err != nil { - return err - } - defer conn.Exec(context.Background(), "DROP TABLE IF EXISTS pg_temp.cleanup_candidates,pg_temp.cleanup_keep") - var candidates, kept int64 - if err = conn.QueryRow(ctx, "SELECT (SELECT count(*) FROM cleanup_candidates),(SELECT count(*) FROM cleanup_keep)").Scan(&candidates, &kept); err != nil { - return err - } - fmt.Printf("dry_run=%t candidate_compositions=%d protected_hashes=%d batch_size=%d\n", !apply, candidates, kept, batch) - if !apply { - return nil - } - var deleted int64 - remaining := candidates - for { - var n, processed int64 - // Each statement is its own atomic transaction. Family references are also - // rechecked here and enforced by their retained foreign keys. - err = conn.QueryRow(ctx, `WITH batch AS (SELECT army_hash FROM cleanup_candidates ORDER BY army_hash LIMIT $1), - gone AS (DELETE FROM public.army_compositions c USING batch b WHERE c.army_hash=b.army_hash - AND NOT EXISTS(SELECT 1 FROM public.army_families f WHERE f.anchor_army_hash=c.army_hash) - AND NOT EXISTS(SELECT 1 FROM public.army_family_members m WHERE m.army_hash=c.army_hash) RETURNING c.army_hash), - consumed AS (DELETE FROM cleanup_candidates c USING batch b WHERE c.army_hash=b.army_hash RETURNING c.army_hash) - SELECT (SELECT count(*) FROM gone),(SELECT count(*) FROM consumed)`, batch).Scan(&n, &processed) - if err != nil { - return fmt.Errorf("cleanup stopped after %d committed deletes: %w", deleted, err) - } - deleted += n - remaining -= processed - fmt.Printf("deleted=%d remaining_candidates=%d\n", deleted, remaining) - if remaining == 0 { - break - } - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(100 * time.Millisecond): - } - } - var missing int64 - if err = conn.QueryRow(ctx, `SELECT count(*) FROM cleanup_keep k WHERE NOT EXISTS(SELECT 1 FROM public.army_compositions c WHERE c.army_hash=k.army_hash)`).Scan(&missing); err != nil { - return err - } - if missing != 0 { - return fmt.Errorf("verification found %d protected hashes without compositions", missing) - } - fmt.Printf("cleanup complete: deleted=%d protected_hashes_missing=0\n", deleted) - return nil -} diff --git a/database/migrations/cleanup_army_compositions_test.go b/database/migrations/cleanup_army_compositions_test.go deleted file mode 100644 index ea1dcd9..0000000 --- a/database/migrations/cleanup_army_compositions_test.go +++ /dev/null @@ -1,93 +0,0 @@ -//go:build ignore - -package main - -import ( - "github.com/ClashKingInc/DevKit/database/migrations/migrateutil" - "github.com/jackc/pgx/v5" - "os" - "os/exec" - "testing" -) - -func TestCleanupDefaultsAndApplyGuard(t *testing.T) { - apply, batch, err := cleanupSettings(nil) - if err != nil || apply || batch != 1000 { - t.Fatal(apply, batch, err) - } - if _, _, err := cleanupSettings(map[string]string{"ARMY_COMPOSITION_CLEANUP_APPLY": "true"}); err == nil { - t.Fatal("must require writers paused") - } - apply, _, err = cleanupSettings(map[string]string{"ARMY_COMPOSITION_CLEANUP_APPLY": "true", "ARMY_COMPOSITION_WRITERS_PAUSED": "true"}) - if err != nil || !apply { - t.Fatal(apply, err) - } - for _, value := range []string{"0", "5001", "bad"} { - if _, _, err := cleanupSettings(map[string]string{"ARMY_COMPOSITION_CLEANUP_BATCH_SIZE": value}); err == nil { - t.Fatal(value) - } - } -} - -func TestCleanupDisposableTimescale(t *testing.T) { - if os.Getenv("CLASHKING_DISPOSABLE_TIMESCALE") != "1" { - t.Skip("requires disposable Goose fixture") - } - dsn := os.Getenv("TEST_DATABASE_URL") - goose := func(args ...string) { - t.Helper() - cmd := exec.Command("goose", append([]string{"-env", "/dev/null", "-dir", "../timescale", "postgres", dsn}, args...)...) - if out, err := cmd.CombinedOutput(); err != nil { - t.Fatalf("goose: %v: %s", err, out) - } - } - goose("down-to", "11") - conn, err := pgx.Connect(t.Context(), dsn) - if err != nil { - t.Fatal(err) - } - defer conn.Close(t.Context()) - _, err = conn.Exec(t.Context(), `INSERT INTO army_compositions(army_hash,normalized_share_code) - SELECT decode(repeat(n::text,64),'hex'),'u'||n||'x0' FROM generate_series(1,4) n; - INSERT INTO army_families(anchor_army_hash,representative_share_code,family_name,source) - VALUES(decode(repeat('3',64),'hex'),'u3x0','Protected family','fallback'); - INSERT INTO battles_ranked(player_tag,opponent_tag,battle_time,direction,battle_mode,player_town_hall,opponent_town_hall,stars,destruction_percentage,army_hash,share_code) - VALUES('#P0Y','#P0L',now(),'attack','ranked',18,18,3,100,decode(repeat('1',64),'hex'),'u1x0'), - ('#P0Y','#P0L',now(),'attack','legend',18,18,3,100,decode(repeat('2',64),'hex'),'u2x0');`) - if err != nil { - t.Fatal(err) - } - goose("up-to", "12") - _, err = conn.Exec(t.Context(), `INSERT INTO battles_ranked(player_tag,opponent_tag,battle_time,direction,battle_mode,player_town_hall,opponent_town_hall,stars,destruction_percentage,army_hash,share_code) - VALUES('#P0Y','#P0L',now()+interval '1 second','attack','ranked',18,18,3,100,decode(repeat('5',64),'hex'),'u5x0')`) - if err != nil { - t.Fatal("Ranked insert after upgrade:", err) - } - cfg := migrateutil.Config{TimescaleURL: dsn, Env: map[string]string{}} - if err = cleanupArmyCompositions(t.Context(), cfg); err != nil { - t.Fatal(err) - } - var count int - if err = conn.QueryRow(t.Context(), "SELECT count(*) FROM army_compositions").Scan(&count); err != nil || count != 4 { - t.Fatalf("dry run mutated data: %d %v", count, err) - } - cfg.Env = map[string]string{"ARMY_COMPOSITION_CLEANUP_APPLY": "true", "ARMY_COMPOSITION_WRITERS_PAUSED": "true", "ARMY_COMPOSITION_CLEANUP_BATCH_SIZE": "1"} - if err = cleanupArmyCompositions(t.Context(), cfg); err != nil { - t.Fatal(err) - } - if err = conn.QueryRow(t.Context(), "SELECT count(*) FROM army_compositions").Scan(&count); err != nil || count != 2 { - t.Fatalf("unexpected remaining: %d %v", count, err) - } - if err = conn.QueryRow(t.Context(), "SELECT count(*) FROM battles_ranked").Scan(&count); err != nil || count != 3 { - t.Fatalf("raw history changed: %d %v", count, err) - } - if _, err = conn.Exec(t.Context(), "DELETE FROM army_compositions WHERE army_hash=decode(repeat('3',64),'hex')"); err == nil { - t.Fatal("family FK lost") - } - if _, err = conn.Exec(t.Context(), "UPDATE army_compositions SET normalized_share_code='u99x0'"); err == nil { - t.Fatal("immutability lost") - } - if err = cleanupArmyCompositions(t.Context(), cfg); err != nil { - t.Fatal("rerun:", err) - } -} diff --git a/database/migrations/legend_composition_scope_schema_test.go b/database/migrations/legend_composition_scope_schema_test.go deleted file mode 100644 index 3997ee9..0000000 --- a/database/migrations/legend_composition_scope_schema_test.go +++ /dev/null @@ -1,25 +0,0 @@ -package main - -import ( - "os" - "strings" - "testing" -) - -func TestLegendCompositionMigrationPreservesRawHistoryAndFamilyGuards(t *testing.T) { - data, err := os.ReadFile("../timescale/012_legend_only_army_compositions.sql") - if err != nil { - t.Fatal(err) - } - up := strings.Split(string(data), "-- +goose Down")[0] - for _, required := range []string{"DROP CONSTRAINT battles_ranked_army_hash_fkey", "DROP CONSTRAINT battles_ranked_share_code_hash_fkey", "BEFORE UPDATE ON public.army_compositions", "lock_timeout = '5s'"} { - if !strings.Contains(up, required) { - t.Fatalf("missing %s", required) - } - } - for _, forbidden := range []string{"DELETE FROM", "TRUNCATE TABLE", "DROP TABLE", "DROP COLUMN", "ALTER TABLE public.army_families", "ALTER TABLE public.army_family_members", "DROP TRIGGER army_compositions_no_truncate"} { - if strings.Contains(up, forbidden) { - t.Fatalf("unexpected destructive operation: %s", forbidden) - } - } -} diff --git a/database/migrations/war_archive_schema_test.go b/database/migrations/war_archive_schema_test.go index c3abbc0..f6dbb31 100644 --- a/database/migrations/war_archive_schema_test.go +++ b/database/migrations/war_archive_schema_test.go @@ -28,6 +28,10 @@ func TestTimescaleMigrationsAreExplicitlyNumbered(t *testing.T) { "013_battle_player_time_identity.sql", "014_ranked_defense_loot_nullable.sql", "015_army_code_family_compatibility.sql", + "016_remove_cwl_season_statistics.sql", + "017_final_operational_contract.sql", + "018_personal_base_library.sql", + "019_unlimited_personal_bases.sql", } if len(files) != len(want) { t.Fatalf("Timescale migrations = %v, want %v", files, want) diff --git a/database/schema/army_code_family_migration_test.go b/database/schema/army_code_family_migration_test.go index e2c3716..f1ac98b 100644 --- a/database/schema/army_code_family_migration_test.go +++ b/database/schema/army_code_family_migration_test.go @@ -266,4 +266,76 @@ func TestArmyCodeFamilyCompatibilityMigration(t *testing.T) { t.Fatal("rollback discarded v2-only data") } check(`SELECT max(version_id)=15 FROM goose_db_version WHERE is_applied`) + + // Complete the code-only representative so the final FK can preserve it, + // then prove a data-bearing 015 -> 019 conversion across every contract. + run(`INSERT INTO army_compositions(army_hash,normalized_share_code) + VALUES(decode(repeat('dd',32),'hex'),'u3x0'); + INSERT INTO auth_users(user_id,provider) VALUES('100','discord'); + INSERT INTO player_links(tag,is_verified,source,user_id) VALUES('#P0Y',true,'api_token','100'); + INSERT INTO mobile_push_devices( + user_id,device_id,platform,provider,environment,token_ciphertext,token_hash, + war_attacks_enabled,war_state_enabled,war_reminders_enabled,raid_reminders_enabled, + events_enabled,announcements_enabled,monthly_support_enabled,reminder_timings,raid_reminder_timings + ) VALUES('100','iphone','ios','fcm','production','cipher','hash',true,false,true,true,true,false,true,'{60}','{120}'); + INSERT INTO mobile_notification_accounts(user_id,player_tag) VALUES('100','#P0Y'); + INSERT INTO mobile_notification_deliveries(user_id,notification_key) VALUES('100','old-delivery'); + INSERT INTO bases(id,message_id,base_link,downloaders,images,description,upvoter_ids,downvoter_ids) + VALUES('00000000-0000-4000-8000-000000000001','123456789012345678', + 'https://link.clashofclans.com/en?action=OpenLayout&id=TH17%3AHV%3AAAAA', + '{100}','{https://api.clashk.ing/v2/media/base.png}','Legacy','{100}','{}'); + INSERT INTO basic_clan(tag,name,public_war_log,war_wins,member_count,badge_token,troops_donated,troops_received) + VALUES('#2PP','Clan',true,1,1,'badge',0,0); + INSERT INTO basic_player(tag,name,league_id,clan_tag,townhall_level,trophies) + VALUES('#P0Y','Player',105000036,'#2PP',18,6500); + INSERT INTO leaderboard_history_player_home( + location_id,date,player_tag,player_name,exp_level,trophies,attack_wins,defense_wins, + rank,previous_rank,clan_tag,clan_name,clan_badge_token,league_id + ) VALUES + ('global','2026-09-10','#P0Y','Historical Player',250,6400,21,7,12,15, + '#2PP','Historical Clan','historical-badge',29000022), + ('32000006','2026-09-10','#P0L','Local Player',175,5900,11,9,3,4, + NULL,NULL,NULL,29000021);`) + + if err := migrate("up-to", "19"); err != nil { + t.Fatal("final contract upgrade:", err) + } + check(`SELECT max(version_id)=19 FROM goose_db_version WHERE is_applied`) + check(`SELECT duration_seconds=0 AND direction=1 AND battle_mode=2 + FROM battles_ranked WHERE share_code='u3x0'`) + check(`SELECT EXISTS(SELECT 1 FROM army_compositions WHERE share_code='u3x0') + AND NOT EXISTS(SELECT 1 FROM information_schema.columns + WHERE table_schema='public' AND column_name IN ('army_hash','anchor_army_hash'))`) + check(`SELECT war_attacks_enabled AND NOT war_state_enabled AND war_reminders_enabled + AND raid_reminders_enabled AND events_enabled AND NOT announcements_enabled + AND monthly_support_enabled AND NOT legend_defenses_enabled + AND reminder_timings='{60}'::integer[] AND raid_reminder_timings='{120}'::integer[] + FROM mobile_notification_preferences WHERE user_id='100'`) + check(`SELECT to_regclass('public.mobile_notification_deliveries') IS NULL`) + check(`SELECT base.id=1 AND image.image_url='https://api.clashk.ing/v2/media/base.png' + AND counts.download_count=1 AND counts.upvote_count=1 AND counts.downvote_count=0 + FROM bases base JOIN base_images image ON image.base_id=base.id + JOIN base_public_counts counts ON counts.base_id=base.id + WHERE base.message_id='123456789012345678'`) + check(`SELECT to_regclass('public.user_saved_bases') IS NOT NULL + AND to_regclass('public.user_base_slots') IS NULL + AND to_regclass('public.base_downloaders') IS NULL + AND (SELECT downloads ? '100' FROM bases WHERE message_id='123456789012345678')`) + check(`SELECT name='Player' AND trophies=6500 AND global_rank=1 + AND clan_tag='#2PP' AND clan_name='Clan' FROM legend_rankings_current WHERE tag='#P0Y'`) + check(`SELECT count(*)=2 FROM leaderboard_history_player_home WHERE date='2026-09-10'`) + check(`SELECT player_name='Historical Player' AND exp_level=250 AND trophies=6400 + AND attack_wins=21 AND defense_wins=7 AND rank=12 AND previous_rank=15 + AND clan_tag='#2PP' AND clan_name='Historical Clan' + AND clan_badge_token='historical-badge' AND league_id=29000022 + FROM leaderboard_history_player_home + WHERE location_id='global' AND date='2026-09-10' AND player_tag='#P0Y'`) + check(`SELECT player_name='Local Player' AND rank=3 AND clan_tag IS NULL + FROM leaderboard_history_player_home + WHERE location_id='32000006' AND date='2026-09-10' AND player_tag='#P0L'`) + run(`INSERT INTO legend_rankings_history(day,tag,global_rank,trophies) + VALUES('2026-09-10','#P0Y',1,6500)`) + check(`SELECT global_rank=1 AND trophies=6500 + FROM legend_rankings_history WHERE day='2026-09-10' AND tag='#P0Y'`) + check(`SELECT count(*)=2 FROM leaderboard_history_player_home WHERE date='2026-09-10'`) } diff --git a/database/schema/battle_player_time_test.go b/database/schema/battle_player_time_test.go index ac5cfa6..6f66432 100644 --- a/database/schema/battle_player_time_test.go +++ b/database/schema/battle_player_time_test.go @@ -2,8 +2,6 @@ package schema import ( "context" - "os" - "strings" "testing" ) @@ -22,104 +20,18 @@ func TestBattlePlayerTimeIdentity(t *testing.T) { if definition != "PRIMARY KEY (player_tag, battle_time)" { t.Fatal(definition) } - insert := `INSERT INTO battles_ranked(player_tag,opponent_tag,battle_time,direction,battle_mode,player_town_hall,opponent_town_hall,stars,destruction_percentage,looted_resources,army_hash) VALUES($1,$2,'2026-09-09T00:00:00Z',$3,'legend',18,17,3,100,CASE WHEN $3='defense' THEN NULL ELSE '{}'::jsonb END,decode(repeat('ab',32),'hex')) ON CONFLICT(player_tag,battle_time) DO NOTHING` - for _, v := range [][3]string{{"#P0", "#Y2", "attack"}, {"#Y2", "#P0", "defense"}} { - tag, err := tx.Exec(ctx, insert, v[0], v[1], v[2]) + insert := `INSERT INTO battles_ranked(player_tag,opponent_tag,battle_time,direction,battle_mode,player_town_hall,opponent_town_hall,stars,destruction_percentage,looted_resources) VALUES($1,$2,'2026-09-09T00:00:00Z',$3::smallint,2,18,17,3,100,CASE WHEN $3::smallint=2 THEN NULL ELSE '{}'::jsonb END) ON CONFLICT(player_tag,battle_time) DO NOTHING` + for _, v := range []struct { + player, opponent string + direction int16 + }{{"#P0", "#Y2", 1}, {"#Y2", "#P0", 2}} { + tag, err := tx.Exec(ctx, insert, v.player, v.opponent, v.direction) if err != nil || tag.RowsAffected() != 1 { t.Fatalf("insert: %v %v", tag, err) } } - tag, err := tx.Exec(ctx, insert, "#P0", "#G9", "defense") + tag, err := tx.Exec(ctx, insert, "#P0", "#G9", int16(2)) if err != nil || tag.RowsAffected() != 0 { t.Fatalf("collision: %v %v", tag, err) } } - -func TestBattleIdentityMigrationRejectsCollisionsWithoutDataLoss(t *testing.T) { - conn := disposableConn(t) - ctx := context.Background() - tx, err := conn.Begin(ctx) - if err != nil { - t.Fatal(err) - } - defer tx.Rollback(ctx) - raw, err := os.ReadFile("../timescale/013_battle_player_time_identity.sql") - if err != nil { - t.Fatal(err) - } - parts := strings.Split(string(raw), "-- +goose Down") - if _, err = tx.Exec(ctx, parts[1]); err != nil { - t.Fatal(err) - } - if _, err = tx.Exec(ctx, `INSERT INTO battles_ranked(player_tag,opponent_tag,battle_time,direction,battle_mode,player_town_hall,opponent_town_hall,stars,destruction_percentage,looted_resources,army_hash) VALUES('#P0','#Y2','2026-09-09T00:00:00Z','attack','legend',18,17,3,100,'{}',decode(repeat('ab',32),'hex')),('#P0','#G9','2026-09-09T00:00:00Z','defense','legend',18,17,3,100,NULL,decode(repeat('ab',32),'hex'))`); err != nil { - t.Fatal(err) - } - if _, err = tx.Exec(ctx, "SAVEPOINT upgrade"); err != nil { - t.Fatal(err) - } - if _, err = tx.Exec(ctx, parts[0]); err == nil { - t.Fatal("migration accepted collisions") - } - if _, err = tx.Exec(ctx, "ROLLBACK TO SAVEPOINT upgrade"); err != nil { - t.Fatal(err) - } - var n int - if err = tx.QueryRow(ctx, `SELECT count(*) FROM battles_ranked WHERE player_tag='#P0' AND battle_time='2026-09-09T00:00:00Z'`).Scan(&n); err != nil || n != 2 { - t.Fatalf("rows=%d err=%v", n, err) - } -} - -func TestExplicitBattleReset(t *testing.T) { - for _, withFamily := range []bool{false, true} { - t.Run(map[bool]string{false: "empty dependents", true: "retained family"}[withFamily], func(t *testing.T) { - conn := disposableConn(t) - ctx := context.Background() - tx, err := conn.Begin(ctx) - if err != nil { - t.Fatal(err) - } - defer tx.Rollback(ctx) - _, err = tx.Exec(ctx, `INSERT INTO army_compositions(army_hash,normalized_share_code) VALUES(decode(repeat('ac',32),'hex'),'u1x0'); INSERT INTO battles_ranked(player_tag,opponent_tag,battle_time,direction,battle_mode,player_town_hall,opponent_town_hall,stars,destruction_percentage,army_hash) VALUES('#P0','#Y2','2026-09-09T00:00:00Z','attack','legend',18,17,3,100,decode(repeat('ac',32),'hex'))`) - if err != nil { - t.Fatal(err) - } - if withFamily { - _, err = tx.Exec(ctx, `INSERT INTO army_families(anchor_army_hash,representative_share_code,family_name,source) VALUES(decode(repeat('ac',32),'hex'),'u1x0','retained test family','fallback')`) - if err != nil { - t.Fatal(err) - } - } - raw, err := os.ReadFile("../../scripts/reset-battle-history.sql") - if err != nil { - t.Fatal(err) - } - reset := strings.Replace(string(raw), "BEGIN;", "", 1) - reset = strings.Replace(reset, "COMMIT;", "", 1) - if _, err = tx.Exec(ctx, "SAVEPOINT reset_test"); err != nil { - t.Fatal(err) - } - _, err = tx.Exec(ctx, reset) - if withFamily { - if err == nil || !strings.Contains(err.Error(), "Reset refused") { - t.Fatalf("expected refusal: %v", err) - } - if _, err = tx.Exec(ctx, "ROLLBACK TO SAVEPOINT reset_test"); err != nil { - t.Fatal(err) - } - } else if err != nil { - t.Fatal(err) - } - var battles, compositions int - if err = tx.QueryRow(ctx, `SELECT (SELECT count(*) FROM battles_ranked),(SELECT count(*) FROM army_compositions)`).Scan(&battles, &compositions); err != nil { - t.Fatal(err) - } - want := 0 - if withFamily { - want = 1 - } - if battles != want || compositions != want { - t.Fatalf("battles=%d compositions=%d want=%d", battles, compositions, want) - } - }) - } -} diff --git a/database/schema/cwl_season_statistics_test.go b/database/schema/cwl_season_statistics_test.go index fc8acdc..b60d5c5 100644 --- a/database/schema/cwl_season_statistics_test.go +++ b/database/schema/cwl_season_statistics_test.go @@ -8,57 +8,34 @@ import ( "github.com/jackc/pgx/v5" ) -func TestCWLSeasonStatisticsReconciliation(t *testing.T) { +func TestCWLSeasonStatisticsRemoved(t *testing.T) { if os.Getenv("CLASHKING_DISPOSABLE_TIMESCALE") != "1" { t.Skip("requires disposable Timescale fixture") } + if os.Getenv("CLASHKING_TIMESCALE_PROFILE") != "retained-api" { + t.Skip("requires retained-api migration profile") + } ctx := context.Background() conn, err := pgx.Connect(ctx, os.Getenv("TEST_DATABASE_URL")) if err != nil { t.Fatal(err) } defer conn.Close(ctx) - tx, err := conn.Begin(ctx) - if err != nil { - t.Fatal(err) - } - defer tx.Rollback(ctx) - _, err = tx.Exec(ctx, ` -INSERT INTO cwl_groups(cwl_id,season,cwl_league_id,rounds,state,war_size) VALUES - ('AAAAAAAAAAAA','2026-09',48000001,'[]','ended',15), - ('BBBBBBBBBBBB','2026-09',48000001,'[]','ended',15), - ('CCCCCCCCCCCC','2026-09',NULL,'[]','ended',15); -INSERT INTO cwl_group_clans(cwl_id,clan_tag) VALUES - ('AAAAAAAAAAAA','#2PP'),('AAAAAAAAAAAA','#9G2YV'),('BBBBBBBBBBBB','#2PP'); -INSERT INTO cwl_group_members(cwl_id,clan_tag,tag,town_hall) VALUES - ('AAAAAAAAAAAA','#2PP','#2PP',17),('AAAAAAAAAAAA','#2PP','#9G2YV',17), - ('AAAAAAAAAAAA','#9G2YV','#P0Y',16),('BBBBBBBBBBBB','#2PP','#2PP',0); -CALL public.reconcile_cwl_season_statistics(ARRAY['2026-09']);`) + var valid bool + err = conn.QueryRow(ctx, `SELECT + to_regclass('public.cwl_season_statistics') IS NULL + AND to_regprocedure('public.reconcile_cwl_season_statistics(text[])') IS NULL + AND to_regprocedure('public.cwl_town_halls_valid(jsonb)') IS NULL + AND to_regclass('public.cwl_groups') IS NOT NULL + AND to_regclass('public.cwl_group_clans') IS NOT NULL + AND to_regclass('public.cwl_group_members') IS NOT NULL + AND to_regclass('public.league_hitrate_stats') IS NOT NULL + AND to_regclass('public.ranked_league_tier_stats') IS NOT NULL + AND to_regclass('public.legend_daily_stats') IS NOT NULL`).Scan(&valid) if err != nil { t.Fatal(err) } - var groups, clans, members int64 - var halls []byte - if err := tx.QueryRow(ctx, `SELECT group_count,clan_count,registered_player_count,town_halls FROM cwl_season_statistics WHERE season='2026-09' AND cwl_league_id=48000001 AND war_size=15`).Scan(&groups, &clans, &members, &halls); err != nil { - t.Fatal(err) - } - if groups != 2 || clans != 3 || members != 4 || string(halls) != `[{"count": 2, "level": 17}, {"count": 1, "level": 16}]` { - t.Fatalf("stats=%d,%d,%d %s", groups, clans, members, halls) - } - var rows int - if err := tx.QueryRow(ctx, `SELECT count(*) FROM cwl_season_statistics`).Scan(&rows); err != nil || rows != 1 { - t.Fatalf("eligible statistic rows=%d err=%v", rows, err) - } - if _, err := tx.Exec(ctx, `DELETE FROM cwl_group_members WHERE cwl_id='BBBBBBBBBBBB'; CALL public.reconcile_cwl_season_statistics(ARRAY['2026-09']);`); err != nil { - t.Fatal(err) - } - if err := tx.QueryRow(ctx, `SELECT registered_player_count FROM cwl_season_statistics WHERE season='2026-09'`).Scan(&members); err != nil || members != 3 { - t.Fatalf("replacement members=%d err=%v", members, err) - } - for _, invalid := range []string{`{}`, `[{"level":16,"count":1},{"level":17,"count":1}]`, `[{"level":17,"count":-1}]`} { - var valid bool - if err := tx.QueryRow(ctx, `SELECT public.cwl_town_halls_valid($1::jsonb)`, invalid).Scan(&valid); err != nil || valid { - t.Fatalf("invalid town halls accepted: %s err=%v", invalid, err) - } + if !valid { + t.Fatal("CWL season-statistics objects remain or retained source/analytics tables are missing") } } diff --git a/database/schema/final_operational_contract_test.go b/database/schema/final_operational_contract_test.go new file mode 100644 index 0000000..d48187d --- /dev/null +++ b/database/schema/final_operational_contract_test.go @@ -0,0 +1,168 @@ +package schema + +import ( + "context" + "testing" +) + +func TestLegendLeaderboardUsesLivePlayerAndClanIdentity(t *testing.T) { + conn := disposableConn(t) + ctx := context.Background() + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(ctx) + _, err = tx.Exec(ctx, ` + INSERT INTO basic_clan(tag,name,public_war_log,war_wins,member_count,badge_token,troops_donated,troops_received) + VALUES('#2PP','Live Clan',true,1,1,'badge',0,0); + INSERT INTO basic_player(tag,name,league_id,clan_tag,townhall_level,trophies) + VALUES('#P0Y','First',105000036,'#2PP',18,6000),('#P0L','Second',105000036,NULL,18,5900),('#P0G','Not Legend',105000035,NULL,18,7000); + TRUNCATE legend_rankings_current; + INSERT INTO legend_rankings_current(tag,name,trophies,global_rank,clan_tag,clan_name) + SELECT player.tag,player.name,player.trophies,row_number() OVER(ORDER BY player.trophies DESC,player.tag),clan.tag,clan.name + FROM basic_player player LEFT JOIN basic_clan clan ON clan.tag=player.clan_tag + WHERE player.league_id=105000036`) + if err != nil { + t.Fatal(err) + } + var count, firstRank int + var clanName *string + if err = tx.QueryRow(ctx, `SELECT count(*),min(global_rank) FROM legend_rankings_current`).Scan(&count, &firstRank); err != nil { + t.Fatal(err) + } + if count != 2 || firstRank != 1 { + t.Fatalf("rows=%d firstRank=%d", count, firstRank) + } + if err = tx.QueryRow(ctx, `SELECT clan_name FROM legend_rankings_current WHERE tag='#P0L'`).Scan(&clanName); err != nil || clanName != nil { + t.Fatalf("absent clan was snapshotted: %v %v", clanName, err) + } + _, err = tx.Exec(ctx, `INSERT INTO legend_rankings_history(day,tag,global_rank,trophies) VALUES('2026-09-10','#P0Y',1,6000)`) + if err != nil { + t.Fatal(err) + } +} + +func TestNotificationPreferencesAreUserLevelAndLinksReset(t *testing.T) { + conn := disposableConn(t) + ctx := context.Background() + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(ctx) + _, err = tx.Exec(ctx, ` + INSERT INTO auth_users(user_id,provider) VALUES('100','discord'),('200','discord'); + INSERT INTO player_links(tag,is_verified,source,user_id) VALUES('#P0Y',true,'api_token','100'); + INSERT INTO mobile_push_devices(user_id,device_id,platform,provider,environment,token_ciphertext,token_hash) + VALUES('100','iphone','ios','fcm','production','cipher','hash'); + INSERT INTO mobile_notification_preferences(user_id,war_attacks_enabled,war_state_enabled,war_reminders_enabled,legend_defenses_enabled,reminder_timings) + VALUES('100',true,true,true,true,'{60,120}'); + INSERT INTO mobile_notification_accounts(user_id,player_tag) VALUES('100','#P0Y')`) + if err != nil { + t.Fatal(err) + } + if _, err = tx.Exec(ctx, `UPDATE player_links SET user_id='200' WHERE tag='#P0Y'`); err != nil { + t.Fatal(err) + } + var absent bool + if err = tx.QueryRow(ctx, `SELECT NOT EXISTS(SELECT 1 FROM mobile_notification_accounts WHERE player_tag='#P0Y')`).Scan(&absent); err != nil || !absent { + t.Fatalf("transferred account inherited enablement: %v", err) + } + if _, err = tx.Exec(ctx, `INSERT INTO mobile_notification_accounts(user_id,player_tag) VALUES('200','#P0Y')`); err != nil { + t.Fatal(err) + } + if _, err = tx.Exec(ctx, `UPDATE player_links SET is_verified=false WHERE tag='#P0Y'`); err != nil { + t.Fatal(err) + } + if err = tx.QueryRow(ctx, `SELECT NOT EXISTS(SELECT 1 FROM mobile_notification_accounts WHERE player_tag='#P0Y')`).Scan(&absent); err != nil || !absent { + t.Fatalf("unverified account retained enablement: %v", err) + } + if _, err = tx.Exec(ctx, `INSERT INTO mobile_notification_accounts(user_id,player_tag) VALUES('200','#P0Y')`); err == nil { + t.Fatal("unverified account accepted") + } +} + +func TestBasesUseBigintRelationsAndPrivateVotes(t *testing.T) { + conn := disposableConn(t) + ctx := context.Background() + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(ctx) + var baseID int64 + err = tx.QueryRow(ctx, `INSERT INTO bases(message_id,base_link,description) + VALUES('123','https://link.clashofclans.com/en?action=OpenLayout&id=TH17%3AHV%3AAAAA','Legacy row') RETURNING id`).Scan(&baseID) + if err != nil { + t.Fatal(err) + } + _, err = tx.Exec(ctx, `INSERT INTO base_images(base_id,position,image_url) VALUES($1,1,'https://api.clashk.ing/v2/media/base.png')`, baseID) + if err == nil { + _, err = tx.Exec(ctx, `UPDATE bases SET downloads=jsonb_build_object('100','2026-09-15T12:00:00Z') WHERE id=$1`, baseID) + } + if err == nil { + _, err = tx.Exec(ctx, `INSERT INTO base_votes(base_id,user_id,vote) VALUES($1,'100',1)`, baseID) + } + if err == nil { + _, err = tx.Exec(ctx, `UPDATE bases SET server_id='200',channel_id='300' WHERE id=$1`, baseID) + } + if err != nil { + t.Fatal(err) + } + var downloads, upvotes, downvotes int + if err = tx.QueryRow(ctx, `SELECT download_count,upvote_count,downvote_count FROM base_public_counts WHERE base_id=$1`, baseID).Scan(&downloads, &upvotes, &downvotes); err != nil { + t.Fatal(err) + } + if downloads != 1 || upvotes != 1 || downvotes != 0 { + t.Fatalf("counts=%d/%d/%d", downloads, upvotes, downvotes) + } + for _, q := range []string{ + `INSERT INTO bases(message_id,base_link) VALUES('124','https://evil.example/?action=OpenLayout&id=TH17')`, + `INSERT INTO base_images(base_id,position,image_url) VALUES(1,1,'https://example.com/base.png')`, + `INSERT INTO base_votes(base_id,user_id,vote) VALUES(1,'101',0)`, + } { + if _, err = tx.Exec(ctx, `SAVEPOINT invalid`); err != nil { + t.Fatal(err) + } + if _, err = tx.Exec(ctx, q); err == nil { + t.Fatalf("invalid base write accepted: %s", q) + } + if _, err = tx.Exec(ctx, `ROLLBACK TO SAVEPOINT invalid`); err != nil { + t.Fatal(err) + } + } +} + +func TestPersonalBaseLibraryIsUnlimitedAndTyped(t *testing.T) { + conn := disposableConn(t) + ctx := context.Background() + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(ctx) + _, err = tx.Exec(ctx, ` + INSERT INTO auth_users(user_id,provider) VALUES('owner','discord'); + INSERT INTO bases(message_id,base_link,description) + SELECT (900000000000000000+value)::text, + 'https://link.clashofclans.com/en?action=OpenLayout&id=TH17%3AWB%3A'||value, + 'Personal base '||value + FROM generate_series(1,25) value; + INSERT INTO user_saved_bases(user_id,base_id,kind) + SELECT 'owner',id,CASE id%3 WHEN 0 THEN NULL WHEN 1 THEN 'war' ELSE 'legend' END + FROM bases WHERE description LIKE 'Personal base %'`) + if err != nil { + t.Fatal(err) + } + var savedCount int + if err = tx.QueryRow(ctx, `SELECT count(*) FROM user_saved_bases WHERE user_id='owner'`).Scan(&savedCount); err != nil || savedCount != 25 { + t.Fatalf("unlimited saved rows=%d err=%v", savedCount, err) + } + if _, err = tx.Exec(ctx, `UPDATE user_saved_bases SET kind='farming' WHERE user_id='owner'`); err == nil { + t.Fatal("invalid saved-base kind accepted") + } + if err = tx.Rollback(ctx); err != nil { + t.Fatal(err) + } +} diff --git a/database/schema/personal_base_storage_migration_test.go b/database/schema/personal_base_storage_migration_test.go new file mode 100644 index 0000000..f7f3dae --- /dev/null +++ b/database/schema/personal_base_storage_migration_test.go @@ -0,0 +1,123 @@ +package schema + +import ( + "context" + "os" + "os/exec" + "testing" + + "github.com/jackc/pgx/v5" +) + +func TestPersonalBaseStorageMigration(t *testing.T) { + if os.Getenv("CLASHKING_DISPOSABLE_TIMESCALE") != "1" { + t.Skip("requires disposable Timescale fixture") + } + if os.Getenv("CLASHKING_TIMESCALE_PROFILE") != "baseline-018" { + t.Skip("requires baseline-018 migration profile") + } + ctx := context.Background() + conn, err := pgx.Connect(ctx, os.Getenv("TEST_DATABASE_URL")) + if err != nil { + t.Fatal(err) + } + defer conn.Close(ctx) + run := func(sql string, args ...any) { + t.Helper() + if _, err := conn.Exec(ctx, sql, args...); err != nil { + t.Fatal(err) + } + } + check := func(sql string, args ...any) { + t.Helper() + var ok bool + if err := conn.QueryRow(ctx, sql, args...).Scan(&ok); err != nil || !ok { + t.Fatalf("check failed: %s (%v)", sql, err) + } + } + reject := func(sql string, args ...any) { + t.Helper() + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(ctx) + if _, err = tx.Exec(ctx, sql, args...); err == nil { + t.Fatalf("invalid write accepted: %s", sql) + } + } + migrate := func(command ...string) { + t.Helper() + args := append([]string{"-env", "/dev/null", "-dir", "../timescale"}, command...) + cmd := exec.Command("goose", args...) + cmd.Env = append(os.Environ(), "GOOSE_DRIVER=postgres", "GOOSE_DBSTRING="+os.Getenv("TEST_DATABASE_URL"), "GOOSE_TABLE=goose_db_version") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("migration failed: %v\n%s", err, out) + } + } + + check(`SELECT max(version_id)=18 FROM goose_db_version WHERE is_applied`) + run(` + INSERT INTO auth_users(user_id,provider) VALUES('owner','discord'); + INSERT INTO player_links(tag,is_verified,source,user_id) VALUES('#P0Y',true,'api_token','owner'); + INSERT INTO bases(message_id,base_link,description) VALUES + ('700000000000000001','https://link.clashofclans.com/en?action=OpenLayout&id=TH17%3AWB%3AONE','One'), + ('700000000000000002','https://link.clashofclans.com/en?action=OpenLayout&id=TH17%3AWB%3ATWO','Two'); + INSERT INTO base_downloaders(base_id,user_id,downloaded_at) + SELECT id,'111111111111111111','2026-01-02T03:04:05.123456Z'::timestamptz FROM bases WHERE message_id='700000000000000001' + UNION ALL + SELECT id,'222222222222222222','2026-02-03T04:05:06Z'::timestamptz FROM bases WHERE message_id='700000000000000001'; + INSERT INTO base_votes(base_id,user_id,vote) + SELECT id,'111111111111111111',1 FROM bases WHERE message_id='700000000000000001' + UNION ALL + SELECT id,'222222222222222222',-1 FROM bases WHERE message_id='700000000000000001'; + INSERT INTO user_saved_bases(user_id,base_id) + SELECT 'owner',id FROM bases WHERE message_id IN ('700000000000000001','700000000000000002'); + INSERT INTO user_base_slots(user_id,player_tag,slot_kind,slot_number,base_id) + SELECT 'owner','#P0Y','war',1,id FROM bases WHERE message_id='700000000000000001';`) + + migrate("up-to", "19") + check(`SELECT max(version_id)=19 FROM goose_db_version WHERE is_applied`) + check(`SELECT to_regclass('public.base_downloaders') IS NULL + AND to_regclass('public.user_base_slots') IS NULL + AND to_regprocedure('public.require_verified_base_slot()') IS NULL + AND to_regprocedure('public.reset_base_slots_on_link_change()') IS NULL`) + check(`SELECT kind IS NULL FROM user_saved_bases + WHERE user_id='owner' AND base_id=(SELECT id FROM bases WHERE message_id='700000000000000001')`) + check(`SELECT (SELECT count(*) FROM jsonb_object_keys(downloads))=2 + AND (downloads->>'111111111111111111')::timestamptz='2026-01-02T03:04:05.123456Z'::timestamptz + AND (downloads->>'222222222222222222')::timestamptz='2026-02-03T04:05:06Z'::timestamptz + FROM bases WHERE message_id='700000000000000001'`) + check(`SELECT download_count=2 AND upvote_count=1 AND downvote_count=1 + FROM base_public_counts WHERE base_id=(SELECT id FROM bases WHERE message_id='700000000000000001')`) + + run(`UPDATE user_saved_bases SET kind='war' WHERE user_id='owner'`) + reject(`UPDATE user_saved_bases SET kind='farming' WHERE user_id='owner'`) + reject(`UPDATE bases SET downloads=downloads-'111111111111111111' WHERE message_id='700000000000000001'`) + reject(`UPDATE bases SET downloads=jsonb_set(downloads,'{111111111111111111}','"2026-09-15T00:00:00Z"') + WHERE message_id='700000000000000001'`) + reject(`UPDATE bases SET downloads='[]'::jsonb WHERE message_id='700000000000000002'`) + reject(`UPDATE bases SET downloads='{"not-a-user":"2026-09-15T00:00:00Z"}'::jsonb WHERE message_id='700000000000000002'`) + reject(`UPDATE bases SET downloads='{"333333333333333333":"not-a-time"}'::jsonb WHERE message_id='700000000000000002'`) + + run(`UPDATE bases SET downloads=CASE + WHEN downloads ? '333333333333333333' THEN downloads + ELSE downloads||jsonb_build_object('333333333333333333','2026-03-04T05:06:07Z') END + WHERE message_id='700000000000000001'`) + run(`UPDATE bases SET downloads=CASE + WHEN downloads ? '333333333333333333' THEN downloads + ELSE downloads||jsonb_build_object('333333333333333333','2026-09-15T00:00:00Z') END + WHERE message_id='700000000000000001'`) + check(`SELECT download_count=3 + AND downloads->>'333333333333333333'='2026-03-04T05:06:07Z' + FROM bases JOIN base_public_counts counts ON counts.base_id=bases.id + WHERE message_id='700000000000000001'`) + + run(`UPDATE user_saved_bases SET saved_at=now()-interval '100 days' WHERE user_id='owner'; + DELETE FROM user_saved_bases WHERE user_id='owner' AND saved_at < now()-interval '90 days'; + INSERT INTO user_saved_bases(user_id,base_id,kind) + SELECT 'owner',id,'legend' FROM bases WHERE message_id='700000000000000001'`) + check(`SELECT download_count=3 AND downloads ? '111111111111111111' + FROM bases JOIN base_public_counts counts ON counts.base_id=bases.id + WHERE message_id='700000000000000001'`) +} diff --git a/database/schema/ranked_battle_history_test.go b/database/schema/ranked_battle_history_test.go index 2f44eff..d40213e 100644 --- a/database/schema/ranked_battle_history_test.go +++ b/database/schema/ranked_battle_history_test.go @@ -2,9 +2,6 @@ package schema import ( "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" "os" "strings" "testing" @@ -12,28 +9,6 @@ import ( "github.com/jackc/pgx/v5" ) -func TestArmyHashV2Vectors(t *testing.T) { - raw, err := os.ReadFile("../../contracts/army-hash-v2.json") - if err != nil { - t.Fatal(err) - } - var contract struct { - Vectors []struct { - Normalized string `json:"normalized_share_code"` - Hex string `json:"sha256_hex"` - } `json:"vectors"` - } - if err = json.Unmarshal(raw, &contract); err != nil { - t.Fatal(err) - } - for _, vector := range contract.Vectors { - digest := sha256.Sum256(append([]byte{2}, []byte(vector.Normalized)...)) - if hex.EncodeToString(digest[:]) != vector.Hex { - t.Fatalf("invalid hash vector for %q", vector.Normalized) - } - } -} - func disposableConn(t *testing.T) *pgx.Conn { t.Helper() if os.Getenv("CLASHKING_DISPOSABLE_TIMESCALE") != "1" { @@ -53,13 +28,13 @@ func TestFinalBattleLeagueSchema(t *testing.T) { checks := []string{ `SELECT count(*)=2 FROM timescaledb_information.hypertables WHERE hypertable_name IN ('battles_farming','battles_ranked')`, `SELECT count(*)=4 FROM timescaledb_information.jobs WHERE hypertable_name IN ('battles_farming','battles_ranked') AND proc_name IN ('policy_compression','policy_retention')`, - `SELECT count(*)=0 FROM timescaledb_information.hypertables WHERE hypertable_name IN ('league_hitrate_stats','ranked_league_tier_stats','legend_daily_stats','army_family_daily_stats')`, - `SELECT count(*)=0 FROM timescaledb_information.jobs WHERE hypertable_name IN ('league_hitrate_stats','ranked_league_tier_stats','legend_daily_stats','army_family_daily_stats')`, - `SELECT to_regclass('public.ranked_league_groups') IS NULL AND to_regclass('public.ranked_item_presence_slots') IS NULL AND to_regclass('public.ranked_army_stats_prefix') IS NULL`, - `SELECT count(*)=9 FROM information_schema.columns WHERE table_schema='public' AND table_name='ranked_league_group_members' AND column_name IN ('season_id','group_tag','league_tier_id','player_tag','player_name','town_hall','placement','league_trophies','maximum_battle_count')`, - `SELECT count(*)=6 FROM information_schema.columns WHERE table_schema='public' AND table_name='ranked_league_group_members' AND column_name IN ('attack_win_count','attack_loss_count','attack_star_count','defense_win_count','defense_loss_count','defense_star_count')`, - `SELECT count(*)=0 FROM information_schema.columns WHERE table_schema='public' AND table_name='ranked_league_group_members' AND column_name IN ('clan_tag','clan_name','observed_at','missing_at','promoted','demoted','state')`, - `SELECT to_regclass('public.legend_history') IS NOT NULL`, + `SELECT to_regclass('public.army_family_daily_stats_v2') IS NULL AND to_regclass('public.legend_daily_stats_v2') IS NULL`, + `SELECT count(*)=0 FROM information_schema.columns WHERE table_schema='public' AND column_name IN ('army_hash','anchor_army_hash','parser_version')`, + `SELECT data_type='smallint' AND is_nullable='NO' FROM information_schema.columns WHERE table_schema='public' AND table_name='battles_ranked' AND column_name='duration_seconds'`, + `SELECT data_type='smallint' FROM information_schema.columns WHERE table_schema='public' AND table_name='battles_ranked' AND column_name='battle_mode'`, + `SELECT to_regclass('public.legend_rankings_current') IS NOT NULL + AND to_regclass('public.legend_rankings_history') IS NOT NULL + AND to_regclass('public.leaderboard_history_player_home') IS NOT NULL`, } for _, q := range checks { var ok bool @@ -77,17 +52,12 @@ func TestRankedPerspectivesCountOnePhysicalAttackOnce(t *testing.T) { t.Fatal(err) } defer tx.Rollback(ctx) - hash := make([]byte, 32) - _, err = tx.Exec(ctx, `INSERT INTO army_compositions(army_hash,normalized_share_code,main_troops,spells,heroes,equipment,pet_assignments) VALUES($1,'u1x1', '[{"id":1,"quantity":1}]','[{"id":2,"quantity":1,"clanCastle":false},{"id":2,"quantity":1,"clanCastle":true}]','{1}','[{"equipmentId":3,"heroId":1}]','[{"petId":4,"heroId":1}]')`, hash) - if err != nil { - t.Fatal(err) - } - _, err = tx.Exec(ctx, `INSERT INTO battles_ranked(player_tag,opponent_tag,battle_time,direction,battle_mode,player_town_hall,opponent_town_hall,stars,destruction_percentage,duration_seconds,looted_resources,share_code,army_hash) VALUES ('#2PP','#9G2YV','2026-09-08T12:00:00Z','attack','ranked',17,17,3,100,120,'{"gold":1}','u1x1',$1),('#9G2YV','#2PP','2026-09-08T12:00:00Z','defense','ranked',17,17,3,100,120,NULL,'u1x1',$1)`, hash) + _, err = tx.Exec(ctx, `INSERT INTO battles_ranked(player_tag,opponent_tag,battle_time,direction,battle_mode,player_town_hall,opponent_town_hall,stars,destruction_percentage,duration_seconds,looted_resources,share_code) VALUES ('#2PP','#9G2YV','2026-09-08T12:00:00Z',1,1,17,17,3,100,120,'{"gold":1}','u1x1'),('#9G2YV','#2PP','2026-09-08T12:00:00Z',2,1,17,17,3,100,120,NULL,'u1x1')`) if err != nil { t.Fatal(err) } var all, attacks int - if err := tx.QueryRow(ctx, `SELECT count(*),count(*) FILTER(WHERE direction='attack') FROM battles_ranked WHERE battle_time='2026-09-08T12:00:00Z'`).Scan(&all, &attacks); err != nil { + if err := tx.QueryRow(ctx, `SELECT count(*),count(*) FILTER(WHERE direction=1) FROM battles_ranked WHERE battle_time='2026-09-08T12:00:00Z'`).Scan(&all, &attacks); err != nil { t.Fatal(err) } if all != 2 || attacks != 1 { @@ -95,7 +65,7 @@ func TestRankedPerspectivesCountOnePhysicalAttackOnce(t *testing.T) { } } -func TestRankedGroupMemberConvergesOnCorrectedGroup(t *testing.T) { +func TestArmyCompositionFamilyAndDailyTotals(t *testing.T) { conn := disposableConn(t) ctx := context.Background() tx, err := conn.Begin(ctx) @@ -103,53 +73,38 @@ func TestRankedGroupMemberConvergesOnCorrectedGroup(t *testing.T) { t.Fatal(err) } defer tx.Rollback(ctx) - _, err = tx.Exec(ctx, `INSERT INTO ranked_league_group_members(season_id,group_tag,league_tier_id,player_tag,player_name,placement,league_trophies,attack_win_count,attack_loss_count,defense_win_count,defense_loss_count) VALUES(1,'#2PP',1,'#9G2YV','Player',1,1000,1,0,1,0) ON CONFLICT(season_id,player_tag) DO UPDATE SET group_tag=EXCLUDED.group_tag`) + var familyID int64 + err = tx.QueryRow(ctx, `WITH composition AS ( + INSERT INTO army_compositions(share_code,main_troops,clan_castle_troops,spells,heroes,equipment,pet_assignments,siege_machine_id) + VALUES('u1x1','[{"id":1,"quantity":10}]','[{"id":2,"quantity":1}]','[{"id":3,"quantity":2,"clanCastle":false}]','{4}','[{"equipmentId":5,"heroId":4}]','[{"petId":6,"heroId":4}]',7) + RETURNING share_code) + INSERT INTO army_families(representative_share_code,name) SELECT share_code,'Root Riders' FROM composition RETURNING family_id`).Scan(&familyID) if err != nil { t.Fatal(err) } - _, err = tx.Exec(ctx, `INSERT INTO ranked_league_group_members(season_id,group_tag,league_tier_id,player_tag,player_name,placement,league_trophies,attack_win_count,attack_loss_count,defense_win_count,defense_loss_count) VALUES(1,'#P0Y',1,'#9G2YV','Player',1,1000,1,0,1,0) ON CONFLICT(season_id,player_tag) DO UPDATE SET group_tag=EXCLUDED.group_tag`) - if err != nil { - t.Fatal(err) + _, err = tx.Exec(ctx, `INSERT INTO army_family_members(share_code,family_id) VALUES('u1x1',$1)`, familyID) + if err == nil { + _, err = tx.Exec(ctx, `INSERT INTO army_family_daily_stats(day,cohort,family_id,attack_count,distinct_player_count,zero_star_count,one_star_count,two_star_count,three_star_count,destruction_percentage_sum,duration_seconds_sum) + VALUES('2026-09-10','top_200',$1,2,2,0,0,1,1,180,240)`, familyID) } - var group string - var count int - if err = tx.QueryRow(ctx, `SELECT max(group_tag),count(*) FROM ranked_league_group_members WHERE season_id=1 AND player_tag='#9G2YV'`).Scan(&group, &count); err != nil { - t.Fatal(err) - } - if group != "#P0Y" || count != 1 { - t.Fatalf("group=%s rows=%d", group, count) + if err == nil { + _, err = tx.Exec(ctx, `INSERT INTO legend_daily_stats(day,cohort,attack_count,distinct_player_count,zero_star_count,one_star_count,two_star_count,three_star_count,destruction_percentage_sum,duration_seconds_sum,hero_stats,pet_stats,equipment_stats,pet_hero_assignments) + VALUES('2026-09-10','top_200',2,2,0,0,1,1,180,240,'[{"id":4,"uses":2,"triples":1}]','[{"id":6,"uses":2,"triples":1}]','[{"id":5,"uses":2,"triples":1}]','[{"petId":6,"heroId":4,"uses":2,"triples":1}]')`) } -} - -func TestArmyIdentityAndFamilyAssignmentAreImmutable(t *testing.T) { - conn := disposableConn(t) - ctx := context.Background() - tx, err := conn.Begin(ctx) if err != nil { t.Fatal(err) } - defer tx.Rollback(ctx) - hash := make([]byte, 32) for _, q := range []string{ - `INSERT INTO army_compositions(army_hash,normalized_share_code) VALUES($1,'u1x1')`, - `INSERT INTO army_families(anchor_army_hash,representative_share_code,family_name,source,named_by_subject) VALUES($1,'u1x1','Root Riders','admin','subject-1')`, - `INSERT INTO army_family_members(army_hash,anchor_army_hash,troop_housing_similarity,spell_capacity_similarity,heroes_exact,equipment_similarity,equipment_difference_count,matching_version) VALUES($1,$1,1,1,true,1,0,'v1')`, + `INSERT INTO army_family_daily_stats(day,cohort,family_id,attack_count,distinct_player_count,zero_star_count,one_star_count,two_star_count,three_star_count,destruction_percentage_sum,duration_seconds_sum) VALUES('2026-09-11','defense',1,1,1,0,0,0,1,100,120)`, + `INSERT INTO legend_daily_stats(day,cohort,attack_count,distinct_player_count,zero_star_count,one_star_count,two_star_count,three_star_count,destruction_percentage_sum,duration_seconds_sum,hero_stats) VALUES('2026-09-11','legend_i',1,1,0,0,0,1,100,120,'[{"id":4,"uses":2,"triples":1}]')`, } { - if _, err = tx.Exec(ctx, q, hash); err != nil { - t.Fatal(err) - } - } - if _, err = tx.Exec(ctx, `UPDATE army_families SET family_name='Root Riders Updated'`); err != nil { - t.Fatalf("mutable family metadata rejected: %v", err) - } - for _, q := range []string{`UPDATE army_compositions SET normalized_share_code='changed'`, `UPDATE army_families SET anchor_army_hash=decode(repeat('01',32),'hex')`, `DELETE FROM army_family_members`} { - if _, err = tx.Exec(ctx, `SAVEPOINT immutable`); err != nil { + if _, err = tx.Exec(ctx, `SAVEPOINT invalid`); err != nil { t.Fatal(err) } if _, err = tx.Exec(ctx, q); err == nil { - t.Fatalf("immutable write accepted: %s", q) + t.Fatalf("invalid aggregate accepted: %s", q) } - if _, err = tx.Exec(ctx, `ROLLBACK TO SAVEPOINT immutable`); err != nil { + if _, err = tx.Exec(ctx, `ROLLBACK TO SAVEPOINT invalid`); err != nil { t.Fatal(err) } } @@ -163,11 +118,7 @@ func TestRankedQueryPlansUsePlayerAndAttackIndexes(t *testing.T) { t.Fatal(err) } defer tx.Rollback(ctx) - hash := make([]byte, 32) - if _, err = tx.Exec(ctx, `INSERT INTO army_compositions(army_hash,normalized_share_code) VALUES($1,'plan')`, hash); err != nil { - t.Fatal(err) - } - if _, err = tx.Exec(ctx, `INSERT INTO battles_ranked(player_tag,opponent_tag,battle_time,direction,battle_mode,player_town_hall,opponent_town_hall,stars,destruction_percentage,army_hash) VALUES('#2PP','#9G2YV',now(),'attack','ranked',17,17,3,100,$1)`, hash); err != nil { + if _, err = tx.Exec(ctx, `INSERT INTO battles_ranked(player_tag,opponent_tag,battle_time,direction,battle_mode,player_town_hall,opponent_town_hall,stars,destruction_percentage,looted_resources) VALUES('#2PP','#9G2YV',now(),1,1,17,17,3,100,'{}')`); err != nil { t.Fatal(err) } if _, err = tx.Exec(ctx, `SET LOCAL enable_seqscan=off`); err != nil { @@ -175,7 +126,7 @@ func TestRankedQueryPlansUsePlayerAndAttackIndexes(t *testing.T) { } for _, tc := range []struct{ q, want string }{ {`EXPLAIN (COSTS OFF) SELECT * FROM battles_ranked WHERE player_tag='#2PP' AND battle_time >= now()-interval '30 days' ORDER BY battle_time DESC`, `idx_battles_ranked_player_time`}, - {`EXPLAIN (COSTS OFF) SELECT count(*) FROM battles_ranked WHERE battle_mode='ranked' AND direction='attack' AND battle_time >= now()-interval '30 days'`, `idx_battles_ranked_attacks_time`}, + {`EXPLAIN (COSTS OFF) SELECT count(*) FROM battles_ranked WHERE battle_mode=1 AND direction=1 AND battle_time >= now()-interval '30 days'`, `idx_battles_ranked_attacks_time`}, } { rows, e := tx.Query(ctx, tc.q) if e != nil { diff --git a/database/timescale/016_remove_cwl_season_statistics.sql b/database/timescale/016_remove_cwl_season_statistics.sql new file mode 100644 index 0000000..5ec8f16 --- /dev/null +++ b/database/timescale/016_remove_cwl_season_statistics.sql @@ -0,0 +1,15 @@ +-- +goose Up +-- The CWL population aggregate was rejected after migration 010 was published. +-- Its source CWL tables remain authoritative and are intentionally untouched. +DROP PROCEDURE IF EXISTS public.reconcile_cwl_season_statistics(text[]); +DROP TABLE IF EXISTS public.cwl_season_statistics; +DROP FUNCTION IF EXISTS public.cwl_town_halls_valid(jsonb); + +-- +goose Down +-- +goose StatementBegin +DO $$ +BEGIN + RAISE EXCEPTION 'migration 016 is irreversible: the rejected CWL season-statistics contract must not be restored by rollback'; +END +$$; +-- +goose StatementEnd diff --git a/database/timescale/017_final_operational_contract.sql b/database/timescale/017_final_operational_contract.sql new file mode 100644 index 0000000..276f7b5 --- /dev/null +++ b/database/timescale/017_final_operational_contract.sql @@ -0,0 +1,552 @@ +-- +goose Up +-- Final shared storage contract for battle analytics, Legend leaderboards, +-- mobile notification preferences, and Discord-backed base layouts. +SET LOCAL lock_timeout = '5s'; +SET LOCAL statement_timeout = '15min'; + +-- Reject values that cannot be represented without silently changing them. +-- Legacy NULL durations intentionally become zero. +-- +goose StatementBegin +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 FROM public.battles_farming + WHERE duration_seconds < 0 OR duration_seconds > 32767 + ) OR EXISTS ( + SELECT 1 FROM public.battles_ranked + WHERE duration_seconds < 0 OR duration_seconds > 32767 + ) THEN + RAISE EXCEPTION 'migration 017 found a battle duration outside the smallint range'; + END IF; +END +$$; +-- +goose StatementEnd + +SELECT remove_compression_policy('battles_farming', if_exists => TRUE); +SELECT remove_compression_policy('battles_ranked', if_exists => TRUE); +SELECT decompress_chunk(chunk, if_compressed => TRUE) +FROM show_chunks('public.battles_farming') chunk; +SELECT decompress_chunk(chunk, if_compressed => TRUE) +FROM show_chunks('public.battles_ranked') chunk; + +UPDATE public.battles_farming SET duration_seconds = 0 WHERE duration_seconds IS NULL; +ALTER TABLE public.battles_farming + DROP CONSTRAINT battles_farming_duration_check, + ALTER COLUMN duration_seconds TYPE smallint USING duration_seconds::smallint, + ALTER COLUMN duration_seconds SET DEFAULT 0, + ALTER COLUMN duration_seconds SET NOT NULL, + ADD CONSTRAINT battles_farming_duration_check CHECK (duration_seconds BETWEEN 0 AND 32767); + +UPDATE public.battles_ranked SET duration_seconds = 0 WHERE duration_seconds IS NULL; +DROP INDEX public.idx_battles_ranked_attacks_time; +DROP INDEX public.idx_battles_ranked_player_mode_time; +DROP INDEX public.idx_battles_ranked_player_direction_time; +ALTER TABLE public.battles_ranked + DROP CONSTRAINT battles_ranked_direction_check, + DROP CONSTRAINT battles_ranked_mode_check, + DROP CONSTRAINT battles_ranked_duration_check, + DROP CONSTRAINT battles_ranked_loot_check, + DROP CONSTRAINT battles_ranked_army_hash_length, + DROP COLUMN army_hash, + ALTER COLUMN direction TYPE smallint USING CASE direction WHEN 'attack' THEN 1 WHEN 'defense' THEN 2 END, + ALTER COLUMN battle_mode TYPE smallint USING CASE battle_mode WHEN 'ranked' THEN 1 WHEN 'legend' THEN 2 END, + ALTER COLUMN duration_seconds TYPE smallint USING duration_seconds::smallint, + ALTER COLUMN duration_seconds SET DEFAULT 0, + ALTER COLUMN duration_seconds SET NOT NULL, + ADD CONSTRAINT battles_ranked_direction_check CHECK (direction IN (1,2)), + ADD CONSTRAINT battles_ranked_mode_check CHECK (battle_mode IN (1,2)), + ADD CONSTRAINT battles_ranked_duration_check CHECK (duration_seconds BETWEEN 0 AND 32767), + ADD CONSTRAINT battles_ranked_loot_check CHECK ( + (direction = 1 AND looted_resources IS NOT NULL + AND public.battle_looted_resources_valid(looted_resources)) + OR (direction = 2 AND looted_resources IS NULL) + ); +CREATE INDEX idx_battles_ranked_player_mode_time + ON public.battles_ranked (player_tag, battle_mode, battle_time DESC); +CREATE INDEX idx_battles_ranked_player_direction_time + ON public.battles_ranked (player_tag, direction, battle_time DESC); +CREATE INDEX idx_battles_ranked_attacks_time + ON public.battles_ranked (battle_mode, battle_time DESC, player_town_hall, opponent_town_hall, share_code) + WHERE direction = 1; +ALTER TABLE public.battles_ranked SET ( + timescaledb.compress, + timescaledb.compress_orderby = 'battle_time DESC', + timescaledb.compress_segmentby = 'player_tag,battle_mode,direction' +); +SELECT add_compression_policy('battles_farming', compress_after => INTERVAL '30 days', if_not_exists => TRUE); +SELECT add_compression_policy('battles_ranked', compress_after => INTERVAL '30 days', if_not_exists => TRUE); + +-- Preserve only authoritative composition/family identity data. Daily totals +-- are derived and must be rebuilt because the old tables have no cohort key. +-- +goose StatementBegin +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 FROM public.army_families family + LEFT JOIN public.army_compositions composition + ON composition.normalized_share_code=family.representative_share_code + WHERE composition.normalized_share_code IS NULL + ) OR EXISTS ( + SELECT 1 FROM public.army_family_members member + LEFT JOIN public.army_compositions composition + ON composition.normalized_share_code=member.share_code + WHERE composition.normalized_share_code IS NULL + ) THEN + RAISE EXCEPTION 'migration 017 requires parsed compositions for every family representative and member'; + END IF; +END +$$; +-- +goose StatementEnd + +CREATE TEMP TABLE migration_017_compositions ON COMMIT DROP AS +SELECT normalized_share_code AS share_code, main_troops, clan_castle_troops, + spells, heroes, equipment, pet_assignments, siege_machine_id, created_at +FROM public.army_compositions; +CREATE TEMP TABLE migration_017_families ON COMMIT DROP AS +SELECT family_id, representative_share_code, + COALESCE(name, NULLIF(regexp_replace(btrim(family_name), '[[:space:]]+', ' ', 'g'), '')) AS name, + created_at, updated_at +FROM public.army_families; +CREATE TEMP TABLE migration_017_members ON COMMIT DROP AS +SELECT share_code, family_id, assigned_at +FROM public.army_family_members; + +DROP TABLE public.legend_daily_stats_v2; +DROP TABLE public.army_family_daily_stats_v2; +DROP TABLE public.army_family_daily_stats; +DROP TABLE public.legend_daily_stats; +DROP TABLE public.army_family_members; +DROP TABLE public.army_families; +DROP TABLE public.army_compositions; +DROP FUNCTION public.prepare_compatible_army_family_member(); +DROP FUNCTION public.prepare_compatible_army_family(); +DROP FUNCTION public.protect_army_family_anchor(); +DROP FUNCTION public.reject_army_identity_mutation(); + +CREATE TABLE public.army_compositions ( + share_code text PRIMARY KEY, + main_troops jsonb NOT NULL DEFAULT '[]', + clan_castle_troops jsonb NOT NULL DEFAULT '[]', + spells jsonb NOT NULL DEFAULT '[]', + heroes integer[] NOT NULL DEFAULT '{}', + equipment jsonb NOT NULL DEFAULT '[]', + pet_assignments jsonb NOT NULL DEFAULT '[]', + siege_machine_id integer, + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT army_compositions_share_code_check CHECK (btrim(share_code) <> ''), + CONSTRAINT army_compositions_main_troops_check CHECK (public.army_component_rows_valid(main_troops,'troop')), + CONSTRAINT army_compositions_clan_castle_troops_check CHECK (public.army_component_rows_valid(clan_castle_troops,'clan_castle_troop')), + CONSTRAINT army_compositions_spells_check CHECK (public.army_component_rows_valid(spells,'spell')), + CONSTRAINT army_compositions_heroes_check CHECK (public.army_hero_ids_valid(heroes)), + CONSTRAINT army_compositions_equipment_check CHECK (public.army_component_rows_valid(equipment,'equipment')), + CONSTRAINT army_compositions_pets_check CHECK (public.army_component_rows_valid(pet_assignments,'pet')), + CONSTRAINT army_compositions_siege_check CHECK (siege_machine_id IS NULL OR siege_machine_id >= 0) +); +INSERT INTO public.army_compositions +SELECT * FROM migration_017_compositions; + +CREATE TABLE public.army_families ( + family_id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + representative_share_code text NOT NULL UNIQUE REFERENCES public.army_compositions(share_code), + name text, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT army_families_name_check CHECK ( + name IS NULL OR (name <> '' AND char_length(name) <= 120 + AND name = regexp_replace(btrim(name), '[[:space:]]+', ' ', 'g')) + ) +); +CREATE UNIQUE INDEX army_families_name_unique ON public.army_families (lower(name)) WHERE name IS NOT NULL; +INSERT INTO public.army_families(family_id,representative_share_code,name,created_at,updated_at) + OVERRIDING SYSTEM VALUE +SELECT family_id,representative_share_code,name,created_at,updated_at +FROM migration_017_families; +SELECT setval( + pg_get_serial_sequence('public.army_families','family_id'), + COALESCE((SELECT max(family_id) FROM public.army_families),1), + EXISTS (SELECT 1 FROM public.army_families) +); + +CREATE TABLE public.army_family_members ( + share_code text PRIMARY KEY REFERENCES public.army_compositions(share_code) ON DELETE CASCADE, + family_id bigint NOT NULL REFERENCES public.army_families(family_id) ON DELETE CASCADE, + assigned_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX army_family_members_family_code_idx + ON public.army_family_members (family_id, share_code); +INSERT INTO public.army_family_members +SELECT * FROM migration_017_members; + +CREATE TABLE public.army_family_daily_stats ( + day date NOT NULL, + cohort text NOT NULL, + family_id bigint NOT NULL REFERENCES public.army_families(family_id) ON DELETE CASCADE, + attack_count bigint NOT NULL, + distinct_player_count bigint NOT NULL, + zero_star_count bigint NOT NULL, + one_star_count bigint NOT NULL, + two_star_count bigint NOT NULL, + three_star_count bigint NOT NULL, + destruction_percentage_sum bigint NOT NULL, + duration_seconds_sum bigint NOT NULL, + PRIMARY KEY (day, cohort, family_id), + CONSTRAINT army_family_daily_stats_cohort_check CHECK (cohort IN ('legend_i','top_1000','top_200')), + CONSTRAINT army_family_daily_stats_counts_check CHECK ( + attack_count >= 0 AND distinct_player_count BETWEEN 0 AND attack_count + AND zero_star_count >= 0 AND one_star_count >= 0 + AND two_star_count >= 0 AND three_star_count >= 0 + AND attack_count = zero_star_count + one_star_count + two_star_count + three_star_count + ), + CONSTRAINT army_family_daily_stats_sums_check CHECK ( + destruction_percentage_sum BETWEEN 0 AND attack_count * 100 + AND duration_seconds_sum BETWEEN 0 AND attack_count * 32767 + ) +); +CREATE INDEX army_family_daily_stats_family_day_idx + ON public.army_family_daily_stats (family_id, day DESC, cohort); + +CREATE TABLE public.legend_daily_stats ( + day date NOT NULL, + cohort text NOT NULL, + attack_count bigint NOT NULL, + distinct_player_count bigint NOT NULL, + zero_star_count bigint NOT NULL, + one_star_count bigint NOT NULL, + two_star_count bigint NOT NULL, + three_star_count bigint NOT NULL, + destruction_percentage_sum bigint NOT NULL, + duration_seconds_sum bigint NOT NULL, + hero_stats jsonb NOT NULL DEFAULT '[]', + pet_stats jsonb NOT NULL DEFAULT '[]', + equipment_stats jsonb NOT NULL DEFAULT '[]', + pet_hero_assignments jsonb NOT NULL DEFAULT '[]', + PRIMARY KEY (day, cohort), + CONSTRAINT legend_daily_stats_cohort_check CHECK (cohort IN ('legend_i','top_1000','top_200')), + CONSTRAINT legend_daily_stats_counts_check CHECK ( + attack_count >= 0 AND distinct_player_count BETWEEN 0 AND attack_count + AND zero_star_count >= 0 AND one_star_count >= 0 + AND two_star_count >= 0 AND three_star_count >= 0 + AND attack_count = zero_star_count + one_star_count + two_star_count + three_star_count + ), + CONSTRAINT legend_daily_stats_sums_check CHECK ( + destruction_percentage_sum BETWEEN 0 AND attack_count * 100 + AND duration_seconds_sum BETWEEN 0 AND attack_count * 32767 + ), + CONSTRAINT legend_daily_stats_heroes_check CHECK (public.item_usage_triples_within_attack_count(hero_stats,attack_count)), + CONSTRAINT legend_daily_stats_pets_check CHECK (public.item_usage_triples_within_attack_count(pet_stats,attack_count)), + CONSTRAINT legend_daily_stats_equipment_check CHECK (public.item_usage_triples_within_attack_count(equipment_stats,attack_count)), + CONSTRAINT legend_daily_stats_pet_hero_check CHECK (public.pet_hero_usage_triples_within_attack_count(pet_hero_assignments,attack_count)) +); + +COMMENT ON COLUMN public.battles_ranked.battle_mode IS '1=ranked, 2=legend.'; +COMMENT ON COLUMN public.battles_ranked.direction IS '1=attack, 2=defense. Aggregate writers count direction 1 only.'; +COMMENT ON TABLE public.army_family_daily_stats IS 'Attack-only totals for half-open 05:10 UTC days, grouped by cohort and family.'; +COMMENT ON TABLE public.legend_daily_stats IS 'Attack-only totals and item usage for half-open 05:10 UTC days, grouped by cohort.'; + +-- Tracking replaces this compact materialization only after a complete player +-- refresh. Preserve the relation in place because api_global_counts depends on +-- its object identity. +TRUNCATE TABLE public.legend_rankings_current; +DROP INDEX public.idx_legend_rankings_current_rank; +ALTER TABLE public.legend_rankings_current + RENAME COLUMN player_tag TO tag; +ALTER TABLE public.legend_rankings_current + RENAME COLUMN player_name TO name; +ALTER TABLE public.legend_rankings_current + RENAME COLUMN rank TO global_rank; +ALTER TABLE public.legend_rankings_current + DROP COLUMN data, + DROP COLUMN updated_at, + ALTER COLUMN name DROP DEFAULT, + ALTER COLUMN clan_name DROP DEFAULT, + ALTER COLUMN clan_name DROP NOT NULL, + ADD CONSTRAINT legend_rankings_current_tag_check CHECK (tag ~ '^#[0289PYLQGRJCUV]{1,15}$'), + ADD CONSTRAINT legend_rankings_current_name_check CHECK (btrim(name) <> ''), + ADD CONSTRAINT legend_rankings_current_rank_check CHECK (global_rank > 0), + ADD CONSTRAINT legend_rankings_current_trophies_check CHECK (trophies >= 0), + ADD CONSTRAINT legend_rankings_current_clan_check CHECK ( + (clan_tag IS NULL AND clan_name IS NULL) + OR (clan_tag IS NOT NULL AND clan_name IS NOT NULL + AND btrim(clan_tag) <> '' AND btrim(clan_name) <> '') + ), + ADD UNIQUE (global_rank); +CREATE INDEX idx_legend_rankings_current_trophies + ON public.legend_rankings_current (trophies DESC, tag); +INSERT INTO public.legend_rankings_current(tag,name,trophies,global_rank,clan_tag,clan_name) +SELECT player.tag,player.name,player.trophies, + row_number() OVER (ORDER BY player.trophies DESC,player.tag)::integer, + clan.tag,clan.name +FROM public.basic_player player +LEFT JOIN public.basic_clan clan ON clan.tag=player.clan_tag +WHERE player.league_id=105000036; + +-- Official leaderboard history keeps its complete location/player/clan/league +-- snapshots. Custom daily Legend ranks have a separate compact lifecycle. +CREATE TABLE public.legend_rankings_history ( + day date NOT NULL, + tag text NOT NULL, + global_rank integer NOT NULL, + trophies integer NOT NULL, + PRIMARY KEY (day, tag), + UNIQUE (day, global_rank), + CONSTRAINT legend_rankings_history_tag_check CHECK (tag ~ '^#[0289PYLQGRJCUV]{1,15}$'), + CONSTRAINT legend_rankings_history_rank_check CHECK (global_rank > 0), + CONSTRAINT legend_rankings_history_trophies_check CHECK (trophies >= 0) +); +CREATE INDEX idx_legend_rankings_history_player + ON public.legend_rankings_history (tag,day DESC); + +-- Notification account selection is valid only while the same user owns a +-- currently verified link. Ownership changes delete the selection first. +DROP TABLE public.mobile_notification_deliveries; +CREATE TABLE public.mobile_notification_preferences ( + user_id text PRIMARY KEY REFERENCES public.auth_users(user_id) ON DELETE CASCADE, + war_attacks_enabled boolean NOT NULL DEFAULT false, + war_state_enabled boolean NOT NULL DEFAULT false, + war_reminders_enabled boolean NOT NULL DEFAULT false, + raid_reminders_enabled boolean NOT NULL DEFAULT false, + events_enabled boolean NOT NULL DEFAULT false, + announcements_enabled boolean NOT NULL DEFAULT false, + monthly_support_enabled boolean NOT NULL DEFAULT false, + legend_defenses_enabled boolean NOT NULL DEFAULT false, + reminder_timings integer[] NOT NULL DEFAULT '{}', + raid_reminder_timings integer[] NOT NULL DEFAULT '{}', + updated_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT mobile_notification_preferences_war_timings_check CHECK ( + cardinality(reminder_timings) <= 3 + AND array_position(reminder_timings,NULL) IS NULL + AND 0 < ALL(reminder_timings) AND 2820 >= ALL(reminder_timings) + ), + CONSTRAINT mobile_notification_preferences_raid_timings_check CHECK ( + cardinality(raid_reminder_timings) <= 3 + AND array_position(raid_reminder_timings,NULL) IS NULL + AND 0 < ALL(raid_reminder_timings) AND 4320 >= ALL(raid_reminder_timings) + ) +); +INSERT INTO public.mobile_notification_preferences( + user_id,war_attacks_enabled,war_state_enabled,war_reminders_enabled, + raid_reminders_enabled,events_enabled,announcements_enabled,monthly_support_enabled, + reminder_timings,raid_reminder_timings,updated_at +) +SELECT users.user_id, + bool_or(device.war_attacks_enabled), bool_or(device.war_state_enabled), + bool_or(device.war_reminders_enabled), bool_or(device.raid_reminders_enabled), + bool_or(device.events_enabled), bool_or(device.announcements_enabled), + bool_or(device.monthly_support_enabled), + (SELECT recent.reminder_timings FROM public.mobile_push_devices recent + WHERE recent.user_id=users.user_id ORDER BY recent.last_seen_at DESC, recent.device_id LIMIT 1), + (SELECT recent.raid_reminder_timings FROM public.mobile_push_devices recent + WHERE recent.user_id=users.user_id ORDER BY recent.last_seen_at DESC, recent.device_id LIMIT 1), + now() +FROM ( + SELECT DISTINCT device.user_id + FROM public.mobile_push_devices device + JOIN public.auth_users auth ON auth.user_id=device.user_id +) users +JOIN public.mobile_push_devices device ON device.user_id=users.user_id +GROUP BY users.user_id; + +DROP INDEX public.idx_mobile_push_devices_announcements; +ALTER TABLE public.mobile_push_devices + DROP CONSTRAINT mobile_push_devices_reminder_timings_check, + DROP CONSTRAINT mobile_push_devices_raid_reminder_timings_check, + DROP COLUMN war_attacks_enabled, + DROP COLUMN war_state_enabled, + DROP COLUMN war_reminders_enabled, + DROP COLUMN raid_reminders_enabled, + DROP COLUMN events_enabled, + DROP COLUMN announcements_enabled, + DROP COLUMN monthly_support_enabled, + DROP COLUMN reminder_timings, + DROP COLUMN raid_reminder_timings; + +DROP INDEX public.idx_mobile_notification_accounts_delivery; +DROP INDEX public.idx_mobile_notification_accounts_player; +CREATE TEMP TABLE migration_017_notification_accounts ON COMMIT DROP AS +SELECT account.user_id, account.player_tag, account.active AS enabled, + account.created_at, account.updated_at +FROM public.mobile_notification_accounts account +JOIN public.player_links link + ON link.tag=account.player_tag AND link.user_id=account.user_id AND link.is_verified +JOIN public.auth_users auth ON auth.user_id=account.user_id; +DROP TABLE public.mobile_notification_accounts; +ALTER TABLE public.player_links ADD CONSTRAINT player_links_tag_user_key UNIQUE (tag,user_id); +CREATE TABLE public.mobile_notification_accounts ( + user_id text NOT NULL REFERENCES public.auth_users(user_id) ON DELETE CASCADE, + player_tag text NOT NULL, + enabled boolean NOT NULL DEFAULT true, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (user_id,player_tag), + FOREIGN KEY (player_tag,user_id) REFERENCES public.player_links(tag,user_id) ON DELETE CASCADE +); +INSERT INTO public.mobile_notification_accounts SELECT * FROM migration_017_notification_accounts; +CREATE INDEX idx_mobile_notification_accounts_delivery + ON public.mobile_notification_accounts (player_tag,user_id) WHERE enabled; + +-- +goose StatementBegin +CREATE FUNCTION public.require_verified_notification_account() +RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM public.player_links + WHERE tag=NEW.player_tag AND user_id=NEW.user_id AND is_verified + ) THEN + RAISE EXCEPTION 'notification account requires a currently verified owned link' + USING ERRCODE='23514'; + END IF; + NEW.updated_at := clock_timestamp(); + RETURN NEW; +END +$$; +-- +goose StatementEnd +CREATE TRIGGER mobile_notification_accounts_verified + BEFORE INSERT OR UPDATE ON public.mobile_notification_accounts + FOR EACH ROW EXECUTE FUNCTION public.require_verified_notification_account(); + +-- +goose StatementBegin +CREATE FUNCTION public.reset_notification_account_on_link_change() +RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF NEW.user_id IS DISTINCT FROM OLD.user_id OR NEW.is_verified IS DISTINCT FROM OLD.is_verified THEN + DELETE FROM public.mobile_notification_accounts + WHERE player_tag=OLD.tag AND user_id=OLD.user_id; + END IF; + RETURN NEW; +END +$$; +-- +goose StatementEnd +CREATE TRIGGER player_links_reset_notification_account + BEFORE UPDATE OF user_id,is_verified ON public.player_links + FOR EACH ROW EXECUTE FUNCTION public.reset_notification_account_on_link_change(); + +-- Only valid layout links are imported. Arrays are normalized into private +-- per-user relation tables; API responses expose counts, not voter identities. +-- +goose StatementBegin +CREATE FUNCTION public.base_layout_link_valid(value text) +RETURNS boolean LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE AS $$ +DECLARE + action_count integer; + action_valid boolean; + id_count integer; +BEGIN + IF value !~ '^https://link[.]clashofclans[.]com/[^?#]*[?][^#]+$' THEN + RETURN false; + END IF; + SELECT count(*), COALESCE(bool_and(captures[1]='OpenLayout'),false) + INTO action_count,action_valid + FROM regexp_matches(value,'[?&]action=([^&#]*)','g') AS matches(captures); + SELECT count(*) INTO id_count + FROM regexp_matches(value,'[?&]id=([^&#]+)','g'); + RETURN action_count=1 AND action_valid AND id_count=1; +END +$$; +-- +goose StatementEnd + +CREATE TEMP TABLE migration_017_bases ON COMMIT DROP AS +SELECT row_number() OVER (ORDER BY created_at,id)::bigint AS id, + message_id,base_link,created_at,server_id,channel_id,description, + images,downloaders,upvoter_ids,downvoter_ids +FROM public.bases +WHERE public.base_layout_link_valid(base_link) + AND message_id ~ '^[0-9]+$' + AND (server_id IS NULL OR server_id ~ '^[0-9]+$') + AND (channel_id IS NULL OR channel_id ~ '^[0-9]+$'); +DROP TABLE public.bases; +CREATE TABLE public.bases ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + base_link text NOT NULL, + message_id text NOT NULL, + server_id text, + channel_id text, + description text NOT NULL DEFAULT '', + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT bases_link_check CHECK (public.base_layout_link_valid(base_link)), + CONSTRAINT bases_message_id_check CHECK (message_id ~ '^[0-9]+$'), + CONSTRAINT bases_server_id_check CHECK (server_id IS NULL OR server_id ~ '^[0-9]+$'), + CONSTRAINT bases_channel_id_check CHECK (channel_id IS NULL OR channel_id ~ '^[0-9]+$'), + CONSTRAINT bases_message_location_check CHECK ((server_id IS NULL) = (channel_id IS NULL)), + CONSTRAINT bases_description_length_check CHECK (char_length(description) <= 1000), + UNIQUE (message_id) +); +INSERT INTO public.bases(id,base_link,message_id,server_id,channel_id,description,created_at) + OVERRIDING SYSTEM VALUE +SELECT id,base_link,message_id,server_id,channel_id,description,created_at +FROM migration_017_bases; +SELECT setval( + pg_get_serial_sequence('public.bases','id'), + COALESCE((SELECT max(id) FROM public.bases),1), + EXISTS (SELECT 1 FROM public.bases) +); + +CREATE TABLE public.base_images ( + base_id bigint NOT NULL REFERENCES public.bases(id) ON DELETE CASCADE, + position smallint NOT NULL, + image_url text NOT NULL, + PRIMARY KEY (base_id,position), + UNIQUE (base_id,image_url), + CONSTRAINT base_images_position_check CHECK (position BETWEEN 1 AND 4), + CONSTRAINT base_images_owned_url_check CHECK ( + image_url ~ '^https://api[.]clashk[.]ing/v2/media/[A-Za-z0-9][A-Za-z0-9._-]*$' + ) +); +INSERT INTO public.base_images(base_id,position,image_url) +SELECT base.id, image.ordinality::smallint, image.url +FROM migration_017_bases base +CROSS JOIN LATERAL unnest(base.images) WITH ORDINALITY image(url,ordinality) +WHERE image.ordinality <= 4 + AND image.url ~ '^https://api[.]clashk[.]ing/v2/media/[A-Za-z0-9][A-Za-z0-9._-]*$' +ON CONFLICT DO NOTHING; + +CREATE TABLE public.base_downloaders ( + base_id bigint NOT NULL REFERENCES public.bases(id) ON DELETE CASCADE, + user_id text NOT NULL, + downloaded_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (base_id,user_id), + CONSTRAINT base_downloaders_user_check CHECK (user_id ~ '^[0-9]+$') +); +INSERT INTO public.base_downloaders(base_id,user_id) +SELECT base.id,user_id +FROM migration_017_bases base CROSS JOIN LATERAL unnest(base.downloaders) user_id +WHERE user_id ~ '^[0-9]+$' +ON CONFLICT DO NOTHING; + +CREATE TABLE public.base_votes ( + base_id bigint NOT NULL REFERENCES public.bases(id) ON DELETE CASCADE, + user_id text NOT NULL, + vote smallint NOT NULL, + updated_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (base_id,user_id), + CONSTRAINT base_votes_user_check CHECK (user_id ~ '^[0-9]+$'), + CONSTRAINT base_votes_value_check CHECK (vote IN (-1,1)) +); +INSERT INTO public.base_votes(base_id,user_id,vote) +SELECT base.id,user_id,1 +FROM migration_017_bases base CROSS JOIN LATERAL unnest(base.upvoter_ids) user_id +WHERE user_id ~ '^[0-9]+$' +UNION ALL +SELECT base.id,user_id,-1 +FROM migration_017_bases base CROSS JOIN LATERAL unnest(base.downvoter_ids) user_id +WHERE user_id ~ '^[0-9]+$'; + +CREATE VIEW public.base_public_counts AS +SELECT base.id AS base_id, + (SELECT count(*) FROM public.base_downloaders downloader WHERE downloader.base_id=base.id) AS download_count, + (SELECT count(*) FROM public.base_votes vote WHERE vote.base_id=base.id AND vote.vote=1) AS upvote_count, + (SELECT count(*) FROM public.base_votes vote WHERE vote.base_id=base.id AND vote.vote=-1) AS downvote_count +FROM public.bases base; + +COMMENT ON TABLE public.base_votes IS 'Private per-user vote state. Public readers use base_public_counts.'; +COMMENT ON TABLE public.mobile_notification_accounts IS 'User-level enablement scoped to currently verified owned player links.'; +COMMENT ON TABLE public.mobile_notification_preferences IS 'User-level categories and reminder timings shared by every enabled device.'; + +-- +goose Down +-- The migration deliberately removes legacy identities and derived totals. A +-- downgrade cannot reconstruct those values, so fail transactionally. +-- +goose StatementBegin +DO $$ +BEGIN + RAISE EXCEPTION 'migration 017 is irreversible: final identities and user-level settings cannot be losslessly downgraded'; +END +$$; +-- +goose StatementEnd diff --git a/database/timescale/018_personal_base_library.sql b/database/timescale/018_personal_base_library.sql new file mode 100644 index 0000000..e785a87 --- /dev/null +++ b/database/timescale/018_personal_base_library.sql @@ -0,0 +1,84 @@ +-- +goose Up +-- A saved row is the authenticated user's durable reference to an existing +-- shared base. Download and explicit-save flows both upsert this same identity; +-- no layout payload is copied into a user-owned table. +CREATE TABLE public.user_saved_bases ( + user_id text NOT NULL REFERENCES public.auth_users(user_id) ON DELETE CASCADE, + base_id bigint NOT NULL REFERENCES public.bases(id) ON DELETE CASCADE, + saved_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (user_id,base_id) +); +CREATE INDEX idx_user_saved_bases_recent + ON public.user_saved_bases (user_id,saved_at DESC,base_id DESC); + +-- Slots are per verified linked game account. Reassigning a slot is an upsert +-- on the primary key; clearing it is a delete. A base may occupy one War and +-- one Legend slot for the same account, but cannot be duplicated within a kind. +CREATE TABLE public.user_base_slots ( + user_id text NOT NULL, + player_tag text NOT NULL, + slot_kind text NOT NULL, + slot_number smallint NOT NULL, + base_id bigint NOT NULL, + assigned_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (user_id,player_tag,slot_kind,slot_number), + UNIQUE (user_id,player_tag,slot_kind,base_id), + FOREIGN KEY (user_id,base_id) + REFERENCES public.user_saved_bases(user_id,base_id) ON DELETE CASCADE, + FOREIGN KEY (player_tag,user_id) + REFERENCES public.player_links(tag,user_id) ON DELETE CASCADE, + CONSTRAINT user_base_slots_kind_check CHECK (slot_kind IN ('war','legend')), + CONSTRAINT user_base_slots_number_check CHECK (slot_number BETWEEN 1 AND 3) +); + +-- +goose StatementBegin +CREATE FUNCTION public.require_verified_base_slot() +RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM public.player_links + WHERE tag=NEW.player_tag AND user_id=NEW.user_id AND is_verified + ) THEN + RAISE EXCEPTION 'base slot requires a currently verified owned link' + USING ERRCODE='23514'; + END IF; + NEW.assigned_at := clock_timestamp(); + RETURN NEW; +END +$$; +-- +goose StatementEnd +CREATE TRIGGER user_base_slots_verified + BEFORE INSERT OR UPDATE ON public.user_base_slots + FOR EACH ROW EXECUTE FUNCTION public.require_verified_base_slot(); + +-- The composite ownership FK prevents a transfer while old slots exist. Clear +-- them before ownership or verification changes so a new owner never inherits +-- another user's assignments. +-- +goose StatementBegin +CREATE FUNCTION public.reset_base_slots_on_link_change() +RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF NEW.user_id IS DISTINCT FROM OLD.user_id OR NEW.is_verified IS DISTINCT FROM OLD.is_verified THEN + DELETE FROM public.user_base_slots + WHERE player_tag=OLD.tag AND user_id=OLD.user_id; + END IF; + RETURN NEW; +END +$$; +-- +goose StatementEnd +CREATE TRIGGER player_links_reset_base_slots + BEFORE UPDATE OF user_id,is_verified ON public.player_links + FOR EACH ROW EXECUTE FUNCTION public.reset_base_slots_on_link_change(); + +COMMENT ON TABLE public.user_saved_bases IS + 'Authenticated-user references to shared bases. Downloads and explicit saves use one durable identity.'; +COMMENT ON TABLE public.user_base_slots IS + 'War and Legend slots 1-3 for currently verified player links; slot rows never own base payloads.'; + +-- +goose Down +DROP TRIGGER player_links_reset_base_slots ON public.player_links; +DROP FUNCTION public.reset_base_slots_on_link_change(); +DROP TRIGGER user_base_slots_verified ON public.user_base_slots; +DROP FUNCTION public.require_verified_base_slot(); +DROP TABLE public.user_base_slots; +DROP TABLE public.user_saved_bases; diff --git a/database/timescale/019_unlimited_personal_bases.sql b/database/timescale/019_unlimited_personal_bases.sql new file mode 100644 index 0000000..54986b6 --- /dev/null +++ b/database/timescale/019_unlimited_personal_bases.sql @@ -0,0 +1,100 @@ +-- +goose Up +ALTER TABLE public.user_saved_bases + ADD COLUMN kind text, + ADD CONSTRAINT user_saved_bases_kind_check CHECK (kind IN ('war','legend')); + +DROP TRIGGER player_links_reset_base_slots ON public.player_links; +DROP FUNCTION public.reset_base_slots_on_link_change(); +DROP TRIGGER user_base_slots_verified ON public.user_base_slots; +DROP FUNCTION public.require_verified_base_slot(); +DROP TABLE public.user_base_slots; + +-- Download identity belongs to the shared base. Each JSON object key is one +-- Discord user ID and its value is that user's immutable first-download time. +-- +goose StatementBegin +CREATE FUNCTION public.base_downloads_valid(downloads_value jsonb) +RETURNS boolean LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE AS $$ +DECLARE + item record; + parsed_at timestamptz; +BEGIN + IF jsonb_typeof(downloads_value) <> 'object' THEN + RETURN false; + END IF; + FOR item IN SELECT entry.key,entry.value FROM jsonb_each(downloads_value) entry + LOOP + IF item.key !~ '^[0-9]+$' + OR jsonb_typeof(item.value) <> 'string' + OR item.value #>> '{}' !~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}([.][0-9]{1,6})?(Z|[+-][0-9]{2}:[0-9]{2})$' + THEN + RETURN false; + END IF; + BEGIN + parsed_at := (item.value #>> '{}')::timestamptz; + EXCEPTION WHEN others THEN + RETURN false; + END; + END LOOP; + RETURN true; +END +$$; +-- +goose StatementEnd + +DROP VIEW public.base_public_counts; +ALTER TABLE public.bases + ADD COLUMN downloads jsonb NOT NULL DEFAULT '{}'::jsonb; +UPDATE public.bases base +SET downloads=migrated.downloads +FROM ( + SELECT base_id,jsonb_object_agg(user_id,to_jsonb(downloaded_at) ORDER BY user_id) AS downloads + FROM public.base_downloaders + GROUP BY base_id +) migrated +WHERE migrated.base_id=base.id; +ALTER TABLE public.bases + ADD CONSTRAINT bases_downloads_check CHECK (public.base_downloads_valid(downloads)); + +-- Existing keys may neither disappear nor change value. New keys remain an +-- ordinary atomic jsonb_set operation, so repeat clicks preserve first time. +-- +goose StatementBegin +CREATE FUNCTION public.preserve_base_first_downloads() +RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF EXISTS ( + SELECT 1 FROM jsonb_each(OLD.downloads) prior + WHERE NEW.downloads -> prior.key IS DISTINCT FROM prior.value + ) THEN + RAISE EXCEPTION 'existing base download identities and timestamps are immutable' + USING ERRCODE='23514'; + END IF; + RETURN NEW; +END +$$; +-- +goose StatementEnd +CREATE TRIGGER bases_preserve_first_downloads + BEFORE UPDATE OF downloads ON public.bases + FOR EACH ROW EXECUTE FUNCTION public.preserve_base_first_downloads(); + +DROP TABLE public.base_downloaders; +CREATE VIEW public.base_public_counts AS +SELECT base.id AS base_id, + (SELECT count(*) FROM jsonb_object_keys(base.downloads)) AS download_count, + (SELECT count(*) FROM public.base_votes vote WHERE vote.base_id=base.id AND vote.vote=1) AS upvote_count, + (SELECT count(*) FROM public.base_votes vote WHERE vote.base_id=base.id AND vote.vote=-1) AS downvote_count +FROM public.bases base; + +COMMENT ON COLUMN public.user_saved_bases.kind IS + 'Optional user label: war or legend. NULL means the saved base is not labeled yet.'; +COMMENT ON COLUMN public.bases.downloads IS + 'Discord user ID to immutable first-download ISO timestamp; independent of personal save retention.'; + +-- +goose Down +-- Removed numeric slot assignments cannot be reconstructed, so this migration +-- is intentionally forward-only. +-- +goose StatementBegin +DO $$ +BEGIN + RAISE EXCEPTION 'migration 019 is irreversible: removed personal base slots cannot be reconstructed'; +END +$$; +-- +goose StatementEnd diff --git a/database/timescale/README.md b/database/timescale/README.md index 2d3461b..fe3e44b 100644 --- a/database/timescale/README.md +++ b/database/timescale/README.md @@ -56,15 +56,17 @@ have target progress to report. ## Battle and league analytics -Migration 008 adds one-year farming history, two-perspective Ranked/Legend history, and immutable exact army compositions. Raw Ranked/Legend rows are compressed after 30 days; aggregate queries count only `direction = 'attack'` so the defense perspective does not double results. Migration 008 also reshapes the existing `ranked_league_group_members` table to match the source counters directly. +Migration 008 adds one-year farming history, two-perspective Ranked/Legend history, and exact army compositions. Raw Ranked/Legend rows are compressed after 30 days. Migration 008 also reshapes the existing `ranked_league_group_members` table to match the source counters directly. Migration 009 adds permanent normal-PostgreSQL rollups for league hit rates, Ranked tier populations, Legend daily item usage, and immutable army-family assignments. It does not add a Ranked group parent table, item presence registry, prefix tables, or compression policies for rollups. See [the complete storage contract](../../docs/ranked-battle-history.md). -`cwl_season_statistics` remains a separate normal PostgreSQL summary refreshed from the existing CWL group, clan, and member tables. See [the reconciliation contract](../../docs/cwl-season-statistics.md). The legacy `battlelogs`, its continuous aggregate, and `legend_history` remain during the consumer cutover. +Migration 016 removes the rejected CWL season-statistics table, reconciliation procedure, and JSON validator while leaving the canonical CWL group, clan, and member tables unchanged. Migration 009's public war and league rollups are unrelated and remain available. See [the cross-repository removal contract](../../docs/cwl-season-statistics-removal.md). + +Migration 017 is the irreversible final contract: numeric battle codes, non-null smallint durations, share-code composition/family identity, cohort-keyed unsuffixed aggregates, compact current/daily leaderboard storage, user-level notification preferences, and relational bigint bases. See [the consumer contract](../../docs/final-operational-contract.md) and [the battle contract](../../docs/ranked-battle-history.md). ## Index Notes -Farming and Ranked/Legend uniqueness is `(player_tag,battle_time)` after migration 013. Tracking stores only the requested player's perspective; `battle_mode` remains available for filtering. Player/time and player/mode/time indexes serve history, while a partial mode/time index containing only `direction = 'attack'` serves aggregate scans. +Farming and Ranked/Legend uniqueness is `(player_tag,battle_time)` after migration 013. Tracking stores only the requested player's perspective; migration 017 uses `battle_mode` 1/2 and `direction` 1/2. Player/time and player/mode/time indexes serve history, while a partial mode/time index containing only `direction = 1` serves aggregate scans. ## Global Clan Changes @@ -160,19 +162,23 @@ league ID; the API deletes the row after it fills that clan's missing ## Mobile Push State Mobile push state is current-state SQL data, not a hypertable. `mobile_push_devices` -stores one FCM token and its notification preferences per -`(user_id, device_id, provider, environment)`, with a unique token hash for idempotent -registration. Store the encrypted token in `token_ciphertext`, use `token_hash` only for -lookup/dedupe, and use `enabled` as the sole device-wide master notification switch. The -same row stores the category booleans and up to three reminder timings expressed as integer -minutes from 1 through 2,820. +stores one FCM token per `(user_id, device_id, provider, environment)`, with a unique token +hash for idempotent registration. Store the encrypted token in `token_ciphertext`, use +`token_hash` only for lookup/dedupe, and use `enabled` as the sole device-wide master +notification switch. Device rows do not store notification categories or reminder timings. + +`mobile_notification_preferences` stores category booleans and reminder timings once per +user. Every enabled device for that user shares those preferences. War timings allow up to +three integer-minute values from 1 through 2,820; raid timings allow up to three values from +1 through 4,320. `mobile_notification_accounts` stores the user-wide enabled verified player accounts; each row is authoritatively sourced from a verified player link. Player bookmarks do not create notification accounts. Clan notifications derive from verified players' current clans rather than a separate clan toggle. -Delivery requires an enabled device with an authorized or provisional OS authorization -status and the relevant category enabled on that device. +Delivery requires an enabled device and the relevant category enabled in that user's +preferences. Runtime delivery and deduplication use Valkey Streams; there is no SQL +notification-delivery or outbox table. `admin_posts.presentation_type` distinguishes block-based articles from hosted interactive stories. `show_on_home` controls carousel inclusion, while `pinned_on_home` keeps a post @@ -186,10 +192,16 @@ ahead of newer home posts without hiding those newer posts. `010_cwl_season_statistics.sql` adds rerunnable CWL population summaries, `013_battle_player_time_identity.sql` fixes raw identity at player/time, `014_ranked_defense_loot_nullable.sql` opens the bounded defense-loot cleanup, -and `015_army_code_family_compatibility.sql` adds code/family-ID identity plus +`015_army_code_family_compatibility.sql` adds code/family-ID identity plus parallel shifted-day aggregates without deleting the old history. +`016_remove_cwl_season_statistics.sql` then removes the rejected CWL-only +aggregate without changing the retained war and league analytics. +`017_final_operational_contract.sql` removes compatibility storage and establishes +the final cross-repository contracts. `018_personal_base_library.sql` introduces +authenticated saved-base references, and `019_unlimited_personal_bases.sql` supersedes +its slot model with an unlimited nullable-kind library and base-owned download history. Do not use the former 007–028 fixture numbering. See [the schema decisions](../../docs/worker-api-schema.md), [the disposable upgrade test](../../RETAINED_API_FIXTURE.md). -Migration 013 replaces the Ranked/Legend primary key without deleting data. Existing collisions cause a transactional failure. Migration 014 changes no rows; with the old writer paused, run the bounded defense-only cleanup companion before migration 015. Keep the writer paused until a binary that stores defense loot as SQL NULL is deployed. Migration 015 refuses incomplete cleanup, retains the player/time primary key, makes legacy hashes nullable, and introduces `army_family_daily_stats_v2` and `legend_daily_stats_v2` for the 05:10 UTC shifted-day meaning. See `docs/ranked-battle-history.md` for the complete schemas and rollback limits. +Migration 013 replaces the Ranked/Legend primary key without deleting data. Migration 014 and the bounded cleanup prepare nullable defense loot; migration 015 is the temporary compatibility bridge. Migration 017 removes that bridge, maps legacy NULL duration to zero, and requires coordinated final writers before ingestion resumes. diff --git a/docs/cross-repo-contracts.md b/docs/cross-repo-contracts.md index 8e3db1c..f075a53 100644 --- a/docs/cross-repo-contracts.md +++ b/docs/cross-repo-contracts.md @@ -52,6 +52,21 @@ database retention. surface. New app archive content uses `admin_posts`; the legacy announcement API keeps its own table until its callers are migrated. +### CWL season statistics removal + +Migration `016_remove_cwl_season_statistics.sql` removes only +`cwl_season_statistics`, `reconcile_cwl_season_statistics(text[])`, and +`cwl_town_halls_valid(jsonb)`. The current Tracking checkout has no caller, but +the deployed Tracking revision must still be checked for a scheduled or manual +procedure invocation before migration 016 is applied. The current API checkout +has no query against the removed table; it must add migration 016 to the expected +local schema inventory in `scripts/local-api-database.mjs`. + +The API's `/v2/stats/cwl` performance route does not use this table and remains +supported. Migration 009's public war hit-rate, Ranked population, Legend usage, +and army family aggregates also remain supported and must not be removed as part +of this cutover. + ## Validation checklist - [ ] Authoritative schema or service updated diff --git a/docs/cwl-season-statistics-removal.md b/docs/cwl-season-statistics-removal.md new file mode 100644 index 0000000..6a2638b --- /dev/null +++ b/docs/cwl-season-statistics-removal.md @@ -0,0 +1,24 @@ +# CWL season-statistics removal + +Migration `016_remove_cwl_season_statistics.sql` is the forward removal for the +CWL-only population aggregate introduced by migration 010. Applied migration +010 remains unchanged in Goose history; migration 016 drops its derived table, +reconciliation procedure, and private JSON validator. The canonical +`cwl_groups`, `cwl_group_clans`, and `cwl_group_members` sources remain. + +## Consumer cutover + +1. Tracking must remove the scheduled refresh and every direct + `CALL public.reconcile_cwl_season_statistics(...)` invocation. No caller is + present in the current primary Tracking checkout, so confirm the deployed + revision has the same boundary before applying the migration. +2. The current API checkout has no query against the removed table. Update its + hard-coded retained migration inventory in `scripts/local-api-database.mjs` + to include migration 016 so local database startup accepts the authoritative + profile. +3. Apply migration 016 through Goose only after both consumers are deployed. + +The API's `/v2/stats/cwl` performance route and public war hit-rate and summary +support are separate from this removed population aggregate. Migration 009's +`league_hitrate_stats`, `ranked_league_tier_stats`, `legend_daily_stats`, army +family tables, and their v2 successors remain supported. diff --git a/docs/cwl-season-statistics.md b/docs/cwl-season-statistics.md deleted file mode 100644 index 2fa6336..0000000 --- a/docs/cwl-season-statistics.md +++ /dev/null @@ -1,36 +0,0 @@ -# CWL season statistics - -Migration `010_cwl_season_statistics.sql` adds one ordinary PostgreSQL table. -It does not add archive storage, refresh state, or finalized-state columns. - -| Column | Type | Rule | -|---|---|---| -| `season` | `text` | PK; `YYYY-MM` | -| `cwl_league_id` | `integer` | PK; positive | -| `war_size` | `smallint` | PK; 1–50 | -| `group_count` | `bigint` | nonnegative eligible groups | -| `clan_count` | `bigint` | nonnegative distinct group/clan registrations | -| `registered_player_count` | `bigint` | nonnegative distinct group/player registrations | -| `town_halls` | `jsonb` | descending unique `[{"level":17,"count":10}]` values | -| `refreshed_at` | `timestamptz` | replacement time | - -`reconcile_cwl_season_statistics(text[])` takes advisory transaction lock -`4850467623902124044`, then deletes and rebuilds each selected season inside the -calling transaction. It calculates groups, clans, members, and town halls in -separate CTEs so joining source tables cannot multiply totals. Groups without a -positive league ID or a 1–50 war size are excluded; incomplete groups otherwise -contribute the partial data currently stored. - -Run one of the explicit rerunnable scopes with the same database connection -settings used for Goose: - -```sh -psql "$DATABASE_URL" --set scope=current --file scripts/reconcile-cwl-season-statistics.sql -psql "$DATABASE_URL" --set scope=previous --file scripts/reconcile-cwl-season-statistics.sql -psql "$DATABASE_URL" --set scope=current_previous --file scripts/reconcile-cwl-season-statistics.sql -psql "$DATABASE_URL" --set scope=all --file scripts/reconcile-cwl-season-statistics.sql -``` - -Passing `NULL` directly to the procedure reconciles every source or previously -materialized season. A specific `text[]` reconciles exactly those seasons and -removes stale rows if an eligible source group no longer exists. diff --git a/docs/final-operational-contract.md b/docs/final-operational-contract.md new file mode 100644 index 0000000..d454331 --- /dev/null +++ b/docs/final-operational-contract.md @@ -0,0 +1,71 @@ +# Final operational consumer contract + +Migration 017 finalizes the shared schemas consumed by Tracking, API, Bot, Dashboard, and App. Migration 018 introduced authenticated personal references and account-scoped slots; migration 019 supersedes the slot model with unlimited labeled saved bases and base-owned download history. Applied migrations 001–018 stay immutable; production execution remains a separate approval. + +## Legend leaderboards + +Tracking atomically replaces `legend_rankings_current` only after its complete refresh loop. Its final columns are `tag`, `name`, `trophies`, `global_rank`, nullable `clan_tag`, and nullable `clan_name`; `tag` is the primary key and `global_rank` is unique. Tracking selects current Legend I players with `basic_player.league_id=105000036`, orders by trophies descending then tag, and joins `basic_clan` for clan identity. If that join does not resolve, both clan fields are NULL. + +`leaderboard_history_player_home` remains the official detailed home-village leaderboard history. Migration 017 does not alter its `(location_id,date,player_tag)` identity or its player name, experience, trophy, win, rank, previous-rank, clan, and league snapshots. + +`legend_rankings_history` is the separate compact custom daily Legend snapshot with `(day,tag)` primary key, unique `(day,global_rank)`, and `global_rank,trophies`. Tracking populates it from the completed `legend_rankings_current` refresh; official leaderboard history is never used as a lossy source for this custom table. + +```json +{ + "current": {"tag":"#P0Y","name":"Player","trophies":6500,"globalRank":12,"clan":{"tag":"#2PP","name":"Clan"}}, + "history": [{"day":"2026-09-10","globalRank":15,"trophies":6420}] +} +``` + +There are no stored trophy buckets, coverage flags, stale flags, freshness timestamps, or separate Legend per-player daily table. + +## Mobile notifications + +`mobile_push_devices` owns device/token identity and its per-device `enabled` switch. It has no category or timing fields. `mobile_notification_preferences` owns one user's `war_attacks_enabled`, `war_state_enabled`, `war_reminders_enabled`, `raid_reminders_enabled`, `events_enabled`, `announcements_enabled`, `monthly_support_enabled`, `legend_defenses_enabled`, `reminder_timings`, and `raid_reminder_timings`. + +Migration 017 promotes each existing category with logical OR across a user's devices and takes timing arrays from that user's most recently seen device. The new Legend-defense switch starts false. There is no Legend-attack category. + +`mobile_notification_accounts` owns user-level per-player enablement. A row is accepted only when `player_links` has the same `(tag,user_id)` and is currently verified. Unlink, unverification, or ownership transfer deletes the enablement before the link changes, so a new owner never inherits it. Every enabled device for that user receives the same currently enabled verified accounts and preferences. + +Valkey Streams provide runtime delivery and deduplication. Migration 017 drops `mobile_notification_deliveries`; no SQL outbox or delivery-history table replaces it. + +## Base layouts + +`bases.id` is generated bigint identity. A valid row has an official HTTPS `OpenLayout` link, Discord `message_id`, optional paired `server_id/channel_id`, description up to 1,000 characters, creation time, and a `downloads` JSON object. `base_images` owns up to four ordered `https://api.clashk.ing/v2/media/...` URLs. `base_votes` privately stores one `-1` or `1` vote per `(base_id,user_id)`; public readers use `base_public_counts`. + +Migration 017 imports only valid legacy layout links, assigns bigint IDs in deterministic `(created_at,uuid)` order, filters image URLs to the ClashKing media namespace, and normalizes valid Discord user IDs into downloader/vote rows. Migration 019 moves each downloader's original timestamp onto `bases.downloads` and drops `base_downloaders`. It retains no UUID, Mongo ID, or legacy audit field. + +First-click conversion is a two-phase state machine without a legacy status column: + +1. The API resolves the row by unique `message_id`. Both location fields NULL means incomplete. +2. The API copies attachments and idempotently stages owned URLs in `base_images`. A copy failure leaves the row incomplete. +3. The Bot edits the Discord components. An edit failure leaves staged images intact and the location fields NULL, so retry does not recopy successful images. +4. After the Discord edit succeeds, the API sets `server_id` and `channel_id` together. This is the only completion transition. + +```json +{ + "id":"42","messageId":"123456789012345678","serverId":"234567890123456789","channelId":"345678901234567890", + "baseLink":"https://link.clashofclans.com/en?action=OpenLayout&id=TH17%3AHV%3AAAAA", + "images":["https://api.clashk.ing/v2/media/base.png"],"description":"Anti-three-star layout", + "downloadCount":10,"upvotes":8,"downvotes":1 +} +``` + +### Unlimited personal library + +`user_saved_bases` is the authenticated user's unlimited durable library. Its identity is `(user_id,base_id)`, where `user_id` references `auth_users` and `base_id` references the existing shared `bases` row. Nullable text `kind` is exactly `war`, `legend`, or NULL when the user has not labeled it. No base link, image, Discord location, vote, or count is duplicated. + +`bases.downloads` is a JSON object keyed by Discord user ID, with the immutable first-download ISO timestamp as its value. Migration 019 validates the object shape and timestamp values, and rejects removal or replacement of an existing key. A repeated posted-button click leaves the first timestamp unchanged while the personal save upsert can restore a deleted `user_saved_bases` row. + +Unsave and the 90-day personal-library cleanup delete only `user_saved_bases`; lifetime download identity remains on `bases`. Deleting the authenticated user clears that user's saved references, while deleting the shared base clears its saved references and votes through existing foreign keys. Migration 019 removes `user_base_slots` and its verification/link-change triggers and functions entirely. + +```json +{ + "savedBase": {"baseId":"42","kind":"legend","savedAt":"2026-09-11T06:00:00Z"}, + "downloads": {"123456789012345678":"2026-09-10T18:05:04.123Z"} +} +``` + +## War and CWL storage + +Migration 017 does not change war or CWL storage. `wars` retains its `archive_pack_id`, `archive_offset`, and `archive_compressed_bytes` locator triple; `war_archive_packs` and `war_archive_pending` remain authoritative for wars.clashk.ing pack publication. Canonical CWL groups, clans, members, rounds, wars, standings, and history remain available to `/v2/stats/cwl`. Migration 016 removes only the rejected `cwl_season_statistics` table, procedure, and validator, so no consumer may write or read that surface. diff --git a/docs/legend-composition-cleanup.md b/docs/legend-composition-cleanup.md deleted file mode 100644 index b3a14a5..0000000 --- a/docs/legend-composition-cleanup.md +++ /dev/null @@ -1,23 +0,0 @@ -# Legend-only composition rollout - -1. Apply Goose migration 012. It changes constraints/triggers only; it deletes no data. Family foreign keys, update immutability, and the no-truncate trigger remain. -2. Deploy Tracking's Legend-only composition writer. The old writer remains compatible with 012 during rollout. -3. Wait for operator confirmation of that deployment. Run dry-run from the local machine against production via an SSH tunnel. -4. Before apply, pause all battle ingestion and family closeouts for the duration of cleanup. This is required because the protected Legend-hash set is computed once. A live writer could otherwise introduce a reference after that snapshot. Resume workers afterward. - -From `database/`, with the existing migration connection environment pointing to the intended database: - -```sh -go run migrations/cleanup_army_compositions.go -``` - -Only after deployment confirmation, pausing writers, and reviewing the candidate count: - -```sh -ARMY_COMPOSITION_CLEANUP_APPLY=true ARMY_COMPOSITION_WRITERS_PAUSED=true \ -go run migrations/cleanup_army_compositions.go -``` - -The script uses temporary protected/candidate hash tables, a singleton advisory lock, a 5-second lock timeout, a 120-second statement timeout, and independently committed batches of 1,000 deletes. Batch size can be set to 1–5,000 with `ARMY_COMPOSITION_CLEANUP_BATCH_SIZE`. It does not delete battle rows, change schemas, rewrite tables, or run VACUUM FULL. An interrupted run is safe to restart while writers remain paused. Existing family references are rechecked and protected by foreign keys. The final check verifies all protected hashes remain. - -Regular vacuum can reuse freed space internally. Returning space to the filesystem is deliberately outside this operation. Rolling migration 012 back requires restoring compositions for all retained raw battle hashes first; Goose Down fails transactionally if those references are missing. diff --git a/docs/ranked-battle-history.md b/docs/ranked-battle-history.md index c2545a2..aeb2b69 100644 --- a/docs/ranked-battle-history.md +++ b/docs/ranked-battle-history.md @@ -1,165 +1,87 @@ -# Battle history and share-code army families - -Migrations 008–013 establish retained raw battle history and the requested-player identity `(player_tag,battle_time)`. Migration 014 opens the nullable defense-loot transition. After the bounded cleanup, migration 015 adds share-code family identity and parallel daily tables for the new Legend-day meaning. Applied migrations 008, 009, 012, and 013 are never rewritten. - -## Rollout order - -1. Apply Goose through 014 only. This changes no rows. -2. Pause the old battlelog writer and keep it paused through the schema/binary cutover. -3. Run `go run ./migrations/clear_ranked_defense_loot.go` from `database/` first as a dry run, then with `RANKED_DEFENSE_LOOT_CLEANUP_APPLY=true`, `RANKED_BATTLELOG_WRITERS_PAUSED=true`, and `RANKED_DEFENSE_LOOT_CLEANUP_DATABASE=`. The companion requires Goose version 14, refuses compressed chunks, and commits bounded `(player_tag,battle_time)` batches. -4. Apply Goose through 015. It refuses to apply if any defense still has non-null loot. -5. Deploy compatible code-only battle ingestion and the new family closeout before resuming the writer. The old closeout must remain paused because its hash tables and old day semantics are no longer authoritative. -6. Populate and compare the `*_v2` aggregates from retained raw data before switching readers. - -Migration 015 can roll back only while no new code-only rows, cleared defense rows, name edits, or v2 aggregates would be lost. Migration 014 cannot roll back while any ranked row contains null loot. Production execution and consumer deployment remain separate approvals. - -## Data flow - -```mermaid -flowchart LR - C[Clash battlelog response] --> W[Tracking normalizes the share code] - W -->|farming attack| F[battles_farming] - W -->|requested Ranked or Legend perspective| R[battles_ranked] - R -->|Legend attack rows in 05:10 window| D[Daily closeout] - M[army_family_members: code to family] --> D - A[army_families: fixed representative] --> D - D --> N[New permanent assignments] - D --> FS[army_family_daily_stats_v2] - D --> LS[legend_daily_stats_v2] +# Final battle and army analytics contract + +Migration 017 is the destructive finalization after migrations 008–016. It keeps raw battle history, replaces hash identity with canonical share codes, removes all `*_v2` tables, and establishes one unsuffixed aggregate contract. It is intentionally irreversible because removed hashes, legacy family provenance, and incompatible aggregate dimensions cannot be reconstructed. + +## Codes and day boundary + +| Field | Code | Meaning | +|---|---:|---| +| `battle_mode` | `1` | Ranked | +| `battle_mode` | `2` | Legend | +| `direction` | `1` | Attack; the only direction included in aggregates | +| `direction` | `2` | Defense | +| `cohort` | `legend_i` | All observed Legend I attackers | +| `cohort` | `top_1000` | The global top 1,000 subset | +| `cohort` | `top_200` | The global top 200 subset | + +Day D is the half-open interval `[D 05:10:00 UTC, D+1 05:10:00 UTC)`. Cohorts overlap by design, so one attack may contribute once to each qualifying cohort. No defense aggregate or direction dimension exists. + +## Raw battles + +`battles_farming` keeps `(player_tag,battle_time)` identity, `stars`, `destruction_percentage`, `looted_resources`, and nullable `share_code`. `duration_seconds` is `smallint NOT NULL DEFAULT 0`; migration 017 maps historical NULL to zero and refuses values outside `0..32767`. + +`battles_ranked` keeps `(player_tag,battle_time)` identity plus opponent, mode, direction, Town Hall, stars, destruction, loot, and nullable share code. Attack rows require a valid loot object; defense rows require SQL NULL. `duration_seconds` has the same non-null smallint contract. There is no army hash or parser version. + +```sql +INSERT INTO battles_ranked( + player_tag,opponent_tag,battle_time,direction,battle_mode, + player_town_hall,opponent_town_hall,stars,destruction_percentage, + duration_seconds,looted_resources,share_code +) VALUES ( + '#P0Y','#P0L','2026-09-10T12:00:00Z',1,2, + 18,18,3,100,121,'{"gold":1000,"elixir":900}','u1x1' +); +``` + +Tracking writes the actual requested-player perspective and never synthesizes the opposite perspective. It writes duration directly, including zero; `NULLIF(duration_seconds,0)` is invalid under this contract. + +## Compositions and families + +`army_compositions` is keyed by `share_code` and stores `main_troops`, `clan_castle_troops`, `spells`, sorted hero IDs, equipment assignments, pet assignments, optional siege-machine ID, and creation time. A share-code upsert may use `ON CONFLICT (share_code) DO NOTHING`; no hash calculation or compatibility row is allowed. + +```sql +INSERT INTO army_compositions( + share_code,main_troops,clan_castle_troops,spells,heroes, + equipment,pet_assignments,siege_machine_id +) VALUES ( + 'u1x1','[{"id":1,"quantity":10}]','[{"id":2,"quantity":1}]', + '[{"id":3,"quantity":2,"clanCastle":false}]','{4}', + '[{"equipmentId":5,"heroId":4}]','[{"petId":6,"heroId":4}]',7 +) ON CONFLICT (share_code) DO NOTHING; +``` + +`army_families` uses generated bigint `family_id`, one representative share code, optional normalized name, and timestamps. `army_family_members` is only `share_code`, `family_id`, and `assigned_at`; matching scores and versions are application logic, not durable identity. + +## Daily aggregates + +`army_family_daily_stats` has primary key `(day,cohort,family_id)` and these totals: `attack_count`, `distinct_player_count`, four star counts, `destruction_percentage_sum`, and `duration_seconds_sum`. + +`legend_daily_stats` has primary key `(day,cohort)`, the same totals, and JSON arrays `hero_stats`, `pet_stats`, `equipment_stats`, and `pet_hero_assignments`. Item rows are sorted and use these exact shapes: + +```json +{ + "heroStats": [{"id": 4, "uses": 120, "triples": 45}], + "petStats": [{"id": 6, "uses": 90, "triples": 38}], + "equipmentStats": [{"id": 5, "uses": 100, "triples": 40}], + "petHeroAssignments": [{"petId": 6, "heroId": 4, "uses": 80, "triples": 35}] +} ``` -Tracking stores only the requested player's perspective. It does not create the opposite side. All family and Legend statistics select `battle_mode='legend' AND direction='attack'`; a stored defense never contributes. - -## Raw history - -### `battles_farming` - -| Column | Type | Null/default and rule | -|---|---|---| -| `player_tag` | `text` | required; primary key with `battle_time` | -| `battle_time` | `timestamptz` | required | -| `stars` | `smallint` | required; 0–3 | -| `destruction_percentage` | `smallint` | required; 0–100 | -| `duration_seconds` | `integer` | nullable; nonnegative | -| `looted_resources` | `jsonb` | required; default `{}`; nonnegative `gold`, `elixir`, and `darkElixir` only | -| `share_code` | `text` | nullable; nonblank when present | - -New attack writes combine base and extra Gold, Elixir, and Dark Elixir into the same loot object. Sour Elixir is ignored. Existing attacks are not backfilled. The hypertable uses 30-day chunks, compression after 30 days, and one-year retention. - -### `battles_ranked` after migration 015 - -| Column | Type | Null/default and rule | -|---|---|---| -| `player_tag` | `text` | required; primary key with `battle_time` | -| `battle_time` | `timestamptz` | required | -| `opponent_tag` | `text` | required; different from player | -| `direction` | `text` | required; `attack` or `defense` | -| `battle_mode` | `text` | required; `ranked` or `legend` | -| `player_town_hall` | `smallint` | required; 1–20 | -| `opponent_town_hall` | `smallint` | required; 1–20 | -| `stars` | `smallint` | required; 0–3 | -| `destruction_percentage` | `smallint` | required; 0–100 | -| `duration_seconds` | `integer` | nullable; nonnegative | -| `looted_resources` | `jsonb` | required object for attacks; SQL NULL for defenses | -| `share_code` | `text` | nullable canonical code; nonblank when present | -| `army_hash` | `bytea` | nullable legacy compatibility column; omitted by the new writer | - -The primary key remains exactly `(player_tag,battle_time)`. No migration cleans the known minute-apart source duplicates because requested-player provenance cannot be reconstructed from SQL. Tracking reconciles those against live Clash battle logs. - -The seven-day hypertable retains one year and compresses after 30 days. Existing player/time, player/mode/time, and player/direction/time indexes remain. The bounded attack aggregate index becomes `(battle_mode,battle_time DESC) WHERE direction='attack'`; full share codes are not indexed on every battle. - -Detailed Ranked and Legend endpoints omit loot. The general player battle-history endpoint combines farming and ranked-table attacks, including Legend attacks, and may return the stored attack loot. - -## Permanent family identity - -### `army_families` after migration 015 - -| Column | Type | Null/default and rule | -|---|---|---| -| `family_id` | `bigint identity` | generated primary key; serialize as a decimal string in JSON | -| `representative_share_code` | `text` | required; unique; immutable | -| `name` | `text` | nullable manual name; normalized whitespace; maximum 120; unique case-insensitively | -| `hero_ids` | `integer[]` | required; default `{}`; sorted and duplicate-free; immutable | -| `equipment_ids` | `integer[]` | required; default `{}`; sorted and duplicate-free; immutable | -| `anchor_army_hash` | `bytea` | nullable legacy compatibility key; unique while retained | -| `family_name` | `text` | nullable legacy name | -| `source` | `text` | nullable legacy naming source | -| `named_by_subject` | `text` | nullable legacy provenance | -| `naming_model` | `text` | nullable legacy provenance | -| `naming_prompt_version` | `text` | nullable legacy provenance | -| `created_at` | `timestamptz` | required; default `now()` | -| `updated_at` | `timestamptz` | required; default `now()` | - -Existing families keep their representative, assignment anchor, and normalized existing name. Hero/equipment filters are derived once from the retained representative composition. New families write only the code, optional name, and representative ID arrays; they do not calculate a hash or create an `army_compositions` row. - -The insert compatibility trigger translates only old rows forward: when an old writer supplies `anchor_army_hash`, it copies the old name and derives representative ID arrays from the existing composition. It never synthesizes a hash. Updating `name` to NULL stays NULL because the trigger runs only on insert. - -### `army_family_members` after migration 015 - -| Column | Type | Null/default and rule | -|---|---|---| -| `share_code` | `text` | primary key; canonical exact-army identity | -| `family_id` | `bigint` | required FK to `army_families` | -| `troop_similarity` | `numeric(5,4)` | required; 0.8600–1 inclusive | -| `spell_similarity` | `numeric(5,4)` | required; 0.8000–1 inclusive | -| `equipment_similarity` | `numeric(5,4)` | required; 0.7500–1 inclusive | -| `army_hash` | `bytea` | nullable legacy identity; unique while retained | -| `anchor_army_hash` | `bytea` | nullable legacy family link | -| `troop_housing_similarity` | `numeric(5,4)` | nullable legacy score | -| `spell_capacity_similarity` | `numeric(5,4)` | nullable legacy score | -| `heroes_exact` | `boolean` | nullable legacy field | -| `equipment_difference_count` | `smallint` | nullable legacy field | -| `matching_version` | `text` | nullable legacy field | -| `assigned_at` | `timestamptz` | retained legacy timestamp; default `now()` | - -Assignments and representatives cannot be updated or deleted. New code-only inserts have no hash, matching-version, assignment-version, or equipment-difference requirement. An old hash-based insert is translated forward by looking up its existing composition code and family ID; the trigger never mirrors a new code row back into hash storage. - -The application matcher requires exact hero IDs, housing-weighted main troops at least 0.86, capacity-weighted main plus clan-castle spells at least 0.80, and equipment overlap at least 0.75. Siege machines, clan-castle troops, and pets do not affect matching. - -## Replacement daily tables - -Old `army_family_daily_stats` and `legend_daily_stats` remain unchanged during compatibility because their prior day/tier/Town Hall meaning cannot be relabeled truthfully. - -### `army_family_daily_stats_v2` - -| Column | Type | Rule | -|---|---|---| -| `family_id` | `bigint` | FK; primary key with day | -| `day` | `date` | shifted day; primary key with family | -| `attack_count` | `bigint` | nonnegative | -| `distinct_player_count` | `bigint` | nonnegative | -| `zero_star_count` … `three_star_count` | `bigint` | nonnegative; sum to attacks | -| `destruction_percentage_sum` | `bigint` | 0 through 100 × attacks | -| `duration_seconds_sum` | `bigint` | nonnegative | -| `duration_count` | `bigint` | 0 through attack count | - -Primary key: `(family_id,day)`. Index: `(day,family_id)`. Tracking stores every observed family, including families outside a daily top list. The table is permanent PostgreSQL storage without compression. - -### `legend_daily_stats_v2` - -| Column | Type | Rule | -|---|---|---| -| `day` | `date` | primary key | -| `attack_count` | `bigint` | nonnegative | -| `distinct_player_count` | `bigint` | nonnegative | -| `perfect_320_player_count` | `bigint` | 0 through player count | -| `zero_star_count` … `three_star_count` | `bigint` | nonnegative; sum to attacks | -| `destruction_percentage_sum` | `bigint` | 0 through 100 × attacks | -| `duration_seconds_sum` | `bigint` | nonnegative | -| `duration_count` | `bigint` | 0 through attack count | -| `hero_stats` | `jsonb` | required; default `[]`; sorted `{id,uses,triples}` | -| `pet_stats` | `jsonb` | same item shape | -| `equipment_stats` | `jsonb` | same item shape | -| `pet_hero_assignments` | `jsonb` | sorted `{petId,heroId,uses,triples}` | - -Each item satisfies `0 <= triples <= uses <= attack_count`. Missing codes remain in global attack, star, destruction, duration, and player totals but cannot contribute decoded item or family data. - -## Time and coverage contract - -Day D owns the half-open window `[D 05:10:00 UTC, D+1 05:10:00 UTC)`. The proposed job starts at 05:12 UTC. Adjacent days neither overlap nor leave a gap. - -Only stored Legend attack rows enter both v2 tables. Defenses, lower Ranked tiers, and synthetic opposite perspectives are excluded. Exact multi-day unique-player counts are supported only when the requested range is fully inside retained raw coverage and every completed day has a matching v2 global total, including explicit zero days. Otherwise the API reports player counts as unavailable and rejects a player-minimum filter. - -## Deferred destructive cleanup +Every JSON row satisfies `0 <= triples <= uses <= attack_count`. Duration averages divide `duration_seconds_sum` by `attack_count`; there is no `duration_count`. The migration does not relabel old aggregates because they lack the cohort dimension and use incompatible day semantics; Tracking rebuilds all three cohorts from retained raw attacks. + +Family IDs cross JSON boundaries as decimal strings: + +```json +{ + "day": "2026-09-10", + "cohort": "top_200", + "familyId": "42", + "attackCount": 200, + "distinctPlayerCount": 190, + "starCounts": {"zero": 1, "one": 9, "two": 70, "three": 120}, + "destructionPercentageSum": 18450, + "durationSecondsSum": 24200 +} +``` -A separate migration may remove `army_compositions`, hashes, legacy family/member columns, old naming provenance, and old daily tables only after every reader and writer uses the code/family-ID contract and retained history has been rebuilt and compared. It must also swap the `*_v2` names to their final unsuffixed names. Older aggregates without sufficient raw history require an explicit preservation decision; migration 015 does not delete or relabel them. +The API, Dashboard, and App use only the unsuffixed tables and these names. There are no `_v2` aliases, direction aggregates, per-player daily statistics, army hashes, or parser versions. diff --git a/scripts/benchmark-ranked-battle-history.sql b/scripts/benchmark-ranked-battle-history.sql index caf4de7..714356d 100644 --- a/scripts/benchmark-ranked-battle-history.sql +++ b/scripts/benchmark-ranked-battle-history.sql @@ -9,5 +9,5 @@ ORDER BY battle_time DESC; EXPLAIN (ANALYZE, COSTS OFF, BUFFERS) SELECT count(*) FROM public.battles_ranked -WHERE direction = 'attack' AND battle_time >= now() - interval '30 days'; +WHERE direction = 1 AND battle_time >= now() - interval '30 days'; RESET enable_seqscan; diff --git a/scripts/reconcile-cwl-season-statistics.sql b/scripts/reconcile-cwl-season-statistics.sql deleted file mode 100644 index 5fd9cb6..0000000 --- a/scripts/reconcile-cwl-season-statistics.sql +++ /dev/null @@ -1,29 +0,0 @@ -\set ON_ERROR_STOP on -\if :{?scope} -\else -\echo 'Set scope to current, previous, current_previous, or all.' -\quit 2 -\endif - -SELECT :'scope' IN ('current','previous','current_previous','all') AS valid_scope \gset -\if :valid_scope -\else -\echo 'scope must be current, previous, current_previous, or all.' -\quit 2 -\endif - -BEGIN; -CALL public.reconcile_cwl_season_statistics( - CASE :'scope' - WHEN 'current' THEN ARRAY[to_char(current_timestamp AT TIME ZONE 'UTC','YYYY-MM')] - WHEN 'previous' THEN ARRAY[to_char((current_timestamp AT TIME ZONE 'UTC') - INTERVAL '1 month','YYYY-MM')] - WHEN 'current_previous' THEN ARRAY[ - to_char(current_timestamp AT TIME ZONE 'UTC','YYYY-MM'), - to_char((current_timestamp AT TIME ZONE 'UTC') - INTERVAL '1 month','YYYY-MM') - ] - WHEN 'all' THEN NULL - END -); -COMMIT; - -TABLE public.cwl_season_statistics ORDER BY season DESC,cwl_league_id,war_size; diff --git a/scripts/reset-battle-history.sql b/scripts/reset-battle-history.sql index a0311a0..e495433 100644 --- a/scripts/reset-battle-history.sql +++ b/scripts/reset-battle-history.sql @@ -1,5 +1,5 @@ -- Explicit operator-only reset. Stop battle-log ingestion before running this. --- No CASCADE and no trigger disabling: retained families prevent this reset. +-- No CASCADE: retained families prevent deleting referenced compositions. BEGIN; SET LOCAL lock_timeout = '5s'; SET LOCAL statement_timeout = '60s'; diff --git a/scripts/retained-api-profile.sh b/scripts/retained-api-profile.sh index 910f990..7d52f89 100755 --- a/scripts/retained-api-profile.sh +++ b/scripts/retained-api-profile.sh @@ -1,5 +1,5 @@ # Canonical migration inventory for retained API tests and local development. -# This is the same contiguous 001-015 sequence used for production upgrades. +# This is the same contiguous 001-019 sequence used for production upgrades. fixture_image='timescale/timescaledb:2.29.2-pg18@sha256:9508616d5b941ed931198504c5db3fb47e8f53f790732ea1e889591f1062057c' fixture_sources=( 001_initial_stats.sql @@ -17,4 +17,8 @@ fixture_sources=( 013_battle_player_time_identity.sql 014_ranked_defense_loot_nullable.sql 015_army_code_family_compatibility.sql + 016_remove_cwl_season_statistics.sql + 017_final_operational_contract.sql + 018_personal_base_library.sql + 019_unlimited_personal_bases.sql ) diff --git a/scripts/with-test-timescale.sh b/scripts/with-test-timescale.sh index 262eb23..9481f49 100755 --- a/scripts/with-test-timescale.sh +++ b/scripts/with-test-timescale.sh @@ -2,8 +2,8 @@ # Run one integration suite against an isolated, Goose-migrated Timescale database. set -euo pipefail -if [[ ${1:-} != --profile || ( ${2:-} != retained-api && ${2:-} != baseline-006 && ${2:-} != baseline-013 ) ]]; then - echo 'An explicit --profile retained-api, baseline-006, or baseline-013 is required.' >&2 +if [[ ${1:-} != --profile || ( ${2:-} != retained-api && ${2:-} != baseline-006 && ${2:-} != baseline-013 && ${2:-} != baseline-018 ) ]]; then + echo 'An explicit --profile retained-api, baseline-006, baseline-013, or baseline-018 is required.' >&2 exit 2 fi fixture_profile="$2" @@ -20,6 +20,8 @@ if [[ $fixture_profile == baseline-006 ]]; then fixture_sources=("${fixture_sources[@]:0:6}") elif [[ $fixture_profile == baseline-013 ]]; then fixture_sources=("${fixture_sources[@]:0:13}") +elif [[ $fixture_profile == baseline-018 ]]; then + fixture_sources=("${fixture_sources[@]:0:18}") fi fixture_container='' fixture_child='' diff --git a/scripts/with-test-timescale.test.mjs b/scripts/with-test-timescale.test.mjs index c855418..9d66431 100644 --- a/scripts/with-test-timescale.test.mjs +++ b/scripts/with-test-timescale.test.mjs @@ -79,6 +79,10 @@ test('uses only retained authoritative migrations and its own disposable contain '013_battle_player_time_identity.sql', '014_ranked_defense_loot_nullable.sql', '015_army_code_family_compatibility.sql', + '016_remove_cwl_season_statistics.sql', + '017_final_operational_contract.sql', + '018_personal_base_library.sql', + '019_unlimited_personal_bases.sql', ]); } assert.equal(migrations[1].args.at(-1), 'up'); @@ -94,6 +98,14 @@ test('supports a migration-013 compatibility baseline', () => { assert.equal(migration.migrations.length, 13); }); +test('supports a migration-018 compatibility baseline', () => { + const result = runFixture({}, 'test "\$CLASHKING_TIMESCALE_PROFILE" = baseline-018', ['--profile', 'baseline-018']); + assert.equal(result.status, 0, result.stderr); + const migration = result.calls.find(call => call.tool === 'goose' && call.args.includes('up')); + assert.equal(migration.migrations.at(-1), '018_personal_base_library.sql'); + assert.equal(migration.migrations.length, 18); +}); + test('preserves a failing child exit code and still cleans up', () => { const result = runFixture({}, 'exit 42'); assert.equal(result.status, 42);