diff --git a/.github/workflows/schema-upgrade.yml b/.github/workflows/schema-upgrade.yml index c540978..32ed70c 100644 --- a/.github/workflows/schema-upgrade.yml +++ b/.github/workflows/schema-upgrade.yml @@ -24,3 +24,6 @@ jobs: - name: Apply 006 then verify the seeded-data upgrade working-directory: database run: bash ../scripts/with-test-timescale.sh --profile baseline-006 -- go test ./schema -run '^TestWorkerAPIUpgrade$' -count=1 -v + - name: Apply 013 then verify the army-family compatibility upgrade + working-directory: database + run: bash ../scripts/with-test-timescale.sh --profile baseline-013 -- go test ./schema -run '^TestArmyCodeFamilyCompatibilityMigration$' -count=1 -v diff --git a/RETAINED_API_FIXTURE.md b/RETAINED_API_FIXTURE.md index 86b92d8..4ee8e67 100644 --- a/RETAINED_API_FIXTURE.md +++ b/RETAINED_API_FIXTURE.md @@ -1,6 +1,6 @@ # Retained API database fixture -The canonical Goose sequence is `database/timescale/001` through `011`. +The canonical Goose sequence is `database/timescale/001` through `015`. Production was confirmed by the user to be at 006 before consolidation; files 001–006 are unchanged. `007_worker_api.sql` combines the unapplied Worker/API migrations formerly numbered 007, 008, 009, 010, 012, 013, 020, 022, 023, 027 and @@ -12,7 +12,7 @@ AI-budget mutex, subject mutex, resource ownership ledger and Discord delivery r `bash scripts/with-test-timescale.sh --profile retained-api -- COMMAND [ARG ...]` applies the complete canonical sequence to a fresh local tmpfs Timescale container. The API's existing retained-api invocation is unchanged. -`--profile baseline-006` applies only the immutable baseline for upgrade tests: +`--profile baseline-006` applies only the immutable baseline for Worker/API upgrade tests. `--profile baseline-013` applies through the deployed battle identity so migration 014 cleanup 015 conversion and rollback behavior can be tested with populated data: Cross-process tests that also require Valkey use the repository-owned wrapper: @@ -30,6 +30,7 @@ never reads or reuses retained Valkey coordinates. node --test scripts/with-test-timescale.test.mjs scripts/with-test-integration.test.mjs cd database bash ../scripts/with-test-timescale.sh --profile baseline-006 -- go test ./schema -run TestWorkerAPIUpgrade -count=1 -v +bash ../scripts/with-test-timescale.sh --profile baseline-013 -- go test ./schema -run TestArmyCodeFamilyCompatibilityMigration -count=1 -v ``` Both profiles reject remote Docker hosts, use deterministic test credentials and @@ -41,6 +42,15 @@ Old disposable fixtures at any prior 007–028 version must be recreated through the harness. Do not rename Goose history rows or apply this consolidation over such a database. Any persistent installation beyond 006 needs separate review. + +Migrations 014 and 015 are intentionally not one automatic production step. +Apply 014, pause the old battle writer, run the default-dry-run +`database/migrations/clear_ranked_defense_loot.go` companion until it verifies +zero remaining defense rows, then apply 015 and deploy compatible binaries before +resuming ingestion. The companion requires Goose version 14 exactly, refuses +compressed chunks, uses bounded committed batches, and never changes attack +loot. No fixture run is evidence that this sequence has run in production. + 007 leaves all 006 ticket tables and data unchanged. Stable panel/button IDs, editable names, archival, and stricter component validation remain deferred until the Bot rewrite resolves legacy Discord component IDs and duplicate diff --git a/database/migrations/clear_ranked_defense_loot.go b/database/migrations/clear_ranked_defense_loot.go new file mode 100644 index 0000000..a9d0807 --- /dev/null +++ b/database/migrations/clear_ranked_defense_loot.go @@ -0,0 +1,15 @@ +//go:build ignore + +package main + +import ( + "context" + + "github.com/ClashKingInc/DevKit/database/migrations/migrateutil" +) + +func main() { + migrateutil.Main("clear_ranked_defense_loot", func(ctx context.Context, cfg migrateutil.Config) error { + return migrateutil.ClearRankedDefenseLoot(ctx, cfg) + }) +} diff --git a/database/migrations/migrateutil/ranked_defense_loot.go b/database/migrations/migrateutil/ranked_defense_loot.go new file mode 100644 index 0000000..4b3d8c3 --- /dev/null +++ b/database/migrations/migrateutil/ranked_defense_loot.go @@ -0,0 +1,132 @@ +package migrateutil + +import ( + "context" + "fmt" + "strconv" +) + +func rankedDefenseLootSettings(env map[string]string) (bool, int, string, error) { + apply := env["RANKED_DEFENSE_LOOT_CLEANUP_APPLY"] == "true" + if apply && env["RANKED_BATTLELOG_WRITERS_PAUSED"] != "true" { + return false, 0, "", fmt.Errorf("apply requires RANKED_BATTLELOG_WRITERS_PAUSED=true") + } + expectedDatabase := env["RANKED_DEFENSE_LOOT_CLEANUP_DATABASE"] + if apply && expectedDatabase == "" { + return false, 0, "", fmt.Errorf("apply requires RANKED_DEFENSE_LOOT_CLEANUP_DATABASE") + } + batch := 1000 + if raw := env["RANKED_DEFENSE_LOOT_CLEANUP_BATCH_SIZE"]; raw != "" { + value, err := strconv.Atoi(raw) + if err != nil || value < 1 || value > 10000 { + return false, 0, "", fmt.Errorf("batch size must be 1..10000") + } + batch = value + } + return apply, batch, expectedDatabase, nil +} + +// ClearRankedDefenseLoot clears only existing ranked defense loot in bounded, +// committed batches. Rerunning resumes from rows that still have non-null loot. +func ClearRankedDefenseLoot(ctx context.Context, cfg Config) error { + apply, batch, expectedDatabase, err := rankedDefenseLootSettings(cfg.Env) + if err != nil { + return err + } + pool, err := TimescalePool(ctx, cfg) + if err != nil { + return err + } + defer pool.Close() + conn, err := pool.Acquire(ctx) + if err != nil { + return err + } + defer conn.Release() + var database string + if err = conn.QueryRow(ctx, "SELECT current_database()").Scan(&database); err != nil { + return err + } + if apply && database != expectedDatabase { + return fmt.Errorf("cleanup target is %q, expected %q", database, expectedDatabase) + } + if _, err = conn.Exec(ctx, "SET application_name='ck_ranked_defense_loot_cleanup'; SET lock_timeout='5s'; SET statement_timeout='120s'"); err != nil { + return err + } + var locked bool + if err = conn.QueryRow(ctx, "SELECT pg_try_advisory_lock(719233813)").Scan(&locked); err != nil { + return err + } + if !locked { + return fmt.Errorf("another ranked defense loot cleanup is running") + } + defer conn.Exec(context.Background(), "SELECT pg_advisory_unlock(719233813)") + + var version int64 + if err = conn.QueryRow(ctx, "SELECT max(version_id) FROM goose_db_version WHERE is_applied").Scan(&version); err != nil { + return err + } + if version != 14 { + return fmt.Errorf("cleanup requires Goose version 14 exactly; current version is %d", version) + } + + var compressedChunks int64 + if err = conn.QueryRow(ctx, `SELECT count(*) + FROM timescaledb_information.chunks + WHERE hypertable_schema='public' AND hypertable_name='battles_ranked' + AND is_compressed`).Scan(&compressedChunks); err != nil { + return err + } + if compressedChunks != 0 { + return fmt.Errorf("cleanup refused: battles_ranked has %d compressed chunks", compressedChunks) + } + + var candidates int64 + if err = conn.QueryRow(ctx, `SELECT count(*) FROM public.battles_ranked + WHERE direction='defense' AND looted_resources IS NOT NULL`).Scan(&candidates); err != nil { + return err + } + fmt.Printf("dry_run=%t defense_rows=%d batch_size=%d compressed_chunks=0\n", !apply, candidates, batch) + if !apply { + return nil + } + + var cleared int64 + for { + var count int64 + err = conn.QueryRow(ctx, `WITH batch AS ( + SELECT player_tag,battle_time + FROM public.battles_ranked + WHERE direction='defense' AND looted_resources IS NOT NULL + ORDER BY battle_time,player_tag + LIMIT $1 + FOR UPDATE SKIP LOCKED + ), cleared AS ( + UPDATE public.battles_ranked battle + SET looted_resources=NULL + FROM batch + WHERE (battle.player_tag,battle.battle_time) + = (batch.player_tag,batch.battle_time) + RETURNING 1 + ) + SELECT count(*) FROM cleared`, batch).Scan(&count) + if err != nil { + return fmt.Errorf("cleanup stopped after %d committed rows: %w", cleared, err) + } + cleared += count + fmt.Printf("cleared=%d remaining_at_start=%d\n", cleared, candidates-cleared) + if count == 0 { + break + } + } + var remaining int64 + if err = conn.QueryRow(ctx, `SELECT count(*) FROM public.battles_ranked + WHERE direction='defense' AND looted_resources IS NOT NULL`).Scan(&remaining); err != nil { + return err + } + if remaining != 0 { + return fmt.Errorf("cleanup incomplete: %d defense rows still contain loot", remaining) + } + fmt.Printf("cleanup complete: cleared=%d remaining=0\n", cleared) + return nil +} diff --git a/database/migrations/migrateutil/ranked_defense_loot_test.go b/database/migrations/migrateutil/ranked_defense_loot_test.go new file mode 100644 index 0000000..1215f5b --- /dev/null +++ b/database/migrations/migrateutil/ranked_defense_loot_test.go @@ -0,0 +1,31 @@ +package migrateutil + +import "testing" + +func TestRankedDefenseLootSettings(t *testing.T) { + apply, batch, database, err := rankedDefenseLootSettings(nil) + if err != nil || apply || batch != 1000 || database != "" { + t.Fatal(apply, batch, database, err) + } + if _, _, _, err = rankedDefenseLootSettings(map[string]string{ + "RANKED_DEFENSE_LOOT_CLEANUP_APPLY": "true", + }); err == nil { + t.Fatal("apply accepted without paused writers") + } + apply, batch, database, err = rankedDefenseLootSettings(map[string]string{ + "RANKED_DEFENSE_LOOT_CLEANUP_APPLY": "true", + "RANKED_BATTLELOG_WRITERS_PAUSED": "true", + "RANKED_DEFENSE_LOOT_CLEANUP_BATCH_SIZE": "25", + "RANKED_DEFENSE_LOOT_CLEANUP_DATABASE": "clashking_test", + }) + if err != nil || !apply || batch != 25 || database != "clashking_test" { + t.Fatal(apply, batch, database, err) + } + for _, value := range []string{"0", "10001", "bad"} { + if _, _, _, err = rankedDefenseLootSettings(map[string]string{ + "RANKED_DEFENSE_LOOT_CLEANUP_BATCH_SIZE": value, + }); err == nil { + t.Fatal(value) + } + } +} diff --git a/database/migrations/war_archive_schema_test.go b/database/migrations/war_archive_schema_test.go index b369bb7..c3abbc0 100644 --- a/database/migrations/war_archive_schema_test.go +++ b/database/migrations/war_archive_schema_test.go @@ -26,6 +26,8 @@ func TestTimescaleMigrationsAreExplicitlyNumbered(t *testing.T) { "011_active_verified_players.sql", "012_legend_only_army_compositions.sql", "013_battle_player_time_identity.sql", + "014_ranked_defense_loot_nullable.sql", + "015_army_code_family_compatibility.sql", } if len(files) != len(want) { t.Fatalf("Timescale migrations = %v, want %v", files, want) diff --git a/database/schema/army_code_family_migration_test.go b/database/schema/army_code_family_migration_test.go new file mode 100644 index 0000000..e2c3716 --- /dev/null +++ b/database/schema/army_code_family_migration_test.go @@ -0,0 +1,269 @@ +package schema + +import ( + "context" + "os" + "os/exec" + "testing" + + "github.com/ClashKingInc/DevKit/database/migrations/migrateutil" + "github.com/jackc/pgx/v5" +) + +func TestArmyCodeFamilyCompatibilityMigration(t *testing.T) { + if os.Getenv("CLASHKING_DISPOSABLE_TIMESCALE") != "1" { + t.Skip("requires disposable Timescale fixture") + } + if os.Getenv("CLASHKING_TIMESCALE_PROFILE") != "baseline-013" { + t.Skip("requires baseline-013 migration profile") + } + ctx := context.Background() + conn, err := pgx.Connect(ctx, os.Getenv("TEST_DATABASE_URL")) + if err != nil { + t.Fatal(err) + } + defer conn.Close(ctx) + + run := func(sql string, args ...any) { + t.Helper() + if _, err := conn.Exec(ctx, sql, args...); err != nil { + t.Fatal(err) + } + } + check := func(sql string, args ...any) { + t.Helper() + var ok bool + if err := conn.QueryRow(ctx, sql, args...).Scan(&ok); err != nil || !ok { + t.Fatalf("check failed: %s (%v)", sql, err) + } + } + reject := func(sql string, args ...any) { + t.Helper() + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(ctx) + if _, err := tx.Exec(ctx, sql, args...); err == nil { + t.Fatalf("invalid write accepted: %s", sql) + } + } + migrate := func(command ...string) error { + t.Helper() + args := append([]string{"-env", "/dev/null", "-dir", "../timescale"}, command...) + cmd := exec.Command("goose", args...) + cmd.Env = append(os.Environ(), + "GOOSE_DRIVER=postgres", + "GOOSE_DBSTRING="+os.Getenv("TEST_DATABASE_URL"), + "GOOSE_TABLE=goose_db_version", + ) + output, err := cmd.CombinedOutput() + if err != nil { + t.Log(string(output)) + } + return err + } + + check(`SELECT max(version_id)=13 FROM goose_db_version WHERE is_applied`) + run(`INSERT INTO army_compositions( + army_hash,normalized_share_code,heroes,equipment + ) VALUES + (decode(repeat('aa',32),'hex'),'u1x0','{1,2}','[{"equipmentId":3,"heroId":1},{"equipmentId":4,"heroId":2}]'), + (decode(repeat('bb',32),'hex'),'u2x0','{1,2}','[{"equipmentId":3,"heroId":1}]'); + INSERT INTO army_families( + anchor_army_hash,representative_share_code,family_name,source,named_by_subject + ) VALUES ( + decode(repeat('aa',32),'hex'),'u1x0',' Root Riders ','admin','subject-1' + ); + INSERT INTO army_family_members( + army_hash,anchor_army_hash,troop_housing_similarity,spell_capacity_similarity, + heroes_exact,equipment_similarity,equipment_difference_count,matching_version + ) VALUES ( + decode(repeat('bb',32),'hex'),decode(repeat('aa',32),'hex'),0.9000,0.8000, + true,0.7500,2,'v1' + ); + INSERT INTO army_family_daily_stats( + anchor_army_hash,day,attack_count,distinct_player_count, + zero_star_count,one_star_count,two_star_count,three_star_count, + destruction_percentage_sum,duration_seconds_sum + ) VALUES ( + decode(repeat('aa',32),'hex'),'2026-08-01',2,2,0,0,1,1,180,200 + ); + INSERT INTO legend_daily_stats( + day,league_tier_id,town_hall,attack_count,distinct_player_count, + perfect_320_player_count,zero_star_count,one_star_count,two_star_count, + three_star_count,destruction_percentage_sum,duration_seconds_sum + ) VALUES ('2026-08-01',1,18,2,2,0,0,0,1,1,180,200); + INSERT INTO battles_ranked( + player_tag,opponent_tag,battle_time,direction,battle_mode, + player_town_hall,opponent_town_hall,stars,destruction_percentage, + duration_seconds,share_code,army_hash + ) VALUES + ('#P0Y','#P0L',now()-interval '1 day','attack','legend', + 18,18,3,100,120,'u2x0',decode(repeat('bb',32),'hex')), + ('#P0L','#P0Y',now()-interval '1 day','defense','legend', + 18,18,3,100,120,'u2x0',decode(repeat('bb',32),'hex')); + `) + check(`SELECT count(*)=0 FROM timescaledb_information.chunks + WHERE hypertable_name='battles_ranked' AND is_compressed`) + if err := migrate("up-to", "14"); err != nil { + t.Fatal("nullable defense loot expansion:", err) + } + check(`SELECT max(version_id)=14 FROM goose_db_version WHERE is_applied`) + cfg := migrateutil.Config{TimescaleURL: os.Getenv("TEST_DATABASE_URL"), Env: map[string]string{}} + if err := migrateutil.ClearRankedDefenseLoot(ctx, cfg); err != nil { + t.Fatal("defense loot dry run:", err) + } + check(`SELECT looted_resources='{}'::jsonb FROM battles_ranked WHERE player_tag='#P0L'`) + cfg.Env = map[string]string{ + "RANKED_DEFENSE_LOOT_CLEANUP_APPLY": "true", + "RANKED_BATTLELOG_WRITERS_PAUSED": "true", + "RANKED_DEFENSE_LOOT_CLEANUP_BATCH_SIZE": "1", + "RANKED_DEFENSE_LOOT_CLEANUP_DATABASE": "clashking_test", + } + if err := migrateutil.ClearRankedDefenseLoot(ctx, cfg); err != nil { + t.Fatal("defense loot apply:", err) + } + check(`SELECT looted_resources IS NULL FROM battles_ranked WHERE player_tag='#P0L'`) + check(`SELECT looted_resources='{}'::jsonb FROM battles_ranked WHERE player_tag='#P0Y'`) + + if err := migrate("up-to", "15"); err != nil { + t.Fatal(err) + } + check(`SELECT max(version_id)=15 FROM goose_db_version WHERE is_applied`) + check(`SELECT pg_get_constraintdef(oid)='PRIMARY KEY (family_id)' + FROM pg_constraint WHERE conrelid='public.army_families'::regclass AND contype='p'`) + check(`SELECT pg_get_constraintdef(oid)='PRIMARY KEY (share_code)' + FROM pg_constraint WHERE conrelid='public.army_family_members'::regclass AND contype='p'`) + check(`SELECT family_id IS NOT NULL AND name='Root Riders' + AND hero_ids='{1,2}'::integer[] AND equipment_ids='{3,4}'::integer[] + FROM army_families WHERE representative_share_code='u1x0'`) + check(`SELECT member.share_code='u2x0' AND member.family_id=family.family_id + AND member.troop_similarity=0.9000 AND member.spell_similarity=0.8000 + AND member.equipment_similarity=0.7500 + FROM army_family_members member + JOIN army_families family ON family.representative_share_code='u1x0'`) + check(`SELECT attack_count=2 AND duration_seconds_sum=200 + FROM army_family_daily_stats WHERE day='2026-08-01'`) + check(`SELECT attack_count=2 AND duration_seconds_sum=200 + FROM legend_daily_stats WHERE day='2026-08-01' AND league_tier_id=1 AND town_hall=18`) + check(`SELECT NOT EXISTS(SELECT 1 FROM army_family_daily_stats_v2) + AND NOT EXISTS(SELECT 1 FROM legend_daily_stats_v2)`) + check(`SELECT army_hash=decode(repeat('bb',32),'hex') + FROM battles_ranked WHERE player_tag='#P0Y'`) + if err := migrate("down"); err != nil { + t.Fatal("empty replacement rollback:", err) + } + check(`SELECT max(version_id)=14 FROM goose_db_version WHERE is_applied`) + check(`SELECT to_regclass('public.legend_daily_stats_v2') IS NULL + AND to_regclass('public.army_family_daily_stats_v2') IS NULL`) + check(`SELECT family_name=' Root Riders ' + FROM army_families WHERE anchor_army_hash=decode(repeat('aa',32),'hex')`) + if err := migrate("down"); err == nil { + t.Fatal("nullable-loot rollback discarded cleaned defense state") + } + check(`SELECT max(version_id)=14 FROM goose_db_version WHERE is_applied`) + // Remove the deliberately cleaned defense fixture only inside this disposable + // test so migration 014 can prove its structural rollback path. + run(`DELETE FROM battles_ranked WHERE player_tag='#P0L'`) + if err := migrate("down"); err != nil { + t.Fatal("nullable-loot rollback after fixture cleanup:", err) + } + check(`SELECT max(version_id)=13 FROM goose_db_version WHERE is_applied`) + if err := migrate("up-to", "15"); err != nil { + t.Fatal("repeat compatibility upgrade:", err) + } + + var familyID int64 + if err := conn.QueryRow(ctx, `INSERT INTO army_families( + representative_share_code,name,hero_ids,equipment_ids + ) VALUES ('u3x0',NULL,'{1,2}','{3,4}') RETURNING family_id`).Scan(&familyID); err != nil { + t.Fatal(err) + } + run(`INSERT INTO army_family_members( + share_code,family_id,troop_similarity,spell_similarity,equipment_similarity + ) VALUES ('u3x0',$1,1.0000,1.0000,1.0000)`, familyID) + check(`SELECT anchor_army_hash IS NULL AND family_name IS NULL AND source IS NULL AND name IS NULL + FROM army_families WHERE family_id=$1`, familyID) + check(`SELECT army_hash IS NULL AND anchor_army_hash IS NULL + AND matching_version IS NULL AND equipment_difference_count IS NULL + FROM army_family_members WHERE share_code='u3x0'`) + check(`SELECT NOT EXISTS ( + SELECT 1 FROM army_compositions WHERE normalized_share_code='u3x0' + )`) + + // An old hash writer still receives the new identity columns. + run(`INSERT INTO army_compositions( + army_hash,normalized_share_code,heroes,equipment + ) VALUES ( + decode(repeat('cc',32),'hex'),'u4x0','{5}', + '[{"equipmentId":6,"heroId":5}]' + ); + INSERT INTO army_families( + anchor_army_hash,representative_share_code,family_name,source + ) VALUES ( + decode(repeat('cc',32),'hex'),'u4x0','Legacy Family','fallback' + ); + INSERT INTO army_family_members( + army_hash,anchor_army_hash,troop_housing_similarity,spell_capacity_similarity, + heroes_exact,equipment_similarity,equipment_difference_count,matching_version + ) VALUES ( + decode(repeat('cc',32),'hex'),decode(repeat('cc',32),'hex'), + 1.0000,1.0000,true,1.0000,0,'v1' + );`) + check(`SELECT name='Legacy Family' AND hero_ids='{5}'::integer[] + AND equipment_ids='{6}'::integer[] AND family_id IS NOT NULL + FROM army_families WHERE representative_share_code='u4x0'`) + check(`SELECT share_code='u4x0' AND family_id IS NOT NULL + AND troop_similarity=1 AND spell_similarity=1 + FROM army_family_members WHERE army_hash=decode(repeat('cc',32),'hex')`) + + run(`INSERT INTO battles_ranked( + player_tag,opponent_tag,battle_time,direction,battle_mode, + player_town_hall,opponent_town_hall,stars,destruction_percentage,share_code + ) VALUES ( + '#P0Y','#P0L',now(),'attack','legend',18,18,3,100,'u3x0' + ) ON CONFLICT(player_tag,battle_time) DO NOTHING`) + check(`SELECT army_hash IS NULL AND share_code='u3x0' + FROM battles_ranked WHERE player_tag='#P0Y' ORDER BY battle_time DESC LIMIT 1`) + + run(`UPDATE army_families SET name=NULL + WHERE representative_share_code='u1x0'`) + check(`SELECT name IS NULL AND family_name=' Root Riders ' + FROM army_families WHERE representative_share_code='u1x0'`) + + run(`INSERT INTO army_family_daily_stats_v2( + family_id,day,attack_count,distinct_player_count, + zero_star_count,one_star_count,two_star_count,three_star_count, + destruction_percentage_sum,duration_seconds_sum,duration_count + ) VALUES ($1,'2026-09-09',2,2,0,0,1,1,180,120,1)`, familyID) + run(`INSERT INTO legend_daily_stats_v2( + day,attack_count,distinct_player_count,perfect_320_player_count, + zero_star_count,one_star_count,two_star_count,three_star_count, + destruction_percentage_sum,duration_seconds_sum,duration_count, + hero_stats,pet_stats,equipment_stats,pet_hero_assignments + ) VALUES ( + '2026-09-09',2,2,0,0,0,1,1,180,120,1, + '[{"id":1,"uses":2,"triples":1}]','[]','[]','[]' + )`) + + reject(`INSERT INTO army_family_members( + share_code,family_id,troop_similarity,spell_similarity,equipment_similarity + ) VALUES('bad',$1,0.8599,1,1)`, familyID) + reject(`UPDATE army_families SET representative_share_code='changed' + WHERE family_id=$1`, familyID) + reject(`DELETE FROM army_family_members WHERE share_code='u3x0'`) + reject(`INSERT INTO legend_daily_stats_v2( + day,attack_count,distinct_player_count,perfect_320_player_count, + zero_star_count,one_star_count,two_star_count,three_star_count, + destruction_percentage_sum,duration_seconds_sum,duration_count,hero_stats + ) VALUES( + '2026-09-10',1,1,0,0,0,0,1,100,0,0, + '[{"id":1,"uses":2,"triples":1}]' + )`) + + if err := migrate("down"); err == nil { + t.Fatal("rollback discarded v2-only data") + } + check(`SELECT max(version_id)=15 FROM goose_db_version WHERE is_applied`) +} diff --git a/database/schema/battle_player_time_test.go b/database/schema/battle_player_time_test.go index 401934a..ac5cfa6 100644 --- a/database/schema/battle_player_time_test.go +++ b/database/schema/battle_player_time_test.go @@ -22,7 +22,7 @@ func TestBattlePlayerTimeIdentity(t *testing.T) { if definition != "PRIMARY KEY (player_tag, battle_time)" { t.Fatal(definition) } - insert := `INSERT INTO battles_ranked(player_tag,opponent_tag,battle_time,direction,battle_mode,player_town_hall,opponent_town_hall,stars,destruction_percentage,army_hash) VALUES($1,$2,'2026-09-09T00:00:00Z',$3,'legend',18,17,3,100,decode(repeat('ab',32),'hex')) ON CONFLICT(player_tag,battle_time) DO NOTHING` + insert := `INSERT INTO battles_ranked(player_tag,opponent_tag,battle_time,direction,battle_mode,player_town_hall,opponent_town_hall,stars,destruction_percentage,looted_resources,army_hash) VALUES($1,$2,'2026-09-09T00:00:00Z',$3,'legend',18,17,3,100,CASE WHEN $3='defense' THEN NULL ELSE '{}'::jsonb END,decode(repeat('ab',32),'hex')) ON CONFLICT(player_tag,battle_time) DO NOTHING` for _, v := range [][3]string{{"#P0", "#Y2", "attack"}, {"#Y2", "#P0", "defense"}} { tag, err := tx.Exec(ctx, insert, v[0], v[1], v[2]) if err != nil || tag.RowsAffected() != 1 { @@ -51,7 +51,7 @@ func TestBattleIdentityMigrationRejectsCollisionsWithoutDataLoss(t *testing.T) { if _, err = tx.Exec(ctx, parts[1]); err != nil { t.Fatal(err) } - if _, err = tx.Exec(ctx, `INSERT INTO battles_ranked(player_tag,opponent_tag,battle_time,direction,battle_mode,player_town_hall,opponent_town_hall,stars,destruction_percentage,army_hash) VALUES('#P0','#Y2','2026-09-09T00:00:00Z','attack','legend',18,17,3,100,decode(repeat('ab',32),'hex')),('#P0','#G9','2026-09-09T00:00:00Z','defense','legend',18,17,3,100,decode(repeat('ab',32),'hex'))`); err != nil { + if _, err = tx.Exec(ctx, `INSERT INTO battles_ranked(player_tag,opponent_tag,battle_time,direction,battle_mode,player_town_hall,opponent_town_hall,stars,destruction_percentage,looted_resources,army_hash) VALUES('#P0','#Y2','2026-09-09T00:00:00Z','attack','legend',18,17,3,100,'{}',decode(repeat('ab',32),'hex')),('#P0','#G9','2026-09-09T00:00:00Z','defense','legend',18,17,3,100,NULL,decode(repeat('ab',32),'hex'))`); err != nil { t.Fatal(err) } if _, err = tx.Exec(ctx, "SAVEPOINT upgrade"); err != nil { diff --git a/database/schema/ranked_battle_history_test.go b/database/schema/ranked_battle_history_test.go index 6233009..2f44eff 100644 --- a/database/schema/ranked_battle_history_test.go +++ b/database/schema/ranked_battle_history_test.go @@ -82,7 +82,7 @@ func TestRankedPerspectivesCountOnePhysicalAttackOnce(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = tx.Exec(ctx, `INSERT INTO battles_ranked(player_tag,opponent_tag,battle_time,direction,battle_mode,player_town_hall,opponent_town_hall,stars,destruction_percentage,duration_seconds,looted_resources,share_code,army_hash) VALUES ('#2PP','#9G2YV','2026-09-08T12:00:00Z','attack','ranked',17,17,3,100,120,'{"gold":1}','u1x1',$1),('#9G2YV','#2PP','2026-09-08T12:00:00Z','defense','ranked',17,17,3,100,120,'{"gold":1}','u1x1',$1)`, hash) + _, err = tx.Exec(ctx, `INSERT INTO battles_ranked(player_tag,opponent_tag,battle_time,direction,battle_mode,player_town_hall,opponent_town_hall,stars,destruction_percentage,duration_seconds,looted_resources,share_code,army_hash) VALUES ('#2PP','#9G2YV','2026-09-08T12:00:00Z','attack','ranked',17,17,3,100,120,'{"gold":1}','u1x1',$1),('#9G2YV','#2PP','2026-09-08T12:00:00Z','defense','ranked',17,17,3,100,120,NULL,'u1x1',$1)`, hash) if err != nil { t.Fatal(err) } diff --git a/database/timescale/014_ranked_defense_loot_nullable.sql b/database/timescale/014_ranked_defense_loot_nullable.sql new file mode 100644 index 0000000..29b0847 --- /dev/null +++ b/database/timescale/014_ranked_defense_loot_nullable.sql @@ -0,0 +1,23 @@ +-- +goose Up +-- Expand only the storage nullability needed by the bounded defense-loot +-- cleanup. Existing attacks and defenses remain unchanged by this migration. +SET LOCAL lock_timeout = '5s'; +ALTER TABLE public.battles_ranked + ALTER COLUMN looted_resources DROP NOT NULL; + +-- +goose Down +SET LOCAL lock_timeout = '5s'; +-- +goose StatementBegin +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 FROM public.battles_ranked + WHERE looted_resources IS NULL + ) THEN + RAISE EXCEPTION 'migration 014 rollback refused: ranked rows contain null loot'; + END IF; +END +$$; +-- +goose StatementEnd +ALTER TABLE public.battles_ranked + ALTER COLUMN looted_resources SET NOT NULL; diff --git a/database/timescale/015_army_code_family_compatibility.sql b/database/timescale/015_army_code_family_compatibility.sql new file mode 100644 index 0000000..6770508 --- /dev/null +++ b/database/timescale/015_army_code_family_compatibility.sql @@ -0,0 +1,500 @@ +-- +goose Up +-- Expand the army-family schema for normalized share-code identity while old +-- hash readers and writers are retired. The old aggregate tables retain their +-- original semantics; *_v2 tables use the shifted Legend-day contract. +SET LOCAL lock_timeout = '5s'; +SET LOCAL statement_timeout = '5min'; + +-- +goose StatementBegin +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 FROM public.battles_ranked + WHERE direction = 'defense' AND looted_resources IS NOT NULL + ) THEN + RAISE EXCEPTION 'migration 015 requires clear_ranked_defense_loot to finish first'; + END IF; +END +$$; +-- +goose StatementEnd + +ALTER TABLE public.battles_ranked + ALTER COLUMN army_hash DROP NOT NULL, + DROP CONSTRAINT battles_ranked_loot_check, + ADD CONSTRAINT battles_ranked_loot_check CHECK ( + (direction = 'attack' AND looted_resources IS NOT NULL + AND public.battle_looted_resources_valid(looted_resources)) + OR (direction = 'defense' AND looted_resources IS NULL) + ); +DROP INDEX public.idx_battles_ranked_attacks_time; +CREATE INDEX idx_battles_ranked_attacks_time + ON public.battles_ranked (battle_mode, battle_time DESC) + WHERE direction = 'attack'; + +-- +goose StatementBegin +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM public.army_families family + LEFT JOIN public.army_compositions composition + ON (composition.army_hash, composition.normalized_share_code) + = (family.anchor_army_hash, family.representative_share_code) + WHERE composition.army_hash IS NULL + ) THEN + RAISE EXCEPTION 'army family conversion requires every representative composition'; + END IF; + IF EXISTS ( + SELECT 1 + FROM public.army_families + WHERE char_length(regexp_replace(btrim(family_name), '[[:space:]]+', ' ', 'g')) > 120 + ) THEN + RAISE EXCEPTION 'army family conversion found a name longer than 120 characters'; + END IF; + IF EXISTS ( + SELECT 1 + FROM public.army_families + GROUP BY lower(regexp_replace(btrim(family_name), '[[:space:]]+', ' ', 'g')) + HAVING count(*) > 1 + ) THEN + RAISE EXCEPTION 'army family conversion found names that collide after normalization'; + END IF; +END +$$; +-- +goose StatementEnd + +ALTER TABLE public.army_families + ADD COLUMN family_id bigint GENERATED ALWAYS AS IDENTITY, + ADD COLUMN name text, + ADD COLUMN hero_ids integer[] NOT NULL DEFAULT '{}', + ADD COLUMN equipment_ids integer[] NOT NULL DEFAULT '{}'; + +UPDATE public.army_families family +SET name = regexp_replace(btrim(family.family_name), '[[:space:]]+', ' ', 'g'), + hero_ids = derived.hero_ids, + equipment_ids = derived.equipment_ids +FROM ( + SELECT composition.army_hash, + ARRAY( + SELECT DISTINCT hero_id + FROM unnest(composition.heroes) AS hero(hero_id) + ORDER BY hero_id + ) AS hero_ids, + ARRAY( + SELECT DISTINCT (equipment.value->>'equipmentId')::integer + FROM jsonb_array_elements(composition.equipment) equipment(value) + ORDER BY (equipment.value->>'equipmentId')::integer + ) AS equipment_ids + FROM public.army_compositions composition +) derived +WHERE family.anchor_army_hash = derived.army_hash; + +ALTER TABLE public.army_families + ALTER COLUMN family_name DROP NOT NULL, + ALTER COLUMN source DROP NOT NULL, + ADD CONSTRAINT army_families_representative_share_code_key UNIQUE (representative_share_code), + ADD CONSTRAINT army_families_name_normalized_check CHECK ( + name IS NULL OR ( + name <> '' + AND char_length(name) <= 120 + AND name = regexp_replace(btrim(name), '[[:space:]]+', ' ', 'g') + ) + ), + ADD CONSTRAINT army_families_hero_ids_check CHECK (public.army_hero_ids_valid(hero_ids)), + ADD CONSTRAINT army_families_equipment_ids_check CHECK (public.army_hero_ids_valid(equipment_ids)); +CREATE UNIQUE INDEX army_families_name_v2_unique + ON public.army_families (lower(name)) + WHERE name IS NOT NULL; + +-- Old hash-based inserts are translated forward. New code-based inserts may +-- leave every legacy identity/provenance column null; no hash or composition is +-- synthesized. +ALTER TABLE public.army_family_members + DROP CONSTRAINT army_family_members_anchor_army_hash_fkey; +ALTER TABLE public.army_family_daily_stats + DROP CONSTRAINT army_family_daily_stats_anchor_army_hash_fkey; +ALTER TABLE public.army_families + DROP CONSTRAINT army_families_pkey, + ADD CONSTRAINT army_families_anchor_army_hash_key UNIQUE (anchor_army_hash), + ADD CONSTRAINT army_families_pkey PRIMARY KEY (family_id), + ALTER COLUMN anchor_army_hash DROP NOT NULL; + +ALTER TABLE public.army_family_members + ADD CONSTRAINT army_family_members_anchor_army_hash_fkey + FOREIGN KEY (anchor_army_hash) REFERENCES public.army_families(anchor_army_hash); +ALTER TABLE public.army_family_daily_stats + ADD CONSTRAINT army_family_daily_stats_anchor_army_hash_fkey + FOREIGN KEY (anchor_army_hash) REFERENCES public.army_families(anchor_army_hash); + +-- +goose StatementBegin +CREATE FUNCTION public.prepare_compatible_army_family() +RETURNS trigger LANGUAGE plpgsql AS $$ +DECLARE + composition public.army_compositions%ROWTYPE; +BEGIN + IF NEW.representative_share_code IS NULL OR btrim(NEW.representative_share_code) = '' THEN + RAISE EXCEPTION 'representative share code is required' USING ERRCODE = '23514'; + END IF; + IF NEW.name IS NOT NULL THEN + NEW.name := regexp_replace(btrim(NEW.name), '[[:space:]]+', ' ', 'g'); + IF NEW.name = '' THEN NEW.name := NULL; END IF; + ELSIF NEW.family_name IS NOT NULL THEN + NEW.name := regexp_replace(btrim(NEW.family_name), '[[:space:]]+', ' ', 'g'); + END IF; + + IF NEW.anchor_army_hash IS NOT NULL THEN + SELECT * INTO composition + FROM public.army_compositions + WHERE army_hash = NEW.anchor_army_hash; + IF composition.normalized_share_code IS DISTINCT FROM NEW.representative_share_code THEN + RAISE EXCEPTION 'legacy army hash resolves to a different share code' USING ERRCODE = '23505'; + END IF; + IF cardinality(NEW.hero_ids) = 0 THEN + NEW.hero_ids := composition.heroes; + END IF; + IF cardinality(NEW.equipment_ids) = 0 THEN + NEW.equipment_ids := ARRAY( + SELECT DISTINCT (equipment.value->>'equipmentId')::integer + FROM jsonb_array_elements(composition.equipment) equipment(value) + ORDER BY (equipment.value->>'equipmentId')::integer + ); + END IF; + END IF; + RETURN NEW; +END +$$; +-- +goose StatementEnd + +CREATE TRIGGER army_families_compatibility_insert + BEFORE INSERT ON public.army_families + FOR EACH ROW EXECUTE FUNCTION public.prepare_compatible_army_family(); + +-- +goose StatementBegin +CREATE OR REPLACE FUNCTION public.protect_army_family_anchor() +RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF TG_OP = 'DELETE' THEN + RAISE EXCEPTION 'army family identities cannot be deleted' USING ERRCODE = 'integrity_constraint_violation'; + END IF; + IF NEW.anchor_army_hash IS DISTINCT FROM OLD.anchor_army_hash + OR NEW.family_id IS DISTINCT FROM OLD.family_id + OR NEW.representative_share_code IS DISTINCT FROM OLD.representative_share_code + OR NEW.hero_ids IS DISTINCT FROM OLD.hero_ids + OR NEW.equipment_ids IS DISTINCT FROM OLD.equipment_ids THEN + RAISE EXCEPTION 'army family identities and representatives cannot be changed' + USING ERRCODE = 'integrity_constraint_violation'; + END IF; + NEW.updated_at := clock_timestamp(); + RETURN NEW; +END +$$; +-- +goose StatementEnd + +ALTER TABLE public.army_family_members + ADD COLUMN share_code text, + ADD COLUMN family_id bigint, + ADD COLUMN troop_similarity numeric(5,4), + ADD COLUMN spell_similarity numeric(5,4); + +ALTER TABLE public.army_family_members DISABLE TRIGGER army_family_members_immutable; +UPDATE public.army_family_members member +SET share_code = composition.normalized_share_code, + family_id = family.family_id, + troop_similarity = member.troop_housing_similarity, + spell_similarity = member.spell_capacity_similarity +FROM public.army_compositions composition, + public.army_families family +WHERE composition.army_hash = member.army_hash + AND family.anchor_army_hash = member.anchor_army_hash; +ALTER TABLE public.army_family_members ENABLE TRIGGER army_family_members_immutable; + +-- +goose StatementBegin +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 FROM public.army_family_members + WHERE share_code IS NULL OR family_id IS NULL + OR troop_similarity IS NULL OR spell_similarity IS NULL + ) THEN + RAISE EXCEPTION 'army family member conversion could not resolve a code or family'; + END IF; +END +$$; +-- +goose StatementEnd + +ALTER TABLE public.army_family_members + DROP CONSTRAINT army_family_members_pkey, + ADD CONSTRAINT army_family_members_army_hash_key UNIQUE (army_hash), + ADD CONSTRAINT army_family_members_pkey PRIMARY KEY (share_code), + ALTER COLUMN army_hash DROP NOT NULL, + ALTER COLUMN anchor_army_hash DROP NOT NULL, + ALTER COLUMN troop_housing_similarity DROP NOT NULL, + ALTER COLUMN spell_capacity_similarity DROP NOT NULL, + ALTER COLUMN heroes_exact DROP NOT NULL, + ALTER COLUMN equipment_difference_count DROP NOT NULL, + ALTER COLUMN matching_version DROP NOT NULL, + ALTER COLUMN family_id SET NOT NULL, + ALTER COLUMN troop_similarity SET NOT NULL, + ALTER COLUMN spell_similarity SET NOT NULL, + ADD CONSTRAINT army_family_members_family_id_fkey + FOREIGN KEY (family_id) REFERENCES public.army_families(family_id), + ADD CONSTRAINT army_family_members_code_check CHECK (btrim(share_code) <> ''), + ADD CONSTRAINT army_family_members_v2_similarity_check CHECK ( + troop_similarity BETWEEN 0.8600 AND 1 + AND spell_similarity BETWEEN 0.8000 AND 1 + AND equipment_similarity BETWEEN 0.7500 AND 1 + ); +CREATE INDEX army_family_members_family_code_idx + ON public.army_family_members (family_id, share_code); + +-- +goose StatementBegin +CREATE FUNCTION public.prepare_compatible_army_family_member() +RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF NEW.share_code IS NULL AND NEW.army_hash IS NOT NULL THEN + SELECT normalized_share_code INTO NEW.share_code + FROM public.army_compositions + WHERE army_hash = NEW.army_hash; + END IF; + IF NEW.family_id IS NULL AND NEW.anchor_army_hash IS NOT NULL THEN + SELECT family_id INTO NEW.family_id + FROM public.army_families + WHERE anchor_army_hash = NEW.anchor_army_hash; + END IF; + NEW.troop_similarity := COALESCE(NEW.troop_similarity, NEW.troop_housing_similarity); + NEW.spell_similarity := COALESCE(NEW.spell_similarity, NEW.spell_capacity_similarity); + RETURN NEW; +END +$$; +-- +goose StatementEnd + +CREATE TRIGGER army_family_members_compatibility_insert + BEFORE INSERT ON public.army_family_members + FOR EACH ROW EXECUTE FUNCTION public.prepare_compatible_army_family_member(); + +-- +goose StatementBegin +CREATE FUNCTION public.item_usage_triples_within_attack_count(value jsonb, attack_limit bigint) +RETURNS boolean LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE AS $$ +DECLARE entry jsonb; +BEGIN + IF attack_limit < 0 OR NOT public.item_usage_triples_valid(value) THEN RETURN false; END IF; + FOR entry IN SELECT element FROM jsonb_array_elements(value) item(element) LOOP + IF (entry->>'uses')::bigint > attack_limit THEN RETURN false; END IF; + END LOOP; + RETURN true; +END +$$; + +CREATE FUNCTION public.pet_hero_usage_triples_within_attack_count(value jsonb, attack_limit bigint) +RETURNS boolean LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE AS $$ +DECLARE entry jsonb; +BEGIN + IF attack_limit < 0 OR NOT public.pet_hero_usage_triples_valid(value) THEN RETURN false; END IF; + FOR entry IN SELECT element FROM jsonb_array_elements(value) item(element) LOOP + IF (entry->>'uses')::bigint > attack_limit THEN RETURN false; END IF; + END LOOP; + RETURN true; +END +$$; +-- +goose StatementEnd + +CREATE TABLE public.army_family_daily_stats_v2 ( + family_id bigint NOT NULL REFERENCES public.army_families(family_id), + day date NOT NULL, + attack_count bigint NOT NULL, + distinct_player_count bigint NOT NULL, + zero_star_count bigint NOT NULL, + one_star_count bigint NOT NULL, + two_star_count bigint NOT NULL, + three_star_count bigint NOT NULL, + destruction_percentage_sum bigint NOT NULL, + duration_seconds_sum bigint NOT NULL, + duration_count bigint NOT NULL, + PRIMARY KEY (family_id, day), + CONSTRAINT army_family_daily_stats_v2_counts_check CHECK ( + attack_count >= 0 AND distinct_player_count >= 0 + AND zero_star_count >= 0 AND one_star_count >= 0 + AND two_star_count >= 0 AND three_star_count >= 0 + AND attack_count = zero_star_count + one_star_count + two_star_count + three_star_count + ), + CONSTRAINT army_family_daily_stats_v2_sums_check CHECK ( + destruction_percentage_sum BETWEEN 0 AND attack_count * 100 + AND duration_seconds_sum >= 0 + AND duration_count BETWEEN 0 AND attack_count + ) +); +CREATE INDEX army_family_daily_stats_v2_day_idx + ON public.army_family_daily_stats_v2 (day, family_id); + +CREATE TABLE public.legend_daily_stats_v2 ( + day date PRIMARY KEY, + attack_count bigint NOT NULL, + distinct_player_count bigint NOT NULL, + perfect_320_player_count bigint NOT NULL, + zero_star_count bigint NOT NULL, + one_star_count bigint NOT NULL, + two_star_count bigint NOT NULL, + three_star_count bigint NOT NULL, + destruction_percentage_sum bigint NOT NULL, + duration_seconds_sum bigint NOT NULL, + duration_count bigint NOT NULL, + hero_stats jsonb NOT NULL DEFAULT '[]', + pet_stats jsonb NOT NULL DEFAULT '[]', + equipment_stats jsonb NOT NULL DEFAULT '[]', + pet_hero_assignments jsonb NOT NULL DEFAULT '[]', + CONSTRAINT legend_daily_stats_v2_counts_check CHECK ( + attack_count >= 0 AND distinct_player_count >= 0 + AND perfect_320_player_count BETWEEN 0 AND distinct_player_count + AND zero_star_count >= 0 AND one_star_count >= 0 + AND two_star_count >= 0 AND three_star_count >= 0 + AND attack_count = zero_star_count + one_star_count + two_star_count + three_star_count + ), + CONSTRAINT legend_daily_stats_v2_sums_check CHECK ( + destruction_percentage_sum BETWEEN 0 AND attack_count * 100 + AND duration_seconds_sum >= 0 + AND duration_count BETWEEN 0 AND attack_count + ), + CONSTRAINT legend_daily_stats_v2_heroes_check CHECK ( + public.item_usage_triples_within_attack_count(hero_stats, attack_count) + ), + CONSTRAINT legend_daily_stats_v2_pets_check CHECK ( + public.item_usage_triples_within_attack_count(pet_stats, attack_count) + ), + CONSTRAINT legend_daily_stats_v2_equipment_check CHECK ( + public.item_usage_triples_within_attack_count(equipment_stats, attack_count) + ), + CONSTRAINT legend_daily_stats_v2_pet_hero_check CHECK ( + public.pet_hero_usage_triples_within_attack_count(pet_hero_assignments, attack_count) + ) +); + +COMMENT ON TABLE public.army_family_daily_stats_v2 IS + 'Replacement family totals for half-open 05:10 UTC Legend days; populated from Legend attack rows only.'; +COMMENT ON TABLE public.legend_daily_stats_v2 IS + 'Replacement global totals for half-open 05:10 UTC Legend days; populated from Legend attack rows only.'; +COMMENT ON COLUMN public.army_families.family_id IS + 'Stable numeric family identity; serialize as a decimal string in JSON.'; +COMMENT ON COLUMN public.army_family_members.share_code IS + 'Canonical normalized exact-army identity.'; + +-- +goose Down +SET LOCAL lock_timeout = '5s'; +SET LOCAL statement_timeout = '5min'; + +-- +goose StatementBegin +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM public.army_family_daily_stats_v2) + OR EXISTS (SELECT 1 FROM public.legend_daily_stats_v2) THEN + RAISE EXCEPTION 'migration 015 rollback refused: replacement daily aggregates contain data'; + END IF; + IF EXISTS ( + SELECT 1 FROM public.army_families + WHERE family_name IS NULL OR source IS NULL + OR name IS DISTINCT FROM regexp_replace(btrim(family_name), '[[:space:]]+', ' ', 'g') + ) THEN + RAISE EXCEPTION 'migration 015 rollback refused: family rows require v2 fields'; + END IF; + IF EXISTS ( + SELECT 1 FROM public.army_family_members + WHERE army_hash IS NULL OR anchor_army_hash IS NULL + OR troop_housing_similarity IS NULL OR spell_capacity_similarity IS NULL + OR heroes_exact IS NULL OR equipment_difference_count IS NULL + OR matching_version IS NULL + ) THEN + RAISE EXCEPTION 'migration 015 rollback refused: member rows require v2 fields'; + END IF; + IF EXISTS ( + SELECT 1 FROM public.battles_ranked + WHERE army_hash IS NULL + ) THEN + RAISE EXCEPTION 'migration 015 rollback refused: raw battles without legacy hashes exist'; + END IF; +END +$$; +-- +goose StatementEnd + +DROP TABLE public.legend_daily_stats_v2; +DROP TABLE public.army_family_daily_stats_v2; +DROP FUNCTION public.pet_hero_usage_triples_within_attack_count(jsonb, bigint); +DROP FUNCTION public.item_usage_triples_within_attack_count(jsonb, bigint); + +DROP TRIGGER army_family_members_compatibility_insert ON public.army_family_members; +DROP FUNCTION public.prepare_compatible_army_family_member(); +DROP INDEX public.army_family_members_family_code_idx; +ALTER TABLE public.army_family_members + DROP CONSTRAINT army_family_members_v2_similarity_check, + DROP CONSTRAINT army_family_members_code_check, + DROP CONSTRAINT army_family_members_family_id_fkey, + DROP CONSTRAINT army_family_members_pkey, + DROP CONSTRAINT army_family_members_army_hash_key, + ADD CONSTRAINT army_family_members_pkey PRIMARY KEY (army_hash), + ALTER COLUMN matching_version SET NOT NULL, + ALTER COLUMN equipment_difference_count SET NOT NULL, + ALTER COLUMN heroes_exact SET NOT NULL, + ALTER COLUMN spell_capacity_similarity SET NOT NULL, + ALTER COLUMN troop_housing_similarity SET NOT NULL, + ALTER COLUMN anchor_army_hash SET NOT NULL, + ALTER COLUMN army_hash SET NOT NULL, + DROP COLUMN spell_similarity, + DROP COLUMN troop_similarity, + DROP COLUMN family_id, + DROP COLUMN share_code; + +DROP TRIGGER army_families_compatibility_insert ON public.army_families; +DROP FUNCTION public.prepare_compatible_army_family(); + +ALTER TABLE public.army_family_members + DROP CONSTRAINT army_family_members_anchor_army_hash_fkey; +ALTER TABLE public.army_family_daily_stats + DROP CONSTRAINT army_family_daily_stats_anchor_army_hash_fkey; + +DROP INDEX public.army_families_name_v2_unique; +ALTER TABLE public.army_families + DROP CONSTRAINT army_families_equipment_ids_check, + DROP CONSTRAINT army_families_hero_ids_check, + DROP CONSTRAINT army_families_name_normalized_check, + DROP CONSTRAINT army_families_representative_share_code_key, + DROP CONSTRAINT army_families_pkey, + DROP CONSTRAINT army_families_anchor_army_hash_key, + ADD CONSTRAINT army_families_pkey PRIMARY KEY (anchor_army_hash), + DROP COLUMN equipment_ids, + DROP COLUMN hero_ids, + DROP COLUMN name, + DROP COLUMN family_id, + ALTER COLUMN anchor_army_hash SET NOT NULL, + ALTER COLUMN source SET NOT NULL, + ALTER COLUMN family_name SET NOT NULL; + +ALTER TABLE public.army_family_members + ADD CONSTRAINT army_family_members_anchor_army_hash_fkey + FOREIGN KEY (anchor_army_hash) REFERENCES public.army_families(anchor_army_hash); +ALTER TABLE public.army_family_daily_stats + ADD CONSTRAINT army_family_daily_stats_anchor_army_hash_fkey + FOREIGN KEY (anchor_army_hash) REFERENCES public.army_families(anchor_army_hash); + +-- +goose StatementBegin +CREATE OR REPLACE FUNCTION public.protect_army_family_anchor() +RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF TG_OP = 'DELETE' THEN + RAISE EXCEPTION 'army family anchors cannot be deleted' USING ERRCODE = 'integrity_constraint_violation'; + END IF; + IF NEW.anchor_army_hash <> OLD.anchor_army_hash + OR NEW.representative_share_code <> OLD.representative_share_code THEN + RAISE EXCEPTION 'army family anchors cannot be changed' USING ERRCODE = 'integrity_constraint_violation'; + END IF; + NEW.updated_at := clock_timestamp(); + RETURN NEW; +END +$$; +-- +goose StatementEnd +DROP INDEX public.idx_battles_ranked_attacks_time; +CREATE INDEX idx_battles_ranked_attacks_time + ON public.battles_ranked (battle_mode, battle_time DESC, player_town_hall, opponent_town_hall, army_hash) + WHERE direction = 'attack'; +ALTER TABLE public.battles_ranked + DROP CONSTRAINT battles_ranked_loot_check, + ALTER COLUMN army_hash SET NOT NULL, + ADD CONSTRAINT battles_ranked_loot_check + CHECK (public.battle_looted_resources_valid(looted_resources)); diff --git a/database/timescale/README.md b/database/timescale/README.md index a5967ae..2d3461b 100644 --- a/database/timescale/README.md +++ b/database/timescale/README.md @@ -181,11 +181,15 @@ ahead of newer home posts without hiding those newer posts. ## Worker/API upgrade and production operation `007_worker_api.sql` consolidates the Worker/API upgrade, -`008_ranked_battle_history.sql` adds raw battle and exact-army storage, -`009_league_army_analytics.sql` adds league, Legend, and army-family rollups, and -`010_cwl_season_statistics.sql` adds rerunnable CWL population summaries. +`008_ranked_battle_history.sql` adds retained raw battles, +`009_league_army_analytics.sql` adds the original league/army rollups, +`010_cwl_season_statistics.sql` adds rerunnable CWL population summaries, +`013_battle_player_time_identity.sql` fixes raw identity at player/time, +`014_ranked_defense_loot_nullable.sql` opens the bounded defense-loot cleanup, +and `015_army_code_family_compatibility.sql` adds code/family-ID identity plus +parallel shifted-day aggregates without deleting the old history. Do not use the former 007–028 fixture numbering. See [the schema decisions](../../docs/worker-api-schema.md), [the disposable upgrade test](../../RETAINED_API_FIXTURE.md). -Migration 013 replaces the Ranked/Legend primary key without deleting data. Existing collisions cause a transactional failure. For an explicitly authorized rebuild with the battle-log writer stopped, `scripts/reset-battle-history.sql` clears battles and compositions only when dependent families and battle aggregates are empty; it never cascades. Reset only the `bl:#*` Valkey checkpoints before restarting the corrected writer. +Migration 013 replaces the Ranked/Legend primary key without deleting data. Existing collisions cause a transactional failure. Migration 014 changes no rows; with the old writer paused, run the bounded defense-only cleanup companion before migration 015. Keep the writer paused until a binary that stores defense loot as SQL NULL is deployed. Migration 015 refuses incomplete cleanup, retains the player/time primary key, makes legacy hashes nullable, and introduces `army_family_daily_stats_v2` and `legend_daily_stats_v2` for the 05:10 UTC shifted-day meaning. See `docs/ranked-battle-history.md` for the complete schemas and rollback limits. diff --git a/docs/ranked-battle-history.md b/docs/ranked-battle-history.md index 51385fe..c2545a2 100644 --- a/docs/ranked-battle-history.md +++ b/docs/ranked-battle-history.md @@ -1,155 +1,165 @@ -# Battle, league, and army-family storage +# Battle history and share-code army families -Migrations `008_ranked_battle_history.sql` and `009_league_army_analytics.sql` add the final battle pipeline. The legacy `battlelogs`, `townhall_stats_daily`, and `legend_history` relations remain during cutover. Migration 010 remains the independent CWL statistics migration. +Migrations 008–013 establish retained raw battle history and the requested-player identity `(player_tag,battle_time)`. Migration 014 opens the nullable defense-loot transition. After the bounded cleanup, migration 015 adds share-code family identity and parallel daily tables for the new Legend-day meaning. Applied migrations 008, 009, 012, and 013 are never rewritten. -## Write flow +## Rollout order + +1. Apply Goose through 014 only. This changes no rows. +2. Pause the old battlelog writer and keep it paused through the schema/binary cutover. +3. Run `go run ./migrations/clear_ranked_defense_loot.go` from `database/` first as a dry run, then with `RANKED_DEFENSE_LOOT_CLEANUP_APPLY=true`, `RANKED_BATTLELOG_WRITERS_PAUSED=true`, and `RANKED_DEFENSE_LOOT_CLEANUP_DATABASE=`. The companion requires Goose version 14, refuses compressed chunks, and commits bounded `(player_tag,battle_time)` batches. +4. Apply Goose through 015. It refuses to apply if any defense still has non-null loot. +5. Deploy compatible code-only battle ingestion and the new family closeout before resuming the writer. The old closeout must remain paused because its hash tables and old day semantics are no longer authoritative. +6. Populate and compare the `*_v2` aggregates from retained raw data before switching readers. + +Migration 015 can roll back only while no new code-only rows, cleared defense rows, name edits, or v2 aggregates would be lost. Migration 014 cannot roll back while any ranked row contains null loot. Production execution and consumer deployment remain separate approvals. + +## Data flow ```mermaid flowchart LR - A[Tracking fetches a battle log] --> B{battle kind} - B -->|farming| C[battles_farming: one attack row] - B -->|Ranked or Legend| D[normalize exact share code] - D --> E[army_compositions: immutable exact hash] - E --> F[battles_ranked: attacker perspective] - E --> G[battles_ranked: defender perspective] - F --> H[aggregate transaction] - H --> I[league_hitrate_stats] - H --> J[legend_daily_stats] - H --> K[army_family_daily_stats] - G -. never counted by aggregate SQL .-> H - E --> L[family matcher] - L --> M[army_family_members] - M --> K + C[Clash battlelog response] --> W[Tracking normalizes the share code] + W -->|farming attack| F[battles_farming] + W -->|requested Ranked or Legend perspective| R[battles_ranked] + R -->|Legend attack rows in 05:10 window| D[Daily closeout] + M[army_family_members: code to family] --> D + A[army_families: fixed representative] --> D + D --> N[New permanent assignments] + D --> FS[army_family_daily_stats_v2] + D --> LS[legend_daily_stats_v2] ``` -The two Ranked rows make player history a direct `player_tag + battle_time` lookup. The attacker row is `#2PP -> #9G2YV, direction=attack`; the defender row is `#9G2YV -> #2PP, direction=defense`. Both describe one physical attack, so every aggregate insert or reconciliation query must include `WHERE direction = 'attack'`. +Tracking stores only the requested player's perspective. It does not create the opposite side. All family and Legend statistics select `battle_mode='legend' AND direction='attack'`; a stored defense never contributes. -## Raw and identity tables +## Raw history ### `battles_farming` -One home-village attack per player and timestamp. It is a Timescale hypertable with 30-day chunks, compression after 30 days segmented by player and ordered by time, and one-year retention. - | Column | Type | Null/default and rule | |---|---|---| -| `player_tag` | `text` | not null; canonical tag; PK | -| `battle_time` | `timestamptz` | not null; PK | -| `stars` | `smallint` | not null; 0–3 | -| `destruction_percentage` | `smallint` | not null; 0–100 | +| `player_tag` | `text` | required; primary key with `battle_time` | +| `battle_time` | `timestamptz` | required | +| `stars` | `smallint` | required; 0–3 | +| `destruction_percentage` | `smallint` | required; 0–100 | | `duration_seconds` | `integer` | nullable; nonnegative | -| `looted_resources` | `jsonb` | not null, default `{}`; optional nonnegative `gold`, `elixir`, `darkElixir` integer keys only | +| `looted_resources` | `jsonb` | required; default `{}`; nonnegative `gold`, `elixir`, and `darkElixir` only | | `share_code` | `text` | nullable; nonblank when present | -### `battles_ranked` +New attack writes combine base and extra Gold, Elixir, and Dark Elixir into the same loot object. Sour Elixir is ignored. Existing attacks are not backfilled. The hypertable uses 30-day chunks, compression after 30 days, and one-year retention. -Two player-perspective rows per physical Ranked or Legend attack. It is a Timescale hypertable with seven-day chunks, compression after 30 days segmented by `(player_tag,direction)` and ordered by time, and one-year retention. +### `battles_ranked` after migration 015 | Column | Type | Null/default and rule | |---|---|---| -| `player_tag` | `text` | not null; canonical perspective owner; PK | -| `opponent_tag` | `text` | not null; canonical opponent; PK | -| `battle_time` | `timestamptz` | not null; PK | -| `direction` | `text` | not null; `attack` or `defense`; PK | -| `battle_mode` | `text` | not null; `ranked` or `legend`; PK | -| `player_town_hall` | `smallint` | not null; 1–20 | -| `opponent_town_hall` | `smallint` | not null; 1–20 | -| `stars` | `smallint` | not null; 0–3 | -| `destruction_percentage` | `smallint` | not null; 0–100 | +| `player_tag` | `text` | required; primary key with `battle_time` | +| `battle_time` | `timestamptz` | required | +| `opponent_tag` | `text` | required; different from player | +| `direction` | `text` | required; `attack` or `defense` | +| `battle_mode` | `text` | required; `ranked` or `legend` | +| `player_town_hall` | `smallint` | required; 1–20 | +| `opponent_town_hall` | `smallint` | required; 1–20 | +| `stars` | `smallint` | required; 0–3 | +| `destruction_percentage` | `smallint` | required; 0–100 | | `duration_seconds` | `integer` | nullable; nonnegative | -| `looted_resources` | `jsonb` | not null, default `{}`; same shape as farming | -| `share_code` | `text` | nullable; normalized code used by the attack | -| `army_hash` | `bytea` | not null; 32-byte FK to `army_compositions` | +| `looted_resources` | `jsonb` | required object for attacks; SQL NULL for defenses | +| `share_code` | `text` | nullable canonical code; nonblank when present | +| `army_hash` | `bytea` | nullable legacy compatibility column; omitted by the new writer | -After migration 013, the primary key is `(player_tag,battle_time)`. Each response contributes only the requested player's attack or defense; polling the opponent never synthesizes a second row for that player. Mode, direction, and opponent describe the observation rather than changing its identity. `idx_battles_ranked_player_time` serves all of a player's attacks and defenses, `idx_battles_ranked_player_mode_time` serves exact mode history, `idx_battles_ranked_player_direction_time` serves direction-filtered history, and the partial `idx_battles_ranked_attacks_time` serves mode-specific aggregate scans while physically excluding defense rows. +The primary key remains exactly `(player_tag,battle_time)`. No migration cleans the known minute-apart source duplicates because requested-player provenance cannot be reconstructed from SQL. Tracking reconciles those against live Clash battle logs. -### `army_compositions` - -One immutable exact army definition. `army_hash` is SHA-256 of the normalized share code according to `contracts/army-hash-v2.json`; it is separate from family similarity. - -| Column | Type | Null/default and rule | -|---|---|---| -| `army_hash` | `bytea` | not null; 32-byte PK | -| `normalized_share_code` | `text` | not null; unique and nonblank | -| `main_troops` | `jsonb` | not null, default `[]`; sorted unique `{id,quantity}` rows | -| `clan_castle_troops` | `jsonb` | not null, default `[]`; sorted unique `{id,quantity}` rows | -| `spells` | `jsonb` | not null, default `[]`; sorted unique `{id,quantity,clanCastle}` rows | -| `heroes` | `integer[]` | not null, default `{}`; sorted unique hero IDs | -| `equipment` | `jsonb` | not null, default `[]`; sorted unique `{equipmentId,heroId}` rows | -| `pet_assignments` | `jsonb` | not null, default `[]`; sorted unique `{petId,heroId}` rows | -| `siege_machine_id` | `integer` | nullable; nonnegative | -| `created_at` | `timestamptz` | not null, default `now()` | +The seven-day hypertable retains one year and compresses after 30 days. Existing player/time, player/mode/time, and player/direction/time indexes remain. The bounded attack aggregate index becomes `(battle_mode,battle_time DESC) WHERE direction='attack'`; full share codes are not indexed on every battle. -## League snapshots and aggregate tables +Detailed Ranked and Legend endpoints omit loot. The general player battle-history endpoint combines farming and ranked-table attacks, including Legend attacks, and may return the stored attack loot. -### `ranked_league_group_members` +## Permanent family identity -This existing table remains the only Ranked group relation; there is no `ranked_league_groups` table. Its primary key remains `(season_id,group_tag,player_tag)`, and `UNIQUE (season_id,player_tag)` lets ingestion replace a stale group tag rather than creating a second membership for the same season. During upgrade, existing duplicates are reduced deterministically because the old schema has no observation timestamp: the row with the greatest combined attack/defense result count wins, followed by trophies, tier, placement, and group tag as stable tie-breakers. Counters from different group snapshots are not combined. +### `army_families` after migration 015 | Column | Type | Null/default and rule | |---|---|---| -| `season_id` | `bigint` | not null; PK | -| `group_tag` | `text` | not null; PK | -| `league_tier_id` | `integer` | not null; positive | -| `player_tag` | `text` | not null; PK | -| `player_name` | `text` | not null | -| `town_hall` | `smallint` | nullable; 1–20 | -| `placement` | `integer` | not null; positive | -| `league_trophies` | `integer` | not null; nonnegative | -| `maximum_battle_count` | `smallint` | not null, default 0 | -| `attack_win_count` | `integer` | not null; source counter | -| `attack_loss_count` | `integer` | not null; source counter | -| `attack_star_count` | `integer` | not null, default 0; source counter | -| `defense_win_count` | `integer` | not null; source counter | -| `defense_loss_count` | `integer` | not null; source counter | -| `defense_star_count` | `integer` | not null, default 0; source counter | - -The old clan snapshot columns are removed because the group-member response does not need a second clan identity copy. No lifecycle flags, inferred promotions, missing markers, or state column are stored. - -### `league_hitrate_stats` - -Permanent hit-rate totals. Primary key: `(period_kind,period_start,league_tier_id,town_hall)`. +| `family_id` | `bigint identity` | generated primary key; serialize as a decimal string in JSON | +| `representative_share_code` | `text` | required; unique; immutable | +| `name` | `text` | nullable manual name; normalized whitespace; maximum 120; unique case-insensitively | +| `hero_ids` | `integer[]` | required; default `{}`; sorted and duplicate-free; immutable | +| `equipment_ids` | `integer[]` | required; default `{}`; sorted and duplicate-free; immutable | +| `anchor_army_hash` | `bytea` | nullable legacy compatibility key; unique while retained | +| `family_name` | `text` | nullable legacy name | +| `source` | `text` | nullable legacy naming source | +| `named_by_subject` | `text` | nullable legacy provenance | +| `naming_model` | `text` | nullable legacy provenance | +| `naming_prompt_version` | `text` | nullable legacy provenance | +| `created_at` | `timestamptz` | required; default `now()` | +| `updated_at` | `timestamptz` | required; default `now()` | + +Existing families keep their representative, assignment anchor, and normalized existing name. Hero/equipment filters are derived once from the retained representative composition. New families write only the code, optional name, and representative ID arrays; they do not calculate a hash or create an `army_compositions` row. + +The insert compatibility trigger translates only old rows forward: when an old writer supplies `anchor_army_hash`, it copies the old name and derives representative ID arrays from the existing composition. It never synthesizes a hash. Updating `name` to NULL stays NULL because the trigger runs only on insert. + +### `army_family_members` after migration 015 -| Column | Type | Rule | +| Column | Type | Null/default and rule | |---|---|---| -| `period_kind` | `text` | `ranked_season` or `legend_day` | -| `period_start` | `timestamptz` | exact period boundary | -| `league_tier_id` | `integer` | positive | -| `town_hall` | `smallint` | 1–20 | -| `attack_count` | `bigint` | equals the four star buckets | -| `zero_star_count`…`three_star_count` | `bigint` | nonnegative | -| `refreshed_at` | `timestamptz` | default `now()` | +| `share_code` | `text` | primary key; canonical exact-army identity | +| `family_id` | `bigint` | required FK to `army_families` | +| `troop_similarity` | `numeric(5,4)` | required; 0.8600–1 inclusive | +| `spell_similarity` | `numeric(5,4)` | required; 0.8000–1 inclusive | +| `equipment_similarity` | `numeric(5,4)` | required; 0.7500–1 inclusive | +| `army_hash` | `bytea` | nullable legacy identity; unique while retained | +| `anchor_army_hash` | `bytea` | nullable legacy family link | +| `troop_housing_similarity` | `numeric(5,4)` | nullable legacy score | +| `spell_capacity_similarity` | `numeric(5,4)` | nullable legacy score | +| `heroes_exact` | `boolean` | nullable legacy field | +| `equipment_difference_count` | `smallint` | nullable legacy field | +| `matching_version` | `text` | nullable legacy field | +| `assigned_at` | `timestamptz` | retained legacy timestamp; default `now()` | -### `ranked_league_tier_stats` +Assignments and representatives cannot be updated or deleted. New code-only inserts have no hash, matching-version, assignment-version, or equipment-difference requirement. An old hash-based insert is translated forward by looking up its existing composition code and family ID; the trigger never mirrors a new code row back into hash storage. -Permanent per-season/tier population and trophy distribution. Primary key: `(season_id,league_tier_id)`. +The application matcher requires exact hero IDs, housing-weighted main troops at least 0.86, capacity-weighted main plus clan-castle spells at least 0.80, and equipment overlap at least 0.75. Siege machines, clan-castle troops, and pets do not affect matching. -| Columns | Types and rules | -|---|---| -| `season_id`, `league_tier_id` | `bigint`, `integer`; positive | -| `group_count`, `distinct_player_count`, `participating_player_count` | `bigint`; nonnegative; participating cannot exceed distinct | -| `trophy_p10`, `trophy_p25`, `trophy_p50`, `trophy_p75`, `trophy_p90` | nullable `integer`; ordered when present | -| `town_halls` | `jsonb`; descending `[{"level":17,"count":123}]` | -| `average_group_first_last_trophy_range`, `average_first_second_trophy_gap` | nullable nonnegative `numeric` | -| `refreshed_at` | `timestamptz`, default `now()` | +## Replacement daily tables -### `legend_daily_stats` +Old `army_family_daily_stats` and `legend_daily_stats` remain unchanged during compatibility because their prior day/tier/Town Hall meaning cannot be relabeled truthfully. -Permanent per-day/tier/TH totals. Primary key: `(day,league_tier_id,town_hall)`. It stores attack/player/perfect-320 counts, four star buckets, destruction and duration sums, plus `hero_stats`, `pet_stats`, and `equipment_stats` arrays of `{id,uses,triples}`. `pet_hero_assignments` is sorted `{petId,heroId,uses,triples}` data using numeric IDs. +### `army_family_daily_stats_v2` -These four aggregate relations are normal PostgreSQL tables. They have no Timescale compression or retention policy. +| Column | Type | Rule | +|---|---|---| +| `family_id` | `bigint` | FK; primary key with day | +| `day` | `date` | shifted day; primary key with family | +| `attack_count` | `bigint` | nonnegative | +| `distinct_player_count` | `bigint` | nonnegative | +| `zero_star_count` … `three_star_count` | `bigint` | nonnegative; sum to attacks | +| `destruction_percentage_sum` | `bigint` | 0 through 100 × attacks | +| `duration_seconds_sum` | `bigint` | nonnegative | +| `duration_count` | `bigint` | 0 through attack count | -## Army families +Primary key: `(family_id,day)`. Index: `(day,family_id)`. Tracking stores every observed family, including families outside a daily top list. The table is permanent PostgreSQL storage without compression. -Exact hashes remain immutable. The matcher assigns an exact hash to a family only when main troops are at least 86% similar by housing space, all spells including clan-castle spells are at least 80% similar by spell capacity, hero IDs match exactly, and equipment is at least 75% similar with at most two differences. Clan-castle troops, pets, and siege machines do not affect family matching; pet usage remains available in `legend_daily_stats`. +### `legend_daily_stats_v2` + +| Column | Type | Rule | +|---|---|---| +| `day` | `date` | primary key | +| `attack_count` | `bigint` | nonnegative | +| `distinct_player_count` | `bigint` | nonnegative | +| `perfect_320_player_count` | `bigint` | 0 through player count | +| `zero_star_count` … `three_star_count` | `bigint` | nonnegative; sum to attacks | +| `destruction_percentage_sum` | `bigint` | 0 through 100 × attacks | +| `duration_seconds_sum` | `bigint` | nonnegative | +| `duration_count` | `bigint` | 0 through attack count | +| `hero_stats` | `jsonb` | required; default `[]`; sorted `{id,uses,triples}` | +| `pet_stats` | `jsonb` | same item shape | +| `equipment_stats` | `jsonb` | same item shape | +| `pet_hero_assignments` | `jsonb` | sorted `{petId,heroId,uses,triples}` | -### `army_families` +Each item satisfies `0 <= triples <= uses <= attack_count`. Missing codes remain in global attack, star, destruction, duration, and player totals but cannot contribute decoded item or family data. -The primary key is the exact `anchor_army_hash`, and `(anchor_army_hash,representative_share_code)` must resolve to one `army_compositions` row. `family_name` is case-insensitively unique. `source` is `ai` with model/prompt provenance, `admin` with the naming subject, or `fallback` with null naming provenance when inference fails or collides. The anchor and representative code cannot change; metadata can be renamed and `updated_at` is touched by the schema. +## Time and coverage contract -### `army_family_members` +Day D owns the half-open window `[D 05:10:00 UTC, D+1 05:10:00 UTC)`. The proposed job starts at 05:12 UTC. Adjacent days neither overlap nor leave a gap. -One immutable assignment per exact `army_hash`. It stores `anchor_army_hash`, troop/spell/equipment similarity values, `heroes_exact`, `equipment_difference_count`, `matching_version`, and `assigned_at`. Constraints enforce the approved 86%/80%/exact/75%/two-difference minimums. +Only stored Legend attack rows enter both v2 tables. Defenses, lower Ranked tiers, and synthetic opposite perspectives are excluded. Exact multi-day unique-player counts are supported only when the requested range is fully inside retained raw coverage and every completed day has a matching v2 global total, including explicit zero days. Otherwise the API reports player counts as unavailable and rejects a player-minimum filter. -### `army_family_daily_stats` +## Deferred destructive cleanup -Permanent daily family totals keyed by `(anchor_army_hash,day)`: attack and distinct-player counts, four star buckets, destruction sum, duration sum, and `refreshed_at`. It is a normal PostgreSQL table with no compression policy. +A separate migration may remove `army_compositions`, hashes, legacy family/member columns, old naming provenance, and old daily tables only after every reader and writer uses the code/family-ID contract and retained history has been rebuilt and compared. It must also swap the `*_v2` names to their final unsuffixed names. Older aggregates without sufficient raw history require an explicit preservation decision; migration 015 does not delete or relabel them. diff --git a/scripts/retained-api-profile.sh b/scripts/retained-api-profile.sh index 8ecc57c..910f990 100755 --- a/scripts/retained-api-profile.sh +++ b/scripts/retained-api-profile.sh @@ -1,5 +1,5 @@ # Canonical migration inventory for retained API tests and local development. -# This is the same contiguous 001-013 sequence used for production upgrades. +# This is the same contiguous 001-015 sequence used for production upgrades. fixture_image='timescale/timescaledb:2.29.2-pg18@sha256:9508616d5b941ed931198504c5db3fb47e8f53f790732ea1e889591f1062057c' fixture_sources=( 001_initial_stats.sql @@ -15,4 +15,6 @@ fixture_sources=( 011_active_verified_players.sql 012_legend_only_army_compositions.sql 013_battle_player_time_identity.sql + 014_ranked_defense_loot_nullable.sql + 015_army_code_family_compatibility.sql ) diff --git a/scripts/with-test-timescale.sh b/scripts/with-test-timescale.sh index c52cffd..262eb23 100755 --- a/scripts/with-test-timescale.sh +++ b/scripts/with-test-timescale.sh @@ -2,8 +2,8 @@ # Run one integration suite against an isolated, Goose-migrated Timescale database. set -euo pipefail -if [[ ${1:-} != --profile || ( ${2:-} != retained-api && ${2:-} != baseline-006 ) ]]; then - echo 'An explicit --profile retained-api or baseline-006 is required.' >&2 +if [[ ${1:-} != --profile || ( ${2:-} != retained-api && ${2:-} != baseline-006 && ${2:-} != baseline-013 ) ]]; then + echo 'An explicit --profile retained-api, baseline-006, or baseline-013 is required.' >&2 exit 2 fi fixture_profile="$2" @@ -18,6 +18,8 @@ fixture_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" source "$fixture_root/scripts/retained-api-profile.sh" if [[ $fixture_profile == baseline-006 ]]; then fixture_sources=("${fixture_sources[@]:0:6}") +elif [[ $fixture_profile == baseline-013 ]]; then + fixture_sources=("${fixture_sources[@]:0:13}") fi fixture_container='' fixture_child='' diff --git a/scripts/with-test-timescale.test.mjs b/scripts/with-test-timescale.test.mjs index 016c377..c855418 100644 --- a/scripts/with-test-timescale.test.mjs +++ b/scripts/with-test-timescale.test.mjs @@ -77,6 +77,8 @@ test('uses only retained authoritative migrations and its own disposable contain '011_active_verified_players.sql', '012_legend_only_army_compositions.sql', '013_battle_player_time_identity.sql', + '014_ranked_defense_loot_nullable.sql', + '015_army_code_family_compatibility.sql', ]); } assert.equal(migrations[1].args.at(-1), 'up'); @@ -84,6 +86,14 @@ test('uses only retained authoritative migrations and its own disposable contain assert.deepEqual(result.calls.at(-1).args, ['rm', '--force', '--volumes', container]); }); +test('supports a migration-013 compatibility baseline', () => { + const result = runFixture({}, 'test "\$CLASHKING_TIMESCALE_PROFILE" = baseline-013', ['--profile', 'baseline-013']); + assert.equal(result.status, 0, result.stderr); + const migration = result.calls.find(call => call.tool === 'goose' && call.args.includes('up')); + assert.equal(migration.migrations.at(-1), '013_battle_player_time_identity.sql'); + assert.equal(migration.migrations.length, 13); +}); + test('preserves a failing child exit code and still cleans up', () => { const result = runFixture({}, 'exit 42'); assert.equal(result.status, 42);