From b97658ac539599875e5227a492d56fe457f75e3d Mon Sep 17 00:00:00 2001 From: Luke Policinski Date: Sun, 26 Jul 2026 20:00:55 -0400 Subject: [PATCH 1/3] chore: update how to report players / servers --- hasura/metadata/actions.graphql | 2 + hasura/triggers/match_maps.sql | 10 ++ src/matches/jobs/CancelExpiredMatches.ts | 20 +++ src/matches/jobs/FinalizeStrandedMaps.spec.ts | 128 +++++++++++++++ src/matches/jobs/FinalizeStrandedMaps.ts | 155 ++++++++++++++++++ src/matches/matches.module.ts | 12 ++ src/telemetry/telemetry.service.ts | 47 ++++-- src/telemetry/types/TelemetryPayload.ts | 13 +- test/match-scoring.spec.ts | 28 ++++ test/telemetry.spec.ts | 77 ++++++++- 10 files changed, 469 insertions(+), 23 deletions(-) create mode 100644 src/matches/jobs/FinalizeStrandedMaps.spec.ts create mode 100644 src/matches/jobs/FinalizeStrandedMaps.ts diff --git a/hasura/metadata/actions.graphql b/hasura/metadata/actions.graphql index a9df445b..6460f481 100644 --- a/hasura/metadata/actions.graphql +++ b/hasura/metadata/actions.graphql @@ -1193,7 +1193,9 @@ type TelemetryFleetTotals { matchesImported: Int! matchesImportedMonth: Int! mapsPlayed: Int! + playersKnown: Int! playersRegistered: Int! + playersPlayed: Int! playersActive30d: Int! teams: Int! } diff --git a/hasura/triggers/match_maps.sql b/hasura/triggers/match_maps.sql index c8145db9..5b07e878 100644 --- a/hasura/triggers/match_maps.sql +++ b/hasura/triggers/match_maps.sql @@ -73,6 +73,16 @@ BEGIN END IF; END IF; + -- The auto-cancel deadline reaps matches stuck *in play*. Once the map has + -- left Live the game server owns a multi-minute end-of-map handshake + -- (WaitingForTV for tv_delay, then UploadingDemo, then Finished); letting + -- the deadline fire inside that window cancels the match and kills the + -- server pod before it can report the result. + IF NEW.status IN ('WaitingForTV', 'UploadingDemo', 'Finished', 'Surrendered') + AND OLD.status IS DISTINCT FROM NEW.status THEN + UPDATE matches SET cancels_at = NULL WHERE id = NEW.match_id; + END IF; + IF NEW.status = 'Finished' AND OLD.status IS DISTINCT FROM NEW.status THEN NEW.ended_at = NOW(); END IF; diff --git a/src/matches/jobs/CancelExpiredMatches.ts b/src/matches/jobs/CancelExpiredMatches.ts index 7e893620..a55bd819 100644 --- a/src/matches/jobs/CancelExpiredMatches.ts +++ b/src/matches/jobs/CancelExpiredMatches.ts @@ -7,6 +7,7 @@ import { HasuraService } from "../../hasura/hasura.service"; import { NotificationsService } from "../../notifications/notifications.service"; import { AppConfig } from "../../configs/types/AppConfig"; import { DISCORD_COLORS } from "../../notifications/utilities/constants"; +import { e_match_map_status_enum } from "../../../generated"; @UseQueue("Matches", MatchQueues.ScheduledMatches) export class CancelExpiredMatches extends WorkerHost { @@ -21,6 +22,23 @@ export class CancelExpiredMatches extends WorkerHost { super(); this.appConfig = this.configService.get("app"); } + // A map that has left Live is inside the game server's end-of-map handshake + // (WaitingForTV -> UploadingDemo -> Finished), which runs for tv_delay plus + // the demo upload. Reaping the match there kills the server pod before it can + // report the result, so the map strands and the series never resolves. + // FinalizeStrandedMaps is the backstop if the server dies anyway. + private static notFinalizingMap() { + return { + _not: { + match_maps: { + status: { + _in: ["WaitingForTV", "UploadingDemo"] as e_match_map_status_enum[], + }, + }, + }, + }; + } + async process(): Promise { const { update_matches } = await this.hasura.mutation({ update_matches: { @@ -47,6 +65,7 @@ export class CancelExpiredMatches extends WorkerHost { _lte: new Date(), }, }, + CancelExpiredMatches.notFinalizingMap(), ], }, _set: { @@ -194,6 +213,7 @@ export class CancelExpiredMatches extends WorkerHost { _lte: new Date(), }, }, + CancelExpiredMatches.notFinalizingMap(), ], }, }, diff --git a/src/matches/jobs/FinalizeStrandedMaps.spec.ts b/src/matches/jobs/FinalizeStrandedMaps.spec.ts new file mode 100644 index 00000000..3b332ae3 --- /dev/null +++ b/src/matches/jobs/FinalizeStrandedMaps.spec.ts @@ -0,0 +1,128 @@ +import { FinalizeStrandedMaps } from "./FinalizeStrandedMaps"; + +const secondsAgo = (seconds: number) => + new Date(Date.now() - seconds * 1000).toISOString(); + +const strandedMap = (overrides: Record = {}) => ({ + id: "map-1", + status: "WaitingForTV", + match_id: "match-1", + winning_lineup_id: "lineup-2", + lineup_1_score: 10, + lineup_2_score: 13, + match: { + lineup_1_id: "lineup-1", + lineup_2_id: "lineup-2", + options: { + tv_delay: 115, + }, + }, + rounds: [{ time: secondsAgo(1000) }], + ...overrides, +}); + +describe("FinalizeStrandedMaps", () => { + const logger = { + warn: jest.fn(), + }; + const hasura = { + query: jest.fn(), + mutation: jest.fn(), + }; + + let job: FinalizeStrandedMaps; + let matchMaps: any[]; + + beforeEach(() => { + jest.clearAllMocks(); + matchMaps = []; + hasura.query.mockImplementation(async () => ({ match_maps: matchMaps })); + hasura.mutation.mockResolvedValue({}); + job = new FinalizeStrandedMaps(logger as any, hasura as any); + }); + + it("finishes a map stranded past the handshake deadline with the reported winner", async () => { + matchMaps = [strandedMap()]; + + await expect(job.process()).resolves.toBe(1); + + expect(hasura.mutation).toHaveBeenCalledWith( + expect.objectContaining({ + update_match_maps_by_pk: expect.objectContaining({ + __args: { + pk_columns: { id: "map-1" }, + _set: { + status: "Finished", + winning_lineup_id: "lineup-2", + }, + }, + }), + }), + ); + }); + + it("leaves a map alone while the server is still inside its tv_delay window", async () => { + matchMaps = [strandedMap({ rounds: [{ time: secondsAgo(60) }] })]; + + await expect(job.process()).resolves.toBe(0); + + expect(hasura.mutation).not.toHaveBeenCalled(); + }); + + it("derives the winner from the score when the server never reported one", async () => { + matchMaps = [ + strandedMap({ + winning_lineup_id: null, + lineup_1_score: 13, + lineup_2_score: 4, + }), + ]; + + await expect(job.process()).resolves.toBe(1); + + expect(hasura.mutation).toHaveBeenCalledWith( + expect.objectContaining({ + update_match_maps_by_pk: expect.objectContaining({ + __args: expect.objectContaining({ + _set: { + status: "Finished", + winning_lineup_id: "lineup-1", + }, + }), + }), + }), + ); + }); + + it("finishes a tied map without a winner rather than leaving it stranded", async () => { + matchMaps = [ + strandedMap({ + winning_lineup_id: null, + lineup_1_score: 12, + lineup_2_score: 12, + }), + ]; + + await expect(job.process()).resolves.toBe(1); + + expect(hasura.mutation).toHaveBeenCalledWith( + expect.objectContaining({ + update_match_maps_by_pk: expect.objectContaining({ + __args: expect.objectContaining({ + _set: { + status: "Finished", + }, + }), + }), + }), + ); + }); + + it("skips maps with no rounds, which have no end-of-game anchor", async () => { + matchMaps = [strandedMap({ rounds: [] })]; + + await expect(job.process()).resolves.toBe(0); + + expect(hasura.mutation).not.toHaveBeenCalled(); + }); +}); diff --git a/src/matches/jobs/FinalizeStrandedMaps.ts b/src/matches/jobs/FinalizeStrandedMaps.ts new file mode 100644 index 00000000..0b1347c5 --- /dev/null +++ b/src/matches/jobs/FinalizeStrandedMaps.ts @@ -0,0 +1,155 @@ +import { Logger } from "@nestjs/common"; +import { WorkerHost } from "@nestjs/bullmq"; +import { MatchQueues } from "../enums/MatchQueues"; +import { UseQueue } from "../../utilities/QueueProcessors"; +import { HasuraService } from "../../hasura/hasura.service"; +import { e_match_map_status_enum } from "../../../generated"; + +@UseQueue("Matches", MatchQueues.ScheduledMatches) +export class FinalizeStrandedMaps extends WorkerHost { + // The game server is the only thing that writes Finished, and it does so at + // the very end of a handshake it runs entirely in-process: WaitingForTV for + // tv_delay, then stop demo -> UploadingDemo, then upload -> Finished. If its + // pod dies anywhere in that window (auto-cancel, crash, eviction, node drain) + // the map strands, update_match_state never runs, the series never resolves + // and the server is never released. Past this deadline we finish the map from + // the result the server already published when the game ended. + private static readonly PLUGIN_HANDSHAKE_SECONDS = 20; + private static readonly GRACE_SECONDS = 300; + + constructor( + private readonly logger: Logger, + private readonly hasura: HasuraService, + ) { + super(); + } + + async process(): Promise { + const { match_maps } = await this.hasura.query({ + match_maps: { + __args: { + where: { + status: { + _in: [ + "WaitingForTV", + "UploadingDemo", + ] as e_match_map_status_enum[], + }, + match: { + status: { + _nin: [ + "Finished", + "Canceled", + "Forfeit", + "Tie", + "Surrendered", + ], + }, + }, + }, + }, + id: true, + status: true, + match_id: true, + winning_lineup_id: true, + lineup_1_score: true, + lineup_2_score: true, + match: { + lineup_1_id: true, + lineup_2_id: true, + options: { + tv_delay: true, + }, + }, + rounds: { + __args: { + order_by: [{ time: "desc" }], + limit: 1, + }, + time: true, + }, + }, + }); + + let finalized = 0; + + for (const matchMap of match_maps) { + const lastRoundAt = matchMap.rounds?.at(0)?.time; + + if (!lastRoundAt) { + continue; + } + + const deadline = + new Date(lastRoundAt).getTime() + + ((matchMap.match?.options?.tv_delay ?? 0) + + FinalizeStrandedMaps.PLUGIN_HANDSHAKE_SECONDS + + FinalizeStrandedMaps.GRACE_SECONDS) * + 1000; + + if (Date.now() < deadline) { + continue; + } + + const winningLineupId = this.resolveWinningLineupId(matchMap); + + this.logger.warn( + `[${matchMap.match_id}] map ${matchMap.id} stranded in ${matchMap.status} ` + + `since ${lastRoundAt} — finalizing without the server ` + + `(score ${matchMap.lineup_1_score ?? 0}-${matchMap.lineup_2_score ?? 0}, ` + + `winner ${winningLineupId ?? ""})`, + ); + + await this.hasura.mutation({ + update_match_maps_by_pk: { + __args: { + pk_columns: { + id: matchMap.id, + }, + _set: { + status: "Finished", + ...(winningLineupId + ? { winning_lineup_id: winningLineupId } + : {}), + }, + }, + __typename: true, + }, + }); + + finalized++; + } + + return finalized; + } + + // The server publishes the winner alongside the WaitingForTV status, so the + // stored value is authoritative when present; scores are the fallback for + // maps that stranded before that landed. + private resolveWinningLineupId(matchMap: { + winning_lineup_id?: string | null; + lineup_1_score?: number | null; + lineup_2_score?: number | null; + match?: { + lineup_1_id?: string | null; + lineup_2_id?: string | null; + } | null; + }) { + if (matchMap.winning_lineup_id) { + return matchMap.winning_lineup_id; + } + + const lineup1Score = matchMap.lineup_1_score ?? 0; + const lineup2Score = matchMap.lineup_2_score ?? 0; + + if (lineup1Score > lineup2Score) { + return matchMap.match?.lineup_1_id ?? null; + } + + if (lineup2Score > lineup1Score) { + return matchMap.match?.lineup_2_id ?? null; + } + + return null; + } +} diff --git a/src/matches/matches.module.ts b/src/matches/matches.module.ts index 4d47ff73..f09895fa 100644 --- a/src/matches/matches.module.ts +++ b/src/matches/matches.module.ts @@ -42,6 +42,7 @@ import { getQueuesProcessors } from "../utilities/QueueProcessors"; import { CancelInvalidTournaments } from "./jobs/CancelInvalidTournaments"; import { SocketsModule } from "../sockets/sockets.module"; import { CleanAbandonedMatches } from "./jobs/CleanAbandonedMatches"; +import { FinalizeStrandedMaps } from "./jobs/FinalizeStrandedMaps"; import { ReapIdleDemoSessions } from "./jobs/ReapIdleDemoSessions"; import { PollMediaMtxViewers } from "./jobs/PollMediaMtxViewers"; import { MatchMaking } from "src/matchmaking/matchmaking.module"; @@ -167,6 +168,7 @@ import { LeaguesModule } from "../leagues/leagues.module"; StopOnDemandServer, CancelInvalidTournaments, CleanAbandonedMatches, + FinalizeStrandedMaps, ReapIdleDemoSessions, PollMediaMtxViewers, EloCalculation, @@ -291,6 +293,16 @@ export class MatchesModule implements NestModule { }, ); + void scheduleMatchQueue.add( + FinalizeStrandedMaps.name, + {}, + { + repeat: { + pattern: "* * * * *", + }, + }, + ); + void scheduleMatchQueue.add( ReapIdleDemoSessions.name, {}, diff --git a/src/telemetry/telemetry.service.ts b/src/telemetry/telemetry.service.ts index 6a4ada0d..20858be6 100644 --- a/src/telemetry/telemetry.service.ts +++ b/src/telemetry/telemetry.service.ts @@ -109,6 +109,12 @@ export class TelemetryService { return (name: string) => (alias ? `${alias}.${name}` : name); } + // populate_game_servers materializes a `servers` row for every port pair in + // a node's range the moment the node is enabled, so a node with a 216 port + // range adds 108 rows nobody provisioned. Counting raw rows reports a fleet + // of thousands of servers that do not exist. + private static readonly RealServer = `(game_server_node_id IS NULL OR is_dedicated)`; + private static activeLineups(interval: string) { const recent = `${TelemetryService.nativeMatch("m")} AND m.effective_at >= now() - interval '${interval}'`; @@ -187,7 +193,6 @@ export class TelemetryService { total: counts.servers_total, enabled: counts.servers_enabled, dedicated: counts.servers_dedicated, - on_demand: counts.servers_on_demand, public: counts.servers_public, capacity: counts.servers_capacity, }, @@ -211,7 +216,9 @@ export class TelemetryService { }, }, players: { + known: counts.players_known, registered: counts.players_registered, + played: counts.players_played, active_7d: counts.players_active_7d, active_30d: counts.players_active_30d, teams: counts.teams_total, @@ -332,7 +339,9 @@ export class TelemetryService { coalesce(sum((payload->'matches'->'external'->>'total')::numeric), 0) AS "matchesImported", coalesce(sum((payload->'matches'->'external'->>'month')::numeric), 0) AS "matchesImportedMonth", coalesce(sum((payload->'matches'->>'maps_played')::numeric), 0) AS "mapsPlayed", + coalesce(sum((payload->'players'->>'known')::numeric), 0) AS "playersKnown", coalesce(sum((payload->'players'->>'registered')::numeric), 0) AS "playersRegistered", + coalesce(sum((payload->'players'->>'played')::numeric), 0) AS "playersPlayed", coalesce(sum((payload->'players'->>'active_30d')::numeric), 0) AS "playersActive30d", coalesce(sum((payload->'players'->>'teams')::numeric), 0) AS teams FROM public.telemetry_installs @@ -354,7 +363,9 @@ export class TelemetryService { "matchesImported", "matchesImportedMonth", "mapsPlayed", + "playersKnown", "playersRegistered", + "playersPlayed", "playersActive30d", "teams", ]); @@ -543,7 +554,6 @@ export class TelemetryService { total: int(servers.total), enabled: int(servers.enabled), dedicated: int(servers.dedicated), - on_demand: int(servers.on_demand), public: int(servers.public), capacity: int(servers.capacity), }, @@ -567,7 +577,9 @@ export class TelemetryService { }, }, players: { + known: int(players.known), registered: int(players.registered), + played: int(players.played), active_7d: int(players.active_7d), active_30d: int(players.active_30d), teams: int(players.teams), @@ -780,9 +792,9 @@ export class TelemetryService { } private async getSettings(): Promise> { - const rows = await this.collectQuery>( - `SELECT name, value FROM public.settings`, - ); + const rows = await this.collectQuery< + Array<{ name: string; value: string }> + >(`SELECT name, value FROM public.settings`); return new Map(rows.map(({ name, value }) => [name, value])); } @@ -797,7 +809,9 @@ export class TelemetryService { } private async getMatchesByType(): Promise> { - const rows = await this.collectQuery>( + const rows = await this.collectQuery< + Array<{ type: string; count: string }> + >( `SELECT o.type, count(*) AS count FROM public.matches m JOIN public.match_options o ON o.id = m.match_options_id @@ -860,13 +874,15 @@ export class TelemetryService { (SELECT count(*) FROM public.game_server_nodes WHERE gpu AND gpu_streaming_enabled) AS gpu_stream_nodes, - (SELECT count(*) FROM public.servers) AS servers_total, - (SELECT count(*) FROM public.servers WHERE enabled) AS servers_enabled, + (SELECT count(*) FROM public.servers + WHERE ${TelemetryService.RealServer}) AS servers_total, + (SELECT count(*) FROM public.servers + WHERE ${TelemetryService.RealServer} AND enabled) AS servers_enabled, (SELECT count(*) FROM public.servers WHERE is_dedicated) AS servers_dedicated, (SELECT count(*) FROM public.servers - WHERE game_server_node_id IS NOT NULL) AS servers_on_demand, - (SELECT count(*) FROM public.servers WHERE type <> 'Ranked') AS servers_public, - (SELECT coalesce(sum(max_players), 0) FROM public.servers) AS servers_capacity, + WHERE ${TelemetryService.RealServer} AND type <> 'Ranked') AS servers_public, + (SELECT coalesce(sum(max_players), 0) FROM public.servers + WHERE ${TelemetryService.RealServer}) AS servers_capacity, (SELECT count(*) FROM public.matches) AS matches_created, (SELECT count(*) FROM public.matches WHERE ${TelemetryService.nativeMatch()}) AS matches_ran, @@ -906,7 +922,14 @@ export class TelemetryService { (SELECT count(DISTINCT r.match_id) FROM public.team_scrim_requests r WHERE r.match_id IS NOT NULL) AS matches_scrim, - (SELECT count(*) FROM public.players) AS players_registered, + -- Connect events, lineup syncs, demo imports and sanctions all create a + -- players row, so the table is mostly steam ids that never signed in. + -- last_sign_in_at is only ever written by the Steam login callback. + (SELECT count(*) FROM public.players) AS players_known, + (SELECT count(*) FROM public.players WHERE last_sign_in_at IS NOT NULL) AS players_registered, + -- Stats land per map from live round events and from parsed demos, so + -- this is everyone who played rather than everyone who was rostered. + (SELECT count(DISTINCT steam_id) FROM public.player_match_map_stats) AS players_played, (SELECT count(*) FROM public.teams) AS teams_total, -- An OR across both lineup columns cannot use an index and forces a -- join over every lineup row. Feeding the two sides in separately lets diff --git a/src/telemetry/types/TelemetryPayload.ts b/src/telemetry/types/TelemetryPayload.ts index 7e1350f6..4c617d5f 100644 --- a/src/telemetry/types/TelemetryPayload.ts +++ b/src/telemetry/types/TelemetryPayload.ts @@ -1,4 +1,4 @@ -export const TELEMETRY_SCHEMA_VERSION = 1; +export const TELEMETRY_SCHEMA_VERSION = 2; export type TelemetryFeature = { enabled: boolean | null; @@ -18,11 +18,14 @@ export type TelemetryPayload = { regions: number; gpu: number; }; + // A `servers` row is not always a server. Enabling a game server node + // pre-provisions one row per port pair in its range, so a single node adds + // ~100 rows that are slots waiting on a match rather than machines. None of + // these count them. servers: { total: number; enabled: number; dedicated: number; - on_demand: number; public: number; capacity: number; }; @@ -48,8 +51,14 @@ export type TelemetryPayload = { year: number; }; }; + // `known` is every steam id the panel holds a row for, most of which never + // belonged to a person who signed in — connect events, lineup syncs, demo + // imports and sanctions all create players. `registered` is the subset that + // has signed in at least once. players: { + known: number; registered: number; + played: number; active_7d: number; active_30d: number; teams: number; diff --git a/test/match-scoring.spec.ts b/test/match-scoring.spec.ts index 0cfea283..d269a952 100644 --- a/test/match-scoring.spec.ts +++ b/test/match-scoring.spec.ts @@ -339,6 +339,34 @@ describe("match scoring from rounds (SQL-driven)", () => { expect(paused.cancels_at).toBeNull(); }); + it.each(["WaitingForTV", "UploadingDemo"])( + "a map entering %s disarms the live-match timeout", + async (status) => { + const match = await createLiveMatch(1); + + await postgres.query( + "UPDATE match_maps SET status = 'Live' WHERE id = $1", + [match.mapIds[0]], + ); + const [live] = await postgres.query>( + "SELECT cancels_at FROM matches WHERE id = $1", + [match.id], + ); + expect(live.cancels_at).not.toBeNull(); + + // The end-of-map handshake runs for tv_delay plus the demo upload; the + // deadline firing inside it kills the server before it reports Finished. + await postgres.query("UPDATE match_maps SET status = $2 WHERE id = $1", [ + match.mapIds[0], + status, + ]); + const [finalizing] = await postgres.query< + Array<{ cancels_at: Date | null }> + >("SELECT cancels_at FROM matches WHERE id = $1", [match.id]); + expect(finalizing.cancels_at).toBeNull(); + }, + ); + it("finishing the map stamps the map's ended_at", async () => { const match = await createLiveMatch(1); await recordScore(match.mapIds[0], 13, 7); diff --git a/test/telemetry.spec.ts b/test/telemetry.spec.ts index 2a2e2028..5098ec90 100644 --- a/test/telemetry.spec.ts +++ b/test/telemetry.spec.ts @@ -56,6 +56,14 @@ describe("telemetry (SQL-driven)", () => { beforeAll(async () => { await fx.region("TelemetryRegion"); + // Enabling a node pre-provisions a `servers` row per port pair in its + // range, so this seeds five rows that are slots, not servers. + await postgres.query( + `INSERT INTO game_server_nodes + (id, public_ip, start_port_range, end_port_range, region, status, enabled, label) + VALUES ('telemetry-node', '203.0.113.1', 27015, 27025, 'TelemetryRegion', 'Online', true, 'telemetry-node')`, + ); + const ran = await fx.bareMatch(new Date().toISOString()); await postgres.query( "UPDATE matches SET started_at = now() WHERE id = $1", @@ -82,6 +90,23 @@ describe("telemetry (SQL-driven)", () => { [reimported.matchId, "5stack-1"], ); + // Only the first of these ever signed in; the other two are the rows a + // panel creates for a steam id it saw in a lineup or a demo. + const signedIn = await fx.player("signed-in"); + const ghost = await fx.player("ghost"); + await fx.player("never-seen-again"); + + await postgres.query( + "UPDATE players SET last_sign_in_at = now() WHERE steam_id = $1", + [signedIn], + ); + + await postgres.query( + `INSERT INTO player_match_map_stats (steam_id, match_map_id, match_id, kills) + VALUES ($1, $3, $4, 10), ($2, $3, $4, 4)`, + [signedIn, ghost, ran.mapId, ran.matchId], + ); + payload = await service.collect(); }, 600_000); @@ -110,9 +135,28 @@ describe("telemetry (SQL-driven)", () => { expect(payload.matches.by_source.faceit).toBe(1); }); - it("reports the servers seeded by the region fixture", () => { - expect(payload.servers.total).toBeGreaterThanOrEqual(1); - expect(payload.servers.dedicated).toBeGreaterThanOrEqual(1); + it("keeps a node's pre-provisioned port slots out of the server count", async () => { + const [rows] = await postgres.query>( + "SELECT count(*) FROM servers", + ); + + // Six rows: the region fixture's dedicated server plus the node's five + // port slots. Only the dedicated one is a server anybody runs. + expect(Number(rows.count)).toBe(6); + expect(payload.servers.total).toBe(1); + expect(payload.servers.dedicated).toBe(1); + // 32 is the max_players default every slot carries, and counting them + // reported a capacity of thousands on a panel with one real server. + expect(payload.servers.capacity).toBe(32); + }); + + it("counts only players who have signed in as registered", () => { + expect(payload.players.known).toBe(3); + expect(payload.players.registered).toBe(1); + }); + + it("counts players with stats on at least one map as having played", () => { + expect(payload.players.played).toBe(2); }); it("reports every feature with an enabled flag or a usage count", () => { @@ -164,7 +208,7 @@ describe("telemetry (SQL-driven)", () => { const install = "11111111-2222-3333-4444-555555555555"; const report = (over: Record = {}) => ({ - schema: 1, + schema: 2, install_id: install, installed_at: "2024-01-01T00:00:00.000Z", panel_version: "deadbeef", @@ -174,7 +218,6 @@ describe("telemetry (SQL-driven)", () => { total: 10, enabled: 9, dedicated: 3, - on_demand: 7, public: 4, capacity: 120, }, @@ -192,7 +235,14 @@ describe("telemetry (SQL-driven)", () => { scrim: 25, external: { total: 30, week: 1, month: 3, year: 12 }, }, - players: { registered: 300, active_7d: 40, active_30d: 90, teams: 22 }, + players: { + known: 900, + registered: 300, + played: 210, + active_7d: 40, + active_30d: 90, + teams: 22, + }, features: { events: { enabled: true, count: 4 }, news: { enabled: false, count: 0 }, @@ -304,7 +354,7 @@ describe("telemetry (SQL-driven)", () => { const installB = "bbbbbbbb-0000-4000-8000-000000000002"; const report = (installId: string, matches: number, servers: number) => ({ - schema: 1, + schema: 2, install_id: installId, installed_at: "2024-01-01T00:00:00.000Z", panel_version: "cafebabe", @@ -314,7 +364,6 @@ describe("telemetry (SQL-driven)", () => { total: servers, enabled: servers, dedicated: 1, - on_demand: servers - 1, public: 2, capacity: servers * 10, }, @@ -332,7 +381,14 @@ describe("telemetry (SQL-driven)", () => { scrim: 2, external: { total: matches / 10, week: 0, month: 2, year: 5 }, }, - players: { registered: 50, active_7d: 5, active_30d: 12, teams: 3 }, + players: { + known: 200, + registered: 50, + played: 35, + active_7d: 5, + active_30d: 12, + teams: 3, + }, features: { events: { enabled: installId === installA, count: 3 }, highlights: { enabled: null as boolean | null, count: 10 }, @@ -353,6 +409,9 @@ describe("telemetry (SQL-driven)", () => { expect(stats.totals.servers).toBe(10); expect(stats.totals.serverCapacity).toBe(100); expect(stats.totals.mapsPlayed).toBe(700); + expect(stats.totals.playersKnown).toBe(400); + expect(stats.totals.playersRegistered).toBe(100); + expect(stats.totals.playersPlayed).toBe(70); // Imported matches are summed apart from the ones the panels hosted. expect(stats.totals.matchesImported).toBe(35); expect(stats.totals.matchesImportedMonth).toBe(4); From 3f861c2d5a3c0564fc3ef4f5fec6004c047cd87a Mon Sep 17 00:00:00 2001 From: Luke Policinski Date: Sun, 26 Jul 2026 20:40:51 -0400 Subject: [PATCH 2/3] wip --- hasura/triggers/match_maps.sql | 10 -- src/matches/jobs/CancelExpiredMatches.ts | 20 --- src/matches/jobs/FinalizeStrandedMaps.spec.ts | 128 --------------- src/matches/jobs/FinalizeStrandedMaps.ts | 155 ------------------ src/matches/matches.module.ts | 12 -- test/match-scoring.spec.ts | 28 ---- 6 files changed, 353 deletions(-) delete mode 100644 src/matches/jobs/FinalizeStrandedMaps.spec.ts delete mode 100644 src/matches/jobs/FinalizeStrandedMaps.ts diff --git a/hasura/triggers/match_maps.sql b/hasura/triggers/match_maps.sql index 5b07e878..c8145db9 100644 --- a/hasura/triggers/match_maps.sql +++ b/hasura/triggers/match_maps.sql @@ -73,16 +73,6 @@ BEGIN END IF; END IF; - -- The auto-cancel deadline reaps matches stuck *in play*. Once the map has - -- left Live the game server owns a multi-minute end-of-map handshake - -- (WaitingForTV for tv_delay, then UploadingDemo, then Finished); letting - -- the deadline fire inside that window cancels the match and kills the - -- server pod before it can report the result. - IF NEW.status IN ('WaitingForTV', 'UploadingDemo', 'Finished', 'Surrendered') - AND OLD.status IS DISTINCT FROM NEW.status THEN - UPDATE matches SET cancels_at = NULL WHERE id = NEW.match_id; - END IF; - IF NEW.status = 'Finished' AND OLD.status IS DISTINCT FROM NEW.status THEN NEW.ended_at = NOW(); END IF; diff --git a/src/matches/jobs/CancelExpiredMatches.ts b/src/matches/jobs/CancelExpiredMatches.ts index a55bd819..7e893620 100644 --- a/src/matches/jobs/CancelExpiredMatches.ts +++ b/src/matches/jobs/CancelExpiredMatches.ts @@ -7,7 +7,6 @@ import { HasuraService } from "../../hasura/hasura.service"; import { NotificationsService } from "../../notifications/notifications.service"; import { AppConfig } from "../../configs/types/AppConfig"; import { DISCORD_COLORS } from "../../notifications/utilities/constants"; -import { e_match_map_status_enum } from "../../../generated"; @UseQueue("Matches", MatchQueues.ScheduledMatches) export class CancelExpiredMatches extends WorkerHost { @@ -22,23 +21,6 @@ export class CancelExpiredMatches extends WorkerHost { super(); this.appConfig = this.configService.get("app"); } - // A map that has left Live is inside the game server's end-of-map handshake - // (WaitingForTV -> UploadingDemo -> Finished), which runs for tv_delay plus - // the demo upload. Reaping the match there kills the server pod before it can - // report the result, so the map strands and the series never resolves. - // FinalizeStrandedMaps is the backstop if the server dies anyway. - private static notFinalizingMap() { - return { - _not: { - match_maps: { - status: { - _in: ["WaitingForTV", "UploadingDemo"] as e_match_map_status_enum[], - }, - }, - }, - }; - } - async process(): Promise { const { update_matches } = await this.hasura.mutation({ update_matches: { @@ -65,7 +47,6 @@ export class CancelExpiredMatches extends WorkerHost { _lte: new Date(), }, }, - CancelExpiredMatches.notFinalizingMap(), ], }, _set: { @@ -213,7 +194,6 @@ export class CancelExpiredMatches extends WorkerHost { _lte: new Date(), }, }, - CancelExpiredMatches.notFinalizingMap(), ], }, }, diff --git a/src/matches/jobs/FinalizeStrandedMaps.spec.ts b/src/matches/jobs/FinalizeStrandedMaps.spec.ts deleted file mode 100644 index 3b332ae3..00000000 --- a/src/matches/jobs/FinalizeStrandedMaps.spec.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { FinalizeStrandedMaps } from "./FinalizeStrandedMaps"; - -const secondsAgo = (seconds: number) => - new Date(Date.now() - seconds * 1000).toISOString(); - -const strandedMap = (overrides: Record = {}) => ({ - id: "map-1", - status: "WaitingForTV", - match_id: "match-1", - winning_lineup_id: "lineup-2", - lineup_1_score: 10, - lineup_2_score: 13, - match: { - lineup_1_id: "lineup-1", - lineup_2_id: "lineup-2", - options: { - tv_delay: 115, - }, - }, - rounds: [{ time: secondsAgo(1000) }], - ...overrides, -}); - -describe("FinalizeStrandedMaps", () => { - const logger = { - warn: jest.fn(), - }; - const hasura = { - query: jest.fn(), - mutation: jest.fn(), - }; - - let job: FinalizeStrandedMaps; - let matchMaps: any[]; - - beforeEach(() => { - jest.clearAllMocks(); - matchMaps = []; - hasura.query.mockImplementation(async () => ({ match_maps: matchMaps })); - hasura.mutation.mockResolvedValue({}); - job = new FinalizeStrandedMaps(logger as any, hasura as any); - }); - - it("finishes a map stranded past the handshake deadline with the reported winner", async () => { - matchMaps = [strandedMap()]; - - await expect(job.process()).resolves.toBe(1); - - expect(hasura.mutation).toHaveBeenCalledWith( - expect.objectContaining({ - update_match_maps_by_pk: expect.objectContaining({ - __args: { - pk_columns: { id: "map-1" }, - _set: { - status: "Finished", - winning_lineup_id: "lineup-2", - }, - }, - }), - }), - ); - }); - - it("leaves a map alone while the server is still inside its tv_delay window", async () => { - matchMaps = [strandedMap({ rounds: [{ time: secondsAgo(60) }] })]; - - await expect(job.process()).resolves.toBe(0); - - expect(hasura.mutation).not.toHaveBeenCalled(); - }); - - it("derives the winner from the score when the server never reported one", async () => { - matchMaps = [ - strandedMap({ - winning_lineup_id: null, - lineup_1_score: 13, - lineup_2_score: 4, - }), - ]; - - await expect(job.process()).resolves.toBe(1); - - expect(hasura.mutation).toHaveBeenCalledWith( - expect.objectContaining({ - update_match_maps_by_pk: expect.objectContaining({ - __args: expect.objectContaining({ - _set: { - status: "Finished", - winning_lineup_id: "lineup-1", - }, - }), - }), - }), - ); - }); - - it("finishes a tied map without a winner rather than leaving it stranded", async () => { - matchMaps = [ - strandedMap({ - winning_lineup_id: null, - lineup_1_score: 12, - lineup_2_score: 12, - }), - ]; - - await expect(job.process()).resolves.toBe(1); - - expect(hasura.mutation).toHaveBeenCalledWith( - expect.objectContaining({ - update_match_maps_by_pk: expect.objectContaining({ - __args: expect.objectContaining({ - _set: { - status: "Finished", - }, - }), - }), - }), - ); - }); - - it("skips maps with no rounds, which have no end-of-game anchor", async () => { - matchMaps = [strandedMap({ rounds: [] })]; - - await expect(job.process()).resolves.toBe(0); - - expect(hasura.mutation).not.toHaveBeenCalled(); - }); -}); diff --git a/src/matches/jobs/FinalizeStrandedMaps.ts b/src/matches/jobs/FinalizeStrandedMaps.ts deleted file mode 100644 index 0b1347c5..00000000 --- a/src/matches/jobs/FinalizeStrandedMaps.ts +++ /dev/null @@ -1,155 +0,0 @@ -import { Logger } from "@nestjs/common"; -import { WorkerHost } from "@nestjs/bullmq"; -import { MatchQueues } from "../enums/MatchQueues"; -import { UseQueue } from "../../utilities/QueueProcessors"; -import { HasuraService } from "../../hasura/hasura.service"; -import { e_match_map_status_enum } from "../../../generated"; - -@UseQueue("Matches", MatchQueues.ScheduledMatches) -export class FinalizeStrandedMaps extends WorkerHost { - // The game server is the only thing that writes Finished, and it does so at - // the very end of a handshake it runs entirely in-process: WaitingForTV for - // tv_delay, then stop demo -> UploadingDemo, then upload -> Finished. If its - // pod dies anywhere in that window (auto-cancel, crash, eviction, node drain) - // the map strands, update_match_state never runs, the series never resolves - // and the server is never released. Past this deadline we finish the map from - // the result the server already published when the game ended. - private static readonly PLUGIN_HANDSHAKE_SECONDS = 20; - private static readonly GRACE_SECONDS = 300; - - constructor( - private readonly logger: Logger, - private readonly hasura: HasuraService, - ) { - super(); - } - - async process(): Promise { - const { match_maps } = await this.hasura.query({ - match_maps: { - __args: { - where: { - status: { - _in: [ - "WaitingForTV", - "UploadingDemo", - ] as e_match_map_status_enum[], - }, - match: { - status: { - _nin: [ - "Finished", - "Canceled", - "Forfeit", - "Tie", - "Surrendered", - ], - }, - }, - }, - }, - id: true, - status: true, - match_id: true, - winning_lineup_id: true, - lineup_1_score: true, - lineup_2_score: true, - match: { - lineup_1_id: true, - lineup_2_id: true, - options: { - tv_delay: true, - }, - }, - rounds: { - __args: { - order_by: [{ time: "desc" }], - limit: 1, - }, - time: true, - }, - }, - }); - - let finalized = 0; - - for (const matchMap of match_maps) { - const lastRoundAt = matchMap.rounds?.at(0)?.time; - - if (!lastRoundAt) { - continue; - } - - const deadline = - new Date(lastRoundAt).getTime() + - ((matchMap.match?.options?.tv_delay ?? 0) + - FinalizeStrandedMaps.PLUGIN_HANDSHAKE_SECONDS + - FinalizeStrandedMaps.GRACE_SECONDS) * - 1000; - - if (Date.now() < deadline) { - continue; - } - - const winningLineupId = this.resolveWinningLineupId(matchMap); - - this.logger.warn( - `[${matchMap.match_id}] map ${matchMap.id} stranded in ${matchMap.status} ` + - `since ${lastRoundAt} — finalizing without the server ` + - `(score ${matchMap.lineup_1_score ?? 0}-${matchMap.lineup_2_score ?? 0}, ` + - `winner ${winningLineupId ?? ""})`, - ); - - await this.hasura.mutation({ - update_match_maps_by_pk: { - __args: { - pk_columns: { - id: matchMap.id, - }, - _set: { - status: "Finished", - ...(winningLineupId - ? { winning_lineup_id: winningLineupId } - : {}), - }, - }, - __typename: true, - }, - }); - - finalized++; - } - - return finalized; - } - - // The server publishes the winner alongside the WaitingForTV status, so the - // stored value is authoritative when present; scores are the fallback for - // maps that stranded before that landed. - private resolveWinningLineupId(matchMap: { - winning_lineup_id?: string | null; - lineup_1_score?: number | null; - lineup_2_score?: number | null; - match?: { - lineup_1_id?: string | null; - lineup_2_id?: string | null; - } | null; - }) { - if (matchMap.winning_lineup_id) { - return matchMap.winning_lineup_id; - } - - const lineup1Score = matchMap.lineup_1_score ?? 0; - const lineup2Score = matchMap.lineup_2_score ?? 0; - - if (lineup1Score > lineup2Score) { - return matchMap.match?.lineup_1_id ?? null; - } - - if (lineup2Score > lineup1Score) { - return matchMap.match?.lineup_2_id ?? null; - } - - return null; - } -} diff --git a/src/matches/matches.module.ts b/src/matches/matches.module.ts index f09895fa..4d47ff73 100644 --- a/src/matches/matches.module.ts +++ b/src/matches/matches.module.ts @@ -42,7 +42,6 @@ import { getQueuesProcessors } from "../utilities/QueueProcessors"; import { CancelInvalidTournaments } from "./jobs/CancelInvalidTournaments"; import { SocketsModule } from "../sockets/sockets.module"; import { CleanAbandonedMatches } from "./jobs/CleanAbandonedMatches"; -import { FinalizeStrandedMaps } from "./jobs/FinalizeStrandedMaps"; import { ReapIdleDemoSessions } from "./jobs/ReapIdleDemoSessions"; import { PollMediaMtxViewers } from "./jobs/PollMediaMtxViewers"; import { MatchMaking } from "src/matchmaking/matchmaking.module"; @@ -168,7 +167,6 @@ import { LeaguesModule } from "../leagues/leagues.module"; StopOnDemandServer, CancelInvalidTournaments, CleanAbandonedMatches, - FinalizeStrandedMaps, ReapIdleDemoSessions, PollMediaMtxViewers, EloCalculation, @@ -293,16 +291,6 @@ export class MatchesModule implements NestModule { }, ); - void scheduleMatchQueue.add( - FinalizeStrandedMaps.name, - {}, - { - repeat: { - pattern: "* * * * *", - }, - }, - ); - void scheduleMatchQueue.add( ReapIdleDemoSessions.name, {}, diff --git a/test/match-scoring.spec.ts b/test/match-scoring.spec.ts index d269a952..0cfea283 100644 --- a/test/match-scoring.spec.ts +++ b/test/match-scoring.spec.ts @@ -339,34 +339,6 @@ describe("match scoring from rounds (SQL-driven)", () => { expect(paused.cancels_at).toBeNull(); }); - it.each(["WaitingForTV", "UploadingDemo"])( - "a map entering %s disarms the live-match timeout", - async (status) => { - const match = await createLiveMatch(1); - - await postgres.query( - "UPDATE match_maps SET status = 'Live' WHERE id = $1", - [match.mapIds[0]], - ); - const [live] = await postgres.query>( - "SELECT cancels_at FROM matches WHERE id = $1", - [match.id], - ); - expect(live.cancels_at).not.toBeNull(); - - // The end-of-map handshake runs for tv_delay plus the demo upload; the - // deadline firing inside it kills the server before it reports Finished. - await postgres.query("UPDATE match_maps SET status = $2 WHERE id = $1", [ - match.mapIds[0], - status, - ]); - const [finalizing] = await postgres.query< - Array<{ cancels_at: Date | null }> - >("SELECT cancels_at FROM matches WHERE id = $1", [match.id]); - expect(finalizing.cancels_at).toBeNull(); - }, - ); - it("finishing the map stamps the map's ended_at", async () => { const match = await createLiveMatch(1); await recordScore(match.mapIds[0], 13, 7); From e6c2469d723528ead94e5da5a017b2768c95d65e Mon Sep 17 00:00:00 2001 From: Luke Policinski Date: Sun, 26 Jul 2026 20:49:08 -0400 Subject: [PATCH 3/3] wip --- hasura/metadata/actions.graphql | 1 - src/telemetry/telemetry.service.ts | 6 ------ src/telemetry/types/TelemetryPayload.ts | 1 - test/telemetry.spec.ts | 6 ------ 4 files changed, 14 deletions(-) diff --git a/hasura/metadata/actions.graphql b/hasura/metadata/actions.graphql index 6460f481..a70ccb6a 100644 --- a/hasura/metadata/actions.graphql +++ b/hasura/metadata/actions.graphql @@ -1185,7 +1185,6 @@ type TelemetryFleetTotals { servers: Int! dedicatedServers: Int! publicServers: Int! - serverCapacity: Int! matches: Int! matchesWeek: Int! matchesMonth: Int! diff --git a/src/telemetry/telemetry.service.ts b/src/telemetry/telemetry.service.ts index 20858be6..6072afa8 100644 --- a/src/telemetry/telemetry.service.ts +++ b/src/telemetry/telemetry.service.ts @@ -194,7 +194,6 @@ export class TelemetryService { enabled: counts.servers_enabled, dedicated: counts.servers_dedicated, public: counts.servers_public, - capacity: counts.servers_capacity, }, matches: { total: counts.matches_ran, @@ -331,7 +330,6 @@ export class TelemetryService { coalesce(sum((payload->'servers'->>'total')::numeric), 0) AS servers, coalesce(sum((payload->'servers'->>'dedicated')::numeric), 0) AS "dedicatedServers", coalesce(sum((payload->'servers'->>'public')::numeric), 0) AS "publicServers", - coalesce(sum((payload->'servers'->>'capacity')::numeric), 0) AS "serverCapacity", coalesce(sum((payload->'matches'->>'total')::numeric), 0) AS matches, coalesce(sum((payload->'matches'->>'week')::numeric), 0) AS "matchesWeek", coalesce(sum((payload->'matches'->>'month')::numeric), 0) AS "matchesMonth", @@ -355,7 +353,6 @@ export class TelemetryService { "servers", "dedicatedServers", "publicServers", - "serverCapacity", "matches", "matchesWeek", "matchesMonth", @@ -555,7 +552,6 @@ export class TelemetryService { enabled: int(servers.enabled), dedicated: int(servers.dedicated), public: int(servers.public), - capacity: int(servers.capacity), }, matches: { total: int(matches.total), @@ -881,8 +877,6 @@ export class TelemetryService { (SELECT count(*) FROM public.servers WHERE is_dedicated) AS servers_dedicated, (SELECT count(*) FROM public.servers WHERE ${TelemetryService.RealServer} AND type <> 'Ranked') AS servers_public, - (SELECT coalesce(sum(max_players), 0) FROM public.servers - WHERE ${TelemetryService.RealServer}) AS servers_capacity, (SELECT count(*) FROM public.matches) AS matches_created, (SELECT count(*) FROM public.matches WHERE ${TelemetryService.nativeMatch()}) AS matches_ran, diff --git a/src/telemetry/types/TelemetryPayload.ts b/src/telemetry/types/TelemetryPayload.ts index 4c617d5f..2546e3be 100644 --- a/src/telemetry/types/TelemetryPayload.ts +++ b/src/telemetry/types/TelemetryPayload.ts @@ -27,7 +27,6 @@ export type TelemetryPayload = { enabled: number; dedicated: number; public: number; - capacity: number; }; // Everything outside `external` counts only matches this panel actually ran. // An imported demo is stamped with a started_at, so without the split it diff --git a/test/telemetry.spec.ts b/test/telemetry.spec.ts index 5098ec90..43b1474b 100644 --- a/test/telemetry.spec.ts +++ b/test/telemetry.spec.ts @@ -145,9 +145,6 @@ describe("telemetry (SQL-driven)", () => { expect(Number(rows.count)).toBe(6); expect(payload.servers.total).toBe(1); expect(payload.servers.dedicated).toBe(1); - // 32 is the max_players default every slot carries, and counting them - // reported a capacity of thousands on a panel with one real server. - expect(payload.servers.capacity).toBe(32); }); it("counts only players who have signed in as registered", () => { @@ -219,7 +216,6 @@ describe("telemetry (SQL-driven)", () => { enabled: 9, dedicated: 3, public: 4, - capacity: 120, }, matches: { total: 500, @@ -365,7 +361,6 @@ describe("telemetry (SQL-driven)", () => { enabled: servers, dedicated: 1, public: 2, - capacity: servers * 10, }, matches: { total: matches, @@ -407,7 +402,6 @@ describe("telemetry (SQL-driven)", () => { expect(stats.installs.active24h).toBe(2); expect(stats.totals.matches).toBe(350); expect(stats.totals.servers).toBe(10); - expect(stats.totals.serverCapacity).toBe(100); expect(stats.totals.mapsPlayed).toBe(700); expect(stats.totals.playersKnown).toBe(400); expect(stats.totals.playersRegistered).toBe(100);