diff --git a/src/matchmaking/jobs/ExpandMatchmaking.ts b/src/matchmaking/jobs/ExpandMatchmaking.ts new file mode 100644 index 000000000..07b26a05f --- /dev/null +++ b/src/matchmaking/jobs/ExpandMatchmaking.ts @@ -0,0 +1,23 @@ +import { Job } from "bullmq"; +import { WorkerHost } from "@nestjs/bullmq"; +import { e_match_types_enum } from "generated"; +import { MatchmakingQueues } from "../enums/MatchmakingQueues"; +import { UseQueue } from "../../utilities/QueueProcessors"; +import { MatchmakeService } from "src/matchmaking/matchmake.service"; + +@UseQueue("Matchmaking", MatchmakingQueues.Matchmaking) +export class ExpandMatchmaking extends WorkerHost { + constructor(private readonly matchmaking: MatchmakeService) { + super(); + } + + async process( + job: Job<{ + type: e_match_types_enum; + region: string; + }>, + ): Promise { + const { type, region } = job.data; + await this.matchmaking.matchmake(type, region); + } +} diff --git a/src/matchmaking/matchmake.integration.spec.ts b/src/matchmaking/matchmake.integration.spec.ts new file mode 100644 index 000000000..26dceb0f4 --- /dev/null +++ b/src/matchmaking/matchmake.integration.spec.ts @@ -0,0 +1,814 @@ +import { Test, TestingModule } from "@nestjs/testing"; +import { Logger } from "@nestjs/common"; +import { Queue } from "bullmq"; +import { e_match_types_enum } from "generated"; +import { MatchmakingLobby } from "./types/MatchmakingLobby"; + +jest.mock("../matches/match-assistant/match-assistant.service", () => ({ + MatchAssistantService: jest.fn().mockImplementation(() => ({ + createMatchBasedOnType: jest.fn(), + updateMatchStatus: jest.fn(), + })), +})); + +import { MatchmakeService } from "./matchmake.service"; +import { HasuraService } from "../hasura/hasura.service"; +import { MatchAssistantService } from "../matches/match-assistant/match-assistant.service"; +import { MatchmakingLobbyService } from "./matchmaking-lobby.service"; +import { RedisManagerService } from "../redis/redis-manager/redis-manager.service"; +import { MatchmakingQueues } from "./enums/MatchmakingQueues"; +import { FakeRedis } from "./testing/fakeRedis"; +import { + getMatchmakingQueueCacheKey, + getMatchmakingRankCacheKey, +} from "./utilities/cacheKeys"; +import { ExpectedPlayers } from "src/discord-bot/enums/ExpectedPlayers"; + +const COMPETITIVE: e_match_types_enum = "Competitive"; + +/** + * These drive the real MatchmakeService against an in-memory redis, so the + * assertions are about the queue's actual end state rather than which mocks + * were called. Matchmaking is a critical path: the failures that matter are a + * lobby silently disappearing from the queue, a lobby landing in two matches, + * or a team going out half full. + */ +describe("matchmaking (end to end)", () => { + let service: MatchmakeService; + let redis: FakeRedis; + let lobbyStore: Map; + let confirmations: Array<{ + region: string; + type: e_match_types_enum; + team1: { lobbies: string[]; players: Array<{ steam_id: string; rank: number }> }; + team2: { lobbies: string[]; players: Array<{ steam_id: string; rank: number }> }; + }>; + let confirmationIds: string[]; + let lineupInserts: Array<{ lineupId: string; steamIds: string[] }>; + let matchAssistant: { + createMatchBasedOnType: jest.Mock; + updateMatchStatus: jest.Mock; + }; + let hasura: { query: jest.Mock; mutation: jest.Mock }; + let queue: { add: jest.Mock; remove: jest.Mock }; + + beforeEach(async () => { + redis = new FakeRedis(); + lobbyStore = new Map(); + confirmations = []; + + queue = { add: jest.fn(), remove: jest.fn() }; + + confirmationIds = []; + lineupInserts = []; + + const lobbyService = { + getLobbyDetails: jest.fn(async (lobbyId: string) => { + const lobby = lobbyStore.get(lobbyId); + return lobby ? { ...lobby, players: [...lobby.players] } : null; + }), + setMatchConformationIdForLobby: jest.fn( + async (_lobbyId: string, confirmationId: string) => { + if (!confirmationIds.includes(confirmationId)) { + confirmationIds.push(confirmationId); + } + }, + ), + sendQueueDetailsToLobby: jest.fn(), + // the real implementations zrem from every region the lobby queued for + removeLobbyFromQueue: jest.fn(async (lobbyId: string) => { + const lobby = lobbyStore.get(lobbyId); + for (const region of lobby?.regions ?? []) { + await redis.zrem( + getMatchmakingRankCacheKey(lobby.type, region), + lobbyId, + ); + await redis.zrem( + getMatchmakingQueueCacheKey(lobby.type, region), + lobbyId, + ); + } + }), + removeLobbyDetails: jest.fn(async (lobbyId: string) => { + lobbyStore.delete(lobbyId); + }), + removeConfirmationIdFromLobby: jest.fn(), + }; + + matchAssistant = { + createMatchBasedOnType: jest.fn(async () => ({ + id: "match-1", + lineup_1_id: "lineup-1", + lineup_2_id: "lineup-2", + })), + updateMatchStatus: jest.fn(), + }; + + hasura = { + // sendRegionStats reads the region list on cancel + query: jest.fn(async () => ({ + server_regions: [{ value: "us-east" }, { value: "eu-west" }], + })), + mutation: jest.fn(async (payload: any) => { + const objects = payload?.insert_match_lineup_players?.__args?.objects; + if (objects) { + lineupInserts.push({ + lineupId: objects[0]?.match_lineup_id, + steamIds: objects.map((o: any) => o.steam_id), + }); + } + return {}; + }), + }; + + const module: TestingModule = await Test.createTestingModule({ + providers: [ + { provide: Logger, useValue: new Logger("Test") }, + MatchmakeService, + { provide: HasuraService, useValue: hasura }, + { provide: MatchAssistantService, useValue: matchAssistant }, + { provide: MatchmakingLobbyService, useValue: lobbyService }, + { + provide: RedisManagerService, + useValue: { getConnection: () => redis }, + }, + { + provide: `BullQueue_${MatchmakingQueues.Matchmaking}`, + useValue: queue as unknown as Queue, + }, + ], + }).compile(); + + service = module.get(MatchmakeService); + + // record confirmations but keep the real redis writes, so queue state stays honest + const original = (service as any).createMatchConfirmation.bind(service); + jest + .spyOn(service as any, "createMatchConfirmation") + .mockImplementation(async (...args: any[]) => { + const [region, type, teams] = args; + confirmations.push({ region, type, ...teams }); + return original(region, type, teams); + }); + }); + + // --- helpers + + function makeLobby( + lobbyId: string, + ranks: number[], + options?: { regions?: string[]; waitSeconds?: number; type?: e_match_types_enum }, + ): MatchmakingLobby { + return { + lobbyId, + type: options?.type ?? COMPETITIVE, + regions: options?.regions ?? ["us-east"], + joinedAt: new Date(Date.now() - (options?.waitSeconds ?? 30) * 1000), + players: ranks.map((rank, index) => ({ + steam_id: `${lobbyId}-p${index}`, + rank, + })), + regionPositions: {}, + avgRank: ranks.reduce((acc, rank) => acc + rank, 0) / ranks.length, + }; + } + + async function enqueue(lobbies: MatchmakingLobby[]) { + for (const lobby of lobbies) { + lobbyStore.set(lobby.lobbyId, lobby); + for (const region of lobby.regions) { + await redis.zadd( + getMatchmakingRankCacheKey(lobby.type, region), + lobby.avgRank, + lobby.lobbyId, + ); + await redis.zadd( + getMatchmakingQueueCacheKey(lobby.type, region), + 0, + lobby.lobbyId, + ); + } + } + } + + function queuedIn(region: string, type = COMPETITIVE) { + return redis.members(getMatchmakingRankCacheKey(type, region)); + } + + function matchedLobbies() { + return confirmations.flatMap((c) => [...c.team1.lobbies, ...c.team2.lobbies]); + } + + /** The invariants that must hold no matter what the queue looked like. */ + function assertInvariants( + lobbies: MatchmakingLobby[], + type: e_match_types_enum = COMPETITIVE, + ) { + const half = ExpectedPlayers[type] / 2; + + for (const confirmation of confirmations) { + expect(confirmation.team1.players).toHaveLength(half); + expect(confirmation.team2.players).toHaveLength(half); + + // a party is never split across the two teams + const team1Lobbies = new Set(confirmation.team1.lobbies); + for (const lobbyId of confirmation.team2.lobbies) { + expect(team1Lobbies.has(lobbyId)).toBe(false); + } + + // the players in a team are exactly the players of its lobbies + for (const team of [confirmation.team1, confirmation.team2]) { + const expected = team.lobbies.flatMap( + (lobbyId) => lobbyStore.get(lobbyId).players, + ); + expect(team.players.map((p) => p.steam_id).sort()).toEqual( + expected.map((p) => p.steam_id).sort(), + ); + } + } + + // no lobby is in two matches + const matched = matchedLobbies(); + expect(new Set(matched).size).toBe(matched.length); + + // no player is in two matches + const players = confirmations.flatMap((c) => [ + ...c.team1.players, + ...c.team2.players, + ]); + expect(new Set(players.map((p) => p.steam_id)).size).toBe(players.length); + + // conservation: every lobby is either matched or still queued, never lost + const matchedSet = new Set(matched); + for (const lobby of lobbies) { + const stillQueued = lobby.regions.some((region) => + queuedIn(region, lobby.type).includes(lobby.lobbyId), + ); + expect(matchedSet.has(lobby.lobbyId) !== stillQueued).toBe(true); + } + } + + // --- tests + + describe("queue state", () => { + it("matches ten solo players and empties the queue", async () => { + const lobbies = Array.from({ length: 10 }, (_, i) => + makeLobby(`solo-${i}`, [5000 + i * 10]), + ); + await enqueue(lobbies); + + await service.matchmake(COMPETITIVE, "us-east"); + + expect(confirmations).toHaveLength(1); + expect(queuedIn("us-east")).toHaveLength(0); + assertInvariants(lobbies); + }); + + it("leaves the surplus queued and requeues it intact", async () => { + const lobbies = Array.from({ length: 13 }, (_, i) => + makeLobby(`solo-${i}`, [5000]), + ); + await enqueue(lobbies); + + await service.matchmake(COMPETITIVE, "us-east"); + + expect(confirmations).toHaveLength(1); + expect(queuedIn("us-east")).toHaveLength(3); + assertInvariants(lobbies); + }); + + it("does not touch the queue when there are not enough players", async () => { + const lobbies = Array.from({ length: 9 }, (_, i) => + makeLobby(`solo-${i}`, [5000]), + ); + await enqueue(lobbies); + + await service.matchmake(COMPETITIVE, "us-east"); + + expect(confirmations).toHaveLength(0); + expect(queuedIn("us-east")).toHaveLength(9); + assertInvariants(lobbies); + }); + + it("keeps everyone queued when party sizes cannot fill two lineups", async () => { + // five duos: ten players, but 2s never sum to a lineup of 5 + const lobbies = Array.from({ length: 5 }, (_, i) => + makeLobby(`duo-${i}`, [5000, 5000]), + ); + await enqueue(lobbies); + + await service.matchmake(COMPETITIVE, "us-east"); + + expect(confirmations).toHaveLength(0); + expect(queuedIn("us-east")).toHaveLength(5); + assertInvariants(lobbies); + }); + + it("releases the region lock so a later pass can run", async () => { + await enqueue( + Array.from({ length: 10 }, (_, i) => makeLobby(`solo-${i}`, [5000])), + ); + + await service.matchmake(COMPETITIVE, "us-east"); + expect(await redis.get("matchmaking:lock:us-east")).toBeNull(); + + const second = Array.from({ length: 10 }, (_, i) => + makeLobby(`second-${i}`, [5000]), + ); + await enqueue(second); + await service.matchmake(COMPETITIVE, "us-east"); + + expect(confirmations).toHaveLength(2); + }); + }); + + describe("matches per pass", () => { + it("drains three matches out of a thirty player queue in one pass", async () => { + const lobbies = Array.from({ length: 30 }, (_, i) => + makeLobby(`solo-${i}`, [5000 + (i % 5) * 10]), + ); + await enqueue(lobbies); + + await service.matchmake(COMPETITIVE, "us-east"); + + expect(confirmations).toHaveLength(3); + expect(queuedIn("us-east")).toHaveLength(0); + assertInvariants(lobbies); + }); + + it("matches what it can and leaves the remainder queued", async () => { + const lobbies = Array.from({ length: 25 }, (_, i) => + makeLobby(`solo-${i}`, [5000]), + ); + await enqueue(lobbies); + + await service.matchmake(COMPETITIVE, "us-east"); + + expect(confirmations).toHaveLength(2); + expect(queuedIn("us-east")).toHaveLength(5); + assertInvariants(lobbies); + }); + + it("keeps unused lobbies claimed between matches instead of churning them", async () => { + const lobbies = Array.from({ length: 23 }, (_, i) => + makeLobby(`solo-${i}`, [5000]), + ); + await enqueue(lobbies); + + const requeuedBefore = redis.zaddCount(); + await service.matchmake(COMPETITIVE, "us-east"); + + // two matches leave 3 lobbies over. releasing and re-claiming the + // spares between iterations would requeue them - and push a redundant + // queue update to every one of them - so each should be written back + // once per region key, at the end, and no more. + const leftover = queuedIn("us-east"); + expect(leftover).toHaveLength(3); + expect(redis.zaddCount() - requeuedBefore).toBe(leftover.length * 2); + }); + + it("schedules one deduplicated pass when lobbies could not be claimed", async () => { + const lobbies = Array.from({ length: 20 }, (_, i) => + makeLobby(`solo-${i}`, [5000]), + ); + await enqueue(lobbies); + + // everything is already locked by another worker, so nothing can be + // claimed and the pass has to be retried later + for (const lobby of lobbies) { + await redis.set(`matchmaking:lock:${lobby.lobbyId}`, 1, "EX", 10, "NX"); + } + + await service.matchmake(COMPETITIVE, "us-east"); + + const expandJobs = queue.add.mock.calls.filter( + ([name]) => name === "ExpandMatchmaking", + ); + expect(expandJobs).toHaveLength(1); + + // a fixed jobId so concurrent callers collapse into one pending pass + // instead of stacking timers + expect(expandJobs[0][2]).toMatchObject({ + jobId: "matchmaking.expand.Competitive.us-east", + }); + expect(expandJobs[0][1]).toEqual({ + type: COMPETITIVE, + region: "us-east", + }); + }); + + it("does not schedule another pass once the queue is drained", async () => { + await enqueue( + Array.from({ length: 10 }, (_, i) => makeLobby(`solo-${i}`, [5000])), + ); + + await service.matchmake(COMPETITIVE, "us-east"); + + expect( + queue.add.mock.calls.filter(([name]) => name === "ExpandMatchmaking"), + ).toHaveLength(0); + }); + + it("stops carving when the remainder cannot legally split", async () => { + // the duos sit outside the rank window of the long waiting solos, so the + // first match is the ten solos and the remainder is five duos - ten + // players that can never make two teams of five + const lobbies = [ + ...Array.from({ length: 10 }, (_, i) => + makeLobby(`solo-${i}`, [5000], { waitSeconds: 300 }), + ), + ...Array.from({ length: 5 }, (_, i) => + makeLobby(`duo-${i}`, [9600, 9600], { waitSeconds: 5 }), + ), + ]; + await enqueue(lobbies); + + await service.matchmake(COMPETITIVE, "us-east"); + + expect(confirmations).toHaveLength(1); + expect(queuedIn("us-east").sort()).toEqual([ + "duo-0", + "duo-1", + "duo-2", + "duo-3", + "duo-4", + ]); + assertInvariants(lobbies); + }); + }); + + describe("parties", () => { + it("never splits a party across teams", async () => { + const lobbies = [ + makeLobby("trio", [6000, 6000, 6000]), + makeLobby("duo", [4000, 4000]), + makeLobby("pair", [5000, 5000]), + makeLobby("solo-1", [5000]), + makeLobby("solo-2", [5000]), + makeLobby("solo-3", [5000]), + ]; + await enqueue(lobbies); + + await service.matchmake(COMPETITIVE, "us-east"); + + expect(confirmations).toHaveLength(1); + assertInvariants(lobbies); + }); + + it("splits a full ten stack in house without touching the queue order", async () => { + const party = makeLobby("ten-stack", new Array(10).fill(5000)); + await enqueue([party]); + + await service.matchmake(COMPETITIVE, "us-east"); + + expect(confirmations).toHaveLength(1); + const [confirmation] = confirmations; + expect(confirmation.team1.players).toHaveLength(5); + expect(confirmation.team2.players).toHaveLength(5); + + // the lobby id is recorded once, not once per team, so the confirmation + // does not process it twice on cancel + expect([ + ...confirmation.team1.lobbies, + ...confirmation.team2.lobbies, + ]).toEqual(["ten-stack"]); + }); + + it("balances a wingman queue", async () => { + const lobbies = [ + makeLobby("w1", [6000], { type: "Wingman" }), + makeLobby("w2", [5000], { type: "Wingman" }), + makeLobby("w3", [5000], { type: "Wingman" }), + makeLobby("w4", [4000], { type: "Wingman" }), + ]; + await enqueue(lobbies); + + await service.matchmake("Wingman", "us-east"); + + expect(confirmations).toHaveLength(1); + const [confirmation] = confirmations; + const avg = (players: Array<{ rank: number }>) => + players.reduce((acc, p) => acc + p.rank, 0) / players.length; + expect( + Math.abs(avg(confirmation.team1.players) - avg(confirmation.team2.players)), + ).toBe(0); + assertInvariants(lobbies, "Wingman"); + }); + }); + + describe("multi region", () => { + it("removes a matched lobby from every region it queued for", async () => { + const shared = makeLobby("shared", [5000], { + regions: ["us-east", "eu-west"], + }); + const lobbies = [ + shared, + ...Array.from({ length: 9 }, (_, i) => makeLobby(`solo-${i}`, [5000])), + ]; + await enqueue(lobbies); + + await service.matchmake(COMPETITIVE, "us-east"); + + expect(matchedLobbies()).toContain("shared"); + expect(queuedIn("us-east")).not.toContain("shared"); + expect(queuedIn("eu-west")).not.toContain("shared"); + }); + + it("requeues an unused multi region lobby to all of its regions", async () => { + // the solos have waited longer, so one of them anchors the match and the + // window is measured from 5000, not from the outlier + const shared = makeLobby("shared", [9000], { + regions: ["us-east", "eu-west"], + waitSeconds: 5, + }); + const lobbies = [ + shared, + ...Array.from({ length: 10 }, (_, i) => + makeLobby(`solo-${i}`, [5000], { waitSeconds: 60 }), + ), + ]; + await enqueue(lobbies); + + await service.matchmake(COMPETITIVE, "us-east"); + + // the 9000 is out of the rank window, so it stays queued everywhere + expect(matchedLobbies()).not.toContain("shared"); + expect(queuedIn("us-east")).toContain("shared"); + expect(queuedIn("eu-west")).toContain("shared"); + }); + + it("matches a shared lobby only once when two regions run concurrently", async () => { + const shared = makeLobby("shared", [5000], { + regions: ["us-east", "eu-west"], + }); + await enqueue([ + shared, + ...Array.from({ length: 9 }, (_, i) => + makeLobby(`us-${i}`, [5000], { regions: ["us-east"] }), + ), + ...Array.from({ length: 9 }, (_, i) => + makeLobby(`eu-${i}`, [5000], { regions: ["eu-west"] }), + ), + ]); + + await Promise.all([ + service.matchmake(COMPETITIVE, "us-east"), + service.matchmake(COMPETITIVE, "eu-west"), + ]); + + const matched = matchedLobbies(); + expect(matched.filter((id) => id === "shared")).toHaveLength(1); + expect(new Set(matched).size).toBe(matched.length); + }); + }); + + describe("balance quality", () => { + it("beats the old greedy split on a queue greedy handles badly", async () => { + const ranks = [ + 6000, 5800, 5600, 5400, 5200, 5000, 4800, 4600, 4400, 1200, + ]; + const lobbies = ranks.map((rank, i) => makeLobby(`solo-${i}`, [rank])); + await enqueue(lobbies); + + await service.matchmake(COMPETITIVE, "us-east"); + + const [confirmation] = confirmations; + const avg = (players: Array<{ rank: number }>) => + players.reduce((acc, p) => acc + p.rank, 0) / players.length; + + // greedy produces a 1120 point gap on this exact queue + expect( + Math.abs(avg(confirmation.team1.players) - avg(confirmation.team2.players)), + ).toBe(0); + }); + + it("leaves a freshly queued out of rank player out of the match", async () => { + const lobbies = [ + makeLobby("smurf", [9500], { waitSeconds: 5 }), + ...Array.from({ length: 10 }, (_, i) => + makeLobby(`solo-${i}`, [5000], { waitSeconds: 60 }), + ), + ]; + await enqueue(lobbies); + + await service.matchmake(COMPETITIVE, "us-east"); + + expect(matchedLobbies()).not.toContain("smurf"); + expect(queuedIn("us-east")).toContain("smurf"); + }); + + it("eventually takes the out of rank player once they have waited", async () => { + const lobbies = [ + makeLobby("smurf", [9500], { waitSeconds: 600 }), + ...Array.from({ length: 9 }, (_, i) => + makeLobby(`solo-${i}`, [5000], { waitSeconds: 5 }), + ), + ]; + await enqueue(lobbies); + + await service.matchmake(COMPETITIVE, "us-east"); + + expect(confirmations).toHaveLength(1); + expect(matchedLobbies()).toContain("smurf"); + }); + + it("plays the longest waiting lobby rather than starving it", async () => { + const lobbies = [ + makeLobby("oldest", [5000], { waitSeconds: 900 }), + ...Array.from({ length: 14 }, (_, i) => + makeLobby(`solo-${i}`, [5000], { waitSeconds: 5 }), + ), + ]; + await enqueue(lobbies); + + await service.matchmake(COMPETITIVE, "us-east"); + + expect(matchedLobbies()).toContain("oldest"); + }); + }); + + describe("ready check", () => { + const tenSolos = () => + Array.from({ length: 10 }, (_, i) => makeLobby(`solo-${i}`, [5000])); + + async function confirmAll() { + const [confirmation] = confirmations; + const [confirmationId] = confirmationIds; + + for (const player of [ + ...confirmation.team1.players, + ...confirmation.team2.players, + ]) { + await service.playerConfirmMatchmaking(confirmationId, player.steam_id); + } + + return { confirmation, confirmationId }; + } + + it("creates the match once every player has confirmed", async () => { + await enqueue(tenSolos()); + await service.matchmake(COMPETITIVE, "us-east"); + + const { confirmation } = await confirmAll(); + + expect(matchAssistant.createMatchBasedOnType).toHaveBeenCalledTimes(1); + expect(matchAssistant.updateMatchStatus).toHaveBeenCalledWith( + "match-1", + "Live", + ); + + // each team's players land in its own lineup, and nobody is duplicated + expect(lineupInserts).toHaveLength(2); + const [first, second] = lineupInserts; + expect(first.lineupId).toBe("lineup-1"); + expect(second.lineupId).toBe("lineup-2"); + expect(first.steamIds.sort()).toEqual( + confirmation.team1.players.map((p) => p.steam_id).sort(), + ); + expect(second.steamIds.sort()).toEqual( + confirmation.team2.players.map((p) => p.steam_id).sort(), + ); + expect(new Set([...first.steamIds, ...second.steamIds]).size).toBe(10); + }); + + it("does not create the match until the last player confirms", async () => { + await enqueue(tenSolos()); + await service.matchmake(COMPETITIVE, "us-east"); + + const [confirmation] = confirmations; + const [confirmationId] = confirmationIds; + const players = [ + ...confirmation.team1.players, + ...confirmation.team2.players, + ]; + + for (const player of players.slice(0, 9)) { + await service.playerConfirmMatchmaking(confirmationId, player.steam_id); + } + expect(matchAssistant.createMatchBasedOnType).not.toHaveBeenCalled(); + + await service.playerConfirmMatchmaking( + confirmationId, + players.at(-1).steam_id, + ); + expect(matchAssistant.createMatchBasedOnType).toHaveBeenCalledTimes(1); + }); + + it("drops every lobby from the queue when nobody confirms", async () => { + await enqueue(tenSolos()); + await service.matchmake(COMPETITIVE, "us-east"); + + await service.cancelMatchMaking(confirmationIds[0]); + + expect(matchAssistant.createMatchBasedOnType).not.toHaveBeenCalled(); + expect(queuedIn("us-east")).toHaveLength(0); + }); + + it("requeues only the lobbies whose players were all ready", async () => { + const lobbies = [ + makeLobby("ready-duo", [5000, 5000]), + ...Array.from({ length: 8 }, (_, i) => makeLobby(`solo-${i}`, [5000])), + ]; + await enqueue(lobbies); + await service.matchmake(COMPETITIVE, "us-east"); + + const [confirmationId] = confirmationIds; + + // only the duo readies up + for (const player of lobbyStore.get("ready-duo").players) { + await service.playerConfirmMatchmaking(confirmationId, player.steam_id); + } + + await service.cancelMatchMaking(confirmationId); + + // the duo goes back in the queue; everyone who ignored the ready check + // is dropped out of matchmaking entirely + expect(queuedIn("us-east")).toEqual(["ready-duo"]); + }); + + it("holds the lobby locks while the confirmation is pending", async () => { + await enqueue(tenSolos()); + await service.matchmake(COMPETITIVE, "us-east"); + + // a pending confirmation must keep its lobbies locked, otherwise a + // concurrent pass could put the same players in a second match + expect(await redis.get("matchmaking:lock:solo-0")).not.toBeNull(); + + redis.advanceTime(31_000); + expect(await redis.get("matchmaking:lock:solo-0")).toBeNull(); + }); + + it("lets a requeued lobby be matched again on the next pass", async () => { + await enqueue(tenSolos()); + await service.matchmake(COMPETITIVE, "us-east"); + + const [confirmation] = confirmations; + const [confirmationId] = confirmationIds; + for (const player of [ + ...confirmation.team1.players, + ...confirmation.team2.players, + ]) { + await service.playerConfirmMatchmaking(confirmationId, player.steam_id); + } + // everyone was ready, but the match was cancelled for another reason + await service.cancelMatchMaking(confirmationId); + + expect(queuedIn("us-east")).toHaveLength(10); + + // the confirmation put a 30s ttl on each lobby lock; the next pass can + // only claim them once that has lapsed + redis.advanceTime(31_000); + + confirmations.length = 0; + await service.matchmake(COMPETITIVE, "us-east"); + + expect(confirmations).toHaveLength(1); + expect(queuedIn("us-east")).toHaveLength(0); + }); + }); + + describe("randomized queues", () => { + function mulberry32(seed: number) { + let state = seed; + return () => { + state |= 0; + state = (state + 0x6d2b79f5) | 0; + let t = Math.imul(state ^ (state >>> 15), 1 | state); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; + } + + it("holds every invariant across 60 random queues", async () => { + const random = mulberry32(31337); + + for (let iteration = 0; iteration < 60; iteration++) { + redis = new FakeRedis(); + (service as any).redis = redis; + lobbyStore.clear(); + confirmations.length = 0; + + const lobbies: MatchmakingLobby[] = []; + const lobbyCount = 2 + Math.floor(random() * 14); + for (let i = 0; i < lobbyCount; i++) { + const size = 1 + Math.floor(random() * 5); + lobbies.push( + makeLobby( + `lobby-${i}`, + Array.from( + { length: size }, + () => 2000 + Math.floor(random() * 7000), + ), + { waitSeconds: 5 + Math.floor(random() * 600) }, + ), + ); + } + + await enqueue(lobbies); + await service.matchmake(COMPETITIVE, "us-east"); + + assertInvariants(lobbies); + } + }); + }); +}); diff --git a/src/matchmaking/matchmake.service.spec.ts b/src/matchmaking/matchmake.service.spec.ts index 3dd32514a..a7f28bc73 100644 --- a/src/matchmaking/matchmake.service.spec.ts +++ b/src/matchmaking/matchmake.service.spec.ts @@ -3,6 +3,7 @@ import { Logger } from "@nestjs/common"; import { Queue } from "bullmq"; import { e_match_types_enum } from "generated"; import { MatchmakingLobby } from "./types/MatchmakingLobby"; +import { MatchmakingTeam } from "./types/MatchmakingTeam"; import Redis from "ioredis"; // Mock the problematic modules before importing the service @@ -20,6 +21,8 @@ import { MatchmakingLobbyService } from "./matchmaking-lobby.service"; import { RedisManagerService } from "../redis/redis-manager/redis-manager.service"; import { MatchmakingQueues } from "./enums/MatchmakingQueues"; +type ConfirmationTeams = { team1: MatchmakingTeam; team2: MatchmakingTeam }; + describe("MatchmakeService", () => { let service: MatchmakeService; let mockRedis: jest.Mocked; @@ -188,7 +191,7 @@ describe("MatchmakeService", () => { expect(callArgs[0]).toBe(region); expect(callArgs[1]).toBe(type); - const { team1, team2 } = callArgs[2]; + const { team1, team2 } = callArgs[2] as ConfirmationTeams; // Verify each team has exactly 5 players (half of 10) expect(team1.players.length).toBe(5); @@ -197,10 +200,14 @@ describe("MatchmakeService", () => { // Verify total players in the match is 10 expect(team1.players.length + team2.players.length).toBe(requiredPlayers); - // Note: The method returns 0 after successfully creating a match - // The remaining 5 players would be handled in a recursive call, but that result isn't returned - // The important thing is that exactly 1 match was created with 10 players - expect(result).toBe(0); + // 15 queued, 10 matched, so 5 are left over and reported accurately so + // the caller knows to expand the search + expect(result).toBe(5); + + // the two closest lobbies play; the 1100 lobby stays queued + const matched = [...team1.lobbies, ...team2.lobbies]; + expect(matched.sort()).toEqual(["lobby-1", "lobby-2"]); + expect(Math.abs(team1.avgRank - team2.avgRank)).toBe(50); // Verify claimLobby was called (via redis.eval) for each lobby expect(mockRedis.eval).toHaveBeenCalled(); @@ -399,7 +406,7 @@ describe("MatchmakeService", () => { expect(firstCallArgs[0]).toBe(region); expect(firstCallArgs[1]).toBe(type); - const { team1: team1Match1, team2: team2Match1 } = firstCallArgs[2]; + const { team1: team1Match1, team2: team2Match1 } = firstCallArgs[2] as ConfirmationTeams; expect(team1Match1.players.length).toBe(5); expect(team2Match1.players.length).toBe(5); expect(team1Match1.players.length + team2Match1.players.length).toBe(10); @@ -409,7 +416,7 @@ describe("MatchmakeService", () => { expect(secondCallArgs[0]).toBe(region); expect(secondCallArgs[1]).toBe(type); - const { team1: team1Match2, team2: team2Match2 } = secondCallArgs[2]; + const { team1: team1Match2, team2: team2Match2 } = secondCallArgs[2] as ConfirmationTeams; expect(team1Match2.players.length).toBe(5); expect(team2Match2.players.length).toBe(5); expect(team1Match2.players.length + team2Match2.players.length).toBe(10); @@ -558,73 +565,31 @@ describe("MatchmakeService", () => { expect(callArgs[0]).toBe(region); expect(callArgs[1]).toBe(type); - const { team1, team2 } = callArgs[2]; + const { team1, team2 } = callArgs[2] as ConfirmationTeams; // Verify each team has exactly 5 players expect(team1.players.length).toBe(5); expect(team2.players.length).toBe(5); expect(team1.players.length + team2.players.length).toBe(10); - // Log the team compositions and ranks for inspection - // Extract steam_id from player objects for logging - const team1PlayerIds = team1.players.map((p) => - typeof p === "string" ? p : p.steam_id, - ); - const team2PlayerIds = team2.players.map((p) => - typeof p === "string" ? p : p.steam_id, - ); - console.log( - `Team 1 players: ${team1PlayerIds.join(", ")} | ranks: ${team1.players - .map((p) => (typeof p === "object" ? p.rank : "N/A")) - .join(", ")}`, - ); - console.log( - `Team 2 players: ${team2PlayerIds.join(", ")} | ranks: ${team2.players - .map((p) => (typeof p === "object" ? p.rank : "N/A")) - .join(", ")}`, - ); - console.log(`Team 1 avg rank: ${team1.avgRank}`); - console.log(`Team 2 avg rank: ${team2.avgRank}`); - console.log(`Team 1 lobbies: ${team1.lobbies.join(", ")}`); - console.log(`Team 2 lobbies: ${team2.lobbies.join(", ")}`); - - // Verify that the rank difference between teams is very small (well balanced) - const rankDifference = Math.abs(team1.avgRank - team2.avgRank); - console.log(`Rank difference between teams: ${rankDifference}`); - - // The ranks should be very similar (within 50 points for this test) - // This ensures the ELO matching algorithm is working correctly - // expect(rankDifference).toBeLessThan(50); + // every party is wholly on one team, and every lobby is used + for (const lobby of lobbies) { + const onTeam1 = team1.lobbies.includes(lobby.lobbyId); + const onTeam2 = team2.lobbies.includes(lobby.lobbyId); + expect(onTeam1 !== onTeam2).toBe(true); + } - // Verify all players are accounted for - // Extract steam_id from player objects for comparison + // every player is used exactly once const allMatchedPlayers = [ - ...team1.players.map((p) => (typeof p === "string" ? p : p.steam_id)), - ...team2.players.map((p) => (typeof p === "string" ? p : p.steam_id)), + ...team1.players.map((p) => p.steam_id), + ...team2.players.map((p) => p.steam_id), ]; expect(allMatchedPlayers.sort()).toEqual(allLobbyPlayers.sort()); - // Verify specific players are on the correct teams - // Extract steam_id values for easier checking - const team1SteamIds = team1.players.map((p) => - typeof p === "string" ? p : p.steam_id, - ); - const team2SteamIds = team2.players.map((p) => - typeof p === "string" ? p : p.steam_id, - ); - - expect(team1SteamIds).toContain("steam-1"); - expect(team1SteamIds).toContain("steam-4"); - expect(team1SteamIds).toContain("steam-5"); - expect(team1SteamIds).toContain("steam-6"); - expect(team1SteamIds).toContain("steam-7"); - - // Verify that steam-2 and steam-3 (from lobby-2 with avgRank 4500) are on team 2 - expect(team2SteamIds).toContain("steam-2"); - expect(team2SteamIds).toContain("steam-3"); - expect(team2SteamIds).toContain("steam-8"); - expect(team2SteamIds).toContain("steam-9"); - expect(team2SteamIds).toContain("steam-10"); + // the party sizes here (1/2/1/3/1/2) only allow side totals of 11500, + // 13500, 15500, 16500, 18500 or 20500 out of 32000, so a perfect + // 16000/16000 split does not exist and 200 is the optimum + expect(Math.abs(team1.avgRank - team2.avgRank)).toBe(200); // Result should be 0 since all players were matched expect(result).toBe(0); @@ -686,7 +651,7 @@ describe("MatchmakeService", () => { // Verify eval was called with correct keys const evalCall = mockRedis.eval.mock.calls[0]; - const numKeys = evalCall[1]; + const numKeys = evalCall[1] as number; const keys = evalCall.slice(2, 2 + numKeys); // Should have: 1 lock key + 2 regions * 2 keys (queue + rank) = 5 keys @@ -755,11 +720,12 @@ describe("MatchmakeService", () => { }, ); - // lobby-1 fails to claim (another region got it), lobby-2 and lobby-3 succeed - mockRedis.eval - .mockResolvedValueOnce(0) // lobby-1: already claimed - .mockResolvedValueOnce(1) // lobby-2: claimed - .mockResolvedValueOnce(1); // lobby-3: claimed + // lobby-1 fails to claim (another region got it), lobby-2 and lobby-3 succeed. + // keyed off the lock key rather than call order, since lobbies are now + // claimed in preference order rather than input order + mockRedis.eval.mockImplementation((...args: any[]) => + Promise.resolve(args[2] === "matchmaking:lock:lobby-1" ? 0 : 1), + ); const createMatchConfirmationSpy = jest .spyOn(service as any, "createMatchConfirmation") @@ -770,13 +736,151 @@ describe("MatchmakeService", () => { // Should still create a match from lobby-2 + lobby-3 expect(createMatchConfirmationSpy).toHaveBeenCalledTimes(1); const callArgs = createMatchConfirmationSpy.mock.calls[0]; - const { team1, team2 } = callArgs[2]; + const { team1, team2 } = callArgs[2] as ConfirmationTeams; expect(team1.players.length + team2.players.length).toBe(10); // lobby-1 should NOT be in either team const allLobbies = [...team1.lobbies, ...team2.lobbies]; expect(allLobbies).not.toContain("lobby-1"); + // we never owned lobby-1's lock, so it must not be requeued here + const requeued = mockRedis.zadd.mock.calls.map((call) => call[2] as string); + expect(requeued).not.toContain("lobby-1"); + + createMatchConfirmationSpy.mockRestore(); + }); + }); + + describe("lobby locks", () => { + const region = "us-east"; + const type: e_match_types_enum = "Competitive"; + + const soloLobbies = (count: number, rank = 1000): MatchmakingLobby[] => + Array.from({ length: count }, (_, i) => ({ + lobbyId: `lobby-${i + 1}`, + type, + regions: [region], + players: [{ steam_id: `steam-${i + 1}`, rank }], + avgRank: rank, + joinedAt: new Date(), + regionPositions: {}, + })); + + it("requeues every claimed but unused lobby exactly once", async () => { + const lobbies = soloLobbies(12); + mockMatchmakingLobbyService.getLobbyDetails.mockImplementation( + async (lobbyId: string) => + lobbies.find((l) => l.lobbyId === lobbyId) || null, + ); + + const createMatchConfirmationSpy = jest + .spyOn(service as any, "createMatchConfirmation") + .mockResolvedValue(undefined); + + await (service as any).createMatches(region, type, lobbies); + + const { team1, team2 } = createMatchConfirmationSpy.mock.calls[0][2] as ConfirmationTeams; + const matched = new Set([...team1.lobbies, ...team2.lobbies]); + const requeued = mockRedis.zadd.mock.calls.map((call) => call[2] as string); + + // matched lobbies keep their lock, the confirmation owns them + for (const lobbyId of matched) { + expect(requeued).not.toContain(lobbyId); + } + + // the other two are released, once each (one zadd per region key) + for (const lobby of lobbies) { + if (matched.has(lobby.lobbyId)) { + continue; + } + expect( + requeued.filter((id) => id === lobby.lobbyId), + ).toHaveLength(2); + } + + createMatchConfirmationSpy.mockRestore(); + }); + + it("leaks no locks when creating the confirmation throws", async () => { + const lobbies = soloLobbies(12); + mockMatchmakingLobbyService.getLobbyDetails.mockImplementation( + async (lobbyId: string) => + lobbies.find((l) => l.lobbyId === lobbyId) || null, + ); + + const createMatchConfirmationSpy = jest + .spyOn(service as any, "createMatchConfirmation") + .mockRejectedValue(new Error("redis is down")); + + const result = await (service as any).createMatches( + region, + type, + lobbies, + ); + + const requeued = new Set( + mockRedis.zadd.mock.calls.map((call) => call[2] as string), + ); + for (const lobby of lobbies) { + expect(requeued.has(lobby.lobbyId)).toBe(true); + } + expect(result).toBe(12); + + createMatchConfirmationSpy.mockRestore(); + }); + + it("claims nothing when the party sizes can never fill two lineups", async () => { + // five duos is ten players, but 2s cannot sum to a lineup of 5 + const lobbies: MatchmakingLobby[] = Array.from( + { length: 5 }, + (_, i) => ({ + lobbyId: `duo-${i + 1}`, + type, + regions: [region], + players: [ + { steam_id: `steam-${i * 2 + 1}`, rank: 1000 }, + { steam_id: `steam-${i * 2 + 2}`, rank: 1000 }, + ], + avgRank: 1000, + joinedAt: new Date(), + regionPositions: {}, + }), + ); + + const createMatchConfirmationSpy = jest + .spyOn(service as any, "createMatchConfirmation") + .mockResolvedValue(undefined); + + const result = await (service as any).createMatches( + region, + type, + lobbies, + ); + + expect(createMatchConfirmationSpy).not.toHaveBeenCalled(); + expect(mockRedis.eval).not.toHaveBeenCalled(); + // 0 so the caller does not spin retrying something that cannot resolve + expect(result).toBe(0); + + createMatchConfirmationSpy.mockRestore(); + }); + + it("does not mutate the lobbies it was given", async () => { + const lobbies = soloLobbies(12); + const snapshot = [...lobbies]; + mockMatchmakingLobbyService.getLobbyDetails.mockImplementation( + async (lobbyId: string) => + lobbies.find((l) => l.lobbyId === lobbyId) || null, + ); + + const createMatchConfirmationSpy = jest + .spyOn(service as any, "createMatchConfirmation") + .mockResolvedValue(undefined); + + await (service as any).createMatches(region, type, lobbies); + + expect(lobbies).toEqual(snapshot); + createMatchConfirmationSpy.mockRestore(); }); }); diff --git a/src/matchmaking/matchmake.service.ts b/src/matchmaking/matchmake.service.ts index d3cb10578..268fe2280 100644 --- a/src/matchmaking/matchmake.service.ts +++ b/src/matchmaking/matchmake.service.ts @@ -19,6 +19,24 @@ import { getMatchmakingRankCacheKey, } from "./utilities/cacheKeys"; import { ExpectedPlayers } from "src/discord-bot/enums/ExpectedPlayers"; +import { shuffleSplit } from "./utilities/shuffleSplit"; +import { balanceTeams, canFillTeams } from "./utilities/balanceTeams"; +import { selectMatchCandidates } from "./utilities/selectMatchCandidates"; +import { WINDOW_CAP, winProbability } from "./utilities/matchmakingTuning"; + +function averageRank(players: Array<{ rank: number }>) { + return players.reduce((acc, player) => acc + player.rank, 0) / players.length; +} + +function toMatchmakingTeam(lobbies: MatchmakingLobby[]): MatchmakingTeam { + const players = lobbies.flatMap((lobby) => lobby.players); + + return { + lobbies: lobbies.map((lobby) => lobby.lobbyId), + players, + avgRank: averageRank(players), + }; +} @Injectable() export class MatchmakeService { @@ -42,6 +60,15 @@ export class MatchmakeService { return; } + // a non-finite score makes redis reject the whole zadd, which would silently + // drop the lobby from the queue + if (!Number.isFinite(lobby.avgRank)) { + this.logger.error( + `Cannot queue lobby ${lobbyId} - avgRank is ${lobby.avgRank}`, + ); + return; + } + // store the lobby's rank in a separate sorted set for quick rank matching for (const region of lobby.regions) { await this.redis.zadd( @@ -151,98 +178,21 @@ export class MatchmakeService { "WITHSCORES", ); - let lobbies = await this.processLobbyData(lobbiesData); + const lobbies = await this.processLobbyData(lobbiesData, region); if (lobbies.length === 0) { await this.releaseMatchmakeRegionLock(region); return; } - // sort lobbies by a weighted score combining rank difference and wait time - lobbies = lobbies.sort((a, b) => { - // normalize wait times to 0-1 range (longer wait = higher priority) - const aWaitTime = (Date.now() - a.joinedAt.getTime()) / 1000; - const bWaitTime = (Date.now() - b.joinedAt.getTime()) / 1000; - - const maxWaitTime = Math.max(aWaitTime, bWaitTime); - - const normalizedAWait = aWaitTime / maxWaitTime; - const normalizedBWait = bWaitTime / maxWaitTime; - - // weight rank differences more heavily (0.7) than wait time (0.3) - const rankWeight = 0.7; - const waitWeight = 0.3; - - return ( - rankWeight * b.avgRank + - waitWeight * normalizedBWait - - rankWeight * a.avgRank + - waitWeight * normalizedAWait - ); - }); - - // group lobbies based on rank differences that expand with wait time - const groupedLobbies = []; - let currentGroup = [lobbies.at(0)]; - - for (const currentLobby of lobbies.slice(1)) { - const firstLobbyInGroup = currentGroup.at(0); - - // TODO - check if rank difference feature is enabled - const rankDiffEnabled = false; - - if (!rankDiffEnabled) { - // if rank difference feature is disabled, just add lobbies in order - currentGroup.push(currentLobby); - continue; - } - - // calculate wait time in seconds - const waitTimeSeconds = Math.max( - 10, - Math.floor((Date.now() - firstLobbyInGroup.joinedAt.getTime()) / 1000), - ); - - // maximum allowed rank difference increases proportionally with wait time (100 per minute) - const maxRankDiff = 25 * waitTimeSeconds; - - // check if current lobby's rank is within acceptable range - if ( - Math.abs(currentLobby.avgRank - firstLobbyInGroup.avgRank) <= - maxRankDiff - ) { - currentGroup.push(currentLobby); - continue; - } - - // start new group if rank difference is too high - if (currentGroup.length > 0) { - groupedLobbies.push([...currentGroup]); - } - currentGroup = [currentLobby]; - } - - // add final group - if (currentGroup.length > 0) { - groupedLobbies.push(currentGroup); - } - - const createMatchesPromises = []; - - for (const group of groupedLobbies) { - createMatchesPromises.push(this.createMatches(region, type, group)); - } - - // once all results are returned as false we no longer need to matchmake - const results = await Promise.all(createMatchesPromises).finally(() => { + const totalPlayerNotQueued = await this.createMatches( + region, + type, + lobbies, + ).finally(() => { void this.releaseMatchmakeRegionLock(region); }); - const totalPlayerNotQueued = results.reduce( - (acc, result) => acc + result, - 0, - ); - if (totalPlayerNotQueued < ExpectedPlayers[type]) { await this.releaseMatchmakeRegionLock(region); return; @@ -252,17 +202,47 @@ export class MatchmakeService { `${totalPlayerNotQueued} players not queued, expanding search....`, ); - // randomize the time to prevent all regions from matchingmake at the same time - setTimeout( - () => { - void this.matchmake(type, region); - }, + await this.scheduleExpandedSearch( + type, + region, 10000 + Math.floor(Math.random() * 10000), ); } + /** + * Queues another pass over a region. + * + * A delayed job rather than a setTimeout: the jobId collapses duplicates, so + * the several callers that can all want another pass at once do not stack up + * timers, and a pending pass survives a restart instead of being lost. + */ + private async scheduleExpandedSearch( + type: e_match_types_enum, + region: string, + delay: number, + ) { + try { + await this.queue.add( + "ExpandMatchmaking", + { type, region }, + { + delay, + jobId: `matchmaking.expand.${type}.${region}`, + removeOnComplete: true, + removeOnFail: true, + }, + ); + } catch (error) { + this.logger.error( + `Unable to schedule another matchmaking pass for ${type}/${region}:`, + error, + ); + } + } + private async processLobbyData( lobbiesData: string[], + region: string, ): Promise { const lobbyDetails = []; @@ -285,38 +265,28 @@ export class MatchmakeService { } try { - const shuffledPlayers = [...details.players].sort( - () => Math.random() - 0.5, - ); - const halfLength = Math.floor(shuffledPlayers.length / 2); + // a party that fills the whole match keeps a random split - they + // queued together for a scrim, not for a rating-balanced game + const [players1, players2] = shuffleSplit(details.players); const team1: MatchmakingTeam = { - players: shuffledPlayers.slice(0, halfLength), - lobbies: [], - avgRank: 0, + players: players1, + lobbies: [details.lobbyId], + avgRank: averageRank(players1), }; const team2: MatchmakingTeam = { - players: shuffledPlayers.slice(halfLength), + // both teams come from the same lobby, but the id is only recorded + // once so the confirmation doesn't process it twice + players: players2, lobbies: [], - avgRank: 0, + avgRank: averageRank(players2), }; - team1.lobbies.push(details.lobbyId); - team2.lobbies.push(details.lobbyId); - - team1.avgRank = - team1.players.reduce((acc, player) => acc + player.rank, 0) / - team1.players.length; - team2.avgRank = - team2.players.reduce((acc, player) => acc + player.rank, 0) / - team2.players.length; - - const region = details.regions.at(0); - - await this.createMatchConfirmation(region, details.type, { - team1, - team2, - }); + await this.createMatchConfirmation( + details.regions.includes(region) ? region : details.regions.at(0), + details.type, + { team1, team2 }, + ); } catch (error) { this.logger.error( `Error creating match confirmation for lobby ${details.lobbyId}:`, @@ -329,7 +299,7 @@ export class MatchmakeService { } lobbyDetails.push({ ...details, - avgRank: parseInt(lobbiesData[i + 1]), + avgRank: averageRank(details.players), joinedAt: new Date(details.joinedAt), }); } @@ -337,191 +307,154 @@ export class MatchmakeService { return lobbyDetails; } + /** + * Carves as many matches as possible out of the queue in one pass. + * + * Lobbies stay claimed between iterations: a lobby that misses one match is a + * candidate for the next, and releasing it just to re-claim it would churn the + * queue keys and push a redundant queue update to everyone in it. + * + * Returns the number of players left unmatched, which the caller uses to + * decide whether another pass could help. 0 means "do not bother retrying". + */ private async createMatches( region: string, type: e_match_types_enum, lobbies: Array, ): Promise { const requiredPlayers = ExpectedPlayers[type]; - const totalPlayers = lobbies.reduce( - (acc, lobby) => acc + lobby.players.length, - 0, - ); - - if (lobbies.length === 0) { - return 0; - } - - if (totalPlayers < requiredPlayers) { - return totalPlayers; - } - // try to make as many valid matches as possible - const team1: MatchmakingTeam = { - players: [], - lobbies: [], - avgRank: 0, - }; - const team2: MatchmakingTeam = { - players: [], - lobbies: [], - avgRank: 0, - }; + // lobbies we hold the lock for and have not committed to a match + const claimed = new Map(); + // lobbies we have not tried to claim yet + const untried = new Map( + lobbies.map((lobby) => [lobby.lobbyId, lobby]), + ); - const lobbiesAdded: Array = []; - const playersPerTeam = requiredPlayers / 2; + const countPlayers = (pool: Array) => + pool.reduce((acc, lobby) => acc + lobby.players.length, 0); - let lobbyLocks = new Set(); + try { + for (;;) { + const pool = [...claimed.values(), ...untried.values()]; + const remainingPlayers = countPlayers(pool); - // try to fill teams with available lobbies - // if they are unable to accuire the lock, it means they are already being matched, or another region is trying to matchmake - // we assign lobbies to the team that keeps average elo between teams as close as possible - for (const lobby of lobbies) { - try { - const lock = await this.claimLobby(lobby.lobbyId, lobby); + if (remainingPlayers < requiredPlayers) { + return remainingPlayers; + } - if (!lock) { + // parties are atomic, so a pool can hold enough players and still have + // no legal split - five duos can never make two teams of five. that + // cannot resolve on its own, so report 0 rather than spinning the retry. + if ( + !canFillTeams( + pool.map((lobby) => lobby.players.length), + requiredPlayers, + ) + ) { this.logger.warn( - `Unable to acquire lobby lock for ${lobby.lobbyId} - lobby is already being processed`, + `${type}/${region}: ${remainingPlayers} queued but the party sizes cannot fill two lineups`, ); - continue; + return 0; } - const team1HasRoom = - team1.players.length + lobby.players.length <= playersPerTeam; - const team2HasRoom = - team2.players.length + lobby.players.length <= playersPerTeam; + const selection = selectMatchCandidates(pool, requiredPlayers); - if (!team1HasRoom && !team2HasRoom) { - await this.releaseLobbyAndRequeue(lobby.lobbyId); - continue; + if (!selection) { + return remainingPlayers; } - let targetTeam: MatchmakingTeam; - - if (!team1HasRoom) { - targetTeam = team2; - } else if (!team2HasRoom) { - targetTeam = team1; - } else { - // Calculate current team totals and player counts - const team1TotalRank = team1.players.reduce( - (acc, player) => acc + player.rank, - 0, - ); - const team2TotalRank = team2.players.reduce( - (acc, player) => acc + player.rank, - 0, - ); - const lobbyTotalRank = lobby.players.reduce( - (acc, player) => acc + player.rank, - 0, - ); + // top up the claimed pool in preference order. selectMatchCandidates + // returns every lobby rather than just the window, so lobbies that fail + // to claim are replaced from the tail. + for (const lobby of selection.candidates) { + if (claimed.size >= WINDOW_CAP) { + break; + } - // Calculate what the new averages would be if we add this lobby to each team - const team1NewAvg = - (team1TotalRank + lobbyTotalRank) / - (team1.players.length + lobby.players.length); - const team2NewAvg = - (team2TotalRank + lobbyTotalRank) / - (team2.players.length + lobby.players.length); - - // Calculate current team averages - const team1CurrentAvg = - team1.players.length > 0 - ? team1TotalRank / team1.players.length - : 0; - const team2CurrentAvg = - team2.players.length > 0 - ? team2TotalRank / team2.players.length - : 0; - - const diffIfToTeam1 = Math.abs(team1NewAvg - team2CurrentAvg); - const diffIfToTeam2 = Math.abs(team1CurrentAvg - team2NewAvg); - - targetTeam = diffIfToTeam1 <= diffIfToTeam2 ? team1 : team2; - } + if (claimed.has(lobby.lobbyId)) { + continue; + } - lobbyLocks.add(lobby.lobbyId); + let acquired = false; + try { + acquired = await this.claimLobby(lobby.lobbyId, lobby); + } catch (error) { + this.logger.error(`Error claiming lobby ${lobby.lobbyId}:`, error); + untried.delete(lobby.lobbyId); + continue; + } - targetTeam.players.push(...lobby.players); - targetTeam.lobbies.push(lobby.lobbyId); + untried.delete(lobby.lobbyId); - targetTeam.avgRank = - targetTeam.players.reduce((acc, player) => acc + player.rank, 0) / - targetTeam.players.length; + if (!acquired) { + // another region is matchmaking it - we never owned it, so it must + // not be requeued here + this.logger.warn( + `Unable to acquire lobby lock for ${lobby.lobbyId} - lobby is already being processed`, + ); + continue; + } - lobbiesAdded.push(lobby.lobbyId); - } catch (error) { - this.logger.error(`Error processing lobby ${lobby.lobbyId}:`, error); - // If we acquired a lock but failed to process, release it - if (lobbyLocks.has(lobby.lobbyId)) { - await this.releaseLobbyAndRequeue(lobby.lobbyId); - lobbyLocks.delete(lobby.lobbyId); + claimed.set(lobby.lobbyId, lobby); } - } - } - for (const lobbyId of lobbiesAdded) { - const lobbyIndex = lobbies.findIndex( - (lobby) => lobby.lobbyId === lobbyId, - ); - if (lobbyIndex !== -1) { - lobbies.splice(lobbyIndex, 1); - } - } + // pure from here until the confirmation, so the teams we pick are + // guaranteed to still be ours + const claimedPool = [...claimed.values()]; + const balanced = + balanceTeams(claimedPool, requiredPlayers) ?? + // the anchor itself may be what makes the split impossible + balanceTeams(claimedPool, requiredPlayers, { pinAnchor: false }); - let totalPlayerNotQueued = 0; - // check if we have valid teams for this match - if ( - team1.players.length === playersPerTeam && - team2.players.length === playersPerTeam - ) { - try { - // lobby locks will be released after confimrmation - for (const lobbyId of [...team1.lobbies, ...team2.lobbies]) { - lobbyLocks.delete(lobbyId); + if (!balanced) { + this.logger.warn( + `${type}/${region}: no valid split among ${claimed.size} claimed lobbies`, + ); + return remainingPlayers; } - await this.createMatchConfirmation(region, type, { - team1, - team2, - }); - } catch (error) { - this.logger.error(`Error creating match confirmation:`, error); - // Release all locks if match confirmation fails - for (const lobbyId of [...team1.lobbies, ...team2.lobbies]) { - await this.releaseLobbyAndRequeue(lobbyId); + this.logger.log( + `${type}/${region} matched: elo diff ${balanced.avgRankDifference.toFixed( + 1, + )} (win probability ${winProbability( + balanced.avgRankDifference, + ).toFixed(3)}), spread ${balanced.spread}, cost ${balanced.cost.toFixed( + 1, + )}, ${balanced.nodesVisited} nodes, optimal ${balanced.exhausted}`, + ); + + const matched = [...balanced.team1, ...balanced.team2]; + const team1 = toMatchmakingTeam(balanced.team1); + const team2 = toMatchmakingTeam(balanced.team2); + + // the confirmation re-ttls these locks and owns them from here + for (const lobby of matched) { + claimed.delete(lobby.lobbyId); } - totalPlayerNotQueued = team1.players.length + team2.players.length; - } - } else { - totalPlayerNotQueued = team1.players.length + team2.players.length; - // Release all acquired locks since we can't create a match - for (const lobbyId of [...team1.lobbies, ...team2.lobbies]) { - await this.releaseLobbyAndRequeue(lobbyId); - } - } - // only try to re-matchmake lobbies that we were able to accuire a lock for - const lobbiesToMatch = lobbies.filter((lobby) => - lobbyLocks.has(lobby.lobbyId), - ); - if (lobbiesToMatch.length > 0) { - for (const lobby of lobbiesToMatch) { - await this.releaseLobbyAndRequeue(lobby.lobbyId); + try { + await this.createMatchConfirmation(region, type, { team1, team2 }); + } catch (error) { + this.logger.error(`Error creating match confirmation:`, error); + // ownership comes back to us, so the settle path requeues them + for (const lobby of matched) { + claimed.set(lobby.lobbyId, lobby); + } + return remainingPlayers; + } } - await this.createMatches(region, type, lobbiesToMatch); - } - - // Safety check: ensure all remaining locks are released - if (lobbyLocks.size > 0) { - for (const lobbyId of lobbyLocks) { - await this.releaseLobbyAndRequeue(lobbyId); + } finally { + // single settle path - every lobby we still hold is requeued exactly + // once, even if the loop threw + for (const lobbyId of claimed.keys()) { + try { + await this.releaseLobbyAndRequeue(lobbyId); + } catch (error) { + this.logger.error(`Failed to requeue lobby ${lobbyId}:`, error); + } } } - - return totalPlayerNotQueued; } private async aquireMatchmakeRegionLock(region: string): Promise { @@ -781,11 +714,9 @@ export class MatchmakeService { await this.sendRegionStats(); if (shouldMatchmake) { - // randomize the time to prevent all regions from matchingmake at the same time - setTimeout( - () => { - void this.matchmake(type, region); - }, + await this.scheduleExpandedSearch( + type, + region, Math.floor(Math.random() * 10000), ); } diff --git a/src/matchmaking/matchmaking-lobby.service.spec.ts b/src/matchmaking/matchmaking-lobby.service.spec.ts index 9e0c1367e..60417e373 100644 --- a/src/matchmaking/matchmaking-lobby.service.spec.ts +++ b/src/matchmaking/matchmaking-lobby.service.spec.ts @@ -280,3 +280,108 @@ describe("MatchmakingLobbyService.verifyLobby", () => { ).rejects.toThrow("banned-player is banned"); }); }); + +/** + * The elo snapshot taken when a lobby joins the queue. get_player_elo returns a + * jsonb blob whose per-type value is SQL NULL for a player with no rated games, + * and which has no key at all for Premier/Faceit. Number(null) is 0 and + * Number(undefined) is NaN, so an unguarded read either seeds a brand new + * player at rank 0 or poisons every average in the queue with NaN. + */ +describe("MatchmakingLobbyService.setLobbyDetails", () => { + let service: MatchmakingLobbyService; + let mockHasura: jest.Mocked; + let stored: Record; + + const buildLobby = (steamIds: string[]) => ({ + id: "lobby-1", + players: steamIds.map((steam_id) => ({ + steam_id, + is_banned: false, + matchmaking_cooldown: false, + })), + }); + + const queuedRanks = () => + JSON.parse(stored.details).players.map( + (player: { rank: number }) => player.rank, + ); + + const setup = (elo: Record[]) => { + stored = {}; + mockHasura = { + query: jest.fn().mockResolvedValue({ + players: elo.map((value, index) => ({ + steam_id: `steam-${index + 1}`, + elo: value, + })), + }), + } as any; + + const mockRedisManager = { + getConnection: jest.fn().mockReturnValue({ + hset: jest.fn(async (_key: string, field: string, value: string) => { + stored[field] = value; + return 1; + }), + } as unknown as Redis), + } as any; + + service = new MatchmakingLobbyService( + new Logger("Test"), + mockHasura, + mockRedisManager, + {} as MatchmakeService, + ); + }; + + it("uses the rated elo when the player has one", async () => { + setup([{ competitive: 4200 }]); + + await service.setLobbyDetails(["us-east"], "Competitive", buildLobby(["steam-1"])); + + expect(queuedRanks()).toEqual([4200]); + }); + + it("defaults an unrated player to 5000 rather than 0", async () => { + // get_player_elo_by_type returns NULL for a player with no player_elo rows + setup([{ competitive: null }]); + + await service.setLobbyDetails(["us-east"], "Competitive", buildLobby(["steam-1"])); + + expect(queuedRanks()).toEqual([5000]); + }); + + it("defaults to 5000 when the type is missing from the blob", async () => { + setup([{ competitive: 4200 }]); + + await service.setLobbyDetails(["us-east"], "Premier", buildLobby(["steam-1"])); + + expect(queuedRanks()).toEqual([5000]); + }); + + it("defaults to 5000 when the player has no elo row at all", async () => { + setup([]); + + await service.setLobbyDetails(["us-east"], "Competitive", buildLobby(["steam-1"])); + + expect(queuedRanks()).toEqual([5000]); + }); + + it("keeps avgRank finite when some of the party is unrated", async () => { + setup([{ competitive: 3000 }, { competitive: null }, { competitive: null }]); + + await service.setLobbyDetails( + ["us-east"], + "Competitive", + buildLobby(["steam-1", "steam-2", "steam-3"]), + ); + + const details = JSON.parse(stored.details); + expect(details.players.map((p: { rank: number }) => p.rank)).toEqual([ + 3000, 5000, 5000, + ]); + expect(Number.isFinite(details.avgRank)).toBe(true); + expect(details.avgRank).toBeCloseTo(13000 / 3, 9); + }); +}); diff --git a/src/matchmaking/matchmaking-lobby.service.ts b/src/matchmaking/matchmaking-lobby.service.ts index 8bfd23b7a..3a3efccab 100644 --- a/src/matchmaking/matchmaking-lobby.service.ts +++ b/src/matchmaking/matchmaking-lobby.service.ts @@ -15,6 +15,7 @@ import { getMatchmakingLobbyDetailsCacheKey, } from "./utilities/cacheKeys"; import { JoinQueueError } from "./utilities/joinQueueError"; +import { DEFAULT_ELO } from "./utilities/matchmakingTuning"; import { ExpectedPlayers } from "src/discord-bot/enums/ExpectedPlayers"; @Injectable() @@ -158,12 +159,17 @@ export class MatchmakingLobbyService { const eloMap = new Map(players.map((p) => [p.steam_id, p.elo])); const _players = lobby.players.map(({ steam_id }) => { - const playerElo = eloMap.get(steam_id); - let elo = 5000; - if (playerElo) { - elo = Number(playerElo[type.toLowerCase()]); - } - return { steam_id, rank: elo }; + // get_player_elo returns SQL NULL for a player with no player_elo rows, + // and has no key at all for Premier/Faceit. Number(null) is 0 and + // Number(undefined) is NaN, so both have to fall back explicitly. + const raw = eloMap.get(steam_id)?.[type.toLowerCase()]; + const elo = Number(raw); + return { + steam_id, + rank: raw === null || raw === undefined || !Number.isFinite(elo) + ? DEFAULT_ELO + : elo, + }; }); const matchmakingLobby: MatchmakingLobby = { diff --git a/src/matchmaking/matchmaking.module.ts b/src/matchmaking/matchmaking.module.ts index 1aa8383a3..46a986d85 100644 --- a/src/matchmaking/matchmaking.module.ts +++ b/src/matchmaking/matchmaking.module.ts @@ -15,6 +15,7 @@ import { MatchmakingQueues } from "./enums/MatchmakingQueues"; import { CancelMatchMaking } from "./jobs/CancelMatchMaking"; import { MatchmakingController } from "./matchmaking.controller"; import { MarkPlayerOffline } from "./jobs/MarkPlayerOffline"; +import { ExpandMatchmaking } from "./jobs/ExpandMatchmaking"; @Module({ imports: [ @@ -37,6 +38,7 @@ import { MarkPlayerOffline } from "./jobs/MarkPlayerOffline"; MatchmakingLobbyService, CancelMatchMaking, MarkPlayerOffline, + ExpandMatchmaking, ...getQueuesProcessors("Matchmaking"), loggerFactory(), ], diff --git a/src/matchmaking/testing/fakeRedis.ts b/src/matchmaking/testing/fakeRedis.ts new file mode 100644 index 000000000..e0ef6df6b --- /dev/null +++ b/src/matchmaking/testing/fakeRedis.ts @@ -0,0 +1,219 @@ +/** + * In-memory stand-in for the subset of redis the matchmaking service uses. + * + * Matchmaking's correctness is mostly about queue state - a lobby must never be + * in two matches, and must never fall out of the queue without being matched. + * Asserting that against `jest.fn()` mocks only proves which calls were made, + * not what the queue ended up looking like, so the tests drive this instead. + */ +export class FakeRedis { + private strings = new Map(); + private hashes = new Map>(); + private sortedSets = new Map>(); + + // lock ttls are load bearing in matchmaking - a lobby lock that never expires + // means that lobby can never be matched again - so expiry is modelled against + // a virtual clock the tests advance explicitly + private expiries = new Map(); + private clock = 0; + + public published: Array<{ channel: string; message: string }> = []; + + advanceTime(milliseconds: number) { + this.clock += milliseconds; + } + + private expireIfDue(key: string) { + const deadline = this.expiries.get(key); + if (deadline !== undefined && deadline <= this.clock) { + this.expiries.delete(key); + this.strings.delete(key); + this.hashes.delete(key); + this.sortedSets.delete(key); + return true; + } + return false; + } + + private zset(key: string) { + let set = this.sortedSets.get(key); + if (!set) { + set = new Map(); + this.sortedSets.set(key, set); + } + return set; + } + + private hash(key: string) { + let hash = this.hashes.get(key); + if (!hash) { + hash = new Map(); + this.hashes.set(key, hash); + } + return hash; + } + + async set(key: string, value: unknown, ...args: unknown[]) { + this.expireIfDue(key); + + const nx = args.some( + (arg) => typeof arg === "string" && arg.toUpperCase() === "NX", + ); + + if (nx && this.strings.has(key)) { + return null; + } + + const exIndex = args.findIndex( + (arg) => typeof arg === "string" && arg.toUpperCase() === "EX", + ); + + this.strings.set(key, String(value)); + + if (exIndex !== -1) { + this.expiries.set(key, this.clock + Number(args[exIndex + 1]) * 1000); + } + + return "OK"; + } + + async get(key: string) { + this.expireIfDue(key); + return this.strings.get(key) ?? null; + } + + async del(key: string) { + this.expiries.delete(key); + const existed = + this.strings.delete(key) || + this.hashes.delete(key) || + this.sortedSets.delete(key); + return existed ? 1 : 0; + } + + async expire(key: string, seconds: number) { + this.expireIfDue(key); + + if (seconds === 0) { + return (await this.del(key)) as number; + } + + if (!this.strings.has(key) && !this.hashes.has(key)) { + return 0; + } + + this.expiries.set(key, this.clock + seconds * 1000); + return 1; + } + + private zadds = 0; + + async zadd(key: string, score: number, member: string) { + this.zadds++; + const set = this.zset(key); + const isNew = !set.has(member); + set.set(member, score); + return isNew ? 1 : 0; + } + + /** Total zadds issued, for asserting the queue is not churned needlessly. */ + zaddCount() { + return this.zadds; + } + + async zrem(key: string, member: string) { + return this.zset(key).delete(member) ? 1 : 0; + } + + async zcard(key: string) { + return this.zset(key).size; + } + + async zrange(key: string, start: number, stop: number, withScores?: string) { + const entries = [...this.zset(key).entries()].sort((a, b) => + a[1] !== b[1] ? a[1] - b[1] : a[0].localeCompare(b[0]), + ); + + const end = stop === -1 ? entries.length : stop + 1; + const sliced = entries.slice(start, end); + + if (withScores?.toUpperCase() === "WITHSCORES") { + return sliced.flatMap(([member, score]) => [member, String(score)]); + } + + return sliced.map(([member]) => member); + } + + // ioredis accepts both hset(key, field, value) and hset(key, {field: value}) + async hset(key: string, field: string | Record, value?: unknown) { + const hash = this.hash(key); + + if (typeof field === "object") { + for (const [name, entry] of Object.entries(field)) { + hash.set(name, String(entry)); + } + return Object.keys(field).length; + } + + hash.set(field, String(value)); + return 1; + } + + async hget(key: string, field: string) { + return this.hash(key).get(field) ?? null; + } + + async hgetall(key: string) { + return Object.fromEntries(this.hash(key)); + } + + async hdel(key: string, field: string) { + return this.hash(key).delete(field) ? 1 : 0; + } + + async publish(channel: string, message: string) { + this.published.push({ channel, message }); + return 1; + } + + /** + * The redis EVAL command, not javascript eval. The lua source is ignored, not + * interpreted - this hardcodes the one script matchmaking runs + * (CLAIM_LOBBY_SCRIPT): SET NX the lock, and on success ZREM the lobby from + * every queue key passed in. Atomic here by virtue of being synchronous, + * which is the property the real script buys with lua. + */ + async eval(_script: string, numKeys: number, ...args: unknown[]) { + const keys = args.slice(0, numKeys) as string[]; + const [member, ttl] = args.slice(numKeys) as [string, number]; + + const acquired = await this.set(keys[0], 1, "EX", ttl, "NX"); + if (!acquired) { + return 0; + } + + for (const key of keys.slice(1)) { + await this.zrem(key, member); + } + + return 1; + } + + // --- test helpers + + members(key: string) { + return [...this.zset(key).keys()].sort(); + } + + has(key: string) { + return this.strings.has(key) || this.hashes.has(key); + } + + keys(prefix: string) { + return [ + ...this.strings.keys(), + ...this.hashes.keys(), + ...this.sortedSets.keys(), + ].filter((key) => key.startsWith(prefix)); + } +} diff --git a/src/matchmaking/types/BalancedTeams.ts b/src/matchmaking/types/BalancedTeams.ts new file mode 100644 index 000000000..f8038b67f --- /dev/null +++ b/src/matchmaking/types/BalancedTeams.ts @@ -0,0 +1,27 @@ +import { MatchmakingLobby } from "./MatchmakingLobby"; + +export interface BalancedTeams { + team1: MatchmakingLobby[]; + team2: MatchmakingLobby[]; + /** + * Candidates this match did not take. Informational only - it is NOT the set + * to requeue. createMatches keeps unused lobbies claimed so it can carve the + * next match out of them, and tracks requeue ownership itself so that the + * "no valid split" path (where there is no BalancedTeams at all) is covered + * by the same bookkeeping. + */ + unused: MatchmakingLobby[]; + avgRankDifference: number; + spread: number; + cost: number; + nodesVisited: number; + // false when the search hit the node budget or bailed early on a good enough + // split, so the result is the best seen rather than provably optimal + exhausted: boolean; +} + +export interface MatchCandidates { + anchor: MatchmakingLobby; + // anchor first, then every other lobby in preference order + candidates: MatchmakingLobby[]; +} diff --git a/src/matchmaking/utilities/balanceTeams.spec.ts b/src/matchmaking/utilities/balanceTeams.spec.ts new file mode 100644 index 000000000..1bf3fd1d8 --- /dev/null +++ b/src/matchmaking/utilities/balanceTeams.spec.ts @@ -0,0 +1,659 @@ +import { e_match_types_enum } from "generated"; +import { MatchmakingLobby } from "../types/MatchmakingLobby"; +import { + balanceTeams, + canFillTeams, + greedyAssign, + matchCost, +} from "./balanceTeams"; +import { TIE_EPSILON } from "./matchmakingTuning"; + +const NOW = 1_700_000_000_000; +const COMPETITIVE: e_match_types_enum = "Competitive"; + +let nextLobbyId = 0; + +function lobby( + ranks: number[], + overrides?: { id?: string; waitSeconds?: number; type?: e_match_types_enum }, +): MatchmakingLobby { + const waitSeconds = overrides?.waitSeconds ?? 30; + + return { + type: overrides?.type ?? COMPETITIVE, + regions: ["us-east"], + joinedAt: new Date(NOW - waitSeconds * 1000), + lobbyId: overrides?.id ?? `lobby-${++nextLobbyId}`, + players: ranks.map((rank, index) => ({ + steam_id: `${overrides?.id ?? `lobby-${nextLobbyId}`}-p${index}`, + rank, + })), + regionPositions: {}, + avgRank: ranks.reduce((acc, rank) => acc + rank, 0) / ranks.length, + }; +} + +function solos(ranks: number[]): MatchmakingLobby[] { + return ranks.map((rank) => lobby([rank])); +} + +function teamRanks(team: MatchmakingLobby[]): number[] { + return team.flatMap((entry) => entry.players.map((player) => player.rank)); +} + +function average(values: number[]): number { + return values.reduce((acc, value) => acc + value, 0) / values.length; +} + +function greedyGap( + candidates: MatchmakingLobby[], + requiredPlayers: number, +): number | null { + const result = greedyAssign(candidates, requiredPlayers); + if (!result) { + return null; + } + return Math.abs( + average(teamRanks(result.team1)) - average(teamRanks(result.team2)), + ); +} + +// deterministic prng so the fuzz cases are reproducible +function mulberry32(seed: number) { + let state = seed; + return () => { + state |= 0; + state = (state + 0x6d2b79f5) | 0; + let t = Math.imul(state ^ (state >>> 15), 1 | state); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +const options = { now: NOW, random: () => 0 }; + +describe("balanceTeams", () => { + beforeEach(() => { + nextLobbyId = 0; + }); + + describe("beats the previous greedy assignment", () => { + it("finds a perfect split where greedy is off by 1120 elo", () => { + // total 48000, so a 24000/24000 split exists: + // 6000 5800 5600 5400 1200 vs 5200 5000 4800 4600 4400 + // greedy places the 1200 last and can never move it. + const ranks = [ + 6000, 5800, 5600, 5400, 5200, 5000, 4800, 4600, 4400, 1200, + ]; + const candidates = solos(ranks); + + expect(greedyGap(candidates, 10)).toBe(1120); + + const result = balanceTeams(candidates, 10, options); + + expect(result).not.toBeNull(); + expect(result.avgRankDifference).toBe(0); + expect(result.team1.length + result.team2.length).toBe(10); + expect(result.unused).toHaveLength(0); + }); + + it("skips a lobby entirely, which greedy has no move for", () => { + // the duo of 9000s poisons any team it lands on; the right answer is to + // leave it in the queue and play the ten 5000s. + const anchor = lobby([5000], { id: "anchor" }); + const duo = lobby([9000, 9000], { id: "duo" }); + const rest = solos(new Array(12).fill(5000)); + const candidates = [anchor, duo, ...rest]; + + const greedy = greedyGap(candidates, 10); + expect(greedy).toBeGreaterThan(0); + + const result = balanceTeams(candidates, 10, options); + + expect(result.avgRankDifference).toBe(0); + expect(result.spread).toBe(0); + expect(result.unused.map((entry) => entry.lobbyId)).toContain("duo"); + expect(teamRanks([...result.team1, ...result.team2])).not.toContain(9000); + }); + + it("never returns a costlier match than greedy across random queues", () => { + const random = mulberry32(20260726); + let compared = 0; + + for (let iteration = 0; iteration < 500; iteration++) { + nextLobbyId = 0; + const lobbyCount = 3 + Math.floor(random() * 14); + const candidates: MatchmakingLobby[] = []; + + for (let i = 0; i < lobbyCount; i++) { + const size = 1 + Math.floor(random() * 5); + const ranks = Array.from( + { length: size }, + () => 2000 + Math.floor(random() * 7000), + ); + candidates.push(lobby(ranks, { waitSeconds: 10 })); + } + + const greedy = greedyAssign(candidates, 10); + const result = balanceTeams(candidates, 10, { + now: NOW, + random: () => 0, + }); + + // greedy has no anchor concept, so only compare when it happened to + // keep the anchor - otherwise the two are solving different problems + if ( + !greedy || + ![...greedy.team1, ...greedy.team2].includes(candidates[0]) + ) { + continue; + } + + compared++; + expect(result).not.toBeNull(); + expect(result.cost).toBeLessThanOrEqual( + matchCost(greedy.team1, greedy.team2, 10, NOW).cost + 1e-9, + ); + } + + expect(compared).toBeGreaterThan(50); + }); + + it("reports a cost that matches the split it returned", () => { + const candidates = solos([ + 6000, 5800, 5600, 5400, 5200, 5000, 4800, 4600, 4400, 1200, + ]); + const result = balanceTeams(candidates, 10, options); + + expect(result.cost).toBeCloseTo( + matchCost(result.team1, result.team2, 10, NOW).cost, + 9, + ); + }); + }); + + describe("constraints", () => { + it("keeps parties together and on opposite teams when forced", () => { + // two trios plus four solos: the only legal shape puts one trio on each + // side, 18000+10000 vs 12000+10000 + const candidates = [ + lobby([6000, 6000, 6000], { id: "trio-high" }), + lobby([4000, 4000, 4000], { id: "trio-low" }), + ...solos([5000, 5000, 5000, 5000]), + ]; + + const result = balanceTeams(candidates, 10, options); + + expect(result).not.toBeNull(); + expect(result.team1).toHaveLength(3); + expect(result.team2).toHaveLength(3); + expect(result.avgRankDifference).toBe(1200); + + const high = [...result.team1, ...result.team2].find( + (entry) => entry.lobbyId === "trio-high", + ); + const low = [...result.team1, ...result.team2].find( + (entry) => entry.lobbyId === "trio-low", + ); + expect(high).toBeDefined(); + expect(low).toBeDefined(); + expect(result.team1.includes(high)).not.toBe(result.team1.includes(low)); + }); + + it("returns null when party sizes cannot fill two lineups", () => { + // five duos is ten players, but 2s cannot sum to 5 + const candidates = [ + lobby([5000, 5000]), + lobby([5000, 5000]), + lobby([5000, 5000]), + lobby([5000, 5000]), + lobby([5000, 5000]), + ]; + + expect(canFillTeams([2, 2, 2, 2, 2], 10)).toBe(false); + expect(balanceTeams(candidates, 10, options)).toBeNull(); + }); + + it("always plays the anchor", () => { + const random = mulberry32(7); + + for (let iteration = 0; iteration < 200; iteration++) { + nextLobbyId = 0; + const candidates: MatchmakingLobby[] = []; + for (let i = 0; i < 6 + Math.floor(random() * 8); i++) { + const size = 1 + Math.floor(random() * 5); + candidates.push( + lobby( + Array.from( + { length: size }, + () => 2000 + Math.floor(random() * 7000), + ), + { waitSeconds: 10 }, + ), + ); + } + + const result = balanceTeams(candidates, 10, { + now: NOW, + random: () => 0, + }); + if (!result) { + continue; + } + + const played = [...result.team1, ...result.team2]; + expect(played).toContain(candidates[0]); + } + }); + + it("holds its invariants across random queues", () => { + const random = mulberry32(99); + + for (let iteration = 0; iteration < 300; iteration++) { + nextLobbyId = 0; + const candidates: MatchmakingLobby[] = []; + for (let i = 0; i < 4 + Math.floor(random() * 12); i++) { + const size = 1 + Math.floor(random() * 5); + candidates.push( + lobby( + Array.from( + { length: size }, + () => 2000 + Math.floor(random() * 7000), + ), + { waitSeconds: 10 }, + ), + ); + } + + const result = balanceTeams(candidates, 10, { + now: NOW, + random: () => 0, + }); + if (!result) { + continue; + } + + expect(teamRanks(result.team1)).toHaveLength(5); + expect(teamRanks(result.team2)).toHaveLength(5); + + const played = [...result.team1, ...result.team2]; + expect(new Set(played).size).toBe(played.length); + expect(new Set([...played, ...result.unused]).size).toBe( + candidates.length, + ); + + const steamIds = played.flatMap((entry) => + entry.players.map((player) => player.steam_id), + ); + expect(new Set(steamIds).size).toBe(10); + } + }); + }); + + describe("objective", () => { + it("rejects an equal-average split made of extremes", () => { + // {10000,10000,2000,2000,2000} vs {5200 x5} both average 5200, but the + // spread term prices it out in favour of the tight cohort + const candidates = [ + lobby([5200], { id: "anchor" }), + ...solos(new Array(9).fill(5200)), + ...solos([10000, 10000, 2000, 2000, 2000]), + ]; + + const result = balanceTeams(candidates, 10, options); + + expect(result.avgRankDifference).toBe(0); + expect(result.spread).toBe(0); + const played = teamRanks([...result.team1, ...result.team2]); + expect(played).not.toContain(10000); + expect(played).not.toContain(2000); + }); + + it("balances wingman (2v2)", () => { + const candidates = solos([6000, 5000, 5000, 4000]); + + expect(greedyGap(candidates, 4)).toBe(1000); + + const result = balanceTeams(candidates, 4, options); + + expect(result.avgRankDifference).toBe(0); + expect(result.team1).toHaveLength(2); + expect(result.team2).toHaveLength(2); + }); + + it("balances duel (1v1) and leaves the mismatch queued", () => { + const candidates = [ + lobby([5000], { id: "anchor" }), + lobby([5100], { id: "close" }), + lobby([8000], { id: "far" }), + ]; + + const result = balanceTeams(candidates, 2, options); + + expect(result.avgRankDifference).toBe(100); + expect(result.unused.map((entry) => entry.lobbyId)).toEqual(["far"]); + }); + }); + + describe("variety", () => { + const compositionOf = (result: { team1: MatchmakingLobby[] }) => + result.team1 + .map((entry) => entry.lobbyId) + .sort() + .join(","); + + it("does not hand the same ten players the same teams every time", () => { + const random = mulberry32(1234); + const candidates = solos(new Array(10).fill(5000)); + const seen = new Set(); + + for (let i = 0; i < 200; i++) { + seen.add( + compositionOf(balanceTeams(candidates, 10, { now: NOW, random })), + ); + } + + // 126 distinct splits exist for ten interchangeable players. a fixed + // candidate order would return the same one every time, which is what a + // recurring group would experience as "always the same teams". + expect(seen.size).toBeGreaterThan(50); + }); + + it("finds every perfectly balanced split once ratings have spread out", () => { + const random = mulberry32(99); + const candidates = [ + ...solos([6200, 5900, 5600]), + ...solos([5100, 5000, 5000, 4900]), + ...solos([4400, 4200, 3900]), + ]; + const seen = new Set(); + + for (let i = 0; i < 200; i++) { + const result = balanceTeams(candidates, 10, { now: NOW, random }); + seen.add(compositionOf(result)); + expect(result.avgRankDifference).toBe(0); + } + + // exhaustive enumeration of this rating set says exactly 5 of the 126 + // partitions are dead even. variety here is bounded by the ratings, not + // by the search - and it reaches all 5 rather than settling on one. + expect(seen.size).toBe(5); + }); + + it("splits the strongest players up instead of stacking them", () => { + const random = mulberry32(5); + const strong = ["s0", "s1", "s2"]; + const candidates = [ + ...strong.map((id) => lobby([7000], { id })), + ...solos(new Array(7).fill(4000)), + ]; + + for (let i = 0; i < 100; i++) { + const result = balanceTeams(candidates, 10, { now: NOW, random }); + const onTeam1 = new Set(result.team1.map((entry) => entry.lobbyId)); + const stacked = strong.filter((id) => onTeam1.has(id)).length; + + // three strong on one side averages 5200 vs 4000; two versus one + // averages 4800 vs 4400, so the balancer always breaks them up + expect(stacked === 0 || stacked === 3).toBe(false); + } + }); + + it("keeps mixing the winners as their ratings pull away", () => { + // the three strongest keep winning and the three weakest keep losing, so + // the spread widens every round. the teams should keep churning rather + // than settling into a fixed lineup. + const random = mulberry32(2026); + const elo = new Map( + Array.from({ length: 10 }, (_, i) => [`p${i}`, 5000]), + ); + const winners = new Set(["p0", "p1", "p2"]); + const losers = new Set(["p7", "p8", "p9"]); + + const seen = new Set(); + let stackedRounds = 0; + let widestGap = 0; + + for (let round = 0; round < 40; round++) { + const candidates = [...elo.entries()].map(([id, rank]) => + lobby([rank], { id }), + ); + + const result = balanceTeams(candidates, 10, { now: NOW, random }); + expect(result).not.toBeNull(); + + seen.add(compositionOf(result)); + widestGap = Math.max(widestGap, result.avgRankDifference); + + const onTeam1 = new Set(result.team1.map((entry) => entry.lobbyId)); + const together = [...winners].filter((id) => onTeam1.has(id)).length; + if (together === 0 || together === 3) { + stackedRounds++; + } + + for (const id of elo.keys()) { + if (winners.has(id)) { + elo.set(id, elo.get(id) + 40); + } else if (losers.has(id)) { + elo.set(id, elo.get(id) - 40); + } + } + } + + // ratings really did diverge over the run + expect(elo.get("p0") - elo.get("p9")).toBe(3200); + + // and the balancer kept producing different lineups rather than locking in + expect(seen.size).toBeGreaterThan(20); + expect(stackedRounds).toBe(0); + + // while still keeping every one of those games close + expect(widestGap).toBeLessThan(200); + }); + + it("rotates which of the top three play together as the field pulls apart", () => { + // the three strongest win every game and the three weakest lose every one + // at a different rate, so the ratings spread unevenly and dead even splits + // stop existing. counting lineups is not enough here - the ratings change + // every round, so even a fixed candidate order produces new lineups. what + // matters is whether the same two of the top three keep landing together. + const winners = ["p0", "p1", "p2"]; + const losers = new Set(["p7", "p8", "p9"]); + const pairs = [ + ["p0", "p1"], + ["p0", "p2"], + ["p1", "p2"], + ]; + const rounds = 40; + + for (const seed of [2026, 7, 555, 31337]) { + const random = mulberry32(seed); + const elo = new Map( + Array.from({ length: 10 }, (_, i) => [`p${i}`, 5000]), + ); + + const pairedRounds = new Map(pairs.map(([a, b]) => [`${a}|${b}`, 0])); + const teammates = new Map(winners.map((id) => [id, new Set()])); + const seen = new Set(); + let longestRepeat = 0; + let repeat = 0; + let previousCarve = ""; + let widestGap = 0; + + for (let round = 0; round < rounds; round++) { + const candidates = [...elo.entries()].map(([id, rank]) => + lobby([rank], { id }), + ); + + const result = balanceTeams(candidates, 10, { now: NOW, random }); + expect(result).not.toBeNull(); + + const onTeam1 = new Set(result.team1.map((entry) => entry.lobbyId)); + const sideOf = (id: string) => (onTeam1.has(id) ? 1 : 2); + + seen.add(compositionOf(result)); + widestGap = Math.max(widestGap, result.avgRankDifference); + + for (const [a, b] of pairs) { + if (sideOf(a) === sideOf(b)) { + const key = `${a}|${b}`; + pairedRounds.set(key, pairedRounds.get(key) + 1); + } + } + + for (const id of winners) { + for (const other of elo.keys()) { + if (other !== id && sideOf(other) === sideOf(id)) { + teammates.get(id).add(other); + } + } + } + + // which of the top three sat together, independent of side + const carve = winners + .map((id) => (sideOf(id) === sideOf(winners[0]) ? "a" : "b")) + .join(""); + repeat = carve === previousCarve ? repeat + 1 : 1; + previousCarve = carve; + longestRepeat = Math.max(longestRepeat, repeat); + + for (const id of elo.keys()) { + if (winners.includes(id)) { + elo.set(id, elo.get(id) + 40); + } else if (losers.has(id)) { + elo.set(id, elo.get(id) - 25); + } + } + } + + // the field really did pull apart, and unevenly + expect(elo.get("p0") - elo.get("p9")).toBe(2600); + + // none of the top three ends up with a fixed circle - each one plays + // alongside every other player at some point over the run + for (const id of winners) { + expect(teammates.get(id).size).toBe(9); + } + + // an even rotation puts each pair together a third of the time. the + // bounds are loose, but a pair that has calcified sits at one extreme: + // before the tie-break was randomised one pair shared a team 36 rounds + // out of 40 while another never shared one at all. + for (const together of pairedRounds.values()) { + expect(together).toBeGreaterThan(3); + expect(together).toBeLessThan(28); + } + + // and no single carve of the top three survives for long + expect(longestRepeat).toBeLessThanOrEqual(12); + expect(seen.size).toBeGreaterThanOrEqual(18); + + // the spread is absorbed rather than pushed into the scoreline + expect(widestGap).toBeLessThan(150); + } + }); + + it("collects alternatives before bailing out on a tightly packed field", () => { + // every rating here is distinct but only 60 elo separates first from + // last, so no two players sit the same distance from the reference and + // the random tie-break never gets a tie to break. the only thing keeping + // this group off identical teams is the search collecting alternatives + // instead of returning the first good split it walks into - bailing on + // the first leaf pins this exact field to one lineup, on every seed. + const tight = [5021, 5015, 5011, 5003, 4996, 4988, 4983, 4975, 4968, 4961]; + + for (const seed of [1, 99, 2026]) { + const random = mulberry32(seed); + const candidates = tight.map((rank, index) => + lobby([rank], { id: `p${index}` }), + ); + const seen = new Set(); + let widestGap = 0; + + for (let i = 0; i < 200; i++) { + const result = balanceTeams(candidates, 10, { now: NOW, random }); + seen.add(compositionOf(result)); + widestGap = Math.max(widestGap, result.avgRankDifference); + } + + expect(seen.size).toBeGreaterThanOrEqual(12); + + // the variety is not bought with worse games. these alternatives are + // all inside TIE_EPSILON, so the widest is a rounding error next to + // the 60 elo the field already spans + expect(widestGap).toBeLessThanOrEqual(TIE_EPSILON); + } + }); + }); + + describe("search bounds", () => { + it("falls back to the greedy seed when the node budget is exhausted", () => { + const candidates = solos([ + 6000, 5800, 5600, 5400, 5200, 5000, 4800, 4600, 4400, 1200, + ]); + + const result = balanceTeams(candidates, 10, { + now: NOW, + random: () => 0, + nodeBudget: 1, + }); + + expect(result).not.toBeNull(); + expect(result.exhausted).toBe(false); + expect(result.team1).toHaveLength(5); + expect(result.team2).toHaveLength(5); + // still no worse than today's behaviour + expect(result.avgRankDifference).toBeLessThanOrEqual(1120); + }); + + it("stays inside the node budget on a full window of solo lobbies", () => { + const random = mulberry32(4242); + const candidates = Array.from({ length: 16 }, () => + lobby([2000 + Math.floor(random() * 7000)], { waitSeconds: 10 }), + ); + + const result = balanceTeams(candidates, 10, { + now: NOW, + random: () => 0, + }); + + expect(result).not.toBeNull(); + expect(result.nodesVisited).toBeLessThanOrEqual(150_000); + }); + + it("is deterministic when the tie-break rng is fixed", () => { + const candidates = solos([ + 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, + ]); + + const runs = Array.from({ length: 10 }, () => + balanceTeams(candidates, 10, { now: NOW, random: () => 0 }) + .team1.map((entry) => entry.lobbyId) + .sort() + .join(","), + ); + + expect(new Set(runs).size).toBe(1); + }); + }); +}); + +describe("canFillTeams", () => { + it.each([ + [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 10, true], + [[5, 5], 10, true], + [[3, 2, 3, 2], 10, true], + [[2, 2, 2, 2, 2], 10, false], + [[3, 3, 3, 3], 10, false], + [[4, 4, 4], 10, false], + [[4, 4, 1, 1], 10, true], + [[3, 2, 4, 1], 10, true], + [[1, 1, 1, 1], 4, true], + [[2, 2], 4, true], + [[1, 1], 2, true], + ])("canFillTeams(%p, %i) === %p", (sizes, requiredPlayers, expected) => { + expect(canFillTeams(sizes as number[], requiredPlayers as number)).toBe( + expected, + ); + }); +}); diff --git a/src/matchmaking/utilities/balanceTeams.ts b/src/matchmaking/utilities/balanceTeams.ts new file mode 100644 index 000000000..73e8fd9ee --- /dev/null +++ b/src/matchmaking/utilities/balanceTeams.ts @@ -0,0 +1,554 @@ +import { BalancedTeams } from "../types/BalancedTeams"; +import { MatchmakingLobby } from "../types/MatchmakingLobby"; +import { joinedAtMillis } from "./selectMatchCandidates"; +import { + EARLY_EXIT_COST, + NODE_BUDGET, + SPREAD_FREE_BAND, + SPREAD_WEIGHT, + TIE_EPSILON, + WAIT_SATURATION_SECONDS, + WAIT_WEIGHT, +} from "./matchmakingTuning"; + +const UNUSED = 0; +const TEAM_1 = 1; +const TEAM_2 = 2; + +const MAX_TIED_SOLUTIONS = 16; + +export interface BalanceTeamsOptions { + now?: number; + nodeBudget?: number; + random?: () => number; + // candidates[0] is the longest waiting lobby and is forced into the match + pinAnchor?: boolean; +} + +interface LobbyStats { + size: number; + rankSum: number; + minRank: number; + maxRank: number; + waitMillis: number; +} + +/** + * Can these lobby sizes be split into two teams of exactly requiredPlayers / 2, + * given that a lobby can never be split across teams? + */ +export function canFillTeams(sizes: number[], requiredPlayers: number): boolean { + const half = requiredPlayers / 2; + + if (!Number.isInteger(half) || half <= 0) { + return false; + } + + const width = half + 1; + let reach = new Uint8Array(width * width); + reach[0] = 1; + + for (const size of sizes) { + if (size <= 0 || size > half) { + continue; + } + + const next = Uint8Array.from(reach); + for (let a = 0; a <= half; a++) { + for (let b = 0; b <= half; b++) { + if (!reach[a * width + b]) { + continue; + } + if (a + size <= half) { + next[(a + size) * width + b] = 1; + } + if (b + size <= half) { + next[a * width + b + size] = 1; + } + } + } + reach = next; + } + + return reach[half * width + half] === 1; +} + +/** + * The pre-existing greedy assignment, kept so the search can be seeded with it. + * That seed is what guarantees the new balancer is never worse than the old + * behaviour, even when the node budget truncates the search - do not remove it. + */ +export function greedyAssign( + candidates: MatchmakingLobby[], + requiredPlayers: number, +): { team1: MatchmakingLobby[]; team2: MatchmakingLobby[] } | null { + const half = requiredPlayers / 2; + const team1: MatchmakingLobby[] = []; + const team2: MatchmakingLobby[] = []; + + let count1 = 0; + let count2 = 0; + let sum1 = 0; + let sum2 = 0; + + for (const lobby of candidates) { + const size = lobby.players.length; + const hasRoom1 = count1 + size <= half; + const hasRoom2 = count2 + size <= half; + + if (!hasRoom1 && !hasRoom2) { + continue; + } + + const lobbySum = lobby.players.reduce((acc, player) => acc + player.rank, 0); + + let target: typeof TEAM_1 | typeof TEAM_2; + if (!hasRoom1) { + target = TEAM_2; + } else if (!hasRoom2) { + target = TEAM_1; + } else { + const new1 = (sum1 + lobbySum) / (count1 + size); + const new2 = (sum2 + lobbySum) / (count2 + size); + const current1 = count1 > 0 ? sum1 / count1 : 0; + const current2 = count2 > 0 ? sum2 / count2 : 0; + target = + Math.abs(new1 - current2) <= Math.abs(current1 - new2) + ? TEAM_1 + : TEAM_2; + } + + if (target === TEAM_1) { + team1.push(lobby); + count1 += size; + sum1 += lobbySum; + } else { + team2.push(lobby); + count2 += size; + sum2 += lobbySum; + } + } + + if (count1 !== half || count2 !== half) { + return null; + } + + return { team1, team2 }; +} + +/** The objective, over a finished match. Exported so callers and tests score the same way. */ +export function matchCost( + team1: MatchmakingLobby[], + team2: MatchmakingLobby[], + requiredPlayers: number, + now: number = Date.now(), +) { + const half = requiredPlayers / 2; + const played = [...team1, ...team2]; + const ranks = played.flatMap((lobby) => + lobby.players.map((player) => player.rank), + ); + const sumOf = (team: MatchmakingLobby[]) => + team.reduce( + (acc, lobby) => + acc + lobby.players.reduce((total, player) => total + player.rank, 0), + 0, + ); + + const avgRankDifference = Math.abs(sumOf(team1) - sumOf(team2)) / half; + const spread = Math.max(...ranks) - Math.min(...ranks); + const quality = + avgRankDifference + + SPREAD_WEIGHT * Math.max(0, spread - SPREAD_FREE_BAND); + + const waitMillis = played.reduce( + (acc, lobby) => + acc + + Math.max(0, now - joinedAtMillis(lobby.joinedAt)) * lobby.players.length, + 0, + ); + const avgWaitSeconds = waitMillis / requiredPlayers / 1000; + const wait = + WAIT_WEIGHT * + (1 - Math.min(1, Math.max(0, avgWaitSeconds / WAIT_SATURATION_SECONDS))); + + return { cost: quality + wait, quality, avgRankDifference, spread }; +} + +/** + * Chooses which of `candidates` play and how they split into two even teams. + * + * Both teams always end up with exactly h = requiredPlayers / 2 players, so + * |avg(team1) - avg(team2)| reduces to |sum1 - sum2| / h and the primary + * objective is an integer subset-difference problem. Each lobby independently + * goes to team1, team2 or unused, which means one search picks the cohort and + * the split together - the two cannot be separated, because the spread term + * only distinguishes cohorts while the balance term only distinguishes splits. + * + * Pure and synchronous. Returns null when no exact partition exists. + */ +export function balanceTeams( + candidates: MatchmakingLobby[], + requiredPlayers: number, + options?: BalanceTeamsOptions, +): BalancedTeams | null { + const half = requiredPlayers / 2; + + if (!Number.isInteger(half) || half <= 0) { + return null; + } + + const now = options?.now ?? Date.now(); + const nodeBudget = options?.nodeBudget ?? NODE_BUDGET; + const random = options?.random ?? Math.random; + const pinAnchor = options?.pinAnchor ?? true; + + const usable = candidates.filter( + (lobby) => lobby.players.length > 0 && lobby.players.length <= half, + ); + + if (!canFillTeams( + usable.map((lobby) => lobby.players.length), + requiredPlayers, + )) { + return null; + } + + const anchorId = pinAnchor ? candidates.at(0)?.lobbyId : undefined; + const anchor = anchorId + ? usable.find((lobby) => lobby.lobbyId === anchorId) + : undefined; + + // depth first search only finds good cohorts early if the lobbies that pair + // best with the anchor come first. outliers sort to the tail, where the + // "leave it queued" branch gets reached long before the node budget runs out. + const reference = anchor?.avgRank ?? usable.at(0)?.avgRank ?? 0; + + // equally good candidates are ordered randomly rather than by id. the search + // returns the first optimal split it reaches, so a fixed order would hand the + // same ten players the same teams every night. injecting a constant rng makes + // this a stable sort, and therefore deterministic, for tests. + const jitter = new Map(usable.map((lobby) => [lobby.lobbyId, random()])); + + const order = [...usable].sort((a, b) => { + if (a === anchor) { + return -1; + } + if (b === anchor) { + return 1; + } + + const rankDiff = + Math.abs(a.avgRank - reference) - Math.abs(b.avgRank - reference); + return rankDiff !== 0 + ? rankDiff + : jitter.get(a.lobbyId) - jitter.get(b.lobbyId); + }); + + const size = order.length; + const anchorIndex = anchor ? order.indexOf(anchor) : -1; + + const stats: LobbyStats[] = order.map((lobby) => { + const ranks = lobby.players.map((player) => player.rank); + return { + size: lobby.players.length, + rankSum: ranks.reduce((acc, rank) => acc + rank, 0), + minRank: Math.min(...ranks), + maxRank: Math.max(...ranks), + waitMillis: Math.max(0, now - joinedAtMillis(lobby.joinedAt)), + }; + }); + + // --- suffix precomputation. the candidate order is fixed, so the set of + // lobbies still available at depth d is always exactly order[d..size-1]. + const width = half + 1; + const suffixPlayers = new Array(size + 1).fill(0); + const suffixReach: Uint8Array[] = new Array(size + 1); + const suffixTopSum: number[][] = new Array(size + 1); + const suffixBottomSum: number[][] = new Array(size + 1); + + suffixReach[size] = new Uint8Array(width * width); + suffixReach[size][0] = 1; + suffixTopSum[size] = new Array(half + 1).fill(0); + suffixBottomSum[size] = new Array(half + 1).fill(0); + + const suffixRanks: number[] = []; + for (let d = size - 1; d >= 0; d--) { + suffixPlayers[d] = suffixPlayers[d + 1] + stats[d].size; + + const previous = suffixReach[d + 1]; + const reach = Uint8Array.from(previous); + for (let a = 0; a <= half; a++) { + for (let b = 0; b <= half; b++) { + if (!previous[a * width + b]) { + continue; + } + if (a + stats[d].size <= half) { + reach[(a + stats[d].size) * width + b] = 1; + } + if (b + stats[d].size <= half) { + reach[a * width + b + stats[d].size] = 1; + } + } + } + suffixReach[d] = reach; + + suffixRanks.push(...order[d].players.map((player) => player.rank)); + suffixRanks.sort((a, b) => a - b); + + const top = new Array(half + 1).fill(0); + const bottom = new Array(half + 1).fill(0); + for (let k = 1; k <= half; k++) { + top[k] = + k <= suffixRanks.length + ? top[k - 1] + suffixRanks[suffixRanks.length - k] + : top[k - 1]; + bottom[k] = + k <= suffixRanks.length ? bottom[k - 1] + suffixRanks[k - 1] : bottom[k - 1]; + } + suffixTopSum[d] = top; + suffixBottomSum[d] = bottom; + } + + // balance + spread. the part of the objective that measures how good the + // game itself is, which is what the early exit is judged on. + const quality = ( + sum1: number, + sum2: number, + minRank: number, + maxRank: number, + ) => + Math.abs(sum1 - sum2) / half + + SPREAD_WEIGHT * Math.max(0, maxRank - minRank - SPREAD_FREE_BAND); + + const cost = ( + sum1: number, + sum2: number, + minRank: number, + maxRank: number, + waitMillis: number, + ) => { + const avgWaitSeconds = waitMillis / requiredPlayers / 1000; + const wait = + WAIT_WEIGHT * + (1 - Math.min(1, Math.max(0, avgWaitSeconds / WAIT_SATURATION_SECONDS))); + + return quality(sum1, sum2, minRank, maxRank) + wait; + }; + + const assign = new Uint8Array(size); + let reservoir: Uint8Array[] = []; + let bestCost = Number.POSITIVE_INFINITY; + let bestQuality = Number.POSITIVE_INFINITY; + let nodesVisited = 0; + let exhausted = true; + let stop = false; + + const seed = greedyAssign(order, requiredPlayers); + if (seed) { + const seeded = new Uint8Array(size); + let sum1 = 0; + let sum2 = 0; + let minRank = Number.POSITIVE_INFINITY; + let maxRank = Number.NEGATIVE_INFINITY; + let waitMillis = 0; + + for (let i = 0; i < size; i++) { + const inTeam1 = seed.team1.includes(order[i]); + const inTeam2 = seed.team2.includes(order[i]); + if (!inTeam1 && !inTeam2) { + continue; + } + seeded[i] = inTeam1 ? TEAM_1 : TEAM_2; + if (inTeam1) { + sum1 += stats[i].rankSum; + } else { + sum2 += stats[i].rankSum; + } + minRank = Math.min(minRank, stats[i].minRank); + maxRank = Math.max(maxRank, stats[i].maxRank); + waitMillis += stats[i].waitMillis * stats[i].size; + } + + // the greedy pass has no concept of an anchor, so its answer is only a + // usable seed when it happens to include one + if (anchorIndex < 0 || seeded[anchorIndex] !== UNUSED) { + bestCost = cost(sum1, sum2, minRank, maxRank, waitMillis); + bestQuality = quality(sum1, sum2, minRank, maxRank); + reservoir = [seeded]; + } + } + + const record = ( + sum1: number, + sum2: number, + minRank: number, + maxRank: number, + waitMillis: number, + ) => { + const candidateCost = cost(sum1, sum2, minRank, maxRank, waitMillis); + + if (candidateCost < bestCost - TIE_EPSILON) { + bestCost = candidateCost; + bestQuality = quality(sum1, sum2, minRank, maxRank); + reservoir = [Uint8Array.from(assign)]; + } else if (candidateCost <= bestCost + TIE_EPSILON) { + if (candidateCost < bestCost) { + bestCost = candidateCost; + bestQuality = quality(sum1, sum2, minRank, maxRank); + } + if (reservoir.length < MAX_TIED_SOLUTIONS) { + reservoir.push(Uint8Array.from(assign)); + } + } + + // judged on quality, not cost - the wait term is a per-cohort offset that + // would otherwise make this fire always or never. holding off until the + // reservoir is full matters: the greedy seed can already be optimal, and + // bailing on the first leaf would leave nothing to pick between. + if (bestQuality <= EARLY_EXIT_COST && reservoir.length >= MAX_TIED_SOLUTIONS) { + exhausted = false; + stop = true; + } + }; + + const search = ( + depth: number, + count1: number, + count2: number, + sum1: number, + sum2: number, + minRank: number, + maxRank: number, + waitMillis: number, + ) => { + if (stop) { + return; + } + + if (++nodesVisited > nodeBudget) { + exhausted = false; + stop = true; + return; + } + + if (count1 === half && count2 === half) { + record(sum1, sum2, minRank, maxRank, waitMillis); + return; + } + + if (depth === size) { + return; + } + + const need1 = half - count1; + const need2 = half - count2; + + if (suffixPlayers[depth] < need1 + need2) { + return; + } + + if (!suffixReach[depth][need1 * width + need2]) { + return; + } + + // admissible lower bound: the widest and narrowest final rank gap still + // reachable, ignoring party atomicity, plus the spread already committed. + // the wait term is always >= 0 so leaving it out only loosens the bound. + const delta = sum1 - sum2; + const maxDelta = + delta + suffixTopSum[depth][need1] - suffixBottomSum[depth][need2]; + const minDelta = + delta + suffixBottomSum[depth][need1] - suffixTopSum[depth][need2]; + const reachableGap = + minDelta <= 0 && maxDelta >= 0 + ? 0 + : Math.min(Math.abs(minDelta), Math.abs(maxDelta)); + + const committedSpread = + maxRank > minRank + ? SPREAD_WEIGHT * Math.max(0, maxRank - minRank - SPREAD_FREE_BAND) + : 0; + + if (reachableGap / half + committedSpread >= bestCost + TIE_EPSILON) { + return; + } + + const stat = stats[depth]; + // nothing assigned yet: putting the first lobby on team2 just mirrors + // team1, so only explore one of them + const mirrored = count1 === 0 && count2 === 0; + const sides = + mirrored || delta <= 0 ? [TEAM_1, TEAM_2] : [TEAM_2, TEAM_1]; + + for (const side of sides) { + if (mirrored && side === TEAM_2) { + continue; + } + if (side === TEAM_1 && count1 + stat.size > half) { + continue; + } + if (side === TEAM_2 && count2 + stat.size > half) { + continue; + } + + assign[depth] = side; + search( + depth + 1, + side === TEAM_1 ? count1 + stat.size : count1, + side === TEAM_2 ? count2 + stat.size : count2, + side === TEAM_1 ? sum1 + stat.rankSum : sum1, + side === TEAM_2 ? sum2 + stat.rankSum : sum2, + Math.min(minRank, stat.minRank), + Math.max(maxRank, stat.maxRank), + waitMillis + stat.waitMillis * stat.size, + ); + assign[depth] = UNUSED; + } + + // the anchor has to play, so it never gets the unused branch + if (depth === anchorIndex) { + return; + } + + search(depth + 1, count1, count2, sum1, sum2, minRank, maxRank, waitMillis); + }; + + search( + 0, + 0, + 0, + 0, + 0, + Number.POSITIVE_INFINITY, + Number.NEGATIVE_INFINITY, + 0, + ); + + if (reservoir.length === 0) { + return null; + } + + const chosen = + reservoir[Math.min(reservoir.length - 1, Math.floor(random() * reservoir.length))]; + + const team1 = order.filter((_, index) => chosen[index] === TEAM_1); + const team2 = order.filter((_, index) => chosen[index] === TEAM_2); + const used = new Set([...team1, ...team2]); + const unused = candidates.filter((lobby) => !used.has(lobby)); + + // scored from the split actually returned, which may be any of the tied + // solutions rather than the one that last set bestCost + const scored = matchCost(team1, team2, requiredPlayers, now); + + return { + team1, + team2, + unused, + avgRankDifference: scored.avgRankDifference, + spread: scored.spread, + cost: scored.cost, + nodesVisited, + exhausted, + }; +} diff --git a/src/matchmaking/utilities/matchmakingTuning.ts b/src/matchmaking/utilities/matchmakingTuning.ts new file mode 100644 index 000000000..f190cd6e5 --- /dev/null +++ b/src/matchmaking/utilities/matchmakingTuning.ts @@ -0,0 +1,38 @@ +// mirrors _default_elo / _scale_factor in hasura/functions/match/match_player_elo.sql +export const DEFAULT_ELO = 5000; +export const ELO_SCALE_FACTOR = 4000; + +// cost weights - every term is in "elo points of team average difference" +export const SPREAD_WEIGHT = 0.15; +export const SPREAD_FREE_BAND = 1000; +export const WAIT_WEIGHT = 25; +export const WAIT_SATURATION_SECONDS = 120; + +// splits within TIE_EPSILON of each other are treated as equally good and +// picked between at random, so a recurring group doesn't get the same teams +// every night +export const TIE_EPSILON = 10; +export const EARLY_EXIT_COST = 5; + +// max lobbies claimed and searched per match, and the hard ceiling on search +// nodes. 16 lobbies is ~630k leaves unpruned for 5v5, roughly 10ms of cpu. +export const WINDOW_CAP = 16; +export const NODE_BUDGET = 150_000; + +// the acceptable rank gap to the longest waiting lobby, widening the longer +// they have been queued +export const RANK_WINDOW_BASE = 500; +export const RANK_WINDOW_GROWTH_PER_SECOND = 25; +export const RANK_WINDOW_MAX = 4000; + +export function getRankWindow(waitSeconds: number) { + return Math.min( + RANK_WINDOW_MAX, + RANK_WINDOW_BASE + RANK_WINDOW_GROWTH_PER_SECOND * Math.max(0, waitSeconds), + ); +} + +/** Expected win rate for the stronger team, using the same curve as the elo engine. */ +export function winProbability(avgRankDifference: number) { + return 1 / (1 + Math.pow(10, -Math.abs(avgRankDifference) / ELO_SCALE_FACTOR)); +} diff --git a/src/matchmaking/utilities/selectMatchCandidates.spec.ts b/src/matchmaking/utilities/selectMatchCandidates.spec.ts new file mode 100644 index 000000000..04135a380 --- /dev/null +++ b/src/matchmaking/utilities/selectMatchCandidates.spec.ts @@ -0,0 +1,149 @@ +import { e_match_types_enum } from "generated"; +import { MatchmakingLobby } from "../types/MatchmakingLobby"; +import { selectMatchCandidates } from "./selectMatchCandidates"; +import { RANK_WINDOW_BASE } from "./matchmakingTuning"; + +const NOW = 1_700_000_000_000; +const COMPETITIVE: e_match_types_enum = "Competitive"; + +function lobby( + id: string, + ranks: number[], + waitSeconds = 10, +): MatchmakingLobby { + return { + type: COMPETITIVE, + regions: ["us-east"], + joinedAt: new Date(NOW - waitSeconds * 1000), + lobbyId: id, + players: ranks.map((rank, index) => ({ + steam_id: `${id}-p${index}`, + rank, + })), + regionPositions: {}, + avgRank: ranks.reduce((acc, rank) => acc + rank, 0) / ranks.length, + }; +} + +function solos(count: number, rank: number, waitSeconds = 10) { + return Array.from({ length: count }, (_, index) => + lobby(`solo-${index}`, [rank], waitSeconds), + ); +} + +describe("selectMatchCandidates", () => { + it("anchors on the longest waiting lobby", () => { + const oldest = lobby("oldest", [5000], 300); + const result = selectMatchCandidates( + [...solos(10, 5000, 10), oldest], + 10, + { now: NOW }, + ); + + expect(result.anchor).toBe(oldest); + expect(result.candidates[0]).toBe(oldest); + }); + + it("breaks equal wait times on lobby id so the order is stable", () => { + const a = lobby("aaa", [5000], 10); + const b = lobby("bbb", [5000], 10); + const result = selectMatchCandidates([b, a, ...solos(9, 5000)], 10, { + now: NOW, + }); + + expect(result.anchor.lobbyId).toBe("aaa"); + }); + + it("orders the rest by rank proximity to the anchor", () => { + const anchor = lobby("anchor", [5000], 300); + const near = lobby("near", [5100]); + const mid = lobby("mid", [5300]); + const far = lobby("far", [5450]); + + const result = selectMatchCandidates( + [far, mid, near, anchor, ...solos(7, 5000)], + 10, + { now: NOW }, + ); + + const ordered = result.candidates.map((entry) => entry.lobbyId); + expect(ordered[0]).toBe("anchor"); + expect(ordered.indexOf("near")).toBeLessThan(ordered.indexOf("mid")); + expect(ordered.indexOf("mid")).toBeLessThan(ordered.indexOf("far")); + }); + + it("drops lobbies too large to fit a lineup", () => { + const oversized = lobby("oversized", [5000, 5000, 5000, 5000, 5000, 5000]); + const result = selectMatchCandidates( + [oversized, ...solos(10, 5000)], + 10, + { now: NOW }, + ); + + expect(result.candidates.map((entry) => entry.lobbyId)).not.toContain( + "oversized", + ); + }); + + it("returns null when the queue cannot reach a full match", () => { + expect(selectMatchCandidates(solos(9, 5000), 10, { now: NOW })).toBeNull(); + }); + + it("enforces the rank window when it can still fill a match", () => { + const anchor = lobby("anchor", [5000], 10); + const smurf = lobby("smurf", [9000], 10); + const result = selectMatchCandidates( + [anchor, smurf, ...solos(10, 5000)], + 10, + { now: NOW }, + ); + + expect(Math.abs(smurf.avgRank - anchor.avgRank)).toBeGreaterThan( + RANK_WINDOW_BASE, + ); + expect(result.candidates.map((entry) => entry.lobbyId)).not.toContain( + "smurf", + ); + }); + + it("widens the window the longer the anchor has waited", () => { + const anchor = lobby("anchor", [5000], 300); + const distant = lobby("distant", [7000], 10); + const result = selectMatchCandidates( + [anchor, distant, ...solos(10, 5000)], + 10, + { now: NOW }, + ); + + // 300s of waiting opens the window past a 2000 point gap + expect(result.candidates.map((entry) => entry.lobbyId)).toContain( + "distant", + ); + }); + + it("demotes rather than drops out of window lobbies on a thin queue", () => { + const anchor = lobby("anchor", [5000], 10); + const distant = lobby("distant", [9000], 10); + const result = selectMatchCandidates( + [anchor, distant, ...solos(8, 5000)], + 10, + { now: NOW }, + ); + + // only 10 players exist, so the queue plays rather than stalling + expect(result.candidates).toHaveLength(10); + expect(result.candidates.at(-1).lobbyId).toBe("distant"); + }); + + it("returns null on a thin queue when the window is hard enforced", () => { + const anchor = lobby("anchor", [5000], 10); + const distant = lobby("distant", [9000], 10); + + expect( + selectMatchCandidates([anchor, distant, ...solos(8, 5000)], 10, { + now: NOW, + softWindow: false, + }), + ).toBeNull(); + }); +}); diff --git a/src/matchmaking/utilities/selectMatchCandidates.ts b/src/matchmaking/utilities/selectMatchCandidates.ts new file mode 100644 index 000000000..447dc017f --- /dev/null +++ b/src/matchmaking/utilities/selectMatchCandidates.ts @@ -0,0 +1,95 @@ +import { MatchmakingLobby } from "../types/MatchmakingLobby"; +import { MatchCandidates } from "../types/BalancedTeams"; +import { getRankWindow } from "./matchmakingTuning"; + +export interface SelectMatchCandidatesOptions { + now?: number; + // when false, out of window lobbies are dropped even if that means no match + softWindow?: boolean; +} + +/** joinedAt comes back off redis as a string, despite the type on MatchmakingLobby. */ +export function joinedAtMillis(joinedAt: Date | string) { + const millis = + joinedAt instanceof Date + ? joinedAt.getTime() + : new Date(joinedAt).getTime(); + + return Number.isFinite(millis) ? millis : 0; +} + +function totalPlayers(lobbies: MatchmakingLobby[]) { + return lobbies.reduce((acc, lobby) => acc + lobby.players.length, 0); +} + +/** + * Picks the anchor - the longest waiting lobby, which must end up in the match + * so nobody starves at the tail of the queue - and orders every other lobby by + * how well it pairs with that anchor. + * + * Returns the full ordered list rather than a truncated window; the caller + * claims from the front and stops at WINDOW_CAP, so lobbies that fail to claim + * are topped up from the tail. + */ +export function selectMatchCandidates( + lobbies: MatchmakingLobby[], + requiredPlayers: number, + options?: SelectMatchCandidatesOptions, +): MatchCandidates | null { + const now = options?.now ?? Date.now(); + const softWindow = options?.softWindow ?? true; + const playersPerTeam = requiredPlayers / 2; + + // a lobby larger than a lineup can never fit; one that fills the whole match + // was already handled by the self split path + const usable = lobbies.filter( + (lobby) => + lobby.players.length > 0 && lobby.players.length <= playersPerTeam, + ); + + if (totalPlayers(usable) < requiredPlayers) { + return null; + } + + const [anchor] = [...usable].sort((a, b) => { + const waitDiff = joinedAtMillis(a.joinedAt) - joinedAtMillis(b.joinedAt); + return waitDiff !== 0 ? waitDiff : a.lobbyId.localeCompare(b.lobbyId); + }); + + const window = getRankWindow((now - joinedAtMillis(anchor.joinedAt)) / 1000); + + const rest = usable + .filter((lobby) => lobby.lobbyId !== anchor.lobbyId) + .sort((a, b) => { + const rankDiff = + Math.abs(a.avgRank - anchor.avgRank) - + Math.abs(b.avgRank - anchor.avgRank); + if (rankDiff !== 0) { + return rankDiff; + } + + const waitDiff = joinedAtMillis(a.joinedAt) - joinedAtMillis(b.joinedAt); + return waitDiff !== 0 ? waitDiff : a.lobbyId.localeCompare(b.lobbyId); + }); + + const inWindow = rest.filter( + (lobby) => Math.abs(lobby.avgRank - anchor.avgRank) <= window, + ); + + // enforce the window whenever it can still produce a match, otherwise demote + // the out of window lobbies instead of dropping them so a thin queue still + // plays rather than waiting for the window to widen + if (totalPlayers([anchor, ...inWindow]) >= requiredPlayers) { + return { anchor, candidates: [anchor, ...inWindow] }; + } + + if (!softWindow) { + return null; + } + + const outOfWindow = rest.filter( + (lobby) => Math.abs(lobby.avgRank - anchor.avgRank) > window, + ); + + return { anchor, candidates: [anchor, ...inWindow, ...outOfWindow] }; +} diff --git a/src/matchmaking/utilities/shuffleSplit.spec.ts b/src/matchmaking/utilities/shuffleSplit.spec.ts new file mode 100644 index 000000000..863e41007 --- /dev/null +++ b/src/matchmaking/utilities/shuffleSplit.spec.ts @@ -0,0 +1,45 @@ +import { shuffleSplit } from "./shuffleSplit"; + +describe("shuffleSplit", () => { + const players = Array.from({ length: 10 }, (_, i) => ({ id: `p${i}` })); + + it("splits evenly and uses every item exactly once", () => { + const [team1, team2] = shuffleSplit(players); + + expect(team1).toHaveLength(5); + expect(team2).toHaveLength(5); + expect(new Set([...team1, ...team2]).size).toBe(10); + }); + + it("does not mutate the input", () => { + const snapshot = [...players]; + shuffleSplit(players); + expect(players).toEqual(snapshot); + }); + + it("splits a wingman party", () => { + const [team1, team2] = shuffleSplit(players.slice(0, 4)); + expect(team1).toHaveLength(2); + expect(team2).toHaveLength(2); + }); + + it("is uniform, unlike the sort based shuffle it replaced", () => { + // sort(() => Math.random() - 0.5) leaves items near where they started, so + // p0 lands on team 1 far more often than half the time + const counts = new Map(); + const runs = 20000; + + for (let i = 0; i < runs; i++) { + const [team1] = shuffleSplit(players); + for (const player of team1) { + counts.set(player.id, (counts.get(player.id) ?? 0) + 1); + } + } + + // every player should land on team 1 about half the time + for (const player of players) { + expect(counts.get(player.id) / runs).toBeGreaterThan(0.45); + expect(counts.get(player.id) / runs).toBeLessThan(0.55); + } + }); +}); diff --git a/src/matchmaking/utilities/shuffleSplit.ts b/src/matchmaking/utilities/shuffleSplit.ts new file mode 100644 index 000000000..20041f1b4 --- /dev/null +++ b/src/matchmaking/utilities/shuffleSplit.ts @@ -0,0 +1,20 @@ +/** + * Fisher-Yates shuffle, then an even cut. Replaces `sort(() => Math.random() - 0.5)`, + * which is not a uniform shuffle - it biases toward leaving items near where + * they started, so a party would keep getting similar teams. + */ +export function shuffleSplit( + items: T[], + random: () => number = Math.random, +): [T[], T[]] { + const shuffled = [...items]; + + for (let i = shuffled.length - 1; i > 0; i--) { + const j = Math.floor(random() * (i + 1)); + [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]]; + } + + const half = Math.floor(shuffled.length / 2); + + return [shuffled.slice(0, half), shuffled.slice(half)]; +} diff --git a/tsconfig.build.json b/tsconfig.build.json index 8b763ad9b..4079dce4f 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -1,4 +1,11 @@ { "extends": "./tsconfig.json", - "exclude": ["node_modules", "test", "dist", "**/*spec.ts", "generated"] + "exclude": [ + "node_modules", + "test", + "dist", + "**/*spec.ts", + "**/testing/**", + "generated" + ] }