From 40fb632db1e44da909b3eb5e25326e3e84fb7cd9 Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Mon, 10 Aug 2026 22:34:48 +0200 Subject: [PATCH 1/4] feat(lobby): add mesh retries and gate QuickMatch startup Retry incomplete full-mesh checks with correlated attempts and gate QuickMatch startup on a successful mesh result. Preserve legacy client compatibility and requeue players when automatic setup fails. --- GenOnlineService/Constants.cs | 27 +- .../WebSocket/WebSocketController.cs | 22 +- GenOnlineService/LobbyManager.cs | 229 ++++++++++++++- GenOnlineService/MatchmakingManager.cs | 269 ++++++++++++++---- 4 files changed, 456 insertions(+), 91 deletions(-) diff --git a/GenOnlineService/Constants.cs b/GenOnlineService/Constants.cs index 06f02cc..6d570b0 100644 --- a/GenOnlineService/Constants.cs +++ b/GenOnlineService/Constants.cs @@ -2543,9 +2543,11 @@ public enum EWebSocketMessageID SOCIAL_CANT_ADD_FRIEND_LIST_FULL = 38, PROBE_RESP = 39, AC_REGISTER_PLAYER = 40, - AC_DEREGISTER_PLAYER = 41, - WS_KEEPALIVE = 42, - WS_KEEPALIVE_CLIENT = 43 + AC_DEREGISTER_PLAYER = 41, + WS_KEEPALIVE = 42, + WS_KEEPALIVE_CLIENT = 43, + MATCHMAKING_ACTION_REQUEUE = 44, + MATCHMAKING_ACTION_SETUP_PROGRESS = 45 }; public static class UserPresence @@ -2653,8 +2655,16 @@ public class WebSocketMessage_NameChange : WebSocketMessage public string name { get; set; } = String.Empty; } - public class WebSocketMessage_FullMeshConnectivityCheckResponseFromUser : WebSocketMessage - { + public class WebSocketMessage_FullMeshConnectivityCheckRequest : WebSocketMessage + { + public Int64 mesh_check_id { get; set; } + public int attempt { get; set; } + } + + public class WebSocketMessage_FullMeshConnectivityCheckResponseFromUser : WebSocketMessage + { + public Int64 mesh_check_id { get; set; } + public int attempt { get; set; } public List connectivity_map { get; set; } = new(); } @@ -2829,4 +2839,9 @@ public class WebSocketMessage_MatchmakerStartGame : WebSocketMessage } -} \ No newline at end of file + public class WebSocketMessage_MatchmakerSetupProgress : WebSocketMessage + { + public int timeout_ms { get; set; } + } + +} diff --git a/GenOnlineService/Controllers/WebSocket/WebSocketController.cs b/GenOnlineService/Controllers/WebSocket/WebSocketController.cs index cd56f79..7175e0d 100644 --- a/GenOnlineService/Controllers/WebSocket/WebSocketController.cs +++ b/GenOnlineService/Controllers/WebSocket/WebSocketController.cs @@ -843,25 +843,7 @@ private async Task ProcessWSMessage(UserWebSocketInstance sourceWS, UserSession lobbyInfo.StartFullMeshConnectivityCheck(); // start full mesh connectivity checks - WebSocketMessage_Simple startCommand = new WebSocketMessage_Simple(); - startCommand.msg_id = (int)EWebSocketMessageID.FULL_MESH_CONNECTIVITY_CHECK_RESPONSE; - - // Serialize once before broadcasting - byte[] bytesJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(startCommand)); - - foreach (LobbyMember lobbyMember in lobbyInfo.Members) - { - if (lobbyMember != null) - { - if (lobbyMember.GetSession().TryGetTarget(out UserSession? sess)) - { - if (sess != null) - { - sess.QueueWebsocketSend(bytesJSON); - } - } - } - } + lobbyInfo.SendFullMeshConnectivityCheckRequestToMembers(); } else if (msgID == EWebSocketMessageID.FULL_MESH_CONNECTIVITY_CHECK_RESPONSE) { @@ -875,7 +857,7 @@ private async Task ProcessWSMessage(UserWebSocketInstance sourceWS, UserSession Lobby? lobby = _lobbyManager.GetLobby(sourceUserSession.currentLobbyID); if (lobby != null) { - await lobby.StoreFullMeshConnectivityResponse(sourceUserSession.m_UserID, fullMeshMsg.connectivity_map); + await lobby.StoreFullMeshConnectivityResponse(sourceUserSession.m_UserID, fullMeshMsg); } } } diff --git a/GenOnlineService/LobbyManager.cs b/GenOnlineService/LobbyManager.cs index 61e5b79..6759910 100644 --- a/GenOnlineService/LobbyManager.cs +++ b/GenOnlineService/LobbyManager.cs @@ -56,9 +56,33 @@ public class Lobby [JsonIgnore] public Int64 TimeStartFullMeshChecks { get; private set; } = -1; - private const int MSToWaitForFullMeshChecks = 5000; // really shouldnt take more than 5 seconds... this might even be too much + private const int MSToWaitForFullMeshChecks = 5000; + private const int MaxFullMeshCheckAttempts = 2; + private const int MSBeforeFullMeshCheckRetry = 3000; + public const int MaxFullMeshConnectivityCheckDurationMS = + (MSToWaitForFullMeshChecks * MaxFullMeshCheckAttempts) + + (MSBeforeFullMeshCheckRetry * (MaxFullMeshCheckAttempts - 1)); + + private readonly object m_FullMeshCheckLock = new(); + + [JsonIgnore] + public bool? LastFullMeshConnectivityCheckOutcome { get; private set; } = null; [JsonIgnore] + public int FullMeshCheckAttempt { get; private set; } = 0; + + [JsonIgnore] + public Int64 FullMeshCheckID { get; private set; } = 0; + + [JsonIgnore] + private Int64 m_TimeToRetryFullMeshChecks = -1; + + [JsonIgnore] + private bool m_bCurrentAttemptHasLegacyResponse = false; + + private static Int64 s_NextFullMeshCheckID = 0; + + [JsonIgnore] public ConcurrentDictionary> FullMeshConnectivityChecks { get; set; } = new(); @@ -142,24 +166,143 @@ public async Task RegisterProbeResponse_Malformed_Type2(Int64 userID) // End AC Probes public void StartFullMeshConnectivityCheck() + { + lock (m_FullMeshCheckLock) + { + FullMeshCheckID = Interlocked.Increment(ref s_NextFullMeshCheckID); + FullMeshCheckAttempt = 1; + m_TimeToRetryFullMeshChecks = -1; + LastFullMeshConnectivityCheckOutcome = null; + BeginFullMeshConnectivityCheckAttempt(); + } + } + + private void BeginFullMeshConnectivityCheckAttempt() { PendingFullMeshConnectivityChecks = true; - FullMeshConnectivityChecks = new(); + FullMeshConnectivityChecks = new(); + m_bCurrentAttemptHasLegacyResponse = false; TimeStartFullMeshChecks = Environment.TickCount64; } - public async Task StoreFullMeshConnectivityResponse(Int64 sourceUser, List connectivityMap) + public void SendFullMeshConnectivityCheckRequestToMembers() + { + WebSocketMessage_FullMeshConnectivityCheckRequest startCommand = new WebSocketMessage_FullMeshConnectivityCheckRequest(); + startCommand.msg_id = (int)EWebSocketMessageID.FULL_MESH_CONNECTIVITY_CHECK_RESPONSE; + startCommand.mesh_check_id = FullMeshCheckID; + startCommand.attempt = FullMeshCheckAttempt; + byte[] bytesJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(startCommand)); + + foreach (LobbyMember member in Members) + { + if (member.GetSession().TryGetTarget(out UserSession? session) && session != null) + { + session.QueueWebsocketSend(bytesJSON); + } + } + } + + // Re-issues signalling between the pairs that failed to connect. This is the same handshake a player + // gets when they join, which is why manually rejoining the lobby often repairs a broken mesh. + private void RestartSignallingForMissingConnections(List lstMissingConnections) { - FullMeshConnectivityChecks[sourceUser] = new ConcurrentList(connectivityMap); + HashSet<(Int64, Int64)> alreadyResignalled = new(); - // check again for being done - await ProcessPendingFullMeshConnectivityChecks(); + foreach (MissingConnectionEntry missingConnection in lstMissingConnections) + { + Int64 lowUserID = Math.Min(missingConnection.source_user_id, missingConnection.target_user_id); + Int64 highUserID = Math.Max(missingConnection.source_user_id, missingConnection.target_user_id); + + if (!alreadyResignalled.Add((lowUserID, highUserID))) + { + continue; + } + + LobbyMember? sourceMember = GetMemberFromUserID(missingConnection.source_user_id); + LobbyMember? targetMember = GetMemberFromUserID(missingConnection.target_user_id); + + if (sourceMember == null || targetMember == null) + { + continue; + } + + Console.WriteLine("[Lobby {0}] Re-signalling {1} <-> {2} before mesh check retry", LobbyID, sourceMember.UserID, targetMember.UserID); + + SendStartSignallingToMember(sourceMember, targetMember); + SendStartSignallingToMember(targetMember, sourceMember); + } } - public async Task ProcessPendingFullMeshConnectivityChecks() + private void SendStartSignallingToMember(LobbyMember recipient, LobbyMember peer) { - // TODO: Add a timeout to this - if (PendingFullMeshConnectivityChecks) + if (recipient.GetSession().TryGetTarget(out UserSession? recipientSession) && recipientSession != null) + { + WebSocketMessage_NetworkStartSignalling signallingMsg = new WebSocketMessage_NetworkStartSignalling(); + signallingMsg.msg_id = (int)EWebSocketMessageID.NETWORK_CONNECTION_START_SIGNALLING; + signallingMsg.lobby_id = LobbyID; + signallingMsg.user_id = peer.UserID; + signallingMsg.preferred_port = peer.Port; + signallingMsg.middleware_id = peer.MiddlewareUserID; + recipientSession.QueueWebsocketSend(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(signallingMsg))); + } + } + + public Task StoreFullMeshConnectivityResponse(Int64 sourceUser, WebSocketMessage_FullMeshConnectivityCheckResponseFromUser response) + { + lock (m_FullMeshCheckLock) + { + bool bLegacyResponse = response.mesh_check_id == 0 && response.attempt == 0; + bool bMatchesCurrentAttempt = bLegacyResponse + ? FullMeshCheckAttempt == 1 + : response.mesh_check_id == FullMeshCheckID && response.attempt == FullMeshCheckAttempt; + + LobbyMember? sourceMember = GetMemberFromUserID(sourceUser); + if (PendingFullMeshConnectivityChecks + && m_TimeToRetryFullMeshChecks == -1 + && sourceMember?.IsHuman() == true + && bMatchesCurrentAttempt) + { + m_bCurrentAttemptHasLegacyResponse |= bLegacyResponse; + FullMeshConnectivityChecks[sourceUser] = new ConcurrentList(response.connectivity_map); + } + + ProcessPendingFullMeshConnectivityChecksInternal(); + } + + return Task.CompletedTask; + } + + public Task ProcessPendingFullMeshConnectivityChecks() + { + lock (m_FullMeshCheckLock) + { + ProcessPendingFullMeshConnectivityChecksInternal(); + } + + return Task.CompletedTask; + } + + private void ProcessPendingFullMeshConnectivityChecksInternal() + { + if (!PendingFullMeshConnectivityChecks) + { + return; + } + + // Give re-signalled connections time to establish before starting the retry. + if (m_TimeToRetryFullMeshChecks != -1) + { + if (Environment.TickCount64 < m_TimeToRetryFullMeshChecks) + { + return; + } + + m_TimeToRetryFullMeshChecks = -1; + BeginFullMeshConnectivityCheckAttempt(); + SendFullMeshConnectivityCheckRequestToMembers(); + return; + } + { bool bDoneChecks = false; int totalMapEntriesExpected = GetNumberOfHumans(); @@ -209,6 +352,24 @@ public async Task ProcessPendingFullMeshConnectivityChecks() } } + // A member that never reported cannot be assumed connected, so treat them as missing to everyone. + foreach (LobbyMember member in Members) + { + if (member.IsHuman() && !FullMeshConnectivityChecks.ContainsKey(member.UserID)) + { + foreach (LobbyMember otherMember in Members) + { + if (otherMember.IsHuman() && otherMember.UserID != member.UserID) + { + MissingConnectionEntry missingConnectionEntry = new(); + missingConnectionEntry.source_user_id = member.UserID; + missingConnectionEntry.target_user_id = otherMember.UserID; + lstMissingConnections.Add(missingConnectionEntry); + } + } + } + } + bool bDisableMeshCheck = false; if (Program.g_Config != null) { @@ -222,6 +383,18 @@ public async Task ProcessPendingFullMeshConnectivityChecks() + bool bMeshComplete = bDisableMeshCheck || lstMissingConnections.Count == 0; + + bool bAllMembersReportedCurrentAttempt = FullMeshConnectivityChecks.Count == totalMapEntriesExpected; + bool bCanSafelyRetry = bAllMembersReportedCurrentAttempt && !m_bCurrentAttemptHasLegacyResponse; + if (!bMeshComplete && bCanSafelyRetry && FullMeshCheckAttempt < MaxFullMeshCheckAttempts) + { + ++FullMeshCheckAttempt; + RestartSignallingForMissingConnections(lstMissingConnections); + m_TimeToRetryFullMeshChecks = Environment.TickCount64 + MSBeforeFullMeshCheckRetry; + return; + } + // inform host that we are done // start full mesh connectivity checks WebSocketMessage_FullMeshConnectivityCheckOutcome outcome = new WebSocketMessage_FullMeshConnectivityCheckOutcome(); @@ -238,6 +411,8 @@ public async Task ProcessPendingFullMeshConnectivityChecks() outcome.missing_connections = lstMissingConnections; } + LastFullMeshConnectivityCheckOutcome = outcome.mesh_complete; + // TODO_EFCORE: Later, these should really use lobby list instead of getting session from ID // send to host @@ -251,6 +426,7 @@ public async Task ProcessPendingFullMeshConnectivityChecks() // reset state PendingFullMeshConnectivityChecks = false; TimeStartFullMeshChecks = -1; + m_TimeToRetryFullMeshChecks = -1; } } } @@ -634,6 +810,8 @@ private void CalculateNextProbeTime(bool bIsFirstProbe) public async Task Tick() { + await ProcessPendingFullMeshConnectivityChecks(); + if (m_NextProbe != 0 && Environment.TickCount64 >= m_NextProbe) { // send probe @@ -915,6 +1093,37 @@ public async Task AddMember(UserSession playerSession, string strDisplayNa } } + public void SendPeerTeardownToDepartingMember(LobbyMember departingMember) + { + if (!departingMember.GetSession().TryGetTarget(out UserSession? departingSession) || departingSession == null) + { + return; + } + + foreach (LobbyMember remoteMember in Members) + { + if (remoteMember.SlotState != EPlayerType.SLOT_PLAYER || remoteMember.UserID == departingMember.UserID) + { + continue; + } + + WebSocketMessage_ACDeregisterPlayer remotePlayerAcMsg = new WebSocketMessage_ACDeregisterPlayer(); + remotePlayerAcMsg.msg_id = (int)EWebSocketMessageID.AC_DEREGISTER_PLAYER; + remotePlayerAcMsg.user_id = remoteMember.UserID; + remotePlayerAcMsg.mwid = remoteMember.MiddlewareUserID; + departingSession.QueueWebsocketSend(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(remotePlayerAcMsg))); + + if (State != ELobbyState.INGAME) + { + WebSocketMessage_NetworkDisconnectPlayer remotePlayerMsg = new WebSocketMessage_NetworkDisconnectPlayer(); + remotePlayerMsg.msg_id = (int)EWebSocketMessageID.NETWORK_CONNECTION_DISCONNECT_PLAYER; + remotePlayerMsg.lobby_id = LobbyID; + remotePlayerMsg.user_id = remoteMember.UserID; + departingSession.QueueWebsocketSend(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(remotePlayerMsg))); + } + } + } + public async Task RemoveMember(LobbyMember member) { // TODO_LOBBY: Optimize this @@ -1763,4 +1972,4 @@ public bool IsUserInLobby(Lobby lobby, Int64 user_id) return member != null; } } -} \ No newline at end of file +} diff --git a/GenOnlineService/MatchmakingManager.cs b/GenOnlineService/MatchmakingManager.cs index 72c4cab..cae1ff5 100644 --- a/GenOnlineService/MatchmakingManager.cs +++ b/GenOnlineService/MatchmakingManager.cs @@ -296,14 +296,17 @@ public class MatchmakingBucket private Int64 m_timeReachedMinPlayers = -1; private bool m_bReachedMinPlayers = false; - private bool m_bWaitingOnLobbyJoins = false; - private bool m_bHasStartedCountdown = false; - private bool m_bMergedAway = false; + private bool m_bWaitingOnLobbyJoins = false; + private bool m_bHasStartedCountdown = false; + private bool m_bWaitingOnMeshConnectivityChecks = false; + private volatile bool m_bAutoStartInvalidated = false; + private bool m_bPendingDeletion = false; + private bool m_bMergedAway = false; // a bucket that has been merged into another bucket (or that has already been handed a lobby) must never // accept or donate members again, otherwise a player ends up in two buckets and gets sent to two lobbies - public bool IsMergedAway() { return m_bMergedAway; } - public bool IsLockedForMatchStart() { return m_bHasStartedCountdown || m_bWaitingOnLobbyJoins; } + public bool IsMergedAway() { return m_bMergedAway; } + public bool IsLockedForMatchStart() { return m_bHasStartedCountdown || m_bWaitingOnLobbyJoins || m_bWaitingOnMeshConnectivityChecks || m_bPendingDeletion; } public UInt16 PlaylistID { get; private set; } public int MinPlayers { get; private set; } @@ -473,6 +476,11 @@ public bool DoMapSelectionsIntersect(ConcurrentList lstRhs) public bool CanMergeWithOtherBucket(MatchmakingBucket bucketToMerge) { + if (m_bPendingDeletion || bucketToMerge.m_bPendingDeletion) + { + return false; + } + // playlist must match if (bucketToMerge.PlaylistID != this.PlaylistID) { @@ -586,6 +594,11 @@ public bool RemovePlayer(UserSession playerSession) { if (member.Is(playerSession)) { + if (m_bWaitingOnLobbyJoins || m_bHasStartedCountdown || m_bWaitingOnMeshConnectivityChecks) + { + m_bAutoStartInvalidated = true; + } + m_lstMembers.Remove(member); return true; } @@ -599,7 +612,7 @@ public int CurrentMemberCount() return m_lstMembers.Count; } - // members whose UserSession has been collected/disconnected are dead weight - they inflate the member count, + // members whose UserSession has been collected/disconnected are dead weight - they inflate the member count, // which both blocks the "everyone joined the lobby" check and skews the average elo public void PruneDeadMembers() { @@ -608,9 +621,14 @@ public void PruneDeadMembers() if (member.GetAssociatedSession() == null) { m_lstMembers.Remove(member); - } - } - } + } + } + } + + internal void MarkPendingDeletion() + { + m_bPendingDeletion = true; + } // TODO_EFCORE: Shared User data, and session<->websocket could be weakrefs public bool IsJoiningUserBlockedByOrHasBlockedAnyBucketMember(UserSession? joiningUserSession, Int64 joining_user) @@ -644,9 +662,9 @@ public bool IsJoiningUserBlockedByOrHasBlockedAnyBucketMember(UserSession? joini } public bool HasSpaceForUsers(int numUsers, UInt32 exe_crc, UInt32 ini_crc, EKnownAnticheatID anticheatID) - { - // stale bucket that was merged into another one - if (m_bMergedAway) + { + // stale buckets must never accept new members + if (m_bMergedAway || m_bPendingDeletion) { return false; } @@ -771,16 +789,140 @@ public Int64 GetLobbyID() return m_LobbyID; } - Int64 m_LobbyID = -1; - Int64 m_StartTime = -1; - Int64 m_timeStartedWaitingOnLobbyJoins = -1; + Int64 m_LobbyID = -1; + Int64 m_StartTime = -1; + Int64 m_timeStartedWaitingOnLobbyJoins = -1; + + // how long we give everyone to actually connect to the QuickMatch lobby before we give up on the stragglers + private const Int64 c_LobbyJoinTimeoutMSec = 45000; + + private async Task StartGameAfterSuccessfulMeshCheck(Lobby lobby) + { + Console.WriteLine("START GAME"); + + WebSocketMessage_MatchmakerStartGame startGameAction = new WebSocketMessage_MatchmakerStartGame(); + startGameAction.msg_id = (int)EWebSocketMessageID.MATCHMAKING_ACTION_START_GAME; + byte[] bytesJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(startGameAction)); + + foreach (MatchmakingBucketMember member in m_lstMembers) + { + UserSession? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + memberSession.QueueWebsocketSend(bytesJSON); + } + } + + await lobby.UpdateState(ELobbyState.INGAME); + MatchmakingManager.DestroyBucket(this); + } - // how long we give everyone to actually connect to the QuickMatch lobby before we give up on the stragglers - private const Int64 c_LobbyJoinTimeoutMSec = 45000; - public async Task Tick() + private async Task TriggerFullMeshConnectivityChecks(Lobby lobby) { - // merged into another bucket - our members live there now, ticking would create a second lobby for them - if (m_bMergedAway) + lobby.StartFullMeshConnectivityCheck(); + + const int MeshCheckClientTimeoutMarginMS = 2000; + WebSocketMessage_MatchmakerSetupProgress setupProgress = new WebSocketMessage_MatchmakerSetupProgress(); + setupProgress.msg_id = (int)EWebSocketMessageID.MATCHMAKING_ACTION_SETUP_PROGRESS; + setupProgress.timeout_ms = Lobby.MaxFullMeshConnectivityCheckDurationMS + MeshCheckClientTimeoutMarginMS; + byte[] setupProgressJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(setupProgress)); + + foreach (MatchmakingBucketMember member in m_lstMembers) + { + UserSession? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + memberSession.QueueWebsocketSend(setupProgressJSON); + } + } + + lobby.SendFullMeshConnectivityCheckRequestToMembers(); + + foreach (MatchmakingBucketMember member in m_lstMembers) + { + UserSession? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + await SendMatchmakingMessage(memberSession, "Running full mesh connectivity checks before game start..."); + } + } + + m_bWaitingOnMeshConnectivityChecks = true; + } + + private async Task AbortQuickMatchAutoStart(string reason) + { + List sessionsToRequeue = new(); + foreach (MatchmakingBucketMember member in m_lstMembers) + { + UserSession? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + sessionsToRequeue.Add(memberSession); + } + } + + m_StartTime = -1; + m_bWaitingOnLobbyJoins = false; + m_bHasStartedCountdown = false; + m_bWaitingOnMeshConnectivityChecks = false; + m_bAutoStartInvalidated = false; + + LobbyManager lobbyManager = ServiceLocator.Services.GetRequiredService(); + Lobby? quickMatchLobby = lobbyManager.GetLobby(m_LobbyID); + if (quickMatchLobby != null) + { + foreach (UserSession memberSession in sessionsToRequeue) + { + LobbyMember? lobbyMember = quickMatchLobby.GetMemberFromUserID(memberSession.m_UserID); + if (lobbyMember != null) + { + // Legacy clients do not understand the requeue action, but they can still tear down + // peer and anti-cheat connections before joining the next temporary lobby. + quickMatchLobby.SendPeerTeardownToDepartingMember(lobbyMember); + await quickMatchLobby.RemoveMember(lobbyMember); + } + } + } + + WebSocketMessage_Simple requeueAction = new WebSocketMessage_Simple(); + requeueAction.msg_id = (int)EWebSocketMessageID.MATCHMAKING_ACTION_REQUEUE; + byte[] requeueActionJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(requeueAction)); + + foreach (UserSession memberSession in sessionsToRequeue) + { + memberSession.UpdateSessionLobbyID(-1); + memberSession.QueueWebsocketSend(requeueActionJSON); + + bool bAlreadyQueued = false; + foreach (WeakReference wrSession in lstSessions) + { + if (wrSession.TryGetTarget(out UserSession? pendingSession) && pendingSession == memberSession) + { + bAlreadyQueued = true; + break; + } + } + + if (!bAlreadyQueued) + { + lstSessions.Add(new WeakReference(memberSession)); + } + + await SendMatchmakingMessage(memberSession, reason); + await SendMatchmakingMessage(memberSession, "Re-queueing you into matchmaking..."); + } + + m_lstMembers.Clear(); + m_LobbyID = -1; + + MatchmakingManager.DestroyBucket(this); + } + + public async Task Tick() + { + // merged/deleted buckets must not create another lobby or continue a committed setup + if (m_bMergedAway || m_bPendingDeletion) { return; } @@ -791,14 +933,48 @@ public async Task Tick() // never reach the "everyone is in the lobby" condition PruneDeadMembers(); - // TODO_QUICKMATCH: What if the playlist is null? is this even possible since we validated before creating the bucket - if (g_Playlists.TryGetValue(PlaylistID, out Playlist? playlist)) - { - // nobody left? clean ourselves up rather than lingering as a ghost bucket - if (CurrentMemberCount() == 0 && !m_bWaitingOnLobbyJoins && !m_bHasStartedCountdown) + // TODO_QUICKMATCH: What if the playlist is null? is this even possible since we validated before creating the bucket + if (g_Playlists.TryGetValue(PlaylistID, out Playlist? playlist)) + { + // nobody left? clean ourselves up rather than lingering as a ghost bucket + if (CurrentMemberCount() == 0 && !m_bWaitingOnLobbyJoins && !m_bHasStartedCountdown) + { + MatchmakingManager.DestroyBucket(this); + return; + } + + if (m_bAutoStartInvalidated) + { + await AbortQuickMatchAutoStart("QuickMatch auto-start was aborted because a player left during match setup."); + return; + } + + if (m_bWaitingOnMeshConnectivityChecks) { - MatchmakingManager.DestroyBucket(this); - return; + Lobby? lobbyDuringMeshCheck = lobbyManager.GetLobby(m_LobbyID); + if (lobbyDuringMeshCheck == null) + { + await AbortQuickMatchAutoStart("QuickMatch auto-start was aborted because the temporary lobby no longer exists."); + return; + } + + await lobbyDuringMeshCheck.ProcessPendingFullMeshConnectivityChecks(); + + if (!lobbyDuringMeshCheck.PendingFullMeshConnectivityChecks) + { + m_bWaitingOnMeshConnectivityChecks = false; + + if (lobbyDuringMeshCheck.LastFullMeshConnectivityCheckOutcome == true) + { + await StartGameAfterSuccessfulMeshCheck(lobbyDuringMeshCheck); + } + else + { + await AbortQuickMatchAutoStart("QuickMatch auto-start was aborted because not all players were fully mesh-connected."); + } + } + + return; } // do we need to start? @@ -1077,39 +1253,21 @@ await SendMatchmakingMessage(memberSession, } - // TODO_QUICKMATCH: Do full mesh connectivity check + handle not being connected + // trigger full mesh check before issuing the final quickmatch start command // do we have a countdown? if (m_StartTime != -1) { if (Environment.TickCount64 >= m_StartTime) { m_StartTime = -1; - - Console.WriteLine("START GAME"); - - // send start - WebSocketMessage_MatchmakerStartGame startGameAction = new WebSocketMessage_MatchmakerStartGame(); - startGameAction.msg_id = (int)EWebSocketMessageID.MATCHMAKING_ACTION_START_GAME; - byte[] bytesJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(startGameAction)); - - foreach (MatchmakingBucketMember member in m_lstMembers) - { - UserSession? memberSession = member.GetAssociatedSession(); - if (memberSession != null) - { - memberSession.QueueWebsocketSend(bytesJSON); - } - } - - // start match + create placeholder match Lobby? lobby = lobbyManager.GetLobby(m_LobbyID); - if (lobby != null) + if (lobby == null) { - await lobby.UpdateState(ELobbyState.INGAME); + await AbortQuickMatchAutoStart("QuickMatch auto-start was aborted because the temporary lobby no longer exists."); + return; } - // destroy the bucket - MatchmakingManager.DestroyBucket(this); + await TriggerFullMeshConnectivityChecks(lobby); } } } @@ -1447,6 +1605,7 @@ public static async Task Tick() private static ConcurrentList m_lstBucketsPendingDeletion = new(); public static void DestroyBucket(MatchmakingBucket bucket) { + bucket.MarkPendingDeletion(); m_lstBucketsPendingDeletion.Add(bucket); } @@ -1498,10 +1657,10 @@ private static void RemoveSessionFromPendingList(UserSession plr) } } - private static void RemovePlayerFromAllBuckets(UserSession plr) - { - var lobbyManager = ServiceLocator.Services.GetRequiredService(); - + private static void RemovePlayerFromAllBuckets(UserSession plr) + { + var lobbyManager = ServiceLocator.Services.GetRequiredService(); + // also remove from any bucket we are in to avoid ghost buckets foreach (var kvPair in m_dictMatchmakingBuckets) { @@ -1547,4 +1706,4 @@ public static void DeregisterPlayer(UserSession plr) Console.WriteLine("[Source 4] User {0} Leave Any Lobby", plr.m_UserID); lobbyManager.LeaveAnyLobby(plr.m_UserID); } -} \ No newline at end of file +} From 29ad6b4e6757900d6a60f336ed83100ed126f3b4 Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Tue, 11 Aug 2026 02:15:30 +0200 Subject: [PATCH 2/4] fix(matchmaking): harden setup concurrency and recovery Serialize registration, cancellation, assignment, and requeue operations. Make lobby removal and bucket cleanup safe under concurrent disconnect and teardown paths. --- GenOnlineService/Constants.cs | 18 +- .../Matchmaking/MatchmakingController.cs | 14 +- .../WebSocket/WebSocketController.cs | 2 +- GenOnlineService/LobbyManager.cs | 377 ++++---- GenOnlineService/MatchmakingManager.cs | 844 +++++++++++------- 5 files changed, 747 insertions(+), 508 deletions(-) diff --git a/GenOnlineService/Constants.cs b/GenOnlineService/Constants.cs index 6d570b0..956478f 100644 --- a/GenOnlineService/Constants.cs +++ b/GenOnlineService/Constants.cs @@ -880,9 +880,11 @@ public class UserSession private string ACExeCRC = String.Empty; - // Matchmaking data - public UInt16 MatchmakingPlaylistID = 0; - public ConcurrentList MatchmakingMapIndicies = new(); + // Matchmaking data + public UInt16 MatchmakingPlaylistID = 0; + public ConcurrentList MatchmakingMapIndicies = new(); + internal bool IsRegisteredForMatchmaking { get; set; } = false; + internal SemaphoreSlim MatchmakingStateLock { get; } = new(1, 1); // NOTE: These are not set on login, only when in quickmatch! public UInt32 ExeCRC = 0; @@ -1356,11 +1358,11 @@ public static async Task FullyDestroyPlayerSession(Int64 user_id, UserSession? u await lobbyManager.CleanupUserLobbiesNotStarted(user_id); - // remove from any matchmaking - if (userData != null) - { - MatchmakingManager.DeregisterPlayer(userData); - } + // remove from any matchmaking + if (userData != null) + { + await MatchmakingManager.DeregisterPlayer(userData); + } // TODO: Client needs to handle this... itll start returning 404 } diff --git a/GenOnlineService/Controllers/Matchmaking/MatchmakingController.cs b/GenOnlineService/Controllers/Matchmaking/MatchmakingController.cs index 3a9794d..16c131c 100644 --- a/GenOnlineService/Controllers/Matchmaking/MatchmakingController.cs +++ b/GenOnlineService/Controllers/Matchmaking/MatchmakingController.cs @@ -118,9 +118,9 @@ public void Put_Widen() } } - [HttpDelete] - [Authorize(Roles = "GameClient")] - public void Delete() + [HttpDelete] + [Authorize(Roles = "GameClient")] + public async Task Delete() { Int64 user_id = TokenHelper.GetUserID(this); EUserSessionType sessionType = TokenHelper.GetSessionType(this); @@ -128,10 +128,10 @@ public void Delete() { UserSession? playerSession = WebSocketManager.GetSessionFromUser(user_id, sessionType); - if (playerSession != null) - { - MatchmakingManager.DeregisterPlayer(playerSession); - } + if (playerSession != null) + { + await MatchmakingManager.DeregisterPlayer(playerSession); + } } } diff --git a/GenOnlineService/Controllers/WebSocket/WebSocketController.cs b/GenOnlineService/Controllers/WebSocket/WebSocketController.cs index 7175e0d..1f6c047 100644 --- a/GenOnlineService/Controllers/WebSocket/WebSocketController.cs +++ b/GenOnlineService/Controllers/WebSocket/WebSocketController.cs @@ -843,7 +843,7 @@ private async Task ProcessWSMessage(UserWebSocketInstance sourceWS, UserSession lobbyInfo.StartFullMeshConnectivityCheck(); // start full mesh connectivity checks - lobbyInfo.SendFullMeshConnectivityCheckRequestToMembers(); + lobbyInfo.SendFullMeshConnectivityCheckRequestToMembers(); } else if (msgID == EWebSocketMessageID.FULL_MESH_CONNECTIVITY_CHECK_RESPONSE) { diff --git a/GenOnlineService/LobbyManager.cs b/GenOnlineService/LobbyManager.cs index 6759910..4a6dd99 100644 --- a/GenOnlineService/LobbyManager.cs +++ b/GenOnlineService/LobbyManager.cs @@ -35,9 +35,35 @@ using System.Threading.Tasks; using System.Xml.Linq; -namespace GenOnlineService -{ - public class Lobby +namespace GenOnlineService +{ + internal static class FullMeshCheckProtocol + { + // TODO: Remove the zero-valued response compatibility path when legacy + // clients are no longer supported. + internal static bool IsLegacyResponse(WebSocketMessage_FullMeshConnectivityCheckResponseFromUser response) + { + return response.mesh_check_id == 0 && response.attempt == 0; + } + + internal static bool MatchesCurrentAttempt( + WebSocketMessage_FullMeshConnectivityCheckResponseFromUser response, + Int64 currentCheckID, + int currentAttempt) + { + return IsLegacyResponse(response) + ? currentAttempt == 1 + : response.mesh_check_id == currentCheckID && response.attempt == currentAttempt; + } + + internal static bool ShouldRetry(bool meshComplete, bool hasLegacyResponse, int currentAttempt, int maxAttempts) + { + // TODO: Remove legacy retry suppression together with the legacy response path. + return !meshComplete && !hasLegacyResponse && currentAttempt < maxAttempts; + } + } + + public class Lobby { public Int64 LobbyID { get; private set; } = -1; public Int64 Owner { get; private set; } = -1; @@ -62,21 +88,21 @@ public class Lobby public const int MaxFullMeshConnectivityCheckDurationMS = (MSToWaitForFullMeshChecks * MaxFullMeshCheckAttempts) + (MSBeforeFullMeshCheckRetry * (MaxFullMeshCheckAttempts - 1)); - - private readonly object m_FullMeshCheckLock = new(); - - [JsonIgnore] - public bool? LastFullMeshConnectivityCheckOutcome { get; private set; } = null; + + private readonly object m_FullMeshCheckLock = new(); + + [JsonIgnore] + public bool? LastFullMeshConnectivityCheckOutcome { get; private set; } = null; [JsonIgnore] - public int FullMeshCheckAttempt { get; private set; } = 0; - + public int FullMeshCheckAttempt { get; private set; } = 0; + [JsonIgnore] public Int64 FullMeshCheckID { get; private set; } = 0; [JsonIgnore] private Int64 m_TimeToRetryFullMeshChecks = -1; - + [JsonIgnore] private bool m_bCurrentAttemptHasLegacyResponse = false; @@ -167,94 +193,95 @@ public async Task RegisterProbeResponse_Malformed_Type2(Int64 userID) // End AC Probes public void StartFullMeshConnectivityCheck() { - lock (m_FullMeshCheckLock) + lock (m_FullMeshCheckLock) { FullMeshCheckID = Interlocked.Increment(ref s_NextFullMeshCheckID); - FullMeshCheckAttempt = 1; - m_TimeToRetryFullMeshChecks = -1; - LastFullMeshConnectivityCheckOutcome = null; - BeginFullMeshConnectivityCheckAttempt(); - } - } - - private void BeginFullMeshConnectivityCheckAttempt() - { + FullMeshCheckAttempt = 1; + m_TimeToRetryFullMeshChecks = -1; + LastFullMeshConnectivityCheckOutcome = null; + BeginFullMeshConnectivityCheckAttempt(); + } + } + + private void BeginFullMeshConnectivityCheckAttempt() + { PendingFullMeshConnectivityChecks = true; FullMeshConnectivityChecks = new(); m_bCurrentAttemptHasLegacyResponse = false; TimeStartFullMeshChecks = Environment.TickCount64; } - public void SendFullMeshConnectivityCheckRequestToMembers() + public void SendFullMeshConnectivityCheckRequestToMembers() { WebSocketMessage_FullMeshConnectivityCheckRequest startCommand = new WebSocketMessage_FullMeshConnectivityCheckRequest(); startCommand.msg_id = (int)EWebSocketMessageID.FULL_MESH_CONNECTIVITY_CHECK_RESPONSE; startCommand.mesh_check_id = FullMeshCheckID; startCommand.attempt = FullMeshCheckAttempt; - byte[] bytesJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(startCommand)); - - foreach (LobbyMember member in Members) - { - if (member.GetSession().TryGetTarget(out UserSession? session) && session != null) - { - session.QueueWebsocketSend(bytesJSON); - } - } - } - - // Re-issues signalling between the pairs that failed to connect. This is the same handshake a player - // gets when they join, which is why manually rejoining the lobby often repairs a broken mesh. - private void RestartSignallingForMissingConnections(List lstMissingConnections) - { - HashSet<(Int64, Int64)> alreadyResignalled = new(); - - foreach (MissingConnectionEntry missingConnection in lstMissingConnections) - { - Int64 lowUserID = Math.Min(missingConnection.source_user_id, missingConnection.target_user_id); - Int64 highUserID = Math.Max(missingConnection.source_user_id, missingConnection.target_user_id); - - if (!alreadyResignalled.Add((lowUserID, highUserID))) - { - continue; - } - - LobbyMember? sourceMember = GetMemberFromUserID(missingConnection.source_user_id); - LobbyMember? targetMember = GetMemberFromUserID(missingConnection.target_user_id); - - if (sourceMember == null || targetMember == null) - { - continue; - } - - Console.WriteLine("[Lobby {0}] Re-signalling {1} <-> {2} before mesh check retry", LobbyID, sourceMember.UserID, targetMember.UserID); - - SendStartSignallingToMember(sourceMember, targetMember); - SendStartSignallingToMember(targetMember, sourceMember); - } - } - - private void SendStartSignallingToMember(LobbyMember recipient, LobbyMember peer) - { - if (recipient.GetSession().TryGetTarget(out UserSession? recipientSession) && recipientSession != null) - { - WebSocketMessage_NetworkStartSignalling signallingMsg = new WebSocketMessage_NetworkStartSignalling(); - signallingMsg.msg_id = (int)EWebSocketMessageID.NETWORK_CONNECTION_START_SIGNALLING; - signallingMsg.lobby_id = LobbyID; - signallingMsg.user_id = peer.UserID; - signallingMsg.preferred_port = peer.Port; - signallingMsg.middleware_id = peer.MiddlewareUserID; - recipientSession.QueueWebsocketSend(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(signallingMsg))); - } - } - + byte[] bytesJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(startCommand)); + + foreach (LobbyMember member in Members) + { + if (member.GetSession().TryGetTarget(out UserSession? session) && session != null) + { + session.QueueWebsocketSend(bytesJSON); + } + } + } + + // Re-issues signalling between the pairs that failed to connect. This is the same handshake a player + // gets when they join, which is why manually rejoining the lobby often repairs a broken mesh. + private void RestartSignallingForMissingConnections(List lstMissingConnections) + { + HashSet<(Int64, Int64)> alreadyResignalled = new(); + + foreach (MissingConnectionEntry missingConnection in lstMissingConnections) + { + Int64 lowUserID = Math.Min(missingConnection.source_user_id, missingConnection.target_user_id); + Int64 highUserID = Math.Max(missingConnection.source_user_id, missingConnection.target_user_id); + + if (!alreadyResignalled.Add((lowUserID, highUserID))) + { + continue; + } + + LobbyMember? sourceMember = GetMemberFromUserID(missingConnection.source_user_id); + LobbyMember? targetMember = GetMemberFromUserID(missingConnection.target_user_id); + + if (sourceMember == null || targetMember == null) + { + continue; + } + + Console.WriteLine("[Lobby {0}] Re-signalling {1} <-> {2} before mesh check retry", LobbyID, sourceMember.UserID, targetMember.UserID); + + SendStartSignallingToMember(sourceMember, targetMember); + SendStartSignallingToMember(targetMember, sourceMember); + } + } + + private void SendStartSignallingToMember(LobbyMember recipient, LobbyMember peer) + { + if (recipient.GetSession().TryGetTarget(out UserSession? recipientSession) && recipientSession != null) + { + WebSocketMessage_NetworkStartSignalling signallingMsg = new WebSocketMessage_NetworkStartSignalling(); + signallingMsg.msg_id = (int)EWebSocketMessageID.NETWORK_CONNECTION_START_SIGNALLING; + signallingMsg.lobby_id = LobbyID; + signallingMsg.user_id = peer.UserID; + signallingMsg.preferred_port = peer.Port; + signallingMsg.middleware_id = peer.MiddlewareUserID; + recipientSession.QueueWebsocketSend(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(signallingMsg))); + } + } + public Task StoreFullMeshConnectivityResponse(Int64 sourceUser, WebSocketMessage_FullMeshConnectivityCheckResponseFromUser response) { lock (m_FullMeshCheckLock) { - bool bLegacyResponse = response.mesh_check_id == 0 && response.attempt == 0; - bool bMatchesCurrentAttempt = bLegacyResponse - ? FullMeshCheckAttempt == 1 - : response.mesh_check_id == FullMeshCheckID && response.attempt == FullMeshCheckAttempt; + bool bLegacyResponse = FullMeshCheckProtocol.IsLegacyResponse(response); + bool bMatchesCurrentAttempt = FullMeshCheckProtocol.MatchesCurrentAttempt( + response, + FullMeshCheckID, + FullMeshCheckAttempt); LobbyMember? sourceMember = GetMemberFromUserID(sourceUser); if (PendingFullMeshConnectivityChecks @@ -264,45 +291,45 @@ public Task StoreFullMeshConnectivityResponse(Int64 sourceUser, WebSocketMessage { m_bCurrentAttemptHasLegacyResponse |= bLegacyResponse; FullMeshConnectivityChecks[sourceUser] = new ConcurrentList(response.connectivity_map); - } - - ProcessPendingFullMeshConnectivityChecksInternal(); - } - - return Task.CompletedTask; - } - - public Task ProcessPendingFullMeshConnectivityChecks() - { - lock (m_FullMeshCheckLock) - { - ProcessPendingFullMeshConnectivityChecksInternal(); - } - - return Task.CompletedTask; - } - - private void ProcessPendingFullMeshConnectivityChecksInternal() - { - if (!PendingFullMeshConnectivityChecks) - { - return; - } - - // Give re-signalled connections time to establish before starting the retry. - if (m_TimeToRetryFullMeshChecks != -1) - { - if (Environment.TickCount64 < m_TimeToRetryFullMeshChecks) - { - return; - } - - m_TimeToRetryFullMeshChecks = -1; - BeginFullMeshConnectivityCheckAttempt(); - SendFullMeshConnectivityCheckRequestToMembers(); - return; - } - + } + + ProcessPendingFullMeshConnectivityChecksInternal(); + } + + return Task.CompletedTask; + } + + public Task ProcessPendingFullMeshConnectivityChecks() + { + lock (m_FullMeshCheckLock) + { + ProcessPendingFullMeshConnectivityChecksInternal(); + } + + return Task.CompletedTask; + } + + private void ProcessPendingFullMeshConnectivityChecksInternal() + { + if (!PendingFullMeshConnectivityChecks) + { + return; + } + + // Give re-signalled connections time to establish before starting the retry. + if (m_TimeToRetryFullMeshChecks != -1) + { + if (Environment.TickCount64 < m_TimeToRetryFullMeshChecks) + { + return; + } + + m_TimeToRetryFullMeshChecks = -1; + BeginFullMeshConnectivityCheckAttempt(); + SendFullMeshConnectivityCheckRequestToMembers(); + return; + } + { bool bDoneChecks = false; int totalMapEntriesExpected = GetNumberOfHumans(); @@ -352,24 +379,24 @@ private void ProcessPendingFullMeshConnectivityChecksInternal() } } - // A member that never reported cannot be assumed connected, so treat them as missing to everyone. - foreach (LobbyMember member in Members) - { - if (member.IsHuman() && !FullMeshConnectivityChecks.ContainsKey(member.UserID)) - { - foreach (LobbyMember otherMember in Members) - { - if (otherMember.IsHuman() && otherMember.UserID != member.UserID) - { - MissingConnectionEntry missingConnectionEntry = new(); - missingConnectionEntry.source_user_id = member.UserID; - missingConnectionEntry.target_user_id = otherMember.UserID; - lstMissingConnections.Add(missingConnectionEntry); - } - } - } - } - + // A member that never reported cannot be assumed connected, so treat them as missing to everyone. + foreach (LobbyMember member in Members) + { + if (member.IsHuman() && !FullMeshConnectivityChecks.ContainsKey(member.UserID)) + { + foreach (LobbyMember otherMember in Members) + { + if (otherMember.IsHuman() && otherMember.UserID != member.UserID) + { + MissingConnectionEntry missingConnectionEntry = new(); + missingConnectionEntry.source_user_id = member.UserID; + missingConnectionEntry.target_user_id = otherMember.UserID; + lstMissingConnections.Add(missingConnectionEntry); + } + } + } + } + bool bDisableMeshCheck = false; if (Program.g_Config != null) { @@ -383,18 +410,20 @@ private void ProcessPendingFullMeshConnectivityChecksInternal() - bool bMeshComplete = bDisableMeshCheck || lstMissingConnections.Count == 0; - - bool bAllMembersReportedCurrentAttempt = FullMeshConnectivityChecks.Count == totalMapEntriesExpected; - bool bCanSafelyRetry = bAllMembersReportedCurrentAttempt && !m_bCurrentAttemptHasLegacyResponse; - if (!bMeshComplete && bCanSafelyRetry && FullMeshCheckAttempt < MaxFullMeshCheckAttempts) - { - ++FullMeshCheckAttempt; - RestartSignallingForMissingConnections(lstMissingConnections); - m_TimeToRetryFullMeshChecks = Environment.TickCount64 + MSBeforeFullMeshCheckRetry; - return; - } - + bool bMeshComplete = bDisableMeshCheck || lstMissingConnections.Count == 0; + + if (FullMeshCheckProtocol.ShouldRetry( + bMeshComplete, + m_bCurrentAttemptHasLegacyResponse, + FullMeshCheckAttempt, + MaxFullMeshCheckAttempts)) + { + ++FullMeshCheckAttempt; + RestartSignallingForMissingConnections(lstMissingConnections); + m_TimeToRetryFullMeshChecks = Environment.TickCount64 + MSBeforeFullMeshCheckRetry; + return; + } + // inform host that we are done // start full mesh connectivity checks WebSocketMessage_FullMeshConnectivityCheckOutcome outcome = new WebSocketMessage_FullMeshConnectivityCheckOutcome(); @@ -411,8 +440,8 @@ private void ProcessPendingFullMeshConnectivityChecksInternal() outcome.missing_connections = lstMissingConnections; } - LastFullMeshConnectivityCheckOutcome = outcome.mesh_complete; - + LastFullMeshConnectivityCheckOutcome = outcome.mesh_complete; + // TODO_EFCORE: Later, these should really use lobby list instead of getting session from ID // send to host @@ -426,7 +455,7 @@ private void ProcessPendingFullMeshConnectivityChecksInternal() // reset state PendingFullMeshConnectivityChecks = false; TimeStartFullMeshChecks = -1; - m_TimeToRetryFullMeshChecks = -1; + m_TimeToRetryFullMeshChecks = -1; } } } @@ -810,8 +839,8 @@ private void CalculateNextProbeTime(bool bIsFirstProbe) public async Task Tick() { - await ProcessPendingFullMeshConnectivityChecks(); - + await ProcessPendingFullMeshConnectivityChecks(); + if (m_NextProbe != 0 && Environment.TickCount64 >= m_NextProbe) { // send probe @@ -1124,16 +1153,32 @@ public void SendPeerTeardownToDepartingMember(LobbyMember departingMember) } } - public async Task RemoveMember(LobbyMember member) - { - // TODO_LOBBY: Optimize this - Int64 UserID = member.UserID; - - Console.WriteLine("User {0} left lobby {1}", UserID, LobbyID); - - LobbyMember placeholderMember = new LobbyMember(this, null, -1, String.Empty, String.Empty, 0, -1, -1, -1, EPlayerType.SLOT_OPEN, member.SlotIndex, true); - Members[member.SlotIndex] = placeholderMember; - TimeMemberLeft[UserID] = DateTime.UtcNow; + public async Task RemoveMember(LobbyMember member) + { + // Matchmaking cancellation and the client's explicit lobby leave can arrive concurrently. + // Claim the slot once so teardown, host migration, and destruction callbacks stay idempotent. + await g_SlotLock.WaitAsync(); + try + { + if (member.SlotIndex < 0 + || member.SlotIndex >= Members.Length + || !ReferenceEquals(Members[member.SlotIndex], member)) + { + return; + } + + LobbyMember placeholderMember = new LobbyMember(this, null, -1, String.Empty, String.Empty, 0, -1, -1, -1, EPlayerType.SLOT_OPEN, member.SlotIndex, true); + Members[member.SlotIndex] = placeholderMember; + TimeMemberLeft[member.UserID] = DateTime.UtcNow; + } + finally + { + g_SlotLock.Release(); + } + + // TODO_LOBBY: Optimize this + Int64 UserID = member.UserID; + Console.WriteLine("User {0} left lobby {1}", UserID, LobbyID); // AC dergister WebSocketMessage_ACDeregisterPlayer remotePlayerAcMsg = new WebSocketMessage_ACDeregisterPlayer(); diff --git a/GenOnlineService/MatchmakingManager.cs b/GenOnlineService/MatchmakingManager.cs index cae1ff5..806dd3f 100644 --- a/GenOnlineService/MatchmakingManager.cs +++ b/GenOnlineService/MatchmakingManager.cs @@ -294,19 +294,43 @@ public class MatchmakingBucket private ConcurrentList m_lstMembers = new(); public ConcurrentList lstMapIndices { get; set; } = new(); - private Int64 m_timeReachedMinPlayers = -1; - private bool m_bReachedMinPlayers = false; + private Int64 m_timeReachedMinPlayers = -1; + private bool m_bReachedMinPlayers = false; private bool m_bWaitingOnLobbyJoins = false; private bool m_bHasStartedCountdown = false; private bool m_bWaitingOnMeshConnectivityChecks = false; - private volatile bool m_bAutoStartInvalidated = false; + private bool m_bAutoStartInvalidated = false; private bool m_bPendingDeletion = false; private bool m_bMergedAway = false; - - // a bucket that has been merged into another bucket (or that has already been handed a lobby) must never - // accept or donate members again, otherwise a player ends up in two buckets and gets sent to two lobbies - public bool IsMergedAway() { return m_bMergedAway; } - public bool IsLockedForMatchStart() { return m_bHasStartedCountdown || m_bWaitingOnLobbyJoins || m_bWaitingOnMeshConnectivityChecks || m_bPendingDeletion; } + private bool m_bAbortInProgress = false; + private bool m_bStartCommitted = false; + private readonly object m_StateLock = new(); + + // a bucket that has been merged into another bucket (or that has already been handed a lobby) must never + // accept or donate members again, otherwise a player ends up in two buckets and gets sent to two lobbies + public bool IsMergedAway() + { + lock (m_StateLock) + { + return m_bMergedAway; + } + } + + public bool IsLockedForMatchStart() + { + lock (m_StateLock) + { + return m_bHasStartedCountdown || m_bWaitingOnLobbyJoins || m_bWaitingOnMeshConnectivityChecks || m_bPendingDeletion; + } + } + + private bool IsPendingDeletion() + { + lock (m_StateLock) + { + return m_bPendingDeletion; + } + } public UInt16 PlaylistID { get; private set; } public int MinPlayers { get; private set; } @@ -474,13 +498,13 @@ public bool DoMapSelectionsIntersect(ConcurrentList lstRhs) return false; } - public bool CanMergeWithOtherBucket(MatchmakingBucket bucketToMerge) - { - if (m_bPendingDeletion || bucketToMerge.m_bPendingDeletion) - { - return false; - } - + public bool CanMergeWithOtherBucket(MatchmakingBucket bucketToMerge) + { + if (IsPendingDeletion() || bucketToMerge.IsPendingDeletion()) + { + return false; + } + // playlist must match if (bucketToMerge.PlaylistID != this.PlaylistID) { @@ -488,7 +512,7 @@ public bool CanMergeWithOtherBucket(MatchmakingBucket bucketToMerge) } // either bucket already merged away this tick? it is stale, never touch it again - if (m_bMergedAway || bucketToMerge.m_bMergedAway) + if (IsMergedAway() || bucketToMerge.IsMergedAway()) { return false; } @@ -559,8 +583,11 @@ public async Task MergeWithOtherBucket(MatchmakingBucket bucketToMerge) } // the source bucket is now empty and flagged so it can never merge/accept players again - bucketToMerge.m_bMergedAway = true; - bucketToMerge.m_lstMembers.Clear(); + lock (bucketToMerge.m_StateLock) + { + bucketToMerge.m_bMergedAway = true; + } + bucketToMerge.m_lstMembers.Clear(); // nothing else to copy... everything else should match since we were a merge candidate @@ -588,20 +615,30 @@ public bool HasPlayer(UserSession playerSession) return false; } - public bool RemovePlayer(UserSession playerSession) + public bool RemovePlayer(UserSession playerSession, out bool bCancellationRejected) { - foreach (MatchmakingBucketMember member in m_lstMembers) - { - if (member.Is(playerSession)) - { - if (m_bWaitingOnLobbyJoins || m_bHasStartedCountdown || m_bWaitingOnMeshConnectivityChecks) + bCancellationRejected = false; + lock (m_StateLock) + { + foreach (MatchmakingBucketMember member in m_lstMembers) + { + if (member.Is(playerSession)) { - m_bAutoStartInvalidated = true; - } + if (m_bStartCommitted) + { + bCancellationRejected = true; + return false; + } - m_lstMembers.Remove(member); - return true; - } + if (m_bWaitingOnLobbyJoins || m_bHasStartedCountdown || m_bWaitingOnMeshConnectivityChecks) + { + m_bAutoStartInvalidated = true; + } + + m_lstMembers.Remove(member); + return true; + } + } } return false; @@ -613,21 +650,36 @@ public int CurrentMemberCount() } // members whose UserSession has been collected/disconnected are dead weight - they inflate the member count, - // which both blocks the "everyone joined the lobby" check and skews the average elo - public void PruneDeadMembers() - { - foreach (MatchmakingBucketMember member in m_lstMembers) - { - if (member.GetAssociatedSession() == null) - { - m_lstMembers.Remove(member); + // which both blocks the "everyone joined the lobby" check and skews the average elo + public bool PruneDeadMembers() + { + bool bInvalidatedSetup = false; + lock (m_StateLock) + { + foreach (MatchmakingBucketMember member in m_lstMembers) + { + if (member.GetAssociatedSession() == null) + { + if (m_bWaitingOnLobbyJoins || m_bHasStartedCountdown || m_bWaitingOnMeshConnectivityChecks) + { + m_bAutoStartInvalidated = true; + bInvalidatedSetup = true; + } + + m_lstMembers.Remove(member); + } } } + + return bInvalidatedSetup; } internal void MarkPendingDeletion() { - m_bPendingDeletion = true; + lock (m_StateLock) + { + m_bPendingDeletion = true; + } } // TODO_EFCORE: Shared User data, and session<->websocket could be weakrefs @@ -661,10 +713,10 @@ public bool IsJoiningUserBlockedByOrHasBlockedAnyBucketMember(UserSession? joini return false; } - public bool HasSpaceForUsers(int numUsers, UInt32 exe_crc, UInt32 ini_crc, EKnownAnticheatID anticheatID) + public bool HasSpaceForUsers(int numUsers, UInt32 exe_crc, UInt32 ini_crc, EKnownAnticheatID anticheatID) { // stale buckets must never accept new members - if (m_bMergedAway || m_bPendingDeletion) + if (IsMergedAway() || IsPendingDeletion()) { return false; } @@ -719,9 +771,9 @@ public int GetAvgElo() { SharedUserData? memberUserData = GenOnlineService.WebSocketManager.GetSharedDataForUser(memberSession.m_UserID); - if (memberUserData != null) - { - avgElo += memberUserData.GameStats.EloRating; + if (memberUserData?.GameStats != null) + { + avgElo += memberUserData.GameStats.EloRating; ++numContributingMembers; } } @@ -797,29 +849,35 @@ public Int64 GetLobbyID() private const Int64 c_LobbyJoinTimeoutMSec = 45000; private async Task StartGameAfterSuccessfulMeshCheck(Lobby lobby) - { - Console.WriteLine("START GAME"); - - WebSocketMessage_MatchmakerStartGame startGameAction = new WebSocketMessage_MatchmakerStartGame(); - startGameAction.msg_id = (int)EWebSocketMessageID.MATCHMAKING_ACTION_START_GAME; - byte[] bytesJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(startGameAction)); - - foreach (MatchmakingBucketMember member in m_lstMembers) - { - UserSession? memberSession = member.GetAssociatedSession(); - if (memberSession != null) - { - memberSession.QueueWebsocketSend(bytesJSON); - } - } - - await lobby.UpdateState(ELobbyState.INGAME); - MatchmakingManager.DestroyBucket(this); - } - - private async Task TriggerFullMeshConnectivityChecks(Lobby lobby) - { + { + Console.WriteLine("START GAME"); + + WebSocketMessage_MatchmakerStartGame startGameAction = new WebSocketMessage_MatchmakerStartGame(); + startGameAction.msg_id = (int)EWebSocketMessageID.MATCHMAKING_ACTION_START_GAME; + byte[] bytesJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(startGameAction)); + + foreach (MatchmakingBucketMember member in m_lstMembers) + { + UserSession? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + memberSession.QueueWebsocketSend(bytesJSON); + } + } + + await lobby.UpdateState(ELobbyState.INGAME); + MatchmakingManager.DestroyBucket(this); + } + + private async Task TriggerFullMeshConnectivityChecks(Lobby lobby) + { lobby.StartFullMeshConnectivityCheck(); + lock (m_StateLock) + { + // Publish the state transition before any notification work. If a send fails, the + // regular lobby timeout still completes or aborts the setup instead of stranding it. + m_bWaitingOnMeshConnectivityChecks = true; + } const int MeshCheckClientTimeoutMarginMS = 2000; WebSocketMessage_MatchmakerSetupProgress setupProgress = new WebSocketMessage_MatchmakerSetupProgress(); @@ -836,142 +894,189 @@ private async Task TriggerFullMeshConnectivityChecks(Lobby lobby) } } - lobby.SendFullMeshConnectivityCheckRequestToMembers(); - - foreach (MatchmakingBucketMember member in m_lstMembers) - { - UserSession? memberSession = member.GetAssociatedSession(); - if (memberSession != null) - { - await SendMatchmakingMessage(memberSession, "Running full mesh connectivity checks before game start..."); - } - } - - m_bWaitingOnMeshConnectivityChecks = true; - } - - private async Task AbortQuickMatchAutoStart(string reason) - { - List sessionsToRequeue = new(); - foreach (MatchmakingBucketMember member in m_lstMembers) + lobby.SendFullMeshConnectivityCheckRequestToMembers(); + + foreach (MatchmakingBucketMember member in m_lstMembers) { - UserSession? memberSession = member.GetAssociatedSession(); - if (memberSession != null) - { - sessionsToRequeue.Add(memberSession); - } + UserSession? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + await SendMatchmakingMessage(memberSession, "Running full mesh connectivity checks before game start..."); + } } - - m_StartTime = -1; - m_bWaitingOnLobbyJoins = false; - m_bHasStartedCountdown = false; - m_bWaitingOnMeshConnectivityChecks = false; - m_bAutoStartInvalidated = false; - - LobbyManager lobbyManager = ServiceLocator.Services.GetRequiredService(); - Lobby? quickMatchLobby = lobbyManager.GetLobby(m_LobbyID); - if (quickMatchLobby != null) + } + + private async Task AbortQuickMatchAutoStart(string reason) + { + List sessionsToRequeue = new(); + lock (m_StateLock) + { + if (m_bAbortInProgress || m_bStartCommitted) + { + return; + } + + m_bAbortInProgress = true; + m_bPendingDeletion = true; + + foreach (MatchmakingBucketMember member in m_lstMembers) + { + UserSession? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + sessionsToRequeue.Add(memberSession); + } + } + + m_StartTime = -1; + m_bWaitingOnLobbyJoins = false; + m_bHasStartedCountdown = false; + m_bWaitingOnMeshConnectivityChecks = false; + } + + LobbyManager lobbyManager = ServiceLocator.Services.GetRequiredService(); + Lobby? quickMatchLobby = lobbyManager.GetLobby(m_LobbyID); + if (quickMatchLobby != null) { - foreach (UserSession memberSession in sessionsToRequeue) + foreach (UserSession memberSession in sessionsToRequeue) { - LobbyMember? lobbyMember = quickMatchLobby.GetMemberFromUserID(memberSession.m_UserID); - if (lobbyMember != null) - { + LobbyMember? lobbyMember = quickMatchLobby.GetMemberFromUserID(memberSession.m_UserID); + if (lobbyMember != null) + { + // TODO: Remove this fallback once all supported clients handle MATCHMAKING_ACTION_REQUEUE. // Legacy clients do not understand the requeue action, but they can still tear down // peer and anti-cheat connections before joining the next temporary lobby. quickMatchLobby.SendPeerTeardownToDepartingMember(lobbyMember); - await quickMatchLobby.RemoveMember(lobbyMember); - } - } - } - + await quickMatchLobby.RemoveMember(lobbyMember); + } + } + } + WebSocketMessage_Simple requeueAction = new WebSocketMessage_Simple(); requeueAction.msg_id = (int)EWebSocketMessageID.MATCHMAKING_ACTION_REQUEUE; byte[] requeueActionJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(requeueAction)); - foreach (UserSession memberSession in sessionsToRequeue) - { - memberSession.UpdateSessionLobbyID(-1); - memberSession.QueueWebsocketSend(requeueActionJSON); - - bool bAlreadyQueued = false; - foreach (WeakReference wrSession in lstSessions) - { - if (wrSession.TryGetTarget(out UserSession? pendingSession) && pendingSession == memberSession) - { - bAlreadyQueued = true; - break; - } - } - - if (!bAlreadyQueued) - { - lstSessions.Add(new WeakReference(memberSession)); - } - - await SendMatchmakingMessage(memberSession, reason); - await SendMatchmakingMessage(memberSession, "Re-queueing you into matchmaking..."); - } - - m_lstMembers.Clear(); - m_LobbyID = -1; - - MatchmakingManager.DestroyBucket(this); - } - + foreach (UserSession memberSession in sessionsToRequeue) + { + if (await TryRequeueRegisteredPlayer(memberSession, requeueActionJSON)) + { + await SendMatchmakingMessage(memberSession, reason); + await SendMatchmakingMessage(memberSession, "Re-queueing you into matchmaking..."); + } + } + + m_lstMembers.Clear(); + m_LobbyID = -1; + + MatchmakingManager.DestroyBucket(this); + } + public async Task Tick() { + bool bPendingDeletion; + bool bAutoStartInvalidated; + bool bWaitingOnMeshConnectivityChecks; + bool bWaitingOnLobbyJoins; + bool bHasStartedCountdown; + lock (m_StateLock) + { + bPendingDeletion = m_bPendingDeletion; + bAutoStartInvalidated = m_bAutoStartInvalidated; + bWaitingOnMeshConnectivityChecks = m_bWaitingOnMeshConnectivityChecks; + bWaitingOnLobbyJoins = m_bWaitingOnLobbyJoins; + bHasStartedCountdown = m_bHasStartedCountdown; + } + // merged/deleted buckets must not create another lobby or continue a committed setup - if (m_bMergedAway || m_bPendingDeletion) - { - return; - } - - var lobbyManager = ServiceLocator.Services.GetRequiredService(); - - // drop any members whose session has gone away, otherwise they are counted forever and the bucket can - // never reach the "everyone is in the lobby" condition - PruneDeadMembers(); - + if (IsMergedAway() || bPendingDeletion) + { + return; + } + + var lobbyManager = ServiceLocator.Services.GetRequiredService(); + + // drop any members whose session has gone away, otherwise they are counted forever and the bucket can + // never reach the "everyone is in the lobby" condition + if (PruneDeadMembers()) + { + bAutoStartInvalidated = true; + } + // TODO_QUICKMATCH: What if the playlist is null? is this even possible since we validated before creating the bucket if (g_Playlists.TryGetValue(PlaylistID, out Playlist? playlist)) { // nobody left? clean ourselves up rather than lingering as a ghost bucket - if (CurrentMemberCount() == 0 && !m_bWaitingOnLobbyJoins && !m_bHasStartedCountdown) + if (CurrentMemberCount() == 0 && !bWaitingOnLobbyJoins && !bHasStartedCountdown) { MatchmakingManager.DestroyBucket(this); return; } - if (m_bAutoStartInvalidated) + if (bAutoStartInvalidated) { await AbortQuickMatchAutoStart("QuickMatch auto-start was aborted because a player left during match setup."); return; } - if (m_bWaitingOnMeshConnectivityChecks) - { - Lobby? lobbyDuringMeshCheck = lobbyManager.GetLobby(m_LobbyID); - if (lobbyDuringMeshCheck == null) - { - await AbortQuickMatchAutoStart("QuickMatch auto-start was aborted because the temporary lobby no longer exists."); - return; - } - - await lobbyDuringMeshCheck.ProcessPendingFullMeshConnectivityChecks(); - - if (!lobbyDuringMeshCheck.PendingFullMeshConnectivityChecks) - { - m_bWaitingOnMeshConnectivityChecks = false; - - if (lobbyDuringMeshCheck.LastFullMeshConnectivityCheckOutcome == true) - { - await StartGameAfterSuccessfulMeshCheck(lobbyDuringMeshCheck); - } - else - { - await AbortQuickMatchAutoStart("QuickMatch auto-start was aborted because not all players were fully mesh-connected."); - } + if (bWaitingOnMeshConnectivityChecks) + { + Lobby? lobbyDuringMeshCheck = lobbyManager.GetLobby(m_LobbyID); + if (lobbyDuringMeshCheck == null) + { + await AbortQuickMatchAutoStart("QuickMatch auto-start was aborted because the temporary lobby no longer exists."); + return; + } + + await lobbyDuringMeshCheck.ProcessPendingFullMeshConnectivityChecks(); + + if (!lobbyDuringMeshCheck.PendingFullMeshConnectivityChecks) + { + // Start the mesh check alongside the existing five-second countdown. A successful + // check still waits for the countdown, while a failed check can abort immediately. + if (lobbyDuringMeshCheck.LastFullMeshConnectivityCheckOutcome == true + && m_StartTime != -1 + && Environment.TickCount64 < m_StartTime) + { + return; + } + + bool bStartGame; + bool bAbortStart; + bool bInvalidatedAtDecision; + lock (m_StateLock) + { + bInvalidatedAtDecision = m_bAutoStartInvalidated; + if (m_bStartCommitted || m_bAbortInProgress || m_bPendingDeletion) + { + bStartGame = false; + bAbortStart = false; + } + else + { + m_bWaitingOnMeshConnectivityChecks = false; + m_bHasStartedCountdown = false; + m_StartTime = -1; + bStartGame = !bInvalidatedAtDecision && lobbyDuringMeshCheck.LastFullMeshConnectivityCheckOutcome == true; + bAbortStart = !bStartGame; + if (bStartGame) + { + m_bStartCommitted = true; + m_bPendingDeletion = true; + } + } + } + + if (bStartGame) + { + await StartGameAfterSuccessfulMeshCheck(lobbyDuringMeshCheck); + } + else if (bAbortStart) + { + string reason = bInvalidatedAtDecision + ? "QuickMatch auto-start was aborted because a player left during match setup." + : "QuickMatch auto-start was aborted because not all players were fully mesh-connected."; + await AbortQuickMatchAutoStart(reason); + } } return; @@ -1040,11 +1145,14 @@ await SendMatchmakingMessage(memberSession, if (bMinPlayersCountdownExpired || CurrentMemberCount() >= DesiredPlayers) { // reset min player countdown - m_bReachedMinPlayers = false; - m_timeReachedMinPlayers = -1; - - m_bWaitingOnLobbyJoins = true; - m_timeStartedWaitingOnLobbyJoins = Environment.TickCount64; + m_bReachedMinPlayers = false; + m_timeReachedMinPlayers = -1; + + lock (m_StateLock) + { + m_bWaitingOnLobbyJoins = true; + m_timeStartedWaitingOnLobbyJoins = Environment.TickCount64; + } // tell everyone UserSession? dummyHostUser = null; @@ -1098,10 +1206,10 @@ await SendMatchmakingMessage(memberSession, if (memberSession != null) { memberSession.QueueWebsocketSend(bytesJSON); - } - } - } - } + } + } + } + } } } else @@ -1211,7 +1319,12 @@ await SendMatchmakingMessage(memberSession, m_timeStartedWaitingOnLobbyJoins = -1; // wait 5 sec - m_StartTime = Environment.TickCount64 + 5000; + lock (m_StateLock) + { + m_StartTime = Environment.TickCount64 + 5000; + m_bWaitingOnLobbyJoins = false; + m_bHasStartedCountdown = true; + } foreach (MatchmakingBucketMember member in m_lstMembers) { UserSession? memberSession = member.GetAssociatedSession(); @@ -1221,9 +1334,6 @@ await SendMatchmakingMessage(memberSession, } } - m_bWaitingOnLobbyJoins = false; - m_bHasStartedCountdown = true; - // finalize the teams const int playlistMaxPlayerPerTeam = 2; bool bIsFFA = true; @@ -1246,31 +1356,16 @@ await SendMatchmakingMessage(memberSession, { teamID = 0; } - } - } - } + } + } + + await TriggerFullMeshConnectivityChecks(lobby); + } } } - // trigger full mesh check before issuing the final quickmatch start command - // do we have a countdown? - if (m_StartTime != -1) - { - if (Environment.TickCount64 >= m_StartTime) - { - m_StartTime = -1; - Lobby? lobby = lobbyManager.GetLobby(m_LobbyID); - if (lobby == null) - { - await AbortQuickMatchAutoStart("QuickMatch auto-start was aborted because the temporary lobby no longer exists."); - return; - } - - await TriggerFullMeshConnectivityChecks(lobby); - } - } - } + } } } @@ -1464,19 +1559,22 @@ public static async Task Tick() } // queue for deletion - m_lstBucketsPendingDeletion.AddRange(lstBucketsMergedNeedingDeleted); - - // cleanup any pending destruction (cannot do this in tick, collection will be modified) - foreach (MatchmakingBucket bucket in m_lstBucketsPendingDeletion) - { - if (m_dictMatchmakingBuckets.TryGetValue(bucket.PlaylistID, out var bucketBag)) + foreach (MatchmakingBucket bucket in lstBucketsMergedNeedingDeleted) + { + m_bucketsPendingDeletion.Enqueue(bucket); + } + + // Drain the queue rather than enumerating and clearing a shared list. A concurrent cancellation can + // enqueue a bucket while cleanup is running, and clearing the list would otherwise lose that request. + while (m_bucketsPendingDeletion.TryDequeue(out MatchmakingBucket? bucket)) + { + if (m_dictMatchmakingBuckets.TryGetValue(bucket.PlaylistID, out var bucketBag)) { // ConcurrentBag doesn't support Remove, so we filter and rebuild var remainingBuckets = bucketBag.Where(b => b != bucket).ToList(); - m_dictMatchmakingBuckets[bucket.PlaylistID] = new ConcurrentBag(remainingBuckets); - } - } - m_lstBucketsPendingDeletion.Clear(); + m_dictMatchmakingBuckets[bucket.PlaylistID] = new ConcurrentBag(remainingBuckets); + } + } List> lstDestroy = new(); foreach (WeakReference wrSession in lstSessions) @@ -1484,19 +1582,39 @@ public static async Task Tick() if (!wrSession.TryGetTarget(out UserSession? thisSession) || thisSession == null) { lstDestroy.Add(wrSession); - } - else - { - SharedUserData? thisSessionUserData = GenOnlineService.WebSocketManager.GetSharedDataForUser(thisSession.m_UserID); - - if (thisSessionUserData == null) - { - lstDestroy.Add(wrSession); - } - else - { - if (g_Playlists.TryGetValue(thisSession.MatchmakingPlaylistID, out Playlist? playlist)) - { + } + else + { + await thisSession.MatchmakingStateLock.WaitAsync(); + try + { + // A cancellation can remove the weak reference while this tick is iterating a snapshot. + // Re-check registration while holding the per-session gate before assigning any bucket. + if (!thisSession.IsRegisteredForMatchmaking || !IsPendingSession(thisSession)) + { + lstDestroy.Add(wrSession); + continue; + } + + SharedUserData? thisSessionUserData = GenOnlineService.WebSocketManager.GetSharedDataForUser(thisSession.m_UserID); + + if (thisSessionUserData == null) + { + lstDestroy.Add(wrSession); + } + else + { + PlayerStats? thisSessionStats = thisSessionUserData.GameStats; + if (thisSessionStats == null) + { + thisSession.IsRegisteredForMatchmaking = false; + lstDestroy.Add(wrSession); + await SendMatchmakingMessage(thisSession, "Matchmaking could not start because your player statistics are unavailable. Please try again."); + continue; + } + + if (g_Playlists.TryGetValue(thisSession.MatchmakingPlaylistID, out Playlist? playlist)) + { // TODO_MATCHAMAKING: Better way of tracking this, we need to know who is already in a bucket // Was the user in a bucket? if so theres nothing to do in terms of bucket management @@ -1541,8 +1659,8 @@ public static async Task Tick() } // must be within initial elo threshold for a join, otherwise we'll make a bucket and try to merge buckets using the elo iteration expansion algorithm - int eloExpansionToUse = (mmBucket.GetAvgElo() >= EloConfig.HighEloThreshold || thisSessionUserData.GameStats.EloRating >= EloConfig.HighEloThreshold) ? EloConfig.EloExpansionValue_HighELO : EloConfig.EloExpansionValue_Standard; - if (mmBucket.IsAvgEloWithinThreshold(thisSessionUserData.GameStats.EloRating, eloExpansionToUse)) + int eloExpansionToUse = (mmBucket.GetAvgElo() >= EloConfig.HighEloThreshold || thisSessionStats.EloRating >= EloConfig.HighEloThreshold) ? EloConfig.EloExpansionValue_HighELO : EloConfig.EloExpansionValue_Standard; + if (mmBucket.IsAvgEloWithinThreshold(thisSessionStats.EloRating, eloExpansionToUse)) { // TODO_MATCHMAKING: Squads if (mmBucket.HasSpaceForUsers(1, thisSession.ExeCRC, thisSession.IniCRC, thisSession.AnticheatID)) @@ -1579,14 +1697,19 @@ public static async Task Tick() lstDestroy.Add(wrSession); } } - } - else - { - // invalid playlist somehow - lstDestroy.Add(wrSession); - } - } - } + } + else + { + // invalid playlist somehow + lstDestroy.Add(wrSession); + } + } + } + finally + { + thisSession.MatchmakingStateLock.Release(); + } + } } // now remove @@ -1602,17 +1725,66 @@ public static async Task Tick() // TODO_MATCHMAKING: Deregister player if they disconnect or leave quickmatch private static ConcurrentList> lstSessions = new(); - private static ConcurrentList m_lstBucketsPendingDeletion = new(); - public static void DestroyBucket(MatchmakingBucket bucket) - { - bucket.MarkPendingDeletion(); - m_lstBucketsPendingDeletion.Add(bucket); - } - - public static async Task RegisterPlayer(UserSession plr, UInt16 playlistID, List mapIndices, UInt32 exe_crc, UInt32 ini_crc, EKnownAnticheatID anticheatID) - { - // validate the request - a bad playlist or out of range map index from a client must never reach a bucket - if (!g_Playlists.TryGetValue(playlistID, out Playlist? playlist)) + private static bool IsPendingSession(UserSession session) + { + foreach (WeakReference wrSession in lstSessions) + { + if (wrSession.TryGetTarget(out UserSession? pendingSession) && ReferenceEquals(pendingSession, session)) + { + return true; + } + } + + return false; + } + + private static void RemovePendingSession(UserSession session) + { + foreach (WeakReference wrSession in lstSessions.ToList()) + { + if (!wrSession.TryGetTarget(out UserSession? pendingSession) || ReferenceEquals(pendingSession, session)) + { + lstSessions.Remove(wrSession); + } + } + } + + private static async Task TryRequeueRegisteredPlayer(UserSession session, byte[] requeueActionJSON) + { + await session.MatchmakingStateLock.WaitAsync(); + try + { + if (!session.IsRegisteredForMatchmaking) + { + return false; + } + + if (!IsPendingSession(session)) + { + lstSessions.Add(new WeakReference(session)); + } + + session.UpdateSessionLobbyID(-1); + session.QueueWebsocketSend(requeueActionJSON); + return true; + } + finally + { + session.MatchmakingStateLock.Release(); + } + } + + private static ConcurrentQueue m_bucketsPendingDeletion = new(); + public static void DestroyBucket(MatchmakingBucket bucket) + { + bucket.MarkPendingDeletion(); + m_bucketsPendingDeletion.Enqueue(bucket); + } + + public static async Task RegisterPlayer(UserSession plr, UInt16 playlistID, List mapIndices, UInt32 exe_crc, UInt32 ini_crc, EKnownAnticheatID anticheatID) + { + // validate the request - a bad playlist or out of range map index from a client must never reach a bucket + if (!g_Playlists.TryGetValue(playlistID, out Playlist? playlist)) { await SendMatchmakingMessage(plr, "That playlist is not available. Matchmaking was not started."); return; @@ -1627,83 +1799,103 @@ public static async Task RegisterPlayer(UserSession plr, UInt16 playlistID, List if (validatedMapIndices.Count < minSelectedMaps) { await SendMatchmakingMessage(plr, String.Format("You must select at least {0} valid map(s) to matchmake in this playlist.", minSelectedMaps)); - return; - } - - // make sure a re-register (or a duplicate request) cannot leave the player queued twice, or queued while - // still sat in an existing bucket - either would matchmake them into two lobbies at once - RemoveSessionFromPendingList(plr); - RemovePlayerFromAllBuckets(plr); - - plr.MatchmakingPlaylistID = playlistID; - plr.MatchmakingMapIndicies = new ConcurrentList(validatedMapIndices); - plr.ExeCRC = exe_crc; - plr.IniCRC = ini_crc; - plr.AnticheatID = anticheatID; - lstSessions.Add(new WeakReference(plr)); - - await SendMatchmakingMessage(plr, "Started matchmaking... Searching for players..."); - } - - // NOTE: WeakReference does not implement value equality, so entries must be matched by their target session - private static void RemoveSessionFromPendingList(UserSession plr) - { - foreach (WeakReference wrSession in lstSessions) - { - if (!wrSession.TryGetTarget(out UserSession? thisSession) || thisSession == null || thisSession == plr) - { - lstSessions.Remove(wrSession); - } - } - } - - private static void RemovePlayerFromAllBuckets(UserSession plr) + return; + } + + bool bCancellationRejected; + await plr.MatchmakingStateLock.WaitAsync(); + try + { + // A duplicate registration must not leave the player queued twice or in two buckets. + plr.IsRegisteredForMatchmaking = false; + RemovePendingSession(plr); + bCancellationRejected = await RemovePlayerFromAllBuckets(plr); + + if (!bCancellationRejected) + { + plr.MatchmakingPlaylistID = playlistID; + plr.MatchmakingMapIndicies = new ConcurrentList(validatedMapIndices); + plr.ExeCRC = exe_crc; + plr.IniCRC = ini_crc; + plr.AnticheatID = anticheatID; + plr.IsRegisteredForMatchmaking = true; + lstSessions.Add(new WeakReference(plr)); + } + } + finally + { + plr.MatchmakingStateLock.Release(); + } + + if (bCancellationRejected) + { + await SendMatchmakingMessage(plr, "Matchmaking cannot be restarted because your game is already starting."); + return; + } + + await SendMatchmakingMessage(plr, "Started matchmaking... Searching for players..."); + } + + private static async Task RemovePlayerFromAllBuckets(UserSession plr) { var lobbyManager = ServiceLocator.Services.GetRequiredService(); + bool bCancellationRejected = false; - // also remove from any bucket we are in to avoid ghost buckets - foreach (var kvPair in m_dictMatchmakingBuckets) - { - foreach (MatchmakingBucket mmBucket in kvPair.Value) - { - if (mmBucket.HasPlayer(plr)) - { - // remove from QM lobby too - Lobby? lobby = lobbyManager.GetLobby(mmBucket.GetLobbyID()); - if (lobby != null) - { - LobbyMember? lobbyMember = lobby.GetMemberFromUserID(plr.m_UserID); - if (lobbyMember != null) - { - Console.WriteLine("User {0} Leave MM Lobby", plr.m_UserID); - lobby.RemoveMember(lobbyMember); - } - } - - // remove player - mmBucket.RemovePlayer(plr); - - // if we're the last player, destroy the bucket - if (mmBucket.CurrentMemberCount() == 0) - { - DestroyBucket(mmBucket); - } - } - } - } - } - - public static void DeregisterPlayer(UserSession plr) - { - var lobbyManager = ServiceLocator.Services.GetRequiredService(); - RemoveSessionFromPendingList(plr); - - // TODO_QUICKMATCH: What happens if the game is going to start? we should handle that, right now people probably goto game solo - - RemovePlayerFromAllBuckets(plr); - - // leave QM lobby too - Console.WriteLine("[Source 4] User {0} Leave Any Lobby", plr.m_UserID); - lobbyManager.LeaveAnyLobby(plr.m_UserID); + // also remove from any bucket we are in to avoid ghost buckets + foreach (var kvPair in m_dictMatchmakingBuckets) + { + foreach (MatchmakingBucket mmBucket in kvPair.Value) + { + bool bRemoved = mmBucket.RemovePlayer(plr, out bool bBucketCancellationRejected); + bCancellationRejected |= bBucketCancellationRejected; + + if (bRemoved) + { + // remove from QM lobby too + Lobby? lobby = lobbyManager.GetLobby(mmBucket.GetLobbyID()); + if (lobby != null) + { + LobbyMember? lobbyMember = lobby.GetMemberFromUserID(plr.m_UserID); + if (lobbyMember != null) + { + Console.WriteLine("User {0} Leave MM Lobby", plr.m_UserID); + await lobby.RemoveMember(lobbyMember); + } + } + + // if we're the last player, destroy the bucket + if (mmBucket.CurrentMemberCount() == 0) + { + DestroyBucket(mmBucket); + } + } + } + } + + return bCancellationRejected; + } + + public static async Task DeregisterPlayer(UserSession plr) + { + var lobbyManager = ServiceLocator.Services.GetRequiredService(); + bool bCancellationRejected; + await plr.MatchmakingStateLock.WaitAsync(); + try + { + plr.IsRegisteredForMatchmaking = false; + RemovePendingSession(plr); + bCancellationRejected = await RemovePlayerFromAllBuckets(plr); + } + finally + { + plr.MatchmakingStateLock.Release(); + } + + // leave QM lobby too + if (!bCancellationRejected) + { + Console.WriteLine("[Source 4] User {0} Leave Any Lobby", plr.m_UserID); + await lobbyManager.LeaveAnyLobby(plr.m_UserID); + } } } From 70b0309f6c594b6df0c9c4f01120dc902aad24c2 Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Fri, 21 Aug 2026 10:40:53 +0200 Subject: [PATCH 3/4] fix(matchmaking): delete aborted quickmatch lobbies --- GenOnlineService/MatchmakingManager.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/GenOnlineService/MatchmakingManager.cs b/GenOnlineService/MatchmakingManager.cs index 806dd3f..769305d 100644 --- a/GenOnlineService/MatchmakingManager.cs +++ b/GenOnlineService/MatchmakingManager.cs @@ -939,7 +939,7 @@ private async Task AbortQuickMatchAutoStart(string reason) if (quickMatchLobby != null) { foreach (UserSession memberSession in sessionsToRequeue) - { + { LobbyMember? lobbyMember = quickMatchLobby.GetMemberFromUserID(memberSession.m_UserID); if (lobbyMember != null) { @@ -950,6 +950,9 @@ private async Task AbortQuickMatchAutoStart(string reason) await quickMatchLobby.RemoveMember(lobbyMember); } } + + // Clean up slots left behind by disconnected sessions. + await lobbyManager.DeleteLobby(quickMatchLobby); } WebSocketMessage_Simple requeueAction = new WebSocketMessage_Simple(); From bce3778bf59e8ae71d68372aa4ef843999f172ea Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Fri, 21 Aug 2026 11:00:31 +0200 Subject: [PATCH 4/4] fix(lobby): prevent joins after deletion --- GenOnlineService/LobbyManager.cs | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/GenOnlineService/LobbyManager.cs b/GenOnlineService/LobbyManager.cs index 4a6dd99..af294c9 100644 --- a/GenOnlineService/LobbyManager.cs +++ b/GenOnlineService/LobbyManager.cs @@ -925,10 +925,15 @@ public async Task Tick() public async Task AddMember(UserSession playerSession, string strDisplayName, UInt16 userPreferredPort, bool bHasMap, UserLobbyPreferences lobbyPrefs) { // NOTE: AddMember is called async, so timing + slot determination could result in players being inserted in the same slot - await g_SlotLock.WaitAsync(); - try - { - // NOTE: this must be inside the lock, otherwise two concurrent joins for the same user can both pass + await g_SlotLock.WaitAsync(); + try + { + if (State != ELobbyState.GAME_SETUP) + { + return false; + } + + // NOTE: this must be inside the lock, otherwise two concurrent joins for the same user can both pass // the check and end up occupying two slots LobbyMember? existingMember = GetMemberFromUserID(playerSession.m_UserID); if (existingMember != null) // we're already in this lobby @@ -1413,7 +1418,15 @@ public bool HadAIAtStart() public async Task UpdateState(ELobbyState state) { - State = state; + await g_SlotLock.WaitAsync(); + try + { + State = state; + } + finally + { + g_SlotLock.Release(); + } // if start, init our AC probe if (state == ELobbyState.INGAME)