diff --git a/GenOnlineService/Database/Database.MatchHistory.cs b/GenOnlineService/Database/Database.MatchHistory.cs index aead105..45e3d7f 100644 --- a/GenOnlineService/Database/Database.MatchHistory.cs +++ b/GenOnlineService/Database/Database.MatchHistory.cs @@ -19,7 +19,6 @@ using GenOnlineService; using GenOnlineService.Controllers; using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Metadata.Builders; using Microsoft.EntityFrameworkCore.Query; using System; @@ -51,7 +50,6 @@ public class MatchHistoryEntry public uint ExeCRC { get; set; } public uint IniCRC { get; set; } public string? MapPath { get; set; } - // JSON slots public string? MemberSlot0 { get; set; } public string? MemberSlot1 { get; set; } @@ -158,6 +156,31 @@ public void Configure(EntityTypeBuilder entity) } } +public class ExternalPublicationEntry +{ + public long MatchId { get; set; } + public DateTime? NextAttemptAt { get; set; } + public DateTime? PublishedAt { get; set; } + public int Attempts { get; set; } + public string? LastError { get; set; } +} + +public class ExternalPublicationConfiguration : IEntityTypeConfiguration +{ + public void Configure(EntityTypeBuilder entity) + { + entity.ToTable("external_publication"); + entity.HasKey(e => e.MatchId); + entity.Property(e => e.MatchId).HasColumnName("match_id").ValueGeneratedNever(); + entity.Property(e => e.NextAttemptAt).HasColumnName("next_attempt_at").HasColumnType("datetime"); + entity.Property(e => e.PublishedAt).HasColumnName("published_at").HasColumnType("datetime"); + entity.Property(e => e.Attempts).HasColumnName("attempts").HasDefaultValue(0); + entity.Property(e => e.LastError).HasColumnName("last_error").HasMaxLength(512); + entity.HasIndex(e => new { e.PublishedAt, e.NextAttemptAt }) + .HasDatabaseName("ix_external_publication_pending"); + } +} + // TODO_EFCORE: put everything in below namespace namespace GenOnlineService { @@ -231,6 +254,8 @@ public MatchdataMemberModel() namespace Database { + public readonly record struct ExternalPublicationWorkItem(long MatchId, int Attempt); + // TODO_EFCORE: Consider moving to zero-serialization model public static class MatchHistory { @@ -299,7 +324,11 @@ private static Expression, SetPropertyC - private static async Task _getMemberSlot(AppDbContext db, long matchId, int slotIndex) + private static async Task _getMemberSlot( + AppDbContext db, + long matchId, + int slotIndex, + CancellationToken cancellationToken = default) { if (slotIndex < 0 || slotIndex > 7) return null; @@ -307,7 +336,7 @@ private static Expression, SetPropertyC return await db.MatchHistory .Where(m => m.MatchId == matchId) .Select(_slotSelectors[slotIndex]) - .FirstOrDefaultAsync(); + .FirstOrDefaultAsync(cancellationToken); } @@ -532,8 +561,9 @@ public static async Task CreatePlaceholderMatchHistory( } public static async Task DetermineLobbyWinnerIfNotPresent( - AppDbContext db, - GenOnlineService.Lobby lobby) + AppDbContext db, + GenOnlineService.Lobby lobby, + CancellationToken cancellationToken = default) { if (lobby == null || lobby.MatchID == 0) return; @@ -568,7 +598,7 @@ public static async Task DetermineLobbyWinnerIfNotPresent( if (kv.Value.side == Constants.OBSERVER_SIDE_VALUE) continue; - await UpdateMatchHistorySetWinFlag(db, lobby.MatchID, kv.Key, false); + await UpdateMatchHistorySetWinFlag(db, lobby.MatchID, kv.Key, false, cancellationToken); } return; } @@ -625,7 +655,7 @@ public static async Task DetermineLobbyWinnerIfNotPresent( ? kv.Value.team == conclusiveWinningTeam.Value : kv.Key == conclusiveWinningSlot; - await UpdateMatchHistorySetWinFlag(db, lobby.MatchID, kv.Key, isWinner); + await UpdateMatchHistorySetWinFlag(db, lobby.MatchID, kv.Key, isWinner, cancellationToken); } return; @@ -687,7 +717,7 @@ public static async Task DetermineLobbyWinnerIfNotPresent( if (kv.Value.side == Constants.OBSERVER_SIDE_VALUE) continue; - await UpdateMatchHistorySetWinFlag(db, lobby.MatchID, kv.Key, false); + await UpdateMatchHistorySetWinFlag(db, lobby.MatchID, kv.Key, false, cancellationToken); } return; } @@ -707,21 +737,22 @@ public static async Task DetermineLobbyWinnerIfNotPresent( (winningTeam != -1 && model.team == winningTeam); Console.WriteLine($"[WinnerDet] Match={lobby.MatchID}: marking slot={kv.Key} user={model.user_id} as {(isWinner ? "WINNER" : "loser")}."); - await UpdateMatchHistorySetWinFlag(db, lobby.MatchID, kv.Key, isWinner); + await UpdateMatchHistorySetWinFlag(db, lobby.MatchID, kv.Key, isWinner, cancellationToken); } } catch (Exception ex) { Console.WriteLine($"[ERROR] DetermineLobbyWinnerIfNotPresent failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + throw; } } public static async Task UpdateMatchHistorySetWinFlag( - AppDbContext db, - ulong matchId, - int slotIndex, - bool won) + AppDbContext db, + ulong matchId, + int slotIndex, + bool won, + CancellationToken cancellationToken = default) { if (matchId == 0 || slotIndex < 0 || slotIndex > 7) return; @@ -729,7 +760,7 @@ public static async Task UpdateMatchHistorySetWinFlag( try { // 1. Load the JSON for this slot - string? json = await _getMemberSlot(db, (long)matchId, slotIndex); + string? json = await _getMemberSlot(db, (long)matchId, slotIndex, cancellationToken); if (string.IsNullOrEmpty(json)) return; @@ -749,14 +780,17 @@ public static async Task UpdateMatchHistorySetWinFlag( var setter = BuildSetter(slotIndex, updatedJson); // 6. Execute update (single SQL UPDATE) - await db.MatchHistory + int updated = await db.MatchHistory .Where(m => m.MatchId == (long)matchId) - .ExecuteUpdateAsync(setter); + .ExecuteUpdateAsync(setter, cancellationToken); + + if (updated != 1) + throw new InvalidOperationException($"Match history entry {matchId} disappeared while its winner was being finalized."); } catch (Exception ex) { Console.WriteLine($"[ERROR] UpdateMatchHistorySetWinFlag failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + throw; } } @@ -932,30 +966,145 @@ public static async Task GetHighestMatchID(AppDbContext db) } // Called when a lobby is deleted, thats the true end of a match - public static async Task CommitLobbyToMatchHistory(AppDbContext db, GenOnlineService.Lobby lobby) + public static async Task FinalizeAndScheduleExternalPublication( + AppDbContext db, + GenOnlineService.Lobby lobby, + CancellationToken cancellationToken = default) { if (lobby.MatchID == 0) return; - try - { - await db.MatchHistory - .Where(m => m.MatchId == (long)lobby.MatchID && !m.Finished) - .ExecuteUpdateAsync(s => s - .SetProperty(m => m.Finished, true) - .SetProperty(m => m.TimeFinished, DateTime.UtcNow)); - } - catch (Exception ex) + await using var transaction = await db.Database.BeginTransactionAsync(cancellationToken); + + await CommitLobbyToMatchHistory(db, lobby, cancellationToken); + await DetermineLobbyWinnerIfNotPresent(db, lobby, cancellationToken); + await ScheduleExternalPublication(db, lobby.MatchID, cancellationToken); + + await transaction.CommitAsync(cancellationToken); + } + + private static async Task CommitLobbyToMatchHistory( + AppDbContext db, + GenOnlineService.Lobby lobby, + CancellationToken cancellationToken) + { + if (lobby.MatchID == 0) + return; + + await db.MatchHistory + .Where(m => m.MatchId == (long)lobby.MatchID && !m.Finished) + .ExecuteUpdateAsync(s => s + .SetProperty(m => m.Finished, true) + .SetProperty(m => m.TimeFinished, DateTime.UtcNow), cancellationToken); + + bool matchExists = await db.MatchHistory + .AnyAsync(m => m.MatchId == (long)lobby.MatchID && m.Finished, cancellationToken); + + if (!matchExists) + throw new InvalidOperationException($"Finished match history entry {lobby.MatchID} was not found."); + } + + private static async Task ScheduleExternalPublication( + AppDbContext db, + ulong matchId, + CancellationToken cancellationToken) + { + if (matchId == 0) + return; + + bool exists = await db.ExternalPublications.AnyAsync(p => p.MatchId == (long)matchId, cancellationToken); + if (!exists) { - Console.WriteLine($"[ERROR] CommitLobbyToMatchHistory failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + db.ExternalPublications.Add(new ExternalPublicationEntry + { + MatchId = (long)matchId, + NextAttemptAt = DateTime.UtcNow + }); + await db.SaveChangesAsync(cancellationToken); } + + bool publicationIsDurable = await db.MatchHistory.AnyAsync( + m => m.MatchId == (long)matchId && m.Finished, + cancellationToken); + publicationIsDurable &= await db.ExternalPublications.AnyAsync( + p => p.MatchId == (long)matchId, + cancellationToken); + + if (!publicationIsDurable) + throw new InvalidOperationException($"Match history entry {matchId} could not be scheduled for external publication."); + } + + public static async Task> GetPendingExternalPublications( + AppDbContext db, + DateTime utcNow, + int maxCount, + CancellationToken cancellationToken) + { + List pending = await db.ExternalPublications + .Where(p => p.PublishedAt == null && p.NextAttemptAt != null && p.NextAttemptAt <= utcNow) + .OrderBy(p => p.NextAttemptAt) + .ThenBy(p => p.MatchId) + .Take(maxCount) + .ToListAsync(cancellationToken); + + return pending + .Select(item => new ExternalPublicationWorkItem(item.MatchId, item.Attempts + 1)) + .ToList(); + } + + public static async Task MarkExternalPublicationSucceeded( + AppDbContext db, + ulong matchId, + int attempt, + CancellationToken cancellationToken) + { + if (matchId == 0) + throw new ArgumentOutOfRangeException(nameof(matchId)); + + int updated = await db.ExternalPublications + .Where(p => p.MatchId == (long)matchId && p.PublishedAt == null) + .ExecuteUpdateAsync(s => s + .SetProperty(p => p.PublishedAt, DateTime.UtcNow) + .SetProperty(p => p.NextAttemptAt, (DateTime?)null) + .SetProperty(p => p.Attempts, attempt) + .SetProperty(p => p.LastError, (string?)null), cancellationToken); + + if (updated != 1) + throw new InvalidOperationException($"Match history entry {matchId} could not be acknowledged after external publication."); + } + + public static async Task MarkExternalPublicationFailed( + AppDbContext db, + ulong matchId, + int attempt, + DateTime? nextAttemptAt, + string? errorMessage, + CancellationToken cancellationToken) + { + if (matchId == 0) + throw new ArgumentOutOfRangeException(nameof(matchId)); + + string? truncatedError = errorMessage; + if (!string.IsNullOrEmpty(truncatedError) && truncatedError.Length > 512) + truncatedError = truncatedError[..512]; + + int updated = await db.ExternalPublications + .Where(p => p.MatchId == (long)matchId && p.PublishedAt == null) + .ExecuteUpdateAsync(s => s + .SetProperty(p => p.NextAttemptAt, nextAttemptAt) + .SetProperty(p => p.Attempts, attempt) + .SetProperty(p => p.LastError, truncatedError), cancellationToken); + + if (updated != 1) + throw new InvalidOperationException($"Match history entry {matchId} could not be rescheduled after publication failure."); } internal static async Task LoadMatchHistoryEntryAsync( - AppDbContext db, long matchId) + AppDbContext db, + long matchId, + CancellationToken cancellationToken = default) { - var row = await db.MatchHistory.FirstOrDefaultAsync(m => m.MatchId == matchId); + var row = await db.MatchHistory.FirstOrDefaultAsync(m => m.MatchId == matchId, cancellationToken); if (row == null) return null; diff --git a/GenOnlineService/Database/Database.User.cs b/GenOnlineService/Database/Database.User.cs index 1e796e8..aa28555 100644 --- a/GenOnlineService/Database/Database.User.cs +++ b/GenOnlineService/Database/Database.User.cs @@ -731,7 +731,11 @@ public static async Task SetFavorite_Color( } } - public static async Task SaveELOData(AppDbContext db, long userId, EloData newEloData) + public static async Task SaveELOData( + AppDbContext db, + long userId, + EloData newEloData, + CancellationToken cancellationToken = default) { try { @@ -740,8 +744,12 @@ await db.Users .ExecuteUpdateAsync(setters => setters .SetProperty(u => u.EloRating, newEloData.Rating) .SetProperty(u => u.MonthlyEloRating, newEloData.MonthlyRating) - .SetProperty(u => u.EloNumberOfMatches, newEloData.NumMatches) - ); + .SetProperty(u => u.EloNumberOfMatches, newEloData.NumMatches), + cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; } catch (Exception ex) { @@ -750,5 +758,25 @@ await db.Users } } + public static async Task SaveExternalELOData( + AppDbContext db, + long userId, + EloData newEloData, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(db); + ArgumentNullException.ThrowIfNull(newEloData); + + int updated = await db.Users + .Where(u => u.ID == userId) + .ExecuteUpdateAsync(setters => setters + .SetProperty(u => u.EloRating, newEloData.Rating) + .SetProperty(u => u.MonthlyEloRating, newEloData.MonthlyRating) + .SetProperty(u => u.EloNumberOfMatches, newEloData.NumMatches), + cancellationToken); + + return updated == 1; + } + } -} \ No newline at end of file +} diff --git a/GenOnlineService/Database/Database.cs b/GenOnlineService/Database/Database.cs index 9b22772..88a840f 100644 --- a/GenOnlineService/Database/Database.cs +++ b/GenOnlineService/Database/Database.cs @@ -31,6 +31,7 @@ public class AppDbContext : DbContext public DbSet ServiceStats => Set(); public DbSet PendingLogins => Set(); public DbSet MatchHistory => Set(); + public DbSet ExternalPublications => Set(); public DbSet UserStats => Set(); public DbSet Friends => Set(); @@ -65,6 +66,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) modelBuilder.ApplyConfiguration(new ServiceStatsConfiguration()); modelBuilder.ApplyConfiguration(new PendingLoginConfiguration()); modelBuilder.ApplyConfiguration(new MatchHistoryConfiguration()); + modelBuilder.ApplyConfiguration(new ExternalPublicationConfiguration()); modelBuilder.ApplyConfiguration(new UserStatsConfiguration()); modelBuilder.ApplyConfiguration(new FriendConfiguration()); modelBuilder.ApplyConfiguration(new FriendRequestConfiguration()); diff --git a/GenOnlineService/Database_Structure/structure.sql b/GenOnlineService/Database_Structure/structure.sql index 57199e3..1f6cd16 100644 --- a/GenOnlineService/Database_Structure/structure.sql +++ b/GenOnlineService/Database_Structure/structure.sql @@ -125,6 +125,16 @@ CREATE TABLE IF NOT EXISTS `match_history` ( PRIMARY KEY (`match_id`) ) ENGINE=InnoDB AUTO_INCREMENT=563766 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; +CREATE TABLE IF NOT EXISTS `external_publication` ( + `match_id` bigint(20) NOT NULL, + `next_attempt_at` datetime DEFAULT NULL, + `published_at` datetime DEFAULT NULL, + `attempts` int(11) NOT NULL DEFAULT 0, + `last_error` varchar(512) DEFAULT NULL, + PRIMARY KEY (`match_id`), + KEY `ix_external_publication_pending` (`published_at`,`next_attempt_at`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; + -- Data exporting was unselected. -- Dumping structure for table go_production.pending_logins diff --git a/GenOnlineService/Database_Structure/upgrade_20260816_external_leaderboard_publication.sql b/GenOnlineService/Database_Structure/upgrade_20260816_external_leaderboard_publication.sql new file mode 100644 index 0000000..bd872e7 --- /dev/null +++ b/GenOnlineService/Database_Structure/upgrade_20260816_external_leaderboard_publication.sql @@ -0,0 +1,9 @@ +CREATE TABLE IF NOT EXISTS `external_publication` ( + `match_id` bigint(20) NOT NULL, + `next_attempt_at` datetime DEFAULT NULL, + `published_at` datetime DEFAULT NULL, + `attempts` int(11) NOT NULL DEFAULT 0, + `last_error` varchar(512) DEFAULT NULL, + PRIMARY KEY (`match_id`), + KEY `ix_external_publication_pending` (`published_at`,`next_attempt_at`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; diff --git a/GenOnlineService/ExternalLeaderboardPublicationQueue.cs b/GenOnlineService/ExternalLeaderboardPublicationQueue.cs new file mode 100644 index 0000000..5efe8b1 --- /dev/null +++ b/GenOnlineService/ExternalLeaderboardPublicationQueue.cs @@ -0,0 +1,288 @@ +using GenOnlineService.Controllers; +using Microsoft.EntityFrameworkCore; +using System.Net; +using System.Text.Json; + +namespace GenOnlineService +{ + public sealed class ExternalLeaderboardPublicationWorker : BackgroundService + { + private static readonly TimeSpan c_PollInterval = TimeSpan.FromSeconds(1); + private static readonly TimeSpan[] c_RetryDelays = new[] + { + TimeSpan.FromSeconds(2), + TimeSpan.FromSeconds(4), + TimeSpan.FromSeconds(15), + TimeSpan.FromSeconds(30), + TimeSpan.FromMinutes(1), + TimeSpan.FromMinutes(2), + TimeSpan.FromMinutes(2) + }; + private const int c_MaxBatchSize = 8; + + private readonly IDbContextFactory _dbFactory; + + public ExternalLeaderboardPublicationWorker(IDbContextFactory dbFactory) + { + _dbFactory = dbFactory; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (!stoppingToken.IsCancellationRequested) + { + try + { + int processed = await PublishPendingMatches(stoppingToken); + if (processed > 0) + continue; + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + catch (Exception ex) + { + Console.WriteLine($"[ERROR] External leaderboard publication poll failed: {ex}"); + SentrySdk.CaptureException(ex); + } + + try + { + await Task.Delay(c_PollInterval, stoppingToken); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + } + } + + private async Task PublishPendingMatches(CancellationToken stoppingToken) + { + // This worker is intentionally a single consumer. Use a distributed queue before + // running more than one service instance. + List pending; + await using (AppDbContext db = await _dbFactory.CreateDbContextAsync(stoppingToken)) + { + pending = await Database.MatchHistory.GetPendingExternalPublications( + db, + DateTime.UtcNow, + c_MaxBatchSize, + stoppingToken); + } + + foreach (Database.ExternalPublicationWorkItem item in pending) + { + stoppingToken.ThrowIfCancellationRequested(); + + try + { + await PublishItem(item, stoppingToken); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + Console.WriteLine($"[ERROR] External publication bookkeeping failed for match {item.MatchId}: {ex}"); + SentrySdk.CaptureException(ex); + throw; + } + } + + return pending.Count; + } + + private async Task PublishItem( + Database.ExternalPublicationWorkItem item, + CancellationToken stoppingToken) + { + try + { + MatchHistory_Entry? matchEntry; + await using (AppDbContext loadDb = await _dbFactory.CreateDbContextAsync(stoppingToken)) + { + matchEntry = await Database.MatchHistory.LoadMatchHistoryEntryAsync( + loadDb, + item.MatchId, + stoppingToken); + } + + if (matchEntry == null) + throw new InvalidOperationException($"MatchHistory entry not found for match ID {item.MatchId}."); + + string responseBody = await ExternalLeaderboardsClient.PostMatchResultAsync(matchEntry, stoppingToken); + try + { + await ApplyRatingsResponse(matchEntry, responseBody, stoppingToken); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + Console.WriteLine($"[WARNING] Match {item.MatchId} was ingested, but its optional ratings response could not be applied: {ex}"); + SentrySdk.CaptureException(ex); + } + + await using AppDbContext completionDb = await _dbFactory.CreateDbContextAsync(stoppingToken); + await Database.MatchHistory.MarkExternalPublicationSucceeded( + completionDb, + (ulong)item.MatchId, + item.Attempt, + stoppingToken); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + Console.WriteLine($"[WARNING] Failed to publish match {item.MatchId} to the external leaderboard (attempt {item.Attempt}): {ex.Message}"); + await RecordFailure(item, ex, stoppingToken); + } + } + + private async Task ApplyRatingsResponse( + MatchHistory_Entry matchEntry, + string responseBody, + CancellationToken stoppingToken) + { + long matchId = matchEntry.match_id; + if (string.IsNullOrWhiteSpace(responseBody)) + { + if (matchEntry.lobby_type == ELobbyType.QuickMatch) + Console.WriteLine($"[WARNING] External Match Ingest response for QuickMatch {matchId} contained no ratings body."); + return; + } + + EloRefreshResponse? refreshResponse; + try + { + refreshResponse = JsonSerializer.Deserialize(responseBody); + } + catch (JsonException ex) + { + Console.WriteLine($"[WARNING] External Match Ingest response for match {matchId} could not be deserialized: {ex.Message}"); + SentrySdk.CaptureException(ex); + return; + } + + if (refreshResponse?.data == null) + { + Console.WriteLine($"[WARNING] External Match Ingest response for match {matchId} contained no ratings data."); + return; + } + + HashSet expectedPlayerIds = matchEntry.members + .Where(member => member.HasValue) + .Select(member => member.GetValueOrDefault().user_id) + .ToHashSet(); + List<(long UserId, EloData Data)> pendingUpdates = new(); + + foreach ((long userId, EloRefreshEntry updatedPlayer) in refreshResponse.data) + { + if (!expectedPlayerIds.Contains(userId)) + { + Console.WriteLine($"[WARNING] External Match Ingest response for match {matchId} contained unexpected player_id {userId}; skipping (ELO left unchanged)."); + continue; + } + + if (updatedPlayer?.overall == null || updatedPlayer.season == null) + { + Console.WriteLine($"[WARNING] External Match Ingest response for match {matchId} contained incomplete ratings for player_id {userId}; skipping."); + continue; + } + + pendingUpdates.Add(( + userId, + new EloData( + updatedPlayer.overall.rating, + updatedPlayer.season.rating, + updatedPlayer.overall.matches))); + } + + if (pendingUpdates.Count == 0) + return; + + List<(long UserId, EloData Data)> savedUpdates = new(pendingUpdates.Count); + await using (AppDbContext db = await _dbFactory.CreateDbContextAsync(stoppingToken)) + await using (var transaction = await db.Database.BeginTransactionAsync(stoppingToken)) + { + foreach ((long userId, EloData data) in pendingUpdates) + { + bool saved = await Database.Users.SaveExternalELOData( + db, + userId, + data, + stoppingToken); + + if (saved) + savedUpdates.Add((userId, data)); + else + Console.WriteLine($"[WARNING] External Match Ingest response for match {matchId} referenced missing user_id {userId}; skipping."); + } + + await transaction.CommitAsync(stoppingToken); + } + + foreach ((long userId, EloData data) in savedUpdates) + { + var sharedData = WebSocketManager.GetSharedDataForUser(userId); + if (sharedData?.GameStats == null) + continue; + + sharedData.GameStats.EloRating = data.Rating; + sharedData.GameStats.EloMatches = data.NumMatches; + sharedData.GameStats.MonthlyEloRating = data.MonthlyRating; + } + } + + private async Task RecordFailure( + Database.ExternalPublicationWorkItem item, + Exception publicationException, + CancellationToken stoppingToken) + { + bool retry = IsRetryable(publicationException) && item.Attempt <= c_RetryDelays.Length; + DateTime? nextAttemptAt = retry + ? DateTime.UtcNow.Add(c_RetryDelays[item.Attempt - 1]) + : null; + await using AppDbContext db = await _dbFactory.CreateDbContextAsync(stoppingToken); + + await Database.MatchHistory.MarkExternalPublicationFailed( + db, + (ulong)item.MatchId, + item.Attempt, + nextAttemptAt, + publicationException.Message, + stoppingToken); + + if (!retry) + { + Console.WriteLine($"[ERROR] External publication for match {item.MatchId} stopped after {item.Attempt} attempts."); + SentrySdk.CaptureException(publicationException); + } + } + + private static bool IsRetryable(Exception exception) + { + if (exception is HttpRequestException httpException) + { + if (httpException.StatusCode is not HttpStatusCode status) + return true; + + if ((int)status >= 400 && (int)status < 500) + return status is HttpStatusCode.RequestTimeout or HttpStatusCode.TooManyRequests; + + return true; + } + + // Database, configuration and other infrastructure failures are retried. Only + // explicit permanent HTTP client errors stop immediately. + return true; + } + } +} diff --git a/GenOnlineService/ExternalLeaderboardsClient.cs b/GenOnlineService/ExternalLeaderboardsClient.cs index 4f14452..af697f3 100644 --- a/GenOnlineService/ExternalLeaderboardsClient.cs +++ b/GenOnlineService/ExternalLeaderboardsClient.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using System.Linq; using System.Net; using System.Net.Http; using System.Net.Http.Headers; @@ -10,20 +9,20 @@ using System.Text; using System.Text.Json; using System.Threading.Tasks; +using GenOnlineService.Controllers; using Microsoft.Extensions.Configuration; -using Polly; namespace GenOnlineService { public class EloRefreshResponse { - public Dictionary data { get; set; } + public Dictionary data { get; set; } = null!; } public class EloRefreshEntry { - public EloRefreshRating overall { get; set; } - public EloRefreshRating season { get; set; } + public EloRefreshRating overall { get; set; } = null!; + public EloRefreshRating season { get; set; } = null!; } public class EloRefreshRating @@ -34,52 +33,49 @@ public class EloRefreshRating public static class ExternalLeaderboardsClient { - private static void GetExternalLeaderboardsConfig(out string postUrl, out string getUrl, out string postToken, out string getToken) + private static IConfigurationSection GetExternalLeaderboardsConfigSection() { - postUrl = string.Empty; - getUrl = string.Empty; - postToken = string.Empty; - getToken = string.Empty; - if (Program.g_Config == null) - { throw new Exception("Config not loaded"); - } - IConfigurationSection? configSection = Program.g_Config.GetSection("ExternalLeaderboards"); - if (configSection == null) - { + IConfigurationSection configSection = Program.g_Config.GetSection("ExternalLeaderboards"); + if (!configSection.Exists()) throw new Exception("ExternalLeaderboards section missing in config"); - } + + return configSection; + } + + private static void GetExternalLeaderboardsPostConfig(out string postUrl, out string postToken) + { + IConfigurationSection configSection = GetExternalLeaderboardsConfigSection(); string? sectionPostUrl = configSection.GetValue("PostUrl"); - string? sectionGetUrl = configSection.GetValue("GetUrl"); string? sectionPostToken = configSection.GetValue("PostToken"); - string? sectionGetToken = configSection.GetValue("GetToken"); if (string.IsNullOrEmpty(sectionPostUrl)) - { throw new Exception("ExternalLeaderboards PostUrl missing in config"); - } - - if (string.IsNullOrEmpty(sectionGetUrl)) - { - throw new Exception("ExternalLeaderboards GetUrl missing in config"); - } if (string.IsNullOrEmpty(sectionPostToken)) - { throw new Exception("ExternalLeaderboards PostToken missing in config"); - } + + postUrl = sectionPostUrl; + postToken = sectionPostToken; + } + + private static void GetExternalLeaderboardsConfig(out string getUrl, out string getToken) + { + IConfigurationSection configSection = GetExternalLeaderboardsConfigSection(); + + string? sectionGetUrl = configSection.GetValue("GetUrl"); + string? sectionGetToken = configSection.GetValue("GetToken"); + + if (string.IsNullOrEmpty(sectionGetUrl)) + throw new Exception("ExternalLeaderboards GetUrl missing in config"); if (string.IsNullOrEmpty(sectionGetToken)) - { throw new Exception("ExternalLeaderboards GetToken missing in config"); - } - postUrl = sectionPostUrl; getUrl = sectionGetUrl; - postToken = sectionPostToken; getToken = sectionGetToken; } @@ -122,126 +118,55 @@ private static SocketsHttpHandler CreateLeaderboardsHandler() }; } - public static async Task PostMatchResultAsync(AppDbContext db, Lobby lobby) + public static async Task PostMatchResultAsync(MatchHistory_Entry matchEntry, CancellationToken cancellationToken = default) { - if (lobby.MatchID == 0) - return; + ArgumentNullException.ThrowIfNull(matchEntry); - try - { - GetExternalLeaderboardsConfig(out string postUrl, out _, out string postToken, out _); - - // Load the match payload - var matchEntry = await Database.MatchHistory.LoadMatchHistoryEntryAsync(db, (long)lobby.MatchID); - if (matchEntry == null) - { - Console.WriteLine($"[WARNING] MatchHistory entry not found for match ID {lobby.MatchID}"); - return; - } - - // Serialize payload to JSON - string payloadJson = JsonSerializer.Serialize(matchEntry); - - // Configure Polly wait-and-retry policy with exponential backoff on HTTP/Socket errors - var retryPolicy = Policy - .Handle() - .Or() - .Or() - .WaitAndRetryAsync(new[] - { - TimeSpan.FromSeconds(2), - TimeSpan.FromSeconds(4), - TimeSpan.FromSeconds(15), - TimeSpan.FromSeconds(30), - TimeSpan.FromMinutes(1), - TimeSpan.FromMinutes(2), - TimeSpan.FromMinutes(2) - }, (exception, timeSpan, retryCount, context) => - { - Console.WriteLine($"[WARNING] External Match ingest POST failed (attempt {retryCount}). Retrying in {timeSpan.TotalSeconds}s. Error: {exception.Message}"); - }); + long matchID = matchEntry.match_id; + if (matchID <= 0) + throw new ArgumentOutOfRangeException(nameof(matchEntry), "Match ID must be greater than zero."); - HttpResponseMessage? response = null; - string? responseBody = null; - - await retryPolicy.ExecuteAsync(async () => - { - HttpClient client = g_LeaderboardsClient.Value; - - using (var request = new HttpRequestMessage(HttpMethod.Post, postUrl)) - { - request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", postToken); - request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); - request.Content = new StringContent(payloadJson, Encoding.UTF8, "application/json"); - - var sw = Stopwatch.StartNew(); - using (response = await client.SendAsync(request)) - { - sw.Stop(); + GetExternalLeaderboardsPostConfig(out string postUrl, out string postToken); - Console.WriteLine($"[INFO] External Match Ingest POST Response for match {lobby.MatchID} was received in {sw.ElapsedMilliseconds}ms (status: {response.StatusCode})."); + // The external ingest endpoint must deduplicate repeated submissions by match_id. + string payloadJson = JsonSerializer.Serialize(matchEntry); - // Explicitly verify response success inside execution block to ensure retry triggers on HTTP error statuses - response.EnsureSuccessStatusCode(); + string responseBody = string.Empty; + HttpClient client = g_LeaderboardsClient.Value; - // NOTE: read the body here, the response is disposed once we leave this block - responseBody = await response.Content.ReadAsStringAsync(); - } - } - }); - - if (responseBody == null) - { - Console.WriteLine($"[ERROR] External Match Ingest POST failed for match {lobby.MatchID}."); - return; - } - - var refreshResponse = JsonSerializer.Deserialize(responseBody); - if (refreshResponse?.data == null) - { - Console.WriteLine($"[WARNING] External Match Ingest response body contains no data or could not be deserialized: {responseBody}"); - return; - } - - // Only player IDs that were actually part of this match are valid recipients of an ELO update. - var expectedPlayerIds = new HashSet(matchEntry.members.Where(m => m.HasValue).Select(m => m.Value.user_id)); + using (var request = new HttpRequestMessage(HttpMethod.Post, postUrl)) + { + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", postToken); + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + request.Content = new StringContent(payloadJson, Encoding.UTF8, "application/json"); - foreach (var (userId, updatedPlayer) in refreshResponse.data) + var sw = Stopwatch.StartNew(); + using (HttpResponseMessage response = await client.SendAsync(request, cancellationToken)) { - if (!expectedPlayerIds.Contains(userId)) - { - Console.WriteLine($"[WARNING] External Match Ingest response for match {lobby.MatchID} contained unexpected player_id {userId}; skipping (ELO left unchanged)."); - continue; - } + sw.Stop(); - int newRating = updatedPlayer.overall.rating; - int newMatches = updatedPlayer.overall.matches; - int newMonthlyRating = updatedPlayer.season.rating; + Console.WriteLine($"[INFO] External Match Ingest POST Response for match {matchID} was received in {sw.ElapsedMilliseconds}ms (status: {response.StatusCode})."); - // Update in-memory session cache if the player is online - var sharedData = WebSocketManager.GetSharedDataForUser(userId); - if (sharedData?.GameStats != null) + responseBody = await response.Content.ReadAsStringAsync(cancellationToken); + if (!response.IsSuccessStatusCode) { - sharedData.GameStats.EloRating = newRating; - sharedData.GameStats.EloMatches = newMatches; - sharedData.GameStats.MonthlyEloRating = newMonthlyRating; + string errorBody = responseBody.Length <= 256 ? responseBody : responseBody[..256]; + throw new HttpRequestException( + $"External Match Ingest returned {(int)response.StatusCode} ({response.StatusCode}): {errorBody}", + null, + response.StatusCode); } - - // Call SaveELOData to persist as fallback - await Database.Users.SaveELOData(db, userId, new EloData(newRating, newMonthlyRating, newMatches)); } } - catch (Exception ex) - { - Console.WriteLine($"[ERROR] Exception during External Match Ingest POST: {ex.Message}"); - } + + return responseBody; } public static async Task GetEloFromApi(long playerId) { try { - GetExternalLeaderboardsConfig(out _, out string getUrl, out _, out string getToken); + GetExternalLeaderboardsConfig(out string getUrl, out string getToken); string requestUrl = getUrl.Replace("{playerId}", playerId.ToString()); diff --git a/GenOnlineService/GenOnlineService.csproj b/GenOnlineService/GenOnlineService.csproj index 29a5013..b09890d 100644 --- a/GenOnlineService/GenOnlineService.csproj +++ b/GenOnlineService/GenOnlineService.csproj @@ -49,7 +49,6 @@ - diff --git a/GenOnlineService/LobbyManager.cs b/GenOnlineService/LobbyManager.cs index 61e5b79..9ef05f7 100644 --- a/GenOnlineService/LobbyManager.cs +++ b/GenOnlineService/LobbyManager.cs @@ -492,9 +492,6 @@ public Lobby(Int64 lobby_id, UserSession owner, string name, ELobbyState state, Members[i] = placeholderMember; } - using var scope = ServiceLocator.Services.CreateScope(); - var factory = scope.ServiceProvider.GetRequiredService>(); - _db = factory.CreateDbContext(); } public event Action? OnLobbyNeedsDestroyed; @@ -1129,7 +1126,6 @@ public void ForceReady() private int m_cachedAtStart_numOpen = -1; private int m_cachedAtStart_numClosed = -1; private int m_cachedAtStart_numAI = -1; - private AppDbContext _db; // TODO: Really, client also shouldnt upload data we arent going to process in this situation, its wasteful public bool WasPVPAtStart() @@ -1177,7 +1173,10 @@ public async Task UpdateState(ELobbyState state) try { // create placeholder - await Database.MatchHistory.CreatePlaceholderMatchHistory(_db, this); + using var scope = ServiceLocator.Services.CreateScope(); + var factory = scope.ServiceProvider.GetRequiredService>(); + await using var db = await factory.CreateDbContextAsync(); + await Database.MatchHistory.CreatePlaceholderMatchHistory(db, this); } catch (Exception ex) { @@ -1406,15 +1405,10 @@ public class LobbyManager private Int64 m_NextLobbyID = 0; private readonly IServiceProvider _services; - private readonly AppDbContext _db; public LobbyManager(IServiceProvider services) { _services = services; - - var scope = _services.CreateScope(); - var factory = scope.ServiceProvider.GetRequiredService>(); - _db = factory.CreateDbContext(); } public async Task Cleanup() @@ -1720,15 +1714,19 @@ public async Task DeleteLobby(Lobby lobby) { try { + using var scope = _services.CreateScope(); + var factory = scope.ServiceProvider.GetRequiredService>(); + await using var db = await factory.CreateDbContextAsync(); + if (lobby.State != ELobbyState.COMPLETE) { // make done await lobby.UpdateState(ELobbyState.COMPLETE); - - // attempt to commit it - await Database.MatchHistory.CommitLobbyToMatchHistory(_db, lobby); } + // Persist publication before removing the lobby. + await Database.MatchHistory.FinalizeAndScheduleExternalPublication(db, lobby); + // delete bool bRemoved = m_dictLobbies.Remove(lobby.LobbyID, out _); await WebSocketManager.SendNewOrDeletedLobbyToAllNetworkRoomMembers(lobby.NetworkRoomID); @@ -1738,13 +1736,6 @@ public async Task DeleteLobby(Lobby lobby) { // unsubscribe from self-destruct event lobby.OnLobbyNeedsDestroyed -= HandleLobbyNeedsDestroyed; - - // make sure we have a winner - await Database.MatchHistory.DetermineLobbyWinnerIfNotPresent(_db, lobby); - - // Post match result to external leaderboard API for every lobby type. - // Only QuickMatch responses are expected to carry a ratings body. - await ExternalLeaderboardsClient.PostMatchResultAsync(_db, lobby); } return bRemoved; diff --git a/GenOnlineService/Program.cs b/GenOnlineService/Program.cs index b9b823d..84951a3 100644 --- a/GenOnlineService/Program.cs +++ b/GenOnlineService/Program.cs @@ -862,6 +862,7 @@ public static async Task Main(string[] args) g_Discord = new DiscordBot(); } + builder.Services.AddHostedService(); builder.Services.AddSingleton(); var rateLimitingSettings = Program.g_Config.GetSection("RateLimiting"); diff --git a/README.md b/README.md index 4418bde..0588860 100644 --- a/README.md +++ b/README.md @@ -39,4 +39,5 @@ GeneralsOnline Game Services Code provides RESTful web services which act as a r - Build the solution for x64, Windows (or your architecture & OS if different) - Edit appsettings.json and fill out any TODO sections (e.g. token settings, database settings) - Import the SQL structure to your database (GenOnlineService\Database_Structure\structure.sql) +- When upgrading an existing database, apply new `GenOnlineService\Database_Structure\upgrade_*.sql` files in date order before starting the updated service - Run GenOnlineService.exe