Skip to content

fix(leaderboards): move match publication to a durable background queue #42

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
tintinhamans wants to merge 2 commits into
base: main
Choose a base branch
from
Open

fix(leaderboards): move match publication to a durable background queue #42

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f7e0b1e
fix(leaderboards): move match publication to a durable background queue
tintinhamans Aug 20, 2026
f6cb699
refactor: rename external leaderboard GET config helper
tintinhamans Aug 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
215 changes: 182 additions & 33 deletions GenOnlineService/Database/Database.MatchHistory.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
using GenOnlineService;
using GenOnlineService.Controllers;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Metadata.Builders;
using Microsoft.EntityFrameworkCore.Query;
using System;
Expand Down Expand Up @@ -51,7 +50,6 @@ public class MatchHistoryEntry
public uint ExeCRC { get; set; }
public uint IniCRC { get; set; }
public string? MapPath { get; set; }

// JSON slots
public string? MemberSlot0 { get; set; }
public string? MemberSlot1 { get; set; }
Expand Down Expand Up @@ -158,6 +156,31 @@ public void Configure(EntityTypeBuilder<MatchHistoryEntry> entity)
}
}

public class ExternalPublicationEntry
{
public long MatchId { get; set; }
public DateTime? NextAttemptAt { get; set; }
public DateTime? PublishedAt { get; set; }
public int Attempts { get; set; }
public string? LastError { get; set; }
}

public class ExternalPublicationConfiguration : IEntityTypeConfiguration<ExternalPublicationEntry>
{
public void Configure(EntityTypeBuilder<ExternalPublicationEntry> entity)
{
entity.ToTable("external_publication");
entity.HasKey(e => e.MatchId);
entity.Property(e => e.MatchId).HasColumnName("match_id").ValueGeneratedNever();
entity.Property(e => e.NextAttemptAt).HasColumnName("next_attempt_at").HasColumnType("datetime");
entity.Property(e => e.PublishedAt).HasColumnName("published_at").HasColumnType("datetime");
entity.Property(e => e.Attempts).HasColumnName("attempts").HasDefaultValue(0);
entity.Property(e => e.LastError).HasColumnName("last_error").HasMaxLength(512);
entity.HasIndex(e => new { e.PublishedAt, e.NextAttemptAt })
.HasDatabaseName("ix_external_publication_pending");
}
}

// TODO_EFCORE: put everything in below namespace
namespace GenOnlineService
{
Expand Down Expand Up @@ -231,6 +254,8 @@ public MatchdataMemberModel()

namespace Database
{
public readonly record struct ExternalPublicationWorkItem(long MatchId, int Attempt);

// TODO_EFCORE: Consider moving to zero-serialization model
public static class MatchHistory
{
Expand Down Expand Up @@ -299,15 +324,19 @@ private static Expression<Func<SetPropertyCalls<MatchHistoryEntry>, SetPropertyC



private static async Task<string?> _getMemberSlot(AppDbContext db, long matchId, int slotIndex)
private static async Task<string?> _getMemberSlot(
AppDbContext db,
long matchId,
int slotIndex,
CancellationToken cancellationToken = default)
{
if (slotIndex < 0 || slotIndex > 7)
return null;

return await db.MatchHistory
.Where(m => m.MatchId == matchId)
.Select(_slotSelectors[slotIndex])
.FirstOrDefaultAsync();
.FirstOrDefaultAsync(cancellationToken);
}


Expand Down Expand Up @@ -532,8 +561,9 @@ public static async Task<ulong> CreatePlaceholderMatchHistory(
}

public static async Task DetermineLobbyWinnerIfNotPresent(
AppDbContext db,
GenOnlineService.Lobby lobby)
AppDbContext db,
GenOnlineService.Lobby lobby,
CancellationToken cancellationToken = default)
{
if (lobby == null || lobby.MatchID == 0)
return;
Expand Down Expand Up @@ -568,7 +598,7 @@ public static async Task DetermineLobbyWinnerIfNotPresent(
if (kv.Value.side == Constants.OBSERVER_SIDE_VALUE)
continue;

await UpdateMatchHistorySetWinFlag(db, lobby.MatchID, kv.Key, false);
await UpdateMatchHistorySetWinFlag(db, lobby.MatchID, kv.Key, false, cancellationToken);
}
return;
}
Expand Down Expand Up @@ -625,7 +655,7 @@ public static async Task DetermineLobbyWinnerIfNotPresent(
? kv.Value.team == conclusiveWinningTeam.Value
: kv.Key == conclusiveWinningSlot;

await UpdateMatchHistorySetWinFlag(db, lobby.MatchID, kv.Key, isWinner);
await UpdateMatchHistorySetWinFlag(db, lobby.MatchID, kv.Key, isWinner, cancellationToken);
}

return;
Expand Down Expand Up @@ -687,7 +717,7 @@ public static async Task DetermineLobbyWinnerIfNotPresent(
if (kv.Value.side == Constants.OBSERVER_SIDE_VALUE)
continue;

await UpdateMatchHistorySetWinFlag(db, lobby.MatchID, kv.Key, false);
await UpdateMatchHistorySetWinFlag(db, lobby.MatchID, kv.Key, false, cancellationToken);
}
return;
}
Expand All @@ -707,29 +737,30 @@ public static async Task DetermineLobbyWinnerIfNotPresent(
(winningTeam != -1 && model.team == winningTeam);

Console.WriteLine($"[WinnerDet] Match={lobby.MatchID}: marking slot={kv.Key} user={model.user_id} as {(isWinner ? "WINNER" : "loser")}.");
await UpdateMatchHistorySetWinFlag(db, lobby.MatchID, kv.Key, isWinner);
await UpdateMatchHistorySetWinFlag(db, lobby.MatchID, kv.Key, isWinner, cancellationToken);
}
}
catch (Exception ex)
{
Console.WriteLine($"[ERROR] DetermineLobbyWinnerIfNotPresent failed: {ex.Message}");
SentrySdk.CaptureException(ex);
throw;

@Skyaero42 Skyaero42 Aug 19, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure if you want to throw here.

Is it caught somewhere else?
Does it have to be thrown or can the application continue after it has been caught at this level?

Multiple times in this PR.

@tintinhamans tintinhamans Aug 20, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This prevents the exception from being swallowed. It bubbles up to DeleteLobby, logs to Sentry, and returns false, so the lobby persists and the cleanup will retry.

}
}

public static async Task UpdateMatchHistorySetWinFlag(
AppDbContext db,
ulong matchId,
int slotIndex,
bool won)
AppDbContext db,
ulong matchId,
int slotIndex,
bool won,
CancellationToken cancellationToken = default)
{
if (matchId == 0 || slotIndex < 0 || slotIndex > 7)
return;

try
{
// 1. Load the JSON for this slot
string? json = await _getMemberSlot(db, (long)matchId, slotIndex);
string? json = await _getMemberSlot(db, (long)matchId, slotIndex, cancellationToken);
if (string.IsNullOrEmpty(json))
return;

Expand All @@ -749,14 +780,17 @@ public static async Task UpdateMatchHistorySetWinFlag(
var setter = BuildSetter(slotIndex, updatedJson);

// 6. Execute update (single SQL UPDATE)
await db.MatchHistory
int updated = await db.MatchHistory
.Where(m => m.MatchId == (long)matchId)
.ExecuteUpdateAsync(setter);
.ExecuteUpdateAsync(setter, cancellationToken);

if (updated != 1)
throw new InvalidOperationException($"Match history entry {matchId} disappeared while its winner was being finalized.");
}
catch (Exception ex)
{
Console.WriteLine($"[ERROR] UpdateMatchHistorySetWinFlag failed: {ex.Message}");
SentrySdk.CaptureException(ex);
throw;
}
}

Expand Down Expand Up @@ -932,30 +966,145 @@ public static async Task<long> GetHighestMatchID(AppDbContext db)
}

// Called when a lobby is deleted, thats the true end of a match
public static async Task CommitLobbyToMatchHistory(AppDbContext db, GenOnlineService.Lobby lobby)
public static async Task FinalizeAndScheduleExternalPublication(
AppDbContext db,
GenOnlineService.Lobby lobby,
CancellationToken cancellationToken = default)
{
if (lobby.MatchID == 0)
return;

try
{
await db.MatchHistory
.Where(m => m.MatchId == (long)lobby.MatchID && !m.Finished)
.ExecuteUpdateAsync(s => s
.SetProperty(m => m.Finished, true)
.SetProperty(m => m.TimeFinished, DateTime.UtcNow));
}
catch (Exception ex)
await using var transaction = await db.Database.BeginTransactionAsync(cancellationToken);

await CommitLobbyToMatchHistory(db, lobby, cancellationToken);
await DetermineLobbyWinnerIfNotPresent(db, lobby, cancellationToken);
await ScheduleExternalPublication(db, lobby.MatchID, cancellationToken);

await transaction.CommitAsync(cancellationToken);
}

private static async Task CommitLobbyToMatchHistory(
AppDbContext db,
GenOnlineService.Lobby lobby,
CancellationToken cancellationToken)
{
if (lobby.MatchID == 0)
return;

await db.MatchHistory
.Where(m => m.MatchId == (long)lobby.MatchID && !m.Finished)
.ExecuteUpdateAsync(s => s
.SetProperty(m => m.Finished, true)
.SetProperty(m => m.TimeFinished, DateTime.UtcNow), cancellationToken);

bool matchExists = await db.MatchHistory
.AnyAsync(m => m.MatchId == (long)lobby.MatchID && m.Finished, cancellationToken);

if (!matchExists)
throw new InvalidOperationException($"Finished match history entry {lobby.MatchID} was not found.");
}

private static async Task ScheduleExternalPublication(
AppDbContext db,
ulong matchId,
CancellationToken cancellationToken)
{
if (matchId == 0)
return;

bool exists = await db.ExternalPublications.AnyAsync(p => p.MatchId == (long)matchId, cancellationToken);
if (!exists)
{
Console.WriteLine($"[ERROR] CommitLobbyToMatchHistory failed: {ex.Message}");
SentrySdk.CaptureException(ex);
db.ExternalPublications.Add(new ExternalPublicationEntry
{
MatchId = (long)matchId,
NextAttemptAt = DateTime.UtcNow
});
await db.SaveChangesAsync(cancellationToken);
}

bool publicationIsDurable = await db.MatchHistory.AnyAsync(
m => m.MatchId == (long)matchId && m.Finished,
cancellationToken);
publicationIsDurable &= await db.ExternalPublications.AnyAsync(
p => p.MatchId == (long)matchId,
cancellationToken);

if (!publicationIsDurable)
throw new InvalidOperationException($"Match history entry {matchId} could not be scheduled for external publication.");
}

public static async Task<List<ExternalPublicationWorkItem>> GetPendingExternalPublications(
AppDbContext db,
DateTime utcNow,
int maxCount,
CancellationToken cancellationToken)
{
List<ExternalPublicationEntry> pending = await db.ExternalPublications
.Where(p => p.PublishedAt == null && p.NextAttemptAt != null && p.NextAttemptAt <= utcNow)
.OrderBy(p => p.NextAttemptAt)
.ThenBy(p => p.MatchId)
.Take(maxCount)
.ToListAsync(cancellationToken);

return pending
.Select(item => new ExternalPublicationWorkItem(item.MatchId, item.Attempts + 1))
.ToList();
}

public static async Task MarkExternalPublicationSucceeded(
AppDbContext db,
ulong matchId,
int attempt,
CancellationToken cancellationToken)
{
if (matchId == 0)
throw new ArgumentOutOfRangeException(nameof(matchId));

int updated = await db.ExternalPublications
.Where(p => p.MatchId == (long)matchId && p.PublishedAt == null)
.ExecuteUpdateAsync(s => s
.SetProperty(p => p.PublishedAt, DateTime.UtcNow)
.SetProperty(p => p.NextAttemptAt, (DateTime?)null)
.SetProperty(p => p.Attempts, attempt)
.SetProperty(p => p.LastError, (string?)null), cancellationToken);

if (updated != 1)
throw new InvalidOperationException($"Match history entry {matchId} could not be acknowledged after external publication.");
}

public static async Task MarkExternalPublicationFailed(
AppDbContext db,
ulong matchId,
int attempt,
DateTime? nextAttemptAt,
string? errorMessage,
CancellationToken cancellationToken)
{
if (matchId == 0)
throw new ArgumentOutOfRangeException(nameof(matchId));

string? truncatedError = errorMessage;
if (!string.IsNullOrEmpty(truncatedError) && truncatedError.Length > 512)
truncatedError = truncatedError[..512];

int updated = await db.ExternalPublications
.Where(p => p.MatchId == (long)matchId && p.PublishedAt == null)
.ExecuteUpdateAsync(s => s
.SetProperty(p => p.NextAttemptAt, nextAttemptAt)
.SetProperty(p => p.Attempts, attempt)
.SetProperty(p => p.LastError, truncatedError), cancellationToken);

if (updated != 1)
throw new InvalidOperationException($"Match history entry {matchId} could not be rescheduled after publication failure.");
}

internal static async Task<MatchHistory_Entry?> LoadMatchHistoryEntryAsync(
AppDbContext db, long matchId)
AppDbContext db,
long matchId,
CancellationToken cancellationToken = default)
{
var row = await db.MatchHistory.FirstOrDefaultAsync(m => m.MatchId == matchId);
var row = await db.MatchHistory.FirstOrDefaultAsync(m => m.MatchId == matchId, cancellationToken);
if (row == null)
return null;

Expand Down
36 changes: 32 additions & 4 deletions GenOnlineService/Database/Database.User.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -731,7 +731,11 @@ public static async Task SetFavorite_Color(
}
}

public static async Task SaveELOData(AppDbContext db, long userId, EloData newEloData)
public static async Task SaveELOData(
AppDbContext db,
long userId,
EloData newEloData,
CancellationToken cancellationToken = default)
{
try
{
Expand All @@ -740,8 +744,12 @@ await db.Users
.ExecuteUpdateAsync(setters => setters
.SetProperty(u => u.EloRating, newEloData.Rating)
.SetProperty(u => u.MonthlyEloRating, newEloData.MonthlyRating)
.SetProperty(u => u.EloNumberOfMatches, newEloData.NumMatches)
);
.SetProperty(u => u.EloNumberOfMatches, newEloData.NumMatches),
cancellationToken);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (Exception ex)
{
Expand All @@ -750,5 +758,25 @@ await db.Users
}
}

public static async Task<bool> SaveExternalELOData(
AppDbContext db,
long userId,
EloData newEloData,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(db);
ArgumentNullException.ThrowIfNull(newEloData);

int updated = await db.Users
.Where(u => u.ID == userId)
.ExecuteUpdateAsync(setters => setters
.SetProperty(u => u.EloRating, newEloData.Rating)
.SetProperty(u => u.MonthlyEloRating, newEloData.MonthlyRating)
.SetProperty(u => u.EloNumberOfMatches, newEloData.NumMatches),
cancellationToken);

return updated == 1;
}

}
}
}
Loading
Loading