From 96c4ea0637d48421d61d0775d9b78c1b7c30b3ca Mon Sep 17 00:00:00 2001
From: Toastie <toastie@toastiet0ast.com>
Date: Thu, 6 Feb 2025 12:34:22 +1300
Subject: [PATCH] wip reimplementation of xp gain loop
---
src/EllieBot/Db/Models/DiscordUser.cs | 10 +-
src/EllieBot/Modules/Xp/XpService.cs | 401 ++++++++++----------------
2 files changed, 154 insertions(+), 257 deletions(-)
diff --git a/src/EllieBot/Db/Models/DiscordUser.cs b/src/EllieBot/Db/Models/DiscordUser.cs
index 0c51df4..f400694 100644
--- a/src/EllieBot/Db/Models/DiscordUser.cs
+++ b/src/EllieBot/Db/Models/DiscordUser.cs
@@ -1,17 +1,17 @@
-#nullable disable
namespace EllieBot.Db.Models;
// FUTURE remove LastLevelUp from here and UserXpStats
public class DiscordUser : DbEntity
{
+ public const string DEFAULT_USERNAME = "??Unknown";
+
public ulong UserId { get; set; }
- public string Username { get; set; }
- // public string Discriminator { get; set; }
- public string AvatarId { get; set; }
+ public string? Username { get; set; }
+ public string? AvatarId { get; set; }
public int? ClubId { get; set; }
- public ClubInfo Club { get; set; }
+ public ClubInfo? Club { get; set; }
public bool IsClubAdmin { get; set; }
public long TotalXp { get; set; }
diff --git a/src/EllieBot/Modules/Xp/XpService.cs b/src/EllieBot/Modules/Xp/XpService.cs
index 7ba2705..f465cc5 100644
--- a/src/EllieBot/Modules/Xp/XpService.cs
+++ b/src/EllieBot/Modules/Xp/XpService.cs
@@ -1,4 +1,5 @@
#nullable disable warnings
+using System.ComponentModel.DataAnnotations;
using LinqToDB;
using Microsoft.EntityFrameworkCore;
using EllieBot.Common.ModuleBehaviors;
@@ -11,10 +12,12 @@ using SixLabors.ImageSharp.Formats;
using SixLabors.ImageSharp.PixelFormats;
using SixLabors.ImageSharp.Processing;
using System.Threading.Channels;
+using LinqToDB.Data;
using LinqToDB.EntityFrameworkCore;
using LinqToDB.Tools;
using EllieBot.Modules.Administration;
using EllieBot.Modules.Patronage;
+using ArgumentOutOfRangeException = System.ArgumentOutOfRangeException;
using Color = SixLabors.ImageSharp.Color;
using Exception = System.Exception;
using Image = SixLabors.ImageSharp.Image;
@@ -36,7 +39,7 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
private readonly ConcurrentDictionary<ulong, ConcurrentHashSet<ulong>> _excludedChannels = new();
private readonly ConcurrentHashSet<ulong> _excludedServers;
- private XpTemplate _template;
+ private XpTemplate _template = new();
private readonly DiscordSocketClient _client;
private readonly TypedKey<bool> _xpTemplateReloadKey;
@@ -44,9 +47,7 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
private readonly IBotCache _c;
- private readonly QueueRunner _levelUpQueue = new(0, 50);
private readonly Channel<UserXpGainData> _xpGainQueue = Channel.CreateUnbounded<UserXpGainData>();
- private readonly IMessageSenderService _sender;
private readonly INotifySubscriber _notifySub;
private readonly ShardData _shardData;
@@ -62,7 +63,6 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
XpConfigService xpConfig,
IPubSub pubSub,
IPatronageService ps,
- IMessageSenderService sender,
INotifySubscriber notifySub,
ShardData shardData)
{
@@ -74,7 +74,6 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
_httpFactory = http;
_xpConfig = xpConfig;
_pubSub = pubSub;
- _sender = sender;
_notifySub = notifySub;
_shardData = shardData;
_excludedServers = new();
@@ -109,16 +108,16 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
public async Task OnReadyAsync()
{
- _ = Task.Run(() => _levelUpQueue.RunAsync());
-
// initialize ignored
+ ArgumentOutOfRangeException.ThrowIfLessThan(_xpConfig.Data.MessageXpCooldown, 1,
+ nameof(_xpConfig.Data.MessageXpCooldown));
await using (var ctx = _db.GetDbContext())
{
var xps = await ctx.GetTable<XpSettings>()
- .Where(x => Queries.GuildOnShard(x.GuildId, _shardData.TotalShards, _shardData.ShardId))
- .LoadWith(x => x.ExclusionList)
- .ToListAsyncLinqToDB();
+ .Where(x => Queries.GuildOnShard(x.GuildId, _shardData.TotalShards, _shardData.ShardId))
+ .LoadWith(x => x.ExclusionList)
+ .ToListAsyncLinqToDB();
foreach (var xp in xps)
{
@@ -135,173 +134,69 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
}
}
- using var timer = new PeriodicTimer(TimeSpan.FromSeconds(5));
- while (await timer.WaitForNextTickAsync())
+ await Task.WhenAll(UpdateTimer(), PeriodClearTimer());
+
+ return;
+
+ async Task PeriodClearTimer()
{
- await UpdateXp();
+ using var timer = new PeriodicTimer(TimeSpan.FromSeconds(_xpConfig.Data.MessageXpCooldown));
+ while (true)
+ {
+ await timer.WaitForNextTickAsync();
+ _usersGainedInPeriod.Clear();
+ }
+ }
+
+ async Task UpdateTimer()
+ {
+ // todo a bigger loop that runs once every XpTimer
+ using var timer = new PeriodicTimer(TimeSpan.FromSeconds(3));
+ while (await timer.WaitForNextTickAsync())
+ {
+ await UpdateXp();
+ }
}
}
+ /// <summary>
+ /// The current batch of users that will gain xp
+ /// </summary>
+ private readonly ConcurrentHashSet<IGuildUser> _usersBatch = new();
+
private async Task UpdateXp()
{
- try
+ var xpAmount = _xpConfig.Data.XpPerMessage;
+ var currentBatch = _usersBatch.ToArray();
+ _usersBatch.Clear();
+
+ await using var ctx = _db.GetDbContext();
+ await using var lctx = ctx.CreateLinqToDBConnection();
+
+ await using var batchTable = await lctx.CreateTempTableAsync<UserXpBatch>();
+
+ await batchTable.BulkCopyAsync(currentBatch.Select(x => new UserXpBatch()
{
- var reader = _xpGainQueue.Reader;
+ GuildId = x.GuildId,
+ UserId = x.Id,
+ UserName = x.Username,
+ AvatarId = x.DisplayAvatarId,
- // sum up all gains into a single UserCacheItem
- var globalToAdd = new Dictionary<ulong, UserXpGainData>();
- var guildToAdd = new Dictionary<ulong, Dictionary<ulong, UserXpGainData>>();
- while (reader.TryRead(out var item))
- {
- // add global xp to these users
- if (!globalToAdd.TryGetValue(item.User.Id, out var ci))
- globalToAdd[item.User.Id] = item.Clone();
- else
- ci.XpAmount += item.XpAmount;
+ }));
+ await lctx.ExecuteAsync(
+ $"""
+ INSERT INTO ${nameof(DiscordUser)}
+ SELECT ${nameof(UserXpBatch.GuildId)}, ${nameof(UserXpBatch.UserId)}, ${nameof(UserXpBatch.UserName)}, ${nameof(UserXpBatch.AvatarId)}, ${xpAmount}
+ FROM ${nameof(UserXpBatch)}
+ ON CONFLICT(${nameof(DiscordUser.UserId)}, ${nameof(DiscordUser.UserId)}) DO UPDATE SET
+ ${nameof(DiscordUser.Username)} = ${nameof(UserXpBatch)}.${nameof(UserXpBatch.UserName)}
+ ${nameof(DiscordUser.AvatarId)} = ${nameof(UserXpBatch)}.${nameof(UserXpBatch.AvatarId)}
+ ${nameof(DiscordUser.TotalXp)} = ${nameof(DiscordUser.TotalXp)} + ${xpAmount}
+ RETURNING *;
+ """);
- // ad guild xp in these guilds to these users
- if (!guildToAdd.TryGetValue(item.Guild.Id, out var users))
- users = guildToAdd[item.Guild.Id] = new();
-
- if (!users.TryGetValue(item.User.Id, out ci))
- users[item.User.Id] = item.Clone();
- else
- ci.XpAmount += item.XpAmount;
- }
-
- var dus = new List<DiscordUser>(globalToAdd.Count);
- var gxps = new List<UserXpStats>(globalToAdd.Count);
- var conf = _xpConfig.Data;
- await using (var ctx = _db.GetDbContext())
- {
- if (conf.CurrencyPerXp > 0)
- {
- foreach (var user in globalToAdd)
- {
- var amount = (long)(user.Value.XpAmount * conf.CurrencyPerXp);
- if (amount > 0)
- await _cs.AddAsync(user.Key, amount, null);
- }
- }
-
- // update global user xp in batches
- // group by xp amount and update the same amounts at the same time
- foreach (var group in globalToAdd.GroupBy(x => x.Value.XpAmount, x => x.Key))
- {
- var items = await ctx.Set<DiscordUser>()
- .Where(x => group.Contains(x.UserId))
- .UpdateWithOutputAsync(old => new()
- {
- TotalXp = old.TotalXp + group.Key
- },
- (_, n) => n);
-
- await ctx.Set<ClubInfo>()
- .Where(x => x.Members.Any(m => group.Contains(m.UserId)))
- .UpdateAsync(old => new()
- {
- Xp = old.Xp + (group.Key * old.Members.Count(m => group.Contains(m.UserId)))
- });
-
- dus.AddRange(items);
- }
-
- // update guild user xp in batches
- foreach (var (guildId, toAdd) in guildToAdd)
- {
- foreach (var group in toAdd.GroupBy(x => x.Value.XpAmount, x => x.Key))
- {
- var items = await ctx
- .Set<UserXpStats>()
- .Where(x => x.GuildId == guildId)
- .Where(x => group.Contains(x.UserId))
- .UpdateWithOutputAsync(old => new()
- {
- Xp = old.Xp + group.Key
- },
- (_, n) => n);
-
- gxps.AddRange(items);
-
- var missingUserIds = group.Where(userId => !items.Any(x => x.UserId == userId)).ToArray();
- foreach (var userId in missingUserIds)
- {
- await ctx
- .Set<UserXpStats>()
- .ToLinqToDBTable()
- .InsertOrUpdateAsync(() => new UserXpStats()
- {
- UserId = userId,
- GuildId = guildId,
- Xp = group.Key,
- DateAdded = DateTime.UtcNow,
- },
- _ => new()
- {
- },
- () => new()
- {
- UserId = userId
- });
- }
-
- if (missingUserIds.Length > 0)
- {
- var missingItems = await ctx.Set<UserXpStats>()
- .ToLinqToDBTable()
- .Where(x => missingUserIds.Contains(x.UserId))
- .ToArrayAsyncLinqToDB();
-
- gxps.AddRange(missingItems);
- }
- }
- }
- }
-
- foreach (var du in dus)
- {
- var oldLevel = new LevelStats(du.TotalXp - globalToAdd[du.UserId].XpAmount);
- var newLevel = new LevelStats(du.TotalXp);
-
- if (oldLevel.Level != newLevel.Level)
- {
- var item = globalToAdd[du.UserId];
- await _levelUpQueue.EnqueueAsync(
- NotifyUser(item.Guild.Id,
- item.Channel.Id,
- du.UserId,
- false,
- oldLevel.Level,
- newLevel.Level));
- }
- }
-
- foreach (var du in gxps)
- {
- if (guildToAdd.TryGetValue(du.GuildId, out var users)
- && users.TryGetValue(du.UserId, out var xpGainData))
- {
- var oldLevel = new LevelStats(du.Xp - xpGainData.XpAmount);
- var newLevel = new LevelStats(du.Xp);
-
- if (oldLevel.Level < newLevel.Level)
- {
- await _levelUpQueue.EnqueueAsync(
- NotifyUser(xpGainData.Guild.Id,
- xpGainData.Channel.Id,
- du.UserId,
- true,
- oldLevel.Level,
- newLevel.Level));
- }
- }
- }
- }
- catch (Exception ex)
- {
- Log.Error(ex, "Error In the XP update loop");
- }
+ // todo send notifications
}
private Func<Task> NotifyUser(
@@ -564,23 +459,23 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
{
await using var uow = _db.GetDbContext();
return await uow
- .UserXpStats
- .Where(x => x.GuildId == guildId)
- .OrderByDescending(x => x.Xp)
- .Skip(page * 10)
- .Take(10)
- .ToArrayAsyncLinqToDB();
+ .UserXpStats
+ .Where(x => x.GuildId == guildId)
+ .OrderByDescending(x => x.Xp)
+ .Skip(page * 10)
+ .Take(10)
+ .ToArrayAsyncLinqToDB();
}
public async Task<IReadOnlyCollection<UserXpStats>> GetGuildUserXps(ulong guildId, List<ulong> users, int page)
{
await using var uow = _db.GetDbContext();
return await uow.Set<UserXpStats>()
- .Where(x => x.GuildId == guildId && x.UserId.In(users))
- .OrderByDescending(x => x.Xp)
- .Skip(page * 10)
- .Take(10)
- .ToArrayAsyncLinqToDB();
+ .Where(x => x.GuildId == guildId && x.UserId.In(users))
+ .OrderByDescending(x => x.Xp)
+ .Skip(page * 10)
+ .Take(10)
+ .ToArrayAsyncLinqToDB();
}
public async Task<IReadOnlyCollection<DiscordUser>> GetGlobalUserXps(int page)
@@ -588,10 +483,10 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
await using var uow = _db.GetDbContext();
return await uow.GetTable<DiscordUser>()
- .OrderByDescending(x => x.TotalXp)
- .Skip(page * 10)
- .Take(10)
- .ToArrayAsyncLinqToDB();
+ .OrderByDescending(x => x.TotalXp)
+ .Skip(page * 10)
+ .Take(10)
+ .ToArrayAsyncLinqToDB();
}
public async Task<IReadOnlyCollection<DiscordUser>> GetGlobalUserXps(int page, List<ulong> users)
@@ -599,11 +494,11 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
await using var uow = _db.GetDbContext();
return await uow.GetTable<DiscordUser>()
- .Where(x => x.UserId.In(users))
- .OrderByDescending(x => x.TotalXp)
- .Skip(page * 10)
- .Take(10)
- .ToArrayAsyncLinqToDB();
+ .Where(x => x.UserId.In(users))
+ .OrderByDescending(x => x.TotalXp)
+ .Skip(page * 10)
+ .Take(10)
+ .ToArrayAsyncLinqToDB();
}
private Task Client_OnGuildAvailable(SocketGuild guild)
@@ -804,13 +699,7 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
if (!await SetUserRewardedAsync(user.Id))
return;
- await _xpGainQueue.Writer.WriteAsync(new()
- {
- Guild = user.Guild,
- Channel = arg.Channel,
- User = user,
- XpAmount = xp
- });
+ _usersBatch.Add(user);
});
return Task.CompletedTask;
}
@@ -833,11 +722,11 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
{
await using var ctx = _db.GetDbContext();
return await ctx.GetTable<UserXpStats>()
- .Where(x => x.GuildId == guildId && userIds.Contains(x.UserId))
- .UpdateAsync(old => new()
- {
- Xp = old.Xp + amount
- });
+ .Where(x => x.GuildId == guildId && userIds.Contains(x.UserId))
+ .UpdateAsync(old => new()
+ {
+ Xp = old.Xp + amount
+ });
}
public async Task AddXpAsync(ulong userId, ulong guildId, int amount)
@@ -1198,10 +1087,10 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
using (var tempDraw = Image.Load<Rgba32>(avatarData))
{
tempDraw.Mutate(x => x
- .Resize(_template.User.Icon.Size.X, _template.User.Icon.Size.Y)
- .ApplyRoundedCorners(Math.Max(_template.User.Icon.Size.X,
- _template.User.Icon.Size.Y)
- / 2.0f));
+ .Resize(_template.User.Icon.Size.X, _template.User.Icon.Size.Y)
+ .ApplyRoundedCorners(Math.Max(_template.User.Icon.Size.X,
+ _template.User.Icon.Size.Y)
+ / 2.0f));
await using (var stream = await tempDraw.ToStreamAsync())
{
data = stream.ToArray();
@@ -1361,10 +1250,10 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
using (var tempDraw = Image.Load<Rgba32>(imgData))
{
tempDraw.Mutate(x => x
- .Resize(_template.Club.Icon.Size.X, _template.Club.Icon.Size.Y)
- .ApplyRoundedCorners(Math.Max(_template.Club.Icon.Size.X,
- _template.Club.Icon.Size.Y)
- / 2.0f));
+ .Resize(_template.Club.Icon.Size.X, _template.Club.Icon.Size.Y)
+ .ApplyRoundedCorners(Math.Max(_template.Club.Icon.Size.X,
+ _template.Club.Icon.Size.Y)
+ / 2.0f));
await using (var tds = await tempDraw.ToStreamAsync())
{
data = tds.ToArray();
@@ -1395,7 +1284,7 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
{
await using var uow = _db.GetDbContext();
await uow.GetTable<UserXpStats>()
- .DeleteAsync(x => x.UserId == userId && x.GuildId == guildId);
+ .DeleteAsync(x => x.UserId == userId && x.GuildId == guildId);
}
public void XpReset(ulong guildId)
@@ -1409,8 +1298,8 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
{
await using var uow = _db.GetDbContext();
await uow.GetTable<XpSettings>()
- .Where(x => x.GuildId == guildId)
- .DeleteAsync();
+ .Where(x => x.GuildId == guildId)
+ .DeleteAsync();
}
public ValueTask<Dictionary<string, XpConfig.ShopItemInfo>?> GetShopBgs()
@@ -1454,7 +1343,7 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
try
{
if (await ctx.GetTable<XpShopOwnedItem>()
- .AnyAsyncLinqToDB(x => x.UserId == userId && x.ItemKey == key && x.ItemType == type))
+ .AnyAsyncLinqToDB(x => x.UserId == userId && x.ItemKey == key && x.ItemType == type))
return BuyResult.AlreadyOwned;
var item = GetShopItem(type, key);
@@ -1467,14 +1356,14 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
await ctx.GetTable<XpShopOwnedItem>()
- .InsertAsync(() => new XpShopOwnedItem()
- {
- UserId = userId,
- IsUsing = false,
- ItemKey = key,
- ItemType = type,
- DateAdded = DateTime.UtcNow,
- });
+ .InsertAsync(() => new XpShopOwnedItem()
+ {
+ UserId = userId,
+ IsUsing = false,
+ ItemKey = key,
+ ItemType = type,
+ DateAdded = DateTime.UtcNow,
+ });
return BuyResult.Success;
}
@@ -1514,9 +1403,9 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
{
await using var ctx = _db.GetDbContext();
return await ctx.GetTable<XpShopOwnedItem>()
- .AnyAsyncLinqToDB(x => x.UserId == userId
- && x.ItemType == itemType
- && x.ItemKey == key);
+ .AnyAsyncLinqToDB(x => x.UserId == userId
+ && x.ItemType == itemType
+ && x.ItemKey == key);
}
@@ -1527,9 +1416,9 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
{
await using var ctx = _db.GetDbContext();
return await ctx.GetTable<XpShopOwnedItem>()
- .FirstOrDefaultAsyncLinqToDB(x => x.UserId == userId
- && x.ItemType == itemType
- && x.ItemKey == key);
+ .FirstOrDefaultAsyncLinqToDB(x => x.UserId == userId
+ && x.ItemType == itemType
+ && x.ItemKey == key);
}
public async Task<XpShopOwnedItem?> GetItemInUse(
@@ -1538,9 +1427,9 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
{
await using var ctx = _db.GetDbContext();
return await ctx.GetTable<XpShopOwnedItem>()
- .FirstOrDefaultAsyncLinqToDB(x => x.UserId == userId
- && x.ItemType == itemType
- && x.IsUsing);
+ .FirstOrDefaultAsyncLinqToDB(x => x.UserId == userId
+ && x.ItemType == itemType
+ && x.IsUsing);
}
public async Task<bool> UseShopItemAsync(ulong userId, XpShopItemType itemType, string key)
@@ -1564,11 +1453,11 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
if (await OwnsItemAsync(userId, itemType, key))
{
await ctx.GetTable<XpShopOwnedItem>()
- .Where(x => x.UserId == userId && x.ItemType == itemType)
- .UpdateAsync(old => new()
- {
- IsUsing = key == old.ItemKey
- });
+ .Where(x => x.UserId == userId && x.ItemType == itemType)
+ .UpdateAsync(old => new()
+ {
+ IsUsing = key == old.ItemKey
+ });
return true;
}
@@ -1590,9 +1479,9 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
{
await using var ctx = _db.GetDbContext();
return await ctx.GetTable<UserXpStats>()
- .Where(x => x.GuildId == requestGuildId
- && (guildUsers == null || guildUsers.Contains(x.UserId)))
- .CountAsyncLinqToDB();
+ .Where(x => x.GuildId == requestGuildId
+ && (guildUsers == null || guildUsers.Contains(x.UserId)))
+ .CountAsyncLinqToDB();
}
public async Task SetLevelAsync(ulong guildId, ulong userId, int level)
@@ -1600,21 +1489,29 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
var lvlStats = LevelStats.CreateForLevel(level);
await using var ctx = _db.GetDbContext();
await ctx.GetTable<UserXpStats>()
- .InsertOrUpdateAsync(() => new()
- {
- GuildId = guildId,
- UserId = userId,
- Xp = lvlStats.TotalXp,
- DateAdded = DateTime.UtcNow
- },
- (old) => new()
- {
- Xp = lvlStats.TotalXp
- },
- () => new()
- {
- GuildId = guildId,
- UserId = userId
- });
+ .InsertOrUpdateAsync(() => new()
+ {
+ GuildId = guildId,
+ UserId = userId,
+ Xp = lvlStats.TotalXp,
+ DateAdded = DateTime.UtcNow
+ },
+ (old) => new()
+ {
+ Xp = lvlStats.TotalXp
+ },
+ () => new()
+ {
+ GuildId = guildId,
+ UserId = userId
+ });
}
+}
+
+public sealed class UserXpBatch
+{
+ [Key] public ulong UserId { get; set; }
+ public ulong GuildId { get; set; }
+ public string UserName { get; set; } = string.Empty;
+ public string AvatarId { get; set; } = string.Empty;
}
\ No newline at end of file