xp almost fully reimplemented

src/EllieBot/Db/EllieDbService.cs

@@ -89,7 +89,7 @@ public sealed class EllieDbService : DbService
         var applied = await ctx.Database.GetAppliedMigrationsAsync();
 
         // get all .sql file names from the migrations folder
-        var available = Directory.GetFiles("Migrations/Sqlite", "*_*.sql")
+        var available = Directory.GetFiles("Migrations/" + GetMigrationDirectory(ctx.Database), "*_*.sql")
             .Select(x => Path.GetFileNameWithoutExtension(x))
             .OrderBy(x => x);
 
@@ -112,12 +112,17 @@ public sealed class EllieDbService : DbService
     }
 
     private static string GetMigrationPath(DatabaseFacade ctxDatabase, string runnable)
+    {
+        return $"Migrations/{GetMigrationDirectory(ctxDatabase)}/{runnable}.sql";
+    }
+
+    private static string GetMigrationDirectory(DatabaseFacade ctxDatabase)
     {
         if (ctxDatabase.IsSqlite())
-            return $"Migrations/Sqlite/{runnable}.sql";
+            return "Sqlite";
 
         if (ctxDatabase.IsNpgsql())
-            return $"Migrations/PostgreSql/{runnable}.sql";
+            return "PostgreSql";
 
         throw new NotSupportedException("This database type is not supported.");
     }

src/EllieBot/Db/Extensions/GuildConfigExtensions.cs

@@ -71,7 +71,6 @@ public static class GuildConfigExtensions
     }
 
 
-
     public static IEnumerable<GuildConfig> PermissionsForAll(this DbSet<GuildConfig> configs, List<ulong> include)
     {
         var query = configs.AsQueryable().Where(x => include.Contains(x.GuildId)).Include(gc => gc.Permissions);
@@ -106,10 +105,13 @@ public static class GuildConfigExtensions
         return config;
     }
 
-    public static async Task<XpSettings> XpSettingsFor(this DbContext ctx, ulong guildId)
+    public static async Task<XpSettings> XpSettingsFor(this DbContext ctx, ulong guildId,
+        Func<IQueryable<XpSettings>, IQueryable<XpSettings>> includes = default)
     {
-        var srs = await ctx.GetTable<XpSettings>()
+        includes ??= static set => set;
-                           .Where(x => x.GuildId == guildId)
+
+        var srs = await includes(ctx.GetTable<XpSettings>()
+                .Where(x => x.GuildId == guildId))
             .FirstOrDefaultAsyncLinqToDB();
 
         if (srs is not null)
@@ -119,6 +121,7 @@ public static class GuildConfigExtensions
             .InsertWithOutputAsync(() => new()
             {
                 GuildId = guildId,
+                ServerExcluded = false,
             });
 
         return srs;

src/EllieBot/Modules/Xp/XpRewards.cs

@@ -38,7 +38,9 @@ public partial class Xp
             if (page is < 0 or > 100)
                 return;
 
-            var rews = await _service.GetRoleRewardsAsync(ctx.Guild.Id);
+            var xpSettings = await _service.GetFullXpSettingsFor(ctx.Guild.Id);
+            var rews = xpSettings.RoleRewards;
+
             var allRewards = rews.OrderBy(x => x.Level)
                 .Select(x =>
                 {
@@ -60,7 +62,7 @@ public partial class Xp
 
                     return (x.Level, Text: sign + str);
                 })
-                                     .Concat((await _service.GetCurrencyRewardsAsync(ctx.Guild.Id))
+                .Concat(xpSettings.CurrencyRewards
                     .OrderBy(x => x.Level)
                     .Select(x => (x.Level,
                         Format.Bold(x.Amount + _cp.GetCurrencySign()))))

src/EllieBot/Modules/Xp/XpService.cs

@@ -46,11 +46,12 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
     private readonly IPatronageService _ps;
     private readonly IBotCache _c;
 
-
     private readonly Channel<UserXpGainData> _xpGainQueue = Channel.CreateUnbounded<UserXpGainData>();
     private readonly INotifySubscriber _notifySub;
     private readonly ShardData _shardData;
 
+    private readonly QueueRunner _levelUpQueue = new QueueRunner(0, 100);
+
     public XpService(
         DiscordSocketClient client,
         DbService db,
@@ -109,7 +110,8 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
     public async Task OnReadyAsync()
     {
         // initialize ignored
-        ArgumentOutOfRangeException.ThrowIfLessThan(_xpConfig.Data.MessageXpCooldown, 1,
+        ArgumentOutOfRangeException.ThrowIfLessThan(_xpConfig.Data.MessageXpCooldown,
+            1,
             nameof(_xpConfig.Data.MessageXpCooldown));
 
         await using (var ctx = _db.GetDbContext())
@@ -134,28 +136,26 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
             }
         }
 
-        await Task.WhenAll(UpdateTimer(), PeriodClearTimer());
+        await Task.WhenAll(UpdateTimer(), _levelUpQueue.RunAsync());
 
         return;
 
-        async Task PeriodClearTimer()
-        {
-            using var timer = new PeriodicTimer(TimeSpan.FromSeconds(_xpConfig.Data.MessageXpCooldown));
-            while (true)
-            {
-                await timer.WaitForNextTickAsync();
-                _usersGainedInPeriod.Clear();
-            }
-        }
-
         async Task UpdateTimer()
         {
             // todo a bigger loop that runs once every XpTimer
             using var timer = new PeriodicTimer(TimeSpan.FromSeconds(3));
             while (await timer.WaitForNextTickAsync())
+            {
+                try
                 {
                     await UpdateXp();
                 }
+                catch (Exception ex)
+                {
+                    Log.Error(ex, "Error updating xp");
+                    await Task.Delay(30_000);
+                }
+            }
         }
     }
 
@@ -170,33 +170,80 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
         var currentBatch = _usersBatch.ToArray();
         _usersBatch.Clear();
 
+        if (currentBatch.Length == 0)
+            return;
+
+        var ids = currentBatch.Select(x => x.Id).ToArray();
+
         await using var ctx = _db.GetDbContext();
         await using var lctx = ctx.CreateLinqToDBConnection();
 
-        await using var batchTable = await lctx.CreateTempTableAsync<UserXpBatch>();
+        var tempTableName = "xp_batch_" + _shardData.ShardId;
+        await using var batchTable = await lctx.CreateTempTableAsync<UserXpBatch>(tempTableName);
 
         await batchTable.BulkCopyAsync(currentBatch.Select(x => new UserXpBatch()
         {
             GuildId = x.GuildId,
             UserId = x.Id,
-            UserName = x.Username,
+            Username = x.Username,
-            AvatarId = x.DisplayAvatarId,
+            AvatarId = x.DisplayAvatarId
-
         }));
 
         await lctx.ExecuteAsync(
             $"""
-             INSERT INTO ${nameof(DiscordUser)}
+             INSERT INTO UserXpStats (GuildId, UserId, Xp)
-             SELECT ${nameof(UserXpBatch.GuildId)}, ${nameof(UserXpBatch.UserId)}, ${nameof(UserXpBatch.UserName)}, ${nameof(UserXpBatch.AvatarId)}, ${xpAmount}
+             SELECT "{tempTableName}"."GuildId", "{tempTableName}"."UserId", {xpAmount}
-             FROM ${nameof(UserXpBatch)}
+             FROM {tempTableName}
-             ON CONFLICT(${nameof(DiscordUser.UserId)}, ${nameof(DiscordUser.UserId)}) DO UPDATE SET 
+             WHERE TRUE
-              ${nameof(DiscordUser.Username)} = ${nameof(UserXpBatch)}.${nameof(UserXpBatch.UserName)}
+             ON CONFLICT (GuildId, UserId) DO UPDATE 
-              ${nameof(DiscordUser.AvatarId)} = ${nameof(UserXpBatch)}.${nameof(UserXpBatch.AvatarId)}
+             SET 
-              ${nameof(DiscordUser.TotalXp)} = ${nameof(DiscordUser.TotalXp)} + ${xpAmount}
+                 Xp = UserXpStats.Xp + EXCLUDED.Xp;
-             RETURNING *;
              """);
 
-        // todo send notifications
+        await lctx.ExecuteAsync(
+            $"""
+             INSERT INTO DiscordUser (UserId, AvatarId, Username, TotalXp)
+             SELECT "{tempTableName}"."UserId", "{tempTableName}"."AvatarId", "{tempTableName}"."Username", {xpAmount}
+             FROM {tempTableName}
+             WHERE TRUE
+             ON CONFLICT (UserId) DO UPDATE 
+             SET 
+                 Username = EXCLUDED.Username,
+                 AvatarId = EXCLUDED.AvatarId,
+                 TotalXp = DiscordUser.TotalXp + {xpAmount};
+             """);
+
+        foreach (var (guildId, users) in currentBatch.GroupBy(x => x.GuildId)
+                     .ToDictionary(x => x.Key, x => x.AsEnumerable()))
+        {
+            var userIds = users.Select(x => x.Id).ToArray();
+
+            var dbStats = await ctx.GetTable<UserXpStats>()
+                .Where(x => x.GuildId == guildId && userIds.Contains(x.UserId))
+                .OrderByDescending(x => x.Xp)
+                .ToArrayAsyncLinqToDB();
+
+            for (var i = 0; i < dbStats.Length; i++)
+            {
+                var oldStats = new LevelStats(dbStats[i].Xp - xpAmount);
+                var newStats = new LevelStats(dbStats[i].Xp);
+
+                Log.Information("User {User} xp updated from {OldLevel} to {NewLevel}",
+                    dbStats[i].UserId,
+                    oldStats.TotalXp,
+                    newStats.TotalXp);
+
+                if (oldStats.Level < newStats.Level)
+                {
+                    await _levelUpQueue.EnqueueAsync(NotifyUser(guildId,
+                        0,
+                        dbStats[i].UserId,
+                        true,
+                        oldStats.Level,
+                        newStats.Level));
+                }
+            }
+        }
     }
 
     private Func<Task> NotifyUser(
@@ -222,13 +269,9 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
         long oldLevel,
         long newLevel)
     {
-        List<XpRoleReward> rrews;
+        var settings = await GetFullXpSettingsFor(guildId);
-        List<XpCurrencyReward> crews;
+        var rrews = settings.RoleRewards;
-        await using (var ctx = _db.GetDbContext())
+        var crews = settings.CurrencyRewards;
-        {
-            rrews = await ctx.XpSettingsFor(guildId).Fmap(x => x.RoleRewards.ToList());
-            crews = await ctx.XpSettingsFor(guildId).Fmap(x => x.CurrencyRewards.ToList());
-        }
 
         //loop through levels since last level up, so if a high amount of xp is gained, reward are still applied.
         for (var i = oldLevel + 1; i <= newLevel; i++)
@@ -369,7 +412,7 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
     public async Task SetCurrencyReward(ulong guildId, int level, int amount)
     {
         await using var uow = _db.GetDbContext();
-        var settings = await uow.XpSettingsFor(guildId);
+        var settings = await uow.XpSettingsFor(guildId, set => set.LoadWith(x => x.CurrencyRewards));
 
         if (amount <= 0)
         {
@@ -399,22 +442,19 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
         uow.SaveChanges();
     }
 
-    public async Task<IEnumerable<XpCurrencyReward>> GetCurrencyRewardsAsync(ulong id)
+    public async Task<XpSettings> GetFullXpSettingsFor(ulong guildId)
     {
         await using var uow = _db.GetDbContext();
-        return (await uow.XpSettingsFor(id)).CurrencyRewards.ToArray();
+        return await uow.XpSettingsFor(guildId,
-    }
+            set => set
-
+                .LoadWith(x => x.CurrencyRewards)
-    public async Task<IEnumerable<XpRoleReward>> GetRoleRewardsAsync(ulong id)
+                .LoadWith(x => x.RoleRewards));
-    {
-        await using var uow = _db.GetDbContext();
-        return (await uow.XpSettingsFor(id)).RoleRewards.ToArray();
     }
 
     public async Task ResetRoleRewardAsync(ulong guildId, int level)
     {
         await using var uow = _db.GetDbContext();
-        var settings = await uow.XpSettingsFor(guildId);
+        var settings = await uow.XpSettingsFor(guildId, set => set.LoadWith(x => x.RoleRewards));
 
         var toRemove = settings.RoleRewards.FirstOrDefault(x => x.Level == level);
         if (toRemove is not null)
@@ -423,7 +463,7 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
             settings.RoleRewards.Remove(toRemove);
         }
 
-        uow.SaveChanges();
+        await uow.SaveChangesAsync();
     }
 
     public async Task SetRoleRewardAsync(
@@ -433,7 +473,7 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
         bool remove)
     {
         await using var uow = _db.GetDbContext();
-        var settings = await uow.XpSettingsFor(guildId);
+        var settings = await uow.XpSettingsFor(guildId, set => set.LoadWith(x => x.RoleRewards));
 
         var rew = settings.RoleRewards.FirstOrDefault(x => x.Level == level);
 
@@ -806,7 +846,7 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
     {
         var roles = _excludedRoles.GetOrAdd(guildId, _ => new());
         await using var uow = _db.GetDbContext();
-        var xpSetting = await uow.XpSettingsFor(guildId);
+        var xpSetting = await uow.XpSettingsFor(guildId, set => set.LoadWith(x => x.ExclusionList));
         var excludeObj = new ExcludedItem
         {
             ItemId = rId,
@@ -837,7 +877,7 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
     {
         var channels = _excludedChannels.GetOrAdd(guildId, _ => new());
         await using var uow = _db.GetDbContext();
-        var xpSetting = await uow.XpSettingsFor(guildId);
+        var xpSetting = await uow.XpSettingsFor(guildId, set => set.LoadWith(x => x.ExclusionList));
         var excludeObj = new ExcludedItem
         {
             ItemId = chId,
@@ -1510,8 +1550,10 @@ public class XpService : IEService, IReadyExecutor, IExecNoCommand
 
 public sealed class UserXpBatch
 {
-    [Key] public ulong UserId { get; set; }
+    [Key]
+    public ulong UserId { get; set; }
+
     public ulong GuildId { get; set; }
-    public string UserName { get; set; } = string.Empty;
+    public string Username { get; set; } = string.Empty;
     public string AvatarId { get; set; } = string.Empty;
 }

src/EllieBot/Services/GrpcApi/XpSvc.cs

@@ -56,8 +56,9 @@ public class XpSvc : GrpcXp.GrpcXpBase, IGrpcSvc, IEService
                     Name = guild.GetRole(x)?.Name ?? "????"
                 })));
 
-        var curRews = await _xp.GetCurrencyRewardsAsync(request.GuildId);
+        var settings = await _xp.GetFullXpSettingsFor(request.GuildId);
-        var roleRews = await _xp.GetRoleRewardsAsync(request.GuildId);
+        var curRews = settings.CurrencyRewards;
+        var roleRews = settings.RoleRewards;
 
         var rews = curRews.Select(x => new RewItemReply()
         {