From 2138d94d566217812e9d6403113befdc1cc87dec Mon Sep 17 00:00:00 2001
From: Toastie <toastie@toastiet0ast.com>
Date: Fri, 24 Jan 2025 00:46:47 +1300
Subject: [PATCH] wip migration rework
---
src/Ellie.Marmalade/Ellie.Marmalade.csproj | 6 +-
.../EllieBot.Coordinator.csproj | 10 +-
.../EllieBot.GrpcApiBase.csproj | 4 +-
src/EllieBot.Tests/EllieBot.Tests.csproj | 6 +-
src/EllieBot.Voice/EllieBot.Voice.csproj | 4 +-
.../EllieBot.VotesApi.csproj | 2 +-
src/EllieBot/Db/EllieDbService.cs | 5 +-
src/EllieBot/Db/Models/GuildConfig.cs | 5 -
src/EllieBot/EllieBot.csproj | 94 +++++++++----------
src/EllieBot/MigrationRunner.cs | 54 +++++++++++
src/EllieBot/migrate.ps1 | 49 +++++++++-
src/EllieBot/remove-migrations.ps1 | 7 +-
12 files changed, 171 insertions(+), 75 deletions(-)
create mode 100644 src/EllieBot/MigrationRunner.cs
diff --git a/src/Ellie.Marmalade/Ellie.Marmalade.csproj b/src/Ellie.Marmalade/Ellie.Marmalade.csproj
index b64d92b..66e8e61 100644
--- a/src/Ellie.Marmalade/Ellie.Marmalade.csproj
+++ b/src/Ellie.Marmalade/Ellie.Marmalade.csproj
@@ -9,9 +9,9 @@
</PropertyGroup>
<ItemGroup>
- <PackageReference Include="Discord.Net.Core" Version="3.16.0" />
- <PackageReference Include="Serilog" Version="3.1.1" />
- <PackageReference Include="YamlDotNet" Version="15.1.4" />
+ <PackageReference Include="Discord.Net.Core" Version="3.17.1" />
+ <PackageReference Include="Serilog" Version="4.2.0" />
+ <PackageReference Include="YamlDotNet" Version="15.1.6" />
</ItemGroup>
<PropertyGroup Condition=" '$(Version)' == '' ">
diff --git a/src/EllieBot.Coordinator/EllieBot.Coordinator.csproj b/src/EllieBot.Coordinator/EllieBot.Coordinator.csproj
index b4f964d..d009507 100644
--- a/src/EllieBot.Coordinator/EllieBot.Coordinator.csproj
+++ b/src/EllieBot.Coordinator/EllieBot.Coordinator.csproj
@@ -9,11 +9,11 @@
</ItemGroup>
<ItemGroup>
- <PackageReference Include="Grpc.AspNetCore" Version="2.62.0" />
- <PackageReference Include="Serilog" Version="3.1.1" />
- <PackageReference Include="Serilog.Sinks.Console" Version="5.0.1" />
- <PackageReference Include="Serilog.Sinks.File" Version="5.0.0" />
- <PackageReference Include="YamlDotNet" Version="15.1.4" />
+ <PackageReference Include="Grpc.AspNetCore" Version="2.67.0" />
+ <PackageReference Include="Serilog" Version="4.2.0" />
+ <PackageReference Include="Serilog.Sinks.Console" Version="6.0.0" />
+ <PackageReference Include="Serilog.Sinks.File" Version="6.0.0" />
+ <PackageReference Include="YamlDotNet" Version="15.1.6" />
</ItemGroup>
</Project>
diff --git a/src/EllieBot.GrpcApiBase/EllieBot.GrpcApiBase.csproj b/src/EllieBot.GrpcApiBase/EllieBot.GrpcApiBase.csproj
index c941779..16fcb89 100644
--- a/src/EllieBot.GrpcApiBase/EllieBot.GrpcApiBase.csproj
+++ b/src/EllieBot.GrpcApiBase/EllieBot.GrpcApiBase.csproj
@@ -7,9 +7,9 @@
</PropertyGroup>
<ItemGroup>
- <PackageReference Include="Google.Protobuf" Version="3.28.2" />
+ <PackageReference Include="Google.Protobuf" Version="3.29.3" />
<PackageReference Include="Grpc" Version="2.46.6" />
- <PackageReference Include="Grpc.Tools" Version="2.66.0" PrivateAssets="All" />
+ <PackageReference Include="Grpc.Tools" Version="2.69.0" PrivateAssets="All" />
</ItemGroup>
<ItemGroup>
diff --git a/src/EllieBot.Tests/EllieBot.Tests.csproj b/src/EllieBot.Tests/EllieBot.Tests.csproj
index d1d7845..3676e77 100644
--- a/src/EllieBot.Tests/EllieBot.Tests.csproj
+++ b/src/EllieBot.Tests/EllieBot.Tests.csproj
@@ -6,9 +6,9 @@
</PropertyGroup>
<ItemGroup>
- <PackageReference Include="NUnit" Version="3.13.3" />
- <PackageReference Include="NUnit3TestAdapter" Version="4.2.1" />
- <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.2.0" />
+ <PackageReference Include="NUnit" Version="4.3.2" />
+ <PackageReference Include="NUnit3TestAdapter" Version="4.6.0" />
+ <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.12.0" />
</ItemGroup>
<ItemGroup>
diff --git a/src/EllieBot.Voice/EllieBot.Voice.csproj b/src/EllieBot.Voice/EllieBot.Voice.csproj
index b8d42d2..a48b8ff 100644
--- a/src/EllieBot.Voice/EllieBot.Voice.csproj
+++ b/src/EllieBot.Voice/EllieBot.Voice.csproj
@@ -9,7 +9,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
- <PackageReference Include="Serilog" Version="3.1.1" />
- <PackageReference Include="System.Threading.Channels" Version="8.0.0" />
+ <PackageReference Include="Serilog" Version="4.2.0" />
+ <PackageReference Include="System.Threading.Channels" Version="9.0.1" />
</ItemGroup>
</Project>
diff --git a/src/EllieBot.VotesApi/EllieBot.VotesApi.csproj b/src/EllieBot.VotesApi/EllieBot.VotesApi.csproj
index af0ed1f..4a803e1 100644
--- a/src/EllieBot.VotesApi/EllieBot.VotesApi.csproj
+++ b/src/EllieBot.VotesApi/EllieBot.VotesApi.csproj
@@ -7,7 +7,7 @@
<ItemGroup>
<PackageReference Include="MorseCode.ITask" Version="2.0.3" />
- <PackageReference Include="Swashbuckle.AspNetCore" Version="6.5.0" />
+ <PackageReference Include="Swashbuckle.AspNetCore" Version="7.2.0" />
</ItemGroup>
</Project>
diff --git a/src/EllieBot/Db/EllieDbService.cs b/src/EllieBot/Db/EllieDbService.cs
index 6106072..eb45520 100644
--- a/src/EllieBot/Db/EllieDbService.cs
+++ b/src/EllieBot/Db/EllieDbService.cs
@@ -1,6 +1,7 @@
using LinqToDB.Common;
using LinqToDB.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore;
+using EllieBot.Migrations;
namespace EllieBot.Db;
@@ -26,6 +27,8 @@ public sealed class EllieDbService : DbService
var connString = ConnString;
await using var context = CreateRawDbContext(dbType, connString);
+
+ await MigrationRunner.RunMigration(context);
// make sure sqlite db is in wal journal mode
if (context is SqliteContext)
@@ -33,7 +36,7 @@ public sealed class EllieDbService : DbService
await context.Database.ExecuteSqlRawAsync("PRAGMA journal_mode=WAL");
}
- await context.Database.MigrateAsync();
+ // await context.Database.MigrateAsync();
}
public override EllieContext CreateRawDbContext(string dbType, string connString)
diff --git a/src/EllieBot/Db/Models/GuildConfig.cs b/src/EllieBot/Db/Models/GuildConfig.cs
index 76d98a0..d6cfd94 100644
--- a/src/EllieBot/Db/Models/GuildConfig.cs
+++ b/src/EllieBot/Db/Models/GuildConfig.cs
@@ -13,11 +13,6 @@ public class GuildConfig : DbEntity
public string AutoAssignRoleIds { get; set; }
- //todo FUTURE: DELETE, UNUSED
- public bool ExclusiveSelfAssignedRoles { get; set; }
- public bool AutoDeleteSelfAssignedRoleMessages { get; set; }
-
-
//stream notifications
public HashSet<FollowedStream> FollowedStreams { get; set; } = new();
diff --git a/src/EllieBot/EllieBot.csproj b/src/EllieBot/EllieBot.csproj
index 8620d8e..36762f4 100644
--- a/src/EllieBot/EllieBot.csproj
+++ b/src/EllieBot/EllieBot.csproj
@@ -23,81 +23,81 @@
</PropertyGroup>
<ItemGroup>
- <PackageReference Include="AngleSharp" Version="1.1.2">
+ <PackageReference Include="AngleSharp" Version="1.2.0">
<PrivateAssets>all</PrivateAssets>
<Publish>True</Publish>
</PackageReference>
- <PackageReference Include="CodeHollow.FeedReader" Version="1.2.6" />
- <PackageReference Include="CommandLineParser" Version="2.9.1" />
- <PackageReference Include="Discord.Net" Version="3.16.0" />
- <PackageReference Include="CoreCLR-NCalc" Version="3.1.246" />
- <PackageReference Include="Google.Apis.Urlshortener.v1" Version="1.41.1.138" />
- <PackageReference Include="Google.Apis.YouTube.v3" Version="1.68.0.3414" />
- <PackageReference Include="Google.Apis.Customsearch.v1" Version="1.49.0.2084" />
-
- <PackageReference Include="Google.Protobuf" Version="3.28.2" />
+ <PackageReference Include="CodeHollow.FeedReader" Version="1.2.6"/>
+ <PackageReference Include="CommandLineParser" Version="2.9.1"/>
+ <PackageReference Include="Discord.Net" Version="3.17.1" />
+ <PackageReference Include="CoreCLR-NCalc" Version="3.1.253" />
+ <PackageReference Include="Google.Apis.Urlshortener.v1" Version="1.41.1.138"/>
+ <PackageReference Include="Google.Apis.YouTube.v3" Version="1.68.0.3653" />
+ <PackageReference Include="Google.Apis.Customsearch.v1" Version="1.49.0.2084"/>
+
+ <PackageReference Include="Google.Protobuf" Version="3.29.3" />
<PackageReference Include="Grpc" Version="2.46.6" />
- <PackageReference Include="Grpc.Net.Client" Version="2.62.0" />
- <PackageReference Include="Grpc.Tools" Version="2.66.0" PrivateAssets="All" />
-
- <PackageReference Include="Microsoft.CodeAnalysis.CSharp.Scripting" Version="4.5.0" />
+ <PackageReference Include="Grpc.Net.Client" Version="2.67.0" />
+ <PackageReference Include="Grpc.Tools" Version="2.69.0" PrivateAssets="All" />
- <PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
- <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="8.0.0" />
- <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
+ <PackageReference Include="Microsoft.CodeAnalysis.CSharp.Scripting" Version="4.8.0" />
- <PackageReference Include="MorseCode.ITask" Version="2.0.3" />
- <PackageReference Include="NetEscapades.Configuration.Yaml" Version="3.1.0" />
+ <PackageReference Include="Microsoft.Extensions.Configuration" Version="9.0.1" />
+ <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="9.0.1" />
+ <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="9.0.1" />
+
+ <PackageReference Include="MorseCode.ITask" Version="2.0.3"/>
+ <PackageReference Include="NetEscapades.Configuration.Yaml" Version="3.1.0"/>
<!-- DI -->
<!-- <PackageReference Include="Ninject" Version="3.3.6"/>-->
<!-- <PackageReference Include="Ninject.Extensions.Conventions" Version="3.3.0"/>-->
<!-- <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />-->
- <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.1" />
- <PackageReference Include="DryIoc.dll" Version="5.4.3" />
+ <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="9.0.1" />
+ <PackageReference Include="DryIoc.dll" Version="5.4.3"/>
<!-- <PackageReference Include="Scrutor" Version="4.2.0" />-->
- <PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
- <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="8.0.0" />
- <PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="8.0.0" />
- <PackageReference Include="Microsoft.SyndicationFeed.ReaderWriter" Version="1.0.2" />
- <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
- <PackageReference Include="NonBlocking" Version="2.1.2" />
- <PackageReference Include="OneOf" Version="3.0.263" />
- <PackageReference Include="OneOf.SourceGenerator" Version="3.0.263" />
- <PackageReference Include="Serilog.Sinks.Console" Version="5.0.1" />
- <PackageReference Include="Serilog.Sinks.Seq" Version="7.0.1" />
+ <PackageReference Include="Microsoft.Extensions.Http" Version="9.0.1" />
+ <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.1" />
+ <PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="9.0.1" />
+ <PackageReference Include="Microsoft.SyndicationFeed.ReaderWriter" Version="1.0.2"/>
+ <PackageReference Include="Newtonsoft.Json" Version="13.0.3"/>
+ <PackageReference Include="NonBlocking" Version="2.1.2"/>
+ <PackageReference Include="OneOf" Version="3.0.271" />
+ <PackageReference Include="OneOf.SourceGenerator" Version="3.0.271" />
+ <PackageReference Include="Serilog.Sinks.Console" Version="6.0.0" />
+ <PackageReference Include="Serilog.Sinks.Seq" Version="9.0.0" />
- <PackageReference Include="SixLabors.Fonts" Version="2.0.4" />
- <PackageReference Include="SixLabors.ImageSharp" Version="3.1.5" />
- <PackageReference Include="SixLabors.ImageSharp.Drawing" Version="2.1.4" />
- <PackageReference Include="SixLabors.Shapes" Version="1.0.0-beta0009" />
- <PackageReference Include="StackExchange.Redis" Version="2.8.0" />
- <PackageReference Include="YamlDotNet" Version="15.1.4" />
- <PackageReference Include="SharpToken" Version="2.0.3" />
+ <PackageReference Include="SixLabors.Fonts" Version="2.1.0" />
+ <PackageReference Include="SixLabors.ImageSharp" Version="3.1.6" />
+ <PackageReference Include="SixLabors.ImageSharp.Drawing" Version="2.1.5" />
+ <PackageReference Include="SixLabors.Shapes" Version="1.0.0-beta0009"/>
+ <PackageReference Include="StackExchange.Redis" Version="2.8.24" />
+ <PackageReference Include="YamlDotNet" Version="15.1.6" />
+ <PackageReference Include="SharpToken" Version="2.0.3"/>
- <PackageReference Include="JetBrains.Annotations" Version="2023.3.0" />
+ <PackageReference Include="JetBrains.Annotations" Version="2024.3.0" />
<!-- Db-related packages -->
- <PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.8" />
- <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.4">
+ <PackageReference Include="Microsoft.EntityFrameworkCore" Version="9.0.1" />
+ <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="9.0.1">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
- <PackageReference Include="linq2db.EntityFrameworkCore" Version="8.1.0" />
+ <PackageReference Include="linq2db.EntityFrameworkCore" Version="9.0.0" />
- <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.8" />
- <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.4" />
+ <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="9.0.1" />
+ <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="9.0.3" />
- <PackageReference Include="EFCore.NamingConventions" Version="8.0.3" />
+ <PackageReference Include="EFCore.NamingConventions" Version="9.0.0" />
<!-- Used by stream notifications -->
- <PackageReference Include="TwitchLib.Api" Version="3.4.1" />
+ <PackageReference Include="TwitchLib.Api" Version="3.9.0" />
<!-- sqlselectcsv and stock -->
- <PackageReference Include="CsvHelper" Version="32.0.3" />
+ <PackageReference Include="CsvHelper" Version="33.0.1" />
</ItemGroup>
diff --git a/src/EllieBot/MigrationRunner.cs b/src/EllieBot/MigrationRunner.cs
new file mode 100644
index 0000000..bc5a895
--- /dev/null
+++ b/src/EllieBot/MigrationRunner.cs
@@ -0,0 +1,54 @@
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+
+namespace EllieBot.Migrations;
+
+public class MigrationRunner
+{
+ public static async Task RunMigration(DbContext ctx)
+ {
+ // if database doesn't exist, run the baseline migration
+ if (!await ctx.Database.CanConnectAsync())
+ {
+ Log.Information("Database does not exist. Creating a new database...");
+ await ctx.Database.MigrateAsync();
+ }
+
+ // get the latest applied migration
+
+ var applied = await ctx.Database.GetAppliedMigrationsAsync();
+
+ // get all .sql file names from the migrations folder
+ var available = Directory.GetFiles("Migrations/Sqlite", "*_*.sql")
+ .Select(x => Path.GetFileNameWithoutExtension(x))
+ .OrderBy(x => x);
+
+ string lastApplied = applied.Last();
+ Log.Information("Last applied migration: {LastApplied}", lastApplied);
+
+ // apply all mirations with names greater than the last applied
+ foreach (var runnable in available)
+ {
+ if (string.Compare(lastApplied, runnable, StringComparison.Ordinal) < 0)
+ {
+ Log.Warning("Migration {MigrationName} has not been applied yet", runnable);
+
+ var query = await File.ReadAllTextAsync(GetMigrationPath(ctx.Database, runnable));
+ await ctx.Database.ExecuteSqlRawAsync(query);
+ }
+ }
+
+ // run all migrations that have not been applied yet
+ }
+
+ private static string GetMigrationPath(DatabaseFacade ctxDatabase, string runnable)
+ {
+ if (ctxDatabase.IsSqlite())
+ return $"Migrations/Sqlite/{runnable}.sql";
+
+ if (ctxDatabase.IsNpgsql())
+ return $"Migrations/PostgreSql/{runnable}.sql";
+
+ throw new NotSupportedException("This database type is not supported.");
+ }
+}
\ No newline at end of file
diff --git a/src/EllieBot/migrate.ps1 b/src/EllieBot/migrate.ps1
index 62f9b69..7a6d4a0 100644
--- a/src/EllieBot/migrate.ps1
+++ b/src/EllieBot/migrate.ps1
@@ -1,8 +1,49 @@
if ($args.Length -eq 0) {
- Write-Host "Please provide a migration name." -ForegroundColor Red
+ Write-Host "Please provide a migration name." -ForegroundColor Red
}
else {
- $migrationName = $args[0]
- dotnet ef migrations add $migrationName -c SqliteContext -p EllieBot.csproj
- dotnet ef migrations add $migrationName -c PostgreSqlContext -p EllieBot.csproj
+ $migrationName = $args[0]
+
+ # find the first .cs item in the folder and get the name without extension
+ $firstMigration = (Get-ChildItem -Path "Migrations/Sqlite" -Filter *.cs | Select-Object -First 1).BaseName
+ $firstPgMigration = (Get-ChildItem -Path "Migrations/PostgreSql" -Filter *.cs | Select-Object -First 1).BaseName
+
+ dotnet build EllieBot.csproj
+
+ dotnet ef migrations add "${migrationName}_p1" --context SqliteContext --project EllieBot.csproj --no-build
+ dotnet ef migrations add "${migrationName}_p1" --context PostgresqlContext --project EllieBot.csproj --no-build
+
+ dotnet build EllieBot.csproj
+ dotnet ef migrations add $migrationName --context SqliteContext --project EllieBot.csproj --no-build
+
+# dotnet ef migrations add "${migrationName}_p1" --context PostgresqlContext --project EllieBot.csproj --no-build
+# dotnet ef migrations add $migrationName --context PostgresqlContext --project EllieBot.csproj --no-build
+
+ dotnet build EllieBot.csproj
+
+ # list migrations as json, get their id and name
+ $sqliteMigNames = (dotnet ef migrations list --no-build --no-connect --project EllieBot.csproj -c SqliteContext --json | ConvertFrom-Json | Select-Object -Last 2)
+
+ $sqliteMigName = ($sqliteMigNames[1] | Select-Object -ExpandProperty id).Split(' ')[0].Trim()
+ $sqliteTempMigName = ($sqliteMigNames[0] | Select-Object -ExpandProperty id).Split(' ')[0].Trim()
+
+
+# $pgMigrationFullName = (dotnet ef migrations list --no-build --no-connect --project EllieBot.csproj -c PostgresqlContext | Select-Object -Last 1)
+# $pgMigName1 = $sqliteMigNames[0].Split(' ')[0].Trim()
+# $pgMigName2 = $sqliteMigNames[1].Split(' ')[0].Trim()
+
+ dotnet ef migrations script $firstMigration $migrationName --project EllieBot.csproj --context SqliteContext --output Migrations/Sqlite/$sqliteMigName.sql --no-build
+# dotnet ef migrations script $firstMigration $migrationName --project EllieBot.csproj --context PostgresqlContext --output Migrations/PostgreSql/$pgMigName2.sql --no-build
+
+ # delete the old first migration .cs and .Designer.cs
+ Remove-Item "Migrations/Sqlite/$firstMigration.cs"
+ Remove-Item "Migrations/Sqlite/$firstMigration.Designer.cs"
+ Remove-Item "Migrations/Sqlite/${sqliteTempMigName}.cs"
+ Remove-Item "Migrations/Sqlite/${sqliteTempMigName}.Designer.cs"
+
+
+# Remove-Item "Migrations/PostgreSql/$firstPgMigration.cs"
+# Remove-Item "Migrations/PostgreSql/$firstPgMigration.Designer.cs"
+# Remove-Item "Migrations/PostgreSql/${pgMigName1}.cs"
+# Remove-Item "Migrations/PostgreSql/${pgMigName1}.Designer.cs"
}
\ No newline at end of file
diff --git a/src/EllieBot/remove-migrations.ps1 b/src/EllieBot/remove-migrations.ps1
index 56fc594..1732410 100644
--- a/src/EllieBot/remove-migrations.ps1
+++ b/src/EllieBot/remove-migrations.ps1
@@ -1,2 +1,5 @@
-dotnet ef migrations remove -c SqliteContext -f -p src/EllieBot/EllieBot.csproj
-dotnet ef migrations remove -c PostgreSqlContext -f -p src/EllieBot/EllieBot.csproj
\ No newline at end of file
+# get the last file in the migrations folder ending with .sql and remove it
+# order files by name
+
+Get-ChildItem -Path "Migrations/Sqlite" -Filter *.sql -Recurse | Sort-Object Name | Select-Object -Last 1 | Remove-Item
+Get-ChildItem -Path "Migrations/PostgreSql" -Filter *.sql -Recurse | Sort-Object Name | Select-Object -Last 1 | Remove-Item
\ No newline at end of file