EllieBotDevs/elliebot
3
1
Fork 1
Code Issues Pull requests Projects Releases56 Packages3 Wiki Activity

wip migration rework

Browse source
This commit is contained in:
Toastie 2025-01-24 00:46:47 +13:00
parent ddd3441771
commit 2138d94d56
Signed by: toastie_t0ast
GPG key ID: 0861BE54AD481DC7
12 changed files with 171 additions and 75 deletions

6
src/Ellie.Marmalade/Ellie.Marmalade.csproj
View file

@ -9,9 +9,9 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="Discord.Net.Core" Version="3.16.0" /> <PackageReference Include="Discord.Net.Core" Version="3.17.1" />
<PackageReference Include="Serilog" Version="3.1.1" /> <PackageReference Include="Serilog" Version="4.2.0" />
<PackageReference Include="YamlDotNet" Version="15.1.4" /> <PackageReference Include="YamlDotNet" Version="15.1.6" />
</ItemGroup> </ItemGroup>
<PropertyGroup Condition=" '$(Version)' == '' "> <PropertyGroup Condition=" '$(Version)' == '' ">

10
src/EllieBot.Coordinator/EllieBot.Coordinator.csproj
View file

@ -9,11 +9,11 @@
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="Grpc.AspNetCore" Version="2.62.0" /> <PackageReference Include="Grpc.AspNetCore" Version="2.67.0" />
<PackageReference Include="Serilog" Version="3.1.1" /> <PackageReference Include="Serilog" Version="4.2.0" />
<PackageReference Include="Serilog.Sinks.Console" Version="5.0.1" /> <PackageReference Include="Serilog.Sinks.Console" Version="6.0.0" />
<PackageReference Include="Serilog.Sinks.File" Version="5.0.0" /> <PackageReference Include="Serilog.Sinks.File" Version="6.0.0" />
<PackageReference Include="YamlDotNet" Version="15.1.4" /> <PackageReference Include="YamlDotNet" Version="15.1.6" />
</ItemGroup> </ItemGroup>
</Project> </Project>

4
src/EllieBot.GrpcApiBase/EllieBot.GrpcApiBase.csproj
View file

@ -7,9 +7,9 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="Google.Protobuf" Version="3.28.2" /> <PackageReference Include="Google.Protobuf" Version="3.29.3" />
<PackageReference Include="Grpc" Version="2.46.6" /> <PackageReference Include="Grpc" Version="2.46.6" />
<PackageReference Include="Grpc.Tools" Version="2.66.0" PrivateAssets="All" /> <PackageReference Include="Grpc.Tools" Version="2.69.0" PrivateAssets="All" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>

6
src/EllieBot.Tests/EllieBot.Tests.csproj
View file

@ -6,9 +6,9 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="NUnit" Version="3.13.3" /> <PackageReference Include="NUnit" Version="4.3.2" />
<PackageReference Include="NUnit3TestAdapter" Version="4.2.1" /> <PackageReference Include="NUnit3TestAdapter" Version="4.6.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.2.0" /> <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.12.0" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>

4
src/EllieBot.Voice/EllieBot.Voice.csproj
View file

@ -9,7 +9,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" /> <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="Serilog" Version="3.1.1" /> <PackageReference Include="Serilog" Version="4.2.0" />
<PackageReference Include="System.Threading.Channels" Version="8.0.0" /> <PackageReference Include="System.Threading.Channels" Version="9.0.1" />
</ItemGroup> </ItemGroup>
</Project> </Project>

2
src/EllieBot.VotesApi/EllieBot.VotesApi.csproj
View file

@ -7,7 +7,7 @@
<ItemGroup> <ItemGroup>
<PackageReference Include="MorseCode.ITask" Version="2.0.3" /> <PackageReference Include="MorseCode.ITask" Version="2.0.3" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.5.0" /> <PackageReference Include="Swashbuckle.AspNetCore" Version="7.2.0" />
</ItemGroup> </ItemGroup>
</Project> </Project>

5
src/EllieBot/Db/EllieDbService.cs
View file

@ -1,6 +1,7 @@
using LinqToDB.Common; using LinqToDB.Common;
using LinqToDB.EntityFrameworkCore; using LinqToDB.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using EllieBot.Migrations;
namespace EllieBot.Db; namespace EllieBot.Db;
@ -26,6 +27,8 @@ public sealed class EllieDbService : DbService
var connString = ConnString; var connString = ConnString;
await using var context = CreateRawDbContext(dbType, connString); await using var context = CreateRawDbContext(dbType, connString);
await MigrationRunner.RunMigration(context);
// make sure sqlite db is in wal journal mode // make sure sqlite db is in wal journal mode
if (context is SqliteContext) if (context is SqliteContext)
@ -33,7 +36,7 @@ public sealed class EllieDbService : DbService
await context.Database.ExecuteSqlRawAsync("PRAGMA journal_mode=WAL"); await context.Database.ExecuteSqlRawAsync("PRAGMA journal_mode=WAL");
} }
await context.Database.MigrateAsync(); // await context.Database.MigrateAsync();
} }
public override EllieContext CreateRawDbContext(string dbType, string connString) public override EllieContext CreateRawDbContext(string dbType, string connString)

5
src/EllieBot/Db/Models/GuildConfig.cs
View file

@ -13,11 +13,6 @@ public class GuildConfig : DbEntity
public string AutoAssignRoleIds { get; set; } public string AutoAssignRoleIds { get; set; }
//todo FUTURE: DELETE, UNUSED
public bool ExclusiveSelfAssignedRoles { get; set; }
public bool AutoDeleteSelfAssignedRoleMessages { get; set; }
//stream notifications //stream notifications
public HashSet<FollowedStream> FollowedStreams { get; set; } = new(); public HashSet<FollowedStream> FollowedStreams { get; set; } = new();

94
src/EllieBot/EllieBot.csproj
View file

@ -23,81 +23,81 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="AngleSharp" Version="1.1.2"> <PackageReference Include="AngleSharp" Version="1.2.0">
<PrivateAssets>all</PrivateAssets> <PrivateAssets>all</PrivateAssets>
<Publish>True</Publish> <Publish>True</Publish>
</PackageReference> </PackageReference>
<PackageReference Include="CodeHollow.FeedReader" Version="1.2.6" /> <PackageReference Include="CodeHollow.FeedReader" Version="1.2.6"/>
<PackageReference Include="CommandLineParser" Version="2.9.1" /> <PackageReference Include="CommandLineParser" Version="2.9.1"/>
<PackageReference Include="Discord.Net" Version="3.16.0" /> <PackageReference Include="Discord.Net" Version="3.17.1" />
<PackageReference Include="CoreCLR-NCalc" Version="3.1.246" /> <PackageReference Include="CoreCLR-NCalc" Version="3.1.253" />
<PackageReference Include="Google.Apis.Urlshortener.v1" Version="1.41.1.138" /> <PackageReference Include="Google.Apis.Urlshortener.v1" Version="1.41.1.138"/>
<PackageReference Include="Google.Apis.YouTube.v3" Version="1.68.0.3414" /> <PackageReference Include="Google.Apis.YouTube.v3" Version="1.68.0.3653" />
<PackageReference Include="Google.Apis.Customsearch.v1" Version="1.49.0.2084" /> <PackageReference Include="Google.Apis.Customsearch.v1" Version="1.49.0.2084"/>
<PackageReference Include="Google.Protobuf" Version="3.28.2" /> <PackageReference Include="Google.Protobuf" Version="3.29.3" />
<PackageReference Include="Grpc" Version="2.46.6" /> <PackageReference Include="Grpc" Version="2.46.6" />
<PackageReference Include="Grpc.Net.Client" Version="2.62.0" /> <PackageReference Include="Grpc.Net.Client" Version="2.67.0" />
<PackageReference Include="Grpc.Tools" Version="2.66.0" PrivateAssets="All" /> <PackageReference Include="Grpc.Tools" Version="2.69.0" PrivateAssets="All" />
<PackageReference Include="Microsoft.CodeAnalysis.CSharp.Scripting" Version="4.5.0" />
<PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0" /> <PackageReference Include="Microsoft.CodeAnalysis.CSharp.Scripting" Version="4.8.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
<PackageReference Include="MorseCode.ITask" Version="2.0.3" /> <PackageReference Include="Microsoft.Extensions.Configuration" Version="9.0.1" />
<PackageReference Include="NetEscapades.Configuration.Yaml" Version="3.1.0" /> <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="9.0.1" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="9.0.1" />
<PackageReference Include="MorseCode.ITask" Version="2.0.3"/>
<PackageReference Include="NetEscapades.Configuration.Yaml" Version="3.1.0"/>
<!-- DI --> <!-- DI -->
<!-- <PackageReference Include="Ninject" Version="3.3.6"/>--> <!-- <PackageReference Include="Ninject" Version="3.3.6"/>-->
<!-- <PackageReference Include="Ninject.Extensions.Conventions" Version="3.3.0"/>--> <!-- <PackageReference Include="Ninject.Extensions.Conventions" Version="3.3.0"/>-->
<!-- <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />--> <!-- <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />-->
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.1" /> <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="9.0.1" />
<PackageReference Include="DryIoc.dll" Version="5.4.3" /> <PackageReference Include="DryIoc.dll" Version="5.4.3"/>
<!-- <PackageReference Include="Scrutor" Version="4.2.0" />--> <!-- <PackageReference Include="Scrutor" Version="4.2.0" />-->
<PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" /> <PackageReference Include="Microsoft.Extensions.Http" Version="9.0.1" />
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="8.0.0" /> <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.1" />
<PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="8.0.0" /> <PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="9.0.1" />
<PackageReference Include="Microsoft.SyndicationFeed.ReaderWriter" Version="1.0.2" /> <PackageReference Include="Microsoft.SyndicationFeed.ReaderWriter" Version="1.0.2"/>
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" /> <PackageReference Include="Newtonsoft.Json" Version="13.0.3"/>
<PackageReference Include="NonBlocking" Version="2.1.2" /> <PackageReference Include="NonBlocking" Version="2.1.2"/>
<PackageReference Include="OneOf" Version="3.0.263" /> <PackageReference Include="OneOf" Version="3.0.271" />
<PackageReference Include="OneOf.SourceGenerator" Version="3.0.263" /> <PackageReference Include="OneOf.SourceGenerator" Version="3.0.271" />
<PackageReference Include="Serilog.Sinks.Console" Version="5.0.1" /> <PackageReference Include="Serilog.Sinks.Console" Version="6.0.0" />
<PackageReference Include="Serilog.Sinks.Seq" Version="7.0.1" /> <PackageReference Include="Serilog.Sinks.Seq" Version="9.0.0" />
<PackageReference Include="SixLabors.Fonts" Version="2.0.4" /> <PackageReference Include="SixLabors.Fonts" Version="2.1.0" />
<PackageReference Include="SixLabors.ImageSharp" Version="3.1.5" /> <PackageReference Include="SixLabors.ImageSharp" Version="3.1.6" />
<PackageReference Include="SixLabors.ImageSharp.Drawing" Version="2.1.4" /> <PackageReference Include="SixLabors.ImageSharp.Drawing" Version="2.1.5" />
<PackageReference Include="SixLabors.Shapes" Version="1.0.0-beta0009" /> <PackageReference Include="SixLabors.Shapes" Version="1.0.0-beta0009"/>
<PackageReference Include="StackExchange.Redis" Version="2.8.0" /> <PackageReference Include="StackExchange.Redis" Version="2.8.24" />
<PackageReference Include="YamlDotNet" Version="15.1.4" /> <PackageReference Include="YamlDotNet" Version="15.1.6" />
<PackageReference Include="SharpToken" Version="2.0.3" /> <PackageReference Include="SharpToken" Version="2.0.3"/>
<PackageReference Include="JetBrains.Annotations" Version="2023.3.0" /> <PackageReference Include="JetBrains.Annotations" Version="2024.3.0" />
<!-- Db-related packages --> <!-- Db-related packages -->
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.8" /> <PackageReference Include="Microsoft.EntityFrameworkCore" Version="9.0.1" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.4"> <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="9.0.1">
<PrivateAssets>all</PrivateAssets> <PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets> <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference> </PackageReference>
<PackageReference Include="linq2db.EntityFrameworkCore" Version="8.1.0" /> <PackageReference Include="linq2db.EntityFrameworkCore" Version="9.0.0" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.8" /> <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="9.0.1" />
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.4" /> <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="9.0.3" />
<PackageReference Include="EFCore.NamingConventions" Version="8.0.3" /> <PackageReference Include="EFCore.NamingConventions" Version="9.0.0" />
<!-- Used by stream notifications --> <!-- Used by stream notifications -->
<PackageReference Include="TwitchLib.Api" Version="3.4.1" /> <PackageReference Include="TwitchLib.Api" Version="3.9.0" />
<!-- sqlselectcsv and stock --> <!-- sqlselectcsv and stock -->
<PackageReference Include="CsvHelper" Version="32.0.3" /> <PackageReference Include="CsvHelper" Version="33.0.1" />
</ItemGroup> </ItemGroup>

54
src/EllieBot/MigrationRunner.cs Normal file
View file

@ -0,0 +1,54 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
namespace EllieBot.Migrations;
public class MigrationRunner
{
public static async Task RunMigration(DbContext ctx)
{
// if database doesn't exist, run the baseline migration
if (!await ctx.Database.CanConnectAsync())
{
Log.Information("Database does not exist. Creating a new database...");
await ctx.Database.MigrateAsync();
}
// get the latest applied migration
var applied = await ctx.Database.GetAppliedMigrationsAsync();
// get all .sql file names from the migrations folder
var available = Directory.GetFiles("Migrations/Sqlite", "*_*.sql")
.Select(x => Path.GetFileNameWithoutExtension(x))
.OrderBy(x => x);
string lastApplied = applied.Last();
Log.Information("Last applied migration: {LastApplied}", lastApplied);
// apply all mirations with names greater than the last applied
foreach (var runnable in available)
{
if (string.Compare(lastApplied, runnable, StringComparison.Ordinal) < 0)
{
Log.Warning("Migration {MigrationName} has not been applied yet", runnable);
var query = await File.ReadAllTextAsync(GetMigrationPath(ctx.Database, runnable));
await ctx.Database.ExecuteSqlRawAsync(query);
}
}
// run all migrations that have not been applied yet
}
private static string GetMigrationPath(DatabaseFacade ctxDatabase, string runnable)
{
if (ctxDatabase.IsSqlite())
return $"Migrations/Sqlite/{runnable}.sql";
if (ctxDatabase.IsNpgsql())
return $"Migrations/PostgreSql/{runnable}.sql";
throw new NotSupportedException("This database type is not supported.");
}
}

49
src/EllieBot/migrate.ps1
View file

@ -1,8 +1,49 @@
if ($args.Length -eq 0) { if ($args.Length -eq 0) {
Write-Host "Please provide a migration name." -ForegroundColor Red Write-Host "Please provide a migration name." -ForegroundColor Red
} }
else { else {
$migrationName = $args[0] $migrationName = $args[0]
dotnet ef migrations add $migrationName -c SqliteContext -p EllieBot.csproj
dotnet ef migrations add $migrationName -c PostgreSqlContext -p EllieBot.csproj # find the first .cs item in the folder and get the name without extension
$firstMigration = (Get-ChildItem -Path "Migrations/Sqlite" -Filter *.cs | Select-Object -First 1).BaseName
$firstPgMigration = (Get-ChildItem -Path "Migrations/PostgreSql" -Filter *.cs | Select-Object -First 1).BaseName
dotnet build EllieBot.csproj
dotnet ef migrations add "${migrationName}_p1" --context SqliteContext --project EllieBot.csproj --no-build
dotnet ef migrations add "${migrationName}_p1" --context PostgresqlContext --project EllieBot.csproj --no-build
dotnet build EllieBot.csproj
dotnet ef migrations add $migrationName --context SqliteContext --project EllieBot.csproj --no-build
# dotnet ef migrations add "${migrationName}_p1" --context PostgresqlContext --project EllieBot.csproj --no-build
# dotnet ef migrations add $migrationName --context PostgresqlContext --project EllieBot.csproj --no-build
dotnet build EllieBot.csproj
# list migrations as json, get their id and name
$sqliteMigNames = (dotnet ef migrations list --no-build --no-connect --project EllieBot.csproj -c SqliteContext --json | ConvertFrom-Json | Select-Object -Last 2)
$sqliteMigName = ($sqliteMigNames[1] | Select-Object -ExpandProperty id).Split(' ')[0].Trim()
$sqliteTempMigName = ($sqliteMigNames[0] | Select-Object -ExpandProperty id).Split(' ')[0].Trim()
# $pgMigrationFullName = (dotnet ef migrations list --no-build --no-connect --project EllieBot.csproj -c PostgresqlContext | Select-Object -Last 1)
# $pgMigName1 = $sqliteMigNames[0].Split(' ')[0].Trim()
# $pgMigName2 = $sqliteMigNames[1].Split(' ')[0].Trim()
dotnet ef migrations script $firstMigration $migrationName --project EllieBot.csproj --context SqliteContext --output Migrations/Sqlite/$sqliteMigName.sql --no-build
# dotnet ef migrations script $firstMigration $migrationName --project EllieBot.csproj --context PostgresqlContext --output Migrations/PostgreSql/$pgMigName2.sql --no-build
# delete the old first migration .cs and .Designer.cs
Remove-Item "Migrations/Sqlite/$firstMigration.cs"
Remove-Item "Migrations/Sqlite/$firstMigration.Designer.cs"
Remove-Item "Migrations/Sqlite/${sqliteTempMigName}.cs"
Remove-Item "Migrations/Sqlite/${sqliteTempMigName}.Designer.cs"
# Remove-Item "Migrations/PostgreSql/$firstPgMigration.cs"
# Remove-Item "Migrations/PostgreSql/$firstPgMigration.Designer.cs"
# Remove-Item "Migrations/PostgreSql/${pgMigName1}.cs"
# Remove-Item "Migrations/PostgreSql/${pgMigName1}.Designer.cs"
} }

7
src/EllieBot/remove-migrations.ps1
View file

@ -1,2 +1,5 @@
dotnet ef migrations remove -c SqliteContext -f -p src/EllieBot/EllieBot.csproj # get the last file in the migrations folder ending with .sql and remove it
dotnet ef migrations remove -c PostgreSqlContext -f -p src/EllieBot/EllieBot.csproj # order files by name
Get-ChildItem -Path "Migrations/Sqlite" -Filter *.sql -Recurse | Sort-Object Name | Select-Object -Last 1 | Remove-Item
Get-ChildItem -Path "Migrations/PostgreSql" -Filter *.sql -Recurse | Sort-Object Name | Select-Object -Last 1 | Remove-Item