From 491bd7d63159b8b4c7d2d2224502a2cab78fbcc2 Mon Sep 17 00:00:00 2001 From: Emotion Date: Thu, 24 Aug 2023 01:12:08 +1200 Subject: [PATCH] Services time! Signed-off-by: Emotion --- src/Ellie/Services/Impl/BotCredsProvider.cs | 204 ++++++++++++++++ src/Ellie/Services/Impl/GoogleApiService.cs | 227 ++++++++++++++++++ .../GoogleApiService_SupportedLanguages.cs | 160 ++++++++++++ src/Ellie/Services/Impl/ImageCache.cs | 83 +++++++ src/Ellie/Services/Impl/LocalDataCache.cs | 108 +++++++++ src/Ellie/Services/Impl/Localization.cs | 121 ++++++++++ src/Ellie/Services/Impl/PubSub/JsonSeria.cs | 27 +++ src/Ellie/Services/Impl/PubSub/YamlSeria.cs | 39 +++ src/Ellie/Services/Impl/RedisBotCache.cs | 119 +++++++++ .../Services/Impl/RedisBotStringsProvider.cs | 78 ++++++ .../Services/Impl/RemoteGrpcCoordinator.cs | 132 ++++++++++ src/Ellie/_common/NinjectIKernalExtensions.cs | 43 ++++ 12 files changed, 1341 insertions(+) create mode 100644 src/Ellie/Services/Impl/BotCredsProvider.cs create mode 100644 src/Ellie/Services/Impl/GoogleApiService.cs create mode 100644 src/Ellie/Services/Impl/GoogleApiService_SupportedLanguages.cs create mode 100644 src/Ellie/Services/Impl/ImageCache.cs create mode 100644 src/Ellie/Services/Impl/LocalDataCache.cs create mode 100644 src/Ellie/Services/Impl/Localization.cs create mode 100644 src/Ellie/Services/Impl/PubSub/JsonSeria.cs create mode 100644 src/Ellie/Services/Impl/PubSub/YamlSeria.cs create mode 100644 src/Ellie/Services/Impl/RedisBotCache.cs create mode 100644 src/Ellie/Services/Impl/RedisBotStringsProvider.cs create mode 100644 src/Ellie/Services/Impl/RemoteGrpcCoordinator.cs create mode 100644 src/Ellie/_common/NinjectIKernalExtensions.cs diff --git a/src/Ellie/Services/Impl/BotCredsProvider.cs b/src/Ellie/Services/Impl/BotCredsProvider.cs new file mode 100644 index 0000000..e556a02 --- /dev/null +++ b/src/Ellie/Services/Impl/BotCredsProvider.cs @@ -0,0 +1,204 @@ +#nullable disable +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Primitives; +using Ellie.Common.Yml; +using Newtonsoft.Json; + +namespace Ellie.Services; + +public sealed class BotCredsProvider : IBotCredsProvider +{ + private const string CREDS_FILE_NAME = "creds.yml"; + private const string CREDS_EXAMPLE_FILE_NAME = "creds_example.yml"; + + private string CredsPath { get; } + + private string CredsExamplePath { get; } + + private readonly int? _totalShards; + + + private readonly Creds _creds = new(); + private readonly IConfigurationRoot _config; + + + private readonly object _reloadLock = new(); + private readonly IDisposable _changeToken; + + public BotCredsProvider(int? totalShards = null, string credPath = null) + { + _totalShards = totalShards; + + if (!string.IsNullOrWhiteSpace(credPath)) + { + CredsPath = credPath; + CredsExamplePath = Path.Combine(Path.GetDirectoryName(credPath), CREDS_EXAMPLE_FILE_NAME); + } + else + { + CredsPath = Path.Combine(Directory.GetCurrentDirectory(), CREDS_FILE_NAME); + CredsExamplePath = Path.Combine(Directory.GetCurrentDirectory(), CREDS_EXAMPLE_FILE_NAME); + } + + try + { + if (!File.Exists(CredsExamplePath)) + File.WriteAllText(CredsExamplePath, Yaml.Serializer.Serialize(_creds)); + } + catch + { + // this can fail in docker containers + } + + MigrateCredentials(); + + if (!File.Exists(CredsPath)) + { + Log.Warning( + "{CredsPath} is missing. Attempting to load creds from environment variables prefixed with 'Ellie_'. Example is in {CredsExamplePath}", + CredsPath, + CredsExamplePath); + } + + try + { + _config = new ConfigurationBuilder().AddYamlFile(CredsPath, false, true) + .AddEnvironmentVariables("Ellie_") + .Build(); + } + catch (Exception ex) + { + Console.WriteLine(ex.ToString()); + } + + _changeToken = ChangeToken.OnChange(() => _config.GetReloadToken(), Reload); + Reload(); + } + + public void Reload() + { + lock (_reloadLock) + { + _creds.OwnerIds.Clear(); + _config.Bind(_creds); + + if (string.IsNullOrWhiteSpace(_creds.Token)) + { + Log.Error("Token is missing from creds.yml or Environment variables.\nAdd it and restart the program"); + Helpers.ReadErrorAndExit(5); + return; + } + + if (string.IsNullOrWhiteSpace(_creds.RestartCommand?.Cmd) + || string.IsNullOrWhiteSpace(_creds.RestartCommand?.Args)) + { + if (Environment.OSVersion.Platform == PlatformID.Unix) + { + _creds.RestartCommand = new RestartConfig() + { + Args = "dotnet", + Cmd = "Ellie.dll -- {0}" + }; + } + else + { + _creds.RestartCommand = new RestartConfig() + { + Args = "Ellie.exe", + Cmd = "{0}" + }; + } + } + + if (string.IsNullOrWhiteSpace(_creds.RedisOptions)) + _creds.RedisOptions = "127.0.0.1,syncTimeout=3000"; + + // replace the old generated key with the shared key + if (string.IsNullOrWhiteSpace(_creds.CoinmarketcapApiKey) + || _creds.CoinmarketcapApiKey.StartsWith("e79ec505-0913")) + _creds.CoinmarketcapApiKey = "3077537c-7dfb-4d97-9a60-56fc9a9f5035"; + + _creds.TotalShards = _totalShards ?? _creds.TotalShards; + } + } + + public void ModifyCredsFile(Action func) + { + var ymlData = File.ReadAllText(CREDS_FILE_NAME); + var creds = Yaml.Deserializer.Deserialize(ymlData); + + func(creds); + + ymlData = Yaml.Serializer.Serialize(creds); + File.WriteAllText(CREDS_FILE_NAME, ymlData); + } + + private string OldCredsJsonPath + => Path.Combine(Directory.GetCurrentDirectory(), "credentials.json"); + + private string OldCredsJsonBackupPath + => Path.Combine(Directory.GetCurrentDirectory(), "credentials.json.bak"); + + private void MigrateCredentials() + { + if (File.Exists(OldCredsJsonPath)) + { + Log.Information("Migrating old creds..."); + var jsonCredentialsFileText = File.ReadAllText(OldCredsJsonPath); + var oldCreds = JsonConvert.DeserializeObject(jsonCredentialsFileText); + + if (oldCreds is null) + { + Log.Error("Error while reading old credentials file. Make sure that the file is formatted correctly"); + return; + } + + var creds = new Creds + { + Version = 1, + Token = oldCreds.Token, + OwnerIds = oldCreds.OwnerIds.Distinct().ToHashSet(), + GoogleApiKey = oldCreds.GoogleApiKey, + RapidApiKey = oldCreds.MashapeKey, + OsuApiKey = oldCreds.OsuApiKey, + CleverbotApiKey = oldCreds.CleverbotApiKey, + TotalShards = oldCreds.TotalShards <= 1 ? 1 : oldCreds.TotalShards, + Patreon = new Creds.PatreonSettings(oldCreds.PatreonAccessToken, null, null, oldCreds.PatreonCampaignId), + Votes = new Creds.VotesSettings(oldCreds.VotesUrl, oldCreds.VotesToken, string.Empty, string.Empty), + BotListToken = oldCreds.BotListToken, + RedisOptions = oldCreds.RedisOptions, + LocationIqApiKey = oldCreds.LocationIqApiKey, + TimezoneDbApiKey = oldCreds.TimezoneDbApiKey, + CoinmarketcapApiKey = oldCreds.CoinmarketcapApiKey + }; + + File.Move(OldCredsJsonPath, OldCredsJsonBackupPath, true); + File.WriteAllText(CredsPath, Yaml.Serializer.Serialize(creds)); + + Log.Warning( + "Data from credentials.json has been moved to creds.yml\nPlease inspect your creds.yml for correctness"); + } + + if (File.Exists(CREDS_FILE_NAME)) + { + var creds = Yaml.Deserializer.Deserialize(File.ReadAllText(CREDS_FILE_NAME)); + if (creds.Version <= 5) + { + creds.BotCache = BotCacheImplemenation.Redis; + } + if (creds.Version <= 6) + { + creds.Version = 7; + File.WriteAllText(CREDS_FILE_NAME, Yaml.Serializer.Serialize(creds)); + } + } + } + + public IBotCredentials GetCreds() + { + lock (_reloadLock) + { + return _creds; + } + } +} \ No newline at end of file diff --git a/src/Ellie/Services/Impl/GoogleApiService.cs b/src/Ellie/Services/Impl/GoogleApiService.cs new file mode 100644 index 0000000..40d3468 --- /dev/null +++ b/src/Ellie/Services/Impl/GoogleApiService.cs @@ -0,0 +1,227 @@ +#nullable disable +using Google; +using Google.Apis.Services; +using Google.Apis.Urlshortener.v1; +using Google.Apis.YouTube.v3; +using Newtonsoft.Json.Linq; +using System.Net; +using System.Text.RegularExpressions; +using System.Xml; + +namespace Ellie.Services; + +public sealed partial class GoogleApiService : IGoogleApiService, IEService +{ + private static readonly Regex + _plRegex = new(@"(?:youtu\.be\/|list=)(?[\da-zA-Z\-_]*)", RegexOptions.Compiled); + + + private readonly YouTubeService _yt; + private readonly UrlshortenerService _sh; + + //private readonly Regex YtVideoIdRegex = new Regex(@"(?:youtube\.com\/\S*(?:(?:\/e(?:mbed))?\/|watch\?(?:\S*?&?v\=))|youtu\.be\/)(?[a-zA-Z0-9_-]{6,11})", RegexOptions.Compiled); + private readonly IBotCredsProvider _creds; + private readonly IHttpClientFactory _httpFactory; + + public GoogleApiService(IBotCredsProvider creds, IHttpClientFactory factory) : this() + { + _creds = creds; + _httpFactory = factory; + + var bcs = new BaseClientService.Initializer + { + ApplicationName = "Nadeko Bot", + ApiKey = _creds.GetCreds().GoogleApiKey + }; + + _yt = new(bcs); + _sh = new(bcs); + } + + public async Task> GetPlaylistIdsByKeywordsAsync(string keywords, int count = 1) + { + if (string.IsNullOrWhiteSpace(keywords)) + throw new ArgumentNullException(nameof(keywords)); + + if (count <= 0) + throw new ArgumentOutOfRangeException(nameof(count)); + + var match = _plRegex.Match(keywords); + if (match.Length > 1) + return new[] { match.Groups["id"].Value }; + var query = _yt.Search.List("snippet"); + query.MaxResults = count; + query.Type = "playlist"; + query.Q = keywords; + + return (await query.ExecuteAsync()).Items.Select(i => i.Id.PlaylistId); + } + + public async Task> GetRelatedVideosAsync(string id, int count = 1, string user = null) + { + if (string.IsNullOrWhiteSpace(id)) + throw new ArgumentNullException(nameof(id)); + + if (count <= 0) + throw new ArgumentOutOfRangeException(nameof(count)); + + var query = _yt.Search.List("snippet"); + query.MaxResults = count; + query.RelatedToVideoId = id; + query.Type = "video"; + query.QuotaUser = user; + return (await query.ExecuteAsync()).Items.Select(i => "https://www.youtube.com/watch?v=" + i.Id.VideoId); + } + + public async Task> GetVideoLinksByKeywordAsync(string keywords, int count = 1) + { + if (string.IsNullOrWhiteSpace(keywords)) + throw new ArgumentNullException(nameof(keywords)); + + if (count <= 0) + throw new ArgumentOutOfRangeException(nameof(count)); + + var query = _yt.Search.List("snippet"); + query.MaxResults = count; + query.Q = keywords; + query.Type = "video"; + query.SafeSearch = SearchResource.ListRequest.SafeSearchEnum.Strict; + return (await query.ExecuteAsync()).Items.Select(i => "https://www.youtube.com/watch?v=" + i.Id.VideoId); + } + + public async Task> GetVideoInfosByKeywordAsync( + string keywords, + int count = 1) + { + if (string.IsNullOrWhiteSpace(keywords)) + throw new ArgumentNullException(nameof(keywords)); + + if (count <= 0) + throw new ArgumentOutOfRangeException(nameof(count)); + + var query = _yt.Search.List("snippet"); + query.MaxResults = count; + query.Q = keywords; + query.Type = "video"; + return (await query.ExecuteAsync()).Items.Select(i + => (i.Snippet.Title.TrimTo(50), i.Id.VideoId, "https://www.youtube.com/watch?v=" + i.Id.VideoId)); + } + + public Task ShortenUrl(Uri url) + => ShortenUrl(url.ToString()); + + public async Task ShortenUrl(string url) + { + if (string.IsNullOrWhiteSpace(url)) + throw new ArgumentNullException(nameof(url)); + + if (string.IsNullOrWhiteSpace(_creds.GetCreds().GoogleApiKey)) + return url; + + try + { + var response = await _sh.Url.Insert(new() + { + LongUrl = url + }) + .ExecuteAsync(); + return response.Id; + } + catch (GoogleApiException ex) when (ex.HttpStatusCode == HttpStatusCode.Forbidden) + { + return url; + } + catch (Exception ex) + { + Log.Warning(ex, "Error shortening URL"); + return url; + } + } + + public async Task> GetPlaylistTracksAsync(string playlistId, int count = 50) + { + if (string.IsNullOrWhiteSpace(playlistId)) + throw new ArgumentNullException(nameof(playlistId)); + + if (count <= 0) + throw new ArgumentOutOfRangeException(nameof(count)); + + string nextPageToken = null; + + var toReturn = new List(count); + + do + { + var toGet = count > 50 ? 50 : count; + count -= toGet; + + var query = _yt.PlaylistItems.List("contentDetails"); + query.MaxResults = toGet; + query.PlaylistId = playlistId; + query.PageToken = nextPageToken; + + var data = await query.ExecuteAsync(); + + toReturn.AddRange(data.Items.Select(i => i.ContentDetails.VideoId)); + nextPageToken = data.NextPageToken; + } while (count > 0 && !string.IsNullOrWhiteSpace(nextPageToken)); + + return toReturn; + } + + public async Task> GetVideoDurationsAsync(IEnumerable videoIds) + { + var videoIdsList = videoIds as List ?? videoIds.ToList(); + + var toReturn = new Dictionary(); + + if (!videoIdsList.Any()) + return toReturn; + var remaining = videoIdsList.Count; + + do + { + var toGet = remaining > 50 ? 50 : remaining; + remaining -= toGet; + + var q = _yt.Videos.List("contentDetails"); + q.Id = string.Join(",", videoIdsList.Take(toGet)); + videoIdsList = videoIdsList.Skip(toGet).ToList(); + var items = (await q.ExecuteAsync()).Items; + foreach (var i in items) + toReturn.Add(i.Id, XmlConvert.ToTimeSpan(i.ContentDetails.Duration)); + } while (remaining > 0); + + return toReturn; + } + + public async Task Translate(string sourceText, string sourceLanguage, string targetLanguage) + { + string text; + + if (!Languages.ContainsKey(sourceLanguage) || !Languages.ContainsKey(targetLanguage)) + throw new ArgumentException(nameof(sourceLanguage) + "/" + nameof(targetLanguage)); + + + var url = new Uri(string.Format( + "https://translate.googleapis.com/translate_a/single?client=gtx&sl={0}&tl={1}&dt=t&q={2}", + ConvertToLanguageCode(sourceLanguage), + ConvertToLanguageCode(targetLanguage), + WebUtility.UrlEncode(sourceText))); + using (var http = _httpFactory.CreateClient()) + { + http.DefaultRequestHeaders.Add("user-agent", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36"); + text = await http.GetStringAsync(url); + } + + return string.Concat(JArray.Parse(text)[0].Select(x => x[0])); + } + + private string ConvertToLanguageCode(string language) + { + Languages.TryGetValue(language, out var mode); + return mode; + } +} + diff --git a/src/Ellie/Services/Impl/GoogleApiService_SupportedLanguages.cs b/src/Ellie/Services/Impl/GoogleApiService_SupportedLanguages.cs new file mode 100644 index 0000000..43caec7 --- /dev/null +++ b/src/Ellie/Services/Impl/GoogleApiService_SupportedLanguages.cs @@ -0,0 +1,160 @@ +namespace Ellie.Services; + +public sealed partial class GoogleApiService +{ + private const string SUPPORTED = """ + afrikaans af + albanian sq + amharic am + arabic ar + armenian hy + assamese as + aymara ay + azerbaijani az + bambara bm + basque eu + belarusian be + bengali bn + bhojpuri bho + bosnian bs + bulgarian bg + catalan ca + cebuano ceb + chinese zh-CN + chinese-trad zh-TW + corsican co + croatian hr + czech cs + danish da + dhivehi dv + dogri doi + dutch nl + english en + esperanto eo + estonian et + ewe ee + filipino fil + finnish fi + french fr + frisian fy + galician gl + georgian ka + german de + greek el + guarani gn + gujarati gu + haitian ht + hausa ha + hawaiian haw + hebrew he + hindi hi + hmong hmn + hungarian hu + icelandic is + igbo ig + ilocano ilo + indonesian id + irish ga + italian it + japanese ja + javanese jv + kannada kn + kazakh kk + khmer km + kinyarwanda rw + konkani gom + korean ko + krio kri + kurdish ku + kurdish-sor ckb + kyrgyz ky + lao lo + latin la + latvian lv + lingala ln + lithuanian lt + luganda lg + luxembourgish lb + macedonian mk + maithili mai + malagasy mg + malay ms + malayalam ml + maltese mt + maori mi + marathi mr + meiteilon mni-Mtei + mizo lus + mongolian mn + myanmar my + nepali ne + norwegian no + nyanja ny + odia or + oromo om + pashto ps + persian fa + polish pl + portuguese pt + punjabi pa + quechua qu + romanian ro + russian ru + samoan sm + sanskrit sa + scots gd + sepedi nso + serbian sr + sesotho st + shona sn + sindhi sd + sinhala si + slovak sk + slovenian sl + somali so + spanish es + sundanese su + swahili sw + swedish sv + tagalog tl + tajik tg + tamil ta + tatar tt + telugu te + thai th + tigrinya ti + tsonga ts + turkish tr + turkmen tk + twi ak + ukrainian uk + urdu ur + uyghur ug + uzbek uz + vietnamese vi + welsh cy + xhosa xh + yiddish yi + yoruba yo + zulu zu + """; + + + public IReadOnlyDictionary Languages { get; } + + private GoogleApiService() + { + var langs = SUPPORTED.Split("\n") + .Select(x => x.Split(' ')) + .ToDictionary(x => x[0].Trim(), x => x[1].Trim()); + + foreach (var (_, v) in langs.ToArray()) + { + langs.Add(v, v); + } + + Languages = langs; + + } + +} \ No newline at end of file diff --git a/src/Ellie/Services/Impl/ImageCache.cs b/src/Ellie/Services/Impl/ImageCache.cs new file mode 100644 index 0000000..0f513f3 --- /dev/null +++ b/src/Ellie/Services/Impl/ImageCache.cs @@ -0,0 +1,83 @@ +namespace Ellie.Services; + +public sealed class ImageCache : IImageCache, IEService +{ + private readonly IBotCache _cache; + private readonly ImagesConfig _ic; + private readonly Random _rng; + private readonly IHttpClientFactory _httpFactory; + + public ImageCache( + IBotCache cache, + ImagesConfig ic, + IHttpClientFactory httpFactory) + { + _cache = cache; + _ic = ic; + _httpFactory = httpFactory; + _rng = new EllieRandom(); + } + + private static TypedKey GetImageKey(Uri url) + => new($"image:{url}"); + + public async Task GetImageDataAsync(Uri url) + => await _cache.GetOrAddAsync( + GetImageKey(url), + async () => + { + if (url.IsFile) + { + return await File.ReadAllBytesAsync(url.LocalPath); + } + + using var http = _httpFactory.CreateClient(); + var bytes = await http.GetByteArrayAsync(url); + return bytes; + }, + expiry: 48.Hours()); + + private async Task GetRandomImageDataAsync(Uri[] urls) + { + if (urls.Length == 0) + return null; + + var url = urls[_rng.Next(0, urls.Length)]; + + var data = await GetImageDataAsync(url); + return data; + } + + public Task GetHeadsImageAsync() + => GetRandomImageDataAsync(_ic.Data.Coins.Heads); + + public Task GetTailsImageAsync() + => GetRandomImageDataAsync(_ic.Data.Coins.Tails); + + public Task GetCurrencyImageAsync() + => GetRandomImageDataAsync(_ic.Data.Currency); + + public Task GetXpBackgroundImageAsync() + => GetImageDataAsync(_ic.Data.Xp.Bg); + + public Task GetRategirlBgAsync() + => GetImageDataAsync(_ic.Data.Rategirl.Matrix); + + public Task GetRategirlDotAsync() + => GetImageDataAsync(_ic.Data.Rategirl.Dot); + + public Task GetDiceAsync(int num) + => GetImageDataAsync(_ic.Data.Dice[num]); + + public Task GetSlotEmojiAsync(int number) + => GetImageDataAsync(_ic.Data.Slots.Emojis[number]); + + public Task GetSlotBgAsync() + => GetImageDataAsync(_ic.Data.Slots.Bg); + + public Task GetRipBgAsync() + => GetImageDataAsync(_ic.Data.Rip.Bg); + + public Task GetRipOverlayAsync() + => GetImageDataAsync(_ic.Data.Rip.Overlay); +} \ No newline at end of file diff --git a/src/Ellie/Services/Impl/LocalDataCache.cs b/src/Ellie/Services/Impl/LocalDataCache.cs new file mode 100644 index 0000000..da05cab --- /dev/null +++ b/src/Ellie/Services/Impl/LocalDataCache.cs @@ -0,0 +1,108 @@ +using Ellie.Common.Pokemon; +using Ellie.Modules.Games.Common.Trivia; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Ellie.Services; + +public sealed class LocalDataCache : ILocalDataCache, IEService +{ + private const string POKEMON_ABILITIES_FILE = "data/pokemon/pokemon_abilities.json"; + private const string POKEMON_LIST_FILE = "data/pokemon/pokemon_list.json"; + private const string POKEMON_MAP_PATH = "data/pokemon/name-id_map.json"; + private const string QUESTIONS_FILE = "data/trivia_questions.json"; + + private readonly IBotCache _cache; + + private readonly JsonSerializerOptions _opts = new JsonSerializerOptions() + { + AllowTrailingCommas = true, + NumberHandling = JsonNumberHandling.AllowReadingFromString, + PropertyNameCaseInsensitive = true + }; + + public LocalDataCache(IBotCache cache) + => _cache = cache; + + private async Task GetOrCreateCachedDataAsync( + TypedKey key, + string fileName) + => await _cache.GetOrAddAsync(key, + async () => + { + if (!File.Exists(fileName)) + { + Log.Warning($"{fileName} is missing. Relevant data can't be loaded"); + return default; + } + + try + { + await using var stream = File.OpenRead(fileName); + return await JsonSerializer.DeserializeAsync(stream, _opts); + } + catch (Exception ex) + { + Log.Error(ex, + "Error reading {FileName} file: {ErrorMessage}", + fileName, + ex.Message); + + return default; + } + }); + + + private static TypedKey> _pokemonListKey + = new("pokemon:list"); + + public async Task?> GetPokemonsAsync() + => await GetOrCreateCachedDataAsync(_pokemonListKey, POKEMON_LIST_FILE); + + + private static TypedKey> _pokemonAbilitiesKey + = new("pokemon:abilities"); + + public async Task?> GetPokemonAbilitiesAsync() + => await GetOrCreateCachedDataAsync(_pokemonAbilitiesKey, POKEMON_ABILITIES_FILE); + + + private static TypedKey> _pokeMapKey + = new("pokemon:ab_map2"); // 2 because ab_map was storing arrays + + public async Task?> GetPokemonMapAsync() + => await _cache.GetOrAddAsync(_pokeMapKey, + async () => + { + var fileName = POKEMON_MAP_PATH; + if (!File.Exists(fileName)) + { + Log.Warning($"{fileName} is missing. Relevant data can't be loaded"); + return default; + } + + try + { + await using var stream = File.OpenRead(fileName); + var arr = await JsonSerializer.DeserializeAsync(stream, _opts); + + return (IReadOnlyDictionary?)arr?.ToDictionary(x => x.Id, x => x.Name); + } + catch (Exception ex) + { + Log.Error(ex, + "Error reading {FileName} file: {ErrorMessage}", + fileName, + ex.Message); + + return default; + } + }); + + + private static TypedKey _triviaKey + = new("trivia:questions"); + + public async Task GetTriviaQuestionsAsync() + => await GetOrCreateCachedDataAsync(_triviaKey, QUESTIONS_FILE); +} \ No newline at end of file diff --git a/src/Ellie/Services/Impl/Localization.cs b/src/Ellie/Services/Impl/Localization.cs new file mode 100644 index 0000000..1f262ae --- /dev/null +++ b/src/Ellie/Services/Impl/Localization.cs @@ -0,0 +1,121 @@ +#nullable disable +using Ellie.Db; +using Newtonsoft.Json; +using System.Globalization; + +namespace Ellie.Services; + +public class Localization : ILocalization, IEService +{ + private static readonly Dictionary _commandData = + JsonConvert.DeserializeObject>( + File.ReadAllText("./data/strings/commands/commands.en-US.json")); + + private readonly ConcurrentDictionary _guildCultureInfos; + + public IDictionary GuildCultureInfos + => _guildCultureInfos; + + public CultureInfo DefaultCultureInfo + => _bss.Data.DefaultLocale; + + private readonly BotConfigService _bss; + private readonly DbService _db; + + public Localization(BotConfigService bss, Bot bot, DbService db) + { + _bss = bss; + _db = db; + + var cultureInfoNames = bot.AllGuildConfigs.ToDictionary(x => x.GuildId, x => x.Locale); + + _guildCultureInfos = new(cultureInfoNames + .ToDictionary(x => x.Key, + x => + { + CultureInfo cultureInfo = null; + try + { + if (x.Value is null) + return null; + cultureInfo = new(x.Value); + } + catch { } + + return cultureInfo; + }) + .Where(x => x.Value is not null)); + } + + public void SetGuildCulture(IGuild guild, CultureInfo ci) + => SetGuildCulture(guild.Id, ci); + + public void SetGuildCulture(ulong guildId, CultureInfo ci) + { + if (ci.Name == _bss.Data.DefaultLocale.Name) + { + RemoveGuildCulture(guildId); + return; + } + + using (var uow = _db.GetDbContext()) + { + var gc = uow.GuildConfigsForId(guildId, set => set); + gc.Locale = ci.Name; + uow.SaveChanges(); + } + + _guildCultureInfos.AddOrUpdate(guildId, ci, (_, _) => ci); + } + + public void RemoveGuildCulture(IGuild guild) + => RemoveGuildCulture(guild.Id); + + public void RemoveGuildCulture(ulong guildId) + { + if (_guildCultureInfos.TryRemove(guildId, out _)) + { + using var uow = _db.GetDbContext(); + var gc = uow.GuildConfigsForId(guildId, set => set); + gc.Locale = null; + uow.SaveChanges(); + } + } + + public void SetDefaultCulture(CultureInfo ci) + => _bss.ModifyConfig(bs => + { + bs.DefaultLocale = ci; + }); + + public void ResetDefaultCulture() + => SetDefaultCulture(CultureInfo.CurrentCulture); + + public CultureInfo GetCultureInfo(IGuild guild) + => GetCultureInfo(guild?.Id); + + public CultureInfo GetCultureInfo(ulong? guildId) + { + if (guildId is null || !GuildCultureInfos.TryGetValue(guildId.Value, out var info) || info is null) + return _bss.Data.DefaultLocale; + + return info; + } + + public static CommandData LoadCommand(string key) + { + _commandData.TryGetValue(key, out var toReturn); + + if (toReturn is null) + { + return new() + { + Cmd = key, + Desc = key, + Usage = new[] { key } + }; + } + + return toReturn; + } +} \ No newline at end of file diff --git a/src/Ellie/Services/Impl/PubSub/JsonSeria.cs b/src/Ellie/Services/Impl/PubSub/JsonSeria.cs new file mode 100644 index 0000000..dddc4b3 --- /dev/null +++ b/src/Ellie/Services/Impl/PubSub/JsonSeria.cs @@ -0,0 +1,27 @@ +using Ellie.Common.JsonConverters; +using System.Text.Json; + +namespace Ellie.Common; + +public class JsonSeria : ISeria +{ + private readonly JsonSerializerOptions _serializerOptions = new() + { + Converters = + { + new Rgba32Converter(), + new CultureInfoConverter() + } + }; + + public byte[] Serialize(T data) + => JsonSerializer.SerializeToUtf8Bytes(data, _serializerOptions); + + public T? Deserialize(byte[]? data) + { + if (data is null) + return default; + + return JsonSerializer.Deserialize(data, _serializerOptions); + } +} \ No newline at end of file diff --git a/src/Ellie/Services/Impl/PubSub/YamlSeria.cs b/src/Ellie/Services/Impl/PubSub/YamlSeria.cs new file mode 100644 index 0000000..dc73dd0 --- /dev/null +++ b/src/Ellie/Services/Impl/PubSub/YamlSeria.cs @@ -0,0 +1,39 @@ +using Ellie.Common.Configs; +using Ellie.Common.Yml; +using System.Text.RegularExpressions; +using YamlDotNet.Serialization; + +namespace Ellie.Common; + +public class YamlSeria : IConfigSeria +{ + private static readonly Regex _codePointRegex = + new(@"(\\U(?[a-zA-Z0-9]{8})|\\u(?[a-zA-Z0-9]{4})|\\x(?[a-zA-Z0-9]{2}))", + RegexOptions.Compiled); + + private readonly IDeserializer _deserializer; + private readonly ISerializer _serializer; + + public YamlSeria() + { + _serializer = Yaml.Serializer; + _deserializer = Yaml.Deserializer; + } + + public string Serialize(T obj) + where T : notnull + { + var escapedOutput = _serializer.Serialize(obj); + var output = _codePointRegex.Replace(escapedOutput, + me => + { + var str = me.Groups["code"].Value; + var newString = YamlHelper.UnescapeUnicodeCodePoint(str); + return newString; + }); + return output; + } + + public T Deserialize(string data) + => _deserializer.Deserialize(data); +} \ No newline at end of file diff --git a/src/Ellie/Services/Impl/RedisBotCache.cs b/src/Ellie/Services/Impl/RedisBotCache.cs new file mode 100644 index 0000000..51bf2f6 --- /dev/null +++ b/src/Ellie/Services/Impl/RedisBotCache.cs @@ -0,0 +1,119 @@ +using OneOf; +using OneOf.Types; +using StackExchange.Redis; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Ellie.Common; + +public sealed class RedisBotCache : IBotCache +{ + private static readonly Type[] _supportedTypes = new [] + { + typeof(bool), typeof(int), typeof(uint), typeof(long), + typeof(ulong), typeof(float), typeof(double), + typeof(string), typeof(byte[]), typeof(ReadOnlyMemory), typeof(Memory), + typeof(RedisValue), + }; + + private static readonly JsonSerializerOptions _opts = new() + { + PropertyNameCaseInsensitive = true, + NumberHandling = JsonNumberHandling.AllowReadingFromString, + AllowTrailingCommas = true, + IgnoreReadOnlyProperties = false, + }; + private readonly ConnectionMultiplexer _conn; + + public RedisBotCache(ConnectionMultiplexer conn) + { + _conn = conn; + } + + public async ValueTask AddAsync(TypedKey key, T value, TimeSpan? expiry = null, bool overwrite = true) + { + // if a null value is passed, remove the key + if (value is null) + { + await RemoveAsync(key); + return false; + } + + var db = _conn.GetDatabase(); + RedisValue val = IsSupportedType(typeof(T)) + ? RedisValue.Unbox(value) + : JsonSerializer.Serialize(value, _opts); + + var success = await db.StringSetAsync(key.Key, + val, + expiry: expiry, + when: overwrite ? When.Always : When.NotExists); + + return success; + } + + public bool IsSupportedType(Type type) + { + if (type.IsGenericType) + { + var typeDef = type.GetGenericTypeDefinition(); + if (typeDef == typeof(Nullable<>)) + return IsSupportedType(type.GenericTypeArguments[0]); + } + + foreach (var t in _supportedTypes) + { + if (type == t) + return true; + } + + return false; + } + + public async ValueTask> GetAsync(TypedKey key) + { + var db = _conn.GetDatabase(); + var val = await db.StringGetAsync(key.Key); + if (val == default) + return new None(); + + if (IsSupportedType(typeof(T))) + return (T)((IConvertible)val).ToType(typeof(T), null); + + return JsonSerializer.Deserialize(val.ToString(), _opts)!; + } + + public async ValueTask RemoveAsync(TypedKey key) + { + var db = _conn.GetDatabase(); + + return await db.KeyDeleteAsync(key.Key); + } + + public async ValueTask GetOrAddAsync(TypedKey key, Func> createFactory, TimeSpan? expiry = null) + { + var result = await GetAsync(key); + + return await result.Match>( + v => Task.FromResult(v), + async _ => + { + var factoryValue = await createFactory(); + + if (factoryValue is null) + return default; + + await AddAsync(key, factoryValue, expiry); + + // get again to make sure it's the cached value + // and not the late factory value, in case there's a race condition + + var newResult = await GetAsync(key); + + // it's fine to do this, it should blow up if something went wrong. + return newResult.Match( + v => v, + _ => default); + }); + } +} \ No newline at end of file diff --git a/src/Ellie/Services/Impl/RedisBotStringsProvider.cs b/src/Ellie/Services/Impl/RedisBotStringsProvider.cs new file mode 100644 index 0000000..879b689 --- /dev/null +++ b/src/Ellie/Services/Impl/RedisBotStringsProvider.cs @@ -0,0 +1,78 @@ +#nullable disable +using StackExchange.Redis; +using System.Web; + +namespace Ellie.Services; + +/// +/// Uses to load strings into redis hash (only on Shard 0) +/// and retrieves them from redis via +/// +public class RedisBotStringsProvider : IBotStringsProvider +{ + private readonly ConnectionMultiplexer _redis; + private readonly IStringsSource _source; + private readonly IBotCredentials _creds; + + public RedisBotStringsProvider( + ConnectionMultiplexer redis, + DiscordSocketClient discordClient, + IStringsSource source, + IBotCredentials creds) + { + _redis = redis; + _source = source; + _creds = creds; + + if (discordClient.ShardId == 0) + Reload(); + } + + public string GetText(string localeName, string key) + { + var value = _redis.GetDatabase().HashGet($"{_creds.RedisKey()}:responses:{localeName}", key); + return value; + } + + public CommandStrings GetCommandStrings(string localeName, string commandName) + { + string argsStr = _redis.GetDatabase() + .HashGet($"{_creds.RedisKey()}:commands:{localeName}", $"{commandName}::args"); + if (argsStr == default) + return null; + + var descStr = _redis.GetDatabase() + .HashGet($"{_creds.RedisKey()}:commands:{localeName}", $"{commandName}::desc"); + if (descStr == default) + return null; + + var args = argsStr.Split('&').Map(HttpUtility.UrlDecode); + return new() + { + Args = args, + Desc = descStr + }; + } + + public void Reload() + { + var redisDb = _redis.GetDatabase(); + foreach (var (localeName, localeStrings) in _source.GetResponseStrings()) + { + var hashFields = localeStrings.Select(x => new HashEntry(x.Key, x.Value)).ToArray(); + + redisDb.HashSet($"{_creds.RedisKey()}:responses:{localeName}", hashFields); + } + + foreach (var (localeName, localeStrings) in _source.GetCommandStrings()) + { + var hashFields = localeStrings + .Select(x => new HashEntry($"{x.Key}::args", + string.Join('&', x.Value.Args.Map(HttpUtility.UrlEncode)))) + .Concat(localeStrings.Select(x => new HashEntry($"{x.Key}::desc", x.Value.Desc))) + .ToArray(); + + redisDb.HashSet($"{_creds.RedisKey()}:commands:{localeName}", hashFields); + } + } +} diff --git a/src/Ellie/Services/Impl/RemoteGrpcCoordinator.cs b/src/Ellie/Services/Impl/RemoteGrpcCoordinator.cs new file mode 100644 index 0000000..e5dd501 --- /dev/null +++ b/src/Ellie/Services/Impl/RemoteGrpcCoordinator.cs @@ -0,0 +1,132 @@ +#nullable disable +using Grpc.Core; +using Grpc.Net.Client; +using Ellie.Common.ModuleBehaviors; +using Ellie.Coordinator; + +namespace Ellie.Services; + +public class RemoteGrpcCoordinator : ICoordinator, IReadyExecutor +{ + private readonly Coordinator.Coordinator.CoordinatorClient _coordClient; + private readonly DiscordSocketClient _client; + + public RemoteGrpcCoordinator(IBotCredentials creds, DiscordSocketClient client) + { + var coordUrl = string.IsNullOrWhiteSpace(creds.CoordinatorUrl) ? "http://localhost:3442" : creds.CoordinatorUrl; + + var channel = GrpcChannel.ForAddress(coordUrl); + _coordClient = new(channel); + _client = client; + } + + public bool RestartBot() + { + _coordClient.RestartAllShards(new()); + + return true; + } + + public void Die(bool graceful) + => _coordClient.Die(new() + { + Graceful = graceful + }); + + public bool RestartShard(int shardId) + { + _coordClient.RestartShard(new() + { + ShardId = shardId + }); + + return true; + } + + public IList GetAllShardStatuses() + { + var res = _coordClient.GetAllStatuses(new()); + + return res.Statuses.ToArray() + .Map(s => new ShardStatus + { + ConnectionState = FromCoordConnState(s.State), + GuildCount = s.GuildCount, + ShardId = s.ShardId, + LastUpdate = s.LastUpdate.ToDateTime() + }); + } + + public int GetGuildCount() + { + var res = _coordClient.GetAllStatuses(new()); + + return res.Statuses.Sum(x => x.GuildCount); + } + + public async Task Reload() + => await _coordClient.ReloadAsync(new()); + + public Task OnReadyAsync() + { + Task.Run(async () => + { + var gracefulImminent = false; + while (true) + { + try + { + var reply = await _coordClient.HeartbeatAsync(new() + { + State = ToCoordConnState(_client.ConnectionState), + GuildCount = + _client.ConnectionState == ConnectionState.Connected ? _client.Guilds.Count : 0, + ShardId = _client.ShardId + }, + deadline: DateTime.UtcNow + TimeSpan.FromSeconds(10)); + gracefulImminent = reply.GracefulImminent; + } + catch (RpcException ex) + { + if (!gracefulImminent) + { + Log.Warning(ex, + "Hearbeat failed and graceful shutdown was not expected: {Message}", + ex.Message); + break; + } + + Log.Information("Coordinator is restarting gracefully. Waiting..."); + await Task.Delay(30_000); + } + catch (Exception ex) + { + Log.Error(ex, "Unexpected heartbeat exception: {Message}", ex.Message); + break; + } + + await Task.Delay(7500); + } + + Environment.Exit(5); + }); + + return Task.CompletedTask; + } + + private ConnState ToCoordConnState(ConnectionState state) + => state switch + { + ConnectionState.Connecting => ConnState.Connecting, + ConnectionState.Connected => ConnState.Connected, + _ => ConnState.Disconnected + }; + + private ConnectionState FromCoordConnState(ConnState state) + => state switch + { + ConnState.Connecting => ConnectionState.Connecting, + ConnState.Connected => ConnectionState.Connected, + _ => ConnectionState.Disconnected + }; +} \ No newline at end of file diff --git a/src/Ellie/_common/NinjectIKernalExtensions.cs b/src/Ellie/_common/NinjectIKernalExtensions.cs new file mode 100644 index 0000000..55ba742 --- /dev/null +++ b/src/Ellie/_common/NinjectIKernalExtensions.cs @@ -0,0 +1,43 @@ +using Ninject; + +namespace Ellie.Extensions; + +public static class NinjectIKernalExtensions +{ + public static IKernel AddSingleton(this IKernel kernel) + { + kernel.Bind().ToSelf().InSingletonScope(); + return kernel; + } + + public static IKernel AddSingleton(this IKernel kernel) + where TImpl : TInterface + { + kernel.Bind().To().InSingletonScope(); + return kernel; + } + + public static IKernel AddSingleton(this IKernel kernel, TImpl obj) + => kernel.AddSingleton(obj); + + public static IKernel AddSingleton(this IKernel kernel, TImpl obj) + where TImpl : TInterface + { + kernel.Bind().ToConstant(obj).InSingletonScope(); + return kernel; + } + + public static IKernel AddSingleton( + this IKernel kernel, + Func factory) + where TImpl : TInterface + { + kernel.Bind().ToMethod(factory).InSingletonScope(); + return kernel; + } + + public static IKernel AddSingleton( + this IKernel kernel, + Func factory) + => kernel.AddSingleton(factory); +} \ No newline at end of file