311 lines
No EOL
10 KiB
C#
311 lines
No EOL
10 KiB
C#
#nullable disable
|
|
using CodeHollow.FeedReader;
|
|
using CodeHollow.FeedReader.Feeds;
|
|
using LinqToDB;
|
|
using LinqToDB.EntityFrameworkCore;
|
|
using Microsoft.EntityFrameworkCore;
|
|
using EllieBot.Common.ModuleBehaviors;
|
|
using EllieBot.Db.Models;
|
|
using System.Collections.Concurrent;
|
|
|
|
namespace EllieBot.Modules.Searches.Services;
|
|
|
|
public class FeedsService : IEService, IReadyExecutor
|
|
{
|
|
private readonly DbService _db;
|
|
private NonBlocking.ConcurrentDictionary<string, List<FeedSub>> _subs;
|
|
private readonly DiscordSocketClient _client;
|
|
private readonly IMessageSenderService _sender;
|
|
private readonly ShardData _shardData;
|
|
|
|
private readonly NonBlocking.ConcurrentDictionary<string, DateTime> _lastPosts = new();
|
|
private readonly Dictionary<string, uint> _errorCounters = new();
|
|
|
|
public FeedsService(
|
|
DbService db,
|
|
DiscordSocketClient client,
|
|
IMessageSenderService sender,
|
|
ShardData shardData)
|
|
{
|
|
_db = db;
|
|
|
|
|
|
_client = client;
|
|
_sender = sender;
|
|
_shardData = shardData;
|
|
}
|
|
|
|
public async Task OnReadyAsync()
|
|
{
|
|
await using (var uow = _db.GetDbContext())
|
|
{
|
|
var subs = await uow.Set<FeedSub>()
|
|
.AsQueryable()
|
|
.Where(x => Queries.GuildOnShard(x.GuildId, _shardData.TotalShards, _shardData.ShardId))
|
|
.ToListAsyncLinqToDB();
|
|
_subs = subs
|
|
.GroupBy(x => x.Url.ToLower())
|
|
.ToDictionary(x => x.Key, x => x.ToList())
|
|
.ToConcurrent();
|
|
}
|
|
|
|
await TrackFeeds();
|
|
}
|
|
|
|
private void ClearErrors(string url)
|
|
=> _errorCounters.Remove(url);
|
|
|
|
private async Task<uint> AddError(string url, List<int> ids)
|
|
{
|
|
try
|
|
{
|
|
var newValue = _errorCounters[url] = _errorCounters.GetValueOrDefault(url) + 1;
|
|
|
|
if (newValue >= 100)
|
|
{
|
|
// remove from db
|
|
await using var ctx = _db.GetDbContext();
|
|
await ctx.GetTable<FeedSub>()
|
|
.DeleteAsync(x => ids.Contains(x.Id));
|
|
|
|
// remove from the local cache
|
|
_subs.TryRemove(url, out _);
|
|
|
|
// reset the error counter
|
|
ClearErrors(url);
|
|
}
|
|
|
|
return newValue;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.Error(ex, "Error adding rss errors...");
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
private DateTime? GetPubDate(FeedItem item)
|
|
{
|
|
if (item.PublishingDate is not null)
|
|
return item.PublishingDate;
|
|
if (item.SpecificItem is AtomFeedItem atomItem)
|
|
return atomItem.UpdatedDate;
|
|
return null;
|
|
}
|
|
|
|
public async Task<EmbedBuilder> TrackFeeds()
|
|
{
|
|
while (true)
|
|
{
|
|
var allSendTasks = new List<Task>(_subs.Count);
|
|
foreach (var kvp in _subs)
|
|
{
|
|
if (kvp.Value.Count == 0)
|
|
continue;
|
|
|
|
var rssUrl = kvp.Value.First().Url;
|
|
try
|
|
{
|
|
var feed = await FeedReader.ReadAsync(rssUrl);
|
|
|
|
var items = new List<(FeedItem Item, DateTime LastUpdate)>();
|
|
foreach (var item in feed.Items)
|
|
{
|
|
var pubDate = GetPubDate(item);
|
|
|
|
if (pubDate is null)
|
|
continue;
|
|
|
|
items.Add((item, pubDate.Value.ToUniversalTime()));
|
|
|
|
// show at most 3 items if you're behind
|
|
if (items.Count > 2)
|
|
break;
|
|
}
|
|
|
|
if (items.Count == 0)
|
|
continue;
|
|
|
|
if (!_lastPosts.TryGetValue(kvp.Key, out var lastFeedUpdate))
|
|
{
|
|
lastFeedUpdate = _lastPosts[kvp.Key] = items[0].LastUpdate;
|
|
}
|
|
|
|
for (var index = 1; index <= items.Count; index++)
|
|
{
|
|
var (feedItem, itemUpdateDate) = items[^index];
|
|
if (itemUpdateDate <= lastFeedUpdate)
|
|
continue;
|
|
|
|
var embed = _sender.CreateEmbed().WithFooter(rssUrl);
|
|
|
|
_lastPosts[kvp.Key] = itemUpdateDate;
|
|
|
|
var link = feedItem.SpecificItem.Link;
|
|
if (!string.IsNullOrWhiteSpace(link) && Uri.IsWellFormedUriString(link, UriKind.Absolute))
|
|
embed.WithUrl(link);
|
|
|
|
var title = string.IsNullOrWhiteSpace(feedItem.Title) ? "-" : feedItem.Title;
|
|
|
|
var gotImage = false;
|
|
if (feedItem.SpecificItem is MediaRssFeedItem mrfi
|
|
&& (mrfi.Enclosure?.MediaType?.StartsWith("image/") ?? false))
|
|
{
|
|
var imgUrl = mrfi.Enclosure.Url;
|
|
if (!string.IsNullOrWhiteSpace(imgUrl)
|
|
&& Uri.IsWellFormedUriString(imgUrl, UriKind.Absolute))
|
|
{
|
|
embed.WithImageUrl(imgUrl);
|
|
gotImage = true;
|
|
}
|
|
}
|
|
|
|
if (!gotImage && feedItem.SpecificItem is AtomFeedItem afi)
|
|
{
|
|
var previewElement = afi.Element.Elements()
|
|
.FirstOrDefault(x => x.Name.LocalName == "preview");
|
|
|
|
if (previewElement is null)
|
|
{
|
|
previewElement = afi.Element.Elements()
|
|
.FirstOrDefault(x => x.Name.LocalName == "thumbnail");
|
|
}
|
|
|
|
if (previewElement is not null)
|
|
{
|
|
var urlAttribute = previewElement.Attribute("url");
|
|
if (urlAttribute is not null
|
|
&& !string.IsNullOrWhiteSpace(urlAttribute.Value)
|
|
&& Uri.IsWellFormedUriString(urlAttribute.Value, UriKind.Absolute))
|
|
{
|
|
embed.WithImageUrl(urlAttribute.Value);
|
|
gotImage = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
embed.WithTitle(title.TrimTo(256));
|
|
|
|
var desc = feedItem.Description?.StripHtml();
|
|
if (!string.IsNullOrWhiteSpace(feedItem.Description))
|
|
embed.WithDescription(desc.TrimTo(2048));
|
|
|
|
|
|
var tasks = new List<Task>();
|
|
|
|
foreach (var val in kvp.Value)
|
|
{
|
|
var ch = _client.GetGuild(val.GuildId).GetTextChannel(val.ChannelId);
|
|
|
|
if (ch is null)
|
|
continue;
|
|
|
|
var sendTask = _sender.Response(ch)
|
|
.Embed(embed)
|
|
.Text(string.IsNullOrWhiteSpace(val.Message)
|
|
? string.Empty
|
|
: val.Message)
|
|
.SendAsync();
|
|
tasks.Add(sendTask);
|
|
}
|
|
|
|
allSendTasks.Add(tasks.WhenAll());
|
|
|
|
// as data retrieval was successful, reset error counter
|
|
ClearErrors(rssUrl);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
var errorCount = await AddError(rssUrl, kvp.Value.Select(x => x.Id).ToList());
|
|
|
|
Log.Warning("An error occured while getting rss stream ({ErrorCount} / 100) {RssFeed}"
|
|
+ "\n {Message}",
|
|
errorCount,
|
|
rssUrl,
|
|
$"[{ex.GetType().Name}]: {ex.Message}");
|
|
}
|
|
}
|
|
|
|
await Task.WhenAll(Task.WhenAll(allSendTasks), Task.Delay(30000));
|
|
}
|
|
}
|
|
|
|
public List<FeedSub> GetFeeds(ulong guildId)
|
|
{
|
|
using var uow = _db.GetDbContext();
|
|
|
|
return uow.GetTable<FeedSub>()
|
|
.Where(x => x.GuildId == guildId)
|
|
.OrderBy(x => x.Id)
|
|
.ToList();
|
|
}
|
|
|
|
private const int MAX_FEEDS = 10;
|
|
|
|
public async Task<FeedAddResult> AddFeedAsync(
|
|
ulong guildId,
|
|
ulong channelId,
|
|
string rssFeed,
|
|
string message)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(rssFeed, nameof(rssFeed));
|
|
|
|
await using var uow = _db.GetDbContext();
|
|
var feedUrl = rssFeed.Trim();
|
|
if (await uow.GetTable<FeedSub>().AnyAsyncLinqToDB(x => x.GuildId == guildId &&
|
|
x.Url.ToLower() == feedUrl.ToLower()))
|
|
return FeedAddResult.Duplicate;
|
|
|
|
var count = await uow.GetTable<FeedSub>().CountAsyncLinqToDB(x => x.GuildId == guildId);
|
|
if (count >= MAX_FEEDS)
|
|
return FeedAddResult.LimitReached;
|
|
|
|
var fs = await uow.GetTable<FeedSub>()
|
|
.InsertWithOutputAsync(() => new FeedSub
|
|
{
|
|
GuildId = guildId,
|
|
ChannelId = channelId,
|
|
Url = feedUrl,
|
|
Message = message,
|
|
});
|
|
|
|
_subs.AddOrUpdate(fs.Url.ToLower(),
|
|
[fs],
|
|
(_, old) => old.Append(fs).ToList());
|
|
|
|
return FeedAddResult.Success;
|
|
}
|
|
|
|
public bool RemoveFeed(ulong guildId, int index)
|
|
{
|
|
if (index < 0)
|
|
return false;
|
|
|
|
using var uow = _db.GetDbContext();
|
|
var items = uow.Set<FeedSub>()
|
|
.Where(x => x.GuildId == guildId)
|
|
.OrderBy(x => x.Id)
|
|
.ToList();
|
|
|
|
if (items.Count <= index)
|
|
return false;
|
|
|
|
var toRemove = items[index];
|
|
_subs.AddOrUpdate(toRemove.Url.ToLower(),
|
|
[],
|
|
(_, old) => { return old.Where(x => x.Id != toRemove.Id).ToList(); });
|
|
uow.Remove(toRemove);
|
|
uow.SaveChanges();
|
|
|
|
return true;
|
|
}
|
|
}
|
|
|
|
public enum FeedAddResult
|
|
{
|
|
Success,
|
|
LimitReached,
|
|
Invalid,
|
|
Duplicate,
|
|
} |