added youtube live support, but only if you have invidious instance (with a working api) set up in searches.yml

This commit is contained in:
Toastie 2025-03-16 15:26:44 +13:00
parent c8ea928de7
commit 7b4d4b1f8c
Signed by: toastie_t0ast
GPG key ID: 0861BE54AD481DC7
16 changed files with 345 additions and 90 deletions

View file

@ -1,4 +1,3 @@
#nullable disable
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Metadata.Builders;
@ -18,9 +17,10 @@ public class FollowedStream
public int Id { get; set; }
public ulong GuildId { get; set; }
public ulong ChannelId { get; set; }
public string Username { get; set; }
public string Username { get; set; } = string.Empty;
public string? PrettyName { get; set; } = null;
public FType Type { get; set; }
public string Message { get; set; }
public string? Message { get; set; } = null;
protected bool Equals(FollowedStream other)
=> ChannelId == other.ChannelId

View file

@ -0,0 +1,9 @@
START TRANSACTION;
ALTER TABLE discorduser DROP COLUMN notifyonlevelup;
ALTER TABLE followedstream ADD prettyname text;
INSERT INTO "__EFMigrationsHistory" (migrationid, productversion)
VALUES ('20250315193825_fs-prettyname', '9.0.1');
COMMIT;

View file

@ -12,7 +12,7 @@ using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
namespace EllieBot.Migrations.PostgreSql
{
[DbContext(typeof(PostgreSqlContext))]
[Migration("20250310143051_init")]
[Migration("20250315193847_init")]
partial class init
{
/// <inheritdoc />
@ -837,12 +837,6 @@ namespace EllieBot.Migrations.PostgreSql
.HasDefaultValue(false)
.HasColumnName("isclubadmin");
b.Property<int>("NotifyOnLevelUp")
.ValueGeneratedOnAdd()
.HasColumnType("integer")
.HasDefaultValue(0)
.HasColumnName("notifyonlevelup");
b.Property<long>("TotalXp")
.ValueGeneratedOnAdd()
.HasColumnType("bigint")
@ -1088,6 +1082,10 @@ namespace EllieBot.Migrations.PostgreSql
.HasColumnType("text")
.HasColumnName("message");
b.Property<string>("PrettyName")
.HasColumnType("text")
.HasColumnName("prettyname");
b.Property<int>("Type")
.HasColumnType("integer")
.HasColumnName("type");

View file

@ -367,6 +367,7 @@ namespace EllieBot.Migrations.PostgreSql
guildid = table.Column<decimal>(type: "numeric(20,0)", nullable: false),
channelid = table.Column<decimal>(type: "numeric(20,0)", nullable: false),
username = table.Column<string>(type: "text", nullable: true),
prettyname = table.Column<string>(type: "text", nullable: true),
type = table.Column<int>(type: "integer", nullable: false),
message = table.Column<string>(type: "text", nullable: true)
},
@ -1623,7 +1624,6 @@ namespace EllieBot.Migrations.PostgreSql
clubid = table.Column<int>(type: "integer", nullable: true),
isclubadmin = table.Column<bool>(type: "boolean", nullable: false, defaultValue: false),
totalxp = table.Column<long>(type: "bigint", nullable: false, defaultValue: 0L),
notifyonlevelup = table.Column<int>(type: "integer", nullable: false, defaultValue: 0),
currencyamount = table.Column<long>(type: "bigint", nullable: false, defaultValue: 0L),
dateadded = table.Column<DateTime>(type: "timestamp without time zone", nullable: true)
},

View file

@ -834,12 +834,6 @@ namespace EllieBot.Migrations.PostgreSql
.HasDefaultValue(false)
.HasColumnName("isclubadmin");
b.Property<int>("NotifyOnLevelUp")
.ValueGeneratedOnAdd()
.HasColumnType("integer")
.HasDefaultValue(0)
.HasColumnName("notifyonlevelup");
b.Property<long>("TotalXp")
.ValueGeneratedOnAdd()
.HasColumnType("bigint")
@ -1085,6 +1079,10 @@ namespace EllieBot.Migrations.PostgreSql
.HasColumnType("text")
.HasColumnName("message");
b.Property<string>("PrettyName")
.HasColumnType("text")
.HasColumnName("prettyname");
b.Property<int>("Type")
.HasColumnType("integer")
.HasColumnName("type");

View file

@ -0,0 +1,9 @@
BEGIN TRANSACTION;
ALTER TABLE discorduser DROP COLUMN notifyonlevelup;
ALTER TABLE "FollowedStream" ADD "PrettyName" TEXT NULL;
INSERT INTO "__EFMigrationsHistory" ("MigrationId", "ProductVersion")
VALUES ('20250315193822_fs-prettyname', '9.0.1');
COMMIT;

View file

@ -11,7 +11,7 @@ using EllieBot.Db;
namespace EllieBot.Migrations.Sqlite
{
[DbContext(typeof(SqliteContext))]
[Migration("20250310143048_init")]
[Migration("20250315193844_init")]
partial class init
{
/// <inheritdoc />
@ -627,11 +627,6 @@ namespace EllieBot.Migrations.Sqlite
.HasColumnType("INTEGER")
.HasDefaultValue(false);
b.Property<int>("NotifyOnLevelUp")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(0);
b.Property<long>("TotalXp")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
@ -812,6 +807,9 @@ namespace EllieBot.Migrations.Sqlite
b.Property<string>("Message")
.HasColumnType("TEXT");
b.Property<string>("PrettyName")
.HasColumnType("TEXT");
b.Property<int>("Type")
.HasColumnType("INTEGER");

View file

@ -366,6 +366,7 @@ namespace EllieBot.Migrations.Sqlite
GuildId = table.Column<ulong>(type: "INTEGER", nullable: false),
ChannelId = table.Column<ulong>(type: "INTEGER", nullable: false),
Username = table.Column<string>(type: "TEXT", nullable: true),
PrettyName = table.Column<string>(type: "TEXT", nullable: true),
Type = table.Column<int>(type: "INTEGER", nullable: false),
Message = table.Column<string>(type: "TEXT", nullable: true)
},
@ -1625,7 +1626,6 @@ namespace EllieBot.Migrations.Sqlite
ClubId = table.Column<int>(type: "INTEGER", nullable: true),
IsClubAdmin = table.Column<bool>(type: "INTEGER", nullable: false, defaultValue: false),
TotalXp = table.Column<long>(type: "INTEGER", nullable: false, defaultValue: 0L),
NotifyOnLevelUp = table.Column<int>(type: "INTEGER", nullable: false, defaultValue: 0),
CurrencyAmount = table.Column<long>(type: "INTEGER", nullable: false, defaultValue: 0L),
DateAdded = table.Column<DateTime>(type: "TEXT", nullable: true)
},

View file

@ -624,11 +624,6 @@ namespace EllieBot.Migrations.Sqlite
.HasColumnType("INTEGER")
.HasDefaultValue(false);
b.Property<int>("NotifyOnLevelUp")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(0);
b.Property<long>("TotalXp")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
@ -809,6 +804,9 @@ namespace EllieBot.Migrations.Sqlite
b.Property<string>("Message")
.HasColumnType("TEXT");
b.Property<string>("PrettyName")
.HasColumnType("TEXT");
b.Property<int>("Type")
.HasColumnType("INTEGER");

View file

@ -27,9 +27,9 @@ public partial class Searches
var embed = _service.GetEmbed(ctx.Guild.Id, data);
await Response()
.Embed(embed)
.Text(strs.stream_tracked)
.SendAsync();
.Embed(embed)
.Text(strs.stream_tracked)
.SendAsync();
}
[Cmd]
@ -70,27 +70,27 @@ public partial class Searches
var allStreams = await _service.GetAllStreamsAsync((SocketGuild)ctx.Guild);
await Response()
.Paginated()
.Items(allStreams)
.PageSize(12)
.CurrentPage(page)
.Page((elements, cur) =>
{
if (elements.Count == 0)
return CreateEmbed().WithDescription(GetText(strs.streams_none)).WithErrorColor();
.Paginated()
.Items(allStreams)
.PageSize(12)
.CurrentPage(page)
.Page((elements, cur) =>
{
if (elements.Count == 0)
return CreateEmbed().WithDescription(GetText(strs.streams_none)).WithErrorColor();
var eb = CreateEmbed().WithTitle(GetText(strs.streams_follow_title)).WithOkColor();
for (var index = 0; index < elements.Count; index++)
{
var elem = elements[index];
eb.AddField($"**#{index + 1 + (12 * cur)}** {elem.Username.ToLower()}",
$"【{elem.Type}】\n<#{elem.ChannelId}>\n{elem.Message?.TrimTo(50)}",
true);
}
var eb = CreateEmbed().WithTitle(GetText(strs.streams_follow_title)).WithOkColor();
for (var index = 0; index < elements.Count; index++)
{
var elem = elements[index];
eb.AddField($"**#{index + 1 + (12 * cur)}** {(elem.PrettyName ?? elem.Username).ToLower()}",
$"【{elem.Type}】\n<#{elem.ChannelId}>\n{elem.Message?.TrimTo(50)}",
true);
}
return eb;
})
.SendAsync();
return eb;
})
.SendAsync();
}
[Cmd]
@ -176,10 +176,10 @@ public partial class Searches
if (data.IsLive)
{
var embed = _service.GetEmbed(ctx.Guild.Id, data, false);
await Response()
.Confirm(strs.streamer_online(Format.Bold(data.Name),
Format.Bold(data.Viewers.ToString())))
.SendAsync();
.Embed(embed)
.SendAsync();
}
else
await Response().Confirm(strs.streamer_offline(data.Name)).SendAsync();

View file

@ -63,7 +63,7 @@ public sealed class StreamNotificationService : IEService, IReadyExecutor
_repSvc = repSvc;
_shardData = shardData;
_streamTracker = new(httpFactory, creds);
_streamTracker = new(httpFactory, creds, config);
StreamsOnlineKey = new("streams.online");
StreamsOfflineKey = new("streams.offline");
@ -123,11 +123,11 @@ public sealed class StreamNotificationService : IEService, IReadyExecutor
_shardTrackedStreams = followedStreams.GroupBy(x => new
{
x.Type,
Name = x.Username.ToLower()
Name = x.Username
})
.ToList()
.ToDictionary(
x => new StreamDataKey(x.Key.Type, x.Key.Name.ToLower()),
x => new StreamDataKey(x.Key.Type, x.Key.Name),
x => x.GroupBy(y => y.GuildId)
.ToDictionary(y => y.Key,
y => y.AsEnumerable().ToHashSet()));
@ -143,7 +143,7 @@ public sealed class StreamNotificationService : IEService, IReadyExecutor
_trackCounter = allFollowedStreams.GroupBy(x => new
{
x.Type,
Name = x.Username.ToLower()
Name = x.Username
})
.ToDictionary(x => new StreamDataKey(x.Key.Type, x.Key.Name),
x => x.Select(fs => fs.GuildId).ToHashSet());
@ -478,6 +478,7 @@ public sealed class StreamNotificationService : IEService, IReadyExecutor
{
Type = data.StreamType,
Username = data.UniqueName,
PrettyName = data.Name,
ChannelId = channelId,
GuildId = guildId
};

View file

@ -5,5 +5,5 @@ namespace EllieBot.Modules.Searches.Common;
public static class Extensions
{
public static StreamDataKey CreateKey(this FollowedStream fs)
=> new(fs.Type, fs.Username.ToLower());
=> new(fs.Type, fs.Username);
}

View file

@ -14,13 +14,15 @@ public class NotifChecker
public NotifChecker(
IHttpClientFactory httpClientFactory,
IBotCredsProvider credsProvider)
IBotCredsProvider credsProvider,
SearchesConfigService scs)
{
_streamProviders = new Dictionary<FollowedStream.FType, Provider>()
{
{ FollowedStream.FType.Twitch, new TwitchHelixProvider(httpClientFactory, credsProvider) },
{ FollowedStream.FType.Picarto, new PicartoProvider(httpClientFactory) },
{ FollowedStream.FType.Trovo, new TrovoProvider(httpClientFactory, credsProvider) }
{ FollowedStream.FType.Trovo, new TrovoProvider(httpClientFactory, credsProvider) },
{ FollowedStream.FType.Youtube, new YouTubeProvider(httpClientFactory, scs) }
};
_offlineBuffer = new();
}
@ -29,11 +31,11 @@ public class NotifChecker
public IEnumerable<StreamDataKey> GetFailingStreams(TimeSpan duration, bool remove = false)
{
var toReturn = _streamProviders
.SelectMany(prov => prov.Value
.FailingStreams
.Where(fs => DateTime.UtcNow - fs.Value > duration)
.Select(fs => new StreamDataKey(prov.Value.Platform, fs.Key)))
.ToList();
.SelectMany(prov => prov.Value
.FailingStreams
.Where(fs => DateTime.UtcNow - fs.Value > duration)
.Select(fs => new StreamDataKey(prov.Value.Platform, fs.Key)))
.ToList();
if (remove)
{
@ -54,29 +56,29 @@ public class NotifChecker
var allStreamData = GetAllData();
var oldStreamDataDict = allStreamData
// group by type
.GroupBy(entry => entry.Key.Type)
.ToDictionary(entry => entry.Key,
entry => entry.AsEnumerable()
.ToDictionary(x => x.Key.Name, x => x.Value));
// group by type
.GroupBy(entry => entry.Key.Type)
.ToDictionary(entry => entry.Key,
entry => entry.AsEnumerable()
.ToDictionary(x => x.Key.Name, x => x.Value));
var newStreamData = await oldStreamDataDict
.Select(x =>
{
// get all stream data for the streams of this type
if (_streamProviders.TryGetValue(x.Key,
out var provider))
{
return provider.GetStreamDataAsync(x.Value
.Select(entry => entry.Key)
.ToList());
}
.Select(x =>
{
// get all stream data for the streams of this type
if (_streamProviders.TryGetValue(x.Key,
out var provider))
{
return provider.GetStreamDataAsync(x.Value
.Select(entry => entry.Key)
.ToList());
}
// this means there's no provider for this stream data, (and there was before?)
return Task.FromResult<IReadOnlyCollection<StreamData>>(
new List<StreamData>());
})
.WhenAll();
// this means there's no provider for this stream data, (and there was before?)
return Task.FromResult<IReadOnlyCollection<StreamData>>(
new List<StreamData>());
})
.WhenAll();
var newlyOnline = new List<StreamData>();
var newlyOffline = new List<StreamData>();
@ -94,11 +96,11 @@ public class NotifChecker
AddLastData(key, newData, true);
continue;
}
// fill with last known game in case it's empty
if (string.IsNullOrWhiteSpace(newData.Game))
newData.Game = oldData.Game;
AddLastData(key, newData, true);
// if the stream is offline, we need to check if it was
@ -144,6 +146,7 @@ public class NotifChecker
catch (Exception ex)
{
Log.Error(ex, "Error getting stream notifications: {ErrorMessage}", ex.Message);
await Task.Delay(15_000);
}
}
});
@ -155,7 +158,7 @@ public class NotifChecker
_cache[key] = data;
return true;
}
return _cache.TryAdd(key, data);
}

View file

@ -37,6 +37,7 @@ public abstract class Provider
/// </summary>
/// <param name="url">Url of the stream</param>
/// <returns><see cref="StreamData" /> of the specified stream. Null if none found</returns>
public abstract Task<StreamData?> GetStreamDataByUrlAsync(string url);
/// <summary>

View file

@ -0,0 +1,230 @@
using System.Net;
using EllieBot.Db.Models;
using EllieBot.Services;
using System.Text.RegularExpressions;
using System.Net.Http.Json;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Xml.Linq;
using AngleSharp.Browser;
namespace EllieBot.Modules.Searches.Common.StreamNotifications.Providers;
/// <summary>
/// Provider for tracking YouTube livestreams
/// </summary>
public sealed partial class YouTubeProvider : Provider
{
private readonly IHttpClientFactory _httpFactory;
private readonly SearchesConfigService _scs;
private readonly EllieRandom _rng = new();
/// <summary>
/// Regex to match YouTube handles
/// </summary>
/// <returns>Regex</returns>
[GeneratedRegex(@"youtu(?:\.be|be\.com)\/@(?<handle>[^\/\?#]+)")]
private static partial Regex HandleRegex();
// channel id regex
[GeneratedRegex(@"youtu(?:\.be|be\.com)\/channel\/(?<channelid>[^\/\?#]+)")]
private static partial Regex ChannelIdRegex();
/// <summary>
/// Type of the platform.
/// </summary>
public override FollowedStream.FType Platform
=> FollowedStream.FType.Youtube;
/// <summary>
/// Initializes a new instance of the <see cref="YouTubeProvider"/> class.
/// </summary>
/// <param name="httpFactory">The HTTP client factory to create HTTP clients.</param>
public YouTubeProvider(
IHttpClientFactory httpFactory,
SearchesConfigService scs
)
{
_httpFactory = httpFactory;
_scs = scs;
}
/// <summary>
/// Checks whether the specified url is a valid YouTube url.
/// </summary>
/// <param name="url">Url to check</param>
/// <returns>True if valid, otherwise false</returns>
public override Task<bool> IsValidUrl(string url)
{
var success = HandleRegex().IsMatch(url)
|| ChannelIdRegex().IsMatch(url);
return Task.FromResult(success);
}
/// <summary>
/// Gets stream data of the stream on the specified YouTube url
/// </summary>
/// <param name="url">Url of the stream</param>
/// <returns><see cref="StreamData"/> of the specified stream. Null if none found</returns>
public override async Task<StreamData?> GetStreamDataByUrlAsync(string url)
{
var match = ChannelIdRegex().Match(url);
var channelId = string.Empty;
if (!match.Success)
{
var handleMatch = HandleRegex().Match(url);
if (!handleMatch.Success)
return null;
var handle = handleMatch.Groups["handle"].Value;
var instances = _scs.Data.InvidiousInstances;
if (instances is not { Count: > 0 })
return null;
var invInstance = instances[_rng.Next(0, _scs.Data.InvidiousInstances.Count)];
using var client = _httpFactory.CreateClient();
client.BaseAddress = new Uri(invInstance);
using var response = await client.GetAsync($"/@{handle}");
if (!response.IsSuccessStatusCode)
return null;
channelId = response.RequestMessage?.RequestUri?.ToString().Split("/").LastOrDefault();
if (channelId is null)
return null;
}
else
{
channelId = match.Groups["channelid"].Value;
}
return await GetStreamDataAsync(channelId);
}
/// <summary>
/// Gets stream data of the specified YouTube channel
/// </summary>
/// <param name="channelId">Channel ID or name</param>
/// <returns><see cref="StreamData"/> of the channel. Null if none found</returns>
public override async Task<StreamData?> GetStreamDataAsync(string channelId)
{
var instances = _scs.Data.InvidiousInstances;
if (instances is not { Count: > 0 })
return null;
var invInstance = instances[_rng.Next(0, instances.Count)];
var client = _httpFactory.CreateClient();
client.BaseAddress = new Uri(invInstance);
var channel = await client.GetFromJsonAsync<InvidiousChannelResponse>($"/api/v1/channels/{channelId}");
if (channel is null)
return null;
var response =
await client.GetFromJsonAsync<InvChannelStreamsResponse>($"/api/v1/channels/{channelId}/streams");
if (response is null)
return null;
var vid = response.Videos.FirstOrDefault(x => !x.IsUpcoming && x.LengthSeconds == 0);
var isLive = false;
if (vid is null)
{
vid = response.Videos.FirstOrDefault(x => !x.IsUpcoming);
}
else
{
isLive = true;
}
var avatarUrl = channel?.AuthorThumbnails?.Select(x => x.Url).LastOrDefault();
return new StreamData()
{
Game = "Livestream",
Name = vid.Author,
Preview = vid.Thumbnails
.Skip(1)
.Select(x => "https://i.ytimg.com/" + x.Url)
.FirstOrDefault(),
Title = vid.Title,
Viewers = vid.ViewCount,
AvatarUrl = avatarUrl,
IsLive = isLive,
StreamType = FollowedStream.FType.Youtube,
StreamUrl = "https://youtube.com/watch?v=" + vid.VideoId,
UniqueName = vid.AuthorId,
};
}
/// <summary>
/// Gets stream data of all specified YouTube channels
/// </summary>
/// <param name="channelIds">List of channel IDs or names</param>
/// <returns><see cref="StreamData"/> of all users, in the same order. Null for every ID not found.</returns>
public override async Task<IReadOnlyCollection<StreamData>> GetStreamDataAsync(List<string> channelIds)
{
var results = new List<StreamData>(channelIds.Count);
foreach (var group in channelIds.Chunk(5))
{
var streamData = await Task.WhenAll(group.Select(GetStreamDataAsync));
foreach (var data in streamData)
results.Add(data);
}
return results;
}
}
public sealed class InvidiousChannelResponse
{
[JsonPropertyName("authorId")]
public required string AuthorId { get; init; }
[JsonPropertyName("authorThumbnails")]
public required List<InvAuthorThumbnail> AuthorThumbnails { get; init; }
public sealed class InvAuthorThumbnail
{
[JsonPropertyName("url")]
public required string Url { get; init; }
}
}
public sealed class InvChannelStreamsResponse
{
public required List<InvidiousStreamResponse> Videos { get; init; }
}
public sealed class InvidiousStreamResponse
{
[JsonPropertyName("title")]
public required string Title { get; init; }
[JsonPropertyName("videoId")]
public required string VideoId { get; init; }
[JsonPropertyName("lengthSeconds")]
public required int LengthSeconds { get; init; }
[JsonPropertyName("videoThumbnails")]
public required List<InvidiousThumbnail> Thumbnails { get; init; }
[JsonPropertyName("author")]
public required string Author { get; init; }
[JsonPropertyName("authorId")]
public required string AuthorId { get; init; }
[JsonPropertyName("isUpcoming")]
public bool IsUpcoming { get; set; }
[JsonPropertyName("viewCount")]
public int ViewCount { get; set; }
}

View file

@ -40,9 +40,19 @@ public partial class Xp : EllieModule<XpService>
_templateService = templateService;
}
// [Cmd]
// [RequireContext(ContextType.Guild)]
// public async Task ExperienceText([Leftover] IUser? user = null)
// {
// user ??= ctx.User;
// var xp = await _service.GetUserStatsAsync((IGuildUser)user);
// await ctx.Channel.TriggerTypingAsync();
// await ctx.Channel.SendMessageAsync(_templateService.GetXpText(xp));
// }
[Cmd]
[RequireContext(ContextType.Guild)]
public async Task Experience([Leftover] IUser user = null)
public async Task Experience([Leftover] IUser? user = null)
{
user ??= ctx.User;
await ctx.Channel.TriggerTypingAsync();