diff --git a/README.md b/README.md index ef197a5..8c5dac8 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,10 @@ Scrapper Service Осуществляет мониторинг контента: Периодическая проверка отслеживаемых URL на наличие изменений -Парсинг контента с различных источников (GitHub, Stack Overflow, Reddit и др.) +Парсинг контента с поддерживаемых источников: +- GitHub-репозитории — новые issue и pull request: `https://github.com/owner/repo` +- вопросы Stack Overflow — новые ответы и комментарии: `https://stackoverflow.com/questions/12345/...` +- сабреддиты Reddit — новые посты: `https://reddit.com/r/dotnet` Определение изменений (diff detection) Отправка уведомлений в Bot Service при обнаружении обновлений Хранение информации о подписках и состоянии контента @@ -60,7 +63,8 @@ cp src/LinkTracker.Bot.Api/.env.template src/LinkTracker.Bot.Api/.env ``` cp src/LinkTracker.Scrapper.Api/.env.template src/LinkTracker.Scrapper.Api/.env ``` -4. Поставить действительные параметры в .env. +4. Поставить действительные параметры в .env. Для Reddit нужны `Reddit__ClientId` и + `Reddit__ClientSecret` — создаются на https://www.reddit.com/prefs/apps как приложение типа `script`. 5. Создать копию [.env.template](src/LinkTracker.AiAgent.Api/.env.template) с именем .env в каталоге ии-агента. ``` cp src/LinkTracker.AiAgent.Api/.env.template src/LinkTracker.AiAgent.Api/.env diff --git a/src/LinkTracker.Bot.Application/Clients/Scrapper/ScrapperErrorMessages.cs b/src/LinkTracker.Bot.Application/Clients/Scrapper/ScrapperErrorMessages.cs index 93a1efa..c97098a 100644 --- a/src/LinkTracker.Bot.Application/Clients/Scrapper/ScrapperErrorMessages.cs +++ b/src/LinkTracker.Bot.Application/Clients/Scrapper/ScrapperErrorMessages.cs @@ -11,7 +11,8 @@ internal static class ScrapperErrorMessages "Эта ссылка не поддерживается.\n" + "Сейчас можно отслеживать только:\n" + "• GitHub-репозитории: https://github.com/owner/repo\n" + - "• вопросы StackOverflow: https://stackoverflow.com/questions/12345/...", + "• вопросы StackOverflow: https://stackoverflow.com/questions/12345/...\n" + + "• сабреддиты Reddit: https://reddit.com/r/dotnet", [ScrapperErrorCodes.ChatAlreadyExists] = "Привет! Чат уже зарегистрирован. Используй справку /help для просмотра команд", [ScrapperErrorCodes.LinkAlreadyExists] = diff --git a/src/LinkTracker.Scrapper.Api/.env.template b/src/LinkTracker.Scrapper.Api/.env.template index 1dd77e6..6a8f59d 100644 --- a/src/LinkTracker.Scrapper.Api/.env.template +++ b/src/LinkTracker.Scrapper.Api/.env.template @@ -1,2 +1,4 @@ GitHub__Token=ghp_abc +Reddit__ClientId=your-reddit-app-client-id +Reddit__ClientSecret=your-reddit-app-client-secret ServiceAuth__Secret=change-me-to-a-long-random-string diff --git a/src/LinkTracker.Scrapper.Api/appsettings.json b/src/LinkTracker.Scrapper.Api/appsettings.json index 1aa3283..e43e6bf 100644 --- a/src/LinkTracker.Scrapper.Api/appsettings.json +++ b/src/LinkTracker.Scrapper.Api/appsettings.json @@ -49,6 +49,18 @@ "AcquireTimeoutSeconds": 60 } }, + "Reddit": { + "BaseUrl": "https://oauth.reddit.com", + "TokenUrl": "https://www.reddit.com/api/v1/access_token", + "RateLimit": { + "Enabled": true, + "TokenLimit": 100, + "TokensPerPeriod": 100, + "ReplenishmentPeriodSeconds": 60, + "QueueLimit": 1000, + "AcquireTimeoutSeconds": 60 + } + }, "Database": { "Host": "localhost", "Port": 5434, diff --git a/src/LinkTracker.Scrapper.Application/Clients/Reddit/Contracts/RedditListingData.cs b/src/LinkTracker.Scrapper.Application/Clients/Reddit/Contracts/RedditListingData.cs new file mode 100644 index 0000000..45cc2e3 --- /dev/null +++ b/src/LinkTracker.Scrapper.Application/Clients/Reddit/Contracts/RedditListingData.cs @@ -0,0 +1,8 @@ +using System.Text.Json.Serialization; + +namespace LinkTracker.Scrapper.Application.Clients.Reddit.Contracts; + +public sealed class RedditListingData +{ + [JsonPropertyName("children")] public IReadOnlyList> Children { get; init; } = []; +} diff --git a/src/LinkTracker.Scrapper.Application/Clients/Reddit/Contracts/RedditListingEnvelope.cs b/src/LinkTracker.Scrapper.Application/Clients/Reddit/Contracts/RedditListingEnvelope.cs new file mode 100644 index 0000000..4f259e2 --- /dev/null +++ b/src/LinkTracker.Scrapper.Application/Clients/Reddit/Contracts/RedditListingEnvelope.cs @@ -0,0 +1,8 @@ +using System.Text.Json.Serialization; + +namespace LinkTracker.Scrapper.Application.Clients.Reddit.Contracts; + +public sealed class RedditListingEnvelope +{ + [JsonPropertyName("data")] public RedditListingData Data { get; init; } = new(); +} diff --git a/src/LinkTracker.Scrapper.Application/Clients/Reddit/Contracts/RedditPostResponse.cs b/src/LinkTracker.Scrapper.Application/Clients/Reddit/Contracts/RedditPostResponse.cs new file mode 100644 index 0000000..522bc79 --- /dev/null +++ b/src/LinkTracker.Scrapper.Application/Clients/Reddit/Contracts/RedditPostResponse.cs @@ -0,0 +1,20 @@ +using System.Text.Json.Serialization; + +namespace LinkTracker.Scrapper.Application.Clients.Reddit.Contracts; + +public sealed class RedditPostResponse +{ + [JsonPropertyName("id")] public string Id { get; init; } = string.Empty; + + [JsonPropertyName("title")] public string Title { get; init; } = string.Empty; + + [JsonPropertyName("selftext")] public string Selftext { get; init; } = string.Empty; + + [JsonPropertyName("author")] public string Author { get; init; } = string.Empty; + + [JsonPropertyName("permalink")] public string Permalink { get; init; } = string.Empty; + + [JsonPropertyName("created_utc")] public double CreatedUtcSeconds { get; init; } + + public DateTimeOffset CreatedAt => DateTimeOffset.FromUnixTimeSeconds((long)CreatedUtcSeconds); +} diff --git a/src/LinkTracker.Scrapper.Application/Clients/Reddit/Contracts/RedditThingResponse.cs b/src/LinkTracker.Scrapper.Application/Clients/Reddit/Contracts/RedditThingResponse.cs new file mode 100644 index 0000000..028fda2 --- /dev/null +++ b/src/LinkTracker.Scrapper.Application/Clients/Reddit/Contracts/RedditThingResponse.cs @@ -0,0 +1,10 @@ +using System.Text.Json.Serialization; + +namespace LinkTracker.Scrapper.Application.Clients.Reddit.Contracts; + +public sealed class RedditThingResponse +{ + [JsonPropertyName("kind")] public string Kind { get; init; } = string.Empty; + + [JsonPropertyName("data")] public T Data { get; init; } = default!; +} diff --git a/src/LinkTracker.Scrapper.Application/Clients/Reddit/IRedditClient.cs b/src/LinkTracker.Scrapper.Application/Clients/Reddit/IRedditClient.cs new file mode 100644 index 0000000..f478601 --- /dev/null +++ b/src/LinkTracker.Scrapper.Application/Clients/Reddit/IRedditClient.cs @@ -0,0 +1,8 @@ +using LinkTracker.Scrapper.Application.Clients.Reddit.Contracts; + +namespace LinkTracker.Scrapper.Application.Clients.Reddit; + +public interface IRedditClient +{ + Task> GetNewPostsAsync(string subreddit, CancellationToken ct = default); +} diff --git a/src/LinkTracker.Scrapper.Application/Errors/ScrapperErrors.cs b/src/LinkTracker.Scrapper.Application/Errors/ScrapperErrors.cs index 75b1fd2..e379b5c 100644 --- a/src/LinkTracker.Scrapper.Application/Errors/ScrapperErrors.cs +++ b/src/LinkTracker.Scrapper.Application/Errors/ScrapperErrors.cs @@ -82,6 +82,6 @@ public static ApiException UnsupportedLink(Uri link) return new ApiException( HttpStatusCode.BadRequest, ScrapperErrorCodes.UnsupportedLink, - $"Ссылка '{link}' не поддерживается. Сейчас поддерживаются только GitHub repository и StackOverflow question."); + $"Ссылка '{link}' не поддерживается. Сейчас поддерживаются только GitHub repository, StackOverflow question и Reddit subreddit."); } } \ No newline at end of file diff --git a/src/LinkTracker.Scrapper.Application/Models/Updates/LinkEventKind.cs b/src/LinkTracker.Scrapper.Application/Models/Updates/LinkEventKind.cs index f944646..503aea4 100644 --- a/src/LinkTracker.Scrapper.Application/Models/Updates/LinkEventKind.cs +++ b/src/LinkTracker.Scrapper.Application/Models/Updates/LinkEventKind.cs @@ -6,5 +6,6 @@ public enum LinkEventKind PullRequest = 2, QuestionActivity = 3, Answer = 4, - Comment = 5 -} \ No newline at end of file + Comment = 5, + Post = 6 +} diff --git a/src/LinkTracker.Scrapper.Application/Models/Updates/LinkSourceKind.cs b/src/LinkTracker.Scrapper.Application/Models/Updates/LinkSourceKind.cs index b7d5c05..9a4142a 100644 --- a/src/LinkTracker.Scrapper.Application/Models/Updates/LinkSourceKind.cs +++ b/src/LinkTracker.Scrapper.Application/Models/Updates/LinkSourceKind.cs @@ -3,5 +3,6 @@ namespace LinkTracker.Scrapper.Application.Models.Updates; public enum LinkSourceKind { GitHub = 1, - StackOverflow = 2 -} \ No newline at end of file + StackOverflow = 2, + Reddit = 3 +} diff --git a/src/LinkTracker.Scrapper.Application/Registration/ApplicationModule.cs b/src/LinkTracker.Scrapper.Application/Registration/ApplicationModule.cs index 0f844c9..f88c357 100644 --- a/src/LinkTracker.Scrapper.Application/Registration/ApplicationModule.cs +++ b/src/LinkTracker.Scrapper.Application/Registration/ApplicationModule.cs @@ -14,6 +14,7 @@ public static IServiceCollection AddApplication(this IServiceCollection services services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); return services; } diff --git a/src/LinkTracker.Scrapper.Application/Services/Helpers/LinkUpdateResultBuilder.cs b/src/LinkTracker.Scrapper.Application/Services/Helpers/LinkUpdateResultBuilder.cs index 43c3c66..704d74b 100644 --- a/src/LinkTracker.Scrapper.Application/Services/Helpers/LinkUpdateResultBuilder.cs +++ b/src/LinkTracker.Scrapper.Application/Services/Helpers/LinkUpdateResultBuilder.cs @@ -4,9 +4,11 @@ namespace LinkTracker.Scrapper.Application.Services.Helpers; internal static class LinkUpdateResultBuilder { - public static LinkCheckResult InitialState(DateTimeOffset actualLastUpdatedAt) + public static LinkCheckResult InitialState( + DateTimeOffset actualLastUpdatedAt, + string? actualLastEventKey = null) { - return Build(actualLastUpdatedAt, null, []); + return Build(actualLastUpdatedAt, actualLastEventKey, []); } public static LinkCheckResult NoChanges( diff --git a/src/LinkTracker.Scrapper.Application/Services/Updates/Clients/RedditLinkUpdateHandler.cs b/src/LinkTracker.Scrapper.Application/Services/Updates/Clients/RedditLinkUpdateHandler.cs new file mode 100644 index 0000000..47aa416 --- /dev/null +++ b/src/LinkTracker.Scrapper.Application/Services/Updates/Clients/RedditLinkUpdateHandler.cs @@ -0,0 +1,104 @@ +using LinkTracker.Scrapper.Application.Clients.Reddit; +using LinkTracker.Scrapper.Application.Clients.Reddit.Contracts; +using LinkTracker.Scrapper.Application.Models.Updates; +using LinkTracker.Scrapper.Application.Services.Helpers; +using LinkTracker.Scrapper.Storage.Abstractions.Models; +using Microsoft.Extensions.Logging; + +namespace LinkTracker.Scrapper.Application.Services.Updates.Clients; + +public sealed class RedditLinkUpdateHandler( + IRedditClient redditClient, + ILogger logger) : LinkUpdateHandlerBase(logger) +{ + public override bool CanHandle(Uri url) + { + return TryParseSubreddit(url, out _); + } + + protected override async Task InitializeStateAsync( + TrackedLinkSubscription subscription, + CancellationToken ct) + { + TryParseSubreddit(subscription.Url, out var subreddit); + + var posts = await redditClient.GetNewPostsAsync(subreddit, ct); + + if (posts.Count == 0) + { + return LinkUpdateResultBuilder.NoChanges(); + } + + var newest = posts.MaxBy(x => x.CreatedAt)!; + + return LinkUpdateResultBuilder.InitialState(newest.CreatedAt, BuildEventKey(newest)); + } + + protected override async Task> GetNewEventsAsync( + TrackedLinkSubscription subscription, + DateTimeOffset lastSeenAt, + string? lastEventKey, + CancellationToken ct) + { + TryParseSubreddit(subscription.Url, out var subreddit); + + var posts = await redditClient.GetNewPostsAsync(subreddit, ct); + + return posts + .Select(x => MapPostToEvent(x, subscription.Url)) + .Where(x => IsAfterCursor(x, lastSeenAt, lastEventKey)) + .ToArray(); + } + + private static bool TryParseSubreddit(Uri url, out string subreddit) + { + subreddit = string.Empty; + + if (!UriParsingHelper.IsHost(url, "reddit.com")) + { + return false; + } + + var segments = UriParsingHelper.GetPathSegments(url); + if (segments.Length != 2) + { + return false; + } + + if (!string.Equals(segments[0], "r", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + subreddit = segments[1]; + + return !string.IsNullOrWhiteSpace(subreddit); + } + + private static LinkEvent MapPostToEvent(RedditPostResponse post, Uri subredditUrl) + { + return new LinkEvent + { + SourceKind = LinkSourceKind.Reddit, + EventKind = LinkEventKind.Post, + Title = post.Title, + UserName = post.Author, + CreatedAt = post.CreatedAt, + EventKey = BuildEventKey(post), + Body = post.Selftext, + ResourceUrl = BuildResourceUrl(post, subredditUrl) + }; + } + + private static string BuildEventKey(RedditPostResponse post) + { + return $"post:{post.Id}"; + } + + private static Uri BuildResourceUrl(RedditPostResponse post, Uri subredditUrl) + { + return Uri.TryCreate(subredditUrl, post.Permalink, out var resourceUrl) + ? resourceUrl + : subredditUrl; + } +} diff --git a/src/LinkTracker.Scrapper.Application/Services/Updates/LinkEventDescriptionFormatter.cs b/src/LinkTracker.Scrapper.Application/Services/Updates/LinkEventDescriptionFormatter.cs index 9e3bb6c..b76c99a 100644 --- a/src/LinkTracker.Scrapper.Application/Services/Updates/LinkEventDescriptionFormatter.cs +++ b/src/LinkTracker.Scrapper.Application/Services/Updates/LinkEventDescriptionFormatter.cs @@ -50,6 +50,7 @@ private static string FormatEventKind(LinkEventKind eventKind) LinkEventKind.QuestionActivity => "question-activity", LinkEventKind.Answer => "answer", LinkEventKind.Comment => "comment", + LinkEventKind.Post => "post", _ => eventKind.ToString() }; } @@ -60,6 +61,7 @@ private static string FormatSourceKind(LinkSourceKind sourceKind) { LinkSourceKind.GitHub => "GitHub", LinkSourceKind.StackOverflow => "Stack Overflow", + LinkSourceKind.Reddit => "Reddit", _ => sourceKind.ToString() }; } diff --git a/src/LinkTracker.Scrapper.Infrastructure/Clients/Reddit/Contracts/RedditAccessTokenResponse.cs b/src/LinkTracker.Scrapper.Infrastructure/Clients/Reddit/Contracts/RedditAccessTokenResponse.cs new file mode 100644 index 0000000..c9691ee --- /dev/null +++ b/src/LinkTracker.Scrapper.Infrastructure/Clients/Reddit/Contracts/RedditAccessTokenResponse.cs @@ -0,0 +1,10 @@ +using System.Text.Json.Serialization; + +namespace LinkTracker.Scrapper.Infrastructure.Clients.Reddit.Contracts; + +internal sealed class RedditAccessTokenResponse +{ + [JsonPropertyName("access_token")] public string AccessToken { get; init; } = string.Empty; + + [JsonPropertyName("expires_in")] public int ExpiresInSeconds { get; init; } +} diff --git a/src/LinkTracker.Scrapper.Infrastructure/Clients/Reddit/IRedditAccessTokenProvider.cs b/src/LinkTracker.Scrapper.Infrastructure/Clients/Reddit/IRedditAccessTokenProvider.cs new file mode 100644 index 0000000..479aab7 --- /dev/null +++ b/src/LinkTracker.Scrapper.Infrastructure/Clients/Reddit/IRedditAccessTokenProvider.cs @@ -0,0 +1,6 @@ +namespace LinkTracker.Scrapper.Infrastructure.Clients.Reddit; + +internal interface IRedditAccessTokenProvider +{ + Task GetAccessTokenAsync(CancellationToken ct = default); +} diff --git a/src/LinkTracker.Scrapper.Infrastructure/Clients/Reddit/RedditAccessTokenHandler.cs b/src/LinkTracker.Scrapper.Infrastructure/Clients/Reddit/RedditAccessTokenHandler.cs new file mode 100644 index 0000000..21cce28 --- /dev/null +++ b/src/LinkTracker.Scrapper.Infrastructure/Clients/Reddit/RedditAccessTokenHandler.cs @@ -0,0 +1,17 @@ +using System.Net.Http.Headers; + +namespace LinkTracker.Scrapper.Infrastructure.Clients.Reddit; + +internal sealed class RedditAccessTokenHandler(IRedditAccessTokenProvider tokenProvider) : DelegatingHandler +{ + protected override async Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + var accessToken = await tokenProvider.GetAccessTokenAsync(cancellationToken); + + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", accessToken); + + return await base.SendAsync(request, cancellationToken); + } +} diff --git a/src/LinkTracker.Scrapper.Infrastructure/Clients/Reddit/RedditAccessTokenProvider.cs b/src/LinkTracker.Scrapper.Infrastructure/Clients/Reddit/RedditAccessTokenProvider.cs new file mode 100644 index 0000000..c2ba815 --- /dev/null +++ b/src/LinkTracker.Scrapper.Infrastructure/Clients/Reddit/RedditAccessTokenProvider.cs @@ -0,0 +1,86 @@ +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Text; +using LinkTracker.Scrapper.Infrastructure.Clients.Reddit.Contracts; +using LinkTracker.Scrapper.Infrastructure.Configuration.Clients; +using Microsoft.Extensions.Options; + +namespace LinkTracker.Scrapper.Infrastructure.Clients.Reddit; + +internal sealed class RedditAccessTokenProvider( + IHttpClientFactory httpClientFactory, + IOptions options, + TimeProvider timeProvider) : IRedditAccessTokenProvider +{ + public const string HttpClientName = "reddit-token"; + + private static readonly TimeSpan ExpirationSafetyMargin = TimeSpan.FromMinutes(1); + + private readonly RedditOptions _options = options.Value; + private readonly SemaphoreSlim _semaphore = new(1, 1); + + private string? _accessToken; + private DateTimeOffset _expiresAt; + + public async Task GetAccessTokenAsync(CancellationToken ct = default) + { + if (TryGetCachedToken(out var cached)) + { + return cached; + } + + await _semaphore.WaitAsync(ct); + + try + { + if (TryGetCachedToken(out cached)) + { + return cached; + } + + var response = await RequestAccessTokenAsync(ct); + var accessToken = response.AccessToken; + + _accessToken = accessToken; + _expiresAt = timeProvider.GetUtcNow().AddSeconds(response.ExpiresInSeconds) - ExpirationSafetyMargin; + + return accessToken; + } + finally + { + _semaphore.Release(); + } + } + + private bool TryGetCachedToken(out string accessToken) + { + accessToken = _accessToken ?? string.Empty; + + return accessToken.Length > 0 && timeProvider.GetUtcNow() < _expiresAt; + } + + private async Task RequestAccessTokenAsync(CancellationToken ct) + { + using var httpClient = httpClientFactory.CreateClient(HttpClientName); + + using var request = new HttpRequestMessage(HttpMethod.Post, _options.TokenUrl) + { + Content = new FormUrlEncodedContent([ + new KeyValuePair("grant_type", "client_credentials") + ]) + }; + + request.Headers.Authorization = new AuthenticationHeaderValue( + "Basic", + Convert.ToBase64String(Encoding.UTF8.GetBytes($"{_options.ClientId}:{_options.ClientSecret}"))); + + using var response = await httpClient.SendAsync(request, ct); + response.EnsureSuccessStatusCode(); + + var body = await response.Content.ReadFromJsonAsync(ct); + + return body is null || body.AccessToken.Length == 0 + ? throw new InvalidOperationException("Reddit returned an empty access token response.") + : body; + } +} diff --git a/src/LinkTracker.Scrapper.Infrastructure/Clients/Reddit/RedditHttpClient.cs b/src/LinkTracker.Scrapper.Infrastructure/Clients/Reddit/RedditHttpClient.cs new file mode 100644 index 0000000..2e44a09 --- /dev/null +++ b/src/LinkTracker.Scrapper.Infrastructure/Clients/Reddit/RedditHttpClient.cs @@ -0,0 +1,49 @@ +using System.Diagnostics; +using System.Net.Http.Json; +using LinkTracker.Scrapper.Application.Clients.Reddit; +using LinkTracker.Scrapper.Application.Clients.Reddit.Contracts; +using LinkTracker.Scrapper.Infrastructure.Telemetry; + +namespace LinkTracker.Scrapper.Infrastructure.Clients.Reddit; + +public sealed class RedditHttpClient(HttpClient httpClient, ScrapperMetrics metrics) : IRedditClient +{ + private const string Scope = "external_source"; + private const string ScopeType = "reddit.com"; + private const int PageSize = 100; + + public async Task> GetNewPostsAsync( + string subreddit, + CancellationToken ct = default) + { + var sw = Stopwatch.StartNew(); + + try + { + using var response = await httpClient.GetAsync($"/r/{subreddit}/new?limit={PageSize}", ct); + response.EnsureSuccessStatusCode(); + + var body = await response.Content.ReadFromJsonAsync>(ct); + + return body is null + ? throw new InvalidOperationException("Reddit returned an empty response body.") + : body.Data.Children.Select(x => x.Data).ToArray(); + } + catch + { + metrics.Errors.Add( + 1, + new KeyValuePair("scope", Scope), + new KeyValuePair("scope_type", ScopeType), + new KeyValuePair("reason", "exception")); + throw; + } + finally + { + metrics.RequestDuration.Record( + sw.Elapsed.TotalMilliseconds, + new KeyValuePair("scope", Scope), + new KeyValuePair("scope_type", ScopeType)); + } + } +} diff --git a/src/LinkTracker.Scrapper.Infrastructure/Clients/Registration/ClientsModule.cs b/src/LinkTracker.Scrapper.Infrastructure/Clients/Registration/ClientsModule.cs index f253613..864bc65 100644 --- a/src/LinkTracker.Scrapper.Infrastructure/Clients/Registration/ClientsModule.cs +++ b/src/LinkTracker.Scrapper.Infrastructure/Clients/Registration/ClientsModule.cs @@ -7,10 +7,12 @@ using Grpc.Net.Client; using LinkTracker.Grpc; using LinkTracker.Scrapper.Application.Clients.GitHub; +using LinkTracker.Scrapper.Application.Clients.Reddit; using LinkTracker.Scrapper.Application.Clients.StackOverflow; using LinkTracker.Scrapper.Infrastructure.Clients.Bot; using LinkTracker.Scrapper.Infrastructure.Clients.GitHub; using LinkTracker.Scrapper.Infrastructure.Clients.RateLimiting; +using LinkTracker.Scrapper.Infrastructure.Clients.Reddit; using LinkTracker.Scrapper.Infrastructure.Clients.StackOverflow; using LinkTracker.Scrapper.Infrastructure.Configuration.Bot; using LinkTracker.Scrapper.Infrastructure.Configuration.Clients; @@ -33,6 +35,7 @@ public static class ClientsModule { private const string GitHubApiName = "github.com"; private const string StackOverflowApiName = "stackoverflow.com"; + private const string RedditApiName = "reddit.com"; public static IServiceCollection AddClients(this IServiceCollection services, IConfiguration configuration) { @@ -61,6 +64,15 @@ public static IServiceCollection AddClients(this IServiceCollection services, IC .Validate(o => !string.IsNullOrWhiteSpace(o.BaseUrl), "StackOverflow:BaseUrl must be set") .ValidateOnStart(); + services + .AddOptions() + .Bind(configuration.GetSection("Reddit")) + .Validate(o => !string.IsNullOrWhiteSpace(o.BaseUrl), "Reddit:BaseUrl must be set") + .Validate(o => !string.IsNullOrWhiteSpace(o.TokenUrl), "Reddit:TokenUrl must be set") + .Validate(o => !string.IsNullOrWhiteSpace(o.ClientId), "Reddit:ClientId must be set") + .Validate(o => !string.IsNullOrWhiteSpace(o.ClientSecret), "Reddit:ClientSecret must be set") + .ValidateOnStart(); + var httpResilienceOptions = configuration.GetHttpResilienceOptions(); services.TryAddSingleton(TimeProvider.System); @@ -71,8 +83,12 @@ public static IServiceCollection AddClients(this IServiceCollection services, IC var stackOverflowRateLimit = configuration.GetSection("StackOverflow").Get()?.RateLimit ?? new StackOverflowOptions().RateLimit; + var redditRateLimit = configuration.GetSection("Reddit").Get()?.RateLimit + ?? new RedditOptions().RateLimit; + services.AddExternalApiRateLimiter(GitHubApiName, gitHubRateLimit); services.AddExternalApiRateLimiter(StackOverflowApiName, stackOverflowRateLimit); + services.AddExternalApiRateLimiter(RedditApiName, redditRateLimit); services.AddHttpClient((sp, client) => { @@ -162,6 +178,29 @@ public static IServiceCollection AddClients(this IServiceCollection services, IC .AddConfiguredHttpResilience("stackoverflow", httpResilienceOptions) .AddExternalApiRateLimiting(StackOverflowApiName, stackOverflowRateLimit); + services.AddHttpClient(RedditAccessTokenProvider.HttpClientName, client => + { + client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("LinkTracker", "1.0")); + }) + .AddConfiguredHttpResilience("reddit-token", httpResilienceOptions); + + services.AddSingleton(sp => new RedditAccessTokenProvider( + sp.GetRequiredService(), + sp.GetRequiredService>(), + sp.GetRequiredService())); + + services.AddTransient(); + + services.AddHttpClient((sp, client) => + { + var options = sp.GetRequiredService>().Value; + client.BaseAddress = new Uri(options.BaseUrl); + client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("LinkTracker", "1.0")); + }) + .AddHttpMessageHandler() + .AddConfiguredHttpResilience("reddit", httpResilienceOptions) + .AddExternalApiRateLimiting(RedditApiName, redditRateLimit); + return services; } diff --git a/src/LinkTracker.Scrapper.Infrastructure/Configuration/Clients/RedditOptions.cs b/src/LinkTracker.Scrapper.Infrastructure/Configuration/Clients/RedditOptions.cs new file mode 100644 index 0000000..00b58f0 --- /dev/null +++ b/src/LinkTracker.Scrapper.Infrastructure/Configuration/Clients/RedditOptions.cs @@ -0,0 +1,19 @@ +namespace LinkTracker.Scrapper.Infrastructure.Configuration.Clients; + +public sealed class RedditOptions +{ + public string BaseUrl { get; init; } = "https://oauth.reddit.com"; + + public string TokenUrl { get; init; } = "https://www.reddit.com/api/v1/access_token"; + + public string ClientId { get; init; } = string.Empty; + + public string ClientSecret { get; init; } = string.Empty; + + public ExternalApiRateLimitOptions RateLimit { get; init; } = new() + { + TokenLimit = 100, + TokensPerPeriod = 100, + ReplenishmentPeriodSeconds = 60 + }; +} diff --git a/src/LinkTracker.Tests/Bot/Unit/Infrastructure/Clients/Scrapper/ScrapperErrorMessageMapperTests.cs b/src/LinkTracker.Tests/Bot/Unit/Infrastructure/Clients/Scrapper/ScrapperErrorMessageMapperTests.cs index 3d219e9..25d1ae0 100644 --- a/src/LinkTracker.Tests/Bot/Unit/Infrastructure/Clients/Scrapper/ScrapperErrorMessageMapperTests.cs +++ b/src/LinkTracker.Tests/Bot/Unit/Infrastructure/Clients/Scrapper/ScrapperErrorMessageMapperTests.cs @@ -21,6 +21,7 @@ public void TryMap_WhenUnsupportedLink_ReturnsHelpfulMessage() Assert.True(result); Assert.Contains("GitHub", message); Assert.Contains("StackOverflow", message); + Assert.Contains("Reddit", message); } [Fact] diff --git a/src/LinkTracker.Tests/Scrapper/Integration/Http/RedditHttpClientTests.cs b/src/LinkTracker.Tests/Scrapper/Integration/Http/RedditHttpClientTests.cs new file mode 100644 index 0000000..a397966 --- /dev/null +++ b/src/LinkTracker.Tests/Scrapper/Integration/Http/RedditHttpClientTests.cs @@ -0,0 +1,137 @@ +using System.Net; +using LinkTracker.Scrapper.Infrastructure.Clients.Reddit; +using LinkTracker.Scrapper.Infrastructure.Telemetry; + +namespace LinkTracker.Tests.Scrapper.Integration.Http; + +[Trait("Module", "Scrapper")] +[Trait("Category", "Integration")] +public sealed class RedditHttpClientTests(WireMockContainerFixture wireMock) : IClassFixture +{ + [Fact] + public async Task GetNewPosts_WhenResponseIsValid_ReturnsUnwrappedPosts() + { + await wireMock.ResetAsync(); + + await wireMock.StubAsync(new + { + request = new { method = "GET", url = "/r/dotnet/new?limit=100" }, + response = new + { + status = 200, + headers = new Dictionary { ["Content-Type"] = "application/json" }, + jsonBody = new + { + kind = "Listing", + data = new + { + children = new object[] + { + new + { + kind = "t3", + data = new + { + id = "abc123", + title = "Announcing .NET 10", + selftext = "Release notes inside", + author = "alice", + permalink = "/r/dotnet/comments/abc123/announcing_net_10/", + created_utc = 1741348800.5 + } + }, + new + { + kind = "t3", + data = new + { + id = "def456", + title = "Weekly thread", + selftext = "", + author = "bob", + permalink = "/r/dotnet/comments/def456/weekly_thread/", + created_utc = 1741262400.0 + } + } + } + } + } + } + }); + + using var httpClient = new HttpClient(); + httpClient.BaseAddress = new Uri(wireMock.BaseUrl); + + var metrics = new ScrapperMetrics(); + + var sut = new RedditHttpClient(httpClient, metrics); + + var result = await sut.GetNewPostsAsync("dotnet"); + + Assert.Equal(2, result.Count); + + Assert.Equal("abc123", result[0].Id); + Assert.Equal("Announcing .NET 10", result[0].Title); + Assert.Equal("Release notes inside", result[0].Selftext); + Assert.Equal("alice", result[0].Author); + Assert.Equal("/r/dotnet/comments/abc123/announcing_net_10/", result[0].Permalink); + Assert.Equal(DateTimeOffset.FromUnixTimeSeconds(1741348800), result[0].CreatedAt); + + Assert.Equal("def456", result[1].Id); + Assert.Equal(DateTimeOffset.FromUnixTimeSeconds(1741262400), result[1].CreatedAt); + } + + [Fact] + public async Task GetNewPosts_WhenChildrenAreEmpty_ReturnsEmptyList() + { + await wireMock.ResetAsync(); + + await wireMock.StubAsync(new + { + request = new { method = "GET", url = "/r/dotnet/new?limit=100" }, + response = new + { + status = 200, + headers = new Dictionary { ["Content-Type"] = "application/json" }, + body = "{\"kind\":\"Listing\",\"data\":{\"children\":[]}}" + } + }); + + using var httpClient = new HttpClient(); + httpClient.BaseAddress = new Uri(wireMock.BaseUrl); + + var metrics = new ScrapperMetrics(); + + var sut = new RedditHttpClient(httpClient, metrics); + + var result = await sut.GetNewPostsAsync("dotnet"); + + Assert.Empty(result); + } + + [Fact] + public async Task GetNewPosts_WhenStatusIsNotSuccess_ThrowsHttpRequestException() + { + await wireMock.ResetAsync(); + + await wireMock.StubAsync(new + { + request = new { method = "GET", url = "/r/dotnet/new?limit=100" }, + response = new + { + status = (int)HttpStatusCode.TooManyRequests, + headers = new Dictionary { ["Content-Type"] = "application/json" }, + jsonBody = new { message = "Too Many Requests", error = 429 } + } + }); + + using var httpClient = new HttpClient(); + httpClient.BaseAddress = new Uri(wireMock.BaseUrl); + + var metrics = new ScrapperMetrics(); + + var sut = new RedditHttpClient(httpClient, metrics); + + await Assert.ThrowsAsync(() => sut.GetNewPostsAsync("dotnet")); + } +} diff --git a/src/LinkTracker.Tests/Scrapper/Unit/Application/Registration/ApplicationModuleTests.cs b/src/LinkTracker.Tests/Scrapper/Unit/Application/Registration/ApplicationModuleTests.cs new file mode 100644 index 0000000..451188e --- /dev/null +++ b/src/LinkTracker.Tests/Scrapper/Unit/Application/Registration/ApplicationModuleTests.cs @@ -0,0 +1,55 @@ +using LinkTracker.Scrapper.Application.Abstractions.Updates; +using LinkTracker.Scrapper.Application.Clients.GitHub; +using LinkTracker.Scrapper.Application.Clients.Reddit; +using LinkTracker.Scrapper.Application.Clients.StackOverflow; +using LinkTracker.Scrapper.Application.Registration; +using LinkTracker.Scrapper.Application.Services.Updates.Clients; +using LinkTracker.Scrapper.Storage.Abstractions.Models; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using NSubstitute; + +namespace LinkTracker.Tests.Scrapper.Unit.Application.Registration; + +[Trait("Module", "Scrapper")] +[Trait("Category", "Unit")] +public sealed class ApplicationModuleTests +{ + [Fact] + public void AddApplication_RegistersHandlerForEverySupportedSource() + { + using var serviceProvider = BuildServiceProvider(); + + var handlers = serviceProvider.GetServices().ToArray(); + + Assert.Contains(handlers, x => x is GitHubLinkUpdateHandler); + Assert.Contains(handlers, x => x is StackOverflowLinkUpdateHandler); + Assert.Contains(handlers, x => x is RedditLinkUpdateHandler); + } + + [Fact] + public void AddApplication_ResolvedHandlers_AcceptSubredditLink() + { + using var serviceProvider = BuildServiceProvider(); + + var handlers = serviceProvider.GetServices(); + + Assert.Contains(handlers, x => x.CanHandle(new Uri("https://www.reddit.com/r/dotnet"))); + } + + private static ServiceProvider BuildServiceProvider() + { + var services = new ServiceCollection(); + + services.AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)); + services.AddSingleton(Substitute.For()); + services.AddSingleton(Substitute.For()); + services.AddSingleton(Substitute.For()); + services.AddSingleton(Substitute.For()); + + services.AddApplication(); + + return services.BuildServiceProvider(); + } +} diff --git a/src/LinkTracker.Tests/Scrapper/Unit/Application/Services/Updates/Clients/RedditLinkUpdateHandlerTests.cs b/src/LinkTracker.Tests/Scrapper/Unit/Application/Services/Updates/Clients/RedditLinkUpdateHandlerTests.cs new file mode 100644 index 0000000..4898453 --- /dev/null +++ b/src/LinkTracker.Tests/Scrapper/Unit/Application/Services/Updates/Clients/RedditLinkUpdateHandlerTests.cs @@ -0,0 +1,169 @@ +using LinkTracker.Scrapper.Application.Clients.Reddit; +using LinkTracker.Scrapper.Application.Clients.Reddit.Contracts; +using LinkTracker.Scrapper.Application.Models.Updates; +using LinkTracker.Scrapper.Application.Services.Updates; +using LinkTracker.Scrapper.Application.Services.Updates.Clients; +using LinkTracker.Scrapper.Storage.Abstractions.Models; +using Microsoft.Extensions.Logging; +using NSubstitute; + +namespace LinkTracker.Tests.Scrapper.Unit.Application.Services.Updates.Clients; + +[Trait("Module", "Scrapper")] +[Trait("Category", "Unit")] +public sealed class RedditLinkUpdateHandlerTests +{ + private readonly IRedditClient _redditClient = Substitute.For(); + private readonly ILogger _logger = Substitute.For>(); + + [Theory] + [InlineData("https://reddit.com/r/dotnet", true)] + [InlineData("https://www.reddit.com/r/dotnet", true)] + [InlineData("https://www.reddit.com/r/dotnet/", true)] + [InlineData("https://www.reddit.com/r/dotnet/comments/abc123/title", false)] + [InlineData("https://www.reddit.com/r/", false)] + [InlineData("https://www.reddit.com/user/alice", false)] + [InlineData("https://github.com/dotnet/runtime", false)] + public void CanHandle_ReturnsExpectedResult(string rawUrl, bool expected) + { + var sut = new RedditLinkUpdateHandler(_redditClient, _logger); + + Assert.Equal(expected, sut.CanHandle(new Uri(rawUrl))); + } + + [Fact] + public async Task CheckAsync_WhenLastUpdatedAtIsNull_ReturnsInitialStateWithNewestPostCursor() + { + var newestCreatedAt = new DateTimeOffset(2026, 9, 4, 12, 0, 0, TimeSpan.Zero); + + _redditClient.GetNewPostsAsync("dotnet", Arg.Any()) + .Returns([ + CreatePost("older", newestCreatedAt.AddHours(-1)), + CreatePost("newest", newestCreatedAt) + ]); + + var subscription = new TrackedLinkSubscription { Id = 1, Url = new Uri("https://www.reddit.com/r/dotnet"), TgChatIds = [1001L], LastUpdatedAt = null }; + + var sut = new RedditLinkUpdateHandler(_redditClient, _logger); + + var result = await sut.CheckAsync(subscription); + + Assert.False(result.HasChanges); + Assert.Empty(result.Events); + Assert.Equal(newestCreatedAt, result.NewLastUpdatedAt); + Assert.Equal("post:newest", result.NewLastEventKey); + } + + [Fact] + public async Task CheckAsync_WhenSubredditHasNoPosts_ReturnsNoChangesWithoutCursor() + { + _redditClient.GetNewPostsAsync("dotnet", Arg.Any()) + .Returns([]); + + var subscription = new TrackedLinkSubscription { Id = 1, Url = new Uri("https://www.reddit.com/r/dotnet"), TgChatIds = [1001L], LastUpdatedAt = null }; + + var sut = new RedditLinkUpdateHandler(_redditClient, _logger); + + var result = await sut.CheckAsync(subscription); + + Assert.False(result.HasChanges); + Assert.Null(result.NewLastUpdatedAt); + } + + [Fact] + public async Task CheckAsync_WhenAllPostsAreNotAfterCursor_ReturnsNoChangesAndKeepsCursor() + { + var lastSeenAt = new DateTimeOffset(2026, 9, 4, 12, 0, 0, TimeSpan.Zero); + + _redditClient.GetNewPostsAsync("dotnet", Arg.Any()) + .Returns([ + CreatePost("seen", lastSeenAt), + CreatePost("older", lastSeenAt.AddMinutes(-5)) + ]); + + var subscription = new TrackedLinkSubscription { Id = 1, Url = new Uri("https://www.reddit.com/r/dotnet"), TgChatIds = [1001L], LastUpdatedAt = lastSeenAt, LastEventKey = "post:seen" }; + + var sut = new RedditLinkUpdateHandler(_redditClient, _logger); + + var result = await sut.CheckAsync(subscription); + + Assert.False(result.HasChanges); + Assert.Equal(lastSeenAt, result.NewLastUpdatedAt); + Assert.Equal("post:seen", result.NewLastEventKey); + } + + [Fact] + public async Task CheckAsync_WhenNewPostsExist_ReturnsMappedEventsOrderedByCreatedAt() + { + var lastSeenAt = new DateTimeOffset(2026, 9, 4, 12, 0, 0, TimeSpan.Zero); + + _redditClient.GetNewPostsAsync("dotnet", Arg.Any()) + .Returns([ + CreatePost("second", lastSeenAt.AddMinutes(10)), + CreatePost("first", lastSeenAt.AddMinutes(5)), + CreatePost("seen", lastSeenAt.AddMinutes(-5)) + ]); + + var subscription = new TrackedLinkSubscription { Id = 1, Url = new Uri("https://www.reddit.com/r/dotnet"), TgChatIds = [1001L], LastUpdatedAt = lastSeenAt, LastEventKey = "post:old" }; + + var sut = new RedditLinkUpdateHandler(_redditClient, _logger); + + var result = await sut.CheckAsync(subscription); + + Assert.True(result.HasChanges); + Assert.Equal(2, result.Events.Count); + + Assert.Equal("post:first", result.Events[0].EventKey); + Assert.Equal("post:second", result.Events[1].EventKey); + + Assert.Equal(lastSeenAt.AddMinutes(10), result.NewLastUpdatedAt); + Assert.Equal("post:second", result.NewLastEventKey); + + var linkEvent = result.Events[0]; + + Assert.Equal(LinkSourceKind.Reddit, linkEvent.SourceKind); + Assert.Equal(LinkEventKind.Post, linkEvent.EventKind); + Assert.Equal("Title of first", linkEvent.Title); + Assert.Equal("author-first", linkEvent.UserName); + Assert.Equal("Body of first", linkEvent.Body); + Assert.Equal(new Uri("https://www.reddit.com/r/dotnet/comments/first/title/"), linkEvent.ResourceUrl); + } + + [Fact] + public async Task CheckAsync_ProducedEvent_IsFormattedAsRedditPostNotification() + { + var lastSeenAt = new DateTimeOffset(2026, 9, 4, 12, 0, 0, TimeSpan.Zero); + + _redditClient.GetNewPostsAsync("dotnet", Arg.Any()) + .Returns([CreatePost("first", lastSeenAt.AddMinutes(5))]); + + var subscription = new TrackedLinkSubscription { Id = 1, Url = new Uri("https://www.reddit.com/r/dotnet"), TgChatIds = [1001L], LastUpdatedAt = lastSeenAt }; + + var sut = new RedditLinkUpdateHandler(_redditClient, _logger); + + var result = await sut.CheckAsync(subscription); + + var update = LinkUpdatePayloadMapper.ToBotUpdate(subscription, result.Events[0]); + + Assert.Contains("Источник: Reddit", update.Description, StringComparison.Ordinal); + Assert.Contains("Тип: post", update.Description, StringComparison.Ordinal); + Assert.Contains( + "Ссылка: https://www.reddit.com/r/dotnet/comments/first/title/", + update.Description, + StringComparison.Ordinal); + Assert.Equal("author-first", update.Author); + } + + private static RedditPostResponse CreatePost(string id, DateTimeOffset createdAt) + { + return new RedditPostResponse + { + Id = id, + Title = $"Title of {id}", + Selftext = $"Body of {id}", + Author = $"author-{id}", + Permalink = $"/r/dotnet/comments/{id}/title/", + CreatedUtcSeconds = createdAt.ToUnixTimeSeconds() + }; + } +} diff --git a/src/LinkTracker.Tests/Scrapper/Unit/Infrastructure/Clients/Reddit/RedditAccessTokenHandlerTests.cs b/src/LinkTracker.Tests/Scrapper/Unit/Infrastructure/Clients/Reddit/RedditAccessTokenHandlerTests.cs new file mode 100644 index 0000000..fd73ace --- /dev/null +++ b/src/LinkTracker.Tests/Scrapper/Unit/Infrastructure/Clients/Reddit/RedditAccessTokenHandlerTests.cs @@ -0,0 +1,47 @@ +using System.Net; +using LinkTracker.Scrapper.Infrastructure.Clients.Reddit; +using NSubstitute; + +namespace LinkTracker.Tests.Scrapper.Unit.Infrastructure.Clients.Reddit; + +[Trait("Module", "Scrapper")] +[Trait("Category", "Unit")] +public sealed class RedditAccessTokenHandlerTests +{ + [Fact] + public async Task SendAsync_AddsBearerAuthorizationHeader() + { + var tokenProvider = Substitute.For(); + tokenProvider.GetAccessTokenAsync(Arg.Any()).Returns("token-1"); + + var innerHandler = new CapturingHandler(); + + var sut = new RedditAccessTokenHandler(tokenProvider) { InnerHandler = innerHandler }; + + using var invoker = new HttpMessageInvoker(sut); + using var request = new HttpRequestMessage(HttpMethod.Get, "https://oauth.reddit.com/r/dotnet/new"); + + using var response = await invoker.SendAsync(request, CancellationToken.None); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("Bearer", innerHandler.LastAuthorizationScheme); + Assert.Equal("token-1", innerHandler.LastAuthorizationParameter); + } + + private sealed class CapturingHandler : HttpMessageHandler + { + public string? LastAuthorizationScheme { get; private set; } + + public string? LastAuthorizationParameter { get; private set; } + + protected override Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + LastAuthorizationScheme = request.Headers.Authorization?.Scheme; + LastAuthorizationParameter = request.Headers.Authorization?.Parameter; + + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)); + } + } +} diff --git a/src/LinkTracker.Tests/Scrapper/Unit/Infrastructure/Clients/Reddit/RedditAccessTokenProviderTests.cs b/src/LinkTracker.Tests/Scrapper/Unit/Infrastructure/Clients/Reddit/RedditAccessTokenProviderTests.cs new file mode 100644 index 0000000..e673733 --- /dev/null +++ b/src/LinkTracker.Tests/Scrapper/Unit/Infrastructure/Clients/Reddit/RedditAccessTokenProviderTests.cs @@ -0,0 +1,147 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Text; +using LinkTracker.Scrapper.Infrastructure.Clients.Reddit; +using LinkTracker.Scrapper.Infrastructure.Configuration.Clients; +using Microsoft.Extensions.Options; + +namespace LinkTracker.Tests.Scrapper.Unit.Infrastructure.Clients.Reddit; + +[Trait("Module", "Scrapper")] +[Trait("Category", "Unit")] +public sealed class RedditAccessTokenProviderTests +{ + private const string TokenUrl = "https://www.reddit.com/api/v1/access_token"; + + [Fact] + public async Task GetAccessTokenAsync_WhenCalledTwice_RequestsTokenOnce() + { + var handler = new StubHttpMessageHandler(expiresInSeconds: 3600); + var timeProvider = new MutableTimeProvider(new DateTimeOffset(2026, 9, 4, 12, 0, 0, TimeSpan.Zero)); + + var sut = CreateSut(handler, timeProvider); + + var first = await sut.GetAccessTokenAsync(); + var second = await sut.GetAccessTokenAsync(); + + Assert.Equal("token-1", first); + Assert.Equal("token-1", second); + Assert.Equal(1, handler.RequestCount); + } + + [Fact] + public async Task GetAccessTokenAsync_WhenTokenIsAboutToExpire_RequestsNewToken() + { + var handler = new StubHttpMessageHandler(expiresInSeconds: 3600); + var timeProvider = new MutableTimeProvider(new DateTimeOffset(2026, 9, 4, 12, 0, 0, TimeSpan.Zero)); + + var sut = CreateSut(handler, timeProvider); + + var first = await sut.GetAccessTokenAsync(); + + timeProvider.Advance(TimeSpan.FromSeconds(3600 - 30)); + + var second = await sut.GetAccessTokenAsync(); + + Assert.Equal("token-1", first); + Assert.Equal("token-2", second); + Assert.Equal(2, handler.RequestCount); + } + + [Fact] + public async Task GetAccessTokenAsync_SendsBasicAuthorizationAndClientCredentialsGrant() + { + var handler = new StubHttpMessageHandler(expiresInSeconds: 3600); + var timeProvider = new MutableTimeProvider(new DateTimeOffset(2026, 9, 4, 12, 0, 0, TimeSpan.Zero)); + + var sut = CreateSut(handler, timeProvider); + + await sut.GetAccessTokenAsync(); + + var expectedCredentials = Convert.ToBase64String(Encoding.UTF8.GetBytes("client-id:client-secret")); + + Assert.Equal(HttpMethod.Post, handler.LastMethod); + Assert.Equal(new Uri(TokenUrl), handler.LastRequestUri); + Assert.Equal("Basic", handler.LastAuthorization?.Scheme); + Assert.Equal(expectedCredentials, handler.LastAuthorization?.Parameter); + Assert.Equal("grant_type=client_credentials", handler.LastBody); + } + + [Fact] + public async Task GetAccessTokenAsync_WhenStatusIsNotSuccess_ThrowsHttpRequestException() + { + var handler = new StubHttpMessageHandler(expiresInSeconds: 3600) { StatusCode = HttpStatusCode.Unauthorized }; + var timeProvider = new MutableTimeProvider(new DateTimeOffset(2026, 9, 4, 12, 0, 0, TimeSpan.Zero)); + + var sut = CreateSut(handler, timeProvider); + + await Assert.ThrowsAsync(() => sut.GetAccessTokenAsync()); + } + + private static RedditAccessTokenProvider CreateSut(StubHttpMessageHandler handler, TimeProvider timeProvider) + { + var options = Options.Create(new RedditOptions { TokenUrl = TokenUrl, ClientId = "client-id", ClientSecret = "client-secret" }); + + return new RedditAccessTokenProvider(new StubHttpClientFactory(handler), options, timeProvider); + } + + private sealed class StubHttpClientFactory(HttpMessageHandler handler) : IHttpClientFactory + { + public HttpClient CreateClient(string name) + { + return new HttpClient(handler, disposeHandler: false); + } + } + + private sealed class StubHttpMessageHandler(int expiresInSeconds) : HttpMessageHandler + { + public int RequestCount { get; private set; } + + public HttpStatusCode StatusCode { get; init; } = HttpStatusCode.OK; + + public HttpMethod? LastMethod { get; private set; } + + public Uri? LastRequestUri { get; private set; } + + public AuthenticationHeaderValue? LastAuthorization { get; private set; } + + public string? LastBody { get; private set; } + + protected override async Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + RequestCount++; + + LastMethod = request.Method; + LastRequestUri = request.RequestUri; + LastAuthorization = request.Headers.Authorization; + LastBody = request.Content is null + ? null + : await request.Content.ReadAsStringAsync(cancellationToken); + + return new HttpResponseMessage(StatusCode) + { + Content = new StringContent( + $"{{\"access_token\":\"token-{RequestCount}\",\"token_type\":\"bearer\",\"expires_in\":{expiresInSeconds}}}", + Encoding.UTF8, + "application/json") + }; + } + } + + private sealed class MutableTimeProvider(DateTimeOffset utcNow) : TimeProvider + { + private DateTimeOffset _utcNow = utcNow; + + public override DateTimeOffset GetUtcNow() + { + return _utcNow; + } + + public void Advance(TimeSpan delta) + { + _utcNow = _utcNow.Add(delta); + } + } +} diff --git a/src/LinkTracker.Tests/Scrapper/Unit/Infrastructure/Clients/Registration/ClientsModuleRedditTests.cs b/src/LinkTracker.Tests/Scrapper/Unit/Infrastructure/Clients/Registration/ClientsModuleRedditTests.cs new file mode 100644 index 0000000..e700684 --- /dev/null +++ b/src/LinkTracker.Tests/Scrapper/Unit/Infrastructure/Clients/Registration/ClientsModuleRedditTests.cs @@ -0,0 +1,65 @@ +using LinkTracker.Scrapper.Application.Clients.Reddit; +using LinkTracker.Scrapper.Infrastructure.Clients.Reddit; +using LinkTracker.Scrapper.Infrastructure.Clients.Registration; +using LinkTracker.Scrapper.Infrastructure.Configuration.Clients; +using LinkTracker.Scrapper.Infrastructure.Telemetry; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; + +namespace LinkTracker.Tests.Scrapper.Unit.Infrastructure.Clients.Registration; + +[Trait("Module", "Scrapper")] +[Trait("Category", "Unit")] +public sealed class ClientsModuleRedditTests +{ + [Fact] + public void AddClients_ResolvesRedditHttpClient() + { + using var serviceProvider = BuildServiceProvider(); + + var client = serviceProvider.GetRequiredService(); + + Assert.IsType(client); + } + + [Fact] + public void AddClients_BindsRedditOptionsFromConfiguration() + { + using var serviceProvider = BuildServiceProvider(); + + var options = serviceProvider.GetRequiredService>().Value; + + Assert.Equal("https://oauth.reddit.com", options.BaseUrl); + Assert.Equal("https://www.reddit.com/api/v1/access_token", options.TokenUrl); + Assert.Equal("client-id", options.ClientId); + Assert.Equal("client-secret", options.ClientSecret); + Assert.Equal(100, options.RateLimit.TokenLimit); + } + + private static ServiceProvider BuildServiceProvider() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Bot:BaseUrl"] = "http://localhost:8091", + ["Kafka:BootstrapServers"] = "localhost:9092", + ["Kafka:Topic"] = "link.raw-updates", + ["Reddit:ClientId"] = "client-id", + ["Reddit:ClientSecret"] = "client-secret" + }) + .Build(); + + var services = new ServiceCollection(); + + services.AddSingleton(configuration); + services.AddSingleton(); + services.AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)); + + services.AddClients(configuration); + + return services.BuildServiceProvider(); + } +}