add reddit logic

README.md

@@ -27,7 +27,10 @@ Scrapper Service
 Осуществляет мониторинг контента:
 
 Периодическая проверка отслеживаемых URL на наличие изменений
-Парсинг контента с различных источников (GitHub, Stack Overflow, Reddit и др.)
+Парсинг контента с поддерживаемых источников:
+- GitHub-репозитории — новые issue и pull request: `https://github.com/owner/repo`
+- вопросы Stack Overflow — новые ответы и комментарии: `https://stackoverflow.com/questions/12345/...`
+- сабреддиты Reddit — новые посты: `https://reddit.com/r/dotnet`
 Определение изменений (diff detection)
 Отправка уведомлений в Bot Service при обнаружении обновлений
 Хранение информации о подписках и состоянии контента
@@ -60,7 +63,8 @@ cp src/LinkTracker.Bot.Api/.env.template src/LinkTracker.Bot.Api/.env
 ```
 cp src/LinkTracker.Scrapper.Api/.env.template src/LinkTracker.Scrapper.Api/.env
 ```
-4. Поставить действительные параметры в .env.
+4. Поставить действительные параметры в .env. Для Reddit нужны `Reddit__ClientId` и
+   `Reddit__ClientSecret` — создаются на https://www.reddit.com/prefs/apps как приложение типа `script`.
 5. Создать копию [.env.template](src/LinkTracker.AiAgent.Api/.env.template) с именем .env в каталоге ии-агента.
 ```
 cp src/LinkTracker.AiAgent.Api/.env.template src/LinkTracker.AiAgent.Api/.env

src/LinkTracker.Bot.Application/Clients/Scrapper/ScrapperErrorMessages.cs

@@ -11,7 +11,8 @@ internal static class ScrapperErrorMessages
                 "Эта ссылка не поддерживается.\n" +
                 "Сейчас можно отслеживать только:\n" +
                 "• GitHub-репозитории: https://github.com/owner/repo\n" +
-                "• вопросы StackOverflow: https://stackoverflow.com/questions/12345/...",
+                "• вопросы StackOverflow: https://stackoverflow.com/questions/12345/...\n" +
+                "• сабреддиты Reddit: https://reddit.com/r/dotnet",
             [ScrapperErrorCodes.ChatAlreadyExists] =
                 "Привет! Чат уже зарегистрирован. Используй справку /help для просмотра команд",
             [ScrapperErrorCodes.LinkAlreadyExists] =

src/LinkTracker.Scrapper.Api/.env.template

@@ -1,2 +1,4 @@
 GitHub__Token=ghp_abc
+Reddit__ClientId=your-reddit-app-client-id
+Reddit__ClientSecret=your-reddit-app-client-secret
 ServiceAuth__Secret=change-me-to-a-long-random-string

src/LinkTracker.Scrapper.Api/appsettings.json

@@ -49,6 +49,18 @@
       "AcquireTimeoutSeconds": 60
     }
   },
+  "Reddit": {
+    "BaseUrl": "https://oauth.reddit.com",
+    "TokenUrl": "https://www.reddit.com/api/v1/access_token",
+    "RateLimit": {
+      "Enabled": true,
+      "TokenLimit": 100,
+      "TokensPerPeriod": 100,
+      "ReplenishmentPeriodSeconds": 60,
+      "QueueLimit": 1000,
+      "AcquireTimeoutSeconds": 60
+    }
+  },
   "Database": {
     "Host": "localhost",
     "Port": 5434,

src/LinkTracker.Scrapper.Application/Clients/Reddit/Contracts/RedditListingData.cs

@@ -0,0 +1,8 @@
+using System.Text.Json.Serialization;
+
+namespace LinkTracker.Scrapper.Application.Clients.Reddit.Contracts;
+
+public sealed class RedditListingData<T>
+{
+    [JsonPropertyName("children")] public IReadOnlyList<RedditThingResponse<T>> Children { get; init; } = [];
+}

src/LinkTracker.Scrapper.Application/Clients/Reddit/Contracts/RedditListingEnvelope.cs

@@ -0,0 +1,8 @@
+using System.Text.Json.Serialization;
+
+namespace LinkTracker.Scrapper.Application.Clients.Reddit.Contracts;
+
+public sealed class RedditListingEnvelope<T>
+{
+    [JsonPropertyName("data")] public RedditListingData<T> Data { get; init; } = new();
+}

src/LinkTracker.Scrapper.Application/Clients/Reddit/Contracts/RedditPostResponse.cs

@@ -0,0 +1,20 @@
+using System.Text.Json.Serialization;
+
+namespace LinkTracker.Scrapper.Application.Clients.Reddit.Contracts;
+
+public sealed class RedditPostResponse
+{
+    [JsonPropertyName("id")] public string Id { get; init; } = string.Empty;
+
+    [JsonPropertyName("title")] public string Title { get; init; } = string.Empty;
+
+    [JsonPropertyName("selftext")] public string Selftext { get; init; } = string.Empty;
+
+    [JsonPropertyName("author")] public string Author { get; init; } = string.Empty;
+
+    [JsonPropertyName("permalink")] public string Permalink { get; init; } = string.Empty;
+
+    [JsonPropertyName("created_utc")] public double CreatedUtcSeconds { get; init; }
+
+    public DateTimeOffset CreatedAt => DateTimeOffset.FromUnixTimeSeconds((long)CreatedUtcSeconds);
+}

src/LinkTracker.Scrapper.Application/Clients/Reddit/Contracts/RedditThingResponse.cs

@@ -0,0 +1,10 @@
+using System.Text.Json.Serialization;
+
+namespace LinkTracker.Scrapper.Application.Clients.Reddit.Contracts;
+
+public sealed class RedditThingResponse<T>
+{
+    [JsonPropertyName("kind")] public string Kind { get; init; } = string.Empty;
+
+    [JsonPropertyName("data")] public T Data { get; init; } = default!;
+}

src/LinkTracker.Scrapper.Application/Clients/Reddit/IRedditClient.cs

@@ -0,0 +1,8 @@
+using LinkTracker.Scrapper.Application.Clients.Reddit.Contracts;
+
+namespace LinkTracker.Scrapper.Application.Clients.Reddit;
+
+public interface IRedditClient
+{
+    Task<IReadOnlyList<RedditPostResponse>> GetNewPostsAsync(string subreddit, CancellationToken ct = default);
+}

src/LinkTracker.Scrapper.Application/Errors/ScrapperErrors.cs

@@ -82,6 +82,6 @@ public static ApiException UnsupportedLink(Uri link)
         return new ApiException(
             HttpStatusCode.BadRequest,
             ScrapperErrorCodes.UnsupportedLink,
-            $"Ссылка '{link}' не поддерживается. Сейчас поддерживаются только GitHub repository и StackOverflow question.");
+            $"Ссылка '{link}' не поддерживается. Сейчас поддерживаются только GitHub repository, StackOverflow question и Reddit subreddit.");
     }
 }

src/LinkTracker.Scrapper.Application/Models/Updates/LinkEventKind.cs

@@ -6,5 +6,6 @@ public enum LinkEventKind
     PullRequest = 2,
     QuestionActivity = 3,
     Answer = 4,
-    Comment = 5
-}
+    Comment = 5,
+    Post = 6
+}

src/LinkTracker.Scrapper.Application/Models/Updates/LinkSourceKind.cs

@@ -3,5 +3,6 @@ namespace LinkTracker.Scrapper.Application.Models.Updates;
 public enum LinkSourceKind
 {
     GitHub = 1,
-    StackOverflow = 2
-}
+    StackOverflow = 2,
+    Reddit = 3
+}

src/LinkTracker.Scrapper.Application/Registration/ApplicationModule.cs

@@ -14,6 +14,7 @@ public static IServiceCollection AddApplication(this IServiceCollection services
 
         services.AddSingleton<ILinkUpdateHandler, GitHubLinkUpdateHandler>();
         services.AddSingleton<ILinkUpdateHandler, StackOverflowLinkUpdateHandler>();
+        services.AddSingleton<ILinkUpdateHandler, RedditLinkUpdateHandler>();
 
         return services;
     }

src/LinkTracker.Scrapper.Application/Services/Helpers/LinkUpdateResultBuilder.cs

@@ -4,9 +4,11 @@ namespace LinkTracker.Scrapper.Application.Services.Helpers;
 
 internal static class LinkUpdateResultBuilder
 {
-    public static LinkCheckResult InitialState(DateTimeOffset actualLastUpdatedAt)
+    public static LinkCheckResult InitialState(
+        DateTimeOffset actualLastUpdatedAt,
+        string? actualLastEventKey = null)
     {
-        return Build(actualLastUpdatedAt, null, []);
+        return Build(actualLastUpdatedAt, actualLastEventKey, []);
     }
 
     public static LinkCheckResult NoChanges(

src/LinkTracker.Scrapper.Application/Services/Updates/Clients/RedditLinkUpdateHandler.cs

@@ -0,0 +1,104 @@
+using LinkTracker.Scrapper.Application.Clients.Reddit;
+using LinkTracker.Scrapper.Application.Clients.Reddit.Contracts;
+using LinkTracker.Scrapper.Application.Models.Updates;
+using LinkTracker.Scrapper.Application.Services.Helpers;
+using LinkTracker.Scrapper.Storage.Abstractions.Models;
+using Microsoft.Extensions.Logging;
+
+namespace LinkTracker.Scrapper.Application.Services.Updates.Clients;
+
+public sealed class RedditLinkUpdateHandler(
+    IRedditClient redditClient,
+    ILogger<RedditLinkUpdateHandler> logger) : LinkUpdateHandlerBase(logger)
+{
+    public override bool CanHandle(Uri url)
+    {
+        return TryParseSubreddit(url, out _);
+    }
+
+    protected override async Task<LinkCheckResult> InitializeStateAsync(
+        TrackedLinkSubscription subscription,
+        CancellationToken ct)
+    {
+        TryParseSubreddit(subscription.Url, out var subreddit);
+
+        var posts = await redditClient.GetNewPostsAsync(subreddit, ct);
+
+        if (posts.Count == 0)
+        {
+            return LinkUpdateResultBuilder.NoChanges();
+        }
+
+        var newest = posts.MaxBy(x => x.CreatedAt)!;
+
+        return LinkUpdateResultBuilder.InitialState(newest.CreatedAt, BuildEventKey(newest));
+    }
+
+    protected override async Task<IReadOnlyList<LinkEvent>> GetNewEventsAsync(
+        TrackedLinkSubscription subscription,
+        DateTimeOffset lastSeenAt,
+        string? lastEventKey,
+        CancellationToken ct)
+    {
+        TryParseSubreddit(subscription.Url, out var subreddit);
+
+        var posts = await redditClient.GetNewPostsAsync(subreddit, ct);
+
+        return posts
+            .Select(x => MapPostToEvent(x, subscription.Url))
+            .Where(x => IsAfterCursor(x, lastSeenAt, lastEventKey))
+            .ToArray();
+    }
+
+    private static bool TryParseSubreddit(Uri url, out string subreddit)
+    {
+        subreddit = string.Empty;
+
+        if (!UriParsingHelper.IsHost(url, "reddit.com"))
+        {
+            return false;
+        }
+
+        var segments = UriParsingHelper.GetPathSegments(url);
+        if (segments.Length != 2)
+        {
+            return false;
+        }
+
+        if (!string.Equals(segments[0], "r", StringComparison.OrdinalIgnoreCase))
+        {
+            return false;
+        }
+
+        subreddit = segments[1];
+
+        return !string.IsNullOrWhiteSpace(subreddit);
+    }
+
+    private static LinkEvent MapPostToEvent(RedditPostResponse post, Uri subredditUrl)
+    {
+        return new LinkEvent
+        {
+            SourceKind = LinkSourceKind.Reddit,
+            EventKind = LinkEventKind.Post,
+            Title = post.Title,
+            UserName = post.Author,
+            CreatedAt = post.CreatedAt,
+            EventKey = BuildEventKey(post),
+            Body = post.Selftext,
+            ResourceUrl = BuildResourceUrl(post, subredditUrl)
+        };
+    }
+
+    private static string BuildEventKey(RedditPostResponse post)
+    {
+        return $"post:{post.Id}";
+    }
+
+    private static Uri BuildResourceUrl(RedditPostResponse post, Uri subredditUrl)
+    {
+        return Uri.TryCreate(subredditUrl, post.Permalink, out var resourceUrl)
+            ? resourceUrl
+            : subredditUrl;
+    }
+}

src/LinkTracker.Scrapper.Application/Services/Updates/LinkEventDescriptionFormatter.cs

@@ -50,6 +50,7 @@ private static string FormatEventKind(LinkEventKind eventKind)
             LinkEventKind.QuestionActivity => "question-activity",
             LinkEventKind.Answer => "answer",
             LinkEventKind.Comment => "comment",
+            LinkEventKind.Post => "post",
             _ => eventKind.ToString()
         };
     }
@@ -60,6 +61,7 @@ private static string FormatSourceKind(LinkSourceKind sourceKind)
         {
             LinkSourceKind.GitHub => "GitHub",
             LinkSourceKind.StackOverflow => "Stack Overflow",
+            LinkSourceKind.Reddit => "Reddit",
             _ => sourceKind.ToString()
         };
     }

src/LinkTracker.Scrapper.Infrastructure/Clients/Reddit/Contracts/RedditAccessTokenResponse.cs

@@ -0,0 +1,10 @@
+using System.Text.Json.Serialization;
+
+namespace LinkTracker.Scrapper.Infrastructure.Clients.Reddit.Contracts;
+
+internal sealed class RedditAccessTokenResponse
+{
+    [JsonPropertyName("access_token")] public string AccessToken { get; init; } = string.Empty;
+
+    [JsonPropertyName("expires_in")] public int ExpiresInSeconds { get; init; }
+}

src/LinkTracker.Scrapper.Infrastructure/Clients/Reddit/IRedditAccessTokenProvider.cs

@@ -0,0 +1,6 @@
+namespace LinkTracker.Scrapper.Infrastructure.Clients.Reddit;
+
+internal interface IRedditAccessTokenProvider
+{
+    Task<string> GetAccessTokenAsync(CancellationToken ct = default);
+}

src/LinkTracker.Scrapper.Infrastructure/Clients/Reddit/RedditAccessTokenHandler.cs

@@ -0,0 +1,17 @@
+using System.Net.Http.Headers;
+
+namespace LinkTracker.Scrapper.Infrastructure.Clients.Reddit;
+
+internal sealed class RedditAccessTokenHandler(IRedditAccessTokenProvider tokenProvider) : DelegatingHandler
+{
+    protected override async Task<HttpResponseMessage> SendAsync(
+        HttpRequestMessage request,
+        CancellationToken cancellationToken)
+    {
+        var accessToken = await tokenProvider.GetAccessTokenAsync(cancellationToken);
+
+        request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", accessToken);
+
+        return await base.SendAsync(request, cancellationToken);
+    }
+}