clear refs && del dead code && upd dockerignore && upd valkey logic &…

.dockerignore

@@ -4,4 +4,7 @@
 .vs/
 .idea/
 *.user
-*.suo
+*.suo
+**/.env
+**/.env.*
+!**/.env.template

Directory.Build.props

@@ -0,0 +1,14 @@
+<Project>
+
+    <PropertyGroup>
+        <TargetFramework>net9.0</TargetFramework>
+        <LangVersion>latest</LangVersion>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+        <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
+        <EnableNETAnalyzers>true</EnableNETAnalyzers>
+        <AnalysisLevel>latest</AnalysisLevel>
+        <GenerateDocumentationFile>false</GenerateDocumentationFile>
+    </PropertyGroup>
+
+</Project>

OBSERVABILITY.md

@@ -32,7 +32,8 @@ Rate limiter применяется точечно, политикой `public-a
 | `api_requests_total` | Counter | `source` | Счётчик запросов к API (лейбл — шаблон маршрута, не сырой путь) |
 | `request_duration_ms_total` | Histogram | `scope`, `scope_type` | Длительность операции в мс (RED: Duration) |
 | `errors_total` | Counter | `scope`, `scope_type`, `reason` | **Ошибки (RED: Errors)** |
-| `sent_updates_total` | Counter | — | Количество обновлений, отправленных в Bot |
+| `sent_updates_total` | Counter | — | Количество обновлений, фактически отправленных в Bot (в outbox-режиме инкрементируется при успешной отправке из outbox) |
+| `outbox_enqueued_updates_total` | Counter | — | Количество обновлений, записанных в transactional outbox |
 | `db_queries_total` | Counter | `operation` | Количество запросов к БД |
 | `db_errors_total` | Counter | `operation` | Количество ошибок БД |
 | `db_query_duration_ms_total` | Histogram | `operation` | Длительность запроса к БД в мс |
@@ -58,6 +59,20 @@ Rate limiter применяется точечно, политикой `public-a
 | `process_memory_working_set_bytes` | Gauge | — | **Потребление RAM (working set), метрика для алерта** |
 | `process_memory_managed_bytes` | Gauge | — | Управляемая память (GC) |
 
+### AI Agent
+
+| Метрика | Тип | Лейблы | Описание |
+|---|---|---|---|
+| `kafka_consumed_total` | Counter | `topic` | Сообщений обработано из Kafka |
+| `kafka_consume_errors_total` | Counter | `topic` | Ошибок обработки из Kafka |
+| `kafka_consume_duration_ms_total` | Histogram | `topic` | Длительность обработки из Kafka в мс |
+| `kafka_dead_letter_total` | Counter | `topic` | Сообщений отправлено в DLQ |
+| `kafka_dead_letter_errors_total` | Counter | `topic` | Неудачных отправок в DLQ |
+| `summarizations_total` | Counter | — | Успешных суммаризаций через Yandex AI |
+| `summarization_fallbacks_total` | Counter | `reason` | **Суммаризаций, деградировавших до обрезки текста** |
+| `process_memory_working_set_bytes` | Gauge | — | **Потребление RAM (working set), метрика для алерта** |
+| `process_memory_managed_bytes` | Gauge | — | Управляемая память (GC) |
+
 > Histogram-метрики дают в Prometheus три серии: `<name>_bucket`, `<name>_sum`, `<name>_count`.
 
 ## Конфигурация сбора

README.md

@@ -87,9 +87,9 @@ dotnet run --project src/LinkTracker.AiAgent.Api
 
 ### Для запуска в контейнерах
 9. В [bot.Docker.appesettings](src/LinkTracker.Bot.Api/appsettings.Docker.json), [scrapper.Docker.appsettings](src/LinkTracker.Scrapper.Api/appsettings.Docker.json) и [aiagent.Docker.appesettings](src/LinkTracker.AiAgent.Api/appsettings.Docker.json) в ветках Scrapper, Bot и AiAgent соответственно выбрать валидные параметры.
-10. Выполнить поочердено запуск сначала [docker-compose](docker-compose.yml), [Scrapper](src/LinkTracker.Scrapper.Api/Program.cs), а затем [Bot](src/LinkTracker.Bot.Api/Program.cs) с помощью команд.
+10. Поднять инфраструктуру и все три сервиса одной командой — порядок запуска обеспечивают `depends_on` в compose-файлах.
 ```
-docker compose -f docker-compose.yml -f docker-compose.apps.yml up 
+docker compose -f docker-compose.yml -f docker-compose.apps.yml up
 ```
 
 

docker-compose.apps.yml

@@ -41,6 +41,8 @@ services:
         condition: service_completed_successfully
       schema-registry:
         condition: service_started
+      valkey-cluster-init:
+        condition: service_completed_successfully
     expose:
       - "8091"
       - "8092"

migrations/006_links_last_checked_at.sql

@@ -0,0 +1,4 @@
+ALTER TABLE links
+    ADD COLUMN IF NOT EXISTS last_checked_at TIMESTAMPTZ NOT NULL DEFAULT '0001-01-01 00:00:00+00';
+
+CREATE INDEX IF NOT EXISTS ix_links_last_checked_at ON links (last_checked_at, id);

src/LinkTracker.AiAgent.Api/LinkTracker.AiAgent.Api.csproj

@@ -1,10 +1,4 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
-    <PropertyGroup>
-        <TargetFramework>net9.0</TargetFramework>
-        <Nullable>enable</Nullable>
-        <ImplicitUsings>enable</ImplicitUsings>
-    </PropertyGroup>
-
     <ItemGroup>
         <ProjectReference Include="..\LinkTracker.AiAgent.Application\LinkTracker.AiAgent.Application.csproj"/>
         <ProjectReference Include="..\LinkTracker.AiAgent.Infrastructure\LinkTracker.AiAgent.Infrastructure.csproj"/>

src/LinkTracker.AiAgent.Api/appsettings.Docker.json

@@ -52,10 +52,5 @@
     "Grouping": {
       "WindowMs": 30000
     }
-  },
-  "YandexAi": {
-    "BaseUrl": "https://ai.api.cloud.yandex.net",
-    "ModelId": "aliceai-llm",
-    "TimeoutSeconds": 120
   }
 }

src/LinkTracker.AiAgent.Api/appsettings.json

@@ -64,10 +64,5 @@
     "Grouping": {
       "WindowMs": 30000
     }
-  },
-  "YandexAi": {
-    "BaseUrl": "https://ai.api.cloud.yandex.net",
-    "ModelId": "aliceai-llm",
-    "TimeoutSeconds": 120
   }
 }

src/LinkTracker.AiAgent.Application/LinkTracker.AiAgent.Application.csproj

@@ -1,10 +1,4 @@
 <Project Sdk="Microsoft.NET.Sdk">
-    <PropertyGroup>
-        <TargetFramework>net9.0</TargetFramework>
-        <ImplicitUsings>enable</ImplicitUsings>
-        <Nullable>enable</Nullable>
-    </PropertyGroup>
-
     <ItemGroup>
         <ProjectReference Include="..\LinkTracker.Shared\LinkTracker.Shared.csproj"/>
     </ItemGroup>

src/LinkTracker.AiAgent.Application/Telemetry/Abstractions/IAiAgentMetrics.cs

@@ -11,4 +11,8 @@ public interface IAiAgentMetrics
     void IncrementKafkaDeadLetter(string topic);
 
     void IncrementKafkaDeadLetterError(string topic);
+
+    void IncrementSummarization();
+
+    void IncrementSummarizationFallback(string reason);
 }

src/LinkTracker.AiAgent.Infrastructure/Clients/Kafka/RawUpdatesKafkaConsumer.cs

@@ -17,10 +17,16 @@ internal sealed class RawUpdatesKafkaConsumer(
     ILogger<RawUpdatesKafkaConsumer> logger) : BackgroundService
 {
     private static readonly TimeSpan PollTimeout = TimeSpan.FromMilliseconds(500);
+    private static readonly TimeSpan ConsumeErrorBackoff = TimeSpan.FromSeconds(1);
 
     protected override Task ExecuteAsync(CancellationToken stoppingToken)
     {
-        return Task.Run(() => ConsumeLoopAsync(stoppingToken), stoppingToken);
+        return Task.Factory.StartNew(
+                () => ConsumeLoopAsync(stoppingToken),
+                stoppingToken,
+                TaskCreationOptions.LongRunning | TaskCreationOptions.DenyChildAttach,
+                TaskScheduler.Default)
+            .Unwrap();
     }
 
     private async Task ConsumeLoopAsync(CancellationToken stoppingToken)
@@ -45,7 +51,8 @@ private async Task ConsumeLoopAsync(CancellationToken stoppingToken)
                 }
                 catch (ConsumeException ex)
                 {
-                    logger.LogWarning(ex, "Kafka consume завершился ошибкой.");
+                    logger.LogWarning(ex, "Kafka consume завершился ошибкой. Пауза перед повтором.");
+                    await Task.Delay(ConsumeErrorBackoff, stoppingToken);
                     continue;
                 }
 

src/LinkTracker.AiAgent.Infrastructure/Clients/Registration/ClientsModule.cs

@@ -24,11 +24,10 @@ public static IServiceCollection AddAiAgentInfrastructure(
         services
             .AddOptions<AiAgentOptions>()
             .Bind(configuration.GetSection("AiAgent"))
-            .Validate(o => o.Filtering != null, "AiAgent:Filtering must be set")
-            .Validate(o => o.Summarization != null, "AiAgent:Summarization must be set")
-            .Validate(o => o.Prioritization != null, "AiAgent:Prioritization must be set")
-            .Validate(o => o.Grouping.WindowMs > 0, "AiAgent:Grouping:WindowMs must be greater that 0")
-            .Validate(o => o.Grouping.FlushIntervalMs > 0, "AiAgent:Grouping.FlushIntervalMs must be greater that 0")
+            .Validate(o => o.Filtering.MinLength >= 0, "AiAgent:Filtering:MinLength must not be negative")
+            .Validate(o => o.Summarization.Threshold > 0, "AiAgent:Summarization:Threshold must be greater than 0")
+            .Validate(o => o.Grouping.WindowMs > 0, "AiAgent:Grouping:WindowMs must be greater than 0")
+            .Validate(o => o.Grouping.FlushIntervalMs > 0, "AiAgent:Grouping:FlushIntervalMs must be greater than 0")
             .ValidateOnStart();
 
         services
@@ -52,6 +51,13 @@ public static IServiceCollection AddAiAgentInfrastructure(
         services
             .AddOptions<YandexAiOptions>()
             .Bind(configuration.GetSection("YandexAi"))
+            .Validate(o => !string.IsNullOrWhiteSpace(o.ApiKey), "YandexAi:ApiKey must be set")
+            .Validate(o => !string.IsNullOrWhiteSpace(o.FolderId), "YandexAi:FolderId must be set")
+            .Validate(o => !string.IsNullOrWhiteSpace(o.ModelId), "YandexAi:ModelId must be set")
+            .Validate(
+                o => Uri.TryCreate(o.BaseUrl, UriKind.Absolute, out _),
+                "YandexAi:BaseUrl must be an absolute URI")
+            .Validate(o => o.TimeoutSeconds > 0, "YandexAi:TimeoutSeconds must be greater than 0")
             .ValidateOnStart();
 
         services.AddSingleton<KafkaOffsetTracker>();

src/LinkTracker.AiAgent.Infrastructure/Clients/YandexAi/YandexAiHttpClient.cs

@@ -1,6 +1,7 @@
 using System.Net.Http.Headers;
 using System.Net.Http.Json;
 using LinkTracker.AiAgent.Application.Abstractions;
+using LinkTracker.AiAgent.Application.Telemetry.Abstractions;
 using LinkTracker.AiAgent.Infrastructure.Clients.YandexAi.Contracts;
 using LinkTracker.AiAgent.Infrastructure.Configuration.AiAgent;
 using LinkTracker.AiAgent.Infrastructure.Configuration.YandexAi;
@@ -13,8 +14,11 @@ internal sealed class YandexAiHttpClient(
     IHttpClientFactory httpClientFactory,
     IOptions<YandexAiOptions> yandexOptions,
     IOptions<AiAgentOptions> agentOptions,
+    IAiAgentMetrics metrics,
     ILogger<YandexAiHttpClient> logger) : ILinkUpdateSummarizer
 {
+    private const string Instructions = "You are a concise summarizer. Summarize the given update in 2-3 sentences.";
+
     public async Task<string> SummarizeAsync(string text, CancellationToken ct)
     {
         var threshold = agentOptions.Value.Summarization.Threshold;
@@ -26,24 +30,37 @@ public async Task<string> SummarizeAsync(string text, CancellationToken ct)
 
         try
         {
-            return await CallApiAsync(text, ct);
+            var summary = await CallApiAsync(text, ct);
+
+            if (string.IsNullOrWhiteSpace(summary))
+            {
+                logger.LogWarning("Yandex AI вернул пустой ответ. Используется обрезка текста.");
+                metrics.IncrementSummarizationFallback("empty_response");
+
+                return FallbackTruncate(text, threshold);
+            }
+
+            metrics.IncrementSummarization();
+            return summary;
         }
         catch (OperationCanceledException) when (ct.IsCancellationRequested)
         {
             throw;
         }
         catch (Exception ex)
         {
-            logger.LogWarning(ex, "Yandex AI суммаризация завершилась ошибкой ({Type}). Используется заглушка.", ex.GetType().Name);
+            logger.LogWarning(ex, "Yandex AI суммаризация завершилась ошибкой ({Type}). Используется обрезка текста.", ex.GetType().Name);
+            metrics.IncrementSummarizationFallback(ex.GetType().Name);
+
             return FallbackTruncate(text, threshold);
         }
     }
 
-    private async Task<string> CallApiAsync(string text, CancellationToken ct)
+    private async Task<string?> CallApiAsync(string text, CancellationToken ct)
     {
         var opts = yandexOptions.Value;
 
-        var requestBody = new YandexResponsesRequest { Model = $"gpt://{opts.FolderId}/{opts.ModelId}/latest", Instructions = "You are a concise summarizer. Summarize the given update in 2-3 sentences.", Input = text };
+        var requestBody = new YandexResponsesRequest { Model = $"gpt://{opts.FolderId}/{opts.ModelId}/latest", Instructions = Instructions, Input = text };
 
         var httpClient = httpClientFactory.CreateClient(nameof(YandexAiHttpClient));
 
@@ -57,15 +74,7 @@ private async Task<string> CallApiAsync(string text, CancellationToken ct)
 
         var result = await response.Content.ReadFromJsonAsync<YandexResponsesResponse>(ct);
 
-        var responseText = result?.Output?.FirstOrDefault()?.Content?.FirstOrDefault()?.Text;
-
-        if (!string.IsNullOrWhiteSpace(responseText))
-        {
-            return responseText;
-        }
-
-        logger.LogWarning("Yandex AI вернул пустой ответ.");
-        return text;
+        return result?.Output?.FirstOrDefault()?.Content?.FirstOrDefault()?.Text;
     }
 
     private static string FallbackTruncate(string text, int threshold)
@@ -84,4 +93,4 @@ private static string FallbackTruncate(string text, int threshold)
 
         return string.Concat(text.AsSpan(0, cutAt).TrimEnd(), "\n...");
     }
-}
+}

src/LinkTracker.AiAgent.Infrastructure/Configuration/AiAgent/AiAgentOptions.cs

@@ -2,9 +2,9 @@ namespace LinkTracker.AiAgent.Infrastructure.Configuration.AiAgent;
 
 public sealed class AiAgentOptions
 {
-    public FilteringOptions? Filtering { get; set; }
-    public SummarizationOptions? Summarization { get; set; }
-    public PrioritizationOptions? Prioritization { get; set; }
+    public FilteringOptions Filtering { get; set; } = new();
+    public SummarizationOptions Summarization { get; set; } = new();
+    public PrioritizationOptions Prioritization { get; set; } = new();
     public GroupingOptions Grouping { get; set; } = new();
 }
 

src/LinkTracker.AiAgent.Infrastructure/LinkTracker.AiAgent.Infrastructure.csproj

@@ -1,10 +1,4 @@
 <Project Sdk="Microsoft.NET.Sdk">
-    <PropertyGroup>
-        <TargetFramework>net9.0</TargetFramework>
-        <ImplicitUsings>enable</ImplicitUsings>
-        <Nullable>enable</Nullable>
-    </PropertyGroup>
-
     <ItemGroup>
         <ProjectReference Include="..\LinkTracker.AiAgent.Application\LinkTracker.AiAgent.Application.csproj"/>
         <ProjectReference Include="..\LinkTracker.Shared\LinkTracker.Shared.csproj"/>

src/LinkTracker.AiAgent.Infrastructure/Telemetry/AiAgentMetrics.cs

@@ -16,6 +16,8 @@ public sealed class AiAgentMetrics : IAiAgentMetrics, IDisposable
     private readonly Counter<long> _kafkaConsumeErrors;
     private readonly Counter<long> _kafkaDeadLetterErrors;
     private readonly Counter<long> _kafkaDeadLetters;
+    private readonly Counter<long> _summarizationFallbacks;
+    private readonly Counter<long> _summarizations;
 
     private readonly Meter _meter;
 
@@ -45,6 +47,14 @@ public AiAgentMetrics()
             "Длительность обработки сообщения из Kafka в миллисекундах",
             advice: new InstrumentAdvice<double> { HistogramBucketBoundaries = DurationBuckets });
 
+        _summarizations = _meter.CreateCounter<long>(
+            "summarizations_total",
+            description: "Количество успешных суммаризаций через Yandex AI");
+
+        _summarizationFallbacks = _meter.CreateCounter<long>(
+            "summarization_fallbacks_total",
+            description: "Количество суммаризаций, деградировавших до обрезки текста, с разбивкой по причине");
+
         _meter.CreateObservableGauge(
             "process_memory_working_set_bytes",
             static () => Process.GetCurrentProcess().WorkingSet64,
@@ -93,6 +103,18 @@ public void IncrementKafkaDeadLetterError(string topic)
             new KeyValuePair<string, object?>("topic", topic));
     }
 
+    public void IncrementSummarization()
+    {
+        _summarizations.Add(1);
+    }
+
+    public void IncrementSummarizationFallback(string reason)
+    {
+        _summarizationFallbacks.Add(
+            1,
+            new KeyValuePair<string, object?>("reason", reason));
+    }
+
     public void Dispose()
     {
         _meter.Dispose();

src/LinkTracker.Bot.Api/LinkTracker.Bot.Api.csproj

@@ -8,10 +8,4 @@
         <ProjectReference Include="..\LinkTracker.EnvReader\LinkTracker.EnvReader.csproj"/>
     </ItemGroup>
 
-    <PropertyGroup>
-        <TargetFramework>net9.0</TargetFramework>
-        <Nullable>enable</Nullable>
-        <ImplicitUsings>enable</ImplicitUsings>
-    </PropertyGroup>
-
 </Project>

src/LinkTracker.Bot.Api/Program.cs

@@ -29,7 +29,7 @@
 builder.Services.AddCommands();
 builder.Services.AddUpdateRouting();
 builder.Services.AddDialogs();
-builder.Services.AddDialogStorage();
+builder.Services.AddDialogStorage(builder.Configuration);
 builder.Services.AddClients(builder.Configuration);
 builder.Services.AddTelegramPresentation(builder.Configuration);
 builder.Services.AddTelegramNotifications();

src/LinkTracker.Bot.Api/appsettings.Docker.json

@@ -21,6 +21,12 @@
     "Serialization": "Json",
     "SchemaRegistryUrl": "http://schema-registry:8071"
   },
+  "Valkey": {
+    "Enabled": true,
+    "ConnectionString": "valkey-node-1:6379,valkey-node-2:6379,valkey-node-3:6379,password=valkey",
+    "InstanceName": "linktracker",
+    "DialogTtlSeconds": 3600
+  },
   "Resilience": {
     "Http": {
       "TimeoutMilliseconds": 1000,

src/LinkTracker.Bot.Api/appsettings.json

@@ -36,6 +36,12 @@
     "Serialization": "Json",
     "SchemaRegistryUrl": "http://localhost:8071"
   },
+  "Valkey": {
+    "Enabled": true,
+    "ConnectionString": "localhost:7379,localhost:7380,localhost:7381,password=valkey",
+    "InstanceName": "linktracker",
+    "DialogTtlSeconds": 3600
+  },
   "Resilience": {
     "Http": {
       "TimeoutMilliseconds": 1000,