From 671e37328e0050d259f60e61942d265333dcf667 Mon Sep 17 00:00:00 2001 From: Echways Date: Wed, 19 Aug 2026 20:02:38 +0300 Subject: [PATCH] repo cleanup --- .gitlab-ci.yml | 119 ------------------ README.md | 20 ++- .env.template => deploy/.env.template | 0 .../docker-compose.apps.yml | 16 +-- .../docker-compose.yml | 2 + .../grafana/provisioning/alerting/rules.yml | 0 .../provisioning/dashboards/dashboards.yml | 0 .../dashboards/linktracker-dashboard.json | 0 .../provisioning/datasources/prometheus.yml | 0 .../monitoring}/prometheus.yml | 0 OBSERVABILITY.md => docs/OBSERVABILITY.md | 14 +-- example-pql.txt => docs/example-pql.txt | 0 .../Directory.Build.props | 0 src/LinkTracker.AiAgent.Api/Dockerfile | 1 + src/LinkTracker.Bot.Api/Dockerfile | 1 + src/LinkTracker.Scrapper.Api/Dockerfile | 1 + 16 files changed, 36 insertions(+), 138 deletions(-) delete mode 100644 .gitlab-ci.yml rename .env.template => deploy/.env.template (100%) rename docker-compose.apps.yml => deploy/docker-compose.apps.yml (88%) rename docker-compose.yml => deploy/docker-compose.yml (99%) rename {monitoring => deploy/monitoring}/grafana/provisioning/alerting/rules.yml (100%) rename {monitoring => deploy/monitoring}/grafana/provisioning/dashboards/dashboards.yml (100%) rename {monitoring => deploy/monitoring}/grafana/provisioning/dashboards/linktracker-dashboard.json (100%) rename {monitoring => deploy/monitoring}/grafana/provisioning/datasources/prometheus.yml (100%) rename {monitoring => deploy/monitoring}/prometheus.yml (100%) rename OBSERVABILITY.md => docs/OBSERVABILITY.md (92%) rename example-pql.txt => docs/example-pql.txt (100%) rename Directory.Build.props => src/Directory.Build.props (100%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 88458f2..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,119 +0,0 @@ -image: mcr.microsoft.com/dotnet/sdk:9.0 - -stages: - - build - - validate - - test - -variables: - GITLAB_DOCKER_PROXY: gitlab.education.tbank.ru:443/backend-academy-dotnet-2025/lecture-materials/dependency_proxy/containers - NUGET_PACKAGES: "$CI_PROJECT_DIR/.nuget/packages" - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: "1" - DOTNET_NOLOGO: "1" - -workflow: - rules: - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "master"' - when: always - - when: never - -cache: - key: "nuget-$CI_COMMIT_REF_SLUG" - paths: - - .nuget/packages/ - policy: pull-push - -.default_dotnet_job: - before_script: - - dotnet --info - - dotnet restore - -build: - stage: build - extends: .default_dotnet_job - script: - - dotnet build --no-restore - -linter: - stage: validate - extends: .default_dotnet_job - script: - - dotnet format --verify-no-changes --no-restore - -.test_job: - stage: test - extends: .default_dotnet_job - needs: - - job: build - artifacts: false - script: - - dotnet test --no-restore --filter "$TEST_FILTER" - -bot_unit_test: - extends: .test_job - variables: - TEST_FILTER: "Module=Bot&Category=Unit" - -scrapper_unit_test: - extends: .test_job - variables: - TEST_FILTER: "Module=Scrapper&Category=Unit" - -shared_unit_test: - extends: .test_job - variables: - TEST_FILTER: "Module=Shared&Category=Unit" - -aiagent_unit_test: - extends: .test_job - variables: - TEST_FILTER: "Module=AiAgent&Category=Unit" - -shared_integration_test: - extends: .test_job - variables: - TEST_FILTER: "Module=Shared&Category=Integration" - -.dind_test_job: - extends: .test_job - tags: - - dind - services: - - name: docker:29.0.1-dind - alias: docker - variables: - DOCKER_HOST: tcp://docker:2376 - DOCKER_TLS_CERTDIR: "/certs" - DOCKER_TLS_VERIFY: "1" - DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client" - TESTCONTAINERS_RYUK_DISABLED: "true" - before_script: - - dotnet --info - - dotnet restore - - apt-get update - - apt-get install -y ca-certificates curl gnupg - - install -m 0755 -d /etc/apt/keyrings - - curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc - - chmod a+r /etc/apt/keyrings/docker.asc - - . /etc/os-release - - echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian ${VERSION_CODENAME} stable" > /etc/apt/sources.list.d/docker.list - - apt-get update - - apt-get install -y docker-ce-cli - - echo "$CI_JOB_TOKEN" | docker login gitlab.education.tbank.ru:443 -u gitlab-ci-token --password-stdin - - docker version - -bot_integration_test: - extends: .dind_test_job - variables: - TEST_FILTER: "Module=Bot&Category=Integration" - -scrapper_integration_test: - extends: .dind_test_job - variables: - TEST_FILTER: "Module=Scrapper&Category=Integration" - - -aiagent_integration_test: - extends: .dind_test_job - variables: - TEST_FILTER: "Module=AiAgent&Category=Integration" \ No newline at end of file diff --git a/README.md b/README.md index e82d2c7..ef197a5 100644 --- a/README.md +++ b/README.md @@ -69,17 +69,17 @@ cp src/LinkTracker.AiAgent.Api/.env.template src/LinkTracker.AiAgent.Api/.env 7. Задать общий сервисный секрет. Bot и Scrapper аутентифицируют друг друга по нему, поэтому значение должно совпадать в обоих сервисах. ``` -cp .env.template .env +cp deploy/.env.template deploy/.env ``` -8. Поставить в корневой .env своё значение `SERVICE_AUTH_SECRET`, а в +8. Поставить в `deploy/.env` своё значение `SERVICE_AUTH_SECRET`, а в `src/LinkTracker.Bot.Api/.env` и `src/LinkTracker.Scrapper.Api/.env` — то же самое значение в `ServiceAuth__Secret` (нужно для локального запуска без Docker). ### Для локального запуска 9. В [bot.Api.appsettings](src/LinkTracker.Bot.Api/appsettings.json), [scrapper.Api.appsettings](src/LinkTracker.Scrapper.Api/appsettings.json) и [aiagent.Api.appsettings](src/LinkTracker.AiAgent.Api/appsettings.json) в ветках Scrapper, Bot и AiAgent соответственно выбрать валидные параметры. -10. Выполнить поочердено запуск сначала [docker-compose](docker-compose.yml), [Scrapper](src/LinkTracker.Scrapper.Api/Program.cs), [Bot](src/LinkTracker.Bot.Api/Program.cs) и [AiAgent](src/LinkTracker.AiAgent.Api/Program.cs) с помощью команд. +10. Выполнить поочердено запуск сначала [docker-compose](deploy/docker-compose.yml), [Scrapper](src/LinkTracker.Scrapper.Api/Program.cs), [Bot](src/LinkTracker.Bot.Api/Program.cs) и [AiAgent](src/LinkTracker.AiAgent.Api/Program.cs) с помощью команд. ``` -docker compose -f docker-compose.yml up +docker compose -f deploy/docker-compose.yml up dotnet run --project src/LinkTracker.Scrapper.Api dotnet run --project src/LinkTracker.Bot.Api dotnet run --project src/LinkTracker.AiAgent.Api @@ -89,7 +89,7 @@ dotnet run --project src/LinkTracker.AiAgent.Api 9. В [bot.Docker.appesettings](src/LinkTracker.Bot.Api/appsettings.Docker.json), [scrapper.Docker.appsettings](src/LinkTracker.Scrapper.Api/appsettings.Docker.json) и [aiagent.Docker.appesettings](src/LinkTracker.AiAgent.Api/appsettings.Docker.json) в ветках Scrapper, Bot и AiAgent соответственно выбрать валидные параметры. 10. Поднять инфраструктуру и все три сервиса одной командой — порядок запуска обеспечивают `depends_on` в compose-файлах. ``` -docker compose -f docker-compose.yml -f docker-compose.apps.yml up +docker compose -f deploy/docker-compose.yml -f deploy/docker-compose.apps.yml up ``` @@ -116,3 +116,13 @@ dotnet test | p99 | 283.45 ms | 19.76 ms | -93.03% | | Успешные ответы 200 | 25 507 | 29 482 | +3 975 | | Ошибки 500/502/504 | 0 | 0 | без изменений | + +--- +## Структура репозитория + +| Путь | Содержимое | +|---|---| +| [src/](src/) | Исходный код сервисов, общие библиотеки и тесты | +| [migrations/](migrations/) | SQL-миграции схемы Postgres (применяются DbUp при старте Scrapper) | +| [deploy/](deploy/) | Compose-файлы, конфигурация Prometheus и Grafana, шаблон корневого `.env` | +| [docs/](docs/) | [Наблюдаемость](docs/OBSERVABILITY.md) и [примеры PromQL](docs/example-pql.txt) | diff --git a/.env.template b/deploy/.env.template similarity index 100% rename from .env.template rename to deploy/.env.template diff --git a/docker-compose.apps.yml b/deploy/docker-compose.apps.yml similarity index 88% rename from docker-compose.apps.yml rename to deploy/docker-compose.apps.yml index 80606a6..125dbe2 100644 --- a/docker-compose.apps.yml +++ b/deploy/docker-compose.apps.yml @@ -1,10 +1,12 @@ +name: linktracker + x-service-auth-env: &service-auth-env - ServiceAuth__Secret: "${SERVICE_AUTH_SECRET:?SERVICE_AUTH_SECRET must be set in the root .env file}" + ServiceAuth__Secret: "${SERVICE_AUTH_SECRET:?SERVICE_AUTH_SECRET must be set in deploy/.env}" services: scrapper: build: - context: . + context: .. dockerfile: src/LinkTracker.Scrapper.Api/Dockerfile network: host container_name: linktracker-scrapper @@ -22,14 +24,14 @@ services: - "8081" - "8082" env_file: - - src/LinkTracker.Scrapper.Api/.env + - ../src/LinkTracker.Scrapper.Api/.env environment: <<: *service-auth-env ASPNETCORE_ENVIRONMENT: "Docker" bot: build: - context: . + context: .. dockerfile: src/LinkTracker.Bot.Api/Dockerfile network: host container_name: linktracker-bot @@ -48,14 +50,14 @@ services: - "8092" - "8011" env_file: - - src/LinkTracker.Bot.Api/.env + - ../src/LinkTracker.Bot.Api/.env environment: <<: *service-auth-env ASPNETCORE_ENVIRONMENT: "Docker" aiagent: build: - context: . + context: .. dockerfile: src/LinkTracker.AiAgent.Api/Dockerfile network: host container_name: linktracker-aiagent @@ -75,6 +77,6 @@ services: - "8101" - "8102" env_file: - - src/LinkTracker.AiAgent.Api/.env + - ../src/LinkTracker.AiAgent.Api/.env environment: ASPNETCORE_ENVIRONMENT: "Docker" diff --git a/docker-compose.yml b/deploy/docker-compose.yml similarity index 99% rename from docker-compose.yml rename to deploy/docker-compose.yml index b21a14c..0143929 100644 --- a/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -1,3 +1,5 @@ +name: linktracker + x-kafka-common-env: &kafka-common-env KAFKA_PROCESS_ROLES: "broker,controller" KAFKA_CONTROLLER_LISTENER_NAMES: "CONTROLLER" diff --git a/monitoring/grafana/provisioning/alerting/rules.yml b/deploy/monitoring/grafana/provisioning/alerting/rules.yml similarity index 100% rename from monitoring/grafana/provisioning/alerting/rules.yml rename to deploy/monitoring/grafana/provisioning/alerting/rules.yml diff --git a/monitoring/grafana/provisioning/dashboards/dashboards.yml b/deploy/monitoring/grafana/provisioning/dashboards/dashboards.yml similarity index 100% rename from monitoring/grafana/provisioning/dashboards/dashboards.yml rename to deploy/monitoring/grafana/provisioning/dashboards/dashboards.yml diff --git a/monitoring/grafana/provisioning/dashboards/linktracker-dashboard.json b/deploy/monitoring/grafana/provisioning/dashboards/linktracker-dashboard.json similarity index 100% rename from monitoring/grafana/provisioning/dashboards/linktracker-dashboard.json rename to deploy/monitoring/grafana/provisioning/dashboards/linktracker-dashboard.json diff --git a/monitoring/grafana/provisioning/datasources/prometheus.yml b/deploy/monitoring/grafana/provisioning/datasources/prometheus.yml similarity index 100% rename from monitoring/grafana/provisioning/datasources/prometheus.yml rename to deploy/monitoring/grafana/provisioning/datasources/prometheus.yml diff --git a/monitoring/prometheus.yml b/deploy/monitoring/prometheus.yml similarity index 100% rename from monitoring/prometheus.yml rename to deploy/monitoring/prometheus.yml diff --git a/OBSERVABILITY.md b/docs/OBSERVABILITY.md similarity index 92% rename from OBSERVABILITY.md rename to docs/OBSERVABILITY.md index 7170840..873f9ad 100644 --- a/OBSERVABILITY.md +++ b/docs/OBSERVABILITY.md @@ -77,7 +77,7 @@ Rate limiter применяется точечно, политикой `public-a ## Конфигурация сбора -Настраивается на стороне Prometheus в `monitoring/prometheus.yml` — приложениям +Настраивается на стороне Prometheus в `deploy/monitoring/prometheus.yml` — приложениям никакой конфигурации телеметрии не требуется. Лейбл `job` берётся из имени scrape-job (`scrapper` / `bot` / `aiagent`), `instance` — из адреса цели. @@ -89,27 +89,27 @@ Rate limiter применяется точечно, политикой `public-a ## Grafana -Дашборд и алерты провижинятся автоматически из `monitoring/grafana/provisioning`: +Дашборд и алерты провижинятся автоматически из `deploy/monitoring/grafana/provisioning`: - **Datasource**: Prometheus (`uid: prometheus`). -- **Dashboard**: `LinkTracker Observability` (`monitoring/.../dashboards/linktracker-dashboard.json`) — +- **Dashboard**: `LinkTracker Observability` (`deploy/monitoring/.../dashboards/linktracker-dashboard.json`) — панели RED, БД, Kafka, RAM и бизнес-метрики. - **Alert (требование задания)**: правило `High process memory usage` - (`monitoring/.../alerting/rules.yaml`) на метрику `process_memory_working_set_bytes`. + (`deploy/monitoring/.../alerting/rules.yml`) на метрику `process_memory_working_set_bytes`. Срабатывает, когда RAM сервиса (`scrapper`/`bot`) превышает 500 MB дольше 2 минут. ## PromQL запросы -Примеры PromQL для визуализаций — в `example-pql.txt`. +Примеры PromQL для визуализаций — в `docs/example-pql.txt`. ## Запуск мониторинга ```bash # Поднять инфраструктуру (Kafka, Postgres, Valkey, Prometheus, Grafana) -docker compose -f docker-compose.yml up -d +docker compose -f deploy/docker-compose.yml up -d # Поднять приложения -docker compose -f docker-compose.yml -f docker-compose.apps.yml up -d +docker compose -f deploy/docker-compose.yml -f deploy/docker-compose.apps.yml up -d ``` После запуска: diff --git a/example-pql.txt b/docs/example-pql.txt similarity index 100% rename from example-pql.txt rename to docs/example-pql.txt diff --git a/Directory.Build.props b/src/Directory.Build.props similarity index 100% rename from Directory.Build.props rename to src/Directory.Build.props diff --git a/src/LinkTracker.AiAgent.Api/Dockerfile b/src/LinkTracker.AiAgent.Api/Dockerfile index 4c5e24a..15f2548 100644 --- a/src/LinkTracker.AiAgent.Api/Dockerfile +++ b/src/LinkTracker.AiAgent.Api/Dockerfile @@ -2,6 +2,7 @@ FROM mcr.microsoft.com/dotnet/sdk:9.0 AS build WORKDIR /src COPY ["linktracker.sln", "./"] +COPY ["src/Directory.Build.props", "src/"] COPY ["src/LinkTracker.AiAgent.Api", "src/LinkTracker.AiAgent.Api/"] COPY ["src/LinkTracker.AiAgent.Application/LinkTracker.AiAgent.Application.csproj", "src/LinkTracker.AiAgent.Application/"] diff --git a/src/LinkTracker.Bot.Api/Dockerfile b/src/LinkTracker.Bot.Api/Dockerfile index b296653..e191f18 100644 --- a/src/LinkTracker.Bot.Api/Dockerfile +++ b/src/LinkTracker.Bot.Api/Dockerfile @@ -2,6 +2,7 @@ FROM mcr.microsoft.com/dotnet/sdk:9.0 AS build WORKDIR /src COPY ["linktracker.sln", "./"] +COPY ["src/Directory.Build.props", "src/"] COPY ["src/LinkTracker.Bot.Api/LinkTracker.Bot.Api.csproj", "src/LinkTracker.Bot.Api/"] COPY ["src/LinkTracker.Bot.Application/LinkTracker.Bot.Application.csproj", "src/LinkTracker.Bot.Application/"] diff --git a/src/LinkTracker.Scrapper.Api/Dockerfile b/src/LinkTracker.Scrapper.Api/Dockerfile index 0396a81..9ec0fda 100644 --- a/src/LinkTracker.Scrapper.Api/Dockerfile +++ b/src/LinkTracker.Scrapper.Api/Dockerfile @@ -2,6 +2,7 @@ FROM mcr.microsoft.com/dotnet/sdk:9.0 AS build WORKDIR /src COPY ["linktracker.sln", "./"] +COPY ["src/Directory.Build.props", "src/"] COPY ["src/LinkTracker.Scrapper.Api/LinkTracker.Scrapper.Api.csproj", "src/LinkTracker.Scrapper.Api/"] COPY ["src/LinkTracker.Scrapper.Application/LinkTracker.Scrapper.Application.csproj", "src/LinkTracker.Scrapper.Application/"]