Agent/complete phase 8

.env.prod.example

@@ -16,6 +16,15 @@ CONTROL_BIND=127.0.0.1:8080
 # [required] Unique high-entropy secrets owned by the named database/service.
 CONTROL_DB_PASSWORD=replace-with-a-unique-high-entropy-control-db-password
 REDIS_PASSWORD=replace-with-a-unique-high-entropy-valkey-password
+# [required for control and telemetry profiles] Absolute path to a mode-0600 file containing one random metrics bearer token.
+METRICS_TOKEN_FILE=/etc/cdnfoundry/secrets/metrics-token
+
+# [required] Encrypted off-host Restic repository. S3 credentials should permit only this backup prefix.
+RESTIC_REPOSITORY=s3:https://s3.example.com/cdnfoundry-control
+RESTIC_PASSWORD_FILE=/etc/cdnfoundry/secrets/restic-password
+BACKUP_ACCESS_KEY_ID=replace-with-backup-only-access-key
+BACKUP_SECRET_ACCESS_KEY=replace-with-backup-only-secret-key
+BACKUP_DEFAULT_REGION=us-east-1
 
 # [required] ACME account contact. Expiry and validation notices go to this address and the admin panel.
 ACME_CONTACT_EMAIL=cdn-operations@example.com

Makefile

@@ -2,7 +2,7 @@ COMPOSE_DEV := docker compose -f compose.dev.yml
 COMPOSE_PROD := docker compose --env-file .env.prod -f compose.prod.yml
 COMPOSE_PROD_EXAMPLE := docker compose --env-file .env.prod.example -f compose.prod.yml
 
-.PHONY: dev-assets dev-up dev-edge-up dev-edge-status dev-scale-up dev-down dev-migrate dev-pdns-migrate dev-test dev-e2e dev-phase7-e2e dev-scale-e2e dev-logs prod-pull prod-migrate prod-pdns-migrate prod-control prod-dns prod-telemetry prod-edge config-check openapi-check docs-check
+.PHONY: dev-assets dev-up dev-edge-up dev-edge-status dev-scale-up dev-down dev-migrate dev-pdns-migrate dev-test dev-e2e dev-phase7-e2e dev-phase8-e2e dev-phase8-recovery-e2e dev-phase8-upgrade-e2e dev-phase8-throughput-e2e dev-phase8-mmdb-e2e dev-scale-e2e dev-logs prod-pull prod-migrate prod-pdns-migrate prod-control prod-dns prod-telemetry prod-edge config-check openapi-check docs-check
 
 dev-assets:
 	docker build --target frontend-assets-export --output type=local,dest=./core/public/build ./core
@@ -41,6 +41,7 @@ dev-e2e:
 	python3 tests/e2e/phase5_tls.py
 	python3 tests/e2e/phase6_security.py
 	python3 tests/e2e/phase7_analytics.py
+	python3 tests/e2e/phase8_operations.py
 	python3 tests/e2e/phase4_runtime.py
 
 dev-scale-e2e:
@@ -49,6 +50,21 @@ dev-scale-e2e:
 dev-phase7-e2e:
 	python3 tests/e2e/phase7_analytics.py
 
+dev-phase8-e2e:
+	python3 tests/e2e/phase8_operations.py
+
+dev-phase8-recovery-e2e:
+	python3 tests/e2e/phase8_recovery.py
+
+dev-phase8-upgrade-e2e:
+	python3 tests/e2e/phase8_upgrade.py
+
+dev-phase8-throughput-e2e:
+	python3 tests/e2e/phase8_throughput.py
+
+dev-phase8-mmdb-e2e:
+	python3 tests/e2e/phase8_mmdb.py
+
 dev-logs:
 	$(COMPOSE_DEV) logs -f --tail=200
 

compose.dev.yml

@@ -28,6 +28,10 @@ x-core-env: &core-env
   CLICKHOUSE_DATABASE: cdnf
   CLICKHOUSE_USER: cdnf
   CLICKHOUSE_PASSWORD: cdnf-dev-only
+  PROMETHEUS_URL: http://prometheus:9090
+  METRICS_TOKEN_FILE: /run/dev-metrics/metrics-token
+  RESTIC_REPOSITORY: /app/storage/app/restic
+  RESTIC_PASSWORD_FILE: /run/dev-backup/restic-password
 
 x-core: &core
   build:
@@ -45,6 +49,8 @@ x-core: &core
     - core-bootstrap-cache:/app/bootstrap/cache
     - mmdb:/mmdb:ro
     - dev-pki:/run/dev-pki:ro
+    - ./docker/prometheus/dev-metrics-token:/run/dev-metrics/metrics-token:ro
+    - ./docker/backup/dev-restic-password:/run/dev-backup/restic-password:ro
   depends_on:
     dev-pki: { condition: service_completed_successfully }
     vendor-init: { condition: service_completed_successfully }
@@ -213,6 +219,11 @@ services:
       pdns-auth: { condition: service_healthy }
     networks: [dns]
     restart: unless-stopped
+    healthcheck:
+      test: [CMD, python3, -c, "import urllib.request; assert urllib.request.urlopen('http://127.0.0.1:8083/metrics', timeout=2).read(1)"]
+      interval: 10s
+      timeout: 3s
+      retries: 10
 
   pebble:
     image: ghcr.io/letsencrypt/pebble@sha256:ddf230642b1a584f519f32e347de1b05a6e4c1f6c35c1863b33effeab5f78199
@@ -261,8 +272,22 @@ services:
     volumes:
       - ./docker/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
       - ./docker/prometheus/telemetry-alerts.yml:/etc/prometheus/telemetry-alerts.yml:ro
+      - ./docker/prometheus/alerts.test.yml:/etc/prometheus/alerts.test.yml:ro
+      - ./docker/prometheus/dev-metrics-token:/run/secrets/metrics-token:ro
       - prometheus:/prometheus
+    networks: [telemetry, control, dns]
+
+  node-exporter:
+    image: prom/node-exporter:v1.10.2
+    command: [--path.procfs=/host/proc, --path.sysfs=/host/sys, --path.rootfs=/host/root, --collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)]
+    pid: host
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/host/root:ro,rslave
     networks: [telemetry]
+    read_only: true
+    restart: unless-stopped
 
   alertmanager:
     image: prom/alertmanager:v0.32.1
@@ -272,7 +297,7 @@ services:
 
   origin-http:
     image: nginx:1.30.3-alpine
-    command: [sh, -c, "dd if=/dev/zero of=/tmp/large-object bs=1048576 count=2 >/dev/null 2>&1 && exec nginx -g 'daemon off;'"]
+    command: [sh, -c, "dd if=/dev/zero of=/tmp/large-object bs=1048576 count=2 >/dev/null 2>&1 && dd if=/dev/zero of=/tmp/graceful-object bs=8192 count=1 >/dev/null 2>&1 && exec nginx -g 'daemon off;'"]
     tmpfs: [/tmp:size=4m]
     volumes: [./docker/nginx/origin.conf:/etc/nginx/conf.d/default.conf:ro]
     networks: [edge]

compose.prod.yml

@@ -30,6 +30,13 @@ x-core-env: &core-env
   CLICKHOUSE_DATABASE: cdnf
   CLICKHOUSE_USER: cdnf
   CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:?CLICKHOUSE_PASSWORD is required}
+  PROMETHEUS_URL: http://prometheus:9090
+  METRICS_TOKEN_FILE: /run/secrets/metrics-token
+  RESTIC_REPOSITORY: ${RESTIC_REPOSITORY:?RESTIC_REPOSITORY is required}
+  RESTIC_PASSWORD_FILE: /run/secrets/restic-password
+  BACKUP_ACCESS_KEY_ID: ${BACKUP_ACCESS_KEY_ID:?BACKUP_ACCESS_KEY_ID is required}
+  BACKUP_SECRET_ACCESS_KEY: ${BACKUP_SECRET_ACCESS_KEY:?BACKUP_SECRET_ACCESS_KEY is required}
+  BACKUP_DEFAULT_REGION: ${BACKUP_DEFAULT_REGION:-us-east-1}
 
 x-core: &core
   image: ghcr.io/vaheed/cdnfoundry-core:${CDNF_RELEASE:?CDNF_RELEASE must be an immutable commit SHA}
@@ -39,13 +46,16 @@ x-core: &core
     redis: { condition: service_healthy }
   networks: [control, telemetry]
   restart: unless-stopped
+  stop_grace_period: 60s
   read_only: true
   tmpfs: [/tmp]
   volumes:
     - core-storage:/app/storage
     - mmdb:/mmdb:ro
     - ${EDGE_IDENTITY_CA_CERTIFICATE:?EDGE_IDENTITY_CA_CERTIFICATE is required}:/run/secrets/edge-identity-ca.crt:ro
     - ${EDGE_IDENTITY_CA_PRIVATE_KEY:?EDGE_IDENTITY_CA_PRIVATE_KEY is required}:/run/secrets/edge-identity-ca.key:ro
+    - ${METRICS_TOKEN_FILE:?METRICS_TOKEN_FILE is required}:/run/secrets/metrics-token:ro
+    - ${RESTIC_PASSWORD_FILE:?RESTIC_PASSWORD_FILE is required}:/run/secrets/restic-password:ro
 
 services:
   core:
@@ -65,6 +75,8 @@ services:
       core: { condition: service_healthy }
     networks: [control, ingress]
     restart: unless-stopped
+    stop_signal: SIGQUIT
+    stop_grace_period: 30s
     healthcheck:
       test: [CMD, wget, -qO-, http://127.0.0.1:8080/api/health]
       interval: 10s
@@ -83,18 +95,22 @@ services:
       core: { condition: service_healthy }
     networks: [control, ingress]
     restart: unless-stopped
+    stop_signal: SIGQUIT
+    stop_grace_period: 30s
     read_only: true
     tmpfs: [/var/cache/nginx, /var/run]
 
   horizon:
     <<: *core
     profiles: [control]
     command: [php, artisan, horizon]
+    stop_grace_period: 120s
 
   scheduler:
     <<: *core
     profiles: [control]
     command: [php, artisan, schedule:work]
+    stop_grace_period: 30s
 
   migrate:
     <<: *core
@@ -192,6 +208,12 @@ services:
       pdns-auth: { condition: service_healthy }
     networks: [dns-private]
     restart: unless-stopped
+    stop_grace_period: 30s
+    healthcheck:
+      test: [CMD, python3, -c, "import urllib.request; assert urllib.request.urlopen('http://127.0.0.1:8083/metrics', timeout=2).read(1)"]
+      interval: 10s
+      timeout: 3s
+      retries: 10
 
   clickhouse:
     image: clickhouse/clickhouse-server:26.3.12.3-alpine
@@ -226,8 +248,22 @@ services:
     volumes:
       - ./docker/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
       - ./docker/prometheus/telemetry-alerts.yml:/etc/prometheus/telemetry-alerts.yml:ro
+      - ${METRICS_TOKEN_FILE:?METRICS_TOKEN_FILE is required}:/run/secrets/metrics-token:ro
       - prometheus:/prometheus
+    networks: [telemetry, control, dns-private]
+    restart: unless-stopped
+
+  node-exporter:
+    image: prom/node-exporter:v1.10.2
+    profiles: [telemetry]
+    command: [--path.procfs=/host/proc, --path.sysfs=/host/sys, --path.rootfs=/host/root, --collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)]
+    pid: host
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/host/root:ro,rslave
     networks: [telemetry]
+    read_only: true
     restart: unless-stopped
 
   alertmanager:
@@ -262,6 +298,8 @@ services:
       - /usr/local/openresty/nginx/logs:rw,noexec,nosuid,size=16m
     networks: [edge, telemetry]
     restart: unless-stopped
+    stop_signal: SIGQUIT
+    stop_grace_period: 30s
     mem_limit: 2g
     cpus: 2
     pids_limit: 256
@@ -299,6 +337,8 @@ services:
       - /usr/local/openresty/nginx/logs:rw,noexec,nosuid,size=16m
     networks: [edge, telemetry]
     restart: unless-stopped
+    stop_signal: SIGQUIT
+    stop_grace_period: 30s
     mem_limit: 512m
     cpus: 0.5
     pids_limit: 128
@@ -329,6 +369,7 @@ services:
       - ${EDGE_CONTROL_CA_CERTIFICATE:?EDGE_CONTROL_CA_CERTIFICATE is required}:/run/secrets/edge-control-ca.crt:ro
     networks: [edge]
     restart: unless-stopped
+    stop_grace_period: 30s
     read_only: true
     tmpfs: [/tmp]
     mem_limit: 128m

core/Dockerfile

@@ -1,6 +1,6 @@
 FROM php:8.5-fpm-alpine AS php-dependencies
 
-RUN apk add --no-cache icu-libs libpq libzip libmaxminddb nginx supervisor su-exec \
+RUN apk add --no-cache icu-libs libpq libzip libmaxminddb nginx supervisor su-exec postgresql-client restic \
     && apk add --no-cache --virtual .build-deps $PHPIZE_DEPS icu-dev postgresql-dev libzip-dev \
     && docker-php-ext-install intl pcntl pdo_pgsql zip \
     && apk del .build-deps
@@ -40,7 +40,9 @@ RUN composer dump-autoload --no-dev --classmap-authoritative \
     && chown -R www-data:www-data storage bootstrap/cache
 
 COPY docker/php/entrypoint.sh /usr/local/bin/cdnf-entrypoint
-RUN chmod +x /usr/local/bin/cdnf-entrypoint
+COPY docker/backup/create.sh /usr/local/bin/cdnf-backup-create
+COPY docker/backup/restore.sh /usr/local/bin/cdnf-backup-restore
+RUN chmod +x /usr/local/bin/cdnf-entrypoint /usr/local/bin/cdnf-backup-create /usr/local/bin/cdnf-backup-restore
 
 ENTRYPOINT ["cdnf-entrypoint"]
 CMD ["php-fpm", "-F"]

core/app/Console/Commands/CreateControlBackup.php

@@ -0,0 +1,36 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Jobs\CreateControlBackup as CreateControlBackupJob;
+use App\Models\Backup;
+use App\Models\Operation;
+use App\Support\ResticBackupRepository;
+use Illuminate\Console\Command;
+
+class CreateControlBackup extends Command
+{
+    protected $signature = 'backups:create {--wait : Run in this process and wait for repository acknowledgement}';
+
+    protected $description = 'Create an encrypted control PostgreSQL backup in the configured Restic repository';
+
+    public function handle(ResticBackupRepository $repository): int
+    {
+        if (! $repository->configured()) {
+            $this->error('Encrypted off-host backup repository is not configured.');
+
+            return self::FAILURE;
+        }
+        $backup = Backup::query()->create(['status' => 'pending']);
+        $operation = Operation::query()->create(['type' => 'backup.create', 'status' => 'pending', 'input' => ['backup_id' => $backup->id]]);
+        if ($this->option('wait')) {
+            CreateControlBackupJob::dispatchSync($backup->id, $operation->id);
+        } else {
+            CreateControlBackupJob::dispatch($backup->id, $operation->id);
+        }
+        $backup->refresh();
+        $this->line(json_encode(['backup_id' => $backup->id, 'operation_id' => $operation->id, 'status' => $backup->status, 'snapshot_id' => $backup->snapshot_id, 'verified_at' => $backup->verified_at?->toIso8601String()], JSON_THROW_ON_ERROR));
+
+        return $backup->status === 'failed' ? self::FAILURE : self::SUCCESS;
+    }
+}

core/app/Console/Commands/GenerateOpenApi.php

@@ -165,9 +165,10 @@ private function pathParameters(string $path): array
 
         return collect($matches[1])->map(function (string $name): array {
             $schema = match ($name) {
-                'operation', 'edge', 'purge' => ['type' => 'string', 'format' => 'uuid'],
+                'operation', 'edge', 'purge', 'backup', 'job' => ['type' => 'string', 'format' => 'uuid'],
                 'checksum' => ['type' => 'string', 'pattern' => '^[a-f0-9]{64}$'],
                 'group' => ['type' => 'string', 'enum' => array_keys(config('platform.groups', []))],
+                'scope' => ['type' => 'string', 'enum' => ['dns', 'edges', 'tls', 'purges', 'usage']],
                 default => ['type' => 'integer', 'minimum' => 1],
             };
 

core/app/Console/Commands/PruneAuditLogs.php

@@ -0,0 +1,24 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\AuditLog;
+use App\Support\PlatformSettings;
+use Illuminate\Console\Command;
+
+class PruneAuditLogs extends Command
+{
+    protected $signature = 'audit:prune {--batch=1000}';
+
+    protected $description = 'Delete one bounded batch of audit events beyond the configured retention window';
+
+    public function handle(PlatformSettings $settings): int
+    {
+        $batch = max(1, min(10000, (int) $this->option('batch')));
+        $ids = AuditLog::query()->where('created_at', '<', now()->subDays($settings->integer('operations', 'audit_retention_days')))->orderBy('id')->limit($batch)->pluck('id');
+        $deleted = $ids->isEmpty() ? 0 : AuditLog::query()->whereIn('id', $ids)->delete();
+        $this->info("Deleted {$deleted} expired audit events.");
+
+        return self::SUCCESS;
+    }
+}