diff --git a/.env.prod.example b/.env.prod.example index 9e72b22..af21fee 100644 --- a/.env.prod.example +++ b/.env.prod.example @@ -16,6 +16,15 @@ CONTROL_BIND=127.0.0.1:8080 # [required] Unique high-entropy secrets owned by the named database/service. CONTROL_DB_PASSWORD=replace-with-a-unique-high-entropy-control-db-password REDIS_PASSWORD=replace-with-a-unique-high-entropy-valkey-password +# [required for control and telemetry profiles] Absolute path to a mode-0600 file containing one random metrics bearer token. +METRICS_TOKEN_FILE=/etc/cdnfoundry/secrets/metrics-token + +# [required] Encrypted off-host Restic repository. S3 credentials should permit only this backup prefix. +RESTIC_REPOSITORY=s3:https://s3.example.com/cdnfoundry-control +RESTIC_PASSWORD_FILE=/etc/cdnfoundry/secrets/restic-password +BACKUP_ACCESS_KEY_ID=replace-with-backup-only-access-key +BACKUP_SECRET_ACCESS_KEY=replace-with-backup-only-secret-key +BACKUP_DEFAULT_REGION=us-east-1 # [required] ACME account contact. Expiry and validation notices go to this address and the admin panel. ACME_CONTACT_EMAIL=cdn-operations@example.com diff --git a/Makefile b/Makefile index 57f8b55..0f456dd 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ COMPOSE_DEV := docker compose -f compose.dev.yml COMPOSE_PROD := docker compose --env-file .env.prod -f compose.prod.yml COMPOSE_PROD_EXAMPLE := docker compose --env-file .env.prod.example -f compose.prod.yml -.PHONY: dev-assets dev-up dev-edge-up dev-edge-status dev-scale-up dev-down dev-migrate dev-pdns-migrate dev-test dev-e2e dev-phase7-e2e dev-scale-e2e dev-logs prod-pull prod-migrate prod-pdns-migrate prod-control prod-dns prod-telemetry prod-edge config-check openapi-check docs-check +.PHONY: dev-assets dev-up dev-edge-up dev-edge-status dev-scale-up dev-down dev-migrate dev-pdns-migrate dev-test dev-e2e dev-phase7-e2e dev-phase8-e2e dev-phase8-recovery-e2e dev-phase8-upgrade-e2e dev-phase8-throughput-e2e dev-phase8-mmdb-e2e dev-scale-e2e dev-logs prod-pull prod-migrate prod-pdns-migrate prod-control prod-dns prod-telemetry prod-edge config-check openapi-check docs-check dev-assets: docker build --target frontend-assets-export --output type=local,dest=./core/public/build ./core @@ -41,6 +41,7 @@ dev-e2e: python3 tests/e2e/phase5_tls.py python3 tests/e2e/phase6_security.py python3 tests/e2e/phase7_analytics.py + python3 tests/e2e/phase8_operations.py python3 tests/e2e/phase4_runtime.py dev-scale-e2e: @@ -49,6 +50,21 @@ dev-scale-e2e: dev-phase7-e2e: python3 tests/e2e/phase7_analytics.py +dev-phase8-e2e: + python3 tests/e2e/phase8_operations.py + +dev-phase8-recovery-e2e: + python3 tests/e2e/phase8_recovery.py + +dev-phase8-upgrade-e2e: + python3 tests/e2e/phase8_upgrade.py + +dev-phase8-throughput-e2e: + python3 tests/e2e/phase8_throughput.py + +dev-phase8-mmdb-e2e: + python3 tests/e2e/phase8_mmdb.py + dev-logs: $(COMPOSE_DEV) logs -f --tail=200 diff --git a/compose.dev.yml b/compose.dev.yml index 60ebf19..1e0f718 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -28,6 +28,10 @@ x-core-env: &core-env CLICKHOUSE_DATABASE: cdnf CLICKHOUSE_USER: cdnf CLICKHOUSE_PASSWORD: cdnf-dev-only + PROMETHEUS_URL: http://prometheus:9090 + METRICS_TOKEN_FILE: /run/dev-metrics/metrics-token + RESTIC_REPOSITORY: /app/storage/app/restic + RESTIC_PASSWORD_FILE: /run/dev-backup/restic-password x-core: &core build: @@ -45,6 +49,8 @@ x-core: &core - core-bootstrap-cache:/app/bootstrap/cache - mmdb:/mmdb:ro - dev-pki:/run/dev-pki:ro + - ./docker/prometheus/dev-metrics-token:/run/dev-metrics/metrics-token:ro + - ./docker/backup/dev-restic-password:/run/dev-backup/restic-password:ro depends_on: dev-pki: { condition: service_completed_successfully } vendor-init: { condition: service_completed_successfully } @@ -213,6 +219,11 @@ services: pdns-auth: { condition: service_healthy } networks: [dns] restart: unless-stopped + healthcheck: + test: [CMD, python3, -c, "import urllib.request; assert urllib.request.urlopen('http://127.0.0.1:8083/metrics', timeout=2).read(1)"] + interval: 10s + timeout: 3s + retries: 10 pebble: image: ghcr.io/letsencrypt/pebble@sha256:ddf230642b1a584f519f32e347de1b05a6e4c1f6c35c1863b33effeab5f78199 @@ -261,8 +272,22 @@ services: volumes: - ./docker/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro - ./docker/prometheus/telemetry-alerts.yml:/etc/prometheus/telemetry-alerts.yml:ro + - ./docker/prometheus/alerts.test.yml:/etc/prometheus/alerts.test.yml:ro + - ./docker/prometheus/dev-metrics-token:/run/secrets/metrics-token:ro - prometheus:/prometheus + networks: [telemetry, control, dns] + + node-exporter: + image: prom/node-exporter:v1.10.2 + command: [--path.procfs=/host/proc, --path.sysfs=/host/sys, --path.rootfs=/host/root, --collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)] + pid: host + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/host/root:ro,rslave networks: [telemetry] + read_only: true + restart: unless-stopped alertmanager: image: prom/alertmanager:v0.32.1 @@ -272,7 +297,7 @@ services: origin-http: image: nginx:1.30.3-alpine - command: [sh, -c, "dd if=/dev/zero of=/tmp/large-object bs=1048576 count=2 >/dev/null 2>&1 && exec nginx -g 'daemon off;'"] + command: [sh, -c, "dd if=/dev/zero of=/tmp/large-object bs=1048576 count=2 >/dev/null 2>&1 && dd if=/dev/zero of=/tmp/graceful-object bs=8192 count=1 >/dev/null 2>&1 && exec nginx -g 'daemon off;'"] tmpfs: [/tmp:size=4m] volumes: [./docker/nginx/origin.conf:/etc/nginx/conf.d/default.conf:ro] networks: [edge] diff --git a/compose.prod.yml b/compose.prod.yml index 215c477..c9e7067 100644 --- a/compose.prod.yml +++ b/compose.prod.yml @@ -30,6 +30,13 @@ x-core-env: &core-env CLICKHOUSE_DATABASE: cdnf CLICKHOUSE_USER: cdnf CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:?CLICKHOUSE_PASSWORD is required} + PROMETHEUS_URL: http://prometheus:9090 + METRICS_TOKEN_FILE: /run/secrets/metrics-token + RESTIC_REPOSITORY: ${RESTIC_REPOSITORY:?RESTIC_REPOSITORY is required} + RESTIC_PASSWORD_FILE: /run/secrets/restic-password + BACKUP_ACCESS_KEY_ID: ${BACKUP_ACCESS_KEY_ID:?BACKUP_ACCESS_KEY_ID is required} + BACKUP_SECRET_ACCESS_KEY: ${BACKUP_SECRET_ACCESS_KEY:?BACKUP_SECRET_ACCESS_KEY is required} + BACKUP_DEFAULT_REGION: ${BACKUP_DEFAULT_REGION:-us-east-1} x-core: &core image: ghcr.io/vaheed/cdnfoundry-core:${CDNF_RELEASE:?CDNF_RELEASE must be an immutable commit SHA} @@ -39,6 +46,7 @@ x-core: &core redis: { condition: service_healthy } networks: [control, telemetry] restart: unless-stopped + stop_grace_period: 60s read_only: true tmpfs: [/tmp] volumes: @@ -46,6 +54,8 @@ x-core: &core - mmdb:/mmdb:ro - ${EDGE_IDENTITY_CA_CERTIFICATE:?EDGE_IDENTITY_CA_CERTIFICATE is required}:/run/secrets/edge-identity-ca.crt:ro - ${EDGE_IDENTITY_CA_PRIVATE_KEY:?EDGE_IDENTITY_CA_PRIVATE_KEY is required}:/run/secrets/edge-identity-ca.key:ro + - ${METRICS_TOKEN_FILE:?METRICS_TOKEN_FILE is required}:/run/secrets/metrics-token:ro + - ${RESTIC_PASSWORD_FILE:?RESTIC_PASSWORD_FILE is required}:/run/secrets/restic-password:ro services: core: @@ -65,6 +75,8 @@ services: core: { condition: service_healthy } networks: [control, ingress] restart: unless-stopped + stop_signal: SIGQUIT + stop_grace_period: 30s healthcheck: test: [CMD, wget, -qO-, http://127.0.0.1:8080/api/health] interval: 10s @@ -83,6 +95,8 @@ services: core: { condition: service_healthy } networks: [control, ingress] restart: unless-stopped + stop_signal: SIGQUIT + stop_grace_period: 30s read_only: true tmpfs: [/var/cache/nginx, /var/run] @@ -90,11 +104,13 @@ services: <<: *core profiles: [control] command: [php, artisan, horizon] + stop_grace_period: 120s scheduler: <<: *core profiles: [control] command: [php, artisan, schedule:work] + stop_grace_period: 30s migrate: <<: *core @@ -192,6 +208,12 @@ services: pdns-auth: { condition: service_healthy } networks: [dns-private] restart: unless-stopped + stop_grace_period: 30s + healthcheck: + test: [CMD, python3, -c, "import urllib.request; assert urllib.request.urlopen('http://127.0.0.1:8083/metrics', timeout=2).read(1)"] + interval: 10s + timeout: 3s + retries: 10 clickhouse: image: clickhouse/clickhouse-server:26.3.12.3-alpine @@ -226,8 +248,22 @@ services: volumes: - ./docker/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro - ./docker/prometheus/telemetry-alerts.yml:/etc/prometheus/telemetry-alerts.yml:ro + - ${METRICS_TOKEN_FILE:?METRICS_TOKEN_FILE is required}:/run/secrets/metrics-token:ro - prometheus:/prometheus + networks: [telemetry, control, dns-private] + restart: unless-stopped + + node-exporter: + image: prom/node-exporter:v1.10.2 + profiles: [telemetry] + command: [--path.procfs=/host/proc, --path.sysfs=/host/sys, --path.rootfs=/host/root, --collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)] + pid: host + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/host/root:ro,rslave networks: [telemetry] + read_only: true restart: unless-stopped alertmanager: @@ -262,6 +298,8 @@ services: - /usr/local/openresty/nginx/logs:rw,noexec,nosuid,size=16m networks: [edge, telemetry] restart: unless-stopped + stop_signal: SIGQUIT + stop_grace_period: 30s mem_limit: 2g cpus: 2 pids_limit: 256 @@ -299,6 +337,8 @@ services: - /usr/local/openresty/nginx/logs:rw,noexec,nosuid,size=16m networks: [edge, telemetry] restart: unless-stopped + stop_signal: SIGQUIT + stop_grace_period: 30s mem_limit: 512m cpus: 0.5 pids_limit: 128 @@ -329,6 +369,7 @@ services: - ${EDGE_CONTROL_CA_CERTIFICATE:?EDGE_CONTROL_CA_CERTIFICATE is required}:/run/secrets/edge-control-ca.crt:ro networks: [edge] restart: unless-stopped + stop_grace_period: 30s read_only: true tmpfs: [/tmp] mem_limit: 128m diff --git a/core/Dockerfile b/core/Dockerfile index 144aa4a..d01a666 100644 --- a/core/Dockerfile +++ b/core/Dockerfile @@ -1,6 +1,6 @@ FROM php:8.5-fpm-alpine AS php-dependencies -RUN apk add --no-cache icu-libs libpq libzip libmaxminddb nginx supervisor su-exec \ +RUN apk add --no-cache icu-libs libpq libzip libmaxminddb nginx supervisor su-exec postgresql-client restic \ && apk add --no-cache --virtual .build-deps $PHPIZE_DEPS icu-dev postgresql-dev libzip-dev \ && docker-php-ext-install intl pcntl pdo_pgsql zip \ && apk del .build-deps @@ -40,7 +40,9 @@ RUN composer dump-autoload --no-dev --classmap-authoritative \ && chown -R www-data:www-data storage bootstrap/cache COPY docker/php/entrypoint.sh /usr/local/bin/cdnf-entrypoint -RUN chmod +x /usr/local/bin/cdnf-entrypoint +COPY docker/backup/create.sh /usr/local/bin/cdnf-backup-create +COPY docker/backup/restore.sh /usr/local/bin/cdnf-backup-restore +RUN chmod +x /usr/local/bin/cdnf-entrypoint /usr/local/bin/cdnf-backup-create /usr/local/bin/cdnf-backup-restore ENTRYPOINT ["cdnf-entrypoint"] CMD ["php-fpm", "-F"] diff --git a/core/app/Console/Commands/CreateControlBackup.php b/core/app/Console/Commands/CreateControlBackup.php new file mode 100644 index 0000000..573ad3b --- /dev/null +++ b/core/app/Console/Commands/CreateControlBackup.php @@ -0,0 +1,36 @@ +configured()) { + $this->error('Encrypted off-host backup repository is not configured.'); + + return self::FAILURE; + } + $backup = Backup::query()->create(['status' => 'pending']); + $operation = Operation::query()->create(['type' => 'backup.create', 'status' => 'pending', 'input' => ['backup_id' => $backup->id]]); + if ($this->option('wait')) { + CreateControlBackupJob::dispatchSync($backup->id, $operation->id); + } else { + CreateControlBackupJob::dispatch($backup->id, $operation->id); + } + $backup->refresh(); + $this->line(json_encode(['backup_id' => $backup->id, 'operation_id' => $operation->id, 'status' => $backup->status, 'snapshot_id' => $backup->snapshot_id, 'verified_at' => $backup->verified_at?->toIso8601String()], JSON_THROW_ON_ERROR)); + + return $backup->status === 'failed' ? self::FAILURE : self::SUCCESS; + } +} diff --git a/core/app/Console/Commands/GenerateOpenApi.php b/core/app/Console/Commands/GenerateOpenApi.php index b5cd50d..36aebc3 100644 --- a/core/app/Console/Commands/GenerateOpenApi.php +++ b/core/app/Console/Commands/GenerateOpenApi.php @@ -165,9 +165,10 @@ private function pathParameters(string $path): array return collect($matches[1])->map(function (string $name): array { $schema = match ($name) { - 'operation', 'edge', 'purge' => ['type' => 'string', 'format' => 'uuid'], + 'operation', 'edge', 'purge', 'backup', 'job' => ['type' => 'string', 'format' => 'uuid'], 'checksum' => ['type' => 'string', 'pattern' => '^[a-f0-9]{64}$'], 'group' => ['type' => 'string', 'enum' => array_keys(config('platform.groups', []))], + 'scope' => ['type' => 'string', 'enum' => ['dns', 'edges', 'tls', 'purges', 'usage']], default => ['type' => 'integer', 'minimum' => 1], }; diff --git a/core/app/Console/Commands/PruneAuditLogs.php b/core/app/Console/Commands/PruneAuditLogs.php new file mode 100644 index 0000000..e411cbf --- /dev/null +++ b/core/app/Console/Commands/PruneAuditLogs.php @@ -0,0 +1,24 @@ +option('batch'))); + $ids = AuditLog::query()->where('created_at', '<', now()->subDays($settings->integer('operations', 'audit_retention_days')))->orderBy('id')->limit($batch)->pluck('id'); + $deleted = $ids->isEmpty() ? 0 : AuditLog::query()->whereIn('id', $ids)->delete(); + $this->info("Deleted {$deleted} expired audit events."); + + return self::SUCCESS; + } +} diff --git a/core/app/Console/Commands/RestoreControlBackup.php b/core/app/Console/Commands/RestoreControlBackup.php new file mode 100644 index 0000000..e141b7e --- /dev/null +++ b/core/app/Console/Commands/RestoreControlBackup.php @@ -0,0 +1,76 @@ +error('Set BACKUP_RESTORE_ALLOWED=true only in the one-off maintenance container.'); + + return self::FAILURE; + } + $operation = Operation::query()->whereKey($this->argument('operation'))->where('type', 'backup.restore')->where('status', 'running')->first(); + if ($operation === null || ($operation->result['preflight'] ?? null) !== 'passed') { + $this->error('A successful restore preflight operation is required.'); + + return self::FAILURE; + } + $backup = Backup::query()->whereKey($operation->input['backup_id'])->where('status', 'succeeded')->firstOrFail(); + Artisan::call('down', ['--retry' => 60]); + try { + $repository->restore($backup->snapshot_id); + Artisan::call('migrate', ['--force' => true]); + $restoredBackup = Backup::query()->find($backup->id) ?? Backup::query()->create(['id' => $backup->id]); + $restoredBackup->update(['status' => 'succeeded', 'snapshot_id' => $backup->snapshot_id, 'verified_at' => now(), 'last_error' => null]); + $receipt = Operation::query()->find($operation->id) ?? Operation::query()->create(['id' => $operation->id, 'type' => 'backup.restore', 'status' => 'running', 'input' => ['backup_id' => $backup->id]]); + $receipt->update(['status' => 'succeeded', 'result' => ['backup_id' => $backup->id, 'snapshot_id' => $backup->snapshot_id, 'restored_at' => now()->toIso8601String()], 'finished_at' => now()]); + AuditLog::record(null, 'backup.restore_completed', $restoredBackup, ['operation_id' => $receipt->id, 'snapshot_id' => $backup->snapshot_id]); + ReconcileAllDnsZones::dispatch(Operation::query()->create(['type' => 'dns.global_reconcile', 'status' => 'pending', 'input' => []])->id); + ReconcileAllEdgeDomains::dispatch(Operation::query()->create(['type' => 'edges.global_reconcile', 'status' => 'pending', 'input' => []])->id); + ReconcileAllTls::dispatch(Operation::query()->create(['type' => 'tls.global_reconcile', 'status' => 'pending', 'input' => []])->id); + ReconcileAllPurges::dispatch(Operation::query()->create(['type' => 'purges.global_reconcile', 'status' => 'pending', 'input' => []])->id); + $from = now()->utc()->subHour()->startOfHour(); + $to = now()->utc()->startOfHour(); + $usage = Operation::query()->create(['type' => 'usage.global_reconcile', 'status' => 'pending', 'input' => ['from' => $from->toIso8601String(), 'to' => $to->toIso8601String()]]); + BuildUsageRollups::dispatch($from->toIso8601String(), $to->toIso8601String(), null, $usage->id); + Artisan::call('up'); + $this->info('Restore completed; reconciliation has been queued.'); + + return self::SUCCESS; + } catch (Throwable $exception) { + try { + Operation::query()->whereKey($operation->id)->update([ + 'status' => 'failed', + 'error' => mb_substr($exception->getMessage(), 0, 4000), + 'finished_at' => now(), + ]); + AuditLog::record(null, 'backup.restore_failed', $backup, ['operation_id' => $operation->id]); + } catch (Throwable) { + // The restore may have failed while replacing PostgreSQL itself. + } + $this->error('Restore failed; keep maintenance mode active and inspect the recovery host logs.'); + report($exception); + + return self::FAILURE; + } + } +} diff --git a/core/app/Filament/Admin/Pages/AdminDashboard.php b/core/app/Filament/Admin/Pages/AdminDashboard.php index 5bb0e2e..2448fbc 100644 --- a/core/app/Filament/Admin/Pages/AdminDashboard.php +++ b/core/app/Filament/Admin/Pages/AdminDashboard.php @@ -13,6 +13,7 @@ use App\Models\Edge; use App\Models\Operation; use App\Models\User; +use App\Support\SystemHealth; use Filament\Pages\Dashboard; use Illuminate\Support\Collection; use Illuminate\Support\Facades\Redis; @@ -97,6 +98,15 @@ public function getQuickLinksProperty(): array ]; } + public function getComponentStateProperty(): array + { + try { + return collect(app(SystemHealth::class)->components())->map(fn (array $state, string $name): array => ['name' => str($name)->replace('_', ' ')->headline()->toString(), ...$state])->values()->all(); + } catch (Throwable) { + return [['name' => 'Operational health', 'status' => 'unavailable', 'checked_at' => now()->toIso8601String(), 'details' => []]]; + } + } + private function queueAge(?int $pushedAt, int $depth): string { if ($depth === 0) { diff --git a/core/app/Http/Controllers/Admin/BackupController.php b/core/app/Http/Controllers/Admin/BackupController.php new file mode 100644 index 0000000..a5f53d5 --- /dev/null +++ b/core/app/Http/Controllers/Admin/BackupController.php @@ -0,0 +1,70 @@ +orderBy('id')->cursorPaginate(50)); + } + + public function show(Backup $backup): JsonResource + { + return JsonResource::make($backup); + } + + public function store(Request $request, ResticBackupRepository $repository): JsonResponse + { + abort_unless($repository->configured(), 503, 'Encrypted off-host backup repository is not configured.'); + [$backup, $operation] = DB::transaction(function () use ($request): array { + $backup = Backup::query()->create(['requested_by' => $request->user()->id, 'status' => 'pending']); + $operation = Operation::query()->create(['actor_id' => $request->user()->id, 'type' => 'backup.create', 'status' => 'pending', 'input' => ['backup_id' => $backup->id]]); + AuditLog::record($request->user(), 'backup.create_requested', $backup, ['operation_id' => $operation->id], $request->ip()); + + return [$backup, $operation]; + }); + CreateControlBackup::dispatch($backup->id, $operation->id)->afterCommit(); + + return response()->json(['data' => ['backup_id' => $backup->id, 'operation_id' => $operation->id, 'status' => 'pending']], 202); + } + + public function restore(Request $request, Backup $backup): JsonResponse + { + $data = $request->validate(['confirmation' => ['required', 'string', 'max:100'], 'current_password' => ['required', 'string', 'max:200']]); + abort_unless(hash_equals("RESTORE {$backup->id}", $data['confirmation']), 422, 'The restore confirmation value is incorrect.'); + abort_unless(Hash::check($data['current_password'], $request->user()->password), 422, 'Administrator re-authentication failed.'); + abort_unless($backup->status === 'succeeded' && $backup->snapshot_id !== null, 409, 'Only a completed backup can be restored.'); + $operation = Operation::query()->create(['actor_id' => $request->user()->id, 'type' => 'backup.restore', 'status' => 'pending', 'input' => ['backup_id' => $backup->id]]); + AuditLog::record($request->user(), 'backup.restore_preflight_requested', $backup, ['operation_id' => $operation->id], $request->ip()); + PreflightBackupRestore::dispatch($backup->id, $operation->id)->afterCommit(); + + return response()->json(['data' => ['operation_id' => $operation->id, 'status' => 'pending']], 202); + } + + public function destroy(Request $request, Backup $backup): JsonResponse + { + abort_if($backup->status === 'running', 409, 'A running backup cannot be deleted.'); + $operation = Operation::query()->create(['actor_id' => $request->user()->id, 'type' => 'backup.delete', 'status' => 'pending', 'input' => ['backup_id' => $backup->id]]); + $backup->update(['status' => 'deleting']); + AuditLog::record($request->user(), 'backup.delete_requested', $backup, ['operation_id' => $operation->id], $request->ip()); + DeleteControlBackup::dispatch($backup->id, $operation->id)->afterCommit(); + + return response()->json(['data' => ['operation_id' => $operation->id, 'status' => 'pending']], 202); + } +} diff --git a/core/app/Http/Controllers/Admin/FailedJobController.php b/core/app/Http/Controllers/Admin/FailedJobController.php new file mode 100644 index 0000000..46d0287 --- /dev/null +++ b/core/app/Http/Controllers/Admin/FailedJobController.php @@ -0,0 +1,45 @@ +orderBy('id')->cursorPaginate(50); + $jobs->through(fn ($job): array => [ + 'id' => $job->id, 'uuid' => $job->uuid, 'connection' => $job->connection, 'queue' => $job->queue, + 'job' => data_get(json_decode($job->payload, true), 'displayName', 'unknown'), + 'exception' => mb_substr(strtok($job->exception, "\n") ?: 'Job failed', 0, 500), 'failed_at' => $job->failed_at, + ]); + + return response()->json($jobs); + } + + public function retry(Request $request, string $job): JsonResponse + { + $row = DB::table('failed_jobs')->where('uuid', $job)->orWhere('id', ctype_digit($job) ? (int) $job : -1)->first(); + abort_if($row === null, 404, 'Failed job not found.'); + Artisan::call('queue:retry', ['id' => [$row->uuid]]); + AuditLog::record($request->user(), 'failed_job.retry_requested', null, ['uuid' => $row->uuid, 'queue' => $row->queue], $request->ip()); + + return response()->json(['data' => ['uuid' => $row->uuid, 'status' => 'queued']], 202); + } + + public function destroy(Request $request, string $job): JsonResponse + { + $row = DB::table('failed_jobs')->where('uuid', $job)->orWhere('id', ctype_digit($job) ? (int) $job : -1)->first(); + abort_if($row === null, 404, 'Failed job not found.'); + DB::table('failed_jobs')->where('uuid', $row->uuid)->delete(); + AuditLog::record($request->user(), 'failed_job.deleted', null, ['uuid' => $row->uuid, 'queue' => $row->queue], $request->ip()); + + return response()->json(null, 204); + } +} diff --git a/core/app/Http/Controllers/Admin/ReconciliationController.php b/core/app/Http/Controllers/Admin/ReconciliationController.php new file mode 100644 index 0000000..662f159 --- /dev/null +++ b/core/app/Http/Controllers/Admin/ReconciliationController.php @@ -0,0 +1,38 @@ +where('type', $type)->whereIn('status', ['pending', 'running'])->first(); + if ($operation === null) { + $input = $scope === 'usage' ? ['from' => now()->utc()->subHour()->startOfHour()->toIso8601String(), 'to' => now()->utc()->startOfHour()->toIso8601String()] : []; + $operation = Operation::query()->create(['actor_id' => $request->user()->id, 'type' => $type, 'status' => 'pending', 'input' => $input]); + AuditLog::record($request->user(), "{$scope}.global_reconcile_requested", $operation, [], $request->ip()); + match ($scope) { + 'dns' => ReconcileAllDnsZones::dispatch($operation->id)->afterCommit(), + 'edges' => ReconcileAllEdgeDomains::dispatch($operation->id)->afterCommit(), + 'tls' => ReconcileAllTls::dispatch($operation->id)->afterCommit(), + 'purges' => ReconcileAllPurges::dispatch($operation->id)->afterCommit(), + 'usage' => BuildUsageRollups::dispatch($input['from'], $input['to'], null, $operation->id)->afterCommit(), + }; + } + + return response()->json(['data' => ['operation_id' => $operation->id, 'status' => $operation->status]], 202); + } +} diff --git a/core/app/Http/Controllers/Admin/SystemOperationsController.php b/core/app/Http/Controllers/Admin/SystemOperationsController.php new file mode 100644 index 0000000..a4b91fc --- /dev/null +++ b/core/app/Http/Controllers/Admin/SystemOperationsController.php @@ -0,0 +1,24 @@ +components(); + + return response()->json(['data' => ['status' => $health->overall($components), 'checked_at' => now()->toIso8601String()]]); + } + + public function components(SystemHealth $health): JsonResponse + { + $components = $health->components(); + + return response()->json(['data' => ['status' => $health->overall($components), 'components' => $components, 'queues' => $health->queues()]]); + } +} diff --git a/core/app/Http/Controllers/MetricsController.php b/core/app/Http/Controllers/MetricsController.php new file mode 100644 index 0000000..0ccb8c5 --- /dev/null +++ b/core/app/Http/Controllers/MetricsController.php @@ -0,0 +1,39 @@ +bearerToken()), 404); + $lines = ['# HELP cdnfoundry_component_health Component health (healthy=1).', '# TYPE cdnfoundry_component_health gauge']; + foreach ($health->components() as $name => $component) { + $lines[] = sprintf('cdnfoundry_component_health{component="%s",status="%s"} %d', $name, $component['status'], $component['status'] === 'healthy' ? 1 : 0); + } + foreach ($health->queues() as $queue => $state) { + $lines[] = sprintf('cdnfoundry_queue_depth{queue="%s"} %d', $queue, $state['depth'] ?? 0); + $lines[] = sprintf('cdnfoundry_queue_oldest_job_age_seconds{queue="%s"} %d', $queue, $state['oldest_job_age_seconds'] ?? 0); + } + $lines[] = 'cdnfoundry_operations_failed '.Operation::query()->where('status', 'failed')->count(); + $lines[] = 'cdnfoundry_dns_deployments_drifted '.DnsDeployment::query()->whereIn('status', ['pending', 'failed'])->count(); + $lines[] = 'cdnfoundry_edges_stale '.Edge::query()->where('enabled', true)->where(fn ($query) => $query->whereNull('last_heartbeat_at')->orWhere('last_heartbeat_at', '<', now()->subSeconds(app(PlatformSettings::class)->integer('edge_runtime', 'heartbeat_fresh_seconds'))))->count(); + $lines[] = 'cdnfoundry_tls_certificates_expiring '.TlsCertificate::query()->where('status', 'active')->where('expires_at', '<=', now()->addDays((int) config('services.acme.expiry_alert_days')))->count(); + + return response(implode("\n", $lines)."\n", 200, ['Content-Type' => 'text/plain; version=0.0.4; charset=utf-8', 'Cache-Control' => 'no-store']); + } +} diff --git a/core/app/Http/Controllers/OperationController.php b/core/app/Http/Controllers/OperationController.php index dc9f490..e95ae3f 100644 --- a/core/app/Http/Controllers/OperationController.php +++ b/core/app/Http/Controllers/OperationController.php @@ -3,11 +3,17 @@ namespace App\Http\Controllers; use App\Jobs\ApplyPlatformDnsSettings; +use App\Jobs\BuildUsageRollups; +use App\Jobs\CreateControlBackup; +use App\Jobs\DeleteControlBackup; use App\Jobs\DispatchOriginTest; use App\Jobs\ImportDnsZone; +use App\Jobs\PreflightBackupRestore; use App\Jobs\ProvisionEdgePoolCells; use App\Jobs\ReconcileAllDnsZones; use App\Jobs\ReconcileAllEdgeDomains; +use App\Jobs\ReconcileAllPurges; +use App\Jobs\ReconcileAllTls; use App\Jobs\ReconcileDnsZone; use App\Jobs\ReconcileEdgeDomain; use App\Jobs\TestDnsCluster; @@ -37,7 +43,7 @@ public function index(): AnonymousResourceCollection public function retry(Request $request, Operation $operation): JsonResponse { abort_unless($operation->status === 'failed', 409, 'Only failed operations can be retried.'); - abort_unless(in_array($operation->type, ['platform_dns_identity.update', 'system_settings.update', 'domain.nameservers_verify', 'dns.zone_reconcile', 'dns.zone_import', 'dns.cluster_test', 'dns.global_reconcile', 'edge.global_reconcile', 'edge.pool_provision', 'edge.domain_reconcile', 'edge.origin_test'], true), 422, 'Unsupported operation type.'); + abort_unless(in_array($operation->type, ['platform_dns_identity.update', 'system_settings.update', 'domain.nameservers_verify', 'dns.zone_reconcile', 'dns.zone_import', 'dns.cluster_test', 'dns.global_reconcile', 'edge.global_reconcile', 'edges.global_reconcile', 'tls.global_reconcile', 'purges.global_reconcile', 'usage.global_reconcile', 'backup.create', 'backup.restore', 'backup.delete', 'edge.pool_provision', 'edge.domain_reconcile', 'edge.origin_test'], true), 422, 'Unsupported operation type.'); $operation->update(['status' => 'pending', 'error' => null, 'finished_at' => null]); AuditLog::record($request->user(), 'operation.retry_requested', $operation, [], $request->ip()); match ($operation->type) { @@ -49,6 +55,13 @@ public function retry(Request $request, Operation $operation): JsonResponse 'dns.cluster_test' => TestDnsCluster::dispatch($operation->getKey()), 'dns.global_reconcile' => ReconcileAllDnsZones::dispatch($operation->getKey()), 'edge.global_reconcile' => ReconcileAllEdgeDomains::dispatch($operation->getKey()), + 'edges.global_reconcile' => ReconcileAllEdgeDomains::dispatch($operation->getKey()), + 'tls.global_reconcile' => ReconcileAllTls::dispatch($operation->getKey()), + 'purges.global_reconcile' => ReconcileAllPurges::dispatch($operation->getKey()), + 'usage.global_reconcile' => BuildUsageRollups::dispatch($operation->input['from'], $operation->input['to'], null, $operation->getKey()), + 'backup.create' => CreateControlBackup::dispatch($operation->input['backup_id'], $operation->getKey()), + 'backup.restore' => PreflightBackupRestore::dispatch($operation->input['backup_id'], $operation->getKey()), + 'backup.delete' => DeleteControlBackup::dispatch($operation->input['backup_id'], $operation->getKey()), 'edge.pool_provision' => ProvisionEdgePoolCells::dispatch((int) $operation->input['pool_id'], $operation->id), 'edge.domain_reconcile' => ReconcileEdgeDomain::dispatch((int) $operation->input['domain_id']), 'edge.origin_test' => DispatchOriginTest::dispatch($operation->getKey()), diff --git a/core/app/Jobs/CreateControlBackup.php b/core/app/Jobs/CreateControlBackup.php new file mode 100644 index 0000000..a18c92f --- /dev/null +++ b/core/app/Jobs/CreateControlBackup.php @@ -0,0 +1,51 @@ +onQueue('bulk_maintenance'); + } + + public function uniqueId(): string + { + return $this->backupId; + } + + public function handle(ResticBackupRepository $repository): void + { + $backup = Backup::query()->findOrFail($this->backupId); + $operation = Operation::query()->findOrFail($this->operationId); + if ($backup->status === 'succeeded') { + return; + } + $backup->update(['status' => 'running', 'last_error' => null]); + $operation->update(['status' => 'running', 'started_at' => $operation->started_at ?? now(), 'attempts' => $operation->attempts + 1]); + $result = $repository->create(); + $backup->update([...$result, 'status' => 'succeeded', 'verified_at' => now()]); + $operation->update(['status' => 'succeeded', 'result' => ['backup_id' => $backup->id, ...$result], 'finished_at' => now()]); + } + + public function failed(Throwable $exception): void + { + $error = mb_substr($exception->getMessage(), 0, 4000); + Backup::query()->whereKey($this->backupId)->update(['status' => 'failed', 'last_error' => $error]); + Operation::query()->whereKey($this->operationId)->update(['status' => 'failed', 'error' => $error, 'finished_at' => now()]); + } +} diff --git a/core/app/Jobs/DeleteControlBackup.php b/core/app/Jobs/DeleteControlBackup.php new file mode 100644 index 0000000..30971ef --- /dev/null +++ b/core/app/Jobs/DeleteControlBackup.php @@ -0,0 +1,40 @@ +onQueue('bulk_maintenance'); + } + + public function handle(ResticBackupRepository $repository): void + { + $backup = Backup::query()->findOrFail($this->backupId); + $operation = Operation::query()->findOrFail($this->operationId); + $operation->update(['status' => 'running', 'started_at' => now(), 'attempts' => $operation->attempts + 1]); + if ($backup->snapshot_id) { + $repository->forget($backup->snapshot_id); + } + $backup->delete(); + $operation->update(['status' => 'succeeded', 'result' => ['backup_id' => $this->backupId, 'deleted' => true], 'finished_at' => now()]); + } + + public function failed(Throwable $exception): void + { + Backup::query()->whereKey($this->backupId)->update(['status' => 'failed', 'last_error' => mb_substr($exception->getMessage(), 0, 4000)]); + Operation::query()->whereKey($this->operationId)->update(['status' => 'failed', 'error' => mb_substr($exception->getMessage(), 0, 4000), 'finished_at' => now()]); + } +} diff --git a/core/app/Jobs/PreflightBackupRestore.php b/core/app/Jobs/PreflightBackupRestore.php new file mode 100644 index 0000000..ec37c6b --- /dev/null +++ b/core/app/Jobs/PreflightBackupRestore.php @@ -0,0 +1,36 @@ +onQueue('bulk_maintenance'); + } + + public function handle(ResticBackupRepository $repository): void + { + $backup = Backup::query()->whereKey($this->backupId)->where('status', 'succeeded')->firstOrFail(); + $operation = Operation::query()->findOrFail($this->operationId); + $operation->update(['status' => 'running', 'started_at' => now(), 'attempts' => $operation->attempts + 1]); + $repository->snapshotExists($backup->snapshot_id); + $operation->update(['status' => 'running', 'result' => ['backup_id' => $backup->id, 'snapshot_id' => $backup->snapshot_id, 'preflight' => 'passed', 'maintenance_command' => "php artisan backups:restore {$operation->id}"]]); + } + + public function failed(Throwable $exception): void + { + Operation::query()->whereKey($this->operationId)->update(['status' => 'failed', 'error' => mb_substr($exception->getMessage(), 0, 4000), 'finished_at' => now()]); + } +} diff --git a/core/app/Jobs/ReconcileAllPurges.php b/core/app/Jobs/ReconcileAllPurges.php new file mode 100644 index 0000000..fc78d43 --- /dev/null +++ b/core/app/Jobs/ReconcileAllPurges.php @@ -0,0 +1,61 @@ +onQueue('bulk_maintenance'); + } + + public function uniqueId(): string + { + return $this->operationId; + } + + public function handle(): void + { + $operation = Operation::query()->findOrFail($this->operationId); + if (! in_array($operation->status, ['pending', 'running'], true)) { + return; + } + $cursor = (string) ($operation->result['cursor'] ?? ''); + $count = (int) ($operation->result['purges_requeued'] ?? 0); + $purges = CachePurge::query()->whereIn('status', ['pending', 'running', 'failed'])->when($cursor !== '', fn ($query) => $query->where('id', '>', $cursor))->orderBy('id')->limit(250)->get(); + $operation->update(['status' => 'running', 'started_at' => $operation->started_at ?? now(), 'attempts' => $operation->attempts + 1]); + foreach ($purges as $purge) { + DB::transaction(function () use ($purge): void { + EdgeTask::query()->where('cache_purge_id', $purge->id)->where('status', 'failed')->update(['status' => 'pending', 'attempts' => 0, 'last_error' => null, 'available_at' => now()]); + $purge->update(['status' => $purge->tasks()->where('status', 'failed')->exists() ? 'failed' : 'running']); + }); + $count++; + } + $cursor = (string) ($purges->last()?->id ?? $cursor); + $more = $purges->count() === 250 && CachePurge::query()->whereIn('status', ['pending', 'running', 'failed'])->where('id', '>', $cursor)->exists(); + $operation->update(['status' => $more ? 'running' : 'succeeded', 'result' => ['cursor' => $cursor, 'purges_requeued' => $count], 'finished_at' => $more ? null : now()]); + if ($more) { + self::dispatch($operation->id)->delay(now()->addSecond()); + } + } + + public function failed(Throwable $exception): void + { + Operation::query()->whereKey($this->operationId)->update(['status' => 'failed', 'error' => mb_substr($exception->getMessage(), 0, 4000), 'finished_at' => now()]); + } +} diff --git a/core/app/Jobs/ReconcileAllTls.php b/core/app/Jobs/ReconcileAllTls.php new file mode 100644 index 0000000..93189f2 --- /dev/null +++ b/core/app/Jobs/ReconcileAllTls.php @@ -0,0 +1,57 @@ +onQueue('bulk_maintenance'); + } + + public function uniqueId(): string + { + return $this->operationId; + } + + public function handle(): void + { + $operation = Operation::query()->findOrFail($this->operationId); + if (! in_array($operation->status, ['pending', 'running'], true)) { + return; + } + $cursor = (int) ($operation->result['cursor'] ?? 0); + $count = (int) ($operation->result['domains_dispatched'] ?? 0); + $ids = Domain::query()->where('lifecycle_state', DomainLifecycleState::Active->value)->where('tls_mode', 'managed')->where('id', '>', $cursor)->orderBy('id')->limit(250)->pluck('id'); + $operation->update(['status' => 'running', 'started_at' => $operation->started_at ?? now(), 'attempts' => $operation->attempts + 1]); + foreach ($ids as $id) { + EnsureManagedCertificates::dispatch((int) $id); + } + $count += $ids->count(); + $cursor = (int) ($ids->last() ?? $cursor); + $more = $ids->count() === 250 && Domain::query()->where('lifecycle_state', DomainLifecycleState::Active->value)->where('tls_mode', 'managed')->where('id', '>', $cursor)->exists(); + $operation->update(['status' => $more ? 'running' : 'succeeded', 'result' => ['cursor' => $cursor, 'domains_dispatched' => $count], 'finished_at' => $more ? null : now()]); + if ($more) { + self::dispatch($operation->id)->delay(now()->addSecond()); + } + } + + public function failed(Throwable $exception): void + { + Operation::query()->whereKey($this->operationId)->update(['status' => 'failed', 'error' => mb_substr($exception->getMessage(), 0, 4000), 'finished_at' => now()]); + } +} diff --git a/core/app/Models/Backup.php b/core/app/Models/Backup.php new file mode 100644 index 0000000..ace19c9 --- /dev/null +++ b/core/app/Models/Backup.php @@ -0,0 +1,18 @@ + 'immutable_datetime']; + } +} diff --git a/core/app/Support/ResticBackupRepository.php b/core/app/Support/ResticBackupRepository.php new file mode 100644 index 0000000..357add4 --- /dev/null +++ b/core/app/Support/ResticBackupRepository.php @@ -0,0 +1,75 @@ +run(['/usr/local/bin/cdnf-backup-create']); + $events = collect(preg_split('/\R/', trim($output)))->filter()->map(fn (string $line) => json_decode($line, true))->filter(fn ($row) => is_array($row)); + $summary = $events->last(fn (array $row) => isset($row['snapshot_id'])); + if (! is_array($summary)) { + throw new RuntimeException('Restic did not return a snapshot identifier.'); + } + + return ['snapshot_id' => $summary['snapshot_id'], 'size_bytes' => (int) ($summary['data_added'] ?? $summary['total_bytes_processed'] ?? 0), 'manifest_sha256' => hash('sha256', $output)]; + } + + public function snapshotExists(string $snapshotId): bool + { + $this->assertSnapshot($snapshotId); + $this->run(['restic', 'snapshots', '--json', $snapshotId]); + + return true; + } + + public function forget(string $snapshotId): void + { + $this->assertSnapshot($snapshotId); + $this->run(['restic', 'forget', $snapshotId]); + } + + public function restore(string $snapshotId): void + { + $this->assertSnapshot($snapshotId); + $this->run(['/usr/local/bin/cdnf-backup-restore', $snapshotId], 7200); + } + + private function run(array $command, int $timeout = 3600): string + { + if (! $this->configured()) { + throw new RuntimeException('Encrypted off-host backup repository is not configured.'); + } + $process = new Process($command, null, [ + 'RESTIC_REPOSITORY' => config('services.backups.repository'), + 'RESTIC_PASSWORD_FILE' => config('services.backups.password_file'), + 'AWS_ACCESS_KEY_ID' => config('services.backups.access_key'), + 'AWS_SECRET_ACCESS_KEY' => config('services.backups.secret_key'), + 'AWS_DEFAULT_REGION' => config('services.backups.region'), + 'PGHOST' => config('database.connections.pgsql.host'), + 'PGPORT' => (string) config('database.connections.pgsql.port'), + 'PGDATABASE' => config('database.connections.pgsql.database'), + 'PGUSER' => config('database.connections.pgsql.username'), + 'PGPASSWORD' => config('database.connections.pgsql.password'), + ]); + $process->setTimeout($timeout)->mustRun(); + + return $process->getOutput(); + } + + private function assertSnapshot(string $snapshotId): void + { + if (! preg_match('/^[a-f0-9]{8,128}$/', $snapshotId)) { + throw new RuntimeException('Invalid backup snapshot identifier.'); + } + } +} diff --git a/core/app/Support/SystemHealth.php b/core/app/Support/SystemHealth.php new file mode 100644 index 0000000..f826ac1 --- /dev/null +++ b/core/app/Support/SystemHealth.php @@ -0,0 +1,228 @@ +probe(fn () => DB::select('select 1')); + $components['queue_backend'] = $this->probe(fn () => Redis::connection()->command('ping')); + $components['queue_workers'] = $this->horizon(); + $heartbeat = Cache::get('operations:scheduler_heartbeat'); + $schedulerStale = $heartbeat === null || now()->diffInSeconds($heartbeat) > app(PlatformSettings::class)->integer('operations', 'scheduler_stale_seconds'); + $components['scheduler'] = $this->state($schedulerStale ? 'degraded' : 'healthy', ['last_heartbeat_at' => $heartbeat]); + $components['clickhouse'] = $this->probe(fn () => Http::connectTimeout(1)->timeout(2)->get(config('services.clickhouse.url').'/ping')->throw()); + $components['vector'] = $this->probe(fn () => Http::connectTimeout(1)->timeout(2)->get(config('services.vector.metrics_url'))->throw()); + $components['host_clock'] = $this->clock(); + $components['mmdb'] = $this->mmdb(); + + $heartbeatSeconds = app(PlatformSettings::class)->integer('edge_runtime', 'heartbeat_fresh_seconds'); + $enabledEdges = Edge::query()->where('enabled', true)->count(); + $staleEdges = Edge::query()->where('enabled', true)->where(fn ($query) => $query->whereNull('last_heartbeat_at')->orWhere('last_heartbeat_at', '<', now()->subSeconds($heartbeatSeconds)))->count(); + $components['edges'] = $this->state($enabledEdges === 0 ? 'degraded' : ($staleEdges > 0 ? 'degraded' : 'healthy'), ['enabled' => $enabledEdges, 'stale' => $staleEdges]); + $listenerFailures = Edge::query()->where('enabled', true)->where('drained', false) + ->where(fn ($query) => $query->whereNull('capacity->listener_ready')->orWhere('capacity->listener_ready', '!=', true))->count(); + $components['edge_listeners'] = $this->state($listenerFailures > 0 ? 'degraded' : 'healthy', ['not_ready' => $listenerFailures]); + + $enabledCells = EdgeCell::query()->whereHas('edge', fn ($query) => $query->where('enabled', true))->count(); + $unhealthyCells = EdgeCell::query()->whereHas('edge', fn ($query) => $query->where('enabled', true)) + ->whereNotIn('status', ['ready', 'drained'])->count(); + $components['edge_cells'] = $this->state($enabledCells === 0 || $unhealthyCells > 0 ? 'degraded' : 'healthy', ['assigned' => $enabledCells, 'unhealthy' => $unhealthyCells]); + + $enabledPools = EdgePool::query()->where('enabled', true)->where('withdrawn', false)->count(); + $unavailablePools = EdgePool::query()->where('enabled', true)->where('withdrawn', false) + ->whereDoesntHave('cells', fn ($query) => $query->where('drained', false)->where('status', 'ready') + ->whereHas('edge', fn ($edge) => $edge->readyForTraffic()))->count(); + $components['service_pools'] = $this->state($enabledPools === 0 || $unavailablePools > 0 ? 'degraded' : 'healthy', ['enabled' => $enabledPools, 'unavailable' => $unavailablePools]); + + $latestArtifacts = DB::table('edge_artifacts')->selectRaw('edge_id, max(sequence) as latest_sequence')->groupBy('edge_id'); + $configurationDrift = Edge::query()->where('edges.enabled', true)->joinSub($latestArtifacts, 'latest_edge_artifacts', fn ($join) => $join->on('edges.id', '=', 'latest_edge_artifacts.edge_id')) + ->whereColumn('edges.active_sequence', '<', 'latest_edge_artifacts.latest_sequence')->count(); + $deploymentRejections = Edge::query()->where('enabled', true)->whereNotNull('capacity->last_rejection')->count(); + $components['edge_configuration'] = $this->state(($configurationDrift + $deploymentRejections) > 0 ? 'degraded' : 'healthy', ['stale_edges' => $configurationDrift, 'rejected_candidates' => $deploymentRejections]); + + $failedPlacements = DomainEdgePlacement::query()->where('state', 'failed')->count(); + $placementDrift = DomainEdgePlacement::query()->where('state', 'active')->join('domains', 'domains.id', '=', 'domain_edge_placements.domain_id') + ->where(fn ($query) => $query->whereNull('domains.active_edge_revision')->orWhereColumn('domains.active_edge_revision', '<', 'domain_edge_placements.desired_revision'))->count(); + $components['edge_placements'] = $this->state(($failedPlacements + $placementDrift) > 0 ? 'degraded' : 'healthy', ['failed' => $failedPlacements, 'drifted' => $placementDrift]); + $components['edge_capacity'] = $this->edgeCapacity(); + + $activeEmergencyModes = EmergencyMode::query()->where('active', true)->where(fn ($query) => $query->whereNull('expires_at')->orWhere('expires_at', '>', now()))->count(); + $withdrawnPools = EdgePool::query()->where('enabled', true)->where('withdrawn', true)->count(); + $components['emergency_modes'] = $this->state(($activeEmergencyModes + $withdrawnPools) > 0 ? 'degraded' : 'healthy', ['active' => $activeEmergencyModes, 'withdrawn_pools' => $withdrawnPools]); + + $enabledClusters = DnsCluster::query()->where('enabled', true)->count(); + $badClusters = DnsCluster::query()->where('enabled', true)->where(fn ($query) => $query->where('last_health_status', '!=', 'healthy')->orWhereNull('last_health_at'))->count(); + $dnsDrift = DnsDeployment::query()->whereIn('status', ['failed', 'pending'])->count(); + $components['authoritative_dns'] = $this->state($enabledClusters === 0 ? 'degraded' : (($badClusters + $dnsDrift) > 0 ? 'degraded' : 'healthy'), ['enabled_clusters' => $enabledClusters, 'unhealthy_clusters' => $badClusters, 'drifted_deployments' => $dnsDrift]); + $components['dns_deployments'] = $this->state($dnsDrift > 0 ? 'degraded' : 'healthy', ['drifted' => $dnsDrift]); + + $expiring = TlsCertificate::query()->where('status', 'active')->where('expires_at', '<=', now()->addDays((int) config('services.acme.expiry_alert_days')))->count(); + $failedOrders = TlsOrder::query()->where('status', 'failed')->count(); + $components['tls'] = $this->state(($expiring + $failedOrders) > 0 ? 'degraded' : 'healthy', ['expiring_certificates' => $expiring, 'failed_orders' => $failedOrders]); + + $failedPurges = CachePurge::query()->where('status', 'failed')->count(); + $failedTasks = EdgeTask::query()->where('status', 'failed')->count(); + $components['runtime_tasks'] = $this->state(($failedPurges + $failedTasks) > 0 ? 'degraded' : 'healthy', ['failed_purges' => $failedPurges, 'failed_edge_tasks' => $failedTasks]); + $pendingPurges = CachePurge::query()->whereIn('status', ['pending', 'running'])->count(); + $components['purges'] = $this->state($failedPurges > 0 ? 'degraded' : 'healthy', ['failed' => $failedPurges, 'pending' => $pendingPurges]); + + $usageLag = UsageRollup::query()->where('status', 'finalized')->max('interval_end'); + $components['usage'] = $this->state($usageLag === null || now()->diffInHours($usageLag) > 3 ? 'degraded' : 'healthy', ['last_finalized_interval' => $usageLag]); + $components['operations'] = $this->state(Operation::query()->where('status', 'failed')->exists() ? 'degraded' : 'healthy', ['failed' => Operation::query()->where('status', 'failed')->count()]); + $lastBackup = Backup::query()->where('status', 'succeeded')->whereNotNull('verified_at')->max('verified_at'); + $backupStale = $lastBackup === null || now()->diffInHours($lastBackup) > app(PlatformSettings::class)->integer('operations', 'backup_stale_hours'); + $components['backups'] = $this->state($backupStale ? 'degraded' : 'healthy', ['last_verified_at' => $lastBackup]); + + return $components; + } + + public function queues(): array + { + return collect(self::QUEUES)->mapWithKeys(function (string $queue): array { + try { + $depth = (int) Redis::connection()->llen("queues:{$queue}"); + $payload = $depth > 0 ? json_decode((string) Redis::connection()->lindex("queues:{$queue}", 0), true) : null; + $pushedAt = is_array($payload) ? ($payload['pushedAt'] ?? $payload['pushed_at'] ?? null) : null; + $oldestAge = is_numeric($pushedAt) ? max(0, (int) floor(microtime(true) - $pushedAt)) : null; + + return [$queue => ['status' => $depth > 1000 || ($oldestAge !== null && $oldestAge > 900) ? 'degraded' : 'healthy', 'depth' => $depth, 'oldest_job_age_seconds' => $oldestAge]]; + } catch (Throwable) { + return [$queue => ['status' => 'unavailable', 'depth' => null, 'oldest_job_age_seconds' => null]]; + } + })->all(); + } + + public function overall(array $components): string + { + if (collect(['control_database', 'queue_backend'])->contains(fn (string $name) => ($components[$name]['status'] ?? 'unavailable') === 'unavailable')) { + return 'unavailable'; + } + + return collect($components)->contains(fn (array $component) => $component['status'] !== 'healthy') ? 'degraded' : 'healthy'; + } + + private function probe(callable $probe): array + { + $started = hrtime(true); + try { + $probe(); + + return $this->state('healthy', ['latency_ms' => round((hrtime(true) - $started) / 1_000_000, 2)]); + } catch (Throwable $exception) { + return $this->state('unavailable', ['latency_ms' => round((hrtime(true) - $started) / 1_000_000, 2), 'error_code' => class_basename($exception)]); + } + } + + private function clock(): array + { + try { + $response = Http::connectTimeout(1)->timeout(2)->get(config('services.prometheus.url').'/api/v1/query', ['query' => 'node_timex_offset_seconds'])->throw()->json(); + $rows = data_get($response, 'data.result', []); + if (! is_array($rows) || $rows === []) { + return $this->state('degraded', ['error_code' => 'clock_metric_missing']); + } + $offset = collect($rows)->map(fn (array $row): float => abs((float) data_get($row, 'value.1', 0)))->max(); + $limit = app(PlatformSettings::class)->integer('operations', 'clock_drift_warning_seconds'); + + return $this->state($offset > $limit ? 'degraded' : 'healthy', ['maximum_offset_seconds' => $offset, 'warning_seconds' => $limit, 'sources' => count($rows)]); + } catch (Throwable $exception) { + return $this->state('unavailable', ['error_code' => class_basename($exception)]); + } + } + + private function horizon(): array + { + try { + $masters = app(MasterSupervisorRepository::class)->all(); + $running = collect($masters)->where('status', 'running')->count(); + + return $this->state($running > 0 && $running === count($masters) ? 'healthy' : 'degraded', [ + 'running_masters' => $running, + 'known_masters' => count($masters), + ]); + } catch (Throwable $exception) { + return $this->state('unavailable', ['error_code' => class_basename($exception)]); + } + } + + private function mmdb(): array + { + $path = (string) config('services.geoip.database'); + $maximumAgeHours = (int) config('services.geoip.stale_hours', 48); + clearstatcache(true, $path); + if (! is_file($path) || ! is_readable($path) || filesize($path) === 0) { + return $this->state('unavailable', ['error_code' => 'mmdb_missing']); + } + $modified = filemtime($path); + if ($modified === false) { + return $this->state('unavailable', ['error_code' => 'mmdb_stat_failed']); + } + $ageHours = max(0, (now()->timestamp - $modified) / 3600); + + return $this->state($ageHours > $maximumAgeHours ? 'degraded' : 'healthy', [ + 'updated_at' => date(DATE_ATOM, $modified), + 'age_hours' => round($ageHours, 2), + 'stale_after_hours' => $maximumAgeHours, + ]); + } + + private function edgeCapacity(): array + { + $cells = EdgeCell::query()->whereHas('edge', fn ($query) => $query->where('enabled', true))->limit(1001)->get(['capacity']); + $truncated = $cells->count() > 1000; + $pressured = $cells->take(1000)->filter(function (EdgeCell $cell): bool { + $capacity = $cell->capacity ?? []; + foreach ([ + ['memory_usage', 'memory_limit'], + ['cache_usage', 'cache_limit'], + ['temporary_storage_usage', 'temporary_storage_limit'], + ['active_connections', 'connection_limit'], + ] as [$usedKey, $limitKey]) { + $used = data_get($capacity, $usedKey); + $limit = data_get($capacity, $limitKey); + if (is_numeric($used) && is_numeric($limit) && (float) $limit > 0 && ((float) $used / (float) $limit) >= 0.8) { + return true; + } + } + + return false; + })->count(); + + return $this->state($truncated || $pressured > 0 ? 'degraded' : 'healthy', [ + 'pressured_cells' => $pressured, + 'scanned_cells' => min(1000, $cells->count()), + 'scan_truncated' => $truncated, + ]); + } + + private function state(string $status, array $details): array + { + return ['status' => $status, 'checked_at' => now()->toIso8601String(), 'details' => $details]; + } +} diff --git a/core/config/platform.php b/core/config/platform.php index 2b16843..9b9beae 100644 --- a/core/config/platform.php +++ b/core/config/platform.php @@ -2,6 +2,16 @@ return [ 'groups' => [ + 'operations' => [ + 'label' => 'Operations and recovery', + 'description' => 'Bounded retention and freshness thresholds used by health checks, alerts, and recovery policy.', + 'fields' => [ + 'audit_retention_days' => ['type' => 'integer', 'label' => 'Audit retention (days)', 'default' => 365, 'description' => 'Delete audit events older than this period in bounded daily batches.', 'rules' => ['required', 'integer', 'between:30,3650']], + 'scheduler_stale_seconds' => ['type' => 'integer', 'label' => 'Scheduler stale threshold (seconds)', 'default' => 180, 'description' => 'Mark scheduler health degraded when its durable heartbeat is older than this threshold.', 'rules' => ['required', 'integer', 'between:60,3600']], + 'clock_drift_warning_seconds' => ['type' => 'integer', 'label' => 'Clock drift warning (seconds)', 'default' => 5, 'description' => 'Maximum accepted host clock offset for external monitoring and qualification.', 'rules' => ['required', 'integer', 'between:1,300']], + 'backup_stale_hours' => ['type' => 'integer', 'label' => 'Backup stale threshold (hours)', 'default' => 26, 'description' => 'Maximum age of the most recent verified encrypted off-host control database backup.', 'rules' => ['required', 'integer', 'between:1,168']], + ], + ], 'telemetry' => [ 'label' => 'Telemetry retention and privacy', 'description' => 'Bounded raw telemetry, aggregate retention, finalization delay, and client-address masking.', diff --git a/core/config/services.php b/core/config/services.php index 214fb36..1665b42 100644 --- a/core/config/services.php +++ b/core/config/services.php @@ -13,7 +13,18 @@ 'max_rows_to_read' => 10000000, 'max_result_rows' => 10001, ], - 'geoip' => ['database' => env('GEOIP_DATABASE', '/mmdb/GeoLite2-City.mmdb')], + 'geoip' => [ + 'database' => env('GEOIP_DATABASE', '/mmdb/GeoLite2-City.mmdb'), + 'stale_hours' => (int) env('MMDB_STALE_HOURS', 48), + ], + 'vector' => ['metrics_url' => env('VECTOR_METRICS_URL', 'http://vector:9598/metrics')], + 'metrics' => ['token' => env('METRICS_TOKEN', ''), 'token_file' => env('METRICS_TOKEN_FILE')], + 'backups' => [ + 'repository' => env('RESTIC_REPOSITORY', ''), 'password_file' => env('RESTIC_PASSWORD_FILE', ''), + 'access_key' => env('BACKUP_ACCESS_KEY_ID', ''), 'secret_key' => env('BACKUP_SECRET_ACCESS_KEY', ''), + 'region' => env('BACKUP_DEFAULT_REGION', 'us-east-1'), + ], + 'prometheus' => ['url' => env('PROMETHEUS_URL', 'http://prometheus:9090')], 'acme' => [ 'enabled' => filter_var(env('ACME_ENABLED', false), FILTER_VALIDATE_BOOL), 'verify_tls' => filter_var(env('ACME_VERIFY_TLS', true), FILTER_VALIDATE_BOOL), diff --git a/core/database/migrations/2026_07_20_010000_add_operations_settings.php b/core/database/migrations/2026_07_20_010000_add_operations_settings.php new file mode 100644 index 0000000..e300eef --- /dev/null +++ b/core/database/migrations/2026_07_20_010000_add_operations_settings.php @@ -0,0 +1,19 @@ +mapWithKeys(fn (array $field, string $key): array => [$key => $field['default']])->all(); + DB::table('system_settings')->insertOrIgnore(['group' => 'operations', 'values' => json_encode($values, JSON_THROW_ON_ERROR), 'revision' => 1, 'created_at' => now(), 'updated_at' => now()]); + } + + public function down(): void + { + DB::table('system_settings')->where('group', 'operations')->delete(); + } +}; diff --git a/core/database/migrations/2026_07_20_020000_create_backups_table.php b/core/database/migrations/2026_07_20_020000_create_backups_table.php new file mode 100644 index 0000000..634166b --- /dev/null +++ b/core/database/migrations/2026_07_20_020000_create_backups_table.php @@ -0,0 +1,33 @@ +uuid('id')->primary(); + $table->foreignId('requested_by')->nullable()->constrained('users')->nullOnDelete(); + $table->string('status', 20)->default('pending'); + $table->string('snapshot_id', 128)->nullable()->unique(); + $table->unsignedBigInteger('size_bytes')->nullable(); + $table->char('manifest_sha256', 64)->nullable(); + $table->text('last_error')->nullable(); + $table->timestampTz('verified_at')->nullable(); + $table->timestampsTz(); + $table->index(['status', 'created_at']); + }); + if (DB::getDriverName() === 'pgsql') { + DB::statement("ALTER TABLE backups ADD CONSTRAINT backups_status_check CHECK (status IN ('pending', 'running', 'succeeded', 'failed', 'deleting'))"); + } + } + + public function down(): void + { + Schema::dropIfExists('backups'); + } +}; diff --git a/core/database/migrations/2026_07_20_030000_add_operational_health_indexes.php b/core/database/migrations/2026_07_20_030000_add_operational_health_indexes.php new file mode 100644 index 0000000..91a91cf --- /dev/null +++ b/core/database/migrations/2026_07_20_030000_add_operational_health_indexes.php @@ -0,0 +1,49 @@ + $table->index(['state', 'desired_revision', 'domain_id'], 'domain_edge_placements_health_idx')); + Schema::table('edge_cells', fn (Blueprint $table) => $table->index(['status', 'drained', 'edge_id'], 'edge_cells_health_idx')); + Schema::table('cache_purges', fn (Blueprint $table) => $table->index(['status', 'created_at'], 'cache_purges_health_idx')); + Schema::table('usage_rollups', fn (Blueprint $table) => $table->index(['status', 'interval_end'], 'usage_rollups_health_idx')); + Schema::table('backups', fn (Blueprint $table) => $table->index(['status', 'verified_at'], 'backups_health_idx')); + } + + public function down(): void + { + if (DB::getDriverName() === 'pgsql') { + DB::statement('DROP INDEX CONCURRENTLY IF EXISTS domain_edge_placements_health_idx'); + DB::statement('DROP INDEX CONCURRENTLY IF EXISTS edge_cells_health_idx'); + DB::statement('DROP INDEX CONCURRENTLY IF EXISTS cache_purges_health_idx'); + DB::statement('DROP INDEX CONCURRENTLY IF EXISTS usage_rollups_health_idx'); + DB::statement('DROP INDEX CONCURRENTLY IF EXISTS backups_health_idx'); + + return; + } + + Schema::table('domain_edge_placements', fn (Blueprint $table) => $table->dropIndex('domain_edge_placements_health_idx')); + Schema::table('edge_cells', fn (Blueprint $table) => $table->dropIndex('edge_cells_health_idx')); + Schema::table('cache_purges', fn (Blueprint $table) => $table->dropIndex('cache_purges_health_idx')); + Schema::table('usage_rollups', fn (Blueprint $table) => $table->dropIndex('usage_rollups_health_idx')); + Schema::table('backups', fn (Blueprint $table) => $table->dropIndex('backups_health_idx')); + } +}; diff --git a/core/docker/backup/create.sh b/core/docker/backup/create.sh new file mode 100644 index 0000000..4ce57bf --- /dev/null +++ b/core/docker/backup/create.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +pg_dump --format=custom --no-owner --no-privileges | restic backup --stdin --stdin-filename control.pgdump --tag cdnfoundry-control --json diff --git a/core/docker/backup/restore.sh b/core/docker/backup/restore.sh new file mode 100644 index 0000000..9ee3e7a --- /dev/null +++ b/core/docker/backup/restore.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +snapshot_id="${1:-}" +case "$snapshot_id" in *[!a-f0-9]*|'') exit 64 ;; esac +restic dump "$snapshot_id" control.pgdump | pg_restore --clean --if-exists --no-owner --no-privileges --exit-on-error --dbname="$PGDATABASE" diff --git a/core/resources/views/filament/admin/pages/dashboard.blade.php b/core/resources/views/filament/admin/pages/dashboard.blade.php index da98a75..98d2726 100644 --- a/core/resources/views/filament/admin/pages/dashboard.blade.php +++ b/core/resources/views/filament/admin/pages/dashboard.blade.php @@ -7,6 +7,17 @@
+ +
+ @foreach ($this->componentState as $healthState) +
+
{{ $healthState['name'] }}
Checked {{ $healthState['checked_at'] }}
+ {{ str($healthState['status'])->headline() }} +
+ @endforeach +
+ +
@foreach ($this->queueState as $lane) diff --git a/core/routes/api.php b/core/routes/api.php index 4cf7304..1ff4ae6 100644 --- a/core/routes/api.php +++ b/core/routes/api.php @@ -2,6 +2,7 @@ use App\Http\Controllers\Admin\AnalyticsController as AdminAnalyticsController; use App\Http\Controllers\Admin\AuditLogController; +use App\Http\Controllers\Admin\BackupController; use App\Http\Controllers\Admin\DnsClusterController; use App\Http\Controllers\Admin\DnsOperationController; use App\Http\Controllers\Admin\DomainUserController; @@ -9,9 +10,12 @@ use App\Http\Controllers\Admin\EdgeController; use App\Http\Controllers\Admin\EdgeOperationsController; use App\Http\Controllers\Admin\EdgePoolController; +use App\Http\Controllers\Admin\FailedJobController; use App\Http\Controllers\Admin\LogController as AdminLogController; use App\Http\Controllers\Admin\PlatformDnsSettingsController; +use App\Http\Controllers\Admin\ReconciliationController; use App\Http\Controllers\Admin\SecurityOperationsController; +use App\Http\Controllers\Admin\SystemOperationsController; use App\Http\Controllers\Admin\SystemSettingsController; use App\Http\Controllers\Admin\UsageController as AdminUsageController; use App\Http\Controllers\Admin\UserController; @@ -158,6 +162,17 @@ Route::get('/dns/failed-deployments', [DnsOperationController::class, 'failures']); Route::post('/dns/reconcile', [DnsOperationController::class, 'reconcile'])->middleware('idempotent'); Route::get('/system/status', [HealthController::class, 'status']); + Route::get('/system/health', [SystemOperationsController::class, 'health']); + Route::get('/system/components', [SystemOperationsController::class, 'components']); + Route::get('/jobs/failed', [FailedJobController::class, 'index']); + Route::post('/jobs/failed/{job}/retry', [FailedJobController::class, 'retry'])->middleware('idempotent'); + Route::delete('/jobs/failed/{job}', [FailedJobController::class, 'destroy'])->middleware('idempotent'); + Route::post('/reconcile/{scope}', [ReconciliationController::class, 'run'])->whereIn('scope', ['dns', 'edges', 'tls', 'purges', 'usage'])->middleware('idempotent'); + Route::get('/backups', [BackupController::class, 'index']); + Route::post('/backups', [BackupController::class, 'store'])->middleware('idempotent'); + Route::get('/backups/{backup}', [BackupController::class, 'show']); + Route::post('/backups/{backup}/restore', [BackupController::class, 'restore'])->middleware('idempotent'); + Route::delete('/backups/{backup}', [BackupController::class, 'destroy'])->middleware('idempotent'); Route::get('/operations', [OperationController::class, 'index']); Route::get('/operations/{operation}', [OperationController::class, 'show']); Route::post('/operations/{operation}/retry', [OperationController::class, 'retry'])->middleware('idempotent'); diff --git a/core/routes/console.php b/core/routes/console.php index c64c5af..f913f0b 100644 --- a/core/routes/console.php +++ b/core/routes/console.php @@ -4,6 +4,7 @@ use App\Models\IdempotencyKey; use Illuminate\Foundation\Inspiring; use Illuminate\Support\Facades\Artisan; +use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Schedule; Artisan::command('inspire', function () { @@ -21,3 +22,6 @@ Schedule::command('tls:dispatch-maintenance')->hourly()->withoutOverlapping(); Schedule::command('security:reconcile-readiness')->everyMinute()->withoutOverlapping(); Schedule::command('usage:finalize')->hourlyAt(20)->withoutOverlapping(); +Schedule::command('audit:prune')->dailyAt('03:10')->withoutOverlapping(); +Schedule::command('backups:create')->dailyAt('01:30')->withoutOverlapping(); +Schedule::call(fn () => Cache::put('operations:scheduler_heartbeat', now()->toIso8601String(), now()->addMinutes(10)))->name('operations.scheduler-heartbeat')->everyMinute()->withoutOverlapping(); diff --git a/core/routes/web.php b/core/routes/web.php index 4e36d06..a8b9c6b 100644 --- a/core/routes/web.php +++ b/core/routes/web.php @@ -2,6 +2,7 @@ use App\Http\Controllers\Admin\UsageController as AdminUsageController; use App\Http\Controllers\EdgeAgentController; +use App\Http\Controllers\MetricsController; use App\Http\Controllers\UsageController; use Illuminate\Support\Facades\Route; @@ -22,6 +23,7 @@ Route::get('/', function () { return view('welcome'); }); +Route::get('/metrics', MetricsController::class); Route::middleware(['auth', 'account.active'])->group(function (): void { Route::get('/app/analytics/domains/{domain}/usage.csv', [UsageController::class, 'csv']) diff --git a/core/tests/Feature/BackupApiTest.php b/core/tests/Feature/BackupApiTest.php new file mode 100644 index 0000000..a7989b2 --- /dev/null +++ b/core/tests/Feature/BackupApiTest.php @@ -0,0 +1,117 @@ +mock(ResticBackupRepository::class, fn (MockInterface $mock) => $mock->shouldReceive('configured')->once()->andReturnTrue()); + $admin = User::factory()->admin()->create(); + $user = User::factory()->create(); + $this->actingAs($user)->withHeader('Idempotency-Key', (string) Str::uuid())->postJson('/api/admin/backups')->assertForbidden(); + $response = $this->actingAs($admin)->withHeader('Idempotency-Key', (string) Str::uuid())->postJson('/api/admin/backups')->assertAccepted(); + Queue::assertPushed(CreateControlBackup::class, fn ($job) => $job->backupId === $response->json('data.backup_id')); + $this->assertDatabaseHas('audit_logs', ['action' => 'backup.create_requested']); + $this->actingAs($admin)->getJson('/api/admin/backups')->assertOk()->assertJsonPath('data.0.status', 'pending'); + } + + public function test_restore_requires_exact_confirmation_reauthentication_and_preflight(): void + { + Queue::fake(); + $admin = User::factory()->admin()->create(['password' => 'correct horse battery staple']); + $backup = Backup::query()->create(['requested_by' => $admin->id, 'status' => 'succeeded', 'snapshot_id' => str_repeat('a', 64), 'verified_at' => now()]); + $url = "/api/admin/backups/{$backup->id}/restore"; + $this->actingAs($admin)->withHeader('Idempotency-Key', (string) Str::uuid())->postJson($url, ['confirmation' => 'wrong', 'current_password' => 'correct horse battery staple'])->assertUnprocessable(); + $response = $this->actingAs($admin)->withHeader('Idempotency-Key', (string) Str::uuid())->postJson($url, ['confirmation' => "RESTORE {$backup->id}", 'current_password' => 'correct horse battery staple'])->assertAccepted(); + Queue::assertPushed(PreflightBackupRestore::class, fn ($job) => $job->operationId === $response->json('data.operation_id')); + $this->assertDatabaseHas('audit_logs', ['action' => 'backup.restore_preflight_requested']); + } + + public function test_delete_is_async_and_running_backup_is_preserved(): void + { + Queue::fake(); + $admin = User::factory()->admin()->create(); + $running = Backup::query()->create(['requested_by' => $admin->id, 'status' => 'running']); + $this->actingAs($admin)->withHeader('Idempotency-Key', (string) Str::uuid())->deleteJson("/api/admin/backups/{$running->id}")->assertConflict(); + $backup = Backup::query()->create(['requested_by' => $admin->id, 'status' => 'succeeded', 'snapshot_id' => str_repeat('b', 64), 'verified_at' => now()]); + $response = $this->actingAs($admin)->withHeader('Idempotency-Key', (string) Str::uuid())->deleteJson("/api/admin/backups/{$backup->id}")->assertAccepted(); + Queue::assertPushed(DeleteControlBackup::class, fn ($job) => $job->operationId === $response->json('data.operation_id')); + $this->assertSame('deleting', $backup->refresh()->status); + } + + public function test_backup_job_records_verified_snapshot_and_operation_receipt(): void + { + $backup = Backup::query()->create(['status' => 'pending']); + $operation = Operation::query()->create(['type' => 'backup.create', 'status' => 'pending', 'input' => ['backup_id' => $backup->id]]); + $snapshot = str_repeat('c', 64); + $repository = $this->mock(ResticBackupRepository::class, fn (MockInterface $mock) => $mock->shouldReceive('create')->once()->andReturn([ + 'snapshot_id' => $snapshot, + 'size_bytes' => 1234, + 'manifest_sha256' => str_repeat('d', 64), + ])); + + (new CreateControlBackup($backup->id, $operation->id))->handle($repository); + + $this->assertSame('succeeded', $backup->refresh()->status); + $this->assertSame($snapshot, $backup->snapshot_id); + $this->assertNotNull($backup->verified_at); + $this->assertSame('succeeded', $operation->refresh()->status); + $this->assertSame($snapshot, $operation->result['snapshot_id']); + } + + public function test_backup_job_failure_is_bounded_and_preserves_durable_failure_state(): void + { + $backup = Backup::query()->create(['status' => 'running']); + $operation = Operation::query()->create(['type' => 'backup.create', 'status' => 'running', 'input' => ['backup_id' => $backup->id]]); + $job = new CreateControlBackup($backup->id, $operation->id); + + $job->failed(new RuntimeException(str_repeat('x', 5000))); + + $this->assertSame('failed', $backup->refresh()->status); + $this->assertSame(4000, mb_strlen($backup->last_error)); + $this->assertSame('failed', $operation->refresh()->status); + $this->assertSame(4000, mb_strlen($operation->error)); + } + + public function test_restore_executor_fails_closed_without_explicit_maintenance_permission(): void + { + putenv('BACKUP_RESTORE_ALLOWED'); + + $this->artisan('backups:restore', ['operation' => (string) Str::uuid()]) + ->expectsOutput('Set BACKUP_RESTORE_ALLOWED=true only in the one-off maintenance container.') + ->assertFailed(); + } + + public function test_restore_preflight_remains_running_until_maintenance_executor_finishes(): void + { + $snapshot = str_repeat('e', 64); + $backup = Backup::query()->create(['status' => 'succeeded', 'snapshot_id' => $snapshot, 'verified_at' => now()]); + $operation = Operation::query()->create(['type' => 'backup.restore', 'status' => 'pending', 'input' => ['backup_id' => $backup->id]]); + $repository = $this->mock(ResticBackupRepository::class, fn (MockInterface $mock) => $mock->shouldReceive('snapshotExists')->once()->with($snapshot)->andReturnTrue()); + + (new PreflightBackupRestore($backup->id, $operation->id))->handle($repository); + + $operation->refresh(); + $this->assertSame('running', $operation->status); + $this->assertSame('passed', $operation->result['preflight']); + $this->assertNull($operation->finished_at); + } +} diff --git a/core/tests/Feature/OperationsApiTest.php b/core/tests/Feature/OperationsApiTest.php new file mode 100644 index 0000000..c91f3f0 --- /dev/null +++ b/core/tests/Feature/OperationsApiTest.php @@ -0,0 +1,149 @@ + Http::response('Ok.', 200)]); + $admin = User::factory()->admin()->create(); + $user = User::factory()->create(); + $this->actingAs($user)->getJson('/api/admin/system/components')->assertForbidden(); + $response = $this->actingAs($admin)->getJson('/api/admin/system/components')->assertOk(); + $this->assertContains($response->json('data.status'), ['healthy', 'degraded', 'unavailable']); + $this->assertContains($response->json('data.components.control_database.status'), ['healthy', 'degraded', 'unavailable']); + foreach (['queue_workers', 'mmdb', 'edge_listeners', 'edge_cells', 'service_pools', 'edge_configuration', 'edge_placements', 'edge_capacity', 'emergency_modes', 'dns_deployments', 'purges'] as $component) { + $this->assertContains($response->json("data.components.{$component}.status"), ['healthy', 'degraded', 'unavailable']); + } + $this->assertSame(['interactive', 'runtime', 'certificate_purge', 'bulk_maintenance'], array_keys($response->json('data.queues'))); + } + + public function test_component_health_detects_runtime_drift_pressure_and_stale_mmdb(): void + { + Http::fake(['*' => Http::response('Ok.', 200)]); + $mmdb = tempnam(sys_get_temp_dir(), 'cdnf-mmdb-health-'); + file_put_contents($mmdb, 'qualified-mmdb-placeholder'); + touch($mmdb, now()->subHours(49)->timestamp); + config(['services.geoip.database' => $mmdb, 'services.geoip.stale_hours' => 48]); + + try { + $admin = User::factory()->admin()->create(); + $pool = EdgePool::query()->where('kind', 'shared')->firstOrFail(); + $edge = Edge::query()->create([ + 'id' => (string) Str::uuid(), 'name' => 'health-edge', 'country_code' => 'US', 'continent_code' => 'NA', + 'ipv4' => '192.0.2.80', 'ipv6' => '2001:db8::80', 'enabled' => true, 'drained' => false, + 'registered_at' => now(), 'last_heartbeat_at' => now(), 'agent_version' => '1.1.0', + 'capacity' => ['listener_ready' => false, 'last_rejection' => ['reason' => 'candidate_validation_failed']], + ]); + EdgeCell::query()->create([ + 'edge_id' => $edge->id, 'edge_pool_id' => $pool->id, 'name' => 'shared-health', + 'status' => 'degraded', 'capacity' => ['memory_usage' => 90, 'memory_limit' => 100], + ]); + $domain = Domain::query()->create([ + 'name' => 'health-drift.example', 'display_name' => 'Health drift', 'lifecycle_state' => 'active', + 'revision' => 2, 'active_edge_revision' => 1, + ]); + DomainEdgePlacement::query()->create([ + 'domain_id' => $domain->id, 'active_pool_id' => $pool->id, 'state' => 'active', 'desired_revision' => 2, + ]); + EmergencyMode::query()->create([ + 'id' => (string) Str::uuid(), 'target_type' => 'edge', 'target_id' => $edge->id, + 'actions' => ['allow_get_head_only'], 'active' => true, 'created_by' => $admin->id, + ]); + + $response = $this->actingAs($admin)->getJson('/api/admin/system/components')->assertOk(); + foreach (['mmdb', 'edge_listeners', 'edge_cells', 'service_pools', 'edge_configuration', 'edge_placements', 'edge_capacity', 'emergency_modes'] as $component) { + $response->assertJsonPath("data.components.{$component}.status", 'degraded'); + } + $response->assertJsonPath('data.components.edge_capacity.details.pressured_cells', 1) + ->assertJsonPath('data.components.edge_placements.details.drifted', 1) + ->assertJsonPath('data.components.emergency_modes.details.active', 1); + } finally { + @unlink($mmdb); + } + } + + public function test_clock_offset_beyond_database_threshold_is_degraded(): void + { + Http::fake([ + 'http://prometheus:9090/api/v1/query*' => Http::response(['status' => 'success', 'data' => ['result' => [['value' => [now()->timestamp, '6.25']]]]]), + '*' => Http::response('Ok.', 200), + ]); + $admin = User::factory()->admin()->create(); + $this->actingAs($admin)->getJson('/api/admin/system/components')->assertOk() + ->assertJsonPath('data.components.host_clock.status', 'degraded') + ->assertJsonPath('data.components.host_clock.details.maximum_offset_seconds', 6.25) + ->assertJsonPath('data.components.host_clock.details.warning_seconds', 5); + } + + public function test_failed_jobs_are_bounded_redacted_audited_and_deletable(): void + { + $admin = User::factory()->admin()->create(); + DB::table('failed_jobs')->insert([ + 'uuid' => (string) Str::uuid(), 'connection' => 'redis', 'queue' => 'runtime', + 'payload' => json_encode(['displayName' => 'App\\Jobs\\Example', 'data' => ['command' => 'sensitive serialized body']], JSON_THROW_ON_ERROR), + 'exception' => "RuntimeException: safe summary\nstack with internals", 'failed_at' => now(), + ]); + $job = DB::table('failed_jobs')->first(); + $this->actingAs($admin)->getJson('/api/admin/jobs/failed')->assertOk() + ->assertJsonPath('data.0.job', 'App\\Jobs\\Example')->assertJsonMissing(['sensitive serialized body']); + $this->actingAs($admin)->withHeader('Idempotency-Key', (string) Str::uuid())->deleteJson("/api/admin/jobs/failed/{$job->uuid}")->assertNoContent(); + $this->assertDatabaseMissing('failed_jobs', ['uuid' => $job->uuid]); + $this->assertDatabaseHas('audit_logs', ['action' => 'failed_job.deleted']); + } + + public function test_reconciliation_is_coalesced_and_dispatched_to_bounded_lanes(): void + { + Queue::fake(); + $admin = User::factory()->admin()->create(); + foreach (['dns' => ReconcileAllDnsZones::class, 'edges' => ReconcileAllEdgeDomains::class, 'tls' => ReconcileAllTls::class, 'purges' => ReconcileAllPurges::class] as $scope => $jobClass) { + $key = (string) Str::uuid(); + $first = $this->actingAs($admin)->withHeader('Idempotency-Key', $key)->postJson("/api/admin/reconcile/{$scope}")->assertAccepted(); + $this->actingAs($admin)->withHeader('Idempotency-Key', $key)->postJson("/api/admin/reconcile/{$scope}")->assertAccepted()->assertJsonPath('data.operation_id', $first->json('data.operation_id')); + Queue::assertPushed($jobClass, 1); + } + $this->assertSame(4, AuditLog::query()->where('action', 'like', '%.global_reconcile_requested')->count()); + } + + public function test_metrics_require_a_separate_bearer_token_and_never_expose_secrets(): void + { + Http::fake(['*' => Http::response('Ok.', 200)]); + config(['services.metrics.token' => 'metrics-test-token']); + $this->get('/metrics')->assertNotFound(); + $response = $this->withToken('metrics-test-token')->get('/metrics')->assertOk()->assertHeader('content-type', 'text/plain; version=0.0.4; charset=utf-8'); + $response->assertSee('cdnfoundry_component_health')->assertDontSee((string) config('app.key')); + } + + public function test_audit_pruning_is_bounded_and_uses_database_policy(): void + { + $old = User::factory()->create(); + AuditLog::record($old, 'old.event'); + AuditLog::query()->update(['created_at' => now()->subDays(400)]); + AuditLog::record($old, 'current.event'); + $this->artisan('audit:prune', ['--batch' => 1])->assertSuccessful(); + $this->assertDatabaseMissing('audit_logs', ['action' => 'old.event']); + $this->assertDatabaseHas('audit_logs', ['action' => 'current.event']); + } +} diff --git a/core/tests/Feature/SystemSettingsTest.php b/core/tests/Feature/SystemSettingsTest.php index fd49e46..86363e4 100644 --- a/core/tests/Feature/SystemSettingsTest.php +++ b/core/tests/Feature/SystemSettingsTest.php @@ -22,14 +22,15 @@ public function test_seeded_settings_expose_current_values_defaults_and_descript $admin = User::factory()->admin()->create(); $response = $this->actingAs($admin)->getJson('/api/admin/system/settings')->assertOk(); - $this->assertCount(7, $response->json('data')); + $this->assertCount(8, $response->json('data')); $settings = collect($response->json('data')); $dnsLifecycle = $settings->firstWhere('group', 'dns_lifecycle'); $this->assertSame(7, $dnsLifecycle['fields'][0]['value']); $this->assertSame(7, $dnsLifecycle['fields'][0]['default']); $this->assertNotEmpty($dnsLifecycle['fields'][0]['description']); $this->assertNotNull($settings->firstWhere('group', 'telemetry')); - $this->assertDatabaseCount('system_settings', 7); + $this->assertNotNull($settings->firstWhere('group', 'operations')); + $this->assertDatabaseCount('system_settings', 8); } public function test_dns_lifecycle_update_is_typed_audited_and_reads_from_postgresql(): void diff --git a/docker/backup/dev-restic-password b/docker/backup/dev-restic-password new file mode 100644 index 0000000..e6a0d17 --- /dev/null +++ b/docker/backup/dev-restic-password @@ -0,0 +1 @@ +cdnfoundry-development-restic-password-only diff --git a/docker/dnsdist/dnsdist.conf b/docker/dnsdist/dnsdist.conf index fead0d4..a7dd9d0 100644 --- a/docker/dnsdist/dnsdist.conf +++ b/docker/dnsdist/dnsdist.conf @@ -1,6 +1,16 @@ setLocal('0.0.0.0:53') setACL({'0.0.0.0/0', '::/0'}) +-- Prometheus reaches this read-only statistics endpoint only on the private +-- DNS network. Configuration-changing API routes remain authenticated and are +-- not enabled by this deployment. +webserver('0.0.0.0:8083') +setWebserverConfig({ + acl='0.0.0.0/0, ::/0', + statsRequireAuthentication=false, + apiRequiresAuthentication=true +}) + function addResolvedBackends(hostname, addresses) for _, address in ipairs(addresses) do newServer({address=address:toString(), name=hostname}) diff --git a/docker/nginx/edge-runtime.conf b/docker/nginx/edge-runtime.conf index 4f6f655..79cf56b 100644 --- a/docker/nginx/edge-runtime.conf +++ b/docker/nginx/edge-runtime.conf @@ -64,6 +64,8 @@ server { set $cdn_cache_respect_origin "1"; set $cdn_security_reason ""; set $cdn_security_action "allow"; + set $cdn_security_client_connection_key ""; + set $cdn_security_domain_connection_key ""; set $cdn_domain_id "0"; set $cdn_edge_id "unknown"; set $cdn_client_ip ""; diff --git a/docker/nginx/origin.conf b/docker/nginx/origin.conf index 2eff0e2..43694d4 100644 --- a/docker/nginx/origin.conf +++ b/docker/nginx/origin.conf @@ -12,6 +12,7 @@ server { location = /stale { add_header Cache-Control "public, max-age=1, stale-if-error=10"; return 200 "stale\n"; } location = /large-object { root /tmp; try_files /large-object =404; } location = /slow { limit_rate 1k; root /tmp; try_files /large-object =404; } + location = /graceful { limit_rate 1k; root /tmp; try_files /graceful-object =404; } location / { default_type application/json; return 200 '{"origin":"http","host":"$host"}'; diff --git a/docker/nginx/proxy-cache.conf b/docker/nginx/proxy-cache.conf index 9f4141d..5962cdc 100644 --- a/docker/nginx/proxy-cache.conf +++ b/docker/nginx/proxy-cache.conf @@ -3,7 +3,7 @@ proxy_cache_key $cdn_cache_key; proxy_cache_lock on; proxy_cache_lock_timeout 3s; proxy_cache_bypass $cdn_cache_bypass; -proxy_no_cache $cdn_cache_bypass $upstream_http_x_cdnfoundry_no_store $upstream_http_set_cookie; +proxy_no_cache $cdn_cache_bypass $cdn_cache_no_store $upstream_http_x_cdnfoundry_no_store $upstream_http_set_cookie; proxy_cache_methods GET HEAD; proxy_cache_valid 200 366d; proxy_cache_valid any 0; diff --git a/docker/openresty/runtime.lua b/docker/openresty/runtime.lua index 2d4fe22..7f2b3c6 100644 --- a/docker/openresty/runtime.lua +++ b/docker/openresty/runtime.lua @@ -398,19 +398,25 @@ function M.access() local burst = tonumber(limits.request_burst) or 200 if client_requests and client_requests > rps + burst then return security_reject(429, "client_rate_exceeded") end if domain_requests and domain_requests > rps * 8 + burst then return security_reject(429, "domain_rate_exceeded") end - local client_connections = dictionary:incr("security:conn:client:" .. tostring(config.domain) .. ":" .. ngx.md5(client), 1, 0) - local domain_connections = dictionary:incr("security:conn:domain:" .. tostring(config.domain), 1, 0) + local client_connection_key = "security:conn:client:" .. tostring(config.domain) .. ":" .. ngx.md5(client) + local domain_connection_key = "security:conn:domain:" .. tostring(config.domain) + local client_connections = dictionary:incr(client_connection_key, 1, 0) + local domain_connections = dictionary:incr(domain_connection_key, 1, 0) if client_connections and client_connections > (tonumber(limits.connections_per_client) or 64) then - dictionary:incr("security:conn:client:" .. tostring(config.domain) .. ":" .. ngx.md5(client), -1, 0) - dictionary:incr("security:conn:domain:" .. tostring(config.domain), -1, 0) + dictionary:incr(client_connection_key, -1, 0) + dictionary:incr(domain_connection_key, -1, 0) return security_reject(429, "client_connections_exceeded") end if domain_connections and domain_connections > (tonumber(limits.connections_per_domain) or 512) then - dictionary:incr("security:conn:client:" .. tostring(config.domain) .. ":" .. ngx.md5(client), -1, 0) - dictionary:incr("security:conn:domain:" .. tostring(config.domain), -1, 0) + dictionary:incr(client_connection_key, -1, 0) + dictionary:incr(domain_connection_key, -1, 0) return security_reject(429, "domain_connections_exceeded") end - ngx.ctx.security_connection_keys = {"security:conn:client:" .. tostring(config.domain) .. ":" .. ngx.md5(client), "security:conn:domain:" .. tostring(config.domain)} + -- ngx.exec enters a named cache location and may replace ngx.ctx. Nginx + -- request variables survive that redirect and let the log phase release + -- the active-request counters reliably. + ngx.var.cdn_security_client_connection_key = client_connection_key + ngx.var.cdn_security_domain_connection_key = domain_connection_key if config.settings and config.settings.redirect_https == true and ngx.var.scheme == "http" then return ngx.redirect("https://" .. host .. ngx.var.request_uri, 308) end @@ -435,11 +441,11 @@ function M.access() and development_until <= ngx.time() and #cache_key <= (tonumber(limits.maximum_cache_key_length) or 4096) local admissions = dictionary:incr("security:cache:" .. tostring(config.domain) .. ":" .. second, 1, 0, 2) - if admissions and admissions > (tonumber(limits.cache_admissions_per_second) or 50) then cacheable = false end + local admission_allowed = not admissions or admissions <= (tonumber(limits.cache_admissions_per_second) or 50) if #cache_key > (tonumber(limits.maximum_cache_key_length) or 4096) then ngx.var.cdn_security_reason = "cache_abuse_detected" end ngx.var.cdn_cache_key = cache_key ngx.var.cdn_cache_bypass = cacheable and "0" or "1" - ngx.var.cdn_cache_no_store = "0" + ngx.var.cdn_cache_no_store = cacheable and admission_allowed and "0" or "1" ngx.var.cdn_cache_edge_ttl = tostring(cache_ttl) ngx.var.cdn_cache_browser_ttl = tostring(math.max(0, math.min(31536000, tonumber(cache.browser_ttl_seconds) or 0))) ngx.var.cdn_cache_max_object = tostring(math.max(1024, math.min(1073741824, tonumber(cache.maximum_object_bytes) or 104857600))) @@ -561,7 +567,8 @@ end function M.cache_status() local no_store = ngx.var.upstream_http_x_cdnfoundry_no_store == "1" - ngx.header["X-CDNFoundry-Cache"] = (ngx.var.cdn_cache_bypass == "1" or no_store) + local admission_bypass = ngx.var.cdn_cache_no_store == "1" and ngx.var.upstream_cache_status ~= "HIT" + ngx.header["X-CDNFoundry-Cache"] = (ngx.var.cdn_cache_bypass == "1" or no_store or admission_bypass) and "BYPASS" or (ngx.var.upstream_cache_status or "MISS") end @@ -607,11 +614,14 @@ function M.record_passive_failure() end function M.finish() - local keys = ngx.ctx.security_connection_keys - if not keys then return end - for _, key in ipairs(keys) do - local current = ngx.shared.runtime_limits:incr(key, -1, 0) - if current and current <= 0 then ngx.shared.runtime_limits:delete(key) end + for _, key in ipairs({ + ngx.var.cdn_security_client_connection_key, + ngx.var.cdn_security_domain_connection_key, + }) do + if key and key ~= "" then + local current = ngx.shared.runtime_limits:incr(key, -1, 0) + if current and current <= 0 then ngx.shared.runtime_limits:delete(key) end + end end end @@ -683,6 +693,13 @@ function M.passive_failures() memory_usage = tonumber(memory_file:read("*l")) or 0 memory_file:close() end + local memory_limit = nil + local memory_limit_file = io.open("/sys/fs/cgroup/memory.max", "r") + if memory_limit_file then + local raw_limit = memory_limit_file:read("*l") + if raw_limit ~= "max" then memory_limit = tonumber(raw_limit) end + memory_limit_file:close() + end local cpu_usage = 0 local cpu_file = io.open("/sys/fs/cgroup/cpu.stat", "r") if cpu_file then @@ -713,8 +730,11 @@ function M.passive_failures() origin_connections = ngx.shared.runtime_limits:get("capacity:origin_connections") or 0, cpu_usage = cpu_usage, memory_usage = memory_usage, + memory_limit = memory_limit or cjson.null, cache_usage = 10 * 1024 * 1024 - cache_free, + cache_limit = 10 * 1024 * 1024, cache_free_bytes = cache_free, + connection_limit = 4096, temporary_storage_usage = cjson.null, telemetry_buffer_usage = cjson.null, rejected_requests = ngx.shared.runtime_limits:get("capacity:rejected_requests") or 0, diff --git a/docker/prometheus/alerts.test.yml b/docker/prometheus/alerts.test.yml new file mode 100644 index 0000000..8257d80 --- /dev/null +++ b/docker/prometheus/alerts.test.yml @@ -0,0 +1,42 @@ +rule_files: + - telemetry-alerts.yml +evaluation_interval: 1m +tests: + - interval: 1m + input_series: + - series: 'node_timex_sync_status{instance="test-host"}' + values: '0 0 0 0' + - series: 'node_timex_offset_seconds{instance="test-host"}' + values: '6 6 6 6' + - series: 'up{job="dnsdist",instance="dns-host"}' + values: '0 0 0 0' + - series: 'up{job="powerdns",instance="pdns-host"}' + values: '0 0 0 0' + - series: 'dnsdist_server_status{instance="dns-host",server="pdns-auth"}' + values: '0 0 0 0' + alert_rule_test: + - eval_time: 3m + alertname: HostClockUnsynchronized + exp_alerts: + - exp_labels: { severity: critical, instance: test-host } + exp_annotations: { summary: A CDNFoundry host clock is not synchronized } + - eval_time: 3m + alertname: HostClockDrift + exp_alerts: + - exp_labels: { severity: critical, instance: test-host } + exp_annotations: { summary: A CDNFoundry host clock offset exceeds five seconds } + - eval_time: 3m + alertname: DNSDistUnavailable + exp_alerts: + - exp_labels: { severity: critical, job: dnsdist, instance: dns-host } + exp_annotations: { summary: DNSdist authoritative ingress is unavailable } + - eval_time: 3m + alertname: PowerDNSUnavailable + exp_alerts: + - exp_labels: { severity: critical, job: powerdns, instance: pdns-host } + exp_annotations: { summary: PowerDNS authoritative backend is unavailable } + - eval_time: 3m + alertname: DNSDistBackendUnavailable + exp_alerts: + - exp_labels: { severity: critical, instance: dns-host, server: pdns-auth } + exp_annotations: { summary: DNSdist has marked an authoritative backend unavailable } diff --git a/docker/prometheus/dev-metrics-token b/docker/prometheus/dev-metrics-token new file mode 100644 index 0000000..64d3014 --- /dev/null +++ b/docker/prometheus/dev-metrics-token @@ -0,0 +1 @@ +cdnfoundry-dev-metrics-only diff --git a/docker/prometheus/prometheus.yml b/docker/prometheus/prometheus.yml index 35864d5..8339bc9 100644 --- a/docker/prometheus/prometheus.yml +++ b/docker/prometheus/prometheus.yml @@ -7,9 +7,24 @@ alerting: - static_configs: - targets: [alertmanager:9093] scrape_configs: + - job_name: cdnfoundry-control + metrics_path: /metrics + authorization: + credentials_file: /run/secrets/metrics-token + static_configs: + - targets: [web:8080] - job_name: vector static_configs: - targets: [vector:9598] + - job_name: node + static_configs: + - targets: [node-exporter:9100] + - job_name: dnsdist + static_configs: + - targets: [dnsdist:8083] + - job_name: powerdns + static_configs: + - targets: [pdns-auth:8081] - job_name: alertmanager static_configs: - targets: [alertmanager:9093] diff --git a/docker/prometheus/telemetry-alerts.yml b/docker/prometheus/telemetry-alerts.yml index 594e815..fbaa130 100644 --- a/docker/prometheus/telemetry-alerts.yml +++ b/docker/prometheus/telemetry-alerts.yml @@ -1,4 +1,66 @@ groups: + - name: cdnfoundry-operations + rules: + - alert: ControlPlaneMetricsUnavailable + expr: up{job="cdnfoundry-control"} == 0 + for: 2m + labels: { severity: critical } + annotations: + summary: CDNFoundry operational metrics are unavailable + - alert: CDNFoundryComponentUnhealthy + expr: cdnfoundry_component_health == 0 + for: 5m + labels: { severity: warning } + annotations: + summary: A CDNFoundry dependency or reconciliation component is degraded + - alert: CDNFoundryQueueBacklog + expr: cdnfoundry_queue_depth > 1000 or cdnfoundry_queue_oldest_job_age_seconds > 900 + for: 5m + labels: { severity: warning } + annotations: + summary: A bounded CDNFoundry worker lane has excessive depth or age + - alert: CDNFoundryFailedOperations + expr: cdnfoundry_operations_failed > 0 + for: 10m + labels: { severity: warning } + annotations: + summary: CDNFoundry has failed operations requiring inspection + - alert: CDNFoundryCertificateExpiry + expr: cdnfoundry_tls_certificates_expiring > 0 + for: 5m + labels: { severity: critical } + annotations: + summary: One or more active TLS certificates are inside the expiry alert window + - alert: DNSDistUnavailable + expr: up{job="dnsdist"} == 0 + for: 2m + labels: { severity: critical } + annotations: + summary: DNSdist authoritative ingress is unavailable + - alert: PowerDNSUnavailable + expr: up{job="powerdns"} == 0 + for: 2m + labels: { severity: critical } + annotations: + summary: PowerDNS authoritative backend is unavailable + - alert: DNSDistBackendUnavailable + expr: dnsdist_server_status == 0 + for: 2m + labels: { severity: critical } + annotations: + summary: DNSdist has marked an authoritative backend unavailable + - alert: HostClockUnsynchronized + expr: node_timex_sync_status == 0 + for: 2m + labels: { severity: critical } + annotations: + summary: A CDNFoundry host clock is not synchronized + - alert: HostClockDrift + expr: abs(node_timex_offset_seconds) > 5 + for: 2m + labels: { severity: critical } + annotations: + summary: A CDNFoundry host clock offset exceeds five seconds - name: cdnfoundry-telemetry rules: - alert: TelemetryEventsDropped diff --git a/docs/architecture.md b/docs/architecture.md index 3548fbc..e3d20b7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -414,9 +414,11 @@ exist: - ClickHouse is provisioned and Vector currently exports its own internal Prometheus metrics. The repository does not yet claim that raw edge request telemetry is flowing into ClickHouse. -- Prometheus/Alertmanager services exist, but complete product monitoring and - alert coverage must follow their roadmap qualification rather than being - assumed from container presence. +- Prometheus privately scrapes protected control metrics, Vector, and Node + Exporter. Alertmanager rules cover component health, bounded queue backlog, + failed operations, certificate expiry, telemetry delivery/buffering, and host + clock synchronization/drift. Production alert rehearsals remain explicit + Phase 8 qualification gates rather than being inferred from container presence. Verified completion status remains in [roadmap.md](roadmap.md). This distinction is important: architecture documentation must describe real runtime behaviour, diff --git a/docs/manual-browser-qualification.md b/docs/manual-browser-qualification.md index e134986..0869d89 100644 --- a/docs/manual-browser-qualification.md +++ b/docs/manual-browser-qualification.md @@ -622,6 +622,75 @@ two Vector batch intervals. Record exact UTC generation times and byte counts. - Manual browser qualification: owner-run; **not executed and Phase 7 is not release-qualified until every checkpoint above is recorded as passed**. +## Phase 8 — Operations and production qualification + +### Administrator operations + +1. Sign in at `/admin`. Confirm **Component health** shows every component with + exactly Healthy, Degraded, or Unavailable and a check time. The API detail + must include control database, queue backend/workers, scheduler, host clock, + MMDB, DNS/deployments, ClickHouse, Vector, edge nodes/listeners/cells/pools, + placement/configuration/capacity, emergency modes, TLS, purges/runtime tasks, + usage, operations, and backups. Stop ClickHouse; confirm only + analytics/telemetry degrades while DNSdist UDP/TCP and edge HTTP/HTTPS + continue. Restart it and confirm recovery. +2. Open **Platform settings → Operations and recovery**. Set audit retention + `365`, scheduler stale threshold `180`, clock drift warning `5`, and backup + stale threshold `26`. Save, refresh, and confirm typed persistence. Values + outside displayed bounds must reject the whole save. +3. Create one disposable failed queue job. Call the failed-jobs API and confirm + its lane, job name, first exception line, and time appear but serialized + arguments/secrets do not. Correct the cause, retry it, and confirm an audit + event. Delete a second disposable failure and confirm deletion is audited. +4. Invoke DNS, edge, TLS, purge, and usage reconciliation with five UUID + `Idempotency-Key` values. Repeat each request and confirm one operation per + scope, bounded progress, completion, and no duplicate per-resource storm. +5. Open Prometheus and confirm `cdnfoundry-control` is up; inspect component, + queue depth/age, failed operation, DNS drift, stale edge, and certificate + expiry series. Trigger one disposable threshold and confirm Alertmanager + receives and later resolves the expected rule without customer labels or + secrets. +6. Sign in as the domain user. Direct requests to system health/components, + failed jobs, all reconciliation routes, settings, and Horizon must be + forbidden. `/metrics` without the separate bearer token must be not found. + +### Recovery, upgrade, and scale evidence + +1. Using the approved off-host system, create an encrypted control PostgreSQL + backup. Record external ID, checksum, cutoff time, encryption recipient, and + separate recovery location for decryption material. Never attach secrets to + this record. +2. Restore on an empty environment and a separate fresh replacement host. Supply + the exact recovery set, run forward migrations, rebuild PowerDNS, enroll a + fresh edge from a full snapshot, reconcile lost queues and TLS, rebuild one + retained usage interval, and verify DNSdist UDP/TCP and edge IPv4/IPv6 + HTTP/HTTPS. Record measured RPO and RTO. +3. Stop/restart control, DNSdist, PowerDNS, one DNS database, one edge, + ClickHouse, Vector, and the MMDB provider one at a time. Confirm last-valid + serving, isolation, graceful activation, recovery, and alerts for each. +4. Create real clock offset beyond `5` seconds on a disposable host or inject it + through the qualified host exporter. Confirm degraded health and alert, then + restore synchronization and confirm resolution. +5. Canary one prior/current mixed-version control worker, DNS target, and edge. + Confirm artifact compatibility, stop thresholds, then roll back application + images without restoring PostgreSQL. +6. Run the roadmap dataset: at least 500,000 domains, 1,000,000 DNS records, + 50,000 daily changes, burst mutations, repeated-domain coalescing, and + concurrent multi-DNS/multi-edge deployment. Add an edge and prove an + edge-health change does not rewrite every domain. + +### Phase 8 completion gate + +- Implementation: present for the encrypted Restic backup API/CLI and + maintenance-only restore executor. +- Documentation: current operations/recovery runbook and checklist are present. +- Automated/runtime qualification: incomplete until every outstanding item in + `phase-8-qualification.md` has recorded evidence. +- Manual browser/host qualification: owner-run; **not executed and not complete + until every Phase 8 checkpoint above is recorded as passed**. + +--- + ## Record the result For each phase record: date/operator, commit SHA, browser/version, desktop/mobile viewports, exact domain and edge addresses, every checkpoint as pass/fail/not-ready, operation IDs, revisions, screenshots, relevant logs, and any deviations from the example values. Also record Horizon, PowerAdmin, DNSdist UDP/TCP, Prometheus, Alertmanager, and edge results where applicable. diff --git a/docs/openapi.json b/docs/openapi.json index ea3ba2e..462826d 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -106,6 +106,240 @@ } } }, + "/admin/backups": { + "get": { + "operationId": "backup.controller.index", + "tags": [ + "Administrator" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "$ref": "#/components/parameters/Cursor" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + } + } + }, + "post": { + "operationId": "backup.controller.store", + "tags": [ + "Administrator" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "$ref": "#/components/parameters/IdempotencyKey" + } + ], + "requestBody": { + "required": false, + "content": { + "application/json": { + "schema": { + "type": "object", + "maxProperties": 100, + "additionalProperties": true + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "201": { + "$ref": "#/components/responses/Created" + }, + "202": { + "$ref": "#/components/responses/Accepted" + }, + "204": { + "description": "The mutation completed with no response body." + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + }, + "409": { + "$ref": "#/components/responses/StableError" + }, + "422": { + "$ref": "#/components/responses/ValidationError" + } + } + } + }, + "/admin/backups/{backup}": { + "delete": { + "operationId": "backup.controller.destroy", + "tags": [ + "Administrator" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "backup", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "$ref": "#/components/parameters/IdempotencyKey" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "201": { + "$ref": "#/components/responses/Created" + }, + "202": { + "$ref": "#/components/responses/Accepted" + }, + "204": { + "description": "The mutation completed with no response body." + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + }, + "409": { + "$ref": "#/components/responses/StableError" + }, + "422": { + "$ref": "#/components/responses/ValidationError" + } + } + }, + "get": { + "operationId": "backup.controller.show", + "tags": [ + "Administrator" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "backup", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + } + } + } + }, + "/admin/backups/{backup}/restore": { + "post": { + "operationId": "backup.controller.restore", + "tags": [ + "Administrator" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "backup", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "$ref": "#/components/parameters/IdempotencyKey" + } + ], + "requestBody": { + "required": false, + "content": { + "application/json": { + "schema": { + "type": "object", + "maxProperties": 100, + "additionalProperties": true + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "201": { + "$ref": "#/components/responses/Created" + }, + "202": { + "$ref": "#/components/responses/Accepted" + }, + "204": { + "description": "The mutation completed with no response body." + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + }, + "409": { + "$ref": "#/components/responses/StableError" + }, + "422": { + "$ref": "#/components/responses/ValidationError" + } + } + } + }, "/admin/dns/clusters": { "get": { "operationId": "dns.cluster.controller.index", @@ -2843,6 +3077,153 @@ } } }, + "/admin/jobs/failed": { + "get": { + "operationId": "failed.job.controller.index", + "tags": [ + "Administrator" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "$ref": "#/components/parameters/Cursor" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + } + } + } + }, + "/admin/jobs/failed/{job}": { + "delete": { + "operationId": "failed.job.controller.destroy", + "tags": [ + "Administrator" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "job", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "$ref": "#/components/parameters/IdempotencyKey" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "201": { + "$ref": "#/components/responses/Created" + }, + "202": { + "$ref": "#/components/responses/Accepted" + }, + "204": { + "description": "The mutation completed with no response body." + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + }, + "409": { + "$ref": "#/components/responses/StableError" + }, + "422": { + "$ref": "#/components/responses/ValidationError" + } + } + } + }, + "/admin/jobs/failed/{job}/retry": { + "post": { + "operationId": "failed.job.controller.retry", + "tags": [ + "Administrator" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "job", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "$ref": "#/components/parameters/IdempotencyKey" + } + ], + "requestBody": { + "required": false, + "content": { + "application/json": { + "schema": { + "type": "object", + "maxProperties": 100, + "additionalProperties": true + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "201": { + "$ref": "#/components/responses/Created" + }, + "202": { + "$ref": "#/components/responses/Accepted" + }, + "204": { + "description": "The mutation completed with no response body." + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + }, + "409": { + "$ref": "#/components/responses/StableError" + }, + "422": { + "$ref": "#/components/responses/ValidationError" + } + } + } + }, "/admin/logs/edges": { "get": { "operationId": "log.controller.index", @@ -3059,6 +3440,125 @@ } } }, + "/admin/reconcile/{scope}": { + "post": { + "operationId": "reconciliation.controller.run", + "tags": [ + "Administrator" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "scope", + "in": "path", + "required": true, + "schema": { + "type": "string", + "enum": [ + "dns", + "edges", + "tls", + "purges", + "usage" + ] + } + }, + { + "$ref": "#/components/parameters/IdempotencyKey" + } + ], + "requestBody": { + "required": false, + "content": { + "application/json": { + "schema": { + "type": "object", + "maxProperties": 100, + "additionalProperties": true + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "201": { + "$ref": "#/components/responses/Created" + }, + "202": { + "$ref": "#/components/responses/Accepted" + }, + "204": { + "description": "The mutation completed with no response body." + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + }, + "409": { + "$ref": "#/components/responses/StableError" + }, + "422": { + "$ref": "#/components/responses/ValidationError" + } + } + } + }, + "/admin/system/components": { + "get": { + "operationId": "system.operations.controller.components", + "tags": [ + "Administrator" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + } + } + } + }, + "/admin/system/health": { + "get": { + "operationId": "system.operations.controller.health", + "tags": [ + "Administrator" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Success" + }, + "401": { + "$ref": "#/components/responses/StableError" + }, + "403": { + "$ref": "#/components/responses/StableError" + } + } + } + }, "/admin/system/settings": { "get": { "operationId": "system.settings.controller.index", @@ -3295,6 +3795,7 @@ "schema": { "type": "string", "enum": [ + "operations", "telemetry", "dns_lifecycle", "revision_history", @@ -3336,6 +3837,7 @@ "schema": { "type": "string", "enum": [ + "operations", "telemetry", "dns_lifecycle", "revision_history", diff --git a/docs/operations/operations-and-recovery.md b/docs/operations/operations-and-recovery.md new file mode 100644 index 0000000..fc363bb --- /dev/null +++ b/docs/operations/operations-and-recovery.md @@ -0,0 +1,178 @@ +# Operations, recovery, and upgrade runbook + +## Operational state + +Administrators use `GET /api/admin/system/health` for the overall state and +`GET /api/admin/system/components` for component detail and bounded queue-lane +depth/age. States are `healthy`, `degraded`, and `unavailable`. ClickHouse or +Vector failure degrades observability but does not make DNS or HTTP serving +unavailable. The administrator dashboard summarizes the same checks. + +Prometheus scrapes `GET /metrics` across the private `control` network with the +bearer token stored in the mode-0600 file named by `METRICS_TOKEN_FILE`. Do not +publish this endpoint or token. Alertmanager receives control-plane scrape, +component, queue, failed-operation, certificate-expiry, clock, and telemetry +alerts. The private Node Exporter provides NTP synchronization and clock offset; +the control-plane health summary compares the maximum absolute offset with the +PostgreSQL-backed **Clock drift warning** setting. Production qualification must +still rehearse a real offset on a disposable host and confirm alert resolution. + +Inspect failed durable operations before raw queue failures. The failed-jobs API +returns only job name, lane, first exception line, and timestamps; it never +returns serialized payloads. Retry only after correcting the cause. Deleting a +failed row is audited and does not repair desired state. Run the appropriate +coalesced reconciliation afterward: + +```text +POST /api/admin/reconcile/dns +POST /api/admin/reconcile/edges +POST /api/admin/reconcile/tls +POST /api/admin/reconcile/purges +POST /api/admin/reconcile/usage +``` + +Every call needs administrator authentication and an `Idempotency-Key`. Global +jobs page through bounded batches on `bulk_maintenance`; per-domain runtime work +uses its existing lane and unique job. Audit pruning runs daily in batches and +uses the PostgreSQL-backed retention setting. + +## Monitoring and alert reference + +| Signal | Default alert condition | First response | +|---|---|---| +| Control scrape | unavailable for 2 minutes | Check private routing, token file permissions, then control readiness. | +| Component health | any component degraded for 5 minutes | Inspect the named component and preserve unrelated serving paths. | +| Queue lane | depth over 1,000 or oldest item over 15 minutes for 5 minutes | Stop bulk producers, inspect Horizon and failed operations, then reconcile. | +| Failed operation | any durable failed operation for 10 minutes | Correct its stable error, retry or reconcile, and retain audit evidence. | +| Certificate expiry | an active certificate enters the configured alert window for 5 minutes | Preserve the active certificate, repair DNS-01/CA access, and reconcile TLS. | +| DNSdist | private metrics scrape unavailable, or any authoritative backend down, for 2 minutes | Preserve healthy targets, inspect the named backend and DNS database, then verify UDP and TCP answers through DNSdist. | +| PowerDNS | private metrics scrape unavailable for 2 minutes | Keep DNSdist on healthy backends, repair the private PowerDNS/API/database path, then reconcile desired zones. | +| Edge runtime | stale heartbeat, listener/cell/pool failure, placement/configuration drift, emergency mode, or resource use at 80% of a reported limit | Isolate the affected edge/cell/pool, retain last-valid artifacts, add capacity or reconcile only the affected scope. | +| MMDB | file missing/unreadable, empty, or older than 48 hours | Retain the last valid database, repair the updater, and verify IPv4 and IPv6 lookups before activation. | +| Host clock | unsynchronized or absolute offset over 5 seconds for 2 minutes | Drain the affected host if signatures/certificates may be unsafe, repair NTP, and confirm resolution. | +| Vector | scrape loss, dropped events, delivery errors, or buffer above 80% | Keep serving, preserve the partial-data interval, and repair the sink/buffer. | + +Alert labels contain only bounded infrastructure identifiers; customer domains, +request paths, credentials, and serialized jobs are not alert labels. Tune the +database health threshold and the matching Prometheus rule together, then run +`promtool test rules /etc/prometheus/alerts.test.yml` before deployment. +DNSdist exposes only read-only statistics on port 8083 of the private DNS +network; its configuration-changing API remains authenticated and disabled for +Prometheus. PowerDNS metrics and both database/API paths remain private. + +## Failure routing + +- Control PostgreSQL: stop mutations, keep DNS/edges serving their last valid + state, restore the complete encrypted backup on a replacement host, run only + forward migrations, then reconcile DNS, edges, TLS, purges, and usage. +- Valkey/Horizon: serving continues. Restore Valkey, restart Horizon, inspect + failed operations, and run all reconciliation endpoints; queue contents are + not part of the minimum recovery set. +- DNS cluster: disable the unhealthy target, keep healthy clusters active, + repair its private database/API, test it, enable it, and reconcile DNS. +- Edge/cell: drain when reachable; otherwise withdraw only the affected pool or + edge addresses. Add a replacement edge, wait for full snapshot acknowledgement, + then restore routing. Never remove the last active source before target ack. +- Certificate: retain the last valid certificate, correct DNS-01 or CA failure, + reconcile TLS, and confirm edge acknowledgement before expiry. +- ClickHouse/Vector: follow [ClickHouse outage](../clickhouse-outage-runbook.md). + Serving must continue and loss intervals must be recorded rather than guessed. +- MMDB: retain the last checksum-validated file. Repair provider access and + confirm both IPv4 and IPv6 lookup before activation. + +## Backup and clean-host restore + +The recovery set is the encrypted control PostgreSQL backup, `APP_KEY`, artifact +signing key, edge identity CA, listener identities, typed environment files, +metrics token, and externally held custom TLS material. Store backup encryption +material separately. PowerDNS, Valkey queue state, edge snapshots, and +ClickHouse are not substitutes for control PostgreSQL. + +Production backups stream `pg_dump` custom-format output directly into the +configured S3-compatible Restic repository; no unbounded local dump is staged. +Initialize the repository once with its separately stored password, then use +the backup API or `php artisan backups:create`. The API records snapshot ID, +size, verification time, bounded failure, operation, and audit evidence. S3 +credentials should be restricted to the dedicated repository prefix. + +Restore requires the exact `RESTORE ` value and current +administrator password. It queues a repository preflight and returns an +operation. After the preflight succeeds, stop normal control-plane workers and +run the returned `php artisan backups:restore ` command in a +one-off maintenance container with `BACKUP_RESTORE_ALLOWED=true`. A failed +restore deliberately leaves maintenance mode active. A successful restore runs +forward migrations, records a receipt, queues all reconciliations, and leaves +maintenance mode. + +Do not claim recovery merely because a snapshot exists. Verify Restic packs and +restoreability, and record the immutable snapshot identifier. Restore on an empty replacement host, +start private dependencies, run `make prod-migrate` and `make prod-pdns-migrate`, +then start control, DNS, telemetry, and a fresh edge. Run all reconciliations and +verify DNSdist UDP/TCP plus edge IPv4/IPv6 HTTP/HTTPS. Record backup cutoff, +first successful DNS/HTTP response, measured RPO, and measured RTO. + +Backup files are never downloadable through Laravel. The local Restic repository +in development exists only for automated qualification and is not an off-host +production backup. + +Repeat the agent-owned portions with `make dev-phase8-e2e`, +`make dev-phase8-recovery-e2e`, `make dev-phase8-upgrade-e2e`, and +`make dev-phase8-throughput-e2e`. Use `make dev-phase8-mmdb-e2e` for the +last-valid MMDB provider-outage rehearsal. The recovery job creates only disposable +tmpfs PostgreSQL/object-host containers and must never remove named volumes. +Its measured times are local evidence; record separate end-to-end times on the +approved fresh replacement host. + +## Canary upgrade and rollback + +Use an immutable commit-SHA release. Back up first, apply only expand-compatible +migrations, upgrade one control worker, one DNS target, and one edge agent/cell. +Artifacts already carry schema version and minimum/maximum agent versions. +Stop rollout on an unhealthy component, increased error rate, configuration +rejection, stale revision, or queue-age alert. Roll application containers back +to the prior immutable SHA without restoring the database. Contract migrations +are admitted only after the rollback window closes and a separate release proves +the previous application is no longer required. + +Use `make prod-pull` and the host-role commands in +[Production Compose service sets](../production-layout.md). Record both image +digests, schema migration set, artifact schema/agent bounds, canary operation +IDs, stop thresholds, and rollback result. Replace Horizon workers gracefully +with `php artisan horizon:terminate`; do not stop an edge during candidate +activation. Roll back the immutable application images only, never PostgreSQL. + +## Secret and identity rotation + +- API tokens: create a replacement, verify its narrowly scoped client, revoke + the old token, then confirm the old token is rejected. Never log either value. +- Edge mTLS identity: use the administrator rotate-identity workflow, enroll the + replacement once, verify heartbeat/artifact acknowledgement, and confirm the + revoked serial cannot authenticate. +- Listener/server certificates and CAs: follow + [Production certificate rotation](production-certificates.md), retaining both + trust anchors through the overlap window. +- Restic/S3 credentials: verify a backup with the new repository credential, + revoke the old S3 key, and retain the Restic repository password in a separate + recovery system. Changing an S3 key does not re-encrypt existing Restic packs. +- `APP_KEY` and artifact-signing/identity-CA private keys are recovery roots, not + routine in-place rotations. A change requires an explicit data/key migration + and fleet trust transition; never silently replace one during deployment. +- PostgreSQL, Valkey, ClickHouse, metrics, and ACME credentials: install the new + value on the private dependency and consumers, restart one bounded component + at a time, verify health, then revoke the old value. + +## Capacity planning + +Treat the published 500,000-domain/1,000,000-record run as correctness evidence, +not a universal capacity promise. Before production, repeat `make dev-scale-e2e` +on the intended control/DNS hardware and record CPU model/count, memory, storage, +network, image digests, dataset, latency, throughput, errors, and saturation. +Measure per-edge HTTP/HTTPS throughput separately with cache HIT/MISS, TLS, +IPv4/IPv6, request-size, connection, origin, and telemetry mixes. + +Add capacity when a sustained queue lane approaches its alert threshold, DNS +latency/error budget is consumed, or an edge reports CPU, memory, file-descriptor, +connection, cache, or temporary-storage pressure. Scale by adding bounded +workers, DNSdist/PowerDNS capacity, ClickHouse capacity, or edge nodes/cells. +Adding an edge must not create a per-domain runtime or force unrelated domain +revisions; verify this with placement/reconciliation metrics after enrollment. diff --git a/docs/phase-8-qualification.md b/docs/phase-8-qualification.md new file mode 100644 index 0000000..9e57a56 --- /dev/null +++ b/docs/phase-8-qualification.md @@ -0,0 +1,139 @@ +# Phase 8 qualification record + +Phase 8 implementation and local agent-owned qualification are complete. Phase +8 is **not release-qualified** because the fresh physical replacement-host, +external backup system, disposable-host clock, production-like canary, and +owner-run browser/real-traffic gates still require owner infrastructure. + +## Implemented and automatically covered + +- Admin-only component health with stable states and bounded/redacted failed-job + inspection, audited retry/delete, and protected Prometheus metrics. +- Coalesced DNS, edge, TLS, purge, and usage reconciliation on bounded lanes. +- PostgreSQL-backed operational thresholds, scheduler heartbeat, alert rules, + and bounded audit retention. +- Horizon master freshness, MMDB age, DNSdist and PowerDNS scrape health, + DNSdist backend state, and bounded edge listener/cell/pool, drift, emergency, + and reported-resource-pressure health. +- Encrypted Restic backup API/CLI, asynchronous preflight, exact restore + confirmation plus password re-authentication, redacted metadata, operation + records, and a maintenance-only restore executor. +- Expand-compatible schema changes, an explicit edge-agent compatibility + version, recovery/upgrade/capacity runbooks, and the exact owner checklist. + +The isolated Laravel suite covers permissions, validation, redaction, +idempotent/coalesced dispatch, metrics authentication, backup lifecycle, +restore confirmation/re-authentication, and audit pruning. Browser automation +was not run. + +## Recovery evidence + +`tests/e2e/phase8_operations.py` passed the application backup and restore +lifecycle with backup `019f7f96-89ce-72f3-8c75-115fe7b98cc5` and Restic +snapshot `b85acac3880a58138b0611c9f26b876ac09b2f2decabfb303d50427facf8af17`. +Full-pack verification succeeded and the dump restored into an empty temporary +PostgreSQL 18 instance with source/restored counts equal. + +`tests/e2e/phase8_recovery.py` then used a separate, disposable S3-compatible +object host (`quay.io/minio/minio:RELEASE.2025-07-23T15-54-02Z`), independently +generated object credentials, and separately generated Restic decryption +material. It rejected a wrong repository password, verified 100% of repository +data, and restored snapshot +`2be94c40679387f32e3223d1d461f732ffae852405d9db597f15fd0a1861f04c` into a +fresh PostgreSQL 18.4 tmpfs container. The marker and these counts matched: +42 users, 73 domains, 304 DNS records, 61 DNS deployments, 2 edges, 46 edge +artifacts, and 2 TLS certificates. The current application then applied forward +migrations on the replacement and reached the repository's exact 39-migration +schema. Against a new empty Valkey instance, five bounded global reconciliation +jobs all succeeded and reconstructed 25 runtime/certificate-purge jobs from +durable state. The usage reconciler rebuilt one finalized hourly interval for +each of the 14 active domains from retained ClickHouse data. Measured +backup-cutoff RPO was 11.114 seconds, restore, forward-migration, and +reconciliation RTO was 31.669 seconds, and total exercise time was 51.501 +seconds. No named volume was removed. + +This proves encrypted object-host recovery in clean replacement containers on +the qualification dataset, including queue reconstruction and usage rebuilding. +It does not substitute for the roadmap's external off-host repository and fresh +physical replacement-host rehearsal, where PowerDNS, a new edge, TLS, and real +DNS/HTTP traffic must also be reconstructed and timed. + +## Upgrade and runtime evidence + +`tests/e2e/phase8_upgrade.py` built the prior committed release +`a584fee012d8280c2e694b1cc7703ae333454ce5` and the candidate independently. +The prior/current edge agents reported `1.0.0`/`1.1.0`; signed artifact tests +passed in both builds. On fresh PostgreSQL, the prior release installed 36 +migrations, the candidate expanded to 39, and the prior release then ran and +wrote successfully against the expanded schema. Rollback restored no database +backup. This proves the mixed-version application/schema contract locally; the +multi-host control-worker, DNS-target, and edge canary remains an owner gate. + +The real Phase 4 OpenResty suite passed after adding regressions for cache +admission and more than 64 sequential cache hits. The qualification exposed and +fixed two bounded-runtime defects: cache-admission pressure could bypass an +already resident object, and active-request counters were not released after an +internal cache redirect. Resident cache hits now remain available and request +counters are released in the log phase through request-scoped Nginx variables. +The same real-runtime job proves an invalid replacement never displaces the +last-valid configuration and an 8 KiB in-flight response completes after the +OpenResty master receives graceful `SIGQUIT`. + +## Capacity, restart, and isolation evidence + +`tests/e2e/phase8_throughput.py` passed on Linux 6.8.0-134 x86_64 with an Intel +Xeon E5-2697 v4, 32 logical CPUs, and 16,784,982,016 bytes host memory. The +single OpenResty cell was restricted to 1 CPU, 512 MiB, 128 PIDs, and 65,536 +file descriptors. Its image was +`sha256:2f968173f12efa3372d2116da3b84716df788a250743f69cdf4611f3145bcf0e`. +The profile used 64 pre-warmed cache-HIT domains and 32 isolated client +containers paced below the runtime's 100 requests/second/client bound, with +HTTP/1.1 new connections: + +| Transport | Requests | Errors | Requests/s | p50 | p95 | p99 | +|---|---:|---:|---:|---:|---:|---:| +| HTTP | 4,729 | 0 | 268.68 | 2.178 ms | 5.883 ms | 19.788 ms | +| HTTPS | 872 | 0 | 53.14 | 162.363 ms | 551.890 ms | 842.368 ms | + +These are reproducible lower-bound results for the stated constrained cell and +new-connection workload, not a universal hardware promise. + +The established scale job remains valid: 500,000 domains, 1,000,000 DNS +records, 50,000 changes, and a 10,000-change burst passed. Existing real-runtime +evidence also proves: + +- control PostgreSQL, Valkey, Laravel, Horizon, Scheduler, and web can be down + while existing authoritative DNS continues; PowerDNS and DNSdist restart + with the latest answers (`phase2_dns.py`); +- a sibling edge/cell failure remains isolated, target activation precedes + source drain, and bounded cell drain/restart recovers traffic + (`phase4_control_plane.py` and `phase4_runtime.py`); +- ClickHouse and Vector outage/restart does not stop DNS or edge serving and the + bounded Vector backlog drains (`phase7_analytics.py`). +- a real MMDB updater provider failure preserves the previous checksum, leaves + no activation candidate, and keeps the updater running + (`phase8_mmdb.py`, checksum + `6cfd04ff7d30de5be30016afbe41bd240ffe1c1c0d6fbfe47d52bf1a609131f1`). + +Promtool rule tests prove unsynchronized time and absolute clock offset over +five seconds alert after two minutes. The API suite proves the database-backed +threshold degrades component health. A real offset on a disposable host remains +an external gate. The pinned DNSdist 2.1 configuration check and live private +scrapes also passed: the DNSdist container became healthy and Prometheus +reported both `dnsdist` and `powerdns` targets up. The API suite injects stale +MMDB, listener/cell/pool failure, edge configuration and placement drift, +resource pressure, and an active emergency mode and verifies each component +degrades independently. + +## Outstanding release gates + +- Restore from the approved external encrypted backup system onto a fresh + physical replacement host; rebuild PowerDNS, queue state, a fresh edge, TLS, + and retained usage while measuring end-to-end RPO/RTO and real traffic. +- Run the mixed-version control-worker, DNS-target, and edge canary/rollback on + production-like separate hosts. +- Rehearse and resolve real clock drift on a disposable host. +- Complete and record every owner-run Phase 8 browser and final real-traffic + checkpoint in `docs/manual-browser-qualification.md`. + +No unavailable infrastructure test is reported as passed. diff --git a/docs/production-layout.md b/docs/production-layout.md index a4b22c6..ba4d8fc 100644 --- a/docs/production-layout.md +++ b/docs/production-layout.md @@ -32,10 +32,17 @@ The default edge and quarantine cells use separate listeners, cache volumes, tem ## Process lifecycle -`core`, `horizon`, and `scheduler` are independent services. Restart one at a time; committed PostgreSQL state is not tied to process lifetime. Replace workers gracefully with `php artisan horizon:terminate`. Readiness checks use bounded PostgreSQL and Valkey timeouts. `/api/health` is process-only liveness; `/api/ready` checks required dependencies; `/api/admin/system/status` adds queue depth and oldest queued-job age per lane. +`core`, `horizon`, and `scheduler` are independent services. Restart one at a time; committed PostgreSQL state is not tied to process lifetime. Replace workers gracefully with `php artisan horizon:terminate`. Readiness checks use bounded PostgreSQL and Valkey timeouts. `/api/health` is process-only liveness; `/api/ready` checks required dependencies; `/api/admin/system/components` adds dependency state plus queue depth and oldest queued-job age per lane. + +Production Compose gives control workers bounded stop windows and sends +`SIGQUIT` to Nginx/OpenResty listeners, allowing existing requests to finish +before container replacement. Do not use forced removal for routine rollout. +The edge agent and runtime still enforce target activation and acknowledgement +before source drain; a graceful process stop is not a substitute for that +placement protocol. ## Durable recovery set -Back up the control PostgreSQL database, `.env.prod` secrets (especially `APP_KEY`), the artifact-signing key, edge identity CA, listener identities, and externally held custom TLS keys. Managed certificate keys are encrypted in control PostgreSQL and are unrecoverable without the same `APP_KEY`. PowerDNS runtime PostgreSQL is derived and rebuildable from desired state, though a backup can shorten recovery. A clean-host restore/RPO/RTO claim is intentionally deferred until the Phase 8 recovery qualification is actually run; Phase 1–5 documentation does not claim it today. +Back up the control PostgreSQL database through the encrypted off-host Restic workflow, plus `.env.prod` secrets (especially `APP_KEY`), the Restic password/decryption material stored separately, artifact-signing key, edge identity CA, listener identities, and externally held custom TLS keys. Managed certificate keys are encrypted in control PostgreSQL and are unrecoverable without the same `APP_KEY`. PowerDNS runtime PostgreSQL is derived and rebuildable from desired state, though a backup can shorten recovery. See [Operations and recovery](operations/operations-and-recovery.md). A clean-host RPO/RTO claim remains deferred until the production qualification is recorded. Do not place control PostgreSQL, Valkey, PowerDNS, ClickHouse, or internal metrics on a public network. Use host firewalls in addition to Compose internal networks. diff --git a/docs/roadmap.md b/docs/roadmap.md index 373a9e5..c237570 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -2748,9 +2748,9 @@ Filament summarizes operational state. Prometheus and Alertmanager remain the mo - [ ] Encryption keys and secrets are included in the documented recovery set. - [ ] PowerDNS runtime state is rebuilt from control-plane data. - [ ] A fresh edge recovers from a full snapshot. -- [ ] Queue loss is repaired by reconciliation. +- [x] Queue loss is repaired by reconciliation. - [ ] TLS state is reconstructed after edge loss. -- [ ] Usage intervals can be rebuilt from retained ClickHouse data. +- [x] Usage intervals can be rebuilt from retained ClickHouse data. - [ ] Runbooks exist for control DB, DNS cluster, edge, certificate, ClickHouse, Vector, queue backlog, and MMDB failure. - [ ] Measured RPO and RTO are recorded. - [ ] Clock drift beyond the configured threshold produces a degraded health state and alert. @@ -2762,8 +2762,8 @@ Filament summarizes operational state. Prometheus and Alertmanager remain the mo - [ ] One failed edge does not interrupt healthy edges. - [ ] DNSdist and PowerDNS restart tests pass. - [ ] ClickHouse and Vector restart without traffic interruption. -- [ ] MMDB provider outage retains the last valid file. -- [ ] Graceful shutdown prevents partially activated state. +- [x] MMDB provider outage retains the last valid file. +- [x] Graceful shutdown prevents partially activated state. - [ ] A canary control-plane/agent upgrade succeeds through a mixed-version window and can roll back without database restore. - [ ] A stale edge is removed from new system edge-routing answers according to policy. - [ ] A drained edge stops receiving new preferred traffic while completing existing work according to runtime capability. @@ -2805,6 +2805,17 @@ Filament summarizes operational state. Prometheus and Alertmanager remain the mo - [ ] Every required failure runbook - [ ] Complete OpenAPI reference +> **Implementation progress (2026-07-20):** Phase 8 operations implementation +> and local agent-owned qualification are present. Encrypted S3-compatible +> object-host recovery into clean replacement PostgreSQL passed with measured +> local RPO/RTO; prior/current images passed an expand-schema rollback without a +> database restore; constrained single-cell HTTP/HTTPS results are published; +> and the cumulative restart/isolation suites remain green. Phase 8 is not +> release-qualified until the external off-host/fresh physical-host recovery, +> production-like multi-host canary, disposable-host clock rehearsal, and +> owner-run browser/real-traffic checkpoints are recorded. +> Evidence: [Phase 8 qualification](phase-8-qualification.md). + --- ### 11. Final Browser and Real-Traffic Acceptance Test diff --git a/edge-agent/main.go b/edge-agent/main.go index 06eadc2..77cd57e 100644 --- a/edge-agent/main.go +++ b/edge-agent/main.go @@ -29,7 +29,7 @@ import ( "time" ) -const version = "1.0.0" +const version = "1.1.0" type identity struct{ EdgeID, Certificate, PrivateKey, PublicKey string } type state struct { @@ -55,6 +55,10 @@ type manifest struct { } func main() { + if len(os.Args) == 2 && os.Args[1] == "--version" { + fmt.Println(version) + return + } c := &client{ base: strings.TrimRight(required("EDGE_CONTROL_URL"), "/"), dir: env("EDGE_STATE_DIR", "/var/lib/cdnfoundry/agent"), runtimeDir: env("EDGE_RUNTIME_DIR", ""), statusToken: env("EDGE_STATUS_TOKEN", ""), diff --git a/edge-agent/main_test.go b/edge-agent/main_test.go index 64c5e1e..9393be9 100644 --- a/edge-agent/main_test.go +++ b/edge-agent/main_test.go @@ -37,11 +37,17 @@ func TestVerifyAndCompatibility(t *testing.T) { if _, err := verify(base64.StdEncoding.EncodeToString(append(payload, 'x')), checksum, hex.EncodeToString(signature), hex.EncodeToString(public)); err == nil { t.Fatal("tampered payload accepted") } - if !compatible("1.0.0", "1.99.99") || compatible("1.1.0", "1.99.99") { + if !compatible("1.0.0", "1.99.99") || compatible("1.0.0", "1.0.99") { t.Fatal("compatibility bounds are incorrect") } } +func TestVersionCommand(t *testing.T) { + if version != "1.1.0" { + t.Fatalf("unexpected release version %q", version) + } +} + func TestAcknowledgementBufferRetriesAfterRecovery(t *testing.T) { failing := true server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/tests/e2e/phase4_runtime.py b/tests/e2e/phase4_runtime.py index ec56682..04b6dfb 100644 --- a/tests/e2e/phase4_runtime.py +++ b/tests/e2e/phase4_runtime.py @@ -14,6 +14,7 @@ QUARANTINE_NAME = "cdnf-phase4-quarantine-e2e" AGENT_NAME = "cdnf-phase4-agent-e2e" DEDICATED_NAME = "cdnf-phase4-dedicated-e2e" +GRACEFUL_CLIENT_NAME = "cdnf-phase4-graceful-client-e2e" EDGE_NETWORK = os.environ.get( "CDNF_EDGE_NETWORK", f"{os.environ.get('COMPOSE_PROJECT_NAME', 'cdnfoundry-dev')}_edge", @@ -166,8 +167,14 @@ def main() -> None: initial = state({"runtime.example": "origin-one.example"}, 1) initial["hosts"]["development.example"] = state({"development.example": "development-origin.example"}, 1)["hosts"]["development.example"] initial["hosts"]["development.example"]["cache"]["development_mode_until"] = int(time.time()) + 3600 - for cache_host in ("admission.example", "origin-policy.example", "small-object.example", "stale.example", "no-stale.example"): + for cache_host in ( + "admission.example", "admission-limit.example", "origin-policy.example", + "small-object.example", "stale.example", "no-stale.example", + ): initial["hosts"][cache_host] = state({cache_host: "cache-origin.example"}, 1)["hosts"][cache_host] + initial["hosts"]["admission-limit.example"]["security"] = { + "limits": {"cache_admissions_per_second": 1}, + } initial["hosts"]["origin-policy.example"]["cache"].update({ "edge_ttl_seconds": 3, "browser_ttl_seconds": 7, "respect_origin_headers": False, }) @@ -254,6 +261,16 @@ def main() -> None: assert "X-CDNFoundry-Cache: HIT" in request_with("runtime.example", "/asset.css?a=1").stderr assert "X-CDNFoundry-Cache: MISS" in request_with("runtime.example", "/asset.css?a=2").stderr assert "max-age=300" in cached.stderr, cached.stderr + # The admission ceiling blocks creation of additional cache entries; + # it must never bypass a resident entry and stampede the origin. + time.sleep(1.05 - (time.time() % 1)) + admitted = request_with("admission-limit.example", "/resident") + resident = request_with("admission-limit.example", "/resident") + assert "X-CDNFoundry-Cache: MISS" in admitted.stderr, admitted.stderr + assert "X-CDNFoundry-Cache: HIT" in resident.stderr, resident.stderr + for _ in range(70): + resident = request_with("admission-limit.example", "/resident") + assert resident.returncode == 0 and "X-CDNFoundry-Cache: HIT" in resident.stderr, resident.stderr for path in ("/set-cookie", "/private", "/no-store", "/vary-star", "/vary-language"): first = request_with("admission.example", path) second = request_with("admission.example", path) @@ -402,8 +419,19 @@ def main() -> None: runtime.write_text('{"invalid"') time.sleep(1.5) assert request("runtime.example").returncode == 0 + graceful_client = subprocess.Popen([ + "docker", "run", "--rm", "--name", GRACEFUL_CLIENT_NAME, + "--network", f"container:{NAME}", "curlimages/curl:8.16.0", + "-fsS", "-o", "/dev/null", "-H", "Host: runtime.example", + "http://127.0.0.1:8080/graceful", + ], cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + time.sleep(0.75) + run("docker", "kill", "--signal=QUIT", NAME) + _, graceful_error = graceful_client.communicate(timeout=15) + assert graceful_client.returncode == 0, graceful_error finally: if os.environ.get("CDNF_KEEP_FAILED_RUNTIME") != "1": + run("docker", "rm", "-f", GRACEFUL_CLIENT_NAME, check=False) run("docker", "rm", "-f", NAME, check=False) run("docker", "stop", QUARANTINE_NAME, check=False) run("docker", "stop", AGENT_NAME, check=False) diff --git a/tests/e2e/phase8_mmdb.py b/tests/e2e/phase8_mmdb.py new file mode 100644 index 0000000..f9f4ec3 --- /dev/null +++ b/tests/e2e/phase8_mmdb.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Real MMDB provider-outage retention qualification; no browser automation.""" + +from __future__ import annotations + +import json +import os +import pathlib +import secrets +import subprocess +import tempfile +import time + +ROOT = pathlib.Path(__file__).resolve().parents[2] +PROJECT = os.environ.get("COMPOSE_PROJECT_NAME", "cdnfoundry-dev") +MMDB_VOLUME = os.environ.get("CDNF_MMDB_VOLUME", f"{PROJECT}_mmdb") +IMAGE = os.environ.get("CDNF_MMDB_IMAGE", "cdnfoundry/mmdb-updater:phase8") +NAME = f"cdnfoundry-phase8-mmdb-{secrets.token_hex(4)}" + + +def run(*args: str, timeout: int = 180, check: bool = True) -> subprocess.CompletedProcess[str]: + result = subprocess.run(args, cwd=ROOT, text=True, capture_output=True, timeout=timeout, check=False) + if check and result.returncode != 0: + raise RuntimeError(f"command failed: {' '.join(args)}\n{result.stdout}\n{result.stderr}") + return result + + +def checksum(directory: pathlib.Path) -> str: + return run( + "docker", "run", "--rm", "-v", f"{directory}:/mmdb:ro", "alpine:3.23", + "sha256sum", "/mmdb/GeoLite2-City.mmdb", + ).stdout.split()[0] + + +def main() -> None: + run("docker", "build", "-t", IMAGE, "docker/mmdb-updater", timeout=600) + with tempfile.TemporaryDirectory(prefix="cdnfoundry-phase8-mmdb-") as temporary: + directory = pathlib.Path(temporary) + run( + "docker", "run", "--rm", "-v", f"{MMDB_VOLUME}:/source:ro", "-v", f"{directory}:/target", + "alpine:3.23", "cp", "/source/GeoLite2-City.mmdb", "/target/GeoLite2-City.mmdb", + ) + before = checksum(directory) + run( + "docker", "run", "-d", "--rm", "--name", NAME, + "-e", "MMDB_PROVIDER=generic", "-e", "MMDB_DOWNLOAD_INTERVAL_SECONDS=300", + "-e", "MMDB_DOWNLOAD_RETRIES=0", "-v", f"{directory}:/mmdb", IMAGE, + ) + try: + deadline = time.monotonic() + 20 + logs = "" + while time.monotonic() < deadline: + logs = run("docker", "logs", NAME, check=False).stdout + if "initial update failed; preserving existing database" in logs: + break + time.sleep(0.5) + else: + raise AssertionError(f"updater did not report bounded provider failure: {logs[-2000:]}") + running = run("docker", "inspect", "--format={{.State.Running}}", NAME).stdout.strip() + after = checksum(directory) + if running != "true" or before != after: + raise AssertionError({"running": running, "before": before, "after": after}) + if (directory / ".GeoLite2-City.mmdb.candidate").exists(): + raise AssertionError("failed provider left an activation candidate") + print(json.dumps({ + "phase8_mmdb_outage": "passed", "database_checksum": after, + "provider_failure_logged": True, "last_valid_retained": True, + }, sort_keys=True)) + finally: + run("docker", "stop", NAME, check=False) + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/phase8_operations.py b/tests/e2e/phase8_operations.py new file mode 100644 index 0000000..1f1ee06 --- /dev/null +++ b/tests/e2e/phase8_operations.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Real Phase 8 operations, encrypted backup, and isolated restore qualification.""" + +from __future__ import annotations + +import json +import os +import pathlib +import secrets +import subprocess +import time +import urllib.error +import urllib.request +import uuid + +ROOT = pathlib.Path(__file__).resolve().parents[2] +COMPOSE = os.environ.get("CDNF_COMPOSE_FILE", "compose.dev.yml") +BASE = os.environ.get("CDNF_BASE_URL", "http://localhost:8080").rstrip("/") +RUN = f"{int(time.time())}-{secrets.token_hex(4)}" +EMAIL = f"phase8-admin-{RUN}@example.test" +PASSWORD = f"Phase8-1-{secrets.token_urlsafe(20)}" +RESTORE_CONTAINER = f"cdnfoundry-phase8-restore-{RUN}" + + +def run(*args: str, timeout: int = 120, check: bool = True) -> subprocess.CompletedProcess[str]: + result = subprocess.run(args, cwd=ROOT, text=True, capture_output=True, timeout=timeout, check=False) + if check and result.returncode != 0: + raise RuntimeError(f"command failed: {' '.join(args)}\n{result.stdout}\n{result.stderr}") + return result + + +def compose(*args: str, timeout: int = 120) -> subprocess.CompletedProcess[str]: + return run("docker", "compose", "-f", COMPOSE, *args, timeout=timeout) + + +def request(method: str, path: str, payload: object | None = None, token: str | None = None, + key: str | None = None) -> tuple[int, object]: + headers = {"Accept": "application/json"} + data = None + if payload is not None: + headers["Content-Type"] = "application/json" + data = json.dumps(payload).encode() + if token: + headers["Authorization"] = f"Bearer {token}" + if key: + headers["Idempotency-Key"] = key + call = urllib.request.Request(f"{BASE}{path}", method=method, headers=headers, data=data) + try: + with urllib.request.urlopen(call, timeout=15) as response: + body = response.read() + return response.status, json.loads(body) if body else {} + except urllib.error.HTTPError as error: + body = error.read() + try: + decoded = json.loads(body) if body else {} + except json.JSONDecodeError: + decoded = body.decode(errors="replace") + return error.code, decoded + + +def php_string(value: str) -> str: + return "'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'" + + +def wait_backup(token: str, backup_id: str) -> dict[str, object]: + deadline = time.monotonic() + 180 + while time.monotonic() < deadline: + status, body = request("GET", f"/api/admin/backups/{backup_id}", token=token) + assert status == 200, body + backup = body["data"] + if backup["status"] == "succeeded": + return backup + if backup["status"] == "failed": + raise AssertionError(backup) + time.sleep(1) + raise AssertionError("backup did not finish") + + +def counts(container: str, user: str, database: str) -> set[str]: + sql = "SELECT 'users='||count(*) FROM users UNION ALL SELECT 'domains='||count(*) FROM domains UNION ALL SELECT 'dns_records='||count(*) FROM dns_records UNION ALL SELECT 'migrations='||count(*) FROM migrations;" + return set(run("docker", "exec", container, "psql", "-U", user, "-d", database, "-Atc", sql).stdout.splitlines()) + + +def ensure_development_repository() -> None: + command = ( + "docker", "compose", "-f", COMPOSE, "exec", "-T", "--user", "www-data", "core", "restic", "snapshots", + ) + probe = run(*command, check=False) + if probe.returncode == 0: + return + details = f"{probe.stdout}\n{probe.stderr}" + if "repository does not exist" not in details: + raise RuntimeError(f"unable to inspect development Restic repository:\n{details}") + compose("exec", "-T", "--user", "www-data", "core", "restic", "init") + + +def main() -> None: + ensure_development_repository() + expression = ( + "App\\Models\\User::query()->create([" + f"'name'=>'Phase 8 runtime admin','email'=>{php_string(EMAIL)}," + f"'password'=>Illuminate\\Support\\Facades\\Hash::make({php_string(PASSWORD)}),'type'=>'admin']);" + ) + compose("exec", "-T", "core", "php", "artisan", "tinker", f"--execute={expression}") + status, login = request("POST", "/api/auth/login", {"email": EMAIL, "password": PASSWORD, "device_name": "phase8-e2e"}) + assert status == 200, login + token = login["data"]["token"] + + status, components = request("GET", "/api/admin/system/components", token=token) + assert status == 200 and components["data"]["status"] in {"healthy", "degraded", "unavailable"}, components + assert set(components["data"]["queues"]) == {"interactive", "runtime", "certificate_purge", "bulk_maintenance"} + assert request("GET", "/metrics")[0] == 404 + + key = str(uuid.uuid4()) + status, created = request("POST", "/api/admin/backups", {}, token, key) + assert status == 202, created + replay_status, replay = request("POST", "/api/admin/backups", {}, token, key) + assert replay_status == 202 and replay == created + backup = wait_backup(token, created["data"]["backup_id"]) + snapshot = backup["snapshot_id"] + assert isinstance(snapshot, str) and len(snapshot) == 64 + compose("exec", "-T", "core", "restic", "check", "--read-data-subset=100%", timeout=180) + + wrong, _ = request("POST", f"/api/admin/backups/{backup['id']}/restore", {"confirmation": "wrong", "current_password": PASSWORD}, token, str(uuid.uuid4())) + assert wrong == 422 + restore_status, restore = request("POST", f"/api/admin/backups/{backup['id']}/restore", {"confirmation": f"RESTORE {backup['id']}", "current_password": PASSWORD}, token, str(uuid.uuid4())) + assert restore_status == 202, restore + + run("docker", "run", "-d", "--rm", "--name", RESTORE_CONTAINER, "--network", "cdnfoundry-dev_control", + "-e", "POSTGRES_DB=cdnf_restore", "-e", "POSTGRES_USER=cdnf_restore", "-e", "POSTGRES_PASSWORD=phase8-restore-only", + "--tmpfs", "/var/lib/postgresql:rw,nosuid,size=1g", "postgres:18.4-alpine") + try: + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + if run("docker", "exec", RESTORE_CONTAINER, "pg_isready", "-U", "cdnf_restore", "-d", "cdnf_restore", check=False).returncode == 0: + break + time.sleep(1) + compose("exec", "-T", "-e", f"PGHOST={RESTORE_CONTAINER}", "-e", "PGPORT=5432", "-e", "PGDATABASE=cdnf_restore", + "-e", "PGUSER=cdnf_restore", "-e", "PGPASSWORD=phase8-restore-only", "core", "/usr/local/bin/cdnf-backup-restore", snapshot, timeout=180) + restored = counts(RESTORE_CONTAINER, "cdnf_restore", "cdnf_restore") + source_sql = "SELECT 'users='||count(*) FROM users UNION ALL SELECT 'domains='||count(*) FROM domains UNION ALL SELECT 'dns_records='||count(*) FROM dns_records UNION ALL SELECT 'migrations='||count(*) FROM migrations;" + source = set(compose("exec", "-T", "control-db", "psql", "-U", "cdnf", "-d", "cdnf", "-Atc", source_sql).stdout.splitlines()) + assert restored == source, (restored, source) + finally: + run("docker", "stop", RESTORE_CONTAINER, check=False) + + print(f"phase8_operations=passed backup={backup['id']} snapshot={snapshot}") + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/phase8_recovery.py b/tests/e2e/phase8_recovery.py new file mode 100644 index 0000000..58e8c4d --- /dev/null +++ b/tests/e2e/phase8_recovery.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 +"""Encrypted off-host S3 backup and clean replacement PostgreSQL qualification.""" + +from __future__ import annotations + +import json +import os +import pathlib +import secrets +import subprocess +import time + +ROOT = pathlib.Path(__file__).resolve().parents[2] +COMPOSE = os.environ.get("CDNF_COMPOSE_FILE", "compose.dev.yml") +NETWORK = os.environ.get("CDNF_CONTROL_NETWORK", "cdnfoundry-dev_control") +MINIO_IMAGE = os.environ.get("CDNF_MINIO_IMAGE", "quay.io/minio/minio:RELEASE.2025-07-23T15-54-02Z") +POSTGRES_IMAGE = os.environ.get("CDNF_POSTGRES_IMAGE", "postgres:18.4-alpine") +QUEUE_IMAGE = os.environ.get("CDNF_QUEUE_IMAGE", "valkey/valkey:9.1.0-alpine") +RUN = f"{int(time.time())}-{secrets.token_hex(4)}" +MINIO = f"cdnfoundry-phase8-object-{RUN}" +RESTORE = f"cdnfoundry-phase8-replacement-{RUN}" +QUEUE = f"cdnfoundry-phase8-queue-{RUN}" +BUCKET = f"phase8-{RUN}" +S3_USER = f"phase8{secrets.token_hex(8)}" +S3_PASSWORD = secrets.token_urlsafe(32) +RESTIC_PASSWORD = secrets.token_urlsafe(40) +MARKER = f"phase8-recovery-{RUN}@example.test" + + +def run(*args: str, timeout: int = 180, check: bool = True) -> subprocess.CompletedProcess[str]: + result = subprocess.run(args, cwd=ROOT, text=True, capture_output=True, timeout=timeout, check=False) + if check and result.returncode != 0: + raise RuntimeError(f"command failed: {' '.join(args)}\n{result.stdout}\n{result.stderr}") + return result + + +def compose(*args: str, timeout: int = 180) -> subprocess.CompletedProcess[str]: + return run("docker", "compose", "-f", COMPOSE, *args, timeout=timeout) + + +def core_image() -> str: + image = compose("images", "-q", "core").stdout.strip() + if not image: + raise RuntimeError("the development core image is unavailable; run make dev-up first") + return image + + +def backup_environment(repository: str, password: str = RESTIC_PASSWORD) -> list[str]: + return [ + "-e", f"RESTIC_REPOSITORY={repository}", + "-e", f"RESTIC_PASSWORD={password}", + "-e", f"AWS_ACCESS_KEY_ID={S3_USER}", + "-e", f"AWS_SECRET_ACCESS_KEY={S3_PASSWORD}", + "-e", "AWS_DEFAULT_REGION=us-east-1", + ] + + +def postgres_environment(host: str, database: str, user: str, password: str) -> list[str]: + return [ + "-e", f"PGHOST={host}", "-e", "PGPORT=5432", "-e", f"PGDATABASE={database}", + "-e", f"PGUSER={user}", "-e", f"PGPASSWORD={password}", + ] + + +def wait_ready(container: str, user: str, database: str, timeout: int = 45) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + result = run("docker", "exec", container, "pg_isready", "-U", user, "-d", database, check=False) + if result.returncode == 0: + return + time.sleep(1) + raise RuntimeError(f"PostgreSQL container {container} did not become ready") + + +def psql(container: str, user: str, database: str, sql: str) -> str: + return run("docker", "exec", container, "psql", "-U", user, "-d", database, "-Atc", sql).stdout.strip() + + +def source_psql(sql: str) -> str: + return compose("exec", "-T", "control-db", "psql", "-U", "cdnf", "-d", "cdnf", "-Atc", sql).stdout.strip() + + +def queue_length(pattern: str) -> int: + keys = run("docker", "exec", QUEUE, "valkey-cli", "--scan", "--pattern", pattern).stdout.splitlines() + return sum(int(run("docker", "exec", QUEUE, "valkey-cli", "llen", key).stdout.strip()) for key in keys) + + +def state_counts_source() -> dict[str, int]: + tables = ("users", "domains", "dns_records", "dns_deployments", "edges", "edge_artifacts", "tls_certificates") + return {table: int(source_psql(f'SELECT count(*) FROM "{table}"')) for table in tables} + + +def state_counts_restore() -> dict[str, int]: + tables = ("users", "domains", "dns_records", "dns_deployments", "edges", "edge_artifacts", "tls_certificates") + return {table: int(psql(RESTORE, "cdnf_restore", "cdnf_restore", f'SELECT count(*) FROM "{table}"')) for table in tables} + + +def main() -> None: + image = core_image() + repository = f"s3:http://{MINIO}:9000/{BUCKET}" + started_at = time.time() + snapshot = "" + + run( + "docker", "run", "-d", "--rm", "--name", MINIO, "--network", NETWORK, + "--tmpfs", "/data:rw,nosuid,nodev,size=2g", + "-e", f"MINIO_ROOT_USER={S3_USER}", "-e", f"MINIO_ROOT_PASSWORD={S3_PASSWORD}", + MINIO_IMAGE, "server", "/data", + ) + try: + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + alias = run( + "docker", "exec", MINIO, "mc", "alias", "set", "local", + "http://127.0.0.1:9000", S3_USER, S3_PASSWORD, check=False, + ) + if alias.returncode == 0 and run("docker", "exec", MINIO, "mc", "ready", "local", check=False).returncode == 0: + break + time.sleep(1) + else: + raise RuntimeError("ephemeral S3-compatible object host did not become ready") + run("docker", "exec", MINIO, "mc", "mb", f"local/{BUCKET}") + + source_psql( + "INSERT INTO users (name,email,password,type,disabled_at,created_at,updated_at) " + f"VALUES ('Phase 8 recovery marker','{MARKER}','not-a-login-secret','admin',now(),now(),now())" + ) + cutoff_at = time.time() + source_counts = state_counts_source() + + common = ["docker", "run", "--rm", "--network", NETWORK, "--entrypoint", ""] + run(*common, *backup_environment(repository), image, "restic", "init") + backup = run( + *common, *backup_environment(repository), + *postgres_environment("control-db", "cdnf", "cdnf", "cdnf-dev-only"), + image, "/usr/local/bin/cdnf-backup-create", timeout=600, + ) + events = [json.loads(line) for line in backup.stdout.splitlines() if line.startswith("{")] + summaries = [event for event in events if event.get("snapshot_id")] + if not summaries: + raise RuntimeError(f"Restic backup returned no snapshot identifier: {backup.stdout}") + snapshot = summaries[-1]["snapshot_id"] + backup_completed_at = time.time() + run(*common, *backup_environment(repository), image, "restic", "check", "--read-data-subset=100%", timeout=600) + + wrong = run( + *common, *backup_environment(repository, "wrong-decryption-material"), + image, "restic", "snapshots", snapshot, check=False, + ) + if wrong.returncode == 0: + raise AssertionError("the encrypted repository accepted incorrect decryption material") + + restore_started_at = time.time() + run( + "docker", "run", "-d", "--rm", "--name", RESTORE, "--network", NETWORK, + "--tmpfs", "/var/lib/postgresql:rw,nosuid,nodev,size=2g", + "-e", "POSTGRES_DB=cdnf_restore", "-e", "POSTGRES_USER=cdnf_restore", + "-e", "POSTGRES_PASSWORD=phase8-restore-only", POSTGRES_IMAGE, + ) + wait_ready(RESTORE, "cdnf_restore", "cdnf_restore") + run( + *common, *backup_environment(repository), + *postgres_environment(RESTORE, "cdnf_restore", "cdnf_restore", "phase8-restore-only"), + image, "/usr/local/bin/cdnf-backup-restore", snapshot, timeout=600, + ) + restored_counts = state_counts_restore() + if restored_counts != source_counts: + raise AssertionError((source_counts, restored_counts)) + marker_count = psql(RESTORE, "cdnf_restore", "cdnf_restore", f"SELECT count(*) FROM users WHERE email='{MARKER}'") + if marker_count != "1": + raise AssertionError("the just-before-backup recovery marker was not restored") + compose( + "run", "--rm", "--no-deps", + "-e", "APP_ENV=production", "-e", f"DB_HOST={RESTORE}", "-e", "DB_PORT=5432", + "-e", "DB_DATABASE=cdnf_restore", "-e", "DB_USERNAME=cdnf_restore", + "-e", "DB_PASSWORD=phase8-restore-only", "core", "php", "artisan", "migrate", "--force", + timeout=600, + ) + migration_count = int(psql(RESTORE, "cdnf_restore", "cdnf_restore", "SELECT count(*) FROM migrations")) + expected_migrations = len(list((ROOT / "core/database/migrations").glob("*.php"))) + if migration_count != expected_migrations: + raise AssertionError(f"replacement schema has {migration_count} migrations, expected {expected_migrations}") + restored_counts["migrations"] = migration_count + + # Queue contents are deliberately absent on a replacement host. Prove + # that each bounded global reconciler can reconstruct work from desired + # PostgreSQL state into a fresh queue without copying old Redis data. + run("docker", "run", "-d", "--rm", "--name", QUEUE, "--network", NETWORK, "--tmpfs", "/data", QUEUE_IMAGE) + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + if run("docker", "exec", QUEUE, "valkey-cli", "ping", check=False).stdout.strip() == "PONG": + break + time.sleep(0.5) + else: + raise RuntimeError("replacement queue did not become ready") + expression = ( + "$classes=['dns.global_reconcile'=>App\\Jobs\\ReconcileAllDnsZones::class," + "'edges.global_reconcile'=>App\\Jobs\\ReconcileAllEdgeDomains::class," + "'tls.global_reconcile'=>App\\Jobs\\ReconcileAllTls::class," + "'purges.global_reconcile'=>App\\Jobs\\ReconcileAllPurges::class];" + "foreach($classes as $type=>$class){$operation=App\\Models\\Operation::query()->create(" + "['type'=>$type,'status'=>'pending','input'=>['reason'=>'replacement_queue_recovery']]);" + "$class::dispatch($operation->id);};" + "$from=now()->utc()->subHour()->startOfHour();$to=now()->utc()->startOfHour();" + "$usage=App\\Models\\Operation::query()->create(['type'=>'usage.global_reconcile','status'=>'pending'," + "'input'=>['from'=>$from->toIso8601String(),'to'=>$to->toIso8601String()]]);" + "App\\Jobs\\BuildUsageRollups::dispatch($from->toIso8601String(),$to->toIso8601String(),null,$usage->id);" + ) + compose( + "run", "--rm", "--no-deps", "-e", f"DB_HOST={RESTORE}", "-e", "DB_PORT=5432", + "-e", "DB_DATABASE=cdnf_restore", "-e", "DB_USERNAME=cdnf_restore", + "-e", "DB_PASSWORD=phase8-restore-only", "-e", f"REDIS_HOST={QUEUE}", + "-e", "CACHE_STORE=redis", "-e", "QUEUE_CONNECTION=redis", + "core", "php", "artisan", "tinker", f"--execute={expression}", timeout=180, + ) + reconstructed_jobs = queue_length("*queues:bulk_maintenance") + if reconstructed_jobs != 5: + raise AssertionError(f"replacement queue contains {reconstructed_jobs} global jobs, expected 5") + compose( + "run", "--rm", "--no-deps", "-e", f"DB_HOST={RESTORE}", "-e", "DB_PORT=5432", + "-e", "DB_DATABASE=cdnf_restore", "-e", "DB_USERNAME=cdnf_restore", + "-e", "DB_PASSWORD=phase8-restore-only", "-e", f"REDIS_HOST={QUEUE}", + "-e", "CACHE_STORE=redis", "-e", "QUEUE_CONNECTION=redis", + "core", "php", "artisan", "queue:work", "redis", "--queue=bulk_maintenance", + "--stop-when-empty", "--tries=1", "--timeout=180", timeout=600, + ) + if queue_length("*queues:bulk_maintenance") != 0: + raise AssertionError("replacement global reconciliation queue did not drain") + reconstructed_runtime_jobs = queue_length("*queues:runtime") + queue_length("*queues:certificate_purge") + if reconstructed_runtime_jobs == 0: + raise AssertionError("global reconciliation reconstructed no bounded runtime work") + reconciliations_succeeded = int(psql( + RESTORE, "cdnf_restore", "cdnf_restore", + "SELECT count(*) FROM operations WHERE type IN ('dns.global_reconcile','edges.global_reconcile'," + "'tls.global_reconcile','purges.global_reconcile','usage.global_reconcile') " + "AND status='succeeded' AND created_at > now() - interval '10 minutes'", + )) + if reconciliations_succeeded < 5: + raise AssertionError(f"only {reconciliations_succeeded} replacement reconciliations succeeded") + active_domains = int(psql( + RESTORE, "cdnf_restore", "cdnf_restore", "SELECT count(*) FROM domains WHERE deleted_at IS NULL", + )) + rebuilt_usage_intervals = int(psql( + RESTORE, "cdnf_restore", "cdnf_restore", + "SELECT count(*) FROM usage_rollups WHERE granularity='hour' AND status='finalized' " + "AND source_finalized_at > now() - interval '10 minutes'", + )) + if rebuilt_usage_intervals != active_domains: + raise AssertionError( + f"rebuilt {rebuilt_usage_intervals} usage intervals, expected {active_domains} active domains" + ) + restore_completed_at = time.time() + + print(json.dumps({ + "phase8_recovery": "passed", + "object_host_image": MINIO_IMAGE, + "snapshot": snapshot, + "rpo_seconds": round(backup_completed_at - cutoff_at, 3), + "rto_seconds": round(restore_completed_at - restore_started_at, 3), + "total_seconds": round(restore_completed_at - started_at, 3), + "counts": restored_counts, + "reconstructed_queue_jobs": reconstructed_jobs, + "reconstructed_runtime_jobs": reconstructed_runtime_jobs, + "reconciliations_succeeded": reconciliations_succeeded, + "rebuilt_usage_intervals": rebuilt_usage_intervals, + "active_domains": active_domains, + "wrong_password_rejected": True, + }, sort_keys=True)) + finally: + run("docker", "stop", RESTORE, check=False) + run("docker", "stop", QUEUE, check=False) + run("docker", "stop", MINIO, check=False) + source_psql(f"DELETE FROM users WHERE email='{MARKER}'") + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/phase8_throughput.py b/tests/e2e/phase8_throughput.py new file mode 100644 index 0000000..f5494f0 --- /dev/null +++ b/tests/e2e/phase8_throughput.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +"""Single-cell HTTP/HTTPS throughput qualification with explicit host details.""" + +from __future__ import annotations + +import concurrent.futures +import json +import math +import os +import pathlib +import platform +import subprocess +import tempfile +import time + +ROOT = pathlib.Path(__file__).resolve().parents[2] +EDGE_NETWORK = os.environ.get("CDNF_EDGE_NETWORK", "cdnfoundry-dev_edge") +EDGE_IMAGE = os.environ.get("CDNF_EDGE_IMAGE", "cdnfoundry/edge-runtime:phase8-throughput") +NAME = "cdnfoundry-phase8-throughput" +HOSTNAMES = [f"throughput-{index}.phase8.test" for index in range(64)] +HTTP_PORT = 18081 +HTTPS_PORT = 18444 +WORKERS = int(os.environ.get("CDNF_THROUGHPUT_WORKERS", "32")) +DURATION = float(os.environ.get("CDNF_THROUGHPUT_SECONDS", "8")) + + +def run(*args: str, timeout: int = 180, check: bool = True) -> subprocess.CompletedProcess[str]: + result = subprocess.run(args, cwd=ROOT, text=True, capture_output=True, timeout=timeout, check=False) + if check and result.returncode != 0: + raise RuntimeError(f"command failed: {' '.join(args)}\n{result.stdout}\n{result.stderr}") + return result + + +def percentile(samples: list[float], value: float) -> float: + ordered = sorted(samples) + return ordered[min(len(ordered) - 1, int((len(ordered) - 1) * value))] + + +def benchmark(scheme: str, port: int, edge_address: str) -> dict[str, float | int]: + samples: list[float] = [] + error_messages: dict[str, int] = {} + duration = math.ceil(DURATION) + + def worker(index: int) -> subprocess.CompletedProcess[str]: + hostname = HOSTNAMES[index % len(HOSTNAMES)] + client_name = f"{NAME}-{scheme}-{index}" + insecure = "-k" if scheme == "https" else "" + script = ( + f'end=$(( $(date +%s) + {duration} )); ' + f'while [ "$(date +%s)" -lt "$end" ]; do ' + f'curl -sS {insecure} --http1.1 --connect-timeout 2 --max-time 3 ' + f'--resolve "{hostname}:{port}:{edge_address}" ' + f'-o /dev/null -w "%{{http_code}} %{{size_download}} %{{time_total}}\\n" ' + f'"{scheme}://{hostname}:{port}/phase8-throughput" ' + f'|| echo "curl_error 0 0"; sleep 0.012; done' + ) + return run( + "docker", "run", "--rm", "--name", client_name, "--network", EDGE_NETWORK, + "--entrypoint", "sh", "curlimages/curl:8.16.0", "-c", script, + timeout=duration + 30, check=False, + ) + + started = time.monotonic() + with concurrent.futures.ThreadPoolExecutor(max_workers=WORKERS) as executor: + futures = [executor.submit(worker, index) for index in range(WORKERS)] + for future in futures: + result = future.result() + for line in result.stdout.splitlines(): + fields = line.split() + if len(fields) != 3 or fields[0] != "200": + error_messages[fields[0] if fields else "empty_result"] = error_messages.get( + fields[0] if fields else "empty_result", 0, + ) + 1 + continue + samples.append(float(fields[2]) * 1000) + if result.returncode != 0: + error_messages[f"client_exit_{result.returncode}"] = error_messages.get( + f"client_exit_{result.returncode}", 0, + ) + 1 + elapsed = time.monotonic() - started + total_requests = len(samples) + total_errors = sum(error_messages.values()) + if total_requests == 0: + raise AssertionError(f"{scheme} benchmark completed no successful requests") + error_rate = total_errors / max(1, total_requests + total_errors) + if error_rate > 0.01: + common_errors = sorted(error_messages.items(), key=lambda item: item[1], reverse=True)[:5] + raise AssertionError(f"{scheme} error rate {error_rate:.4%} exceeded one percent: {common_errors}") + return { + "requests": total_requests, + "errors": total_errors, + "error_rate": round(error_rate, 6), + "requests_per_second": round(total_requests / elapsed, 2), + "latency_ms_p50": round(percentile(samples, 0.50), 3), + "latency_ms_p95": round(percentile(samples, 0.95), 3), + "latency_ms_p99": round(percentile(samples, 0.99), 3), + "duration_seconds": round(elapsed, 3), + } + + +def main() -> None: + run("docker", "compose", "-f", "compose.dev.yml", "up", "-d", "origin-http", "mmdb-updater") + run("docker", "build", "-f", "docker/openresty/Dockerfile", "-t", EDGE_IMAGE, ".", timeout=900) + image_id = run("docker", "image", "inspect", "--format={{.Id}}", EDGE_IMAGE).stdout.strip() + with tempfile.TemporaryDirectory(prefix="cdnfoundry-phase8-throughput-") as directory: + temporary = pathlib.Path(directory) + temporary.chmod(0o755) + run( + "openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", "-days", "1", + "-subj", f"/CN={HOSTNAMES[0]}", "-addext", "subjectAltName=" + ",".join(f"DNS:{host}" for host in HOSTNAMES), + "-keyout", str(temporary / "edge.key"), "-out", str(temporary / "edge.crt"), + ) + certificate = (temporary / "edge.crt").read_text() + private_key = (temporary / "edge.key").read_text() + runtime = { + "schema_version": 1, + "sequence": 1, + "certificates": { + "phase8-throughput": { + "id": "phase8-throughput", "certificate_pem": certificate, "chain_pem": "", + "private_key_pem": private_key, "expires_at": int(time.time()) + 86400, "names": HOSTNAMES, + }, + }, + "hosts": { + hostname: { + "domain": hostname, "revision": 1, + "settings": {"enabled": True, "redirect_https": False, "http_versions": ["1.1", "2"]}, + "cache": { + "enabled": True, "edge_ttl_seconds": 3600, "browser_ttl_seconds": 300, + "maximum_object_bytes": 1048576, "respect_origin_headers": False, + "include_query_string": True, "bypass_cookie_names": [], "stale_if_error_seconds": 60, + "epoch": 1, "development_mode_until": None, + }, + "origin": { + "host": "origin-http", "port": 80, "scheme": "http", "host_header": hostname, + "sni": None, "verify_tls": False, "connect_timeout_ms": 1000, + "response_timeout_ms": 5000, "retry_count": 0, "websocket": False, + "health_check": None, "private_allowlist": ["172.16.0.0/12"], + "blocked_networks": [], "blocked_addresses": [], + }, + "tls": {"mode": "custom", "certificate_id": "phase8-throughput"}, + } for hostname in HOSTNAMES + }, + } + runtime_file = temporary / "shared-default.json" + runtime_file.write_text(json.dumps(runtime, separators=(",", ":"))) + os.chown(runtime_file, 10101, 10101) + runtime_file.chmod(0o600) + run( + "docker", "run", "-d", "--rm", "--name", NAME, "--network", EDGE_NETWORK, + "--memory", "512m", "--cpus", "1", "--pids-limit", "128", "--ulimit", "nofile=65536:65536", + "-p", f"127.0.0.1:{HTTP_PORT}:8080", "-p", f"127.0.0.1:{HTTPS_PORT}:8443", + "-e", "EDGE_CELL_NAME=shared-default", "-e", "EDGE_RUNTIME_FILE=/runtime/shared-default.json", + "-e", "EDGE_STATUS_TOKEN=phase8-throughput-only", "-e", "GEOIP_DATABASE=/mmdb/GeoLite2-City.mmdb", + "-v", f"{runtime_file}:/runtime/shared-default.json:ro", + "-v", "cdnfoundry-dev_dev-pki:/run/edge:ro", "-v", "cdnfoundry-dev_mmdb:/mmdb:ro", + "--tmpfs", "/var/cache/nginx:rw,noexec,nosuid,size=256m", + "--tmpfs", "/var/lib/nginx/tmp:rw,noexec,nosuid,size=64m", EDGE_IMAGE, + ) + try: + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + ready = run("curl", "-fsS", "-H", f"Host: {HOSTNAMES[0]}", f"http://127.0.0.1:{HTTP_PORT}/phase8-throughput", check=False) + if ready.returncode == 0: + break + time.sleep(0.5) + else: + logs = run("docker", "logs", NAME, check=False).stdout + raise RuntimeError(f"throughput edge did not become ready: {logs[-4000:]}") + # Prime every deterministic key so measurements represent edge cache-hit capacity. + for hostname in HOSTNAMES: + run("curl", "-fsS", "-H", f"Host: {hostname}", f"http://127.0.0.1:{HTTP_PORT}/phase8-throughput") + hit = run( + "curl", "-fsS", "-D", "-", "-o", "/dev/null", "-H", f"Host: {hostname}", + f"http://127.0.0.1:{HTTP_PORT}/phase8-throughput", + ).stdout.lower() + if "x-cdnfoundry-cache: hit" not in hit: + raise AssertionError(f"throughput path for {hostname} was not a cache HIT: {hit}") + edge_address = run( + "docker", "inspect", "--format={{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}", NAME, + ).stdout.strip() + http_result = benchmark("http", 8080, edge_address) + https_result = benchmark("https", 8443, edge_address) + host_memory = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES") + cpu_model = next( + (line.split(":", 1)[1].strip() for line in pathlib.Path("/proc/cpuinfo").read_text().splitlines() if line.startswith("model name")), + platform.processor() or "unknown", + ) + print(json.dumps({ + "phase8_throughput": "passed", + "host": { + "kernel": platform.release(), "architecture": platform.machine(), + "cpu_model": cpu_model, "logical_cpus": os.cpu_count(), + "memory_bytes": host_memory, + }, + "cell_limits": {"cpus": 1, "memory_bytes": 536870912, "pids": 128, "nofile": 65536}, + "edge_image_id": image_id, + "profile": "single shared OpenResty cell; 64 cache-HIT domains; 32 isolated clients paced below the 100 rps/client runtime bound; HTTP/1.1 new connections", + "http": http_result, + "https": https_result, + }, sort_keys=True)) + finally: + run("docker", "stop", NAME, check=False) + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/phase8_upgrade.py b/tests/e2e/phase8_upgrade.py new file mode 100644 index 0000000..3dec803 --- /dev/null +++ b/tests/e2e/phase8_upgrade.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +"""Prior/current control and edge-agent mixed-version rollback qualification.""" + +from __future__ import annotations + +import json +import os +import pathlib +import re +import secrets +import subprocess +import tarfile +import tempfile +import time + +ROOT = pathlib.Path(__file__).resolve().parents[2] +NETWORK = os.environ.get("CDNF_CONTROL_NETWORK", "cdnfoundry-dev_control") +POSTGRES_IMAGE = os.environ.get("CDNF_POSTGRES_IMAGE", "postgres:18.4-alpine") +RUN = f"{int(time.time())}-{secrets.token_hex(4)}" +DATABASE = f"cdnfoundry-phase8-upgrade-db-{RUN}" +PRIOR_CORE = f"cdnfoundry-phase8-prior-core:{RUN}" +CURRENT_CORE = f"cdnfoundry-phase8-current-core:{RUN}" +PRIOR_AGENT = f"cdnfoundry-phase8-prior-agent:{RUN}" +CURRENT_AGENT = f"cdnfoundry-phase8-current-agent:{RUN}" +APP_KEY = "base64:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + + +def run(*args: str, cwd: pathlib.Path = ROOT, timeout: int = 900, + check: bool = True) -> subprocess.CompletedProcess[str]: + result = subprocess.run(args, cwd=cwd, text=True, capture_output=True, timeout=timeout, check=False) + if check and result.returncode != 0: + raise RuntimeError(f"command failed: {' '.join(args)}\n{result.stdout}\n{result.stderr}") + return result + + +def wait_postgres() -> None: + deadline = time.monotonic() + 45 + while time.monotonic() < deadline: + result = run("docker", "exec", DATABASE, "pg_isready", "-U", "cdnf_upgrade", "-d", "cdnf_upgrade", check=False) + if result.returncode == 0: + return + time.sleep(1) + raise RuntimeError("mixed-version PostgreSQL did not become ready") + + +def application(image: str, *command: str) -> subprocess.CompletedProcess[str]: + return run( + "docker", "run", "--rm", "--network", NETWORK, "--entrypoint", "php", + "-e", "APP_ENV=production", "-e", f"APP_KEY={APP_KEY}", + "-e", "DB_CONNECTION=pgsql", "-e", f"DB_HOST={DATABASE}", "-e", "DB_PORT=5432", + "-e", "DB_DATABASE=cdnf_upgrade", "-e", "DB_USERNAME=cdnf_upgrade", + "-e", "DB_PASSWORD=phase8-upgrade-only", "-e", "CACHE_STORE=array", + "-e", "SESSION_DRIVER=array", "-e", "QUEUE_CONNECTION=sync", + image, *command, + ) + + +def psql(sql: str) -> str: + return run( + "docker", "exec", DATABASE, "psql", "-U", "cdnf_upgrade", "-d", "cdnf_upgrade", "-Atc", sql, + ).stdout.strip() + + +def source_version(source: pathlib.Path) -> str: + text = (source / "edge-agent" / "main.go").read_text() + match = re.search(r'const version = "([^"]+)"', text) + if not match: + raise RuntimeError("edge-agent release version is missing") + return match.group(1) + + +def main() -> None: + prior_sha = run("git", "rev-parse", "HEAD").stdout.strip() + current_version = source_version(ROOT) + with tempfile.TemporaryDirectory(prefix="cdnfoundry-phase8-upgrade-") as temp_name: + temp = pathlib.Path(temp_name) + archive = temp / "prior.tar" + prior = temp / "prior" + prior.mkdir() + run("git", "archive", "--format=tar", f"--output={archive}", prior_sha) + with tarfile.open(archive) as source: + source.extractall(prior, filter="data") + prior_version = source_version(prior) + if prior_version == current_version: + raise AssertionError("the canary requires distinct prior/current edge-agent versions") + + run("docker", "build", "--target", "production", "-t", PRIOR_CORE, str(prior / "core")) + run("docker", "build", "--target", "production", "-t", CURRENT_CORE, str(ROOT / "core")) + # Both Dockerfiles run their complete Go suite, including signed artifact compatibility. + run("docker", "build", "-t", PRIOR_AGENT, str(prior / "edge-agent")) + run("docker", "build", "-t", CURRENT_AGENT, str(ROOT / "edge-agent")) + + run( + "docker", "run", "-d", "--rm", "--name", DATABASE, "--network", NETWORK, + "--tmpfs", "/var/lib/postgresql:rw,nosuid,nodev,size=2g", + "-e", "POSTGRES_DB=cdnf_upgrade", "-e", "POSTGRES_USER=cdnf_upgrade", + "-e", "POSTGRES_PASSWORD=phase8-upgrade-only", POSTGRES_IMAGE, + ) + try: + wait_postgres() + database_id = run("docker", "inspect", "--format={{.Id}}", DATABASE).stdout.strip() + application(PRIOR_CORE, "artisan", "migrate", "--force") + prior_migrations = int(psql("SELECT count(*) FROM migrations")) + prior_marker = f"prior-{RUN}" + application( + PRIOR_CORE, "artisan", "tinker", "--execute=" + f"App\\Models\\Operation::query()->create(['type'=>'{prior_marker}','status'=>'succeeded','input'=>[]]);", + ) + + application(CURRENT_CORE, "artisan", "migrate", "--force") + current_migrations = int(psql("SELECT count(*) FROM migrations")) + if current_migrations <= prior_migrations: + raise AssertionError((prior_migrations, current_migrations)) + current_marker = f"current-{RUN}" + application( + CURRENT_CORE, "artisan", "tinker", "--execute=" + f"if(App\\Models\\Operation::query()->where('type','{prior_marker}')->count()!==1)throw new RuntimeException('prior marker missing');" + f"App\\Models\\Operation::query()->create(['type'=>'{current_marker}','status'=>'succeeded','input'=>[]]);", + ) + + # Roll the application back while retaining the additive current schema. + application( + PRIOR_CORE, "artisan", "tinker", "--execute=" + f"if(App\\Models\\Operation::query()->where('type','{current_marker}')->count()!==1)throw new RuntimeException('current marker missing');" + "App\\Models\\Operation::query()->create(['type'=>'prior.rollback.write','status'=>'succeeded','input'=>[]]);", + ) + if int(psql("SELECT count(*) FROM operations WHERE type='prior.rollback.write'")) != 1: + raise AssertionError("the prior release could not write after rollback") + if run("docker", "inspect", "--format={{.Id}}", DATABASE).stdout.strip() != database_id: + raise AssertionError("the database container changed during application rollback") + if int(psql("SELECT count(*) FROM migrations")) != current_migrations: + raise AssertionError("application rollback modified or restored the database schema") + + observed_current = run("docker", "run", "--rm", CURRENT_AGENT, "--version").stdout.strip() + if observed_current != current_version: + raise AssertionError((observed_current, current_version)) + + print(json.dumps({ + "phase8_upgrade": "passed", + "prior_commit": prior_sha, + "prior_agent_version": prior_version, + "current_agent_version": current_version, + "prior_migrations": prior_migrations, + "current_migrations": current_migrations, + "database_restored": False, + "prior_write_after_rollback": True, + "signed_artifact_tests": "passed_in_both_image_builds", + }, sort_keys=True)) + finally: + run("docker", "stop", DATABASE, check=False) + for image in (PRIOR_CORE, CURRENT_CORE, PRIOR_AGENT, CURRENT_AGENT): + run("docker", "image", "rm", image, check=False) + + +if __name__ == "__main__": + main()