diff --git a/compose.dev.yml b/compose.dev.yml index 982156a..808a262 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -332,8 +332,8 @@ services: context: . dockerfile: docker/openresty/Dockerfile environment: - EDGE_CELL_NAME: shared-default - EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/shared-default.json + EDGE_CELL_NAME: cell-01 + EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/cell-01.json EDGE_STATUS_TOKEN: ${CDNF_DEV_EDGE_STATUS_TOKEN:-cdnf-dev-edge-status-only} GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb volumes: @@ -365,8 +365,8 @@ services: context: . dockerfile: docker/openresty/Dockerfile environment: - EDGE_CELL_NAME: quarantine-default - EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/quarantine-default.json + EDGE_CELL_NAME: cell-02 + EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/cell-02.json EDGE_STATUS_TOKEN: ${CDNF_DEV_EDGE_STATUS_TOKEN:-cdnf-dev-edge-status-only} GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb volumes: @@ -403,6 +403,7 @@ services: {"address":"fd00:cd0f:10::10","pool":"shared-default","http":"edge-a:8081","https":"edge-a:8444"}, {"address":"172.28.11.10","pool":"quarantine-default","http":"edge-a-quarantine:8081","https":"edge-a-quarantine:8444"}] EDGE_GATEWAY_STATUS_URL: http://edge-gateway-a:9105/metrics + EDGE_CELL_ASSIGNMENTS: '{"cell-01":"shared-default","cell-02":"quarantine-default"}' EDGE_CELL_STATUS_URLS: http://edge-a:9080/passive-failures,http://edge-a-quarantine:9080/passive-failures EDGE_STATUS_TOKEN: ${CDNF_DEV_EDGE_STATUS_TOKEN:-cdnf-dev-edge-status-only} volumes: @@ -467,8 +468,8 @@ services: context: . dockerfile: docker/openresty/Dockerfile environment: - EDGE_CELL_NAME: shared-default - EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/shared-default.json + EDGE_CELL_NAME: cell-01 + EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/cell-01.json EDGE_STATUS_TOKEN: ${CDNF_DEV_EDGE_STATUS_TOKEN:-cdnf-dev-edge-status-only} GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb volumes: @@ -500,8 +501,8 @@ services: context: . dockerfile: docker/openresty/Dockerfile environment: - EDGE_CELL_NAME: quarantine-default - EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/quarantine-default.json + EDGE_CELL_NAME: cell-02 + EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/cell-02.json EDGE_STATUS_TOKEN: ${CDNF_DEV_EDGE_STATUS_TOKEN:-cdnf-dev-edge-status-only} GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb volumes: @@ -537,6 +538,7 @@ services: [{"address":"172.28.20.10","pool":"shared-default","http":"edge-b:8081","https":"edge-b:8444"}, {"address":"172.28.21.10","pool":"quarantine-default","http":"edge-b-quarantine:8081","https":"edge-b-quarantine:8444"}] EDGE_GATEWAY_STATUS_URL: http://edge-gateway-b:9105/metrics + EDGE_CELL_ASSIGNMENTS: '{"cell-01":"shared-default","cell-02":"quarantine-default"}' EDGE_CELL_STATUS_URLS: http://edge-b:9080/passive-failures,http://edge-b-quarantine:9080/passive-failures EDGE_STATUS_TOKEN: ${CDNF_DEV_EDGE_STATUS_TOKEN:-cdnf-dev-edge-status-only} volumes: diff --git a/compose.prod.yml b/compose.prod.yml index ec6fb4c..2337b29 100644 --- a/compose.prod.yml +++ b/compose.prod.yml @@ -18,6 +18,7 @@ x-core-env: &core-env CACHE_STORE: redis SESSION_DRIVER: redis QUEUE_CONNECTION: redis + REDIS_URL: ${REDIS_URL:-} REDIS_HOST: ${REDIS_HOST:-redis} REDIS_PORT: ${REDIS_PORT:-6379} @@ -44,6 +45,35 @@ x-core-env: &core-env BACKUP_SECRET_ACCESS_KEY: ${BACKUP_SECRET_ACCESS_KEY:?BACKUP_SECRET_ACCESS_KEY is required} BACKUP_DEFAULT_REGION: ${BACKUP_DEFAULT_REGION:-us-east-1} +x-edge-cell: &edge-cell + image: ghcr.io/vaheed/cdnfoundry-edge-runtime:${CDNF_RELEASE:?CDNF_RELEASE must be a published commit SHA or release tag} + profiles: [edge] + volumes: + - ${EDGE_RUNTIME_TLS_CERTIFICATE:?EDGE_RUNTIME_TLS_CERTIFICATE is required}:/run/edge/tls.crt:ro + - ${EDGE_RUNTIME_TLS_PRIVATE_KEY:?EDGE_RUNTIME_TLS_PRIVATE_KEY is required}:/run/edge/tls.key:ro + - edge-state:/var/lib/cdnfoundry/runtime:ro + - mmdb:/mmdb:ro + depends_on: + mmdb-updater: { condition: service_healthy } + read_only: true + tmpfs: + - /var/cache/nginx:rw,noexec,nosuid,size=256m + - /var/lib/nginx/tmp:rw,noexec,nosuid,size=64m + - /usr/local/openresty/nginx/logs:rw,noexec,nosuid,size=16m + networks: [edge, telemetry] + restart: unless-stopped + stop_signal: SIGQUIT + stop_grace_period: 30s + mem_limit: 512m + cpus: 0.5 + pids_limit: 128 + sysctls: { net.ipv4.tcp_syncookies: "1" } + ulimits: + nofile: { soft: 32768, hard: 32768 } + deploy: + resources: + limits: { memory: 512m, cpus: "0.5", pids: 128 } + x-core: &core image: ghcr.io/vaheed/cdnfoundry-core:${CDNF_RELEASE:?CDNF_RELEASE must be a published commit SHA or release tag} environment: *core-env @@ -287,85 +317,51 @@ services: networks: [telemetry] restart: unless-stopped - edge: - image: ghcr.io/vaheed/cdnfoundry-edge-runtime:${CDNF_RELEASE:?CDNF_RELEASE must be a published commit SHA or release tag} - profiles: [edge] - ports: - - "127.0.0.1:18081:8081" - - "127.0.0.1:18444:8444" - environment: - EDGE_CELL_NAME: shared-default - EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/shared-default.json - EDGE_STATUS_TOKEN: ${EDGE_STATUS_TOKEN:?EDGE_STATUS_TOKEN is required} - GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb - volumes: - - ${EDGE_RUNTIME_TLS_CERTIFICATE:?EDGE_RUNTIME_TLS_CERTIFICATE is required}:/run/edge/tls.crt:ro - - ${EDGE_RUNTIME_TLS_PRIVATE_KEY:?EDGE_RUNTIME_TLS_PRIVATE_KEY is required}:/run/edge/tls.key:ro - - edge-state:/var/lib/cdnfoundry/runtime:ro - - mmdb:/mmdb:ro - depends_on: - mmdb-updater: { condition: service_healthy } - read_only: true - tmpfs: - - /var/cache/nginx:rw,noexec,nosuid,size=512m - - /var/lib/nginx/tmp:rw,noexec,nosuid,size=128m - - /usr/local/openresty/nginx/logs:rw,noexec,nosuid,size=16m - networks: [edge, telemetry] - restart: unless-stopped - stop_signal: SIGQUIT - stop_grace_period: 30s - mem_limit: 2g - cpus: 2 - pids_limit: 256 - sysctls: { net.ipv4.tcp_syncookies: "1" } - ulimits: - nofile: { soft: 65536, hard: 65536 } - deploy: - resources: - limits: - memory: 2g - cpus: "2" - pids: 256 + cell-01: &cell-01 + <<: *edge-cell + ports: ["127.0.0.1:18081:8081", "127.0.0.1:18444:8444", "127.0.0.1:19081:9080"] + environment: { EDGE_CELL_NAME: cell-01, EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/cell-01.json, EDGE_STATUS_TOKEN: "${EDGE_STATUS_TOKEN:?EDGE_STATUS_TOKEN is required}", GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb } + networks: + edge: { aliases: [edge] } + telemetry: {} - edge-quarantine: - image: ghcr.io/vaheed/cdnfoundry-edge-runtime:${CDNF_RELEASE:?CDNF_RELEASE must be a published commit SHA or release tag} - profiles: [edge] - ports: - - "127.0.0.1:28081:8081" - - "127.0.0.1:28444:8444" - environment: - EDGE_CELL_NAME: quarantine-default - EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/quarantine-default.json - EDGE_STATUS_TOKEN: ${EDGE_STATUS_TOKEN:?EDGE_STATUS_TOKEN is required} - GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb - volumes: - - ${EDGE_RUNTIME_TLS_CERTIFICATE:?EDGE_RUNTIME_TLS_CERTIFICATE is required}:/run/edge/tls.crt:ro - - ${EDGE_RUNTIME_TLS_PRIVATE_KEY:?EDGE_RUNTIME_TLS_PRIVATE_KEY is required}:/run/edge/tls.key:ro - - edge-state:/var/lib/cdnfoundry/runtime:ro - - mmdb:/mmdb:ro - depends_on: - mmdb-updater: { condition: service_healthy } - read_only: true - tmpfs: - - /var/cache/nginx:rw,noexec,nosuid,size=256m - - /var/lib/nginx/tmp:rw,noexec,nosuid,size=64m - - /usr/local/openresty/nginx/logs:rw,noexec,nosuid,size=16m - networks: [edge, telemetry] - restart: unless-stopped - stop_signal: SIGQUIT - stop_grace_period: 30s - mem_limit: 512m - cpus: 0.5 - pids_limit: 128 - sysctls: { net.ipv4.tcp_syncookies: "1" } - ulimits: - nofile: { soft: 32768, hard: 32768 } - deploy: - resources: - limits: - memory: 512m - cpus: "0.5" - pids: 128 + cell-02: + <<: *edge-cell + ports: ["127.0.0.1:18082:8081", "127.0.0.1:18445:8444", "127.0.0.1:19082:9080"] + environment: { EDGE_CELL_NAME: cell-02, EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/cell-02.json, EDGE_STATUS_TOKEN: "${EDGE_STATUS_TOKEN:?EDGE_STATUS_TOKEN is required}", GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb } + networks: + edge: { aliases: [edge-quarantine] } + telemetry: {} + + cell-03: + <<: *edge-cell + ports: ["127.0.0.1:18083:8081", "127.0.0.1:18446:8444", "127.0.0.1:19083:9080"] + environment: { EDGE_CELL_NAME: cell-03, EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/cell-03.json, EDGE_STATUS_TOKEN: "${EDGE_STATUS_TOKEN:?EDGE_STATUS_TOKEN is required}", GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb } + + cell-04: + <<: *edge-cell + ports: ["127.0.0.1:18084:8081", "127.0.0.1:18447:8444", "127.0.0.1:19084:9080"] + environment: { EDGE_CELL_NAME: cell-04, EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/cell-04.json, EDGE_STATUS_TOKEN: "${EDGE_STATUS_TOKEN:?EDGE_STATUS_TOKEN is required}", GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb } + + cell-05: + <<: *edge-cell + ports: ["127.0.0.1:18085:8081", "127.0.0.1:18448:8444", "127.0.0.1:19085:9080"] + environment: { EDGE_CELL_NAME: cell-05, EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/cell-05.json, EDGE_STATUS_TOKEN: "${EDGE_STATUS_TOKEN:?EDGE_STATUS_TOKEN is required}", GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb } + + cell-06: + <<: *edge-cell + ports: ["127.0.0.1:18086:8081", "127.0.0.1:18449:8444", "127.0.0.1:19086:9080"] + environment: { EDGE_CELL_NAME: cell-06, EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/cell-06.json, EDGE_STATUS_TOKEN: "${EDGE_STATUS_TOKEN:?EDGE_STATUS_TOKEN is required}", GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb } + + cell-07: + <<: *edge-cell + ports: ["127.0.0.1:18087:8081", "127.0.0.1:18450:8444", "127.0.0.1:19087:9080"] + environment: { EDGE_CELL_NAME: cell-07, EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/cell-07.json, EDGE_STATUS_TOKEN: "${EDGE_STATUS_TOKEN:?EDGE_STATUS_TOKEN is required}", GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb } + + cell-08: + <<: *edge-cell + ports: ["127.0.0.1:18088:8081", "127.0.0.1:18451:8444", "127.0.0.1:19088:9080"] + environment: { EDGE_CELL_NAME: cell-08, EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/cell-08.json, EDGE_STATUS_TOKEN: "${EDGE_STATUS_TOKEN:?EDGE_STATUS_TOKEN is required}", GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb } edge-agent: image: ghcr.io/vaheed/cdnfoundry-edge-agent:${CDNF_RELEASE:?CDNF_RELEASE must be a published commit SHA or release tag} @@ -379,7 +375,8 @@ services: EDGE_RUNTIME_DIR: /var/lib/cdnfoundry/runtime EDGE_GATEWAY_BINDINGS: ${EDGE_GATEWAY_BINDINGS:?EDGE_GATEWAY_BINDINGS is required} EDGE_GATEWAY_STATUS_URL: ${EDGE_GATEWAY_STATUS_URL:-http://host-gateway:9105/metrics} - EDGE_CELL_STATUS_URLS: http://edge:9080/passive-failures,http://edge-quarantine:9080/passive-failures + EDGE_CELL_ASSIGNMENTS: '{"cell-01":"shared-default","cell-02":"quarantine-default","cell-03":"","cell-04":"","cell-05":"","cell-06":"","cell-07":"","cell-08":""}' + EDGE_CELL_STATUS_URLS: http://cell-01:9080/passive-failures,http://cell-02:9080/passive-failures,http://cell-03:9080/passive-failures,http://cell-04:9080/passive-failures,http://cell-05:9080/passive-failures,http://cell-06:9080/passive-failures,http://cell-07:9080/passive-failures,http://cell-08:9080/passive-failures EDGE_STATUS_TOKEN: ${EDGE_STATUS_TOKEN:?EDGE_STATUS_TOKEN is required} volumes: - edge-agent-state:/var/lib/cdnfoundry/agent diff --git a/core/app/Filament/Admin/Resources/EdgePools/EdgePoolResource.php b/core/app/Filament/Admin/Resources/EdgePools/EdgePoolResource.php index 95083a1..73c38e0 100644 --- a/core/app/Filament/Admin/Resources/EdgePools/EdgePoolResource.php +++ b/core/app/Filament/Admin/Resources/EdgePools/EdgePoolResource.php @@ -6,11 +6,13 @@ use App\Filament\Admin\Resources\EdgePools\Pages\CreateEdgePool; use App\Filament\Admin\Resources\EdgePools\Pages\EditEdgePool; use App\Filament\Admin\Resources\EdgePools\Pages\ListEdgePools; +use App\Jobs\ProvisionEdgePoolCells; use App\Jobs\ReconcilePlatformDnsIdentity; use App\Models\AuditLog; use App\Models\DomainEdgePlacement; use App\Models\EdgePool; use App\Models\EmergencyMode; +use App\Models\Operation; use App\Models\PlatformDnsSetting; use App\Support\EdgeRoutingCompiler; use Filament\Actions\Action; @@ -60,6 +62,18 @@ public static function table(Table $table): Table TextColumn::make('cells_count')->counts('cells')->label('Edge cells'), TextColumn::make('updated_at')->since()->sortable(), ])->recordActions([ + Action::make('reconcileCells')->label('Reconcile cells')->icon('heroicon-o-arrow-path') + ->requiresConfirmation() + ->action(function (EdgePool $record): void { + $operation = Operation::query()->create([ + 'actor_id' => auth()->id(), 'type' => 'edge.pool_provision', 'status' => 'pending', + 'input' => ['pool_id' => $record->id], + ]); + AuditLog::record(auth()->user(), 'edge.pool_provision_requested', $record, ['operation_id' => $operation->id], request()->ip()); + ProvisionEdgePoolCells::dispatch($record->id, $operation->id); + Notification::make()->info()->title('Cell reconciliation queued') + ->body("Operation {$operation->id} will assign one existing unassigned slot on each missing edge.")->send(); + }), Action::make('enable')->visible(fn (EdgePool $record): bool => ! $record->enabled)->action(function (EdgePool $record): void { $incomplete = $record->cells()->whereHas('edge', fn ($query) => $query->where('enabled', true))->whereNull('service_ipv4')->exists(); if ($incomplete) { diff --git a/core/app/Filament/Admin/Resources/Edges/EdgeResource.php b/core/app/Filament/Admin/Resources/Edges/EdgeResource.php index 22af2c0..60d7d14 100644 --- a/core/app/Filament/Admin/Resources/Edges/EdgeResource.php +++ b/core/app/Filament/Admin/Resources/Edges/EdgeResource.php @@ -41,6 +41,8 @@ public static function form(Schema $schema): Schema { return $schema->components([ TextInput::make('name')->required()->maxLength(100)->unique(ignoreRecord: true), + TextInput::make('cell_slot_count')->label('Cell slots')->numeric()->integer()->required()->minValue(1)->maxValue(32)->default(8)->disabledOn('edit') + ->helperText('Bounded OpenResty slots created during edge installation. This cannot be changed after creation.'), Select::make('country_code')->label('Country')->options(array_combine(GeoVocabulary::countries(), GeoVocabulary::countries()))->searchable()->required(), Select::make('continent_code')->label('Continent')->options(array_combine(GeoVocabulary::CONTINENTS, GeoVocabulary::CONTINENTS))->required(), TextInput::make('ipv4')->label('IPv4')->ipv4()->required()->unique(ignoreRecord: true) diff --git a/core/app/Filament/Admin/Resources/Edges/Pages/CreateEdge.php b/core/app/Filament/Admin/Resources/Edges/Pages/CreateEdge.php index b02722c..903820f 100644 --- a/core/app/Filament/Admin/Resources/Edges/Pages/CreateEdge.php +++ b/core/app/Filament/Admin/Resources/Edges/Pages/CreateEdge.php @@ -26,12 +26,19 @@ protected function handleRecordCreation(array $data): Model $data['bootstrap_token_hash'] = hash('sha256', $this->bootstrapToken); $edge = DB::transaction(function () use ($data): Edge { $edge = Edge::query()->create($data); - $defaultSharedId = EdgePool::query()->where('enabled', true)->where('kind', 'shared')->orderBy('id')->value('id'); - foreach (EdgePool::query()->orderBy('id')->limit(32)->get() as $pool) { + $pools = EdgePool::query()->where('enabled', true)->orderByRaw("CASE WHEN kind = 'shared' THEN 0 WHEN kind = 'quarantine' THEN 1 ELSE 2 END")->orderBy('id')->limit($edge->cell_slot_count)->get()->values(); + for ($slot = 1; $slot <= $edge->cell_slot_count; $slot++) { + $pool = $pools->get($slot - 1); + $name = sprintf('cell-%02d', $slot); $edge->cells()->create([ - 'edge_pool_id' => $pool->id, 'name' => $pool->name, - 'service_ipv4' => $pool->id === $defaultSharedId ? $edge->ipv4 : null, - 'service_ipv6' => $pool->id === $defaultSharedId ? $edge->ipv6 : null, + 'slot' => $slot, 'edge_pool_id' => $pool?->id, 'name' => $name, + 'http_port' => 18080 + $slot, 'https_port' => 18443 + $slot, 'status_port' => 19080 + $slot, + 'runtime_path' => "/var/lib/cdnfoundry/runtime/{$name}.json", + 'cache_path' => "/var/cache/cdnfoundry/{$name}", 'temporary_path' => "/var/lib/cdnfoundry/tmp/{$name}", + 'resource_limits' => ['memory_bytes' => 536870912, 'cpu_millis' => 500, 'pids' => 128, 'cache_bytes' => 268435456, 'temporary_bytes' => 67108864, 'log_bytes' => 16777216], + 'status' => $pool === null ? 'unassigned' : 'assigned', + 'service_ipv4' => $pool?->kind === 'shared' ? $edge->ipv4 : null, + 'service_ipv6' => $pool?->kind === 'shared' ? $edge->ipv6 : null, ]); } AuditLog::record(auth()->user(), 'edge.created', $edge, [], request()->ip()); diff --git a/core/app/Filament/Admin/Resources/Edges/RelationManagers/CellsRelationManager.php b/core/app/Filament/Admin/Resources/Edges/RelationManagers/CellsRelationManager.php index 6fe4c9a..4b925a4 100644 --- a/core/app/Filament/Admin/Resources/Edges/RelationManagers/CellsRelationManager.php +++ b/core/app/Filament/Admin/Resources/Edges/RelationManagers/CellsRelationManager.php @@ -14,6 +14,7 @@ use Filament\Actions\Action; use Filament\Actions\EditAction; use Filament\Forms\Components\CheckboxList; +use Filament\Forms\Components\Select; use Filament\Forms\Components\TextInput; use Filament\Notifications\Notification; use Filament\Resources\RelationManagers\RelationManager; @@ -22,7 +23,6 @@ use Filament\Tables\Columns\TextColumn; use Filament\Tables\Table; use Illuminate\Support\Str; -use Illuminate\Validation\ValidationException; class CellsRelationManager extends RelationManager { @@ -31,6 +31,9 @@ class CellsRelationManager extends RelationManager public function form(Schema $schema): Schema { return $schema->components([ + Select::make('edge_pool_id')->label('Service pool assignment')->relationship('pool', 'name') + ->placeholder('Unassigned')->disabled()->dehydrated(false) + ->helperText('Assignments are managed through service-pool provisioning so every participating edge changes asynchronously and consistently.'), TextInput::make('service_ipv4')->label('Public service IPv4')->ipv4()->required() ->rule(fn () => function (string $attribute, mixed $value, \Closure $fail): void { if (NetworkAddress::isUnsafe((string) $value)) { @@ -54,7 +57,8 @@ public function table(Table $table): Table return $table->description(fn (): string => $this->edgeReadinessDescription()) ->columns([ TextColumn::make('name')->label('Cell')->searchable(), - TextColumn::make('pool.name')->label('Service pool'), + TextColumn::make('slot')->label('Slot')->sortable(), + TextColumn::make('pool.name')->label('Assignment')->placeholder('Unassigned'), TextColumn::make('status')->badge() ->formatStateUsing(fn (string $state, EdgeCell $record): string => $record->drained ? 'Drained' : ucfirst($state)) ->color(fn (string $state, EdgeCell $record): string => match (true) { @@ -75,13 +79,17 @@ public function table(Table $table): Table ? data_get($record->capacity, 'active_connections').' active connections' : 'Connections not reported'), TextColumn::make('capacity.cpu_usage')->label('Resources')->placeholder('Awaiting heartbeat') + ->formatStateUsing(fn (mixed $state): string => is_numeric($state) ? number_format((float) $state, 2).' CPU' : (string) $state) ->description(fn (EdgeCell $record): string => filled(data_get($record->capacity, 'memory_usage')) - ? data_get($record->capacity, 'memory_usage').' memory bytes used' + ? self::formatBytes(data_get($record->capacity, 'memory_usage')).' / '.self::formatBytes(data_get($record->capacity, 'memory_limit', data_get($record->resource_limits, 'memory_bytes'))).' memory' : 'Memory use not reported'), TextColumn::make('capacity.cache_usage')->label('Storage')->placeholder('Awaiting heartbeat') + ->formatStateUsing(fn (mixed $state, EdgeCell $record): string => self::formatBytes($state).' / '.self::formatBytes(data_get($record->capacity, 'cache_limit', data_get($record->resource_limits, 'cache_bytes'))).' cache') ->description(fn (EdgeCell $record): string => filled(data_get($record->capacity, 'temporary_storage_usage')) - ? data_get($record->capacity, 'temporary_storage_usage').' temporary bytes used' + ? self::formatBytes(data_get($record->capacity, 'temporary_storage_usage')).' / '.self::formatBytes(data_get($record->capacity, 'temporary_storage_limit', data_get($record->resource_limits, 'temporary_bytes'))).' temporary' : 'Temporary use not reported'), + TextColumn::make('http_port')->label('Ports')->description(fn (EdgeCell $record): string => "HTTPS {$record->https_port}; status {$record->status_port}"), + TextColumn::make('runtime_path')->label('Runtime path')->toggleable(isToggledHiddenByDefault: true), IconColumn::make('drained')->boolean(), ])->recordActions([ EditAction::make()->mutateDataUsing(fn (array $data, EdgeCell $record): array => EdgeCellAddressData::validate($record, $data)) @@ -112,11 +120,8 @@ public function table(Table $table): Table private static function queue(EdgeCell $cell, string $action): void { - if ($action !== 'drain' && $cell->service_ipv4 === null) { - throw ValidationException::withMessages(['service_ipv4' => 'Configure the cell service addresses before making it available.']); - } if ($action !== 'restart') { - $cell->update(['drained' => $action === 'drain', ...($action === 'undrain' ? ['status' => 'pending'] : [])]); + $cell->update(['drained' => $action === 'drain', ...($action === 'undrain' ? ['status' => $cell->edge_pool_id === null ? 'unassigned' : 'assigned'] : [])]); } $task = EdgeTask::query()->where('edge_id', $cell->edge_id)->where('type', 'cell_'.$action) ->where('status', 'pending')->where('payload->cell_id', $cell->id)->first() ?? EdgeTask::query()->create([ @@ -155,4 +160,21 @@ private function edgeReadinessDescription(): string return 'Agent connected. Capacity values come from the latest authenticated runtime heartbeat.'; } + + private static function formatBytes(mixed $bytes): string + { + if (! is_numeric($bytes)) { + return 'Not reported'; + } + + $value = max(0, (float) $bytes); + $units = ['B', 'KiB', 'MiB', 'GiB', 'TiB']; + $unit = 0; + while ($value >= 1024 && $unit < count($units) - 1) { + $value /= 1024; + $unit++; + } + + return ($unit === 0 ? number_format($value, 0) : number_format($value, $value < 10 ? 2 : 1)).' '.$units[$unit]; + } } diff --git a/core/app/Http/Controllers/Admin/EdgeController.php b/core/app/Http/Controllers/Admin/EdgeController.php index dc7f231..671d990 100644 --- a/core/app/Http/Controllers/Admin/EdgeController.php +++ b/core/app/Http/Controllers/Admin/EdgeController.php @@ -29,17 +29,19 @@ public function show(Edge $edge): JsonResponse public function store(Request $request): JsonResponse { - $data = $request->validate(['name' => ['required', 'string', 'max:100', 'unique:edges'], 'country_code' => ['required', Rule::in(GeoVocabulary::countries())], 'continent_code' => ['required', Rule::in(GeoVocabulary::CONTINENTS)], 'ipv4' => ['required', 'ipv4', 'unique:edges'], 'ipv6' => ['nullable', 'ipv6', 'unique:edges']]); + $data = $request->validate(['name' => ['required', 'string', 'max:100', 'unique:edges'], 'country_code' => ['required', Rule::in(GeoVocabulary::countries())], 'continent_code' => ['required', Rule::in(GeoVocabulary::CONTINENTS)], 'ipv4' => ['required', 'ipv4', 'unique:edges'], 'ipv6' => ['nullable', 'ipv6', 'unique:edges'], 'cell_slot_count' => ['sometimes', 'integer', 'between:1,32']]); abort_if(NetworkAddress::isUnsafe($data['ipv4']) || (isset($data['ipv6']) && NetworkAddress::isUnsafe($data['ipv6'])), 422, 'Edge addresses must be public unicast service addresses.'); $token = Str::random(64); $edge = DB::transaction(function () use ($data, $token, $request): Edge { - $edge = Edge::query()->create(array_merge($data, ['country_code' => strtoupper($data['country_code']), 'continent_code' => strtoupper($data['continent_code']), 'bootstrap_token_hash' => hash('sha256', $token)])); - $defaultSharedId = EdgePool::query()->where('enabled', true)->where('kind', 'shared')->orderBy('id')->value('id'); - foreach (EdgePool::query()->orderBy('id')->get() as $pool) { + $edge = Edge::query()->create(array_merge($data, ['country_code' => strtoupper($data['country_code']), 'continent_code' => strtoupper($data['continent_code']), 'cell_slot_count' => $data['cell_slot_count'] ?? 8, 'bootstrap_token_hash' => hash('sha256', $token)])); + $pools = EdgePool::query()->where('enabled', true)->orderByRaw("CASE WHEN kind = 'shared' THEN 0 WHEN kind = 'quarantine' THEN 1 ELSE 2 END")->orderBy('id')->limit($edge->cell_slot_count)->get()->values(); + for ($slot = 1; $slot <= $edge->cell_slot_count; $slot++) { + $pool = $pools->get($slot - 1); $edge->cells()->create([ - 'edge_pool_id' => $pool->id, 'name' => $pool->name, - 'service_ipv4' => $pool->id === $defaultSharedId ? $edge->ipv4 : null, - 'service_ipv6' => $pool->id === $defaultSharedId ? $edge->ipv6 : null, + 'slot' => $slot, 'edge_pool_id' => $pool?->id, + 'status' => $pool === null ? 'unassigned' : 'assigned', + 'service_ipv4' => $pool?->kind === 'shared' ? $edge->ipv4 : null, + 'service_ipv6' => $pool?->kind === 'shared' ? $edge->ipv6 : null, ]); } AuditLog::record($request->user(), 'edge.created', $edge, [], $request->ip()); diff --git a/core/app/Http/Controllers/Admin/EdgeOperationsController.php b/core/app/Http/Controllers/Admin/EdgeOperationsController.php index 8fcf40d..6660694 100644 --- a/core/app/Http/Controllers/Admin/EdgeOperationsController.php +++ b/core/app/Http/Controllers/Admin/EdgeOperationsController.php @@ -78,10 +78,9 @@ public function updateCell(Request $request, EdgeCell $cell): JsonResponse public function cellAction(Request $request, EdgeCell $cell, string $action): JsonResponse { abort_unless(in_array($action, ['drain', 'undrain', 'restart'], true), 404); - abort_if($action !== 'drain' && $cell->service_ipv4 === null, 409, 'Configure the cell service addresses before making it available.'); $task = DB::transaction(function () use ($action, $cell, $request): EdgeTask { if ($action !== 'restart') { - $cell->update(['drained' => $action === 'drain', ...($action === 'undrain' ? ['status' => 'pending'] : [])]); + $cell->update(['drained' => $action === 'drain', ...($action === 'undrain' ? ['status' => $cell->edge_pool_id === null ? 'unassigned' : 'assigned'] : [])]); } $task = EdgeTask::query()->where('edge_id', $cell->edge_id)->where('type', 'cell_'.$action) ->where('status', 'pending')->where('payload->cell_id', $cell->id)->first() diff --git a/core/app/Http/Controllers/EdgeAgentController.php b/core/app/Http/Controllers/EdgeAgentController.php index ed4594b..d7cc2f3 100644 --- a/core/app/Http/Controllers/EdgeAgentController.php +++ b/core/app/Http/Controllers/EdgeAgentController.php @@ -101,7 +101,7 @@ public function heartbeat(Request $request): JsonResponse 'gateway.connections_rejected' => ['sometimes', 'integer', 'min:0'], 'gateway.errors' => ['sometimes', 'integer', 'min:0'], 'gateway.candidate_rejections' => ['sometimes', 'integer', 'min:0'], - 'cells.*.name' => ['required', 'string', 'max:100', 'distinct'], 'cells.*.status' => ['required', 'in:ready,degraded,failed,drained'], + 'cells.*.name' => ['required', 'regex:/^cell-(0[1-9]|[12][0-9]|3[0-2])$/', 'distinct'], 'cells.*.status' => ['required', 'in:ready,degraded,drained,stopped'], 'cells.*.capacity' => ['required', 'array', 'max:20'], 'noisy_domains' => ['sometimes', 'array', 'max:20'], 'noisy_domains.*.domain_id' => ['required', 'integer', 'exists:domains,id'], 'noisy_domains.*.hostname' => ['nullable', 'string', 'max:253'], @@ -132,8 +132,9 @@ public function heartbeat(Request $request): JsonResponse $edge->cells()->where('name', $cell['name'])->limit(1)->update(['status' => $cell['status'], 'capacity' => $cell['capacity']]); } $reportedNames = collect($data['cells'])->pluck('name'); - $edge->cells()->whereNotIn('name', $reportedNames)->update(['status' => 'degraded', 'capacity' => null]); - $computedReady = $edge->cells()->where('drained', false)->where('status', 'ready')->whereNotNull('service_ipv4')->exists(); + $edge->cells()->whereNotIn('name', $reportedNames)->whereNotNull('edge_pool_id')->update(['status' => 'degraded', 'capacity' => null]); + $edge->cells()->whereNotIn('name', $reportedNames)->whereNull('edge_pool_id')->update(['status' => 'stopped', 'capacity' => null]); + $computedReady = $edge->cells()->whereNotNull('edge_pool_id')->where('drained', false)->where('status', 'ready')->whereNotNull('service_ipv4')->exists(); $listenerReady = $data['listener_ready'] && $computedReady; $edge->update([ 'last_heartbeat_at' => now(), 'agent_version' => $data['agent_version'], @@ -307,7 +308,7 @@ public function taskResult(Request $request, string $task): JsonResponse $action = substr($row->type, 5); $cell->update(match ($action) { 'drain' => ['status' => 'drained', 'drained' => true], - 'undrain' => ['status' => 'pending', 'drained' => false], + 'undrain' => ['status' => $cell->edge_pool_id === null ? 'unassigned' : 'assigned', 'drained' => false], default => [], }); } diff --git a/core/app/Jobs/ProvisionEdgePoolCells.php b/core/app/Jobs/ProvisionEdgePoolCells.php index a626ea0..24ac969 100644 --- a/core/app/Jobs/ProvisionEdgePoolCells.php +++ b/core/app/Jobs/ProvisionEdgePoolCells.php @@ -8,6 +8,7 @@ use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Queue\Queueable; +use Illuminate\Support\Facades\DB; class ProvisionEdgePoolCells implements ShouldBeUniqueUntilProcessing, ShouldQueue { @@ -35,7 +36,27 @@ public function handle(): void ->orderBy('id')->limit(250)->get(); $operation->update(['status' => 'running', 'started_at' => $operation->started_at ?? now(), 'attempts' => $operation->attempts + 1]); foreach ($edges as $edge) { - $edge->cells()->firstOrCreate(['edge_pool_id' => $pool->id], ['name' => $pool->name]); + $assigned = DB::transaction(function () use ($edge, $pool): bool { + if ($edge->cells()->where('edge_pool_id', $pool->id)->exists()) { + return true; + } + $slot = $edge->cells()->whereNull('edge_pool_id')->orderBy('slot')->lockForUpdate()->first(); + if ($slot === null) { + return false; + } + $slot->update(['edge_pool_id' => $pool->id, 'status' => 'assigned']); + + return true; + }); + if (! $assigned) { + $operation->update([ + 'status' => 'failed', 'error' => 'cell_slot_capacity_exhausted', + 'result' => ['pool_id' => $pool->id, 'cursor' => $cursor, 'cells_provisioned' => $provisioned, 'failed_edge_id' => $edge->id], + 'finished_at' => now(), + ]); + + return; + } $provisioned++; $cursor = $edge->id; } diff --git a/core/app/Models/Edge.php b/core/app/Models/Edge.php index ea7a536..054c434 100644 --- a/core/app/Models/Edge.php +++ b/core/app/Models/Edge.php @@ -41,6 +41,6 @@ public function scopeReadyForTraffic(Builder $query): Builder protected function casts(): array { - return ['enabled' => 'boolean', 'drained' => 'boolean', 'capacity' => 'array', 'bootstrap_consumed_at' => 'immutable_datetime', 'identity_revoked_at' => 'immutable_datetime', 'identity_certificate_expires_at' => 'immutable_datetime', 'registered_at' => 'immutable_datetime', 'last_heartbeat_at' => 'immutable_datetime']; + return ['enabled' => 'boolean', 'drained' => 'boolean', 'capacity' => 'array', 'cell_slot_count' => 'integer', 'bootstrap_consumed_at' => 'immutable_datetime', 'identity_revoked_at' => 'immutable_datetime', 'identity_certificate_expires_at' => 'immutable_datetime', 'registered_at' => 'immutable_datetime', 'last_heartbeat_at' => 'immutable_datetime']; } } diff --git a/core/app/Models/EdgeCell.php b/core/app/Models/EdgeCell.php index 182f140..7961f30 100644 --- a/core/app/Models/EdgeCell.php +++ b/core/app/Models/EdgeCell.php @@ -11,7 +11,25 @@ class EdgeCell extends Model protected function casts(): array { - return ['drained' => 'boolean', 'capacity' => 'array']; + return ['drained' => 'boolean', 'capacity' => 'array', 'resource_limits' => 'array']; + } + + protected static function booted(): void + { + static::creating(function (EdgeCell $cell): void { + if ($cell->slot === null) { + $cell->slot = ((int) static::query()->where('edge_id', $cell->edge_id)->max('slot')) + 1; + } + $cell->name = sprintf('cell-%02d', $cell->slot); + $cell->http_port ??= 18080 + $cell->slot; + $cell->https_port ??= 18443 + $cell->slot; + $cell->status_port ??= 19080 + $cell->slot; + $cell->runtime_path ??= "/var/lib/cdnfoundry/runtime/{$cell->name}.json"; + $cell->cache_path ??= "/var/cache/cdnfoundry/{$cell->name}"; + $cell->temporary_path ??= "/var/lib/cdnfoundry/tmp/{$cell->name}"; + $cell->resource_limits ??= ['memory_bytes' => 536870912, 'cpu_millis' => 500, 'pids' => 128, 'cache_bytes' => 268435456, 'temporary_bytes' => 67108864, 'log_bytes' => 16777216]; + $cell->status = $cell->status ?: ($cell->edge_pool_id === null ? 'unassigned' : 'assigned'); + }); } public function edge(): BelongsTo diff --git a/core/database/migrations/2026_07_27_000000_add_bounded_edge_cell_slots.php b/core/database/migrations/2026_07_27_000000_add_bounded_edge_cell_slots.php new file mode 100644 index 0000000..0dfa987 --- /dev/null +++ b/core/database/migrations/2026_07_27_000000_add_bounded_edge_cell_slots.php @@ -0,0 +1,111 @@ +unsignedSmallInteger('cell_slot_count')->default(self::DEFAULT_SLOTS)->after('drained'); + }); + Schema::table('edge_cells', function (Blueprint $table): void { + $table->dropUnique(['edge_id', 'edge_pool_id']); + $table->foreignId('edge_pool_id')->nullable()->change(); + $table->unsignedSmallInteger('slot')->nullable()->after('edge_id'); + $table->unsignedSmallInteger('http_port')->nullable()->after('name'); + $table->unsignedSmallInteger('https_port')->nullable()->after('http_port'); + $table->unsignedSmallInteger('status_port')->nullable()->after('https_port'); + $table->string('runtime_path', 255)->nullable()->after('status_port'); + $table->string('cache_path', 255)->nullable()->after('runtime_path'); + $table->string('temporary_path', 255)->nullable()->after('cache_path'); + $table->json('resource_limits')->nullable()->after('temporary_path'); + }); + if (DB::getDriverName() === 'pgsql') { + DB::statement('ALTER TABLE edge_cells DROP CONSTRAINT IF EXISTS edge_cells_status_check'); + } + + DB::table('edges')->orderBy('id')->get(['id', 'cell_slot_count'])->each(function ($edge): void { + $existing = DB::table('edge_cells')->where('edge_id', $edge->id)->orderByRaw("CASE WHEN name = 'shared-default' THEN 0 WHEN name = 'quarantine-default' THEN 1 ELSE 2 END")->orderBy('id')->get(); + foreach ($existing as $offset => $cell) { + $slot = $offset + 1; + DB::table('edge_cells')->where('id', $cell->id)->update($this->slotAttributes($slot)); + } + for ($slot = $existing->count() + 1; $slot <= (int) $edge->cell_slot_count; $slot++) { + DB::table('edge_cells')->insert([ + 'edge_id' => $edge->id, + 'edge_pool_id' => null, + 'drained' => false, + 'status' => 'stopped', + 'capacity' => null, + 'service_ipv4' => null, + 'service_ipv6' => null, + 'created_at' => now(), + 'updated_at' => now(), + ...$this->slotAttributes($slot), + ]); + } + }); + DB::table('edge_cells')->where('status', 'pending')->update(['status' => 'assigned']); + DB::table('edge_cells')->where('status', 'failed')->update(['status' => 'degraded']); + + Schema::table('edge_cells', function (Blueprint $table): void { + $table->unsignedSmallInteger('slot')->nullable(false)->change(); + $table->unique(['edge_id', 'slot']); + $table->unique(['edge_id', 'edge_pool_id']); + }); + + if (DB::getDriverName() === 'pgsql') { + DB::statement("ALTER TABLE edge_cells ADD CONSTRAINT edge_cells_status_check CHECK (status IN ('assigned', 'unassigned', 'ready', 'degraded', 'drained', 'stopped'))"); + DB::statement('ALTER TABLE edges ADD CONSTRAINT edges_cell_slot_count_check CHECK (cell_slot_count BETWEEN 1 AND 32)'); + DB::statement('ALTER TABLE edge_cells ADD CONSTRAINT edge_cells_slot_check CHECK (slot BETWEEN 1 AND 32)'); + DB::statement('ALTER TABLE edge_cells ADD CONSTRAINT edge_cells_ports_check CHECK (http_port BETWEEN 1 AND 65535 AND https_port BETWEEN 1 AND 65535 AND status_port BETWEEN 1 AND 65535 AND http_port <> https_port AND http_port <> status_port AND https_port <> status_port)'); + DB::statement("ALTER TABLE edge_cells ADD CONSTRAINT edge_cells_name_check CHECK (name = 'cell-' || lpad(slot::text, 2, '0'))"); + DB::statement("ALTER TABLE edge_cells ADD CONSTRAINT edge_cells_paths_check CHECK (runtime_path = '/var/lib/cdnfoundry/runtime/' || name || '.json' AND cache_path = '/var/cache/cdnfoundry/' || name AND temporary_path = '/var/lib/cdnfoundry/tmp/' || name)"); + } + } + + public function down(): void + { + if (DB::getDriverName() === 'pgsql') { + foreach (['edge_cells_paths_check', 'edge_cells_name_check', 'edge_cells_ports_check', 'edge_cells_slot_check'] as $constraint) { + DB::statement("ALTER TABLE edge_cells DROP CONSTRAINT IF EXISTS {$constraint}"); + } + DB::statement('ALTER TABLE edges DROP CONSTRAINT IF EXISTS edges_cell_slot_count_check'); + DB::statement('ALTER TABLE edge_cells DROP CONSTRAINT IF EXISTS edge_cells_status_check'); + DB::statement("ALTER TABLE edge_cells ADD CONSTRAINT edge_cells_status_check CHECK (status IN ('pending', 'ready', 'degraded', 'failed', 'drained'))"); + } + DB::table('edge_cells')->whereNull('edge_pool_id')->delete(); + Schema::table('edge_cells', function (Blueprint $table): void { + $table->dropUnique(['edge_id', 'edge_pool_id']); + $table->dropUnique(['edge_id', 'slot']); + $table->foreignId('edge_pool_id')->nullable(false)->change(); + $table->dropColumn(['slot', 'http_port', 'https_port', 'status_port', 'runtime_path', 'cache_path', 'temporary_path', 'resource_limits']); + $table->unique(['edge_id', 'edge_pool_id']); + }); + Schema::table('edges', fn (Blueprint $table) => $table->dropColumn('cell_slot_count')); + } + + private function slotAttributes(int $slot): array + { + $name = sprintf('cell-%02d', $slot); + + return [ + 'slot' => $slot, + 'name' => $name, + 'http_port' => 18080 + $slot, + 'https_port' => 18443 + $slot, + 'status_port' => 19080 + $slot, + 'runtime_path' => "/var/lib/cdnfoundry/runtime/{$name}.json", + 'cache_path' => "/var/cache/cdnfoundry/{$name}", + 'temporary_path' => "/var/lib/cdnfoundry/tmp/{$name}", + 'resource_limits' => json_encode(['memory_bytes' => 536870912, 'cpu_millis' => 500, 'pids' => 128, 'cache_bytes' => 268435456, 'temporary_bytes' => 67108864, 'log_bytes' => 16777216], JSON_THROW_ON_ERROR), + 'updated_at' => now(), + ]; + } +}; diff --git a/core/tests/Feature/EdgeProxyTest.php b/core/tests/Feature/EdgeProxyTest.php index 5f05505..1dcde25 100644 --- a/core/tests/Feature/EdgeProxyTest.php +++ b/core/tests/Feature/EdgeProxyTest.php @@ -195,6 +195,14 @@ public function test_edge_bootstrap_is_one_time_and_artifacts_require_active_ide $created = $this->actingAs($admin)->postJson('/api/admin/edges', ['name' => 'edge-ir-1', 'country_code' => 'IR', 'continent_code' => 'AS', 'ipv4' => '203.0.113.10', 'ipv6' => '2001:db8::10']) ->assertCreated(); $id = $created->json('data.id'); + $slots = Edge::query()->findOrFail($id)->cells()->orderBy('slot')->get(); + $this->assertCount(8, $slots); + $this->assertSame(['cell-01', 'cell-02', 'cell-03', 'cell-04', 'cell-05', 'cell-06', 'cell-07', 'cell-08'], $slots->pluck('name')->all()); + $this->assertCount(8, $slots->pluck('runtime_path')->unique()); + $this->assertCount(8, $slots->pluck('http_port')->unique()); + $this->assertSame(6, $slots->whereNull('edge_pool_id')->where('status', 'unassigned')->count()); + $this->assertSame(536870912, $slots->first()->resource_limits['memory_bytes']); + $this->actingAs($admin)->postJson('/api/admin/edges', ['name' => 'too-many-slots', 'country_code' => 'IR', 'continent_code' => 'AS', 'ipv4' => '203.0.113.19', 'cell_slot_count' => 33])->assertUnprocessable(); $bootstrap = $created->json('data.bootstrap_token'); $registration = ['edge_id' => $id, 'bootstrap_token' => $bootstrap, 'agent_version' => '1.0.0', 'certificate_request' => $this->certificateRequest($id)]; $registered = $this->postJson('/edge/v1/register', $registration)->assertCreated(); @@ -204,14 +212,14 @@ public function test_edge_bootstrap_is_one_time_and_artifacts_require_active_ide $differentRegistration = [...$registration, 'certificate_request' => $this->certificateRequest($id)]; $this->postJson('/edge/v1/register', $differentRegistration)->assertUnauthorized(); $this->withHeaders($identity)->postJson('/edge/v1/heartbeat', ['agent_version' => '1.0.0', 'listener_ready' => true, 'active_sequence' => 0, 'cells' => [ - ['name' => 'shared-default', 'status' => 'ready', 'capacity' => ['active_connections' => 0, 'memory_usage' => 0]], + ['name' => 'cell-01', 'status' => 'ready', 'capacity' => ['active_connections' => 0, 'memory_usage' => 0]], ], 'gateway' => [ 'ready' => true, 'active_revision' => 0, 'routes' => 2, 'listeners' => 4, 'connections_active' => 0, 'connections_accepted' => 12, 'connections_rejected' => 3, 'errors' => 1, 'candidate_rejections' => 1, ]])->assertOk(); $this->postJson('/edge/v1/register', $registration)->assertUnauthorized(); - $this->assertDatabaseHas('edge_cells', ['edge_id' => $id, 'name' => 'shared-default', 'status' => 'ready']); + $this->assertDatabaseHas('edge_cells', ['edge_id' => $id, 'name' => 'cell-01', 'slot' => 1, 'status' => 'ready']); $this->assertSame(4, Edge::query()->findOrFail($id)->capacity['gateway']['listeners']); [$user, $domain] = $this->ownedDomain(); @@ -234,7 +242,7 @@ public function test_edge_bootstrap_is_one_time_and_artifacts_require_active_ide $artifactCount = EdgeArtifact::query()->where('domain_id', $domain->id)->count(); $hostname = $domain->dnsRecords()->findOrFail($record)->name; $this->withHeaders($identity)->postJson('/edge/v1/heartbeat', ['agent_version' => '1.0.0', 'listener_ready' => true, 'active_sequence' => $artifact->sequence, 'cells' => [ - ['name' => 'shared-default', 'status' => 'ready', 'capacity' => ['active_connections' => 1]], + ['name' => 'cell-01', 'status' => 'ready', 'capacity' => ['active_connections' => 1]], ], 'passive_origins' => [[ 'domain' => $domain->name, 'hostname' => $hostname, 'failure_count' => 2, 'last_status' => 502, 'last_failed_at' => now()->timestamp, @@ -242,11 +250,24 @@ public function test_edge_bootstrap_is_one_time_and_artifacts_require_active_ide $this->assertSame('passive', $domain->dnsRecords()->findOrFail($record)->origin_health['source']); $this->assertSame($artifactCount, EdgeArtifact::query()->where('domain_id', $domain->id)->count()); $this->actingAs($admin)->getJson('/api/admin/edge-routing')->assertOk()->assertJsonPath('data.global.0.id', $id); - $cell = Edge::query()->findOrFail($id)->cells()->firstOrFail(); + $cell = Edge::query()->findOrFail($id)->cells()->whereNotNull('service_ipv4')->firstOrFail(); $this->actingAs($admin)->postJson("/api/admin/edge-cells/{$cell->id}/drain")->assertAccepted(); $this->assertTrue($cell->refresh()->drained); $this->actingAs($admin)->postJson("/api/admin/edge-cells/{$cell->id}/undrain")->assertAccepted(); $this->assertFalse($cell->refresh()->drained); + $undrainTask = EdgeTask::query()->where('type', 'cell_undrain')->where('payload->cell_id', $cell->id)->firstOrFail(); + $this->withHeaders($identity)->postJson("/edge/v1/tasks/{$undrainTask->id}/result", [ + 'status' => 'succeeded', 'result' => ['status' => 'completed'], + ])->assertOk(); + $this->assertSame('assigned', $cell->refresh()->status); + $unassignedCell = Edge::query()->findOrFail($id)->cells()->whereNull('edge_pool_id')->firstOrFail(); + $this->actingAs($admin)->postJson("/api/admin/edge-cells/{$unassignedCell->id}/drain")->assertAccepted(); + $this->actingAs($admin)->postJson("/api/admin/edge-cells/{$unassignedCell->id}/undrain")->assertAccepted(); + $unassignedTask = EdgeTask::query()->where('type', 'cell_undrain')->where('payload->cell_id', $unassignedCell->id)->firstOrFail(); + $this->withHeaders($identity)->postJson("/edge/v1/tasks/{$unassignedTask->id}/result", [ + 'status' => 'succeeded', 'result' => ['status' => 'completed'], + ])->assertOk(); + $this->assertSame('unassigned', $unassignedCell->refresh()->status); $this->actingAs($user)->postJson("/api/domains/{$domain->id}/deploy")->assertAccepted(); $this->actingAs($user)->postJson("/api/domains/{$domain->id}/deploy")->assertAccepted(); $this->assertSame(1, EdgeArtifact::query()->where('edge_id', $id)->where('domain_id', $domain->id)->count()); @@ -265,6 +286,8 @@ public function test_edge_bootstrap_is_one_time_and_artifacts_require_active_ide $poolResponse = $this->actingAs($admin)->postJson('/api/admin/edge-pools', ['name' => 'dedicated-test', 'kind' => 'dedicated'])->assertAccepted(); $pool = $poolResponse->json('data.pool.id'); $dedicatedCell = Edge::query()->findOrFail($id)->cells()->where('edge_pool_id', $pool)->firstOrFail(); + $this->assertSame(8, Edge::query()->findOrFail($id)->cells()->count(), 'Pool provisioning must assign an existing bounded slot.'); + $this->assertSame(3, $dedicatedCell->slot); $this->actingAs($admin)->patchJson("/api/admin/edge-cells/{$dedicatedCell->id}", [ 'service_ipv4' => '203.0.113.20', 'service_ipv6' => '2001:db8::20', ])->assertOk(); @@ -299,8 +322,8 @@ public function test_edge_bootstrap_is_one_time_and_artifacts_require_active_ide $operation = Operation::query()->findOrFail($move->json('data.operation_id')); $moveArtifact = EdgeArtifact::query()->where('edge_id', $id)->where('domain_id', $domain->id)->latest('sequence')->firstOrFail(); $this->withHeaders($identity)->postJson('/edge/v1/heartbeat', ['agent_version' => '1.0.0', 'listener_ready' => true, 'active_sequence' => $artifact->sequence, 'cells' => [ - ['name' => 'shared-default', 'status' => 'ready', 'capacity' => ['active_connections' => 1]], - ['name' => 'dedicated-test', 'status' => 'ready', 'capacity' => ['active_connections' => 0]], + ['name' => 'cell-01', 'status' => 'ready', 'capacity' => ['active_connections' => 1]], + ['name' => $dedicatedCell->name, 'status' => 'ready', 'capacity' => ['active_connections' => 0]], ]])->assertOk(); $this->withHeaders($identity)->postJson('/edge/v1/config/applied', ['sequence' => $moveArtifact->sequence])->assertOk(); $this->assertDatabaseHas('domain_edge_placements', ['domain_id' => $domain->id, 'active_pool_id' => 1, 'target_pool_id' => $pool, 'state' => 'draining']); @@ -343,7 +366,7 @@ public function test_origin_test_tasks_and_phase_four_operations_are_visible_to_ $registered = $this->postJson('/edge/v1/register', ['edge_id' => $edgeId, 'bootstrap_token' => $created->json('data.bootstrap_token'), 'agent_version' => '1.0.0', 'certificate_request' => $this->certificateRequest($edgeId)])->assertCreated(); $identity = $this->edgeIdentityHeaders($registered->json('data.identity_certificate_serial')); $this->withHeaders($identity)->postJson('/edge/v1/heartbeat', ['agent_version' => '1.0.0', 'listener_ready' => true, 'active_sequence' => 0, 'cells' => [ - ['name' => 'shared-default', 'status' => 'ready', 'capacity' => ['active_connections' => 0]], + ['name' => 'cell-01', 'status' => 'ready', 'capacity' => ['active_connections' => 0]], ]])->assertOk(); [$user, $domain] = $this->ownedDomain(); diff --git a/core/tests/Feature/FilamentWorkflowTest.php b/core/tests/Feature/FilamentWorkflowTest.php index cda7f18..7b993a5 100644 --- a/core/tests/Feature/FilamentWorkflowTest.php +++ b/core/tests/Feature/FilamentWorkflowTest.php @@ -5,12 +5,14 @@ use App\Enums\DomainLifecycleState; use App\Filament\Admin\Pages\Telemetry; use App\Filament\Admin\Resources\DnsClusters\Pages\ListDnsClusters; +use App\Filament\Admin\Resources\EdgePools\Pages\ListEdgePools; use App\Filament\Admin\Resources\Edges\Pages\EditEdge; use App\Filament\Admin\Resources\Edges\Pages\ListEdges; use App\Filament\Admin\Resources\Edges\RelationManagers\CellsRelationManager; use App\Filament\Domain\Resources\Domains\Pages\ViewDomain; use App\Filament\Domain\Resources\Domains\RelationManagers\DnsRecordsRelationManager; use App\Jobs\BuildUsageRollups; +use App\Jobs\ProvisionEdgePoolCells; use App\Jobs\ReconcileAllDnsZones; use App\Jobs\ReconcileAllEdgeDomains; use App\Jobs\ReconcileDnsZone; @@ -138,6 +140,15 @@ public function test_cells_show_enrollment_state_and_use_the_same_address_rules_ 'name' => $pool->name, 'service_ipv4' => $edge->ipv4, 'service_ipv6' => $edge->ipv6, + 'capacity' => [ + 'cpu_usage' => 0.25, + 'memory_usage' => 67108864, + 'memory_limit' => 536870912, + 'cache_usage' => 10485760, + 'cache_limit' => 268435456, + 'temporary_storage_usage' => 1048576, + 'temporary_storage_limit' => 67108864, + ], ]); Filament::setCurrentPanel(Filament::getPanel('admin')); $this->actingAs($admin); @@ -146,7 +157,11 @@ public function test_cells_show_enrollment_state_and_use_the_same_address_rules_ 'pageClass' => EditEdge::class, ]); - $component()->assertSee('Awaiting agent enrollment')->assertSee('Awaiting heartbeat'); + $component()->assertSee('Awaiting agent enrollment') + ->assertSee($pool->name) + ->assertSee('64.0 MiB / 512.0 MiB memory') + ->assertSee('10.0 MiB / 256.0 MiB cache') + ->assertSee('1.00 MiB / 64.0 MiB temporary'); $component()->callTableAction('edit', $cell, [ 'service_ipv4' => '10.0.0.10', 'service_ipv6' => '2001:db8::11', @@ -269,6 +284,23 @@ public function test_administrator_can_queue_global_reconciliation_and_bounded_u Queue::assertPushed(BuildUsageRollups::class); } + public function test_administrator_can_reconcile_a_pools_missing_cell_assignments(): void + { + Queue::fake(); + $admin = User::factory()->admin()->create(); + $pool = EdgePool::query()->where('kind', 'shared')->firstOrFail(); + Filament::setCurrentPanel(Filament::getPanel('admin')); + $this->actingAs($admin); + + Livewire::test(ListEdgePools::class) + ->callTableAction('reconcileCells', $pool) + ->assertHasNoFormErrors(); + + $operation = Operation::query()->where('type', 'edge.pool_provision')->firstOrFail(); + $this->assertSame($pool->id, $operation->input['pool_id']); + Queue::assertPushed(ProvisionEdgePoolCells::class, fn (ProvisionEdgePoolCells $job): bool => $job->poolId === $pool->id && $job->operationId === $operation->id); + } + public function test_domain_dns_reconcile_action_reuses_the_policy_aware_endpoint(): void { Queue::fake(); diff --git a/docker/nginx/openresty.conf b/docker/nginx/openresty.conf index b3a120d..8905001 100644 --- a/docker/nginx/openresty.conf +++ b/docker/nginx/openresty.conf @@ -16,6 +16,9 @@ events { http { client_body_temp_path /var/lib/nginx/tmp/client_body 1 2; proxy_temp_path /var/lib/nginx/tmp/proxy; + fastcgi_temp_path /var/lib/nginx/tmp/fastcgi; + uwsgi_temp_path /var/lib/nginx/tmp/uwsgi; + scgi_temp_path /var/lib/nginx/tmp/scgi; proxy_cache_path /var/cache/nginx/content levels=1:2 keys_zone=customer_content:10m max_size=192m inactive=1h use_temp_path=off; include /usr/local/openresty/nginx/conf/mime.types; default_type application/octet-stream; diff --git a/docs/concepts/edges-and-cells.md b/docs/concepts/edges-and-cells.md index 6aa810f..3ec0db8 100644 --- a/docs/concepts/edges-and-cells.md +++ b/docs/concepts/edges-and-cells.md @@ -11,26 +11,42 @@ flowchart TB Placement --> Pool["shared, quarantine, or dedicated pool"] Pool --> EdgeA["Edge A"] Pool --> EdgeB["Edge B"] - EdgeA --> SharedA["Shared cell"] - EdgeA --> QuarantineA["Quarantine cell"] - EdgeB --> SharedB["Shared cell"] - EdgeB --> QuarantineB["Quarantine cell"] + EdgeA --> SharedA["cell-01 assigned to shared"] + EdgeA --> QuarantineA["cell-02 assigned to quarantine"] + EdgeB --> SharedB["cell-01 assigned to shared"] + EdgeB --> QuarantineB["cell-02 assigned to quarantine"] SharedA -->|"assigned domains as data"| RuntimeA["One OpenResty runtime"] SharedB -->|"assigned domains as data"| RuntimeB["One OpenResty runtime"] ``` An edge is one enrolled agent identity and host. A pool is a stable service -class: `shared`, `quarantine`, or exceptional `dedicated`. A cell is the bounded -OpenResty runtime for one pool on one edge. - -The shipped production profile creates two cells per edge host: - -- `shared-default`, with 2 GiB memory and 2 CPU limits; -- `quarantine-default`, with 512 MiB memory and 0.5 CPU limits. - -The edge agent is separate, read-only, non-root, and limited to 128 MiB and 0.25 +class: `shared`, `quarantine`, or exceptional `dedicated`. A cell slot is a +bounded OpenResty runtime with stable identity independent of its optional pool +assignment. + +The shipped production profile creates exactly eight slots per edge host, +`cell-01` through `cell-08`. Fresh control-plane edges default to the same +bounded count and may select 1–32 slots before creation. The count is immutable +after creation. Each slot has unique host ports, runtime/cache/temporary paths, +status, capacity, and resource limits. Initially: + +- `cell-01` is assigned to `shared-default`; +- `cell-02` is assigned to `quarantine-default`; +- the remaining slots are unassigned, hold an empty runtime, and are available + for explicit later placement work. + +Assignment never changes slot identity or storage paths. The registered states +are assigned, unassigned, ready, degraded, drained, and stopped. A missing +assigned runtime becomes degraded; a missing unassigned runtime remains +stopped. + +Every cell is limited to 512 MiB, 0.5 CPU, 128 PIDs, 256 MiB cache temporary +storage, 64 MiB request temporary storage, and 16 MiB logs by the shipped +topology. The edge agent is separate, read-only, non-root, and limited to 128 MiB and 0.25 CPU. It owns identity, artifact validation, atomic runtime files, acknowledgements, -and control tasks. It does not proxy customer traffic. +and control tasks. It does not proxy customer traffic and has no container-engine +socket. Drain, undrain, and bounded restart controls use authenticated private +cell endpoints. ## Enrollment diff --git a/docs/development/testing.md b/docs/development/testing.md index 897964d..deff3e3 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -10,6 +10,10 @@ qualification. It requires the development edge profile and the locally built `cdnfoundry/edge-gateway:qualification` image. Coding agents must not run the separate manual browser checklist. +Run `python3 tests/e2e/cell_inventory.py` for the eight-slot non-browser cell +inventory, isolation, restart, storage-bound, and idle/active overhead +qualification. + ::: danger Database guard Laravel tests must use `APP_ENV=testing`, `DB_CONNECTION=sqlite`, and `DB_DATABASE=:memory:`. Never point migration or truncation tests at the diff --git a/docs/manual-browser-qualification.md b/docs/manual-browser-qualification.md index 50b117d..31b1eef 100644 --- a/docs/manual-browser-qualification.md +++ b/docs/manual-browser-qualification.md @@ -399,6 +399,95 @@ Phase 1 is complete only when every applicable gate is **Passed**. A missing UI, failed configured IPv6 path, failed IPv4-only path, unexecuted scale run, or unrecorded browser result keeps the phase incomplete. +## Phase 2 — Bounded cell inventory + +Do not start this gate until Phase 1 is Passed. Record the edge UUID, configured +slot count, release SHA, host, browser, and timestamps. The shipped topology +uses eight slots. + +### Fresh inventory and authorization + +1. Sign in as an administrator and open **Edge network → Edges → New edge**. + Enter a unique name, country, continent, public IPv4, optional public IPv6, + and **Cell slots = 8**. Save and copy the one-time bootstrap token. +2. Open the new edge and its **Cells** relation. Expect exactly `cell-01` + through `cell-08`, consecutive slot numbers, unique HTTP/HTTPS/status ports, + unique runtime paths, and no extra row. Expect `cell-01` assigned to shared, + `cell-02` assigned to quarantine, and `cell-03`–`cell-08` unassigned. +3. Attempt another edge with slot counts 0 and 33. Expect field validation and + no edge, slot, token, task, or audit side effect. Create a disposable edge + with one slot and expect exactly `cell-01`; drain, disable, and delete it. +4. Enroll the eight-slot edge. Refresh its detail page and Cells relation. + Expect current enrollment, heartbeat, agent version, gateway readiness, and + every running slot's ready/drained state and capacity. No bootstrap secret + may reappear. +5. Sign in as a domain user and request the edge list/detail and cell API URLs. + Expect denial without slot identity, assignment, address, capacity, path, + resource, revision, or failure disclosure. + +### Runtime controls and isolation + +1. Record every cell's status, active revision, assigned domain count, active + connections, CPU, memory, cache, temporary storage, and last restart. +2. Drain `cell-02`. Expect one pending operation/task, then **Drained** only for + that slot. Repeat the action with the same idempotency key through the API; + expect the same operation and no duplicate task. +3. Undrain `cell-02`. Expect pending then ready. Restart `cell-02`; expect its + restart timestamp/generation to advance after a bounded drain while the + edge agent, gateway, `cell-01`, and `cell-03`–`cell-08` remain available. +4. Stop `cell-04` through the operator runtime fixture. Expect it to become + stopped or degraded with a stable reason. Valid traffic targeting another + cell must continue, gateway and agent readiness must remain, and unrelated + revisions/capacity must not reset. +5. Saturate the disposable `cell-04` CPU and memory only up to its cgroup + ceilings. Expect the container limit to hold and another cell's traffic and + status to remain available. Record host and per-cell metrics. +6. Restore `cell-04`. Expect reconciliation to return it to the latest active + revision without editing generated files or replaying unrelated cells. + +### Recovery, bounds, and evidence + +1. Restart the agent. Expect enrollment identity, mutual TLS, acknowledgements, + active sequence, all eight slot files, drained controls, and last-valid + snapshot recovery. No cell-engine socket may be mounted in the agent. +2. Make the control plane unavailable. Restart one cell and keep valid traffic + on another. Expect local serving and previous valid state to continue; after + restoration, expect convergence without duplicate activation. +3. Present an invalid slot mapping and invalid runtime candidate through the + supported fixture. Expect rejection, bounded reason, and previous active + state for every unrelated slot. +4. Confirm each cell has 512 MiB memory, 0.5 CPU, 128 PID, 256 MiB cache, 64 MiB + request-temporary, and 16 MiB log ceilings. Fill each disposable storage area + to its ceiling and expect bounded failure without host filesystem growth or + another cell losing service. +5. Link the eight-slot agent-owned report with host/topology, idle and active + overhead per slot, concurrency, workload, saturation result, accepted limit, + crash isolation, restart, snapshot recovery, IPv4/IPv6, and baseline + regression evidence. + +### Phase 2 completion gate + +Agent-owned implementation, PostgreSQL expand migration, 162 isolated Laravel +tests, Go format/vet/test/build, Compose validation, the cumulative non-browser +baseline/runtime regression, and the eight-slot overhead/isolation test passed +on 2026-07-27. The owner browser run above and the Phase 1 release gate must both +be Passed before changing this phase's release decision from **Blocked**. + +| Gate | Result | Required evidence | +| --- | --- | --- | +| Implementation | Passed | [Bounded inventory design and operations](operations/cell-inventory.md) | +| Unit and feature tests | Passed | 162 Laravel tests / 1,280 assertions and Go format/vet/test/build | +| Real-runtime E2E | Passed | Eight-slot test plus enrollment, mTLS, snapshot, restart, and cumulative baseline runtime suite | +| IPv4 and IPv6 | Passed | Authoritative DNS and edge baseline dual-stack/IPv4-only evidence | +| Scale | Passed | Eight-slot idle/active overhead and isolation report | +| Failure and recovery | Passed | Control outage, restart, retry, rollback, and last-valid cumulative evidence | +| Isolation | Passed | `cell-04` stop left `cell-05` and support process ready | +| Observability | Pending owner run | Runtime metrics passed; cell state/capacity and alert screenshots remain owner evidence | +| Documentation | Passed | User, administrator, reference, deployment, operations, troubleshooting, and runbook checks | +| Manual qualification | Pending owner run | Every exact browser checkpoint above | +| Regression | Passed | Completed baseline and Phase 1 cumulative non-browser checks | +| Release decision | Blocked | Phase 1 and owner-run Phase 2 browser evidence are not Passed | + ## Failure record For every failed or blocked checkpoint, record: diff --git a/docs/operations/cell-inventory.md b/docs/operations/cell-inventory.md new file mode 100644 index 0000000..dfeb5de --- /dev/null +++ b/docs/operations/cell-inventory.md @@ -0,0 +1,106 @@ +--- +title: Bounded cell inventory +description: Install, monitor, control, and recover stable OpenResty cell slots. +--- + +# Bounded cell inventory + +Each production edge installation starts exactly eight generic OpenResty slots, +`cell-01` through `cell-08`. PostgreSQL records the configured slot count and +each slot's identity, assignment, ports, paths, state, capacity, and resource +ceilings. A slot is never created for a domain and its identity never changes +when work moves. + +## Installation contract + +Create the edge with the intended `cell_slot_count` (1–32) before enrollment. +The shipped Compose topology implements the default count of eight. An operator +using a non-default count must render the same consecutive services and update +the agent assignment/status lists before starting the host; a mismatch is a +failed installation, not elastic scaling. + +| Slot | HTTP | HTTPS | Status | Initial assignment | +| --- | ---: | ---: | ---: | --- | +| `cell-01` | 18081 | 18444 | 19081 | `shared-default` | +| `cell-02` | 18082 | 18445 | 19082 | `quarantine-default` | +| `cell-03`–`cell-08` | 18083–18088 | 18446–18451 | 19083–19088 | Unassigned | + +All host publications are loopback diagnostics. Customer traffic enters only +through the gateway's public service addresses. Each container has separate +tmpfs-backed cache, request-temporary, and log storage, plus CPU, memory, PID, +and file-descriptor limits. The agent has its own smaller limits and no Docker +or other container-engine socket. + +## State and controls + +The authenticated heartbeat accepts at most 32 distinct canonical slot names. +Ready, degraded, drained, and stopped are runtime observations; assigned and +unassigned describe durable inventory. An assigned slot omitted from a +heartbeat becomes degraded. An unassigned omitted slot is stopped. Gateway +readiness is authoritative when the gateway is configured and is never tied to +a hardcoded shared-cell name. + +Administrators may drain, undrain, or restart an existing slot. The +control plane commits desired state and queues one coalesced task; the agent +calls only that slot's private authenticated control endpoint. Restart drains +the runtime briefly and advances its restart generation without restarting the +agent, gateway, or unrelated cells. + +Use **Reconcile cells** on a service pool when an existing edge is missing that +pool's Phase 2 assignment. Reconciliation asynchronously claims one existing +unassigned slot per missing edge and never creates another slot. Phase 2 allows +at most one cell from a pool on each edge; assigning several cells from one edge +to the same pool is Phase 3 work. + +## Failure and recovery + +- A stopped or saturated slot leaves the gateway, agent, and other slot cgroups + running. Routes targeting it fail in isolation. +- Invalid signed state or an invalid slot mapping never replaces the agent's + previous active state. +- Agent restart reconstructs every assigned and empty unassigned slot file from + its durable signed snapshot. +- Control-plane outage leaves active gateway and cell state serving locally. +- Cache, temporary, and log tmpfs ceilings prevent disk growth. Loss is safe + because these artifacts are derived. +- Enrollment identity, mutual TLS, acknowledgements, and full snapshot recovery + remain in the independent persistent agent volume. + +Monitor the edge detail and Cells relation for assignment, state, revision, +workload, connections, CPU, memory, cache, temporary storage, and last restart. +Treat any slot-count/name/path mismatch, stale heartbeat, assigned stopped slot, +or resource limit drift as degraded and reconcile the host definition before +returning traffic. + +## Qualification + +Run the non-browser inventory runtime test documented in +[Testing and qualification](../development/testing.md). Record the host CPU and +memory, Docker version, eight-slot idle and active cgroup usage, concurrency, +request rate and latency, the first saturated resource, and the accepted host +limit. Then run the owner-only browser checklist. Coding agents must not run or +claim the browser result. + +On 2026-07-27, `python3 tests/e2e/cell_inventory.py` passed on an x86_64 VMware +host with 32 Intel Xeon E5-2697 v4 vCPUs, 15.6 GiB RAM, and Docker 29.1.3. The +test created exactly eight real read-only OpenResty containers, verified unique +identities and cgroup/tmpfs ceilings, made 160 active health requests, stopped +and restored `cell-04`, and confirmed `cell-05` plus the separate support +process remained ready. The first run found and corrected unsafe default Nginx +temp paths outside the writable quota; the passing run used the production +paths. + +| Measurement | Result | +| --- | --- | +| Idle CPU per slot | 1.21%–1.57% sample | +| Idle memory per slot | 97.25–98.26 MiB | +| Active CPU per slot | 1.11%–1.28% sample | +| Active memory per slot | 98.20–98.75 MiB | +| Active workload | 160 health requests in 40.056 s through serialized Docker exec probes | +| Isolation | Stopped slot did not stop another slot or support process | +| Saturation | Not reached; installation is intentionally capped at eight shipped slots (32 schema maximum) | +| Accepted limit | Eight 512 MiB / 0.5 CPU / 128 PID slots on this topology | + +The serialized probe rate measures active per-process overhead, not customer +throughput. Operators must load-test gateway-to-cell traffic on their host +before admitting workload up to the cgroup ceilings. diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 2f3518f..26e01a0 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -150,9 +150,10 @@ The edge-agent binary also accepts these internal variables: | `EDGE_STATE_DIR` | `/var/lib/cdnfoundry/agent` | Persistent identity, state, controls, acknowledgements | | `EDGE_RUNTIME_DIR` | empty | Active and previous compiled runtime directories | | `EDGE_CELL_STATUS_URLS` | empty | Comma-separated internal cell endpoints | +| `EDGE_CELL_ASSIGNMENTS` | `{}` | JSON object mapping at most 32 stable `cell-NN` names to a pool name or an empty unassigned value | | `EDGE_ONCE` | `false` | Run one sync cycle for diagnostics | -The OpenResty container receives `EDGE_CELL_NAME`, `EDGE_RUNTIME_FILE`, and +Production fixes `EDGE_CELL_ASSIGNMENTS` to eight stable slots. The OpenResty container receives `EDGE_CELL_NAME`, `EDGE_RUNTIME_FILE`, and `EDGE_STATUS_TOKEN` from Compose. These describe a cell and are not customer settings. diff --git a/docs/reference/services-and-ports.md b/docs/reference/services-and-ports.md index 8072831..6e53f70 100644 --- a/docs/reference/services-and-ports.md +++ b/docs/reference/services-and-ports.md @@ -12,7 +12,7 @@ description: Map CDNFoundry Compose services, profiles, listeners, networks, and | `control` | `core`, `web`, `edge-control`, `horizon`, `scheduler`, local `control-db`, local `redis` | | `dns` | `pdns-db`, `pdns-auth`, `dnsdist`, `mmdb-updater` | | `telemetry` | `clickhouse`, `vector`, `prometheus`, `node-exporter`, `alertmanager` | -| `edge` | `edge`, `edge-quarantine`, `edge-agent`, `vector`, `mmdb-updater` | +| `edge` | `cell-01` through `cell-08`, `edge-agent`, `edge-gateway`, `vector`, `mmdb-updater` | | `tools` | explicit `migrate` and `pdns-migrate` one-shot services | `deploy/production/compose.external-control-data.yml` replaces local @@ -27,8 +27,7 @@ gateways. | Browser/API web | `127.0.0.1:8080` | Publish through the control Caddy overlay | | Edge control mTLS | `0.0.0.0:8443` | Restrict to registered edge sources | | DNSdist | `${DNS_BIND_V4}:53` TCP and UDP | Public authoritative DNS | -| Shared cell HTTP/HTTPS | `0.0.0.0:80`, `0.0.0.0:443` | Public customer traffic | -| Quarantine cell | private TCP `8081`, `8444` | Reachable only from its gateway network | +| Cell slot host diagnostics | loopback `18081`–`18088`, `18444`–`18451`, `19081`–`19088` | HTTP, HTTPS, and status; never public | | Edge gateway | operator service IPv4/IPv6 TCP `80`, `443` | Public ingress; TLS passes through | | Gateway metrics | TCP `9105` | Restrict to edge agent and monitoring | | Cell gateway contract | TCP `8081`, `8444` | Private gateway-to-cell network; PROXY protocol version 2 required | diff --git a/docs/roadmap.md b/docs/roadmap.md index 231b43c..aec3059 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -117,12 +117,12 @@ active overhead per slot. **Completion checklist:** -- [ ] Fresh installation creates exactly the configured slots. -- [ ] Every slot has unique identity, paths, ports, health, and limits. -- [ ] One crashed or saturated cell does not stop the gateway, agent, or other cells. -- [ ] Cell restart and rollback preserve unrelated traffic. -- [ ] Cache, temporary, and log storage remain bounded. -- [ ] Existing enrollment, mTLS, and snapshot recovery still pass. +- [x] Fresh installation creates exactly the configured slots. +- [x] Every slot has unique identity, paths, ports, health, and limits. +- [x] One crashed or saturated cell does not stop the gateway, agent, or other cells. +- [x] Cell restart and rollback preserve unrelated traffic. +- [x] Cache, temporary, and log storage remain bounded. +- [x] Existing enrollment, mTLS, and snapshot recovery still pass. - [ ] Tests, scale, documentation, and manual qualification pass. ## Phase 3 — Multi-cell pools and stable placement diff --git a/edge-agent/main.go b/edge-agent/main.go index 93d7d7d..998678e 100644 --- a/edge-agent/main.go +++ b/edge-agent/main.go @@ -29,7 +29,7 @@ import ( "time" ) -const version = "1.1.0" +const version = "1.2.0" type identity struct{ EdgeID, Certificate, PrivateKey, PublicKey string } type state struct { @@ -47,6 +47,7 @@ type client struct { gatewayStatusURL string derivedEnsured bool statusURLs []string + cellAssignments map[string]string http *http.Client id identity } @@ -69,6 +70,14 @@ func main() { gatewayStatusURL: env("EDGE_GATEWAY_STATUS_URL", ""), statusURLs: splitNonempty(env("EDGE_CELL_STATUS_URLS", "")), http: &http.Client{Timeout: 15 * time.Second}, } + if err := json.Unmarshal([]byte(env("EDGE_CELL_ASSIGNMENTS", "{}")), &c.cellAssignments); err != nil || len(c.cellAssignments) > 32 { + fatal(errors.New("EDGE_CELL_ASSIGNMENTS must be an object with at most 32 slots")) + } + for cellName, poolName := range c.cellAssignments { + if !validCellName(cellName) || poolName != "" && !validPoolName(poolName) { + fatal(errors.New("EDGE_CELL_ASSIGNMENTS contains an invalid slot or pool")) + } + } if err := c.configureServerTrust(env("EDGE_CONTROL_CA_CERTIFICATE", "")); err != nil { fatal(err) } @@ -133,6 +142,9 @@ func (c *client) ensureDerivedRuntime(current state) error { return err } } + if err := c.writeCellRuntimes(current.Sequence, pools); err != nil { + return err + } if c.gatewayBindings != "" { gateway, err := compileGateway(current.Sequence, pools, c.gatewayBindings) if err != nil { @@ -767,6 +779,9 @@ func (c *client) activate(s state) error { return c.rollbackActive(active, previous, err) } } + if err := c.writeCellRuntimes(s.Sequence, pools); err != nil { + return c.rollbackActive(active, previous, err) + } if c.gatewayBindings != "" { gateway, err := compileGateway(s.Sequence, pools, c.gatewayBindings) if err != nil { @@ -780,6 +795,56 @@ func (c *client) activate(s state) error { return nil } +func (c *client) writeCellRuntimes(sequence uint64, pools map[string]map[string]any) error { + for cellName, poolName := range c.resolvedCellAssignments(pools) { + runtime := pools[poolName] + if runtime == nil { + runtime = map[string]any{"schema_version": 1, "sequence": sequence, "hosts": map[string]any{}, "certificates": map[string]any{}} + } + if err := atomicJSON(filepath.Join(c.runtimeDir, cellName+".json"), runtime); err != nil { + return err + } + } + return nil +} + +func (c *client) resolvedCellAssignments(pools map[string]map[string]any) map[string]string { + resolved := make(map[string]string, len(c.cellAssignments)) + assigned := map[string]bool{} + emptyCells := []string{} + for cellName, poolName := range c.cellAssignments { + resolved[cellName] = poolName + if poolName == "" { + emptyCells = append(emptyCells, cellName) + } else { + assigned[poolName] = true + } + } + unassignedPools := []string{} + for poolName := range pools { + if !assigned[poolName] { + unassignedPools = append(unassignedPools, poolName) + } + } + sort.Strings(emptyCells) + sort.Strings(unassignedPools) + for index, poolName := range unassignedPools { + if index >= len(emptyCells) { + break + } + resolved[emptyCells[index]] = poolName + } + return resolved +} + +func validCellName(name string) bool { + if len(name) != 7 || !strings.HasPrefix(name, "cell-") { + return false + } + slot, err := strconv.Atoi(name[5:]) + return err == nil && slot >= 1 && slot <= 32 && fmt.Sprintf("cell-%02d", slot) == name +} + func (c *client) rollbackActive(active, previous string, cause error) error { os.RemoveAll(active) if _, err := os.Stat(previous); err == nil { @@ -967,7 +1032,7 @@ func (c *client) heartbeat(sequence uint64) error { cells, failures, security := c.runtimeStatus() listenerReady := false for _, cell := range cells { - if cell["name"] == "shared-default" && cell["status"] == "ready" { + if cell["status"] == "ready" { listenerReady = true } } diff --git a/edge-agent/main_test.go b/edge-agent/main_test.go index 8436dc7..6724fa4 100644 --- a/edge-agent/main_test.go +++ b/edge-agent/main_test.go @@ -43,7 +43,7 @@ func TestVerifyAndCompatibility(t *testing.T) { } func TestVersionCommand(t *testing.T) { - if version != "1.1.0" { + if version != "1.2.0" { t.Fatalf("unexpected release version %q", version) } } @@ -443,6 +443,53 @@ func TestCachePurgeFansOutToEveryAuthenticatedCell(t *testing.T) { } } +func TestWriteCellRuntimesKeepsStableSlotsAndEmptyUnassignedState(t *testing.T) { + dir := t.TempDir() + c := &client{runtimeDir: dir, cellAssignments: map[string]string{"cell-01": "shared-default", "cell-02": ""}} + pools := map[string]map[string]any{ + "shared-default": {"schema_version": 1, "sequence": uint64(9), "hosts": map[string]any{"www.example.test": map[string]any{}}}, + } + if err := c.writeCellRuntimes(9, pools); err != nil { + t.Fatal(err) + } + for _, name := range []string{"cell-01", "cell-02"} { + if _, err := os.Stat(filepath.Join(dir, name+".json")); err != nil { + t.Fatalf("missing stable runtime for %s: %v", name, err) + } + } + var empty map[string]any + body, err := os.ReadFile(filepath.Join(dir, "cell-02.json")) + if err != nil || json.Unmarshal(body, &empty) != nil { + t.Fatalf("invalid unassigned runtime: %v", err) + } + if empty["sequence"] != float64(9) || len(empty["hosts"].(map[string]any)) != 0 { + t.Fatalf("unexpected unassigned runtime: %#v", empty) + } +} + +func TestValidCellNameIsBounded(t *testing.T) { + for _, name := range []string{"cell-01", "cell-08", "cell-32"} { + if !validCellName(name) { + t.Fatalf("expected %s to be valid", name) + } + } + for _, name := range []string{"shared-default", "cell-00", "cell-1", "cell-33", "cell-01-extra"} { + if validCellName(name) { + t.Fatalf("expected %s to be invalid", name) + } + } +} + +func TestUnassignedSlotsReceiveAdditionalRuntimePoolsDeterministically(t *testing.T) { + c := &client{cellAssignments: map[string]string{"cell-01": "shared-default", "cell-02": "", "cell-03": ""}} + resolved := c.resolvedCellAssignments(map[string]map[string]any{ + "shared-default": {}, "reserved-z": {}, "dedicated-a": {}, + }) + if resolved["cell-01"] != "shared-default" || resolved["cell-02"] != "dedicated-a" || resolved["cell-03"] != "reserved-z" { + t.Fatalf("unexpected deterministic assignments: %#v", resolved) + } +} + func signedGzipJSON(t *testing.T, private ed25519.PrivateKey, value any) (string, string, string) { t.Helper() payload, err := json.Marshal(value) diff --git a/tests/e2e/cell_inventory.py b/tests/e2e/cell_inventory.py new file mode 100644 index 0000000..a246c70 --- /dev/null +++ b/tests/e2e/cell_inventory.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Real-runtime qualification for the bounded eight-slot edge inventory.""" + +from __future__ import annotations + +import json +import pathlib +import shutil +import subprocess +import tempfile +import time +import uuid + + +ROOT = pathlib.Path(__file__).resolve().parents[2] +IMAGE = "cdnfoundry/edge-runtime:cell-inventory" +TOKEN = "cell-inventory-qualification-only" + + +def run(*args: str, check: bool = True) -> subprocess.CompletedProcess[str]: + completed = subprocess.run(args, cwd=ROOT, check=False, text=True, capture_output=True) + if check and completed.returncode != 0: + raise RuntimeError(f"command failed ({completed.returncode}): {' '.join(args)}\n{completed.stdout}\n{completed.stderr}") + return completed + + +def inspect(name: str) -> dict: + return json.loads(run("docker", "inspect", name).stdout)[0] + + +def main() -> None: + if shutil.which("docker") is None or shutil.which("openssl") is None: + raise SystemExit("docker and openssl are required") + + suffix = uuid.uuid4().hex[:10] + network = f"cdnf-cell-inventory-{suffix}" + names = [f"cdnf-{suffix}-cell-{slot:02d}" for slot in range(1, 9)] + vector = f"cdnf-{suffix}-vector" + started: list[str] = [] + report: dict[str, object] = {"slots": 8, "network": network} + + with tempfile.TemporaryDirectory(prefix="cdnf-cell-inventory-") as directory: + work = pathlib.Path(directory) + try: + run("docker", "build", "--progress=plain", "--load", "-t", IMAGE, "-f", "docker/openresty/Dockerfile", ".") + run("openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", "-days", "1", "-subj", "/CN=cell-inventory.test", "-keyout", str(work / "tls.key"), "-out", str(work / "tls.crt")) + run("docker", "network", "create", network) + run("docker", "run", "-d", "--name", vector, "--network", network, "--network-alias", "vector", "alpine:3.22", "sleep", "600") + started.append(vector) + + for slot, name in enumerate(names, 1): + runtime = {"schema_version": 1, "sequence": 1, "hosts": {}, "certificates": {}} + runtime_path = work / f"cell-{slot:02d}.json" + runtime_path.write_text(json.dumps(runtime), encoding="utf-8") + run( + "docker", "run", "-d", "--name", name, "--network", network, + "--memory", "512m", "--cpus", "0.5", "--pids-limit", "128", "--read-only", + "--tmpfs", "/var/cache/nginx:rw,noexec,nosuid,size=256m", + "--tmpfs", "/var/lib/nginx/tmp:rw,noexec,nosuid,size=64m", + "--tmpfs", "/usr/local/openresty/nginx/logs:rw,noexec,nosuid,size=16m", + "-e", f"EDGE_CELL_NAME=cell-{slot:02d}", + "-e", f"EDGE_RUNTIME_FILE=/var/lib/cdnfoundry/runtime/cell-{slot:02d}.json", + "-e", f"EDGE_STATUS_TOKEN={TOKEN}", + "-v", f"{runtime_path}:/var/lib/cdnfoundry/runtime/cell-{slot:02d}.json:ro", + "-v", f"{work / 'tls.crt'}:/run/edge/tls.crt:ro", + "-v", f"{work / 'tls.key'}:/run/edge/tls.key:ro", + IMAGE, + ) + started.append(name) + + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + healthy = sum(run("docker", "exec", name, "wget", "-qO-", "http://127.0.0.1:8080/healthz", check=False).returncode == 0 for name in names) + if healthy == 8: + break + time.sleep(1) + if healthy != 8: + diagnostics = run("docker", "logs", names[0], check=False).stdout + run("docker", "logs", names[0], check=False).stderr + raise RuntimeError(f"only {healthy}/8 slots became healthy; first-slot logs:\n{diagnostics}") + + identities = [] + for slot, name in enumerate(names, 1): + metadata = inspect(name) + host = metadata["HostConfig"] + if host["Memory"] != 536870912 or host["NanoCpus"] != 500000000 or host["PidsLimit"] != 128 or not host["ReadonlyRootfs"]: + raise RuntimeError(f"resource limit drift for {name}") + status = json.loads(run("docker", "exec", name, "wget", "-qO-", "--header", f"X-Edge-Status-Token: {TOKEN}", "http://127.0.0.1:9080/passive-failures").stdout) + identities.append(status["cell"]["name"]) + expected = [f"cell-{slot:02d}" for slot in range(1, 9)] + if identities != expected: + raise RuntimeError(f"slot identities differ: {identities}") + + idle_stats = run("docker", "stats", "--no-stream", "--format", "{{.Name}}|{{.CPUPerc}}|{{.MemUsage}}", *names).stdout.strip().splitlines() + started_at = time.monotonic() + for _ in range(20): + for name in names: + run("docker", "exec", name, "wget", "-qO-", "http://127.0.0.1:8080/healthz") + active_seconds = time.monotonic() - started_at + active_stats = run("docker", "stats", "--no-stream", "--format", "{{.Name}}|{{.CPUPerc}}|{{.MemUsage}}", *names).stdout.strip().splitlines() + + run("docker", "stop", "-t", "1", names[3]) + if run("docker", "exec", names[4], "wget", "-qO-", "http://127.0.0.1:8080/healthz", check=False).returncode != 0: + raise RuntimeError("stopping cell-04 affected cell-05") + if inspect(vector)["State"]["Running"] is not True: + raise RuntimeError("cell failure affected the separate support process") + run("docker", "start", names[3]) + + report.update({ + "identities": identities, + "idle_stats": idle_stats, + "active_stats": active_stats, + "health_requests": 160, + "active_seconds": round(active_seconds, 3), + "isolation": "cell-04 stop left cell-05 and support process ready", + "resource_limits": {"memory_bytes": 536870912, "cpu": 0.5, "pids": 128, "cache_bytes": 268435456, "temporary_bytes": 67108864, "log_bytes": 16777216}, + }) + print(json.dumps(report, indent=2)) + print("cell_inventory=passed") + finally: + for name in reversed(started): + run("docker", "rm", "-f", name, check=False) + run("docker", "network", "rm", network, check=False) + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/phase4_control_plane.py b/tests/e2e/phase4_control_plane.py index d306528..dfdd3e2 100644 --- a/tests/e2e/phase4_control_plane.py +++ b/tests/e2e/phase4_control_plane.py @@ -311,13 +311,13 @@ def provision_edge(token: str, name: str, country: str, continent: str, ipv4: st def heartbeat(edge: dict, active_sequence: int, quarantine_ready: bool = False) -> None: cells = [{ - "name": "shared-default", + "name": "cell-01", "status": "ready", "capacity": {"assigned_domain_count": 0, "active_connections": 0, "requests_per_second": 0}, }] if quarantine_ready: cells.append({ - "name": "quarantine-default", + "name": "cell-02", "status": "ready", "capacity": {"assigned_domain_count": 0, "active_connections": 0, "requests_per_second": 0}, })