Agent/phase 2 bounded cell inventory

compose.dev.yml

@@ -332,8 +332,8 @@ services:
       context: .
       dockerfile: docker/openresty/Dockerfile
     environment:
-      EDGE_CELL_NAME: shared-default
-      EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/shared-default.json
+      EDGE_CELL_NAME: cell-01
+      EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/cell-01.json
       EDGE_STATUS_TOKEN: ${CDNF_DEV_EDGE_STATUS_TOKEN:-cdnf-dev-edge-status-only}
       GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb
     volumes:
@@ -365,8 +365,8 @@ services:
       context: .
       dockerfile: docker/openresty/Dockerfile
     environment:
-      EDGE_CELL_NAME: quarantine-default
-      EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/quarantine-default.json
+      EDGE_CELL_NAME: cell-02
+      EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/cell-02.json
       EDGE_STATUS_TOKEN: ${CDNF_DEV_EDGE_STATUS_TOKEN:-cdnf-dev-edge-status-only}
       GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb
     volumes:
@@ -403,6 +403,7 @@ services:
         {"address":"fd00:cd0f:10::10","pool":"shared-default","http":"edge-a:8081","https":"edge-a:8444"},
         {"address":"172.28.11.10","pool":"quarantine-default","http":"edge-a-quarantine:8081","https":"edge-a-quarantine:8444"}]
       EDGE_GATEWAY_STATUS_URL: http://edge-gateway-a:9105/metrics
+      EDGE_CELL_ASSIGNMENTS: '{"cell-01":"shared-default","cell-02":"quarantine-default"}'
       EDGE_CELL_STATUS_URLS: http://edge-a:9080/passive-failures,http://edge-a-quarantine:9080/passive-failures
       EDGE_STATUS_TOKEN: ${CDNF_DEV_EDGE_STATUS_TOKEN:-cdnf-dev-edge-status-only}
     volumes:
@@ -467,8 +468,8 @@ services:
       context: .
       dockerfile: docker/openresty/Dockerfile
     environment:
-      EDGE_CELL_NAME: shared-default
-      EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/shared-default.json
+      EDGE_CELL_NAME: cell-01
+      EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/cell-01.json
       EDGE_STATUS_TOKEN: ${CDNF_DEV_EDGE_STATUS_TOKEN:-cdnf-dev-edge-status-only}
       GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb
     volumes:
@@ -500,8 +501,8 @@ services:
       context: .
       dockerfile: docker/openresty/Dockerfile
     environment:
-      EDGE_CELL_NAME: quarantine-default
-      EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/quarantine-default.json
+      EDGE_CELL_NAME: cell-02
+      EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/cell-02.json
       EDGE_STATUS_TOKEN: ${CDNF_DEV_EDGE_STATUS_TOKEN:-cdnf-dev-edge-status-only}
       GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb
     volumes:
@@ -537,6 +538,7 @@ services:
         [{"address":"172.28.20.10","pool":"shared-default","http":"edge-b:8081","https":"edge-b:8444"},
         {"address":"172.28.21.10","pool":"quarantine-default","http":"edge-b-quarantine:8081","https":"edge-b-quarantine:8444"}]
       EDGE_GATEWAY_STATUS_URL: http://edge-gateway-b:9105/metrics
+      EDGE_CELL_ASSIGNMENTS: '{"cell-01":"shared-default","cell-02":"quarantine-default"}'
       EDGE_CELL_STATUS_URLS: http://edge-b:9080/passive-failures,http://edge-b-quarantine:9080/passive-failures
       EDGE_STATUS_TOKEN: ${CDNF_DEV_EDGE_STATUS_TOKEN:-cdnf-dev-edge-status-only}
     volumes:

compose.prod.yml

@@ -18,6 +18,7 @@ x-core-env: &core-env
   CACHE_STORE: redis
   SESSION_DRIVER: redis
   QUEUE_CONNECTION: redis
+
   REDIS_URL: ${REDIS_URL:-}
   REDIS_HOST: ${REDIS_HOST:-redis}
   REDIS_PORT: ${REDIS_PORT:-6379}
@@ -44,6 +45,35 @@ x-core-env: &core-env
   BACKUP_SECRET_ACCESS_KEY: ${BACKUP_SECRET_ACCESS_KEY:?BACKUP_SECRET_ACCESS_KEY is required}
   BACKUP_DEFAULT_REGION: ${BACKUP_DEFAULT_REGION:-us-east-1}
 
+x-edge-cell: &edge-cell
+  image: ghcr.io/vaheed/cdnfoundry-edge-runtime:${CDNF_RELEASE:?CDNF_RELEASE must be a published commit SHA or release tag}
+  profiles: [edge]
+  volumes:
+    - ${EDGE_RUNTIME_TLS_CERTIFICATE:?EDGE_RUNTIME_TLS_CERTIFICATE is required}:/run/edge/tls.crt:ro
+    - ${EDGE_RUNTIME_TLS_PRIVATE_KEY:?EDGE_RUNTIME_TLS_PRIVATE_KEY is required}:/run/edge/tls.key:ro
+    - edge-state:/var/lib/cdnfoundry/runtime:ro
+    - mmdb:/mmdb:ro
+  depends_on:
+    mmdb-updater: { condition: service_healthy }
+  read_only: true
+  tmpfs:
+    - /var/cache/nginx:rw,noexec,nosuid,size=256m
+    - /var/lib/nginx/tmp:rw,noexec,nosuid,size=64m
+    - /usr/local/openresty/nginx/logs:rw,noexec,nosuid,size=16m
+  networks: [edge, telemetry]
+  restart: unless-stopped
+  stop_signal: SIGQUIT
+  stop_grace_period: 30s
+  mem_limit: 512m
+  cpus: 0.5
+  pids_limit: 128
+  sysctls: { net.ipv4.tcp_syncookies: "1" }
+  ulimits:
+    nofile: { soft: 32768, hard: 32768 }
+  deploy:
+    resources:
+      limits: { memory: 512m, cpus: "0.5", pids: 128 }
+
 x-core: &core
   image: ghcr.io/vaheed/cdnfoundry-core:${CDNF_RELEASE:?CDNF_RELEASE must be a published commit SHA or release tag}
   environment: *core-env
@@ -287,85 +317,51 @@ services:
     networks: [telemetry]
     restart: unless-stopped
 
-  edge:
-    image: ghcr.io/vaheed/cdnfoundry-edge-runtime:${CDNF_RELEASE:?CDNF_RELEASE must be a published commit SHA or release tag}
-    profiles: [edge]
-    ports:
-      - "127.0.0.1:18081:8081"
-      - "127.0.0.1:18444:8444"
-    environment:
-      EDGE_CELL_NAME: shared-default
-      EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/shared-default.json
-      EDGE_STATUS_TOKEN: ${EDGE_STATUS_TOKEN:?EDGE_STATUS_TOKEN is required}
-      GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb
-    volumes:
-      - ${EDGE_RUNTIME_TLS_CERTIFICATE:?EDGE_RUNTIME_TLS_CERTIFICATE is required}:/run/edge/tls.crt:ro
-      - ${EDGE_RUNTIME_TLS_PRIVATE_KEY:?EDGE_RUNTIME_TLS_PRIVATE_KEY is required}:/run/edge/tls.key:ro
-      - edge-state:/var/lib/cdnfoundry/runtime:ro
-      - mmdb:/mmdb:ro
-    depends_on:
-      mmdb-updater: { condition: service_healthy }
-    read_only: true
-    tmpfs:
-      - /var/cache/nginx:rw,noexec,nosuid,size=512m
-      - /var/lib/nginx/tmp:rw,noexec,nosuid,size=128m
-      - /usr/local/openresty/nginx/logs:rw,noexec,nosuid,size=16m
-    networks: [edge, telemetry]
-    restart: unless-stopped
-    stop_signal: SIGQUIT
-    stop_grace_period: 30s
-    mem_limit: 2g
-    cpus: 2
-    pids_limit: 256
-    sysctls: { net.ipv4.tcp_syncookies: "1" }
-    ulimits:
-      nofile: { soft: 65536, hard: 65536 }
-    deploy:
-      resources:
-        limits:
-          memory: 2g
-          cpus: "2"
-          pids: 256
+  cell-01: &cell-01
+    <<: *edge-cell
+    ports: ["127.0.0.1:18081:8081", "127.0.0.1:18444:8444", "127.0.0.1:19081:9080"]
+    environment: { EDGE_CELL_NAME: cell-01, EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/cell-01.json, EDGE_STATUS_TOKEN: "${EDGE_STATUS_TOKEN:?EDGE_STATUS_TOKEN is required}", GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb }
+    networks:
+      edge: { aliases: [edge] }
+      telemetry: {}
 
-  edge-quarantine:
-    image: ghcr.io/vaheed/cdnfoundry-edge-runtime:${CDNF_RELEASE:?CDNF_RELEASE must be a published commit SHA or release tag}
-    profiles: [edge]
-    ports:
-      - "127.0.0.1:28081:8081"
-      - "127.0.0.1:28444:8444"
-    environment:
-      EDGE_CELL_NAME: quarantine-default
-      EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/quarantine-default.json
-      EDGE_STATUS_TOKEN: ${EDGE_STATUS_TOKEN:?EDGE_STATUS_TOKEN is required}
-      GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb
-    volumes:
-      - ${EDGE_RUNTIME_TLS_CERTIFICATE:?EDGE_RUNTIME_TLS_CERTIFICATE is required}:/run/edge/tls.crt:ro
-      - ${EDGE_RUNTIME_TLS_PRIVATE_KEY:?EDGE_RUNTIME_TLS_PRIVATE_KEY is required}:/run/edge/tls.key:ro
-      - edge-state:/var/lib/cdnfoundry/runtime:ro
-      - mmdb:/mmdb:ro
-    depends_on:
-      mmdb-updater: { condition: service_healthy }
-    read_only: true
-    tmpfs:
-      - /var/cache/nginx:rw,noexec,nosuid,size=256m
-      - /var/lib/nginx/tmp:rw,noexec,nosuid,size=64m
-      - /usr/local/openresty/nginx/logs:rw,noexec,nosuid,size=16m
-    networks: [edge, telemetry]
-    restart: unless-stopped
-    stop_signal: SIGQUIT
-    stop_grace_period: 30s
-    mem_limit: 512m
-    cpus: 0.5
-    pids_limit: 128
-    sysctls: { net.ipv4.tcp_syncookies: "1" }
-    ulimits:
-      nofile: { soft: 32768, hard: 32768 }
-    deploy:
-      resources:
-        limits:
-          memory: 512m
-          cpus: "0.5"
-          pids: 128
+  cell-02:
+    <<: *edge-cell
+    ports: ["127.0.0.1:18082:8081", "127.0.0.1:18445:8444", "127.0.0.1:19082:9080"]
+    environment: { EDGE_CELL_NAME: cell-02, EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/cell-02.json, EDGE_STATUS_TOKEN: "${EDGE_STATUS_TOKEN:?EDGE_STATUS_TOKEN is required}", GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb }
+    networks:
+      edge: { aliases: [edge-quarantine] }
+      telemetry: {}
+
+  cell-03:
+    <<: *edge-cell
+    ports: ["127.0.0.1:18083:8081", "127.0.0.1:18446:8444", "127.0.0.1:19083:9080"]
+    environment: { EDGE_CELL_NAME: cell-03, EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/cell-03.json, EDGE_STATUS_TOKEN: "${EDGE_STATUS_TOKEN:?EDGE_STATUS_TOKEN is required}", GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb }
+
+  cell-04:
+    <<: *edge-cell
+    ports: ["127.0.0.1:18084:8081", "127.0.0.1:18447:8444", "127.0.0.1:19084:9080"]
+    environment: { EDGE_CELL_NAME: cell-04, EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/cell-04.json, EDGE_STATUS_TOKEN: "${EDGE_STATUS_TOKEN:?EDGE_STATUS_TOKEN is required}", GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb }
+
+  cell-05:
+    <<: *edge-cell
+    ports: ["127.0.0.1:18085:8081", "127.0.0.1:18448:8444", "127.0.0.1:19085:9080"]
+    environment: { EDGE_CELL_NAME: cell-05, EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/cell-05.json, EDGE_STATUS_TOKEN: "${EDGE_STATUS_TOKEN:?EDGE_STATUS_TOKEN is required}", GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb }
+
+  cell-06:
+    <<: *edge-cell
+    ports: ["127.0.0.1:18086:8081", "127.0.0.1:18449:8444", "127.0.0.1:19086:9080"]
+    environment: { EDGE_CELL_NAME: cell-06, EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/cell-06.json, EDGE_STATUS_TOKEN: "${EDGE_STATUS_TOKEN:?EDGE_STATUS_TOKEN is required}", GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb }
+
+  cell-07:
+    <<: *edge-cell
+    ports: ["127.0.0.1:18087:8081", "127.0.0.1:18450:8444", "127.0.0.1:19087:9080"]
+    environment: { EDGE_CELL_NAME: cell-07, EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/cell-07.json, EDGE_STATUS_TOKEN: "${EDGE_STATUS_TOKEN:?EDGE_STATUS_TOKEN is required}", GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb }
+
+  cell-08:
+    <<: *edge-cell
+    ports: ["127.0.0.1:18088:8081", "127.0.0.1:18451:8444", "127.0.0.1:19088:9080"]
+    environment: { EDGE_CELL_NAME: cell-08, EDGE_RUNTIME_FILE: /var/lib/cdnfoundry/runtime/cell-08.json, EDGE_STATUS_TOKEN: "${EDGE_STATUS_TOKEN:?EDGE_STATUS_TOKEN is required}", GEOIP_DATABASE: /mmdb/GeoLite2-City.mmdb }
 
   edge-agent:
     image: ghcr.io/vaheed/cdnfoundry-edge-agent:${CDNF_RELEASE:?CDNF_RELEASE must be a published commit SHA or release tag}
@@ -379,7 +375,8 @@ services:
       EDGE_RUNTIME_DIR: /var/lib/cdnfoundry/runtime
       EDGE_GATEWAY_BINDINGS: ${EDGE_GATEWAY_BINDINGS:?EDGE_GATEWAY_BINDINGS is required}
       EDGE_GATEWAY_STATUS_URL: ${EDGE_GATEWAY_STATUS_URL:-http://host-gateway:9105/metrics}
-      EDGE_CELL_STATUS_URLS: http://edge:9080/passive-failures,http://edge-quarantine:9080/passive-failures
+      EDGE_CELL_ASSIGNMENTS: '{"cell-01":"shared-default","cell-02":"quarantine-default","cell-03":"","cell-04":"","cell-05":"","cell-06":"","cell-07":"","cell-08":""}'
+      EDGE_CELL_STATUS_URLS: http://cell-01:9080/passive-failures,http://cell-02:9080/passive-failures,http://cell-03:9080/passive-failures,http://cell-04:9080/passive-failures,http://cell-05:9080/passive-failures,http://cell-06:9080/passive-failures,http://cell-07:9080/passive-failures,http://cell-08:9080/passive-failures
       EDGE_STATUS_TOKEN: ${EDGE_STATUS_TOKEN:?EDGE_STATUS_TOKEN is required}
     volumes:
       - edge-agent-state:/var/lib/cdnfoundry/agent

core/app/Filament/Admin/Resources/EdgePools/EdgePoolResource.php

@@ -6,11 +6,13 @@
 use App\Filament\Admin\Resources\EdgePools\Pages\CreateEdgePool;
 use App\Filament\Admin\Resources\EdgePools\Pages\EditEdgePool;
 use App\Filament\Admin\Resources\EdgePools\Pages\ListEdgePools;
+use App\Jobs\ProvisionEdgePoolCells;
 use App\Jobs\ReconcilePlatformDnsIdentity;
 use App\Models\AuditLog;
 use App\Models\DomainEdgePlacement;
 use App\Models\EdgePool;
 use App\Models\EmergencyMode;
+use App\Models\Operation;
 use App\Models\PlatformDnsSetting;
 use App\Support\EdgeRoutingCompiler;
 use Filament\Actions\Action;
@@ -60,6 +62,18 @@ public static function table(Table $table): Table
             TextColumn::make('cells_count')->counts('cells')->label('Edge cells'),
             TextColumn::make('updated_at')->since()->sortable(),
         ])->recordActions([
+            Action::make('reconcileCells')->label('Reconcile cells')->icon('heroicon-o-arrow-path')
+                ->requiresConfirmation()
+                ->action(function (EdgePool $record): void {
+                    $operation = Operation::query()->create([
+                        'actor_id' => auth()->id(), 'type' => 'edge.pool_provision', 'status' => 'pending',
+                        'input' => ['pool_id' => $record->id],
+                    ]);
+                    AuditLog::record(auth()->user(), 'edge.pool_provision_requested', $record, ['operation_id' => $operation->id], request()->ip());
+                    ProvisionEdgePoolCells::dispatch($record->id, $operation->id);
+                    Notification::make()->info()->title('Cell reconciliation queued')
+                        ->body("Operation {$operation->id} will assign one existing unassigned slot on each missing edge.")->send();
+                }),
             Action::make('enable')->visible(fn (EdgePool $record): bool => ! $record->enabled)->action(function (EdgePool $record): void {
                 $incomplete = $record->cells()->whereHas('edge', fn ($query) => $query->where('enabled', true))->whereNull('service_ipv4')->exists();
                 if ($incomplete) {

core/app/Filament/Admin/Resources/Edges/EdgeResource.php

@@ -41,6 +41,8 @@ public static function form(Schema $schema): Schema
     {
         return $schema->components([
             TextInput::make('name')->required()->maxLength(100)->unique(ignoreRecord: true),
+            TextInput::make('cell_slot_count')->label('Cell slots')->numeric()->integer()->required()->minValue(1)->maxValue(32)->default(8)->disabledOn('edit')
+                ->helperText('Bounded OpenResty slots created during edge installation. This cannot be changed after creation.'),
             Select::make('country_code')->label('Country')->options(array_combine(GeoVocabulary::countries(), GeoVocabulary::countries()))->searchable()->required(),
             Select::make('continent_code')->label('Continent')->options(array_combine(GeoVocabulary::CONTINENTS, GeoVocabulary::CONTINENTS))->required(),
             TextInput::make('ipv4')->label('IPv4')->ipv4()->required()->unique(ignoreRecord: true)

core/app/Filament/Admin/Resources/Edges/Pages/CreateEdge.php

@@ -26,12 +26,19 @@ protected function handleRecordCreation(array $data): Model
         $data['bootstrap_token_hash'] = hash('sha256', $this->bootstrapToken);
         $edge = DB::transaction(function () use ($data): Edge {
             $edge = Edge::query()->create($data);
-            $defaultSharedId = EdgePool::query()->where('enabled', true)->where('kind', 'shared')->orderBy('id')->value('id');
-            foreach (EdgePool::query()->orderBy('id')->limit(32)->get() as $pool) {
+            $pools = EdgePool::query()->where('enabled', true)->orderByRaw("CASE WHEN kind = 'shared' THEN 0 WHEN kind = 'quarantine' THEN 1 ELSE 2 END")->orderBy('id')->limit($edge->cell_slot_count)->get()->values();
+            for ($slot = 1; $slot <= $edge->cell_slot_count; $slot++) {
+                $pool = $pools->get($slot - 1);
+                $name = sprintf('cell-%02d', $slot);
                 $edge->cells()->create([
-                    'edge_pool_id' => $pool->id, 'name' => $pool->name,
-                    'service_ipv4' => $pool->id === $defaultSharedId ? $edge->ipv4 : null,
-                    'service_ipv6' => $pool->id === $defaultSharedId ? $edge->ipv6 : null,
+                    'slot' => $slot, 'edge_pool_id' => $pool?->id, 'name' => $name,
+                    'http_port' => 18080 + $slot, 'https_port' => 18443 + $slot, 'status_port' => 19080 + $slot,
+                    'runtime_path' => "/var/lib/cdnfoundry/runtime/{$name}.json",
+                    'cache_path' => "/var/cache/cdnfoundry/{$name}", 'temporary_path' => "/var/lib/cdnfoundry/tmp/{$name}",
+                    'resource_limits' => ['memory_bytes' => 536870912, 'cpu_millis' => 500, 'pids' => 128, 'cache_bytes' => 268435456, 'temporary_bytes' => 67108864, 'log_bytes' => 16777216],
+                    'status' => $pool === null ? 'unassigned' : 'assigned',
+                    'service_ipv4' => $pool?->kind === 'shared' ? $edge->ipv4 : null,
+                    'service_ipv6' => $pool?->kind === 'shared' ? $edge->ipv6 : null,
                 ]);
             }
             AuditLog::record(auth()->user(), 'edge.created', $edge, [], request()->ip());