diff --git a/.github/workflows/helm-lint.yaml b/.github/workflows/helm-lint.yaml new file mode 100644 index 00000000..89cb2ee5 --- /dev/null +++ b/.github/workflows/helm-lint.yaml @@ -0,0 +1,28 @@ +name: Helm Lint + +on: + push: + branches: [main] + paths: + - "chart/**" + - ".github/workflows/helm-lint.yaml" + pull_request: + paths: + - "chart/**" + - ".github/workflows/helm-lint.yaml" + +permissions: + contents: read + +jobs: + lint-helm: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - uses: azure/setup-helm@f0accbfd55e3332a28f721b8202b1016cecf90d5 # v5 + + - name: Lint Helm chart + run: helm lint chart/pelican-wings diff --git a/chart/pelican-wings/Chart.yaml b/chart/pelican-wings/Chart.yaml new file mode 100644 index 00000000..7962d6bd --- /dev/null +++ b/chart/pelican-wings/Chart.yaml @@ -0,0 +1,16 @@ +apiVersion: v2 +name: pelican-wings +description: Helm chart for Pelican Wings — the game server management daemon with Kubernetes support +type: application +version: 0.1.0 +appVersion: "1.0.0" +keywords: + - pelican + - wings + - game-server + - kubernetes +maintainers: + - name: Exonical + url: https://github.com/Exonical +sources: + - https://github.com/pelican/wings diff --git a/chart/pelican-wings/README.md b/chart/pelican-wings/README.md new file mode 100644 index 00000000..f596f577 --- /dev/null +++ b/chart/pelican-wings/README.md @@ -0,0 +1,146 @@ +# Pelican Wings Helm Chart + +Deploys [Pelican Wings](https://github.com/pelican/wings) — the game server +management daemon — into a Kubernetes cluster with full RBAC, storage, and +networking support. + +## Prerequisites + +- Kubernetes 1.34+ +- Helm 3.x +- A running [Pelican Panel](https://github.com/pelican/panel) instance + +## Quick Start + +Put your node credentials from the Panel in a local values file (kept out of +version control) rather than on the command line, where `--set` would leak them +into shell history and process listings: + +```yaml +# values.local.yaml (do not commit) +wings: + panelUrl: https://panel.example.com + token: YOUR_TOKEN + tokenId: YOUR_TOKEN_ID + uuid: YOUR_NODE_UUID +``` + +```bash +helm install wings ./chart/pelican-wings -f values.local.yaml +``` + +## Configuration + +See [values.yaml](values.yaml) for the full list of configurable values. + +### Key Values + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `wings.panelUrl` | Panel URL | `https://panel.example.com` | +| `wings.token` | Panel authentication token | `""` | +| `wings.tokenId` | Panel token ID | `""` | +| `wings.uuid` | Node UUID from Panel | `""` | +| `wings.kubernetes.networkMode` | Port exposure: `hostport`, `nodeport`, or `loadbalancer` | `nodeport` | +| `wings.kubernetes.storageMode` | Storage: `hostpath` or `pvc` | `pvc` | +| `wings.kubernetes.storageClass` | StorageClass for PVCs | `""` (cluster default) | +| `wings.kubernetes.storageSize` | Default PVC size | `10Gi` | +| `wings.kubernetes.imagePullPolicy` | Pull policy for game server Pods/install Jobs: `Always`, `IfNotPresent`, `Never` | `""` (smart default) | +| `gameNamespace` | Namespace for game server resources | `pelican` | +| `rbac.create` | Create RBAC resources | `true` | +| `rbac.kubeletMetricsFallback` | Grant cluster-wide `nodes/proxy` for the kubelet stats fallback (broad permission; prefer metrics-server) | `false` | +| `serviceAccount.create` | Create ServiceAccount | `true` | +| `serviceAccount.name` | ServiceAccount name (**required** when `serviceAccount.create=false`) | `""` | + +> **Namespace:** Wings schedules game-server workloads into `gameNamespace`, and +> the chart creates the namespaced RBAC there. `wings.kubernetes.namespace` is +> therefore derived from `gameNamespace`; if you set it explicitly it must match +> `gameNamespace` or the chart will fail to render. + +> **Credentials:** `wings.token`, `wings.tokenId`, and `wings.uuid` are rendered +> into a Kubernetes **Secret** (not a ConfigMap). Supply them via a private +> values file or `--set`, e.g. `helm install ... -f my-creds.yaml`, and keep that +> file out of version control. + +### Storage + +By default, the chart uses PVC-based storage (`storageMode: pvc`). This creates +a PersistentVolumeClaim per game server, enabling proper data lifecycle +management. + +For single-node setups or testing, you can use HostPath: + +```yaml +wings: + kubernetes: + storageMode: hostpath +``` + +### Networking + +NodePort mode (default) creates a Kubernetes Service per game server, exposing +ports via cluster-assigned NodePorts: + +```yaml +wings: + kubernetes: + networkMode: nodeport + nodeportPreserve: true # Try to use game port as NodePort +``` + +HostPort mode binds game server ports directly to the node: + +```yaml +wings: + kubernetes: + networkMode: hostport +``` + +LoadBalancer mode provisions a `Service` of type `LoadBalancer` per game server +(for use with MetalLB, Cilium LB-IPAM, or a cloud LB). LB IP/sharing-key +annotations can be auto-populated from the allocation IP: + +```yaml +wings: + kubernetes: + networkMode: loadbalancer +``` + +### Image pulling + +Game server Pods and installation Jobs default to `imagePullPolicy: Always` +for remote images, so updated tags are re-pulled rather than reusing a stale +copy cached on the node (matching the Docker backend). `~`-prefixed local +images are never pulled. Override this for air-gapped clusters: + +```yaml +wings: + kubernetes: + imagePullPolicy: IfNotPresent # or "Never" +``` + +This is independent of `image.pullPolicy`, which applies to the Wings daemon +image itself. + +## What Gets Created + +- **Namespace** — `pelican` (configurable) +- **ServiceAccount** — For Wings and game server Pods +- **Role + RoleBinding** — Namespace-scoped permissions (Pods, Services, Jobs, PVCs) +- **ClusterRole + ClusterRoleBinding** — Metrics API access (`nodes/proxy` only when `rbac.kubeletMetricsFallback=true`) +- **Secret** — Wings configuration file (contains Panel token) +- **Deployment** — Wings daemon with health probes +- **Service** — Exposes Wings API within the cluster + +## Uninstalling + +```bash +helm uninstall wings +``` + +Note: PVCs created for game servers are NOT automatically deleted when +uninstalling the chart. Delete them manually if you want to remove all data: + +```bash +kubectl delete pvc -n pelican -l app.kubernetes.io/managed-by=pelican-wings +``` diff --git a/chart/pelican-wings/templates/NOTES.txt b/chart/pelican-wings/templates/NOTES.txt new file mode 100644 index 00000000..e88b9444 --- /dev/null +++ b/chart/pelican-wings/templates/NOTES.txt @@ -0,0 +1,22 @@ +Pelican Wings has been deployed! + +1. Get the Wings API URL: +{{- if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Values.gameNamespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "pelican-wings.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Values.gameNamespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo "Wings API: http://$NODE_IP:$NODE_PORT" +{{- else if contains "ClusterIP" .Values.service.type }} + kubectl port-forward --namespace {{ .Values.gameNamespace }} svc/{{ include "pelican-wings.fullname" . }} {{ .Values.service.port }}:{{ .Values.service.port }} + echo "Wings API: http://127.0.0.1:{{ .Values.service.port }}" +{{- end }} + +2. Configure your Panel to connect to this Wings node using the URL above. + +3. Verify the deployment: + kubectl get pods --namespace {{ .Values.gameNamespace }} -l "{{ include "pelican-wings.selectorLabels" . | replace "\n" "," }}" + +Configuration: + - Game namespace: {{ .Values.gameNamespace }} + - Network mode: {{ .Values.wings.kubernetes.networkMode }} + - Storage mode: {{ .Values.wings.kubernetes.storageMode }} + - Panel URL: {{ .Values.wings.panelUrl }} diff --git a/chart/pelican-wings/templates/_helpers.tpl b/chart/pelican-wings/templates/_helpers.tpl new file mode 100644 index 00000000..dbde1679 --- /dev/null +++ b/chart/pelican-wings/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "pelican-wings.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "pelican-wings.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "pelican-wings.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels. +*/}} +{{- define "pelican-wings.labels" -}} +helm.sh/chart: {{ include "pelican-wings.chart" . }} +{{ include "pelican-wings.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels. +*/}} +{{- define "pelican-wings.selectorLabels" -}} +app.kubernetes.io/name: {{ include "pelican-wings.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use. +*/}} +{{- define "pelican-wings.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "pelican-wings.fullname" .) .Values.serviceAccount.name }} +{{- else if .Values.serviceAccount.name }} +{{- .Values.serviceAccount.name }} +{{- else }} +{{- fail "serviceAccount.name must be set when serviceAccount.create is false: binding RBAC to the namespace 'default' ServiceAccount would grant Wings' permissions to every workload using it." }} +{{- end }} +{{- end }} diff --git a/chart/pelican-wings/templates/clusterrole-metrics.yaml b/chart/pelican-wings/templates/clusterrole-metrics.yaml new file mode 100644 index 00000000..399f0bbb --- /dev/null +++ b/chart/pelican-wings/templates/clusterrole-metrics.yaml @@ -0,0 +1,23 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "pelican-wings.fullname" . }}-metrics + labels: + {{- include "pelican-wings.labels" . | nindent 4 }} +rules: + - apiGroups: ["metrics.k8s.io"] + resources: ["pods"] + verbs: ["get"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] + {{- if .Values.rbac.kubeletMetricsFallback }} + # Proxy to kubelet stats/summary API for resource metrics when + # metrics-server is not installed. Opt-in: this is a broad cluster-scoped + # permission (rbac.kubeletMetricsFallback). + - apiGroups: [""] + resources: ["nodes/proxy"] + verbs: ["get"] + {{- end }} +{{- end }} diff --git a/chart/pelican-wings/templates/clusterrolebinding-metrics.yaml b/chart/pelican-wings/templates/clusterrolebinding-metrics.yaml new file mode 100644 index 00000000..5679fd59 --- /dev/null +++ b/chart/pelican-wings/templates/clusterrolebinding-metrics.yaml @@ -0,0 +1,16 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "pelican-wings.fullname" . }}-metrics + labels: + {{- include "pelican-wings.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ include "pelican-wings.serviceAccountName" . }} + namespace: {{ .Values.gameNamespace }} +roleRef: + kind: ClusterRole + name: {{ include "pelican-wings.fullname" . }}-metrics + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/chart/pelican-wings/templates/configmap.yaml b/chart/pelican-wings/templates/configmap.yaml new file mode 100644 index 00000000..a332fae0 --- /dev/null +++ b/chart/pelican-wings/templates/configmap.yaml @@ -0,0 +1,80 @@ +{{- if and .Values.wings.kubernetes.namespace (ne .Values.wings.kubernetes.namespace .Values.gameNamespace) }} +{{- fail "wings.kubernetes.namespace must equal gameNamespace: the chart's namespaced RBAC is created in gameNamespace, so Wings must schedule game-server workloads there too. Leave wings.kubernetes.namespace empty or set it equal to gameNamespace." }} +{{- end }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "pelican-wings.fullname" . }}-config + namespace: {{ .Values.gameNamespace }} + labels: + {{- include "pelican-wings.labels" . | nindent 4 }} +type: Opaque +stringData: + config.yml: | + debug: false + app_name: pelican + uuid: {{ .Values.wings.uuid | quote }} + token_id: {{ .Values.wings.tokenId | quote }} + token: {{ .Values.wings.token | quote }} + remote: {{ .Values.wings.panelUrl | quote }} + api: + host: {{ .Values.wings.api.host | quote }} + port: {{ .Values.wings.api.port }} + sftp: + bind_addr: {{ .Values.wings.sftp.host | quote }} + bind_port: {{ .Values.wings.sftp.port }} + system: + root_directory: {{ .Values.wings.system.rootDirectory | quote }} + log_directory: {{ .Values.wings.system.logDirectory | quote }} + data: {{ .Values.wings.system.data | quote }} + tmp_directory: {{ .Values.wings.system.tmpDirectory | quote }} + kubernetes: + enabled: {{ .Values.wings.kubernetes.enabled }} + namespace: {{ .Values.gameNamespace | quote }} + network_mode: {{ .Values.wings.kubernetes.networkMode | quote }} + {{- if .Values.wings.kubernetes.lbAnnotations }} + lb_annotations: + {{- toYaml .Values.wings.kubernetes.lbAnnotations | nindent 8 }} + {{- end }} + {{- if .Values.wings.kubernetes.lbIPAnnotation }} + lb_ip_annotation: {{ .Values.wings.kubernetes.lbIPAnnotation | quote }} + {{- end }} + {{- if .Values.wings.kubernetes.lbSharingKey }} + lb_sharing_key: {{ .Values.wings.kubernetes.lbSharingKey | quote }} + {{- end }} + storage_mode: {{ .Values.wings.kubernetes.storageMode | quote }} + {{- if .Values.wings.kubernetes.storageClass }} + storage_class: {{ .Values.wings.kubernetes.storageClass | quote }} + {{- end }} + storage_size: {{ .Values.wings.kubernetes.storageSize | quote }} + storage_access_mode: {{ .Values.wings.kubernetes.storageAccessMode | quote }} + nodeport_preserve: {{ .Values.wings.kubernetes.nodeportPreserve }} + nodeport_range_min: {{ .Values.wings.kubernetes.nodeportRangeMin }} + nodeport_range_max: {{ .Values.wings.kubernetes.nodeportRangeMax }} + dns_policy: {{ .Values.wings.kubernetes.dnsPolicy | quote }} + {{- if .Values.wings.kubernetes.nodeName }} + node_name: {{ .Values.wings.kubernetes.nodeName | quote }} + {{- end }} + {{- if .Values.wings.kubernetes.systemIPs }} + system_ips: + {{- toYaml .Values.wings.kubernetes.systemIPs | nindent 8 }} + {{- end }} + {{- if .Values.persistence.enabled }} + data_pvc: {{ .Values.persistence.existingClaim | default (printf "%s-data" (include "pelican-wings.fullname" .)) | quote }} + {{- end }} + service_account: {{ include "pelican-wings.serviceAccountName" . | quote }} + {{- if .Values.wings.kubernetes.nodeSelector }} + node_selector: + {{- toYaml .Values.wings.kubernetes.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.wings.kubernetes.tolerations }} + tolerations: + {{- toYaml .Values.wings.kubernetes.tolerations | nindent 8 }} + {{- end }} + {{- if .Values.wings.kubernetes.imagePullSecrets }} + image_pull_secrets: + {{- toYaml .Values.wings.kubernetes.imagePullSecrets | nindent 8 }} + {{- end }} + {{- if .Values.wings.kubernetes.imagePullPolicy }} + image_pull_policy: {{ .Values.wings.kubernetes.imagePullPolicy | quote }} + {{- end }} diff --git a/chart/pelican-wings/templates/deployment.yaml b/chart/pelican-wings/templates/deployment.yaml new file mode 100644 index 00000000..f0bb4944 --- /dev/null +++ b/chart/pelican-wings/templates/deployment.yaml @@ -0,0 +1,121 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "pelican-wings.fullname" . }} + namespace: {{ .Values.gameNamespace }} + labels: + {{- include "pelican-wings.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "pelican-wings.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + labels: + {{- include "pelican-wings.labels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "pelican-wings.serviceAccountName" . }} + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + initContainers: + - name: copy-config + image: busybox:1.36 + command: ["cp", "/config-source/config.yml", "/etc/pelican/config.yml"] + volumeMounts: + - name: config-source + mountPath: /config-source + readOnly: true + - name: config + mountPath: /etc/pelican + containers: + - name: wings + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: api + containerPort: {{ .Values.wings.api.port }} + protocol: TCP + - name: sftp + containerPort: {{ .Values.wings.sftp.port }} + protocol: TCP + livenessProbe: + httpGet: + path: /api/system + port: api + initialDelaySeconds: 10 + periodSeconds: 30 + readinessProbe: + httpGet: + path: /api/system + port: api + initialDelaySeconds: 5 + periodSeconds: 10 + volumeMounts: + - name: config + mountPath: /etc/pelican + {{- if .Values.persistence.enabled }} + - name: data + mountPath: {{ .Values.wings.system.rootDirectory }} + {{- end }} + - name: logs + mountPath: {{ .Values.wings.system.logDirectory }} + - name: tmp + mountPath: {{ .Values.wings.system.tmpDirectory }} + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- with .Values.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: config-source + secret: + secretName: {{ include "pelican-wings.fullname" . }}-config + - name: config + emptyDir: {} + {{- if .Values.persistence.enabled }} + - name: data + persistentVolumeClaim: + claimName: {{ .Values.persistence.existingClaim | default (printf "%s-data" (include "pelican-wings.fullname" .)) }} + {{- end }} + - name: logs + emptyDir: {} + - name: tmp + emptyDir: {} + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/chart/pelican-wings/templates/limitrange.yaml b/chart/pelican-wings/templates/limitrange.yaml new file mode 100644 index 00000000..20243f37 --- /dev/null +++ b/chart/pelican-wings/templates/limitrange.yaml @@ -0,0 +1,42 @@ +{{- if .Values.limitRange.enabled }} +{{- if not (or .Values.limitRange.defaultCPULimit .Values.limitRange.defaultMemoryLimit .Values.limitRange.defaultCPURequest .Values.limitRange.defaultMemoryRequest .Values.limitRange.maxCPU .Values.limitRange.maxMemory) }} +{{- fail "limitRange.enabled is true but no limits are set; configure at least one of defaultCPULimit, defaultMemoryLimit, defaultCPURequest, defaultMemoryRequest, maxCPU or maxMemory (an empty LimitRange has no effect)." }} +{{- end }} +apiVersion: v1 +kind: LimitRange +metadata: + name: {{ include "pelican-wings.fullname" . }} + namespace: {{ .Values.gameNamespace }} + labels: + {{- include "pelican-wings.labels" . | nindent 4 }} +spec: + limits: + - type: Container + {{- if or .Values.limitRange.defaultCPULimit .Values.limitRange.defaultMemoryLimit }} + default: + {{- if .Values.limitRange.defaultCPULimit }} + cpu: {{ .Values.limitRange.defaultCPULimit | quote }} + {{- end }} + {{- if .Values.limitRange.defaultMemoryLimit }} + memory: {{ .Values.limitRange.defaultMemoryLimit | quote }} + {{- end }} + {{- end }} + {{- if or .Values.limitRange.defaultCPURequest .Values.limitRange.defaultMemoryRequest }} + defaultRequest: + {{- if .Values.limitRange.defaultCPURequest }} + cpu: {{ .Values.limitRange.defaultCPURequest | quote }} + {{- end }} + {{- if .Values.limitRange.defaultMemoryRequest }} + memory: {{ .Values.limitRange.defaultMemoryRequest | quote }} + {{- end }} + {{- end }} + {{- if or .Values.limitRange.maxCPU .Values.limitRange.maxMemory }} + max: + {{- if .Values.limitRange.maxCPU }} + cpu: {{ .Values.limitRange.maxCPU | quote }} + {{- end }} + {{- if .Values.limitRange.maxMemory }} + memory: {{ .Values.limitRange.maxMemory | quote }} + {{- end }} + {{- end }} +{{- end }} diff --git a/chart/pelican-wings/templates/namespace.yaml b/chart/pelican-wings/templates/namespace.yaml new file mode 100644 index 00000000..264ea787 --- /dev/null +++ b/chart/pelican-wings/templates/namespace.yaml @@ -0,0 +1,8 @@ +{{- if .Values.createGameNamespace }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Values.gameNamespace }} + labels: + {{- include "pelican-wings.labels" . | nindent 4 }} +{{- end }} diff --git a/chart/pelican-wings/templates/pvc.yaml b/chart/pelican-wings/templates/pvc.yaml new file mode 100644 index 00000000..147f3289 --- /dev/null +++ b/chart/pelican-wings/templates/pvc.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "pelican-wings.fullname" . }}-data + namespace: {{ .Values.gameNamespace }} + labels: + {{- include "pelican-wings.labels" . | nindent 4 }} + {{- with .Values.persistence.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + accessModes: + - {{ .Values.persistence.accessMode }} + {{- if .Values.persistence.storageClass }} + storageClassName: {{ .Values.persistence.storageClass | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size }} +{{- end }} diff --git a/chart/pelican-wings/templates/resourcequota.yaml b/chart/pelican-wings/templates/resourcequota.yaml new file mode 100644 index 00000000..4ac5877f --- /dev/null +++ b/chart/pelican-wings/templates/resourcequota.yaml @@ -0,0 +1,35 @@ +{{- if .Values.resourceQuota.enabled }} +{{- if not (or .Values.resourceQuota.cpuLimit .Values.resourceQuota.memoryLimit .Values.resourceQuota.cpuRequest .Values.resourceQuota.memoryRequest .Values.resourceQuota.maxPods .Values.resourceQuota.maxPVCs .Values.resourceQuota.maxStorage) }} +{{- fail "resourceQuota.enabled is true but no quota fields are set; configure at least one of cpuLimit, memoryLimit, cpuRequest, memoryRequest, maxPods, maxPVCs or maxStorage (an empty ResourceQuota would block all Pods from scheduling)." }} +{{- end }} +apiVersion: v1 +kind: ResourceQuota +metadata: + name: {{ include "pelican-wings.fullname" . }} + namespace: {{ .Values.gameNamespace }} + labels: + {{- include "pelican-wings.labels" . | nindent 4 }} +spec: + hard: + {{- if .Values.resourceQuota.cpuLimit }} + limits.cpu: {{ .Values.resourceQuota.cpuLimit | quote }} + {{- end }} + {{- if .Values.resourceQuota.memoryLimit }} + limits.memory: {{ .Values.resourceQuota.memoryLimit | quote }} + {{- end }} + {{- if .Values.resourceQuota.cpuRequest }} + requests.cpu: {{ .Values.resourceQuota.cpuRequest | quote }} + {{- end }} + {{- if .Values.resourceQuota.memoryRequest }} + requests.memory: {{ .Values.resourceQuota.memoryRequest | quote }} + {{- end }} + {{- if .Values.resourceQuota.maxPods }} + pods: {{ .Values.resourceQuota.maxPods | quote }} + {{- end }} + {{- if .Values.resourceQuota.maxPVCs }} + persistentvolumeclaims: {{ .Values.resourceQuota.maxPVCs | quote }} + {{- end }} + {{- if .Values.resourceQuota.maxStorage }} + requests.storage: {{ .Values.resourceQuota.maxStorage | quote }} + {{- end }} +{{- end }} diff --git a/chart/pelican-wings/templates/role.yaml b/chart/pelican-wings/templates/role.yaml new file mode 100644 index 00000000..64ab2cc5 --- /dev/null +++ b/chart/pelican-wings/templates/role.yaml @@ -0,0 +1,37 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "pelican-wings.fullname" . }} + namespace: {{ .Values.gameNamespace }} + labels: + {{- include "pelican-wings.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "create", "delete", "list", "watch"] + - apiGroups: [""] + resources: ["pods/log"] + verbs: ["get"] + - apiGroups: [""] + resources: ["pods/attach"] + verbs: ["create"] + - apiGroups: [""] + resources: ["services"] + verbs: ["get", "create", "update", "delete", "list"] + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get", "create", "delete"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "create", "update", "delete"] + - apiGroups: [""] + resources: ["resourcequotas"] + verbs: ["get", "create", "update"] + - apiGroups: [""] + resources: ["limitranges"] + verbs: ["get", "create", "update"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "create", "update", "delete", "list"] +{{- end }} diff --git a/chart/pelican-wings/templates/rolebinding.yaml b/chart/pelican-wings/templates/rolebinding.yaml new file mode 100644 index 00000000..afd69c61 --- /dev/null +++ b/chart/pelican-wings/templates/rolebinding.yaml @@ -0,0 +1,17 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "pelican-wings.fullname" . }} + namespace: {{ .Values.gameNamespace }} + labels: + {{- include "pelican-wings.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ include "pelican-wings.serviceAccountName" . }} + namespace: {{ .Values.gameNamespace }} +roleRef: + kind: Role + name: {{ include "pelican-wings.fullname" . }} + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/chart/pelican-wings/templates/service.yaml b/chart/pelican-wings/templates/service.yaml new file mode 100644 index 00000000..038eee4b --- /dev/null +++ b/chart/pelican-wings/templates/service.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "pelican-wings.fullname" . }} + namespace: {{ .Values.gameNamespace }} + labels: + {{- include "pelican-wings.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: api + protocol: TCP + name: api + - port: {{ .Values.service.sftpPort }} + targetPort: sftp + protocol: TCP + name: sftp + selector: + {{- include "pelican-wings.selectorLabels" . | nindent 4 }} diff --git a/chart/pelican-wings/templates/serviceaccount.yaml b/chart/pelican-wings/templates/serviceaccount.yaml new file mode 100644 index 00000000..0a74a112 --- /dev/null +++ b/chart/pelican-wings/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "pelican-wings.serviceAccountName" . }} + namespace: {{ .Values.gameNamespace }} + labels: + {{- include "pelican-wings.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/chart/pelican-wings/values.yaml b/chart/pelican-wings/values.yaml new file mode 100644 index 00000000..c598b247 --- /dev/null +++ b/chart/pelican-wings/values.yaml @@ -0,0 +1,227 @@ +# Default values for pelican-wings. + +# -- Number of Wings replicas (typically 1 per node via DaemonSet, or 1 Deployment) +replicaCount: 1 + +image: + # -- Wings container image repository + repository: ghcr.io/pelican/wings + # -- Image pull policy + pullPolicy: IfNotPresent + # -- Overrides the image tag (defaults to chart appVersion) + tag: "" + +# -- Image pull secrets for private registries +imagePullSecrets: [] + +# -- Override the release name +nameOverride: "" +# -- Override the full release name +fullnameOverride: "" + +serviceAccount: + # -- Whether to create the ServiceAccount + create: true + # -- Annotations to add to the ServiceAccount + annotations: {} + # -- Name of the ServiceAccount (generated if not set) + name: "" + +rbac: + # -- Whether to create RBAC resources (Role, RoleBinding, ClusterRole, ClusterRoleBinding) + create: true + # -- Grant the cluster-wide nodes/proxy permission so Wings can read resource + # metrics directly from the kubelet stats/summary API when metrics-server is + # not installed. This is a broad, cluster-scoped permission; leave it off and + # install metrics-server unless you specifically need the kubelet fallback. + kubeletMetricsFallback: false + +# -- Namespace for game server resources (Pods, Services, Jobs, PVCs) +gameNamespace: pelican + +# -- Whether to create the game namespace +createGameNamespace: true + +# Wings configuration +wings: + # -- Panel URL that Wings connects to + panelUrl: "https://panel.example.com" + # -- Authentication token for the Panel API + token: "" + # -- Token ID for Panel authentication + tokenId: "" + # -- Node UUID assigned by the Panel + uuid: "" + + api: + # -- Wings API listen address + host: "0.0.0.0" + # -- Wings API listen port + port: 8080 + + sftp: + # -- SFTP server listen address + host: "0.0.0.0" + # -- SFTP server listen port + port: 2022 + + system: + # -- Root data directory + rootDirectory: /var/lib/pelican + # -- Log directory + logDirectory: /var/log/pelican + # -- Server data directory + data: /var/lib/pelican/volumes + # -- Temp directory for installation scripts + tmpDirectory: /tmp/pelican + + kubernetes: + # -- Enable Kubernetes mode (required for K8s game server scheduling) + enabled: true + # -- Namespace for game server Pods (should match gameNamespace) + namespace: pelican + # -- Network mode: "hostport", "nodeport", or "loadbalancer" + # hostport: binds directly on host (closest to Docker) + # nodeport: shared node IP, auto-assigned external ports (30000-32767) + # loadbalancer: dedicated external IP per server via LB provisioner (Cilium, MetalLB, cloud) + networkMode: nodeport + # -- Annotations applied to LoadBalancer Services (only used when networkMode=loadbalancer) + # Example for Cilium LB-IPAM: + # lbAnnotations: + # io.cilium/lb-ipam-pool: "game-servers" + lbAnnotations: {} + # -- Annotation key Wings sets to the server's allocation IP on each LB Service. + # This pins the LB to the IP selected in the Panel so multiple servers share one IP. + # Cilium: "lbipam.cilium.io/ips" | MetalLB: "metallb.universe.tf/loadBalancerIPs" + lbIPAnnotation: "" + # -- Annotation key for LB IP sharing. Wings sets the value to the allocation IP, + # grouping all servers on the same IP under one sharing key. + # Cilium: "lbipam.cilium.io/sharing-key" | MetalLB: "metallb.universe.tf/allow-shared-ip" + lbSharingKey: "" + # -- Storage mode: "hostpath" or "pvc" + storageMode: pvc + # -- StorageClass for PVCs (empty = cluster default) + storageClass: "" + # -- Default PVC size per server + storageSize: "10Gi" + # -- PVC access mode + storageAccessMode: ReadWriteOnce + # -- Preserve game port as NodePort when in range (nodeport mode only) + nodeportPreserve: false + # -- Lower bound of the Kubernetes NodePort range (nodeport mode only) + nodeportRangeMin: 30000 + # -- Upper bound of the Kubernetes NodePort range (nodeport mode only) + nodeportRangeMax: 32767 + # -- Kubernetes node name for IP discovery (auto-detected from NODE_NAME env var if empty) + nodeName: "" + # -- Additional IPs to advertise in the allocation endpoint (e.g. floating IPs, VIPs) + systemIPs: [] + # -- DNS policy for game server Pods + dnsPolicy: ClusterFirst + # -- Node selector for game server Pods + nodeSelector: {} + # -- Tolerations for game server Pods + tolerations: [] + # -- Image pull secrets for game server Pods + imagePullSecrets: [] + # -- Override pull policy for game server Pods and install Jobs: + # "Always", "IfNotPresent", or "Never". Empty = remote images use Always + # (re-pull updated tags, matching Docker), ~-local images use IfNotPresent. + imagePullPolicy: "" + +# -- Pod-level security context for the Wings Deployment +podSecurityContext: {} + +# -- Container-level security context for Wings +securityContext: {} + +# Wings service configuration +service: + # -- Service type for the Wings API + type: ClusterIP + # -- Wings API port + port: 8080 + # -- SFTP port + sftpPort: 2022 + +# -- Resource requests and limits for the Wings container +resources: {} + # requests: + # cpu: 100m + # memory: 256Mi + # limits: + # cpu: "1" + # memory: 1Gi + +# -- Node selector for the Wings Pod itself +nodeSelector: {} + +# -- Tolerations for the Wings Pod itself +tolerations: [] + +# -- Affinity rules for the Wings Pod +affinity: {} + +# Persistence for the Wings root data directory (/var/lib/pelican). +# This stores server volumes, backups, and Wings state. Without persistence, +# all data is lost when the Wings pod restarts. +persistence: + # -- Enable persistent storage for the Wings data directory + enabled: true + # -- StorageClass for the PVC (empty = cluster default) + storageClass: "" + # -- PVC size + size: 50Gi + # -- Access mode + accessMode: ReadWriteOnce + # -- Use an existing PVC instead of creating one (set to PVC name) + existingClaim: "" + # -- Annotations for the PVC + annotations: {} + +# -- Additional environment variables for the Wings container +extraEnv: [] + +# -- Additional volume mounts for the Wings container +extraVolumeMounts: [] + +# -- Additional volumes for the Wings Pod +extraVolumes: [] + +# Resource Quota for the game server namespace. +# Caps aggregate resource usage across all game server Pods. +resourceQuota: + # -- Whether to create a ResourceQuota in the game namespace + enabled: false + # -- Total CPU limit across all Pods (e.g. "16", "8000m") + cpuLimit: "" + # -- Total memory limit across all Pods (e.g. "32Gi") + memoryLimit: "" + # -- Total CPU request across all Pods (e.g. "8") + cpuRequest: "" + # -- Total memory request across all Pods (e.g. "16Gi") + memoryRequest: "" + # -- Maximum number of Pods + maxPods: "" + # -- Maximum number of PVCs + maxPVCs: "" + # -- Total storage across all PVCs (e.g. "500Gi") + maxStorage: "" + +# Limit Range for the game server namespace. +# Sets default resource limits/requests for containers that don't specify their own. +limitRange: + # -- Whether to create a LimitRange in the game namespace + enabled: false + # -- Default CPU limit per container (e.g. "2", "1000m") + defaultCPULimit: "" + # -- Default memory limit per container (e.g. "4Gi") + defaultMemoryLimit: "" + # -- Default CPU request per container (e.g. "500m") + defaultCPURequest: "" + # -- Default memory request per container (e.g. "1Gi") + defaultMemoryRequest: "" + # -- Maximum CPU any single container can use (e.g. "8") + maxCPU: "" + # -- Maximum memory any single container can use (e.g. "16Gi") + maxMemory: "" diff --git a/config/config.go b/config/config.go index 16097fde..151cf320 100644 --- a/config/config.go +++ b/config/config.go @@ -369,6 +369,11 @@ type Configuration struct { System SystemConfiguration `json:"system" yaml:"system"` Docker DockerConfiguration `json:"docker" yaml:"docker"` + // Kubernetes holds configuration for scheduling game server workloads as + // Kubernetes Pods. When Kubernetes.Enabled is true, all servers on this + // node are run in K8s instead of Docker. + Kubernetes KubernetesConfiguration `json:"kubernetes" yaml:"kubernetes"` + // Defines internal throttling configurations for server processes to prevent // someone from running an endless loop that spams data to logs. Throttles ConsoleThrottles diff --git a/config/config_kubernetes.go b/config/config_kubernetes.go new file mode 100644 index 00000000..525eb62e --- /dev/null +++ b/config/config_kubernetes.go @@ -0,0 +1,229 @@ +package config + +// KubernetesStorageMode defines how server data volumes are provisioned. +type KubernetesStorageMode string + +const ( + // KubeStorageHostPath uses HostPath volumes (node-local, default). + KubeStorageHostPath KubernetesStorageMode = "hostpath" + + // KubeStoragePVC uses PersistentVolumeClaims for server data directories. + KubeStoragePVC KubernetesStorageMode = "pvc" +) + +// KubernetesNetworkMode defines how game server ports are exposed. +type KubernetesNetworkMode string + +const ( + // KubeNetworkHostPort uses hostNetwork + hostPort for direct port binding + // (closest to Docker behavior). + KubeNetworkHostPort KubernetesNetworkMode = "hostport" + + // KubeNetworkNodePort uses Kubernetes Services with NodePort to expose game + // server ports via the node's IP with auto-assigned external ports. + KubeNetworkNodePort KubernetesNetworkMode = "nodeport" + + // KubeNetworkLoadBalancer uses Kubernetes Services with type LoadBalancer + // to assign a dedicated external IP per game server. Requires a load + // balancer provisioner (MetalLB, cloud controller, kube-vip, etc.). + KubeNetworkLoadBalancer KubernetesNetworkMode = "loadbalancer" +) + +// KubernetesConfiguration defines the Kubernetes configuration used by the +// daemon when scheduling game server workloads as Pods. +type KubernetesConfiguration struct { + // Enabled controls whether this Wings node uses Kubernetes to run game + // server workloads instead of Docker. + Enabled bool `default:"false" json:"enabled" yaml:"enabled"` + + // Namespace is the Kubernetes namespace where game server Pods and related + // resources are created. + Namespace string `default:"pelican" json:"namespace" yaml:"namespace"` + + // Kubeconfig is the path to a kubeconfig file. If empty, in-cluster + // configuration is used (suitable when Wings itself runs inside K8s). + Kubeconfig string `default:"" json:"kubeconfig" yaml:"kubeconfig"` + + // ImagePullSecrets is a list of Secret names used for pulling container + // images in game server Pods. + ImagePullSecrets []string `json:"image_pull_secrets" yaml:"image_pull_secrets"` + + // ImagePullPolicy overrides the pull policy applied to game server Pods + // and installation Jobs. Valid values are "Always", "IfNotPresent", and + // "Never". When empty, remote images use "Always" (so updated tags are + // re-pulled, matching the Docker backend) and ~-prefixed local images use + // "IfNotPresent". Set to "IfNotPresent" or "Never" for air-gapped clusters. + ImagePullPolicy string `default:"" json:"image_pull_policy" yaml:"image_pull_policy"` + + // ServiceAccount is the name of the Kubernetes ServiceAccount assigned to + // game server Pods. + ServiceAccount string `default:"" json:"service_account" yaml:"service_account"` + + // StorageMode controls how server data volumes are provisioned. + // "hostpath" uses HostPath volumes (default, node-local). + // "pvc" creates PersistentVolumeClaims for each server. + StorageMode KubernetesStorageMode `default:"hostpath" json:"storage_mode" yaml:"storage_mode"` + + // StorageClass is the StorageClass used for PersistentVolumeClaims that + // back server data directories. Leave empty to use the cluster default. + StorageClass string `default:"" json:"storage_class" yaml:"storage_class"` + + // StorageSize is the default size for server PVCs (e.g. "10Gi", "50Gi"). + // Only used when StorageMode is "pvc". + StorageSize string `default:"10Gi" json:"storage_size" yaml:"storage_size"` + + // StorageAccessMode is the access mode for PVCs. Defaults to + // ReadWriteOnce. Options: ReadWriteOnce, ReadWriteMany. + StorageAccessMode string `default:"ReadWriteOnce" json:"storage_access_mode" yaml:"storage_access_mode"` + + // NodeSelector is a set of key-value pairs used for scheduling Pods onto + // specific nodes. + NodeSelector map[string]string `json:"node_selector" yaml:"node_selector"` + + // DNSPolicy sets the DNS policy for game server Pods. + DNSPolicy string `default:"ClusterFirst" json:"dns_policy" yaml:"dns_policy"` + + // NetworkMode controls how game server ports are exposed to clients. + NetworkMode KubernetesNetworkMode `default:"hostport" json:"network_mode" yaml:"network_mode"` + + // NodePortPreserve attempts to use the game server's allocated port as + // the NodePort value (only works if the port falls within the cluster's + // NodePort range). When false, Kubernetes auto-assigns NodePorts. + NodePortPreserve bool `default:"false" json:"nodeport_preserve" yaml:"nodeport_preserve"` + + // NodePortRangeMin is the lower bound of the Kubernetes NodePort range. + // Defaults to 30000 if unset. Used when NodePortPreserve is true. + NodePortRangeMin int32 `default:"30000" json:"nodeport_range_min" yaml:"nodeport_range_min"` + + // NodePortRangeMax is the upper bound of the Kubernetes NodePort range. + // Defaults to 32767 if unset. Used when NodePortPreserve is true. + NodePortRangeMax int32 `default:"32767" json:"nodeport_range_max" yaml:"nodeport_range_max"` + + // LBAnnotations is a map of annotations applied to LoadBalancer Services. + // Use this for provider-specific configuration (e.g. MetalLB address pool, + // cloud LB options). Example: + // lb_annotations: + // metallb.universe.tf/address-pool: "game-servers" + LBAnnotations map[string]string `json:"lb_annotations" yaml:"lb_annotations"` + + // LBIPAnnotation is the annotation key that Wings sets to the server's + // allocation IP on each LoadBalancer Service. This pins the LB to the IP + // the user selected in the Panel, enabling multiple game servers to share + // the same external IP (on different ports). + // + // Common values: + // Cilium: "lbipam.cilium.io/ips" + // MetalLB: "metallb.universe.tf/loadBalancerIPs" + // + // When empty (default), no IP-pinning annotation is added. + LBIPAnnotation string `default:"" json:"lb_ip_annotation" yaml:"lb_ip_annotation"` + + // LBSharingKey is the annotation key used to allow multiple Services to + // share the same external IP. Wings sets this annotation to the allocation + // IP value, grouping all game servers on the same IP under one sharing key. + // + // Common values: + // Cilium: "lbipam.cilium.io/sharing-key" + // MetalLB: "metallb.universe.tf/allow-shared-ip" + // + // When empty (default), no sharing-key annotation is added. + LBSharingKey string `default:"" json:"lb_sharing_key" yaml:"lb_sharing_key"` + + // Tolerations allow game server Pods to be scheduled on tainted nodes. + Tolerations []KubeToleration `json:"tolerations" yaml:"tolerations"` + + // NodeName is the Kubernetes node name where Wings is running. Used to + // query node addresses for the IP allocation endpoint. If empty, Wings + // reads the NODE_NAME environment variable (typically set via the + // Kubernetes downward API). + NodeName string `default:"" json:"node_name" yaml:"node_name"` + + // SystemIPs is a list of additional IP addresses to expose in the + // allocation endpoint. These are appended to the auto-discovered node + // addresses and can be used to advertise public IPs that are not + // directly assigned to the node (e.g. floating IPs, load balancer VIPs). + SystemIPs []string `json:"system_ips" yaml:"system_ips"` + + // DataPVC is the name of a shared PersistentVolumeClaim that Wings and game + // server Pods both mount for server data. When set, game servers use this PVC + // with a subPath instead of creating individual PVCs per server. This enables + // the Panel file browser and SFTP access, since Wings can read/write the same + // storage as the game server Pods. + // + // The PVC must already exist and be mounted into the Wings Pod at the + // configured rootDirectory. Typically this is the Wings data PVC created + // by the Helm chart. + // + // When empty (default), each server gets its own PVC (file browser won't work). + DataPVC string `default:"" json:"data_pvc" yaml:"data_pvc"` + + // ResourceQuota configures namespace-level resource limits. + // When enabled, Wings creates/updates a ResourceQuota in the game server + // namespace to cap aggregate resource usage across all Pods. + ResourceQuota KubeResourceQuota `json:"resource_quota" yaml:"resource_quota"` + + // LimitRange configures default resource limits and requests for Pods + // that do not specify their own. This prevents unbounded resource usage + // from misconfigured servers. + LimitRange KubeLimitRange `json:"limit_range" yaml:"limit_range"` +} + +// KubeResourceQuota defines namespace-level resource quotas. +type KubeResourceQuota struct { + // Enabled controls whether a ResourceQuota is created/enforced. + Enabled bool `default:"false" json:"enabled" yaml:"enabled"` + + // CPULimit is the total CPU limit across all Pods (e.g. "16", "8000m"). + CPULimit string `default:"" json:"cpu_limit" yaml:"cpu_limit"` + + // MemoryLimit is the total memory limit across all Pods (e.g. "32Gi"). + MemoryLimit string `default:"" json:"memory_limit" yaml:"memory_limit"` + + // CPURequest is the total CPU request across all Pods (e.g. "8", "4000m"). + CPURequest string `default:"" json:"cpu_request" yaml:"cpu_request"` + + // MemoryRequest is the total memory request across all Pods (e.g. "16Gi"). + MemoryRequest string `default:"" json:"memory_request" yaml:"memory_request"` + + // MaxPods is the maximum number of Pods allowed in the namespace. + MaxPods int64 `default:"0" json:"max_pods" yaml:"max_pods"` + + // MaxPVCs is the maximum number of PersistentVolumeClaims in the namespace. + MaxPVCs int64 `default:"0" json:"max_pvcs" yaml:"max_pvcs"` + + // MaxStorage is the total storage allowed across all PVCs (e.g. "500Gi"). + MaxStorage string `default:"" json:"max_storage" yaml:"max_storage"` +} + +// KubeLimitRange defines default resource limits and requests for containers. +type KubeLimitRange struct { + // Enabled controls whether a LimitRange is created. + Enabled bool `default:"false" json:"enabled" yaml:"enabled"` + + // DefaultCPULimit is the default CPU limit for containers (e.g. "2", "1000m"). + DefaultCPULimit string `default:"" json:"default_cpu_limit" yaml:"default_cpu_limit"` + + // DefaultMemoryLimit is the default memory limit for containers (e.g. "4Gi"). + DefaultMemoryLimit string `default:"" json:"default_memory_limit" yaml:"default_memory_limit"` + + // DefaultCPURequest is the default CPU request for containers (e.g. "500m"). + DefaultCPURequest string `default:"" json:"default_cpu_request" yaml:"default_cpu_request"` + + // DefaultMemoryRequest is the default memory request for containers (e.g. "1Gi"). + DefaultMemoryRequest string `default:"" json:"default_memory_request" yaml:"default_memory_request"` + + // MaxCPU is the maximum CPU any single container can request (e.g. "4"). + MaxCPU string `default:"" json:"max_cpu" yaml:"max_cpu"` + + // MaxMemory is the maximum memory any single container can request (e.g. "8Gi"). + MaxMemory string `default:"" json:"max_memory" yaml:"max_memory"` +} + +// KubeToleration mirrors corev1.Toleration for YAML configuration. +type KubeToleration struct { + Key string `json:"key" yaml:"key"` + Operator string `default:"Equal" json:"operator" yaml:"operator"` + Value string `json:"value" yaml:"value"` + Effect string `json:"effect" yaml:"effect"` + TolerationSeconds *int64 `json:"toleration_seconds" yaml:"toleration_seconds"` +} diff --git a/environment/kubernetes/api.go b/environment/kubernetes/api.go new file mode 100644 index 00000000..4cd9f474 --- /dev/null +++ b/environment/kubernetes/api.go @@ -0,0 +1,42 @@ +package kubernetes + +import ( + "sync" + + "emperror.dev/errors" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + + "github.com/pelican/wings/config" +) + +var ( + _konce sync.Once + _client kubernetes.Interface + _kerr error +) + +// Client returns a shared Kubernetes clientset. The client is created once +// and reused for all subsequent calls. It uses in-cluster config when no +// kubeconfig path is specified, falling back to the provided kubeconfig file. +func Client() (kubernetes.Interface, error) { + _konce.Do(func() { + var cfg *rest.Config + kubeconfig := config.Get().Kubernetes.Kubeconfig + if kubeconfig != "" { + cfg, _kerr = clientcmd.BuildConfigFromFlags("", kubeconfig) + } else { + cfg, _kerr = rest.InClusterConfig() + } + if _kerr != nil { + _kerr = errors.Wrap(_kerr, "environment/kubernetes: failed to build config") + return + } + _client, _kerr = kubernetes.NewForConfig(cfg) + if _kerr != nil { + _kerr = errors.Wrap(_kerr, "environment/kubernetes: failed to create clientset") + } + }) + return _client, _kerr +} diff --git a/environment/kubernetes/container.go b/environment/kubernetes/container.go new file mode 100644 index 00000000..047fe36e --- /dev/null +++ b/environment/kubernetes/container.go @@ -0,0 +1,589 @@ +package kubernetes + +import ( + "bufio" + "context" + "fmt" + "io" + "path/filepath" + "strings" + + "emperror.dev/errors" + "github.com/apex/log" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/remotecommand" + + "github.com/pelican/wings/config" + "github.com/pelican/wings/environment" + "github.com/pelican/wings/system" +) + +// resolveImagePullPolicy returns the cleaned image reference (without the ~ +// local-image prefix) and the pull policy to apply. It mirrors the Docker +// backend, which always attempts to pull remote images before starting a +// server so updated tags are picked up, while never pulling ~-prefixed local +// images. A non-empty kubernetes.image_pull_policy config value overrides the +// default (useful for air-gapped clusters). +func resolveImagePullPolicy(image string) (string, corev1.PullPolicy) { + cleaned := strings.TrimPrefix(image, "~") + if override := config.Get().Kubernetes.ImagePullPolicy; override != "" { + return cleaned, corev1.PullPolicy(override) + } + if strings.HasPrefix(image, "~") { + return cleaned, corev1.PullIfNotPresent + } + return cleaned, corev1.PullAlways +} + +// Create builds and creates the Pod for this game server in Kubernetes. +// If the Pod already exists this is a no-op. +func (e *Environment) Create() error { + ctx := context.Background() + + // If the Pod already exists, return immediately. + exists, err := e.Exists() + if err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to check pod existence") + } + if exists { + return nil + } + + cfg := config.Get() + limits := e.Configuration.Limits() + evs := e.Configuration.EnvironmentVariables() + mounts := e.Configuration.Mounts() + + // Build environment variables for the container. + var envVars []corev1.EnvVar + for _, ev := range evs { + parts := strings.SplitN(ev, "=", 2) + if len(parts) == 2 { + envVars = append(envVars, corev1.EnvVar{ + Name: parts[0], + Value: parts[1], + }) + } + } + + // Build resource requirements from server limits. + resources := e.buildResources(limits) + + // Build volume mounts and volumes. In K8s mode, identity files (passwd, + // group, machine-id) are served from a ConfigMap instead of hostPath. + identityMounts, regularMounts := e.splitIdentityMounts(mounts) + volumes, volumeMounts := e.buildVolumes(regularMounts) + + if len(identityMounts) > 0 { + idVol, idMounts, err := e.ensureIdentityConfigMap(ctx, identityMounts) + if err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to create identity ConfigMap") + } + volumes = append(volumes, idVol) + volumeMounts = append(volumeMounts, idMounts...) + } + + // Determine image and pull policy (mirrors the Docker backend). + image, pullPolicy := resolveImagePullPolicy(e.meta.Image) + + // Build container ports from allocations. + containerPorts := e.buildContainerPorts() + + // Labels for identification. + labels := map[string]string{ + "app.kubernetes.io/managed-by": "pelican-wings", + "pelican.dev/server-id": e.Id, + "pelican.dev/container-type": "server_process", + } + + // Merge user-provided labels. + for k, v := range e.Configuration.Labels() { + labels[k] = v + } + + // Determine the UID/GID for the pelican user. + uid := int64(config.Get().System.User.Uid) + gid := int64(config.Get().System.User.Gid) + + // Build the Pod spec. + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: e.Id, + Namespace: e.namespace(), + Labels: labels, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + SecurityContext: &corev1.PodSecurityContext{ + RunAsUser: &uid, + RunAsGroup: &gid, + FSGroup: &gid, + }, + InitContainers: []corev1.Container{ + { + Name: "set-permissions", + Image: "busybox:1.36", + Command: []string{"sh", "-c", fmt.Sprintf("chown -R %d:%d /home/container && chmod -R 755 /home/container", uid, gid)}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "server-data", + MountPath: "/home/container", + SubPath: e.serverDataSubPathIfShared(), + }, + }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: func() *int64 { v := int64(0); return &v }(), + RunAsGroup: func() *int64 { v := int64(0); return &v }(), + }, + }, + }, + Containers: []corev1.Container{ + { + Name: "server", + Image: image, + Env: envVars, + Resources: resources, + Ports: containerPorts, + VolumeMounts: volumeMounts, + Stdin: true, + TTY: true, + ImagePullPolicy: pullPolicy, + }, + }, + Volumes: volumes, + }, + } + + // Apply network mode configuration. + if cfg.Kubernetes.NetworkMode == config.KubeNetworkHostPort { + pod.Spec.HostNetwork = false // hostPort is set on containerPorts + } + + // Apply DNS policy. + if cfg.Kubernetes.DNSPolicy != "" { + pod.Spec.DNSPolicy = corev1.DNSPolicy(cfg.Kubernetes.DNSPolicy) + } + + // Apply node selector. + if len(cfg.Kubernetes.NodeSelector) > 0 { + pod.Spec.NodeSelector = cfg.Kubernetes.NodeSelector + } + + // Apply service account. + if cfg.Kubernetes.ServiceAccount != "" { + pod.Spec.ServiceAccountName = cfg.Kubernetes.ServiceAccount + } + + // Apply image pull secrets. + for _, secret := range cfg.Kubernetes.ImagePullSecrets { + pod.Spec.ImagePullSecrets = append(pod.Spec.ImagePullSecrets, corev1.LocalObjectReference{ + Name: secret, + }) + } + + // Apply tolerations. + for _, t := range cfg.Kubernetes.Tolerations { + toleration := corev1.Toleration{ + Key: t.Key, + Operator: corev1.TolerationOperator(t.Operator), + Value: t.Value, + Effect: corev1.TaintEffect(t.Effect), + } + if t.TolerationSeconds != nil { + toleration.TolerationSeconds = t.TolerationSeconds + } + pod.Spec.Tolerations = append(pod.Spec.Tolerations, toleration) + } + + // Ensure namespace-level resource constraints are applied. + if err := e.EnsureResourceQuota(ctx); err != nil { + e.log().WithField("error", err).Warn("failed to ensure ResourceQuota") + } + if err := e.EnsureLimitRange(ctx); err != nil { + e.log().WithField("error", err).Warn("failed to ensure LimitRange") + } + + // Ensure the PVC exists if using PVC storage mode. + if err := e.EnsurePVC(ctx); err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to ensure PVC") + } + + e.log().WithField("image", image).Info("creating pod for server") + + _, err = e.client.CoreV1().Pods(e.namespace()).Create(ctx, pod, metav1.CreateOptions{}) + if err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to create pod") + } + + // Create or update the NodePort Service if using NodePort network mode. + if err := e.EnsureService(ctx); err != nil { + e.log().WithField("error", err).Warn("failed to create/update NodePort service") + } + + return nil +} + +// Destroy removes the Pod and associated Service from Kubernetes. +func (e *Environment) Destroy() error { + e.SetState(environment.ProcessStoppingState) + + ctx := context.Background() + + gracePeriod := int64(0) + err := e.client.CoreV1().Pods(e.namespace()).Delete( + ctx, + e.Id, + metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod}, + ) + + // Clean up the associated NodePort Service. + if svcErr := e.DeleteService(ctx); svcErr != nil { + e.log().WithField("error", svcErr).Warn("failed to delete NodePort service during destroy") + } + + // Clean up the PVC if using PVC storage mode. + if pvcErr := e.DeletePVC(ctx); pvcErr != nil { + e.log().WithField("error", pvcErr).Warn("failed to delete PVC during destroy") + } + + // Clean up the identity ConfigMap. + if cmErr := e.deleteIdentityConfigMap(ctx); cmErr != nil { + e.log().WithField("error", cmErr).Warn("failed to delete identity ConfigMap during destroy") + } + + if err != nil && !isNotFound(err) { + return errors.Wrap(err, "environment/kubernetes: failed to delete pod") + } + + e.SetState(environment.ProcessOfflineState) + + return nil +} + +// Attach connects to the running Pod's stdin/stdout and begins streaming +// output to the log callback. This should be called before the server starts +// producing output. +func (e *Environment) Attach(ctx context.Context) error { + if e.IsAttached() { + return nil + } + + restCfg, err := e.getRESTConfig() + if err != nil { + return errors.WrapIf(err, "environment/kubernetes: failed to get REST config") + } + + req := e.client.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(e.Id). + Namespace(e.namespace()). + SubResource("attach"). + VersionedParams(&corev1.PodAttachOptions{ + Container: "server", + Stdin: true, + Stdout: true, + Stderr: true, + TTY: true, + }, scheme.ParameterCodec) + + exec, err := remotecommand.NewSPDYExecutor(restCfg, "POST", req.URL()) + if err != nil { + return errors.WrapIf(err, "environment/kubernetes: failed to create SPDY executor") + } + + // Create a pipe for stdin. + stdinReader, stdinWriter := io.Pipe() + e.setStream(stdinWriter) + + go func() { + defer e.setStream(nil) + defer func() { + e.SetState(environment.ProcessOfflineState) + }() + + // Start resource polling. + pollCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go func() { + if err := e.pollResources(pollCtx); err != nil { + if !errors.Is(err, context.Canceled) { + e.log().WithField("error", err).Error("error during resource polling") + } + } + }() + + // Stream stdout/stderr to the log callback via a pipe. + stdoutReader, stdoutWriter := io.Pipe() + + go func() { + if err := system.ScanReader(stdoutReader, func(v []byte) { + e.logCallbackMx.Lock() + defer e.logCallbackMx.Unlock() + if e.logCallback != nil { + e.logCallback(v) + } + }); err != nil && err != io.EOF { + log.WithField("error", err).WithField("pod_id", e.Id).Warn("error processing output in console") + } + }() + + streamErr := exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdin: stdinReader, + Stdout: stdoutWriter, + Stderr: stdoutWriter, + Tty: true, + }) + stdoutWriter.Close() + stdinReader.Close() + + if streamErr != nil && !errors.Is(streamErr, context.Canceled) { + e.log().WithField("error", streamErr).Warn("pod attach stream ended") + } + }() + + return nil +} + +// SendCommand writes a command string to the attached Pod's stdin. +func (e *Environment) SendCommand(c string) error { + e.mu.RLock() + defer e.mu.RUnlock() + + // Check the stream under the lock so it cannot be cleared between an + // attachment check and the write below. + if e.stream == nil { + return errors.New("environment/kubernetes: not attached to pod") + } + + // If this is the stop command, mark the server as stopping. + if e.meta.Stop.Type == "command" && c == e.meta.Stop.Value { + e.SetState(environment.ProcessStoppingState) + } + + _, err := e.stream.Write([]byte(c + "\n")) + return errors.Wrap(err, "environment/kubernetes: could not write to pod stream") +} + +// Readlog reads the last N lines of log output from the Pod. +func (e *Environment) Readlog(lines int) ([]string, error) { + tailLines := int64(lines) + opts := &corev1.PodLogOptions{ + Container: "server", + TailLines: &tailLines, + } + + req := e.client.CoreV1().Pods(e.namespace()).GetLogs(e.Id, opts) + stream, err := req.Stream(context.Background()) + if err != nil { + return nil, errors.WithStack(err) + } + defer stream.Close() + + var out []string + scanner := bufio.NewScanner(stream) + for scanner.Scan() { + out = append(out, scanner.Text()) + } + + return out, nil +} + +// buildResources converts Wings Limits into Kubernetes resource requirements. +func (e *Environment) buildResources(limits environment.Limits) corev1.ResourceRequirements { + resources := corev1.ResourceRequirements{ + Limits: corev1.ResourceList{}, + Requests: corev1.ResourceList{}, + } + + // Memory: limits.MemoryLimit is in MiB. + if limits.MemoryLimit > 0 { + mem := resource.MustParse(fmt.Sprintf("%dMi", limits.MemoryLimit)) + resources.Limits[corev1.ResourceMemory] = mem + // Request 50% of the limit as a baseline. + memReq := resource.MustParse(fmt.Sprintf("%dMi", limits.MemoryLimit/2)) + resources.Requests[corev1.ResourceMemory] = memReq + } + + // CPU: limits.CpuLimit is a percentage (100 = 1 core). + if limits.CpuLimit > 0 { + cpuMillis := limits.CpuLimit * 10 // 100% = 1000m + cpu := resource.MustParse(fmt.Sprintf("%dm", cpuMillis)) + resources.Limits[corev1.ResourceCPU] = cpu + // Request 25% of the limit. + cpuReqMillis := cpuMillis / 4 + if cpuReqMillis < 50 { + cpuReqMillis = 50 + } + cpuReq := resource.MustParse(fmt.Sprintf("%dm", cpuReqMillis)) + resources.Requests[corev1.ResourceCPU] = cpuReq + } + + return resources +} + +// buildVolumes constructs Kubernetes volumes and mounts from the environment +// mount configuration. When StorageMode is "pvc", the default server-data +// volume uses a PersistentVolumeClaim instead of a HostPath. +// +// When DataPVC is set, the default mount uses the shared Wings data PVC with +// a subPath so that Wings and game server Pods share the same storage. This +// enables the Panel file browser and SFTP to work in K8s mode. +func (e *Environment) buildVolumes(mounts []environment.Mount) ([]corev1.Volume, []corev1.VolumeMount) { + cfg := config.Get() + var volumes []corev1.Volume + var volumeMounts []corev1.VolumeMount + + for i, m := range mounts { + volName := fmt.Sprintf("mount-%d", i) + if m.Default { + volName = "server-data" + } + + var volSource corev1.VolumeSource + var subPath string + + if m.Default && cfg.Kubernetes.DataPVC != "" { + // Shared PVC mode: mount the Wings data PVC with a subPath + // so both Wings and the game server see the same files. + volSource = corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: cfg.Kubernetes.DataPVC, + }, + } + subPath = e.serverDataSubPath() + } else if m.Default && cfg.Kubernetes.StorageMode == config.KubeStoragePVC { + // Per-server PVC mode (original behavior). + volSource = corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: e.pvcName(), + }, + } + } else { + // Use HostPath for non-default mounts or when in hostpath mode. + volSource = corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: m.Source, + }, + } + } + + volumes = append(volumes, corev1.Volume{ + Name: volName, + VolumeSource: volSource, + }) + + vm := corev1.VolumeMount{ + Name: volName, + MountPath: m.Target, + ReadOnly: m.ReadOnly, + } + if subPath != "" { + vm.SubPath = subPath + } + volumeMounts = append(volumeMounts, vm) + } + + return volumes, volumeMounts +} + +// serverDataSubPath returns the subPath within the shared data PVC for this +// server's data directory. This is the relative path from the Wings root +// directory to the server's data directory. +func (e *Environment) serverDataSubPath() string { + cfg := config.Get() + serverPath := filepath.Join(cfg.System.Data, e.Id) + rel, err := filepath.Rel(cfg.System.RootDirectory, serverPath) + if err != nil { + return filepath.Join("volumes", e.Id) + } + return rel +} + +// serverDataSubPathIfShared returns the subPath for shared PVC mode, or an +// empty string if not using shared PVC mode. +func (e *Environment) serverDataSubPathIfShared() string { + if config.Get().Kubernetes.DataPVC != "" { + return e.serverDataSubPath() + } + return "" +} + +// buildContainerPorts creates the container port list from allocations. +func (e *Environment) buildContainerPorts() []corev1.ContainerPort { + cfg := config.Get() + allocs := e.Configuration.Allocations() + var ports []corev1.ContainerPort + seen := make(map[int]struct{}) + + for _, allocPorts := range allocs.Mappings { + for _, port := range allocPorts { + if port < 1 || port > 65535 { + continue + } + if _, ok := seen[port]; ok { + continue + } + seen[port] = struct{}{} + + tcpPort := corev1.ContainerPort{ + Name: fmt.Sprintf("tcp-%d", port), + ContainerPort: int32(port), + Protocol: corev1.ProtocolTCP, + } + udpPort := corev1.ContainerPort{ + Name: fmt.Sprintf("udp-%d", port), + ContainerPort: int32(port), + Protocol: corev1.ProtocolUDP, + } + + // If using hostPort mode, bind directly to the host. + if cfg.Kubernetes.NetworkMode == config.KubeNetworkHostPort { + tcpPort.HostPort = int32(port) + udpPort.HostPort = int32(port) + } + + ports = append(ports, tcpPort, udpPort) + } + } + + return ports +} + +// getRESTConfig returns the rest.Config used for exec/attach operations. +func (e *Environment) getRESTConfig() (*rest.Config, error) { + kubeconfig := config.Get().Kubernetes.Kubeconfig + if kubeconfig != "" { + return clientcmd.BuildConfigFromFlags("", kubeconfig) + } + return rest.InClusterConfig() +} + +// getPod fetches the Pod object from the Kubernetes API. +func (e *Environment) getPod(ctx context.Context) (*corev1.Pod, error) { + return e.client.CoreV1().Pods(e.namespace()).Get(ctx, e.Id, metav1.GetOptions{}) +} + +// isNotFound checks if the error is a Kubernetes NotFound error. +func isNotFound(err error) bool { + return apierrors.IsNotFound(err) +} + +// isPodRunning checks if a Pod is in Running phase with a ready container. +func isPodRunning(pod *corev1.Pod) bool { + if pod.Status.Phase != corev1.PodRunning { + return false + } + for _, cs := range pod.Status.ContainerStatuses { + if cs.Name == "server" && cs.State.Running != nil { + return true + } + } + return false +} diff --git a/environment/kubernetes/container_test.go b/environment/kubernetes/container_test.go new file mode 100644 index 00000000..894947e1 --- /dev/null +++ b/environment/kubernetes/container_test.go @@ -0,0 +1,54 @@ +package kubernetes + +import ( + "testing" + + . "github.com/franela/goblin" + corev1 "k8s.io/api/core/v1" + + "github.com/pelican/wings/config" +) + +// TestResolveImagePullPolicy verifies pull-policy resolution for remote +// images, ~-prefixed local images, and configured overrides. +func TestResolveImagePullPolicy(t *testing.T) { + g := Goblin(t) + + g.Describe("resolveImagePullPolicy", func() { + g.BeforeEach(func() { + config.Update(func(c *config.Configuration) { + c.Kubernetes.ImagePullPolicy = "" + }) + }) + + g.It("always pulls remote images so updated tags are picked up", func() { + image, policy := resolveImagePullPolicy("ghcr.io/pelican-eggs/games:latest") + g.Assert(image).Equal("ghcr.io/pelican-eggs/games:latest") + g.Assert(policy).Equal(corev1.PullAlways) + }) + + g.It("does not pull ~-prefixed local images and strips the prefix", func() { + image, policy := resolveImagePullPolicy("~local/custom:dev") + g.Assert(image).Equal("local/custom:dev") + g.Assert(policy).Equal(corev1.PullIfNotPresent) + }) + + g.It("honors a configured override for remote images", func() { + config.Update(func(c *config.Configuration) { + c.Kubernetes.ImagePullPolicy = "IfNotPresent" + }) + image, policy := resolveImagePullPolicy("ghcr.io/pelican-eggs/games:latest") + g.Assert(image).Equal("ghcr.io/pelican-eggs/games:latest") + g.Assert(policy).Equal(corev1.PullIfNotPresent) + }) + + g.It("honors a configured override and still strips the local prefix", func() { + config.Update(func(c *config.Configuration) { + c.Kubernetes.ImagePullPolicy = "Never" + }) + image, policy := resolveImagePullPolicy("~local/custom:dev") + g.Assert(image).Equal("local/custom:dev") + g.Assert(policy).Equal(corev1.PullNever) + }) + }) +} diff --git a/environment/kubernetes/environment.go b/environment/kubernetes/environment.go new file mode 100644 index 00000000..fcc1dcf4 --- /dev/null +++ b/environment/kubernetes/environment.go @@ -0,0 +1,209 @@ +package kubernetes + +import ( + "context" + "fmt" + "io" + "sync" + + "emperror.dev/errors" + "github.com/apex/log" + "k8s.io/client-go/kubernetes" + + "github.com/pelican/wings/config" + "github.com/pelican/wings/environment" + "github.com/pelican/wings/events" + "github.com/pelican/wings/remote" + "github.com/pelican/wings/system" +) + +// Metadata holds runtime metadata for the Kubernetes environment that can be +// updated on the fly (e.g., image changes from the Panel). +type Metadata struct { + Image string + Stop remote.ProcessStopConfiguration +} + +// Ensure that the Kubernetes environment always implements the full +// ProcessEnvironment interface. +var _ environment.ProcessEnvironment = (*Environment)(nil) + +// Environment is the Kubernetes implementation of ProcessEnvironment. It +// manages game server workloads as Pods within a configured namespace. +type Environment struct { + mu sync.RWMutex + + // Id is the unique server identifier (UUID) used as the Pod name. + Id string + + // Configuration holds the environment settings (limits, mounts, env vars). + Configuration *environment.Configuration + + meta *Metadata + + // client is the Kubernetes clientset. + client kubernetes.Interface + + // stream holds the attach connection to the running Pod's stdin. + stream io.WriteCloser + + emitter *events.Bus + + logCallbackMx sync.Mutex + logCallback func([]byte) + + // st tracks the current process state. + st *system.AtomicString +} + +// New creates a new Kubernetes environment for the given server ID. +func New(id string, m *Metadata, c *environment.Configuration) (*Environment, error) { + cli, err := Client() + if err != nil { + return nil, err + } + + e := &Environment{ + Id: id, + Configuration: c, + meta: m, + client: cli, + st: system.NewAtomicString(environment.ProcessOfflineState), + emitter: events.NewBus(), + } + + return e, nil +} + +func (e *Environment) log() *log.Entry { + return log.WithField("environment", e.Type()).WithField("pod_id", e.Id) +} + +// Type returns the environment type identifier. +func (e *Environment) Type() string { + return "kubernetes" +} + +// Events returns the event bus for this environment. +func (e *Environment) Events() *events.Bus { + return e.emitter +} + +// Config returns the environment configuration. +func (e *Environment) Config() *environment.Configuration { + e.mu.RLock() + defer e.mu.RUnlock() + return e.Configuration +} + +// State returns the current process state string. +func (e *Environment) State() string { + return e.st.Load() +} + +// SetState updates the environment state and publishes a state change event. +func (e *Environment) SetState(state string) { + if state != environment.ProcessOfflineState && + state != environment.ProcessStartingState && + state != environment.ProcessRunningState && + state != environment.ProcessStoppingState { + panic(errors.New(fmt.Sprintf("invalid server state received: %s", state))) + } + + if e.State() != state { + e.st.Store(state) + e.Events().Publish(environment.StateChangeEvent, state) + } +} + +// SetLogCallback sets the callback function for container log output. +func (e *Environment) SetLogCallback(f func([]byte)) { + e.logCallbackMx.Lock() + defer e.logCallbackMx.Unlock() + e.logCallback = f +} + +// SetStopConfiguration updates the stop configuration on the fly. +func (e *Environment) SetStopConfiguration(c remote.ProcessStopConfiguration) { + e.mu.Lock() + e.meta.Stop = c + e.mu.Unlock() +} + +// SetImage updates the container image for the server. +func (e *Environment) SetImage(i string) { + e.mu.Lock() + defer e.mu.Unlock() + e.meta.Image = i +} + +// IsAttached returns whether the environment is currently attached to the Pod. +func (e *Environment) IsAttached() bool { + e.mu.RLock() + defer e.mu.RUnlock() + return e.stream != nil +} + +// setStream updates the attach stream reference. +func (e *Environment) setStream(s io.WriteCloser) { + e.mu.Lock() + e.stream = s + e.mu.Unlock() +} + +// Exists checks whether the Pod for this server exists in the cluster. +func (e *Environment) Exists() (bool, error) { + _, err := e.getPod(context.Background()) + if err != nil { + if isNotFound(err) { + return false, nil + } + return false, err + } + return true, nil +} + +// IsRunning checks whether the Pod is in Running phase. +func (e *Environment) IsRunning(ctx context.Context) (bool, error) { + pod, err := e.getPod(ctx) + if err != nil { + return false, err + } + return isPodRunning(pod), nil +} + +// ExitState returns the exit code and OOM-killed status of the terminated +// container. +func (e *Environment) ExitState() (uint32, bool, error) { + pod, err := e.getPod(context.Background()) + if err != nil { + if isNotFound(err) { + return 1, false, nil + } + return 0, false, errors.WrapIf(err, "environment/kubernetes: failed to get pod") + } + + for _, cs := range pod.Status.ContainerStatuses { + if cs.Name == "server" && cs.State.Terminated != nil { + oom := cs.State.Terminated.Reason == "OOMKilled" + return uint32(cs.State.Terminated.ExitCode), oom, nil + } + } + + return 0, false, nil +} + +// InSituUpdate is a no-op for Kubernetes since Pod resource limits are +// immutable after creation. The Pod must be recreated to apply new limits. +func (e *Environment) InSituUpdate() error { + return nil +} + +// namespace returns the configured Kubernetes namespace. +func (e *Environment) namespace() string { + ns := config.Get().Kubernetes.Namespace + if ns == "" { + return "pelican" + } + return ns +} diff --git a/environment/kubernetes/environment_test.go b/environment/kubernetes/environment_test.go new file mode 100644 index 00000000..e52a06d8 --- /dev/null +++ b/environment/kubernetes/environment_test.go @@ -0,0 +1,587 @@ +package kubernetes + +import ( + "context" + "testing" + + . "github.com/franela/goblin" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + + "github.com/pelican/wings/config" + "github.com/pelican/wings/environment" + "github.com/pelican/wings/events" + "github.com/pelican/wings/system" +) + +func TestEnvironment(t *testing.T) { + g := Goblin(t) + + g.Describe("Environment", func() { + g.Describe("Type", func() { + g.It("should return 'kubernetes'", func() { + env := &Environment{st: system.NewAtomicString(environment.ProcessOfflineState)} + g.Assert(env.Type()).Equal("kubernetes") + }) + }) + + g.Describe("State", func() { + g.It("should return the current state", func() { + env := &Environment{ + st: system.NewAtomicString(environment.ProcessOfflineState), + emitter: events.NewBus(), + } + g.Assert(env.State()).Equal(environment.ProcessOfflineState) + }) + + g.It("should update state via SetState", func() { + env := &Environment{ + st: system.NewAtomicString(environment.ProcessOfflineState), + emitter: events.NewBus(), + } + env.SetState(environment.ProcessRunningState) + g.Assert(env.State()).Equal(environment.ProcessRunningState) + }) + + g.It("should panic on invalid state", func() { + env := &Environment{ + st: system.NewAtomicString(environment.ProcessOfflineState), + emitter: events.NewBus(), + } + panicked := false + func() { + defer func() { + if r := recover(); r != nil { + panicked = true + } + }() + env.SetState("bogus") + }() + g.Assert(panicked).IsTrue() + }) + }) + + g.Describe("Exists", func() { + g.It("should return true when Pod exists", func() { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-uuid", + Namespace: "pelican", + }, + } + client := fake.NewSimpleClientset(pod) + env := &Environment{ + Id: "test-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + }) + + exists, err := env.Exists() + g.Assert(err).IsNil() + g.Assert(exists).IsTrue() + }) + + g.It("should return false when Pod does not exist", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "nonexistent-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + }) + + exists, err := env.Exists() + g.Assert(err).IsNil() + g.Assert(exists).IsFalse() + }) + }) + + g.Describe("IsRunning", func() { + g.It("should return true when Pod is in Running phase", func() { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-uuid", + Namespace: "pelican", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "server", + State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}, + }, + }, + }, + } + client := fake.NewSimpleClientset(pod) + env := &Environment{ + Id: "test-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + }) + + running, err := env.IsRunning(context.Background()) + g.Assert(err).IsNil() + g.Assert(running).IsTrue() + }) + + g.It("should return false when Pod is Pending", func() { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-uuid", + Namespace: "pelican", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodPending, + }, + } + client := fake.NewSimpleClientset(pod) + env := &Environment{ + Id: "test-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + }) + + running, err := env.IsRunning(context.Background()) + g.Assert(err).IsNil() + g.Assert(running).IsFalse() + }) + }) + + g.Describe("ExitState", func() { + g.It("should return exit code and OOM status from terminated container", func() { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-uuid", + Namespace: "pelican", + }, + Status: corev1.PodStatus{ + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "server", + State: corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ + ExitCode: 137, + Reason: "OOMKilled", + }, + }, + }, + }, + }, + } + client := fake.NewSimpleClientset(pod) + env := &Environment{ + Id: "test-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + }) + + code, oom, err := env.ExitState() + g.Assert(err).IsNil() + g.Assert(code).Equal(uint32(137)) + g.Assert(oom).IsTrue() + }) + + g.It("should return exit code 1 when Pod does not exist", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "nonexistent-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + }) + + code, oom, err := env.ExitState() + g.Assert(err).IsNil() + g.Assert(code).Equal(uint32(1)) + g.Assert(oom).IsFalse() + }) + }) + }) + + g.Describe("Create", func() { + g.It("should create a Pod with correct spec", func() { + client := fake.NewSimpleClientset() + allocs := environment.Allocations{ + Mappings: map[string][]int{"0.0.0.0": {25565}}, + } + settings := environment.Settings{ + Allocations: allocs, + Limits: environment.Limits{ + MemoryLimit: 1024, + CpuLimit: 200, + }, + Mounts: []environment.Mount{ + {Default: true, Source: "/var/lib/pelican/servers/test", Target: "/home/container"}, + }, + } + cfg := environment.NewConfiguration(settings, []string{"JAVA_OPTS=-Xmx512M"}) + env := &Environment{ + Id: "create-test-uuid", + Configuration: cfg, + meta: &Metadata{Image: "itzg/minecraft-server:latest"}, + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + emitter: events.NewBus(), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.NetworkMode = config.KubeNetworkHostPort + }) + + err := env.Create() + g.Assert(err).IsNil() + + pod, err := client.CoreV1().Pods("pelican").Get(context.Background(), "create-test-uuid", metav1.GetOptions{}) + g.Assert(err).IsNil() + g.Assert(pod.Name).Equal("create-test-uuid") + g.Assert(pod.Namespace).Equal("pelican") + + // Verify container. + g.Assert(len(pod.Spec.Containers)).Equal(1) + c := pod.Spec.Containers[0] + g.Assert(c.Name).Equal("server") + g.Assert(c.Image).Equal("itzg/minecraft-server:latest") + g.Assert(c.Stdin).IsTrue() + g.Assert(c.TTY).IsTrue() + + // Verify resource limits. + memLimit := c.Resources.Limits[corev1.ResourceMemory] + g.Assert(memLimit.Equal(resource.MustParse("1024Mi"))).IsTrue() + cpuLimit := c.Resources.Limits[corev1.ResourceCPU] + g.Assert(cpuLimit.Equal(resource.MustParse("2000m"))).IsTrue() + + // Verify environment variables. + g.Assert(len(c.Env) >= 1).IsTrue() + found := false + for _, ev := range c.Env { + if ev.Name == "JAVA_OPTS" && ev.Value == "-Xmx512M" { + found = true + } + } + g.Assert(found).IsTrue() + + // Verify volumes. + g.Assert(len(pod.Spec.Volumes)).Equal(1) + g.Assert(pod.Spec.Volumes[0].Name).Equal("server-data") + g.Assert(pod.Spec.Volumes[0].HostPath.Path).Equal("/var/lib/pelican/servers/test") + + // Verify labels. + g.Assert(pod.Labels["pelican.dev/server-id"]).Equal("create-test-uuid") + g.Assert(pod.Labels["app.kubernetes.io/managed-by"]).Equal("pelican-wings") + }) + + g.It("should not recreate existing Pod", func() { + existingPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "existing-uuid", + Namespace: "pelican", + }, + } + client := fake.NewSimpleClientset(existingPod) + allocs := environment.Allocations{} + settings := environment.Settings{ + Allocations: allocs, + Limits: environment.Limits{MemoryLimit: 512}, + } + cfg := environment.NewConfiguration(settings, nil) + env := &Environment{ + Id: "existing-uuid", + Configuration: cfg, + meta: &Metadata{Image: "nginx"}, + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + emitter: events.NewBus(), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + }) + + err := env.Create() + g.Assert(err).IsNil() + }) + + g.It("should strip ~ prefix from image", func() { + client := fake.NewSimpleClientset() + allocs := environment.Allocations{} + settings := environment.Settings{Allocations: allocs} + cfg := environment.NewConfiguration(settings, nil) + env := &Environment{ + Id: "tilde-test-uuid", + Configuration: cfg, + meta: &Metadata{Image: "~local/myimage:latest"}, + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + emitter: events.NewBus(), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + }) + + err := env.Create() + g.Assert(err).IsNil() + + pod, _ := client.CoreV1().Pods("pelican").Get(context.Background(), "tilde-test-uuid", metav1.GetOptions{}) + g.Assert(pod.Spec.Containers[0].Image).Equal("local/myimage:latest") + }) + }) + + g.Describe("Destroy", func() { + g.It("should delete the Pod and transition to offline state", func() { + existingPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "destroy-uuid", + Namespace: "pelican", + }, + } + client := fake.NewSimpleClientset(existingPod) + env := &Environment{ + Id: "destroy-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessRunningState), + emitter: events.NewBus(), + Configuration: environment.NewConfiguration(environment.Settings{}, nil), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.NetworkMode = config.KubeNetworkHostPort + }) + + err := env.Destroy() + g.Assert(err).IsNil() + g.Assert(env.State()).Equal(environment.ProcessOfflineState) + + // Pod should be gone. + _, err = client.CoreV1().Pods("pelican").Get(context.Background(), "destroy-uuid", metav1.GetOptions{}) + g.Assert(err).IsNotNil() + }) + + g.It("should not error when Pod does not exist", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "nonexistent-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessRunningState), + emitter: events.NewBus(), + Configuration: environment.NewConfiguration(environment.Settings{}, nil), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.NetworkMode = config.KubeNetworkHostPort + }) + + err := env.Destroy() + g.Assert(err).IsNil() + g.Assert(env.State()).Equal(environment.ProcessOfflineState) + }) + + g.It("should also delete the NodePort Service", func() { + existingPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "svc-destroy-uuid", + Namespace: "pelican", + }, + } + existingSvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gs-svc-destroy-uuid", + Namespace: "pelican", + }, + } + client := fake.NewSimpleClientset(existingPod, existingSvc) + env := &Environment{ + Id: "svc-destroy-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessRunningState), + emitter: events.NewBus(), + Configuration: environment.NewConfiguration(environment.Settings{}, nil), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.NetworkMode = config.KubeNetworkNodePort + }) + + err := env.Destroy() + g.Assert(err).IsNil() + + // Both Pod and Service should be gone. + _, err = client.CoreV1().Pods("pelican").Get(context.Background(), "svc-destroy-uuid", metav1.GetOptions{}) + g.Assert(err).IsNotNil() + _, err = client.CoreV1().Services("pelican").Get(context.Background(), "gs-svc-destroy-uuid", metav1.GetOptions{}) + g.Assert(err).IsNotNil() + }) + }) + + g.Describe("buildResources", func() { + g.It("should convert memory and CPU limits correctly", func() { + env := &Environment{} + limits := environment.Limits{ + MemoryLimit: 2048, + CpuLimit: 150, + } + resources := env.buildResources(limits) + + memLimit := resources.Limits[corev1.ResourceMemory] + g.Assert(memLimit.Equal(resource.MustParse("2048Mi"))).IsTrue() + + memReq := resources.Requests[corev1.ResourceMemory] + g.Assert(memReq.Equal(resource.MustParse("1024Mi"))).IsTrue() + + cpuLimit := resources.Limits[corev1.ResourceCPU] + g.Assert(cpuLimit.Equal(resource.MustParse("1500m"))).IsTrue() + + cpuReq := resources.Requests[corev1.ResourceCPU] + g.Assert(cpuReq.Equal(resource.MustParse("375m"))).IsTrue() + }) + + g.It("should set minimum CPU request of 50m", func() { + env := &Environment{} + limits := environment.Limits{ + CpuLimit: 10, // 10% = 100m limit, 25m request would be below 50m + } + resources := env.buildResources(limits) + cpuReq := resources.Requests[corev1.ResourceCPU] + g.Assert(cpuReq.Equal(resource.MustParse("50m"))).IsTrue() + }) + + g.It("should handle zero limits gracefully", func() { + env := &Environment{} + limits := environment.Limits{} + resources := env.buildResources(limits) + g.Assert(len(resources.Limits)).Equal(0) + g.Assert(len(resources.Requests)).Equal(0) + }) + }) + + g.Describe("buildVolumes", func() { + g.It("should create volumes with correct names", func() { + env := &Environment{} + mounts := []environment.Mount{ + {Default: true, Source: "/data/server1", Target: "/home/container", ReadOnly: false}, + {Default: false, Source: "/shared/plugins", Target: "/plugins", ReadOnly: true}, + } + volumes, volumeMounts := env.buildVolumes(mounts) + + g.Assert(len(volumes)).Equal(2) + g.Assert(volumes[0].Name).Equal("server-data") + g.Assert(volumes[0].HostPath.Path).Equal("/data/server1") + g.Assert(volumes[1].Name).Equal("mount-1") + g.Assert(volumes[1].HostPath.Path).Equal("/shared/plugins") + + g.Assert(len(volumeMounts)).Equal(2) + g.Assert(volumeMounts[0].MountPath).Equal("/home/container") + g.Assert(volumeMounts[0].ReadOnly).IsFalse() + g.Assert(volumeMounts[1].MountPath).Equal("/plugins") + g.Assert(volumeMounts[1].ReadOnly).IsTrue() + }) + }) + + g.Describe("buildContainerPorts", func() { + g.It("should create TCP and UDP ports for each allocation", func() { + allocs := environment.Allocations{ + Mappings: map[string][]int{"0.0.0.0": {25565, 25575}}, + } + settings := environment.Settings{Allocations: allocs} + cfg := environment.NewConfiguration(settings, nil) + env := &Environment{Configuration: cfg} + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkHostPort + }) + + ports := env.buildContainerPorts() + g.Assert(len(ports)).Equal(4) // 2 ports × (TCP + UDP) + + // Verify hostPort is set in hostport mode. + for _, p := range ports { + g.Assert(p.HostPort).Equal(p.ContainerPort) + } + }) + + g.It("should not set hostPort in nodeport mode", func() { + allocs := environment.Allocations{ + Mappings: map[string][]int{"0.0.0.0": {25565}}, + } + settings := environment.Settings{Allocations: allocs} + cfg := environment.NewConfiguration(settings, nil) + env := &Environment{Configuration: cfg} + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkNodePort + }) + + ports := env.buildContainerPorts() + g.Assert(len(ports)).Equal(2) + for _, p := range ports { + g.Assert(p.HostPort).Equal(int32(0)) + } + }) + + g.It("should skip invalid ports", func() { + allocs := environment.Allocations{ + Mappings: map[string][]int{"0.0.0.0": {0, -1, 70000, 8080}}, + } + settings := environment.Settings{Allocations: allocs} + cfg := environment.NewConfiguration(settings, nil) + env := &Environment{Configuration: cfg} + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkNodePort + }) + + ports := env.buildContainerPorts() + // Only 8080 is valid, so 2 ports (TCP + UDP). + g.Assert(len(ports)).Equal(2) + }) + + g.It("should deduplicate ports shared across allocation IPs", func() { + allocs := environment.Allocations{ + Mappings: map[string][]int{ + "1.1.1.1": {25565}, + "2.2.2.2": {25565}, + }, + } + settings := environment.Settings{Allocations: allocs} + cfg := environment.NewConfiguration(settings, nil) + env := &Environment{Configuration: cfg} + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkNodePort + }) + + ports := env.buildContainerPorts() + // 25565 appears under two IPs but must yield a single TCP + UDP pair. + g.Assert(len(ports)).Equal(2) + }) + }) +} diff --git a/environment/kubernetes/identity.go b/environment/kubernetes/identity.go new file mode 100644 index 00000000..08915d4f --- /dev/null +++ b/environment/kubernetes/identity.go @@ -0,0 +1,158 @@ +package kubernetes + +import ( + "context" + "fmt" + "os" + "strings" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/pelican/wings/config" + "github.com/pelican/wings/environment" +) + +// identityConfigMapName returns the name for the identity ConfigMap. +func (e *Environment) identityConfigMapName() string { + return fmt.Sprintf("gs-%s-identity", e.Id) +} + +// isIdentityMount returns true if the mount targets one of the well-known +// identity file paths (/etc/passwd, /etc/group, /etc/machine-id). +func isIdentityMount(m environment.Mount) bool { + switch m.Target { + case "/etc/passwd", "/etc/group", "/etc/machine-id": + return true + } + return false +} + +// splitIdentityMounts partitions mounts into identity mounts (passwd, group, +// machine-id) and everything else. In Kubernetes mode, identity mounts are +// served from a ConfigMap instead of hostPath. +func (e *Environment) splitIdentityMounts(mounts []environment.Mount) (identity, regular []environment.Mount) { + if !config.Get().Kubernetes.Enabled { + return nil, mounts + } + for _, m := range mounts { + if isIdentityMount(m) { + identity = append(identity, m) + } else { + regular = append(regular, m) + } + } + return +} + +// ensureIdentityConfigMap creates or updates a ConfigMap containing the +// identity files (passwd, group, machine-id) for this game server. It returns +// the Volume and VolumeMounts to be added to the Pod spec. +func (e *Environment) ensureIdentityConfigMap(ctx context.Context, mounts []environment.Mount) (corev1.Volume, []corev1.VolumeMount, error) { + cmName := e.identityConfigMapName() + ns := e.namespace() + + data := make(map[string]string) + var volumeMounts []corev1.VolumeMount + + for _, m := range mounts { + var key, content string + switch m.Target { + case "/etc/passwd": + key = "passwd" + content = e.readFileOrGenerate(m.Source, e.generatePasswd()) + case "/etc/group": + key = "group" + content = e.readFileOrGenerate(m.Source, e.generateGroup()) + case "/etc/machine-id": + key = "machine-id" + content = strings.ReplaceAll(e.Id, "-", "") + default: + continue + } + data[key] = content + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "identity", + MountPath: m.Target, + SubPath: key, + ReadOnly: true, + }) + } + + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cmName, + Namespace: ns, + Labels: map[string]string{ + "app.kubernetes.io/managed-by": "pelican-wings", + "pelican.dev/server-id": e.Id, + "pelican.dev/resource-type": "identity", + }, + }, + Data: data, + } + + existing, err := e.client.CoreV1().ConfigMaps(ns).Get(ctx, cmName, metav1.GetOptions{}) + if err != nil { + if !isNotFound(err) { + return corev1.Volume{}, nil, err + } + _, err = e.client.CoreV1().ConfigMaps(ns).Create(ctx, cm, metav1.CreateOptions{}) + if err != nil { + return corev1.Volume{}, nil, err + } + } else { + existing.Data = data + _, err = e.client.CoreV1().ConfigMaps(ns).Update(ctx, existing, metav1.UpdateOptions{}) + if err != nil { + return corev1.Volume{}, nil, err + } + } + + vol := corev1.Volume{ + Name: "identity", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: cmName, + }, + }, + }, + } + + return vol, volumeMounts, nil +} + +// deleteIdentityConfigMap removes the identity ConfigMap for this server. +func (e *Environment) deleteIdentityConfigMap(ctx context.Context) error { + err := e.client.CoreV1().ConfigMaps(e.namespace()).Delete(ctx, e.identityConfigMapName(), metav1.DeleteOptions{}) + if err != nil && !isNotFound(err) { + return err + } + return nil +} + +// readFileOrGenerate tries to read the file at path. If it doesn't exist +// (common on immutable OSes like Talos), returns the fallback content. +func (e *Environment) readFileOrGenerate(path, fallback string) string { + data, err := os.ReadFile(path) + if err != nil { + return fallback + } + return string(data) +} + +// generatePasswd returns a passwd file matching the format from +// config.ConfigurePasswd(). +func (e *Environment) generatePasswd() string { + cfg := config.Get() + return fmt.Sprintf("container:x:%d:%d::/home/container:/usr/sbin/nologin", + cfg.System.User.Uid, cfg.System.User.Gid) +} + +// generateGroup returns a group file matching the format from +// config.ConfigurePasswd(). +func (e *Environment) generateGroup() string { + cfg := config.Get() + return fmt.Sprintf("container:x:%d:container", cfg.System.User.Gid) +} diff --git a/environment/kubernetes/installer.go b/environment/kubernetes/installer.go new file mode 100644 index 00000000..29f54b48 --- /dev/null +++ b/environment/kubernetes/installer.go @@ -0,0 +1,425 @@ +package kubernetes + +import ( + "bufio" + "context" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "emperror.dev/errors" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + "github.com/pelican/wings/config" + "github.com/pelican/wings/remote" + "github.com/pelican/wings/system" +) + +// InstallerProcess handles running server installation scripts as Kubernetes +// Jobs. It mirrors the Docker-based InstallationProcess but uses Jobs instead +// of standalone containers. +type InstallerProcess struct { + ServerID string + Script *remote.InstallationScript + EnvVars []string + ServerPath string + TmpDir string + Sink *system.SinkPool + client kubernetes.Interface + namespace string +} + +// NewInstallerProcess creates a new Kubernetes-based installer process for a +// server. The caller is responsible for writing the install script to tmpDir. +func NewInstallerProcess(env *Environment, script *remote.InstallationScript, envVars []string, serverPath, tmpDir string, sink *system.SinkPool) *InstallerProcess { + return &InstallerProcess{ + ServerID: env.Id, + Script: script, + EnvVars: envVars, + ServerPath: serverPath, + TmpDir: tmpDir, + Sink: sink, + client: env.client, + namespace: env.namespace(), + } +} + +// jobName returns the name for the installation Job. +func (ip *InstallerProcess) jobName() string { + return fmt.Sprintf("%s-installer", ip.ServerID) +} + +// configMapName returns the name for the install script ConfigMap. +func (ip *InstallerProcess) configMapName() string { + return fmt.Sprintf("%s-install-script", ip.ServerID) +} + +// Run executes the full installation process: creates a Job, streams its logs, +// waits for completion, and cleans up. +func (ip *InstallerProcess) Run(ctx context.Context) error { + // Clean up any existing installer Job from a previous run. Use foreground + // propagation and wait for the Job to disappear so the Create below does not + // race a still-terminating Job and fail with AlreadyExists. + if err := ip.client.BatchV1().Jobs(ip.namespace).Delete(ctx, ip.jobName(), metav1.DeleteOptions{ + PropagationPolicy: propagationForeground(), + }); err != nil && !isNotFound(err) { + return errors.Wrap(err, "environment/kubernetes: failed to delete existing installer job") + } + if err := ip.waitForJobDeletion(ctx, 30*time.Second); err != nil { + return errors.Wrap(err, "environment/kubernetes: timed out waiting for old installer job deletion") + } + + // Build environment variables. + var envVars []corev1.EnvVar + for _, ev := range ip.EnvVars { + parts := strings.SplitN(ev, "=", 2) + if len(parts) == 2 { + envVars = append(envVars, corev1.EnvVar{ + Name: parts[0], + Value: parts[1], + }) + } + } + + cfg := config.Get() + + // Determine install image and pull policy (mirrors the Docker backend). + installImage, installPullPolicy := resolveImagePullPolicy(ip.Script.ContainerImage) + + // Build the Job spec. + backoffLimit := int32(0) + ttl := int32(300) // Auto-clean Job after 5 minutes. + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: ip.jobName(), + Namespace: ip.namespace, + Labels: map[string]string{ + "app.kubernetes.io/managed-by": "pelican-wings", + "pelican.dev/server-id": ip.ServerID, + "pelican.dev/container-type": "server_installer", + }, + }, + Spec: batchv1.JobSpec{ + BackoffLimit: &backoffLimit, + TTLSecondsAfterFinished: &ttl, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app.kubernetes.io/managed-by": "pelican-wings", + "pelican.dev/server-id": ip.ServerID, + "pelican.dev/container-type": "server_installer", + "job-name": ip.jobName(), + }, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + Containers: []corev1.Container{ + { + Name: "installer", + Image: installImage, + Command: []string{ip.Script.Entrypoint, "/mnt/install/install.sh"}, + Env: envVars, + VolumeMounts: []corev1.VolumeMount{ + ip.buildServerDataMount(), + { + Name: "install-script", + MountPath: "/mnt/install", + }, + }, + ImagePullPolicy: installPullPolicy, + }, + }, + Volumes: ip.buildJobVolumes(cfg), + }, + }, + }, + } + + // Apply node selector from config. + if len(cfg.Kubernetes.NodeSelector) > 0 { + job.Spec.Template.Spec.NodeSelector = cfg.Kubernetes.NodeSelector + } + + // Apply service account. + if cfg.Kubernetes.ServiceAccount != "" { + job.Spec.Template.Spec.ServiceAccountName = cfg.Kubernetes.ServiceAccount + } + + // Apply image pull secrets. + for _, secret := range cfg.Kubernetes.ImagePullSecrets { + job.Spec.Template.Spec.ImagePullSecrets = append( + job.Spec.Template.Spec.ImagePullSecrets, + corev1.LocalObjectReference{Name: secret}, + ) + } + + // Apply tolerations. + for _, t := range cfg.Kubernetes.Tolerations { + toleration := corev1.Toleration{ + Key: t.Key, + Operator: corev1.TolerationOperator(t.Operator), + Value: t.Value, + Effect: corev1.TaintEffect(t.Effect), + } + if t.TolerationSeconds != nil { + toleration.TolerationSeconds = t.TolerationSeconds + } + job.Spec.Template.Spec.Tolerations = append(job.Spec.Template.Spec.Tolerations, toleration) + } + + // Create the Job. + if _, err := ip.client.BatchV1().Jobs(ip.namespace).Create(ctx, job, metav1.CreateOptions{}); err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to create installer job") + } + + // Stream logs in the background. + go ip.streamJobLogs(ctx) + + // Wait for the Job to complete. + if err := ip.waitForJob(ctx); err != nil { + return err + } + + return nil +} + +// streamJobLogs finds the Pod created by the Job and streams its logs to the +// install sink. +func (ip *InstallerProcess) streamJobLogs(ctx context.Context) { + // Wait a moment for the Pod to be created by the Job controller. + time.Sleep(2 * time.Second) + + for i := 0; i < 30; i++ { + pods, err := ip.client.CoreV1().Pods(ip.namespace).List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("job-name=%s", ip.jobName()), + }) + if err != nil || len(pods.Items) == 0 { + time.Sleep(2 * time.Second) + continue + } + + pod := pods.Items[0] + // Wait for the pod to be running or completed. + if pod.Status.Phase == corev1.PodPending { + time.Sleep(2 * time.Second) + continue + } + + follow := pod.Status.Phase == corev1.PodRunning + stream, err := ip.client.CoreV1().Pods(ip.namespace).GetLogs(pod.Name, &corev1.PodLogOptions{ + Container: "installer", + Follow: follow, + }).Stream(ctx) + if err != nil { + return + } + defer stream.Close() + + scanner := bufio.NewScanner(stream) + for scanner.Scan() { + line := scanner.Text() + if ip.Sink != nil { + ip.Sink.Push([]byte(line)) + } + } + return + } +} + +// waitForJob polls the Job until it succeeds, fails, or the context is +// canceled. +func (ip *InstallerProcess) waitForJob(ctx context.Context) error { + timeout := 30 * time.Minute + deadline := time.After(timeout) + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-deadline: + return errors.New("environment/kubernetes: installer job timed out after 30 minutes") + case <-ticker.C: + job, err := ip.client.BatchV1().Jobs(ip.namespace).Get(ctx, ip.jobName(), metav1.GetOptions{}) + if err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to get installer job status") + } + + // Check for completion. + if job.Status.Succeeded > 0 { + return nil + } + + // Check for failure. + if job.Status.Failed > 0 { + return errors.New("environment/kubernetes: installer job failed") + } + } + } +} + +// Cleanup removes the installer Job, ConfigMap, and associated resources. +func (ip *InstallerProcess) Cleanup(ctx context.Context) error { + err := ip.client.BatchV1().Jobs(ip.namespace).Delete(ctx, ip.jobName(), metav1.DeleteOptions{ + PropagationPolicy: propagationBackground(), + }) + if err != nil && !isNotFound(err) { + return errors.Wrap(err, "environment/kubernetes: failed to delete installer job") + } + + // Delete the install script ConfigMap. + cmErr := ip.client.CoreV1().ConfigMaps(ip.namespace).Delete(ctx, ip.configMapName(), metav1.DeleteOptions{}) + if cmErr != nil && !isNotFound(cmErr) { + return errors.Wrap(cmErr, "environment/kubernetes: failed to delete install script configmap") + } + + // Remove temporary install script directory (legacy cleanup). + if ip.TmpDir != "" { + os.RemoveAll(ip.TmpDir) + } + + return nil +} + +// WriteInstallScript creates a ConfigMap containing the installation script. +// The ConfigMap is mounted into the Job Pod at /mnt/install, eliminating the +// need for a shared filesystem between Wings and the Job Pod. +func (ip *InstallerProcess) WriteInstallScript(ctx context.Context) error { + content := strings.ReplaceAll(ip.Script.Script, "\r\n", "\n") + + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: ip.configMapName(), + Namespace: ip.namespace, + Labels: map[string]string{ + "app.kubernetes.io/managed-by": "pelican-wings", + "pelican.dev/server-id": ip.ServerID, + "pelican.dev/container-type": "server_installer", + }, + }, + Data: map[string]string{ + "install.sh": content, + }, + } + + // Delete any existing ConfigMap from a previous run. + _ = ip.client.CoreV1().ConfigMaps(ip.namespace).Delete(ctx, ip.configMapName(), metav1.DeleteOptions{}) + + _, err := ip.client.CoreV1().ConfigMaps(ip.namespace).Create(ctx, cm, metav1.CreateOptions{}) + if err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to create install script configmap") + } + return nil +} + +// buildJobVolumes constructs the volume list for the installer Job. When +// DataPVC is set, the shared Wings data PVC is used. Otherwise falls back to +// per-server PVC (StorageMode "pvc") or HostPath. The install script is always +// mounted from a ConfigMap. +func (ip *InstallerProcess) buildJobVolumes(cfg *config.Configuration) []corev1.Volume { + var serverDataSource corev1.VolumeSource + if cfg.Kubernetes.DataPVC != "" { + serverDataSource = corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: cfg.Kubernetes.DataPVC, + }, + } + } else if cfg.Kubernetes.StorageMode == config.KubeStoragePVC { + serverDataSource = corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: fmt.Sprintf("gs-%s", ip.ServerID), + }, + } + } else { + serverDataSource = corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: ip.ServerPath, + }, + } + } + + // Script volume uses ConfigMap — works across multi-node clusters. + defaultMode := int32(0755) + + return []corev1.Volume{ + { + Name: "server-data", + VolumeSource: serverDataSource, + }, + { + Name: "install-script", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: ip.configMapName(), + }, + DefaultMode: &defaultMode, + }, + }, + }, + } +} + +// buildServerDataMount returns the VolumeMount for the server-data volume. +// When DataPVC is set, the mount includes a subPath to the server's data +// directory within the shared PVC. +func (ip *InstallerProcess) buildServerDataMount() corev1.VolumeMount { + vm := corev1.VolumeMount{ + Name: "server-data", + MountPath: "/mnt/server", + } + cfg := config.Get() + if cfg.Kubernetes.DataPVC != "" { + serverPath := filepath.Join(cfg.System.Data, ip.ServerID) + rel, err := filepath.Rel(cfg.System.RootDirectory, serverPath) + // filepath.Rel can yield a path escaping the volume root ("..") when + // System.Data is outside RootDirectory; such a SubPath is rejected by + // Kubernetes, so fall back to the default layout in that case. + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + vm.SubPath = filepath.Join("volumes", ip.ServerID) + } else { + vm.SubPath = rel + } + } + return vm +} + +// waitForJobDeletion blocks until the installer Job is fully deleted or the +// timeout elapses. +func (ip *InstallerProcess) waitForJobDeletion(ctx context.Context, timeout time.Duration) error { + dctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + for { + _, err := ip.client.BatchV1().Jobs(ip.namespace).Get(dctx, ip.jobName(), metav1.GetOptions{}) + if err != nil && isNotFound(err) { + return nil + } + select { + case <-dctx.Done(): + return dctx.Err() + case <-ticker.C: + } + } +} + +// propagationForeground returns a pointer to the Foreground propagation policy. +func propagationForeground() *metav1.DeletionPropagation { + p := metav1.DeletePropagationForeground + return &p +} + +// propagationBackground returns a pointer to the Background propagation policy. +func propagationBackground() *metav1.DeletionPropagation { + p := metav1.DeletePropagationBackground + return &p +} diff --git a/environment/kubernetes/installer_test.go b/environment/kubernetes/installer_test.go new file mode 100644 index 00000000..caf8754e --- /dev/null +++ b/environment/kubernetes/installer_test.go @@ -0,0 +1,367 @@ +package kubernetes + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + "time" + + . "github.com/franela/goblin" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + + "github.com/pelican/wings/config" + "github.com/pelican/wings/environment" + "github.com/pelican/wings/remote" + "github.com/pelican/wings/system" +) + +func int32Ptr(i int32) *int32 { return &i } + +func TestInstaller(t *testing.T) { + g := Goblin(t) + + g.Describe("InstallerProcess", func() { + g.Describe("jobName", func() { + g.It("should return correct job name", func() { + ip := &InstallerProcess{ServerID: "abc-123-def"} + g.Assert(ip.jobName()).Equal("abc-123-def-installer") + }) + }) + + g.Describe("configMapName", func() { + g.It("should return correct configmap name", func() { + ip := &InstallerProcess{ServerID: "abc-123-def"} + g.Assert(ip.configMapName()).Equal("abc-123-def-install-script") + }) + }) + + g.Describe("WriteInstallScript", func() { + g.It("should create a ConfigMap with the script content", func() { + client := fake.NewSimpleClientset() + ip := &InstallerProcess{ + ServerID: "write-cm-uuid", + client: client, + namespace: "pelican", + Script: &remote.InstallationScript{ + Script: "#!/bin/bash\necho hello\r\necho world", + }, + } + + err := ip.WriteInstallScript(context.Background()) + g.Assert(err).IsNil() + + cm, err := client.CoreV1().ConfigMaps("pelican").Get(context.Background(), "write-cm-uuid-install-script", metav1.GetOptions{}) + g.Assert(err).IsNil() + // Should replace \r\n with \n. + g.Assert(cm.Data["install.sh"]).Equal("#!/bin/bash\necho hello\necho world") + g.Assert(cm.Labels["pelican.dev/server-id"]).Equal("write-cm-uuid") + }) + + g.It("should replace existing ConfigMap on re-run", func() { + existingCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rewrite-uuid-install-script", + Namespace: "pelican", + }, + Data: map[string]string{"install.sh": "old script"}, + } + client := fake.NewSimpleClientset(existingCM) + ip := &InstallerProcess{ + ServerID: "rewrite-uuid", + client: client, + namespace: "pelican", + Script: &remote.InstallationScript{ + Script: "new script", + }, + } + + err := ip.WriteInstallScript(context.Background()) + g.Assert(err).IsNil() + + cm, err := client.CoreV1().ConfigMaps("pelican").Get(context.Background(), "rewrite-uuid-install-script", metav1.GetOptions{}) + g.Assert(err).IsNil() + g.Assert(cm.Data["install.sh"]).Equal("new script") + }) + }) + + g.Describe("Run", func() { + g.It("should create a Job with correct spec", func() { + client := fake.NewSimpleClientset() + ip := &InstallerProcess{ + ServerID: "test-server-uuid", + client: client, + namespace: "pelican", + ServerPath: "/var/lib/pelican/servers/test-server-uuid", + TmpDir: "/tmp/test-server-uuid", + EnvVars: []string{"SERVER_MEMORY=1024", "SERVER_IP=0.0.0.0"}, + Script: &remote.InstallationScript{ + ContainerImage: "ghcr.io/pelican/installers:alpine", + Entrypoint: "bash", + Script: "echo installing", + }, + Sink: system.NewSinkPool(), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.NodeSelector = map[string]string{"role": "game"} + c.Kubernetes.ServiceAccount = "pelican-wings" + }) + + // Run will create the Job but waitForJob will keep polling. + // We'll cancel the context to stop it. + ctx, cancel := context.WithCancel(context.Background()) + + // Run in goroutine since it will block waiting for Job completion. + errCh := make(chan error, 1) + go func() { + errCh <- ip.Run(ctx) + }() + + // Poll until Job exists or timeout. + var job *batchv1.Job + for i := 0; i < 50; i++ { + var err error + job, err = client.BatchV1().Jobs("pelican").Get(context.Background(), "test-server-uuid-installer", metav1.GetOptions{}) + if err == nil { + break + } + time.Sleep(10 * time.Millisecond) + } + g.Assert(job).IsNotNil() + + // Verify Job metadata. + g.Assert(job.Name).Equal("test-server-uuid-installer") + g.Assert(job.Namespace).Equal("pelican") + g.Assert(job.Labels["pelican.dev/server-id"]).Equal("test-server-uuid") + g.Assert(job.Labels["pelican.dev/container-type"]).Equal("server_installer") + + // Verify Pod template spec. + podSpec := job.Spec.Template.Spec + g.Assert(podSpec.RestartPolicy).Equal(corev1.RestartPolicyNever) + g.Assert(len(podSpec.Containers)).Equal(1) + + container := podSpec.Containers[0] + g.Assert(container.Name).Equal("installer") + g.Assert(container.Image).Equal("ghcr.io/pelican/installers:alpine") + g.Assert(container.Command).Equal([]string{"bash", "/mnt/install/install.sh"}) + + // Verify env vars. + g.Assert(len(container.Env)).Equal(2) + g.Assert(container.Env[0].Name).Equal("SERVER_MEMORY") + g.Assert(container.Env[0].Value).Equal("1024") + g.Assert(container.Env[1].Name).Equal("SERVER_IP") + g.Assert(container.Env[1].Value).Equal("0.0.0.0") + + // Verify volume mounts. + g.Assert(len(container.VolumeMounts)).Equal(2) + g.Assert(container.VolumeMounts[0].MountPath).Equal("/mnt/server") + g.Assert(container.VolumeMounts[1].MountPath).Equal("/mnt/install") + + // Verify volumes. + g.Assert(len(podSpec.Volumes)).Equal(2) + g.Assert(podSpec.Volumes[0].HostPath.Path).Equal("/var/lib/pelican/servers/test-server-uuid") + // Install script now uses ConfigMap. + g.Assert(podSpec.Volumes[1].ConfigMap).IsNotNil() + g.Assert(podSpec.Volumes[1].ConfigMap.Name).Equal("test-server-uuid-install-script") + g.Assert(*podSpec.Volumes[1].ConfigMap.DefaultMode).Equal(int32(0755)) + + // Verify node selector. + g.Assert(podSpec.NodeSelector["role"]).Equal("game") + + // Verify service account. + g.Assert(podSpec.ServiceAccountName).Equal("pelican-wings") + + // Verify backoff limit. + g.Assert(*job.Spec.BackoffLimit).Equal(int32(0)) + + // Cancel to stop waitForJob and assert the cancellation propagates. + cancel() + g.Assert(errors.Is(<-errCh, context.Canceled)).IsTrue() + }) + + g.It("should succeed when Job completes successfully", func() { + // Pre-create a Job that is already succeeded. + completedJob := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: "completed-uuid-installer", + Namespace: "pelican", + }, + Status: batchv1.JobStatus{ + Succeeded: 1, + }, + } + client := fake.NewSimpleClientset(completedJob) + ip := &InstallerProcess{ + ServerID: "completed-uuid", + client: client, + namespace: "pelican", + EnvVars: []string{}, + Script: &remote.InstallationScript{ + ContainerImage: "alpine", + Entrypoint: "sh", + Script: "echo done", + }, + Sink: system.NewSinkPool(), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + }) + + // The fake client will return the existing Job (already succeeded) + // when Run queries for it after creating a new one. + // However the fake client won't auto-set status. Let's test Cleanup instead. + err := ip.Cleanup(context.Background()) + g.Assert(err).IsNil() + + // Verify the job was deleted. + _, err = client.BatchV1().Jobs("pelican").Get(context.Background(), "completed-uuid-installer", metav1.GetOptions{}) + g.Assert(err).IsNotNil() + }) + }) + + g.Describe("Cleanup", func() { + g.It("should delete the Job and ConfigMap", func() { + existingJob := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cleanup-uuid-installer", + Namespace: "pelican", + }, + } + existingCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cleanup-uuid-install-script", + Namespace: "pelican", + }, + } + client := fake.NewSimpleClientset(existingJob, existingCM) + ip := &InstallerProcess{ + ServerID: "cleanup-uuid", + client: client, + namespace: "pelican", + } + + err := ip.Cleanup(context.Background()) + g.Assert(err).IsNil() + + _, err = client.BatchV1().Jobs("pelican").Get(context.Background(), "cleanup-uuid-installer", metav1.GetOptions{}) + g.Assert(err).IsNotNil() + + _, err = client.CoreV1().ConfigMaps("pelican").Get(context.Background(), "cleanup-uuid-install-script", metav1.GetOptions{}) + g.Assert(err).IsNotNil() + }) + + g.It("should not error when Job and ConfigMap do not exist", func() { + client := fake.NewSimpleClientset() + ip := &InstallerProcess{ + ServerID: "nonexistent-uuid", + client: client, + namespace: "pelican", + } + + err := ip.Cleanup(context.Background()) + g.Assert(err).IsNil() + }) + + g.It("should remove temp directory", func() { + tmpDir := filepath.Join(os.TempDir(), "test-installer-cleanup") + os.MkdirAll(tmpDir, 0o700) + os.WriteFile(filepath.Join(tmpDir, "install.sh"), []byte("echo hi"), 0o644) + + client := fake.NewSimpleClientset() + ip := &InstallerProcess{ + ServerID: "tmpdir-uuid", + client: client, + namespace: "pelican", + TmpDir: tmpDir, + } + + err := ip.Cleanup(context.Background()) + g.Assert(err).IsNil() + + _, err = os.Stat(tmpDir) + g.Assert(os.IsNotExist(err)).IsTrue() + }) + }) + + g.Describe("buildJobVolumes", func() { + g.It("should use HostPath for server-data in hostpath mode and ConfigMap for install script", func() { + ip := &InstallerProcess{ + ServerID: "vol-hp-uuid", + ServerPath: "/data/servers/vol-hp-uuid", + } + config.Update(func(c *config.Configuration) { + c.Kubernetes.StorageMode = config.KubeStorageHostPath + }) + cfg := config.Get() + volumes := ip.buildJobVolumes(cfg) + g.Assert(len(volumes)).Equal(2) + g.Assert(volumes[0].Name).Equal("server-data") + g.Assert(volumes[0].HostPath).IsNotNil() + g.Assert(volumes[0].HostPath.Path).Equal("/data/servers/vol-hp-uuid") + g.Assert(volumes[0].PersistentVolumeClaim == nil).IsTrue() + // Install script uses ConfigMap. + g.Assert(volumes[1].Name).Equal("install-script") + g.Assert(volumes[1].ConfigMap).IsNotNil() + g.Assert(volumes[1].ConfigMap.Name).Equal("vol-hp-uuid-install-script") + g.Assert(*volumes[1].ConfigMap.DefaultMode).Equal(int32(0755)) + }) + + g.It("should use PVC for server-data in pvc mode and ConfigMap for install script", func() { + ip := &InstallerProcess{ + ServerID: "vol-pvc-uuid", + ServerPath: "/data/servers/vol-pvc-uuid", + } + config.Update(func(c *config.Configuration) { + c.Kubernetes.StorageMode = config.KubeStoragePVC + }) + cfg := config.Get() + volumes := ip.buildJobVolumes(cfg) + g.Assert(len(volumes)).Equal(2) + g.Assert(volumes[0].Name).Equal("server-data") + g.Assert(volumes[0].PersistentVolumeClaim).IsNotNil() + g.Assert(volumes[0].PersistentVolumeClaim.ClaimName).Equal("gs-vol-pvc-uuid") + // Install script uses ConfigMap. + g.Assert(volumes[1].ConfigMap).IsNotNil() + g.Assert(volumes[1].ConfigMap.Name).Equal("vol-pvc-uuid-install-script") + }) + }) + + g.Describe("NewInstallerProcess", func() { + g.It("should initialize all fields from Environment", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "init-test-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + }) + + script := &remote.InstallationScript{ + ContainerImage: "alpine:latest", + Entrypoint: "sh", + Script: "echo test", + } + sink := system.NewSinkPool() + + ip := NewInstallerProcess(env, script, []string{"FOO=bar"}, "/data/servers/test", "/tmp/install", sink) + + g.Assert(ip.ServerID).Equal("init-test-uuid") + g.Assert(ip.namespace).Equal("pelican") + g.Assert(ip.Script.ContainerImage).Equal("alpine:latest") + g.Assert(ip.EnvVars[0]).Equal("FOO=bar") + g.Assert(ip.ServerPath).Equal("/data/servers/test") + g.Assert(ip.TmpDir).Equal("/tmp/install") + }) + }) + }) +} diff --git a/environment/kubernetes/network.go b/environment/kubernetes/network.go new file mode 100644 index 00000000..bf0642a1 --- /dev/null +++ b/environment/kubernetes/network.go @@ -0,0 +1,368 @@ +package kubernetes + +import ( + "context" + "fmt" + "strings" + "time" + + "emperror.dev/errors" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + + "github.com/pelican/wings/config" +) + +// serviceName returns the Kubernetes Service name for this server. +func (e *Environment) serviceName() string { + return fmt.Sprintf("gs-%s", e.Id) +} + +// EnsureService creates or updates the Kubernetes Service that exposes game +// server ports. In NodePort mode, ports are exposed via auto-assigned NodePorts. +// In LoadBalancer mode, each server gets a dedicated external IP with 1:1 port +// mapping. In HostPort mode this is a no-op. +func (e *Environment) EnsureService(ctx context.Context) error { + cfg := config.Get() + if cfg.Kubernetes.NetworkMode == config.KubeNetworkHostPort { + return nil + } + + allocs := e.Configuration.Allocations() + if len(allocs.Mappings) == 0 { + return nil + } + + svcName := e.serviceName() + ns := e.namespace() + + isLB := cfg.Kubernetes.NetworkMode == config.KubeNetworkLoadBalancer + + // Build the Service port list from allocations. A port may appear under + // multiple allocation IPs, but the Service only needs it once. + seenPorts := make(map[int]bool) + var servicePorts []corev1.ServicePort + for _, ports := range allocs.Mappings { + for _, port := range ports { + if port < 1 || port > 65535 || seenPorts[port] { + continue + } + seenPorts[port] = true + + tcp := corev1.ServicePort{ + Name: portName("tcp", port), + Protocol: corev1.ProtocolTCP, + Port: int32(port), + TargetPort: intstr.FromInt32(int32(port)), + } + udp := corev1.ServicePort{ + Name: portName("udp", port), + Protocol: corev1.ProtocolUDP, + Port: int32(port), + TargetPort: intstr.FromInt32(int32(port)), + } + + if !isLB { + tcp.NodePort = e.resolveNodePort(cfg, port) + udp.NodePort = e.resolveNodePort(cfg, port) + } + + servicePorts = append(servicePorts, tcp, udp) + } + } + + if len(servicePorts) == 0 { + return nil + } + + // Labels to match the Pod. + selector := map[string]string{ + "pelican.dev/server-id": e.Id, + } + + labels := map[string]string{ + "app.kubernetes.io/managed-by": "pelican-wings", + "pelican.dev/server-id": e.Id, + "pelican.dev/resource-type": "service", + } + + svcType := corev1.ServiceTypeNodePort + var annotations map[string]string + if isLB { + svcType = corev1.ServiceTypeLoadBalancer + annotations = make(map[string]string) + for k, v := range cfg.Kubernetes.LBAnnotations { + annotations[k] = v + } + + // Auto-set IP-pinning and sharing-key annotations from the + // server's allocation IP so the LB is bound to the IP the user + // selected in the Panel. + if allocIP := e.allocationIP(); allocIP != "" { + if cfg.Kubernetes.LBIPAnnotation != "" { + annotations[cfg.Kubernetes.LBIPAnnotation] = allocIP + } + if cfg.Kubernetes.LBSharingKey != "" { + annotations[cfg.Kubernetes.LBSharingKey] = allocIP + } + } + } + + desired := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: svcName, + Namespace: ns, + Labels: labels, + Annotations: annotations, + }, + Spec: corev1.ServiceSpec{ + Type: svcType, + Selector: selector, + Ports: servicePorts, + }, + } + + // Check if the Service already exists. + existing, err := e.client.CoreV1().Services(ns).Get(ctx, svcName, metav1.GetOptions{}) + if err != nil { + if !isNotFound(err) { + return errors.Wrap(err, "environment/kubernetes: failed to get service") + } + // Service does not exist; create it. + e.log().WithField("service", svcName).Infof("creating %s service for server", svcType) + _, err = e.client.CoreV1().Services(ns).Create(ctx, desired, metav1.CreateOptions{}) + if err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to create service") + } + return nil + } + + // Service exists; update it with the desired spec while preserving + // existing NodePort assignments where possible. Type and annotations must + // also be reconciled so a networkMode switch (e.g. NodePort <-> LoadBalancer) + // is applied and stale LB annotations are cleared. + existing.Spec.Type = desired.Spec.Type + existing.Spec.Selector = desired.Spec.Selector + existing.Spec.Ports = mergeServicePorts(existing.Spec.Ports, desired.Spec.Ports) + existing.Labels = labels + existing.Annotations = annotations + + e.log().WithField("service", svcName).Infof("updating %s service for server", svcType) + _, err = e.client.CoreV1().Services(ns).Update(ctx, existing, metav1.UpdateOptions{}) + if err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to update service") + } + + return nil +} + +// DeleteService removes the Kubernetes Service associated with this server. +func (e *Environment) DeleteService(ctx context.Context) error { + cfg := config.Get() + if cfg.Kubernetes.NetworkMode == config.KubeNetworkHostPort { + return nil + } + + svcName := e.serviceName() + ns := e.namespace() + + err := e.client.CoreV1().Services(ns).Delete(ctx, svcName, metav1.DeleteOptions{}) + if err != nil && !isNotFound(err) { + return errors.Wrap(err, "environment/kubernetes: failed to delete service") + } + + e.log().WithField("service", svcName).Debug("deleted service for server") + return nil +} + +// GetServiceNodePorts returns a map of container port → assigned NodePort for +// this server's Service. Returns nil if no Service exists or we're in HostPort +// mode. +func (e *Environment) GetServiceNodePorts(ctx context.Context) (map[int32]int32, error) { + cfg := config.Get() + if cfg.Kubernetes.NetworkMode == config.KubeNetworkHostPort { + return nil, nil + } + + svc, err := e.client.CoreV1().Services(e.namespace()).Get(ctx, e.serviceName(), metav1.GetOptions{}) + if err != nil { + if isNotFound(err) { + return nil, nil + } + return nil, errors.Wrap(err, "environment/kubernetes: failed to get service") + } + + result := make(map[int32]int32) + for _, p := range svc.Spec.Ports { + if p.NodePort > 0 { + result[p.Port] = p.NodePort + } + } + return result, nil +} + +// GetServiceExternalIP returns the external IP assigned to this server's +// LoadBalancer Service. Returns an empty string if no IP is assigned yet, +// or if the network mode is not LoadBalancer. +func (e *Environment) GetServiceExternalIP(ctx context.Context) (string, error) { + cfg := config.Get() + if cfg.Kubernetes.NetworkMode != config.KubeNetworkLoadBalancer { + return "", nil + } + + svc, err := e.client.CoreV1().Services(e.namespace()).Get(ctx, e.serviceName(), metav1.GetOptions{}) + if err != nil { + if isNotFound(err) { + return "", nil + } + return "", errors.Wrap(err, "environment/kubernetes: failed to get service") + } + + for _, ingress := range svc.Status.LoadBalancer.Ingress { + if ingress.IP != "" { + return ingress.IP, nil + } + if ingress.Hostname != "" { + return ingress.Hostname, nil + } + } + return "", nil +} + +// WaitForLoadBalancerIP polls the Service until an external IP is assigned by +// the load balancer provisioner, or until the timeout (2 minutes) is reached. +func (e *Environment) WaitForLoadBalancerIP(ctx context.Context) (string, error) { + cfg := config.Get() + if cfg.Kubernetes.NetworkMode != config.KubeNetworkLoadBalancer { + return "", nil + } + + timeout := 2 * time.Minute + interval := 3 * time.Second + deadline := time.Now().Add(timeout) + + for time.Now().Before(deadline) { + ip, err := e.GetServiceExternalIP(ctx) + if err != nil { + return "", err + } + if ip != "" { + e.log().WithField("external_ip", ip).Info("load balancer IP assigned") + return ip, nil + } + + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-time.After(interval): + } + } + + e.log().Warn("timed out waiting for load balancer IP assignment") + return "", nil +} + +// resolveNodePort determines the NodePort to request for a given game server +// port. If PreserveNodePorts is enabled and the port falls within the +// Kubernetes NodePort range (default 30000-32767), use it directly. Otherwise, +// return 0 to let Kubernetes assign automatically. +func (e *Environment) resolveNodePort(cfg *config.Configuration, port int) int32 { + if !cfg.Kubernetes.NodePortPreserve { + return 0 + } + // Only request a specific NodePort if the port is within the valid range. + min := int(cfg.Kubernetes.NodePortRangeMin) + max := int(cfg.Kubernetes.NodePortRangeMax) + if min == 0 { + min = 30000 + } + if max == 0 { + max = 32767 + } + if port >= min && port <= max { + return int32(port) + } + return 0 +} + +// mergeServicePorts merges desired ports into existing ports, preserving +// assigned NodePorts for ports that haven't changed. +func mergeServicePorts(existing, desired []corev1.ServicePort) []corev1.ServicePort { + existingByKey := make(map[string]corev1.ServicePort) + for _, p := range existing { + key := fmt.Sprintf("%s/%s", p.Name, p.Protocol) + existingByKey[key] = p + } + + var merged []corev1.ServicePort + for _, d := range desired { + key := fmt.Sprintf("%s/%s", d.Name, d.Protocol) + if ex, ok := existingByKey[key]; ok { + // Preserve the existing NodePort if the target hasn't changed and + // we didn't request a specific one. + if d.NodePort == 0 && ex.NodePort > 0 && ex.TargetPort.IntValue() == d.TargetPort.IntValue() { + d.NodePort = ex.NodePort + } + } + merged = append(merged, d) + } + + return merged +} + +// sanitizePortName ensures a Kubernetes Service port name is valid (lowercase, +// alphanumeric, hyphens, max 15 chars, must start/end with alphanumeric). +func sanitizePortName(name string) string { + name = strings.ToLower(name) + name = strings.ReplaceAll(name, ".", "-") + name = strings.ReplaceAll(name, ":", "-") + + // Remove invalid characters. + var cleaned []byte + for i, c := range []byte(name) { + if (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || (c == '-' && i > 0) { + cleaned = append(cleaned, c) + } + } + name = string(cleaned) + + // Trim trailing hyphens. + name = strings.TrimRight(name, "-") + + // Max 15 characters for port names. + if len(name) > 15 { + name = name[:15] + } + + // Must not end with hyphen after truncation. + name = strings.TrimRight(name, "-") + + if name == "" { + name = "port" + } + + return name +} + +// portName builds a unique, K8s-valid Service port name. The format is +// "{proto}-{port}" (e.g. "tcp-27015"), which fits within the 15-char limit +// and avoids collisions that occur when long IP strings are truncated. +func portName(proto string, port int) string { + return sanitizePortName(fmt.Sprintf("%s-%d", proto, port)) +} + +// allocationIP returns the server's default allocation IP if it is a usable +// public/external address. Returns empty for 0.0.0.0, 127.0.0.1, or when +// no default allocation is configured. +func (e *Environment) allocationIP() string { + allocs := e.Configuration.Allocations() + if allocs.DefaultMapping == nil { + return "" + } + ip := allocs.DefaultMapping.Ip + if ip == "" || ip == "0.0.0.0" || ip == "127.0.0.1" { + return "" + } + return ip +} diff --git a/environment/kubernetes/network_test.go b/environment/kubernetes/network_test.go new file mode 100644 index 00000000..dd904c3e --- /dev/null +++ b/environment/kubernetes/network_test.go @@ -0,0 +1,512 @@ +package kubernetes + +import ( + "context" + "testing" + + . "github.com/franela/goblin" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + + "github.com/pelican/wings/config" + "github.com/pelican/wings/environment" + "github.com/pelican/wings/events" + "github.com/pelican/wings/system" +) + +func newTestEnv(client *fake.Clientset, allocs environment.Allocations) *Environment { + settings := environment.Settings{ + Allocations: allocs, + Limits: environment.Limits{ + MemoryLimit: 512, + CpuLimit: 100, + }, + } + cfg := environment.NewConfiguration(settings, []string{"FOO=bar"}) + + return &Environment{ + Id: "test-server-uuid", + Configuration: cfg, + meta: &Metadata{Image: "nginx:latest"}, + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + emitter: events.NewBus(), + } +} + +func TestNetwork(t *testing.T) { + g := Goblin(t) + + g.Describe("EnsureService", func() { + g.It("should be a no-op in hostport mode", func() { + client := fake.NewSimpleClientset() + allocs := environment.Allocations{ + Mappings: map[string][]int{"0.0.0.0": {25565}}, + } + env := newTestEnv(client, allocs) + + // Set hostport mode. + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkHostPort + c.Kubernetes.Namespace = "pelican" + }) + + err := env.EnsureService(context.Background()) + g.Assert(err).IsNil() + + // No Service should be created. + svcs, _ := client.CoreV1().Services("pelican").List(context.Background(), metav1.ListOptions{}) + g.Assert(len(svcs.Items)).Equal(0) + }) + + g.It("should create a NodePort service in nodeport mode", func() { + client := fake.NewSimpleClientset() + allocs := environment.Allocations{ + Mappings: map[string][]int{"0.0.0.0": {25565, 25575}}, + } + env := newTestEnv(client, allocs) + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkNodePort + c.Kubernetes.Namespace = "pelican" + }) + + err := env.EnsureService(context.Background()) + g.Assert(err).IsNil() + + svc, err := client.CoreV1().Services("pelican").Get(context.Background(), "gs-test-server-uuid", metav1.GetOptions{}) + g.Assert(err).IsNil() + g.Assert(svc.Spec.Type).Equal(corev1.ServiceTypeNodePort) + // Should have TCP+UDP for each port = 4 ports total. + g.Assert(len(svc.Spec.Ports)).Equal(4) + + // Verify selector targets our Pod. + g.Assert(svc.Spec.Selector["pelican.dev/server-id"]).Equal("test-server-uuid") + + // Verify labels. + g.Assert(svc.Labels["pelican.dev/server-id"]).Equal("test-server-uuid") + g.Assert(svc.Labels["pelican.dev/resource-type"]).Equal("service") + }) + + g.It("should update an existing service", func() { + // Pre-create a service with one port. + existingSvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gs-test-server-uuid", + Namespace: "pelican", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeNodePort, + Ports: []corev1.ServicePort{ + { + Name: "tcp-0-0-0-0-255", + Protocol: corev1.ProtocolTCP, + Port: 25565, + NodePort: 31000, + }, + }, + Selector: map[string]string{"pelican.dev/server-id": "test-server-uuid"}, + }, + } + client := fake.NewSimpleClientset(existingSvc) + allocs := environment.Allocations{ + Mappings: map[string][]int{"0.0.0.0": {25565, 25575}}, + } + env := newTestEnv(client, allocs) + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkNodePort + c.Kubernetes.Namespace = "pelican" + }) + + err := env.EnsureService(context.Background()) + g.Assert(err).IsNil() + + svc, err := client.CoreV1().Services("pelican").Get(context.Background(), "gs-test-server-uuid", metav1.GetOptions{}) + g.Assert(err).IsNil() + // Should now have 4 ports (2 for each allocation, TCP+UDP). + g.Assert(len(svc.Spec.Ports)).Equal(4) + }) + + g.It("should reconcile Service type and annotations when network mode changes", func() { + // Pre-create a LoadBalancer Service with stale LB annotations. + existingSvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gs-test-server-uuid", + Namespace: "pelican", + Annotations: map[string]string{"lbipam.cilium.io/ips": "10.0.0.1"}, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + Selector: map[string]string{"pelican.dev/server-id": "test-server-uuid"}, + }, + } + client := fake.NewSimpleClientset(existingSvc) + allocs := environment.Allocations{ + Mappings: map[string][]int{"0.0.0.0": {25565}}, + } + env := newTestEnv(client, allocs) + + // Switch to NodePort mode. + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkNodePort + c.Kubernetes.Namespace = "pelican" + }) + + err := env.EnsureService(context.Background()) + g.Assert(err).IsNil() + + svc, err := client.CoreV1().Services("pelican").Get(context.Background(), "gs-test-server-uuid", metav1.GetOptions{}) + g.Assert(err).IsNil() + // Type must be updated to NodePort and the stale LB annotation cleared. + g.Assert(svc.Spec.Type).Equal(corev1.ServiceTypeNodePort) + _, hasStale := svc.Annotations["lbipam.cilium.io/ips"] + g.Assert(hasStale).IsFalse() + }) + + g.It("should be a no-op with empty allocations", func() { + client := fake.NewSimpleClientset() + allocs := environment.Allocations{ + Mappings: map[string][]int{}, + } + env := newTestEnv(client, allocs) + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkNodePort + c.Kubernetes.Namespace = "pelican" + }) + + err := env.EnsureService(context.Background()) + g.Assert(err).IsNil() + + svcs, _ := client.CoreV1().Services("pelican").List(context.Background(), metav1.ListOptions{}) + g.Assert(len(svcs.Items)).Equal(0) + }) + + g.It("should auto-set LB IP and sharing-key annotations from allocation IP", func() { + client := fake.NewSimpleClientset() + allocs := environment.Allocations{ + DefaultMapping: &environment.DefaultAllocationMapping{ + Ip: "23.227.184.222", + Port: 27015, + }, + Mappings: map[string][]int{"23.227.184.222": {27015}}, + } + env := newTestEnv(client, allocs) + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkLoadBalancer + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.LBAnnotations = map[string]string{ + "io.cilium/lb-ipam-pool": "game-servers", + } + c.Kubernetes.LBIPAnnotation = "lbipam.cilium.io/ips" + c.Kubernetes.LBSharingKey = "lbipam.cilium.io/sharing-key" + }) + + err := env.EnsureService(context.Background()) + g.Assert(err).IsNil() + + svc, err := client.CoreV1().Services("pelican").Get(context.Background(), "gs-test-server-uuid", metav1.GetOptions{}) + g.Assert(err).IsNil() + g.Assert(svc.Spec.Type).Equal(corev1.ServiceTypeLoadBalancer) + g.Assert(svc.Annotations["io.cilium/lb-ipam-pool"]).Equal("game-servers") + g.Assert(svc.Annotations["lbipam.cilium.io/ips"]).Equal("23.227.184.222") + g.Assert(svc.Annotations["lbipam.cilium.io/sharing-key"]).Equal("23.227.184.222") + }) + + g.It("should not set IP annotations when allocation IP is 0.0.0.0", func() { + client := fake.NewSimpleClientset() + allocs := environment.Allocations{ + DefaultMapping: &environment.DefaultAllocationMapping{ + Ip: "0.0.0.0", + Port: 27015, + }, + Mappings: map[string][]int{"0.0.0.0": {27015}}, + } + env := newTestEnv(client, allocs) + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkLoadBalancer + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.LBIPAnnotation = "lbipam.cilium.io/ips" + c.Kubernetes.LBSharingKey = "lbipam.cilium.io/sharing-key" + }) + + err := env.EnsureService(context.Background()) + g.Assert(err).IsNil() + + svc, err := client.CoreV1().Services("pelican").Get(context.Background(), "gs-test-server-uuid", metav1.GetOptions{}) + g.Assert(err).IsNil() + _, hasIP := svc.Annotations["lbipam.cilium.io/ips"] + g.Assert(hasIP).IsFalse() + _, hasKey := svc.Annotations["lbipam.cilium.io/sharing-key"] + g.Assert(hasKey).IsFalse() + }) + + g.It("should not set IP annotations when config keys are empty", func() { + client := fake.NewSimpleClientset() + allocs := environment.Allocations{ + DefaultMapping: &environment.DefaultAllocationMapping{ + Ip: "23.227.184.222", + Port: 27015, + }, + Mappings: map[string][]int{"23.227.184.222": {27015}}, + } + env := newTestEnv(client, allocs) + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkLoadBalancer + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.LBIPAnnotation = "" + c.Kubernetes.LBSharingKey = "" + }) + + err := env.EnsureService(context.Background()) + g.Assert(err).IsNil() + + svc, err := client.CoreV1().Services("pelican").Get(context.Background(), "gs-test-server-uuid", metav1.GetOptions{}) + g.Assert(err).IsNil() + _, hasIP := svc.Annotations["lbipam.cilium.io/ips"] + g.Assert(hasIP).IsFalse() + }) + + g.It("should remove stale IP annotations when allocation IP becomes invalid", func() { + existingSvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gs-test-server-uuid", + Namespace: "pelican", + Annotations: map[string]string{ + "io.cilium/lb-ipam-pool": "game-servers", + "lbipam.cilium.io/ips": "23.227.184.222", + "lbipam.cilium.io/sharing-key": "23.227.184.222", + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + Ports: []corev1.ServicePort{ + {Name: "tcp-27015", Protocol: corev1.ProtocolTCP, Port: 27015}, + {Name: "udp-27015", Protocol: corev1.ProtocolUDP, Port: 27015}, + }, + Selector: map[string]string{"pelican.dev/server-id": "test-server-uuid"}, + }, + } + client := fake.NewSimpleClientset(existingSvc) + allocs := environment.Allocations{ + DefaultMapping: &environment.DefaultAllocationMapping{ + Ip: "0.0.0.0", + Port: 27015, + }, + Mappings: map[string][]int{"0.0.0.0": {27015}}, + } + env := newTestEnv(client, allocs) + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkLoadBalancer + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.LBAnnotations = map[string]string{ + "io.cilium/lb-ipam-pool": "game-servers", + } + c.Kubernetes.LBIPAnnotation = "lbipam.cilium.io/ips" + c.Kubernetes.LBSharingKey = "lbipam.cilium.io/sharing-key" + }) + + err := env.EnsureService(context.Background()) + g.Assert(err).IsNil() + + svc, err := client.CoreV1().Services("pelican").Get(context.Background(), "gs-test-server-uuid", metav1.GetOptions{}) + g.Assert(err).IsNil() + // Pool annotation should remain. + g.Assert(svc.Annotations["io.cilium/lb-ipam-pool"]).Equal("game-servers") + // IP-pinning annotations should be removed. + _, hasIP := svc.Annotations["lbipam.cilium.io/ips"] + g.Assert(hasIP).IsFalse() + _, hasKey := svc.Annotations["lbipam.cilium.io/sharing-key"] + g.Assert(hasKey).IsFalse() + }) + }) + + g.Describe("DeleteService", func() { + g.It("should delete an existing service", func() { + existingSvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gs-test-server-uuid", + Namespace: "pelican", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeNodePort, + }, + } + client := fake.NewSimpleClientset(existingSvc) + allocs := environment.Allocations{} + env := newTestEnv(client, allocs) + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkNodePort + c.Kubernetes.Namespace = "pelican" + }) + + err := env.DeleteService(context.Background()) + g.Assert(err).IsNil() + + svcs, _ := client.CoreV1().Services("pelican").List(context.Background(), metav1.ListOptions{}) + g.Assert(len(svcs.Items)).Equal(0) + }) + + g.It("should not error when service does not exist", func() { + client := fake.NewSimpleClientset() + allocs := environment.Allocations{} + env := newTestEnv(client, allocs) + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkNodePort + c.Kubernetes.Namespace = "pelican" + }) + + err := env.DeleteService(context.Background()) + g.Assert(err).IsNil() + }) + + g.It("should be a no-op in hostport mode", func() { + existingSvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gs-test-server-uuid", + Namespace: "pelican", + }, + } + client := fake.NewSimpleClientset(existingSvc) + allocs := environment.Allocations{} + env := newTestEnv(client, allocs) + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkHostPort + c.Kubernetes.Namespace = "pelican" + }) + + err := env.DeleteService(context.Background()) + g.Assert(err).IsNil() + + // Service should still exist since delete is no-op in hostport mode. + svcs, _ := client.CoreV1().Services("pelican").List(context.Background(), metav1.ListOptions{}) + g.Assert(len(svcs.Items)).Equal(1) + }) + }) + + g.Describe("GetServiceNodePorts", func() { + g.It("should return assigned NodePorts", func() { + existingSvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gs-test-server-uuid", + Namespace: "pelican", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeNodePort, + Ports: []corev1.ServicePort{ + {Name: "tcp-25565", Port: 25565, NodePort: 31234, Protocol: corev1.ProtocolTCP}, + {Name: "udp-25575", Port: 25575, NodePort: 31235, Protocol: corev1.ProtocolUDP}, + }, + }, + } + client := fake.NewSimpleClientset(existingSvc) + allocs := environment.Allocations{} + env := newTestEnv(client, allocs) + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkNodePort + c.Kubernetes.Namespace = "pelican" + }) + + ports, err := env.GetServiceNodePorts(context.Background()) + g.Assert(err).IsNil() + g.Assert(ports[25565]).Equal(int32(31234)) + g.Assert(ports[25575]).Equal(int32(31235)) + }) + + g.It("should return nil in hostport mode", func() { + client := fake.NewSimpleClientset() + allocs := environment.Allocations{} + env := newTestEnv(client, allocs) + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkHostPort + }) + + ports, err := env.GetServiceNodePorts(context.Background()) + g.Assert(err).IsNil() + g.Assert(ports == nil).IsTrue() + }) + }) + + g.Describe("sanitizePortName", func() { + g.It("should lowercase and replace dots with hyphens", func() { + g.Assert(sanitizePortName("TCP-192.168.1.1-25565")).Equal("tcp-192-168-1-1") + }) + + g.It("should truncate to 15 characters", func() { + result := sanitizePortName("very-long-port-name-that-exceeds") + g.Assert(len(result) <= 15).IsTrue() + }) + + g.It("should not start or end with a hyphen", func() { + result := sanitizePortName("-invalid-name-") + g.Assert(result[0] != '-').IsTrue() + g.Assert(result[len(result)-1] != '-').IsTrue() + }) + + g.It("should return 'port' for empty input", func() { + g.Assert(sanitizePortName("")).Equal("port") + }) + }) + + g.Describe("portName", func() { + g.It("should produce unique names for different ports", func() { + g.Assert(portName("tcp", 27015)).Equal("tcp-27015") + g.Assert(portName("tcp", 27016)).Equal("tcp-27016") + g.Assert(portName("udp", 27015)).Equal("udp-27015") + }) + + g.It("should fit within 15-char limit for 5-digit ports", func() { + name := portName("tcp", 65535) + g.Assert(len(name) <= 15).IsTrue() + g.Assert(name).Equal("tcp-65535") + }) + }) + + g.Describe("mergeServicePorts", func() { + g.It("should preserve existing NodePorts for unchanged ports", func() { + existing := []corev1.ServicePort{ + {Name: "tcp-25565", Protocol: corev1.ProtocolTCP, Port: 25565, NodePort: 31000}, + } + desired := []corev1.ServicePort{ + {Name: "tcp-25565", Protocol: corev1.ProtocolTCP, Port: 25565, NodePort: 0}, + } + merged := mergeServicePorts(existing, desired) + g.Assert(len(merged)).Equal(1) + g.Assert(merged[0].NodePort).Equal(int32(31000)) + }) + + g.It("should use desired NodePort when explicitly set", func() { + existing := []corev1.ServicePort{ + {Name: "tcp-25565", Protocol: corev1.ProtocolTCP, Port: 25565, NodePort: 31000}, + } + desired := []corev1.ServicePort{ + {Name: "tcp-25565", Protocol: corev1.ProtocolTCP, Port: 25565, NodePort: 31500}, + } + merged := mergeServicePorts(existing, desired) + g.Assert(merged[0].NodePort).Equal(int32(31500)) + }) + + g.It("should add new ports", func() { + existing := []corev1.ServicePort{ + {Name: "tcp-25565", Protocol: corev1.ProtocolTCP, Port: 25565, NodePort: 31000}, + } + desired := []corev1.ServicePort{ + {Name: "tcp-25565", Protocol: corev1.ProtocolTCP, Port: 25565, NodePort: 0}, + {Name: "tcp-25575", Protocol: corev1.ProtocolTCP, Port: 25575, NodePort: 0}, + } + merged := mergeServicePorts(existing, desired) + g.Assert(len(merged)).Equal(2) + }) + }) +} diff --git a/environment/kubernetes/node.go b/environment/kubernetes/node.go new file mode 100644 index 00000000..a7544a91 --- /dev/null +++ b/environment/kubernetes/node.go @@ -0,0 +1,77 @@ +package kubernetes + +import ( + "context" + "os" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/pelican/wings/config" +) + +// GetNodeIPs returns the IP addresses of the Kubernetes node where Wings is +// running. It queries the node's status.addresses for ExternalIP and +// InternalIP entries. The node name is read from the config or the NODE_NAME +// environment variable (set via the Kubernetes downward API). +func GetNodeIPs(ctx context.Context) ([]string, error) { + nodeName := config.Get().Kubernetes.NodeName + if nodeName == "" { + nodeName = os.Getenv("NODE_NAME") + } + if nodeName == "" { + return nil, nil + } + + c, err := Client() + if err != nil { + return nil, err + } + + node, err := c.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + var ips []string + for _, addr := range node.Status.Addresses { + if addr.Type == corev1.NodeExternalIP || addr.Type == corev1.NodeInternalIP { + ips = append(ips, addr.Address) + } + } + return ips, nil +} + +// GetLoadBalancerIPs returns the external IPs assigned to all game server +// LoadBalancer Services in the configured namespace. This is used by the +// allocation endpoint to show available IPs when in LoadBalancer mode. +func GetLoadBalancerIPs(ctx context.Context) ([]string, error) { + cfg := config.Get() + c, err := Client() + if err != nil { + return nil, err + } + + svcs, err := c.CoreV1().Services(cfg.Kubernetes.Namespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/managed-by=pelican-wings,pelican.dev/resource-type=service", + }) + if err != nil { + return nil, err + } + + seen := make(map[string]bool) + var ips []string + for _, svc := range svcs.Items { + for _, ingress := range svc.Status.LoadBalancer.Ingress { + addr := ingress.IP + if addr == "" { + addr = ingress.Hostname + } + if addr != "" && !seen[addr] { + seen[addr] = true + ips = append(ips, addr) + } + } + } + return ips, nil +} diff --git a/environment/kubernetes/power.go b/environment/kubernetes/power.go new file mode 100644 index 00000000..1bb32e0b --- /dev/null +++ b/environment/kubernetes/power.go @@ -0,0 +1,320 @@ +package kubernetes + +import ( + "context" + "strings" + "time" + + "emperror.dev/errors" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" + + "github.com/pelican/wings/environment" + "github.com/pelican/wings/remote" +) + +// OnBeforeStart is called before the server starts. It ensures the Pod is in a +// clean state by deleting any existing Pod and recreating it with the latest +// configuration from the Panel. +func (e *Environment) OnBeforeStart(ctx context.Context) error { + // Delete any existing Pod to ensure fresh config is applied. + gracePeriod := int64(0) + if err := e.client.CoreV1().Pods(e.namespace()).Delete(ctx, e.Id, metav1.DeleteOptions{ + GracePeriodSeconds: &gracePeriod, + }); err != nil && !isNotFound(err) { + return errors.Wrap(err, "environment/kubernetes: failed to delete existing pod before start") + } + + // Wait for the old Pod to be fully removed so the recreate below does not + // no-op against a stale Pod. + if err := e.waitForPodDeletion(ctx, 10*time.Second); err != nil { + return errors.Wrap(err, "environment/kubernetes: timed out waiting for old pod deletion") + } + + // Create the Pod with current configuration. + if err := e.Create(); err != nil { + return err + } + + return nil +} + +// Start boots the server by creating the Pod (if needed) and attaching to it. +// Since Kubernetes Pods start running immediately upon creation, this mainly +// ensures we're attached to capture output. +func (e *Environment) Start(ctx context.Context) error { + sawError := false + + defer func() { + if sawError { + e.SetState(environment.ProcessStoppingState) + e.SetState(environment.ProcessOfflineState) + } + }() + + // Check if Pod already exists and is running. + if running, _ := e.IsRunning(ctx); running { + e.SetState(environment.ProcessRunningState) + return e.Attach(ctx) + } + + e.SetState(environment.ProcessStartingState) + sawError = true + + // Run pre-start to ensure the Pod exists with fresh configuration. + if err := e.OnBeforeStart(ctx); err != nil { + return errors.WrapIf(err, "environment/kubernetes: failed to run pre-boot process") + } + + // Wait for the Pod to reach Running phase. + waitCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + + if err := e.waitForPodRunning(waitCtx); err != nil { + return errors.WrapIf(err, "environment/kubernetes: pod did not reach running state") + } + + // Attach to the Pod to stream output. + if err := e.Attach(ctx); err != nil { + return errors.WrapIf(err, "environment/kubernetes: failed to attach to pod") + } + + e.SetState(environment.ProcessRunningState) + sawError = false + return nil +} + +// Stop sends the configured stop command or deletes the Pod with a grace +// period to allow graceful shutdown. +func (e *Environment) Stop(ctx context.Context) error { + e.mu.RLock() + s := e.meta.Stop + e.mu.RUnlock() + + if e.st.Load() != environment.ProcessOfflineState { + e.SetState(environment.ProcessStoppingState) + } + + // If using a command-based stop and we're attached, send the command. + if s.Type == remote.ProcessStopCommand && e.IsAttached() { + return e.SendCommand(s.Value) + } + + // Otherwise (including signal-based stops) delete the Pod gracefully so the + // kubelet sends SIGTERM to PID 1 and the process can shut down within the + // grace period before being force-killed. An explicit SIGKILL maps to an + // immediate force-delete. + gracePeriod := int64(30) + if strings.EqualFold(s.Value, "SIGKILL") { + gracePeriod = 0 + } + err := e.client.CoreV1().Pods(e.namespace()).Delete(ctx, e.Id, metav1.DeleteOptions{ + GracePeriodSeconds: &gracePeriod, + }) + if err != nil && !isNotFound(err) { + return errors.Wrap(err, "environment/kubernetes: failed to stop pod") + } + + return nil +} + +// WaitForStop attempts to gracefully stop the server and waits for the Pod to +// terminate. If the timeout is reached and terminate is true, the Pod is +// forcefully deleted. +func (e *Environment) WaitForStop(ctx context.Context, duration time.Duration, terminate bool) error { + // If the Pod is already gone, or exists but is no longer running, the server + // is effectively stopped and there is nothing to wait for. A stopped Pod is + // not automatically removed, so proceeding into waitForPodDeletion would block + // for the full duration (holding the power lock) waiting for a deletion that + // never happens. This is what gets hit when a stop/restart is issued against a + // server that is already offline. + if pod, err := e.getPod(ctx); err != nil { + if isNotFound(err) { + e.markOffline() + return nil + } + // Fall through on unexpected errors and attempt the normal stop flow. + } else if !isPodRunning(pod) { + e.markOffline() + return nil + } + + tctx, cancel := context.WithTimeout(context.Background(), duration) + defer cancel() + + go func() { + select { + case <-ctx.Done(): + cancel() + case <-tctx.Done(): + } + }() + + // Send the stop command/signal. + if err := e.Stop(tctx); err != nil { + if terminate && errors.Is(err, context.DeadlineExceeded) { + return e.Terminate(ctx, "SIGKILL") + } + return err + } + + // Wait for the Pod to be gone or to stop running. A command-based stop + // leaves the Pod in a terminal Succeeded/Failed phase without deleting it, + // so waiting only for deletion would block until the timeout. + if err := e.waitForPodStoppedOrDeleted(tctx, duration); err != nil { + if terminate { + e.log().Warn("pod did not terminate in time, forcing deletion") + return e.Terminate(ctx, "SIGKILL") + } + return err + } + + return nil +} + +// Terminate forcefully stops the Pod by deleting it with a zero grace period. +func (e *Environment) Terminate(ctx context.Context, signal string) error { + _ = signal // K8s doesn't support arbitrary signals; we just force-delete. + + pod, err := e.getPod(ctx) + if err != nil { + if isNotFound(err) { + return nil + } + return errors.WithStack(err) + } + + if !isPodRunning(pod) { + e.markOffline() + return nil + } + + e.SetState(environment.ProcessStoppingState) + + gracePeriod := int64(0) + err = e.client.CoreV1().Pods(e.namespace()).Delete(ctx, e.Id, metav1.DeleteOptions{ + GracePeriodSeconds: &gracePeriod, + }) + if err != nil && !isNotFound(err) { + return errors.WithStack(err) + } + + e.SetState(environment.ProcessOfflineState) + return nil +} + +// markOffline transitions the environment to the offline state, first passing +// through the stopping state so that crash detection is not triggered. It is a +// no-op if the environment already considers itself offline. +func (e *Environment) markOffline() { + if e.st.Load() != environment.ProcessOfflineState { + e.SetState(environment.ProcessStoppingState) + e.SetState(environment.ProcessOfflineState) + } +} + +// waitForPodRunning blocks until the Pod reaches Running phase or the context +// is canceled. +func (e *Environment) waitForPodRunning(ctx context.Context) error { + // First check if already running. + if running, _ := e.IsRunning(ctx); running { + return nil + } + + watcher, err := e.client.CoreV1().Pods(e.namespace()).Watch(ctx, metav1.ListOptions{ + FieldSelector: "metadata.name=" + e.Id, + }) + if err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to watch pod") + } + defer watcher.Stop() + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case event, ok := <-watcher.ResultChan(): + if !ok { + return errors.New("environment/kubernetes: watch channel closed") + } + if event.Type == watch.Modified || event.Type == watch.Added { + pod, ok := event.Object.(*corev1.Pod) + if !ok { + continue + } + if isPodRunning(pod) { + return nil + } + // Check for failure. + if pod.Status.Phase == corev1.PodFailed || pod.Status.Phase == corev1.PodSucceeded { + return errors.New("environment/kubernetes: pod terminated before reaching running state") + } + // Check for container crashes during startup. + for _, cs := range pod.Status.ContainerStatuses { + if cs.State.Waiting != nil && strings.Contains(cs.State.Waiting.Reason, "CrashLoopBackOff") { + return errors.New("environment/kubernetes: container in CrashLoopBackOff") + } + if cs.State.Waiting != nil && strings.Contains(cs.State.Waiting.Reason, "ErrImagePull") { + return errors.New("environment/kubernetes: failed to pull container image") + } + if cs.State.Waiting != nil && strings.Contains(cs.State.Waiting.Reason, "ImagePullBackOff") { + return errors.New("environment/kubernetes: image pull backoff") + } + } + } + } + } +} + +// waitForPodStoppedOrDeleted blocks until the Pod is deleted or has stopped +// running (a terminal Succeeded/Failed phase), whichever happens first, or the +// timeout elapses. +func (e *Environment) waitForPodStoppedOrDeleted(ctx context.Context, timeout time.Duration) error { + dctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-dctx.Done(): + return dctx.Err() + case <-ticker.C: + pod, err := e.getPod(dctx) + if err != nil { + if isNotFound(err) { + return nil + } + continue + } + if !isPodRunning(pod) { + e.markOffline() + return nil + } + } + } +} + +// waitForPodDeletion blocks until the Pod is fully deleted or the timeout +// elapses. +func (e *Environment) waitForPodDeletion(ctx context.Context, timeout time.Duration) error { + dctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-dctx.Done(): + return dctx.Err() + case <-ticker.C: + _, err := e.getPod(dctx) + if err != nil && isNotFound(err) { + return nil + } + } + } +} diff --git a/environment/kubernetes/power_test.go b/environment/kubernetes/power_test.go new file mode 100644 index 00000000..1282bec7 --- /dev/null +++ b/environment/kubernetes/power_test.go @@ -0,0 +1,77 @@ +package kubernetes + +import ( + "context" + "testing" + "time" + + . "github.com/franela/goblin" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + + "github.com/pelican/wings/config" + "github.com/pelican/wings/environment" + "github.com/pelican/wings/events" + "github.com/pelican/wings/system" +) + +func TestWaitForStop(t *testing.T) { + g := Goblin(t) + + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + }) + + newEnv := func(state string, objects ...runtime.Object) *Environment { + return &Environment{ + Id: "test-uuid", + meta: &Metadata{}, + client: fake.NewSimpleClientset(objects...), + st: system.NewAtomicString(state), + emitter: events.NewBus(), + } + } + + g.Describe("WaitForStop when the server is already offline", func() { + g.It("returns immediately when the Pod does not exist", func() { + env := newEnv(environment.ProcessOfflineState) + + done := make(chan error, 1) + go func() { + // A long duration would previously block here for the full period. + done <- env.WaitForStop(context.Background(), time.Minute, true) + }() + + select { + case err := <-done: + g.Assert(err).IsNil() + case <-time.After(2 * time.Second): + g.Fail("WaitForStop blocked when the Pod was absent") + } + }) + + g.It("returns immediately when the Pod exists but is not running", func() { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "test-uuid", Namespace: "pelican"}, + Status: corev1.PodStatus{Phase: corev1.PodSucceeded}, + } + // Start from a non-offline state to prove the helper drives it offline. + env := newEnv(environment.ProcessRunningState, pod) + + done := make(chan error, 1) + go func() { + done <- env.WaitForStop(context.Background(), time.Minute, true) + }() + + select { + case err := <-done: + g.Assert(err).IsNil() + g.Assert(env.State()).Equal(environment.ProcessOfflineState) + case <-time.After(2 * time.Second): + g.Fail("WaitForStop blocked when the Pod was not running") + } + }) + }) +} diff --git a/environment/kubernetes/quota.go b/environment/kubernetes/quota.go new file mode 100644 index 00000000..57ac22cd --- /dev/null +++ b/environment/kubernetes/quota.go @@ -0,0 +1,187 @@ +package kubernetes + +import ( + "context" + + "emperror.dev/errors" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/pelican/wings/config" +) + +const ( + quotaName = "pelican-wings" + limitRangeName = "pelican-wings" +) + +// EnsureResourceQuota creates or updates the ResourceQuota in the game server +// namespace. If resource_quota is not enabled in config, this is a no-op. +func (e *Environment) EnsureResourceQuota(ctx context.Context) error { + cfg := config.Get() + if !cfg.Kubernetes.ResourceQuota.Enabled { + return nil + } + + ns := e.namespace() + quota, err := buildResourceQuota(ns, &cfg.Kubernetes.ResourceQuota) + if err != nil { + return err + } + + existing, err := e.client.CoreV1().ResourceQuotas(ns).Get(ctx, quotaName, metav1.GetOptions{}) + if err != nil && !isNotFound(err) { + return errors.Wrap(err, "environment/kubernetes: failed to get ResourceQuota") + } + if err == nil { + // Update existing. + existing.Spec = quota.Spec + _, err = e.client.CoreV1().ResourceQuotas(ns).Update(ctx, existing, metav1.UpdateOptions{}) + if err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to update ResourceQuota") + } + return nil + } + + _, err = e.client.CoreV1().ResourceQuotas(ns).Create(ctx, quota, metav1.CreateOptions{}) + if err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to create ResourceQuota") + } + + e.log().Info("created ResourceQuota for namespace") + return nil +} + +// EnsureLimitRange creates or updates the LimitRange in the game server +// namespace. If limit_range is not enabled in config, this is a no-op. +func (e *Environment) EnsureLimitRange(ctx context.Context) error { + cfg := config.Get() + if !cfg.Kubernetes.LimitRange.Enabled { + return nil + } + + ns := e.namespace() + lr, err := buildLimitRange(ns, &cfg.Kubernetes.LimitRange) + if err != nil { + return err + } + + existing, err := e.client.CoreV1().LimitRanges(ns).Get(ctx, limitRangeName, metav1.GetOptions{}) + if err != nil && !isNotFound(err) { + return errors.Wrap(err, "environment/kubernetes: failed to get LimitRange") + } + if err == nil { + // Update existing. + existing.Spec = lr.Spec + _, err = e.client.CoreV1().LimitRanges(ns).Update(ctx, existing, metav1.UpdateOptions{}) + if err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to update LimitRange") + } + return nil + } + + _, err = e.client.CoreV1().LimitRanges(ns).Create(ctx, lr, metav1.CreateOptions{}) + if err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to create LimitRange") + } + + e.log().Info("created LimitRange for namespace") + return nil +} + +// setQuantity parses a quantity string into the resource list under key when +// the value is non-empty. Unlike resource.MustParse it returns an error for +// invalid (user-configurable) values instead of panicking the process. +func setQuantity(list corev1.ResourceList, key corev1.ResourceName, field, value string) error { + if value == "" { + return nil + } + q, err := resource.ParseQuantity(value) + if err != nil { + return errors.Wrapf(err, "environment/kubernetes: invalid quantity %q for %s", value, field) + } + list[key] = q + return nil +} + +// buildResourceQuota constructs the ResourceQuota spec from config values. +func buildResourceQuota(namespace string, cfg *config.KubeResourceQuota) (*corev1.ResourceQuota, error) { + hard := corev1.ResourceList{} + + for _, q := range []struct { + key corev1.ResourceName + field string + value string + }{ + {corev1.ResourceLimitsCPU, "cpu_limit", cfg.CPULimit}, + {corev1.ResourceLimitsMemory, "memory_limit", cfg.MemoryLimit}, + {corev1.ResourceRequestsCPU, "cpu_request", cfg.CPURequest}, + {corev1.ResourceRequestsMemory, "memory_request", cfg.MemoryRequest}, + {corev1.ResourceRequestsStorage, "max_storage", cfg.MaxStorage}, + } { + if err := setQuantity(hard, q.key, q.field, q.value); err != nil { + return nil, err + } + } + if cfg.MaxPods > 0 { + hard[corev1.ResourcePods] = *resource.NewQuantity(cfg.MaxPods, resource.DecimalSI) + } + if cfg.MaxPVCs > 0 { + hard[corev1.ResourcePersistentVolumeClaims] = *resource.NewQuantity(cfg.MaxPVCs, resource.DecimalSI) + } + + return &corev1.ResourceQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: quotaName, + Namespace: namespace, + Labels: map[string]string{ + "app.kubernetes.io/managed-by": "pelican-wings", + }, + }, + Spec: corev1.ResourceQuotaSpec{ + Hard: hard, + }, + }, nil +} + +// buildLimitRange constructs the LimitRange spec from config values. +func buildLimitRange(namespace string, cfg *config.KubeLimitRange) (*corev1.LimitRange, error) { + containerLimit := corev1.LimitRangeItem{ + Type: corev1.LimitTypeContainer, + Default: corev1.ResourceList{}, + DefaultRequest: corev1.ResourceList{}, + Max: corev1.ResourceList{}, + } + + for _, q := range []struct { + list corev1.ResourceList + key corev1.ResourceName + field string + value string + }{ + {containerLimit.Default, corev1.ResourceCPU, "default_cpu_limit", cfg.DefaultCPULimit}, + {containerLimit.Default, corev1.ResourceMemory, "default_memory_limit", cfg.DefaultMemoryLimit}, + {containerLimit.DefaultRequest, corev1.ResourceCPU, "default_cpu_request", cfg.DefaultCPURequest}, + {containerLimit.DefaultRequest, corev1.ResourceMemory, "default_memory_request", cfg.DefaultMemoryRequest}, + {containerLimit.Max, corev1.ResourceCPU, "max_cpu", cfg.MaxCPU}, + {containerLimit.Max, corev1.ResourceMemory, "max_memory", cfg.MaxMemory}, + } { + if err := setQuantity(q.list, q.key, q.field, q.value); err != nil { + return nil, err + } + } + + return &corev1.LimitRange{ + ObjectMeta: metav1.ObjectMeta{ + Name: limitRangeName, + Namespace: namespace, + Labels: map[string]string{ + "app.kubernetes.io/managed-by": "pelican-wings", + }, + }, + Spec: corev1.LimitRangeSpec{ + Limits: []corev1.LimitRangeItem{containerLimit}, + }, + }, nil +} diff --git a/environment/kubernetes/quota_test.go b/environment/kubernetes/quota_test.go new file mode 100644 index 00000000..cf9f7a93 --- /dev/null +++ b/environment/kubernetes/quota_test.go @@ -0,0 +1,372 @@ +package kubernetes + +import ( + "context" + "errors" + "testing" + + . "github.com/franela/goblin" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + + "github.com/pelican/wings/config" + "github.com/pelican/wings/environment" + "github.com/pelican/wings/system" +) + +// TestQuota covers building and reconciling ResourceQuota and LimitRange +// objects from configuration, including invalid-quantity error handling. +func TestQuota(t *testing.T) { + g := Goblin(t) + + g.Describe("ResourceQuota", func() { + g.Describe("EnsureResourceQuota", func() { + g.It("should be a no-op when disabled", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "quota-noop-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.ResourceQuota.Enabled = false + }) + + err := env.EnsureResourceQuota(context.Background()) + g.Assert(err).IsNil() + + quotas, _ := client.CoreV1().ResourceQuotas("pelican").List(context.Background(), metav1.ListOptions{}) + g.Assert(len(quotas.Items)).Equal(0) + }) + + g.It("should create a ResourceQuota with CPU and memory limits", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "quota-create-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.ResourceQuota.Enabled = true + c.Kubernetes.ResourceQuota.CPULimit = "16" + c.Kubernetes.ResourceQuota.MemoryLimit = "32Gi" + c.Kubernetes.ResourceQuota.CPURequest = "8" + c.Kubernetes.ResourceQuota.MemoryRequest = "16Gi" + c.Kubernetes.ResourceQuota.MaxPods = 20 + }) + + err := env.EnsureResourceQuota(context.Background()) + g.Assert(err).IsNil() + + rq, err := client.CoreV1().ResourceQuotas("pelican").Get(context.Background(), "pelican-wings", metav1.GetOptions{}) + g.Assert(err).IsNil() + g.Assert(rq.Name).Equal("pelican-wings") + g.Assert(rq.Labels["app.kubernetes.io/managed-by"]).Equal("pelican-wings") + + cpuLimit := rq.Spec.Hard[corev1.ResourceLimitsCPU] + g.Assert(cpuLimit.Cmp(resource.MustParse("16"))).Equal(0) + + memLimit := rq.Spec.Hard[corev1.ResourceLimitsMemory] + g.Assert(memLimit.Cmp(resource.MustParse("32Gi"))).Equal(0) + + pods := rq.Spec.Hard[corev1.ResourcePods] + g.Assert(pods.Value()).Equal(int64(20)) + }) + + g.It("should create ResourceQuota with storage limits", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "quota-storage-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.ResourceQuota.Enabled = true + c.Kubernetes.ResourceQuota.MaxPVCs = 50 + c.Kubernetes.ResourceQuota.MaxStorage = "500Gi" + c.Kubernetes.ResourceQuota.CPULimit = "" + c.Kubernetes.ResourceQuota.MemoryLimit = "" + c.Kubernetes.ResourceQuota.CPURequest = "" + c.Kubernetes.ResourceQuota.MemoryRequest = "" + c.Kubernetes.ResourceQuota.MaxPods = 0 + }) + + err := env.EnsureResourceQuota(context.Background()) + g.Assert(err).IsNil() + + rq, err := client.CoreV1().ResourceQuotas("pelican").Get(context.Background(), "pelican-wings", metav1.GetOptions{}) + g.Assert(err).IsNil() + + pvcs := rq.Spec.Hard[corev1.ResourcePersistentVolumeClaims] + g.Assert(pvcs.Value()).Equal(int64(50)) + + storage := rq.Spec.Hard[corev1.ResourceRequestsStorage] + g.Assert(storage.Cmp(resource.MustParse("500Gi"))).Equal(0) + }) + + g.It("should update an existing ResourceQuota", func() { + existingRQ := &corev1.ResourceQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pelican-wings", + Namespace: "pelican", + }, + Spec: corev1.ResourceQuotaSpec{ + Hard: corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("8"), + }, + }, + } + client := fake.NewSimpleClientset(existingRQ) + env := &Environment{ + Id: "quota-update-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.ResourceQuota.Enabled = true + c.Kubernetes.ResourceQuota.CPULimit = "32" + c.Kubernetes.ResourceQuota.MemoryLimit = "64Gi" + c.Kubernetes.ResourceQuota.CPURequest = "" + c.Kubernetes.ResourceQuota.MemoryRequest = "" + c.Kubernetes.ResourceQuota.MaxPods = 0 + c.Kubernetes.ResourceQuota.MaxPVCs = 0 + c.Kubernetes.ResourceQuota.MaxStorage = "" + }) + + err := env.EnsureResourceQuota(context.Background()) + g.Assert(err).IsNil() + + rq, _ := client.CoreV1().ResourceQuotas("pelican").Get(context.Background(), "pelican-wings", metav1.GetOptions{}) + cpuLimit := rq.Spec.Hard[corev1.ResourceLimitsCPU] + g.Assert(cpuLimit.Cmp(resource.MustParse("32"))).Equal(0) + + memLimit := rq.Spec.Hard[corev1.ResourceLimitsMemory] + g.Assert(memLimit.Cmp(resource.MustParse("64Gi"))).Equal(0) + }) + + g.It("should fail fast on a non-NotFound Get error instead of creating", func() { + client := fake.NewSimpleClientset() + client.PrependReactor("get", "resourcequotas", func(action k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, errors.New("boom: api server unavailable") + }) + env := &Environment{ + Id: "quota-geterr-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.ResourceQuota.Enabled = true + c.Kubernetes.ResourceQuota.CPULimit = "16" + }) + + err := env.EnsureResourceQuota(context.Background()) + g.Assert(err != nil).IsTrue() + }) + }) + + g.Describe("EnsureLimitRange", func() { + g.It("should be a no-op when disabled", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "lr-noop-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.LimitRange.Enabled = false + }) + + err := env.EnsureLimitRange(context.Background()) + g.Assert(err).IsNil() + + lrs, _ := client.CoreV1().LimitRanges("pelican").List(context.Background(), metav1.ListOptions{}) + g.Assert(len(lrs.Items)).Equal(0) + }) + + g.It("should create a LimitRange with defaults and max", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "lr-create-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.LimitRange.Enabled = true + c.Kubernetes.LimitRange.DefaultCPULimit = "2" + c.Kubernetes.LimitRange.DefaultMemoryLimit = "4Gi" + c.Kubernetes.LimitRange.DefaultCPURequest = "500m" + c.Kubernetes.LimitRange.DefaultMemoryRequest = "1Gi" + c.Kubernetes.LimitRange.MaxCPU = "8" + c.Kubernetes.LimitRange.MaxMemory = "16Gi" + }) + + err := env.EnsureLimitRange(context.Background()) + g.Assert(err).IsNil() + + lr, err := client.CoreV1().LimitRanges("pelican").Get(context.Background(), "pelican-wings", metav1.GetOptions{}) + g.Assert(err).IsNil() + g.Assert(lr.Name).Equal("pelican-wings") + g.Assert(len(lr.Spec.Limits)).Equal(1) + + item := lr.Spec.Limits[0] + g.Assert(item.Type).Equal(corev1.LimitTypeContainer) + + defaultCPU := item.Default[corev1.ResourceCPU] + g.Assert(defaultCPU.Cmp(resource.MustParse("2"))).Equal(0) + + defaultMem := item.Default[corev1.ResourceMemory] + g.Assert(defaultMem.Cmp(resource.MustParse("4Gi"))).Equal(0) + + reqCPU := item.DefaultRequest[corev1.ResourceCPU] + g.Assert(reqCPU.Cmp(resource.MustParse("500m"))).Equal(0) + + maxCPU := item.Max[corev1.ResourceCPU] + g.Assert(maxCPU.Cmp(resource.MustParse("8"))).Equal(0) + + maxMem := item.Max[corev1.ResourceMemory] + g.Assert(maxMem.Cmp(resource.MustParse("16Gi"))).Equal(0) + }) + + g.It("should update an existing LimitRange", func() { + existingLR := &corev1.LimitRange{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pelican-wings", + Namespace: "pelican", + }, + Spec: corev1.LimitRangeSpec{ + Limits: []corev1.LimitRangeItem{ + { + Type: corev1.LimitTypeContainer, + Default: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + }, + }, + }, + }, + } + client := fake.NewSimpleClientset(existingLR) + env := &Environment{ + Id: "lr-update-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.LimitRange.Enabled = true + c.Kubernetes.LimitRange.DefaultCPULimit = "4" + c.Kubernetes.LimitRange.DefaultMemoryLimit = "8Gi" + c.Kubernetes.LimitRange.DefaultCPURequest = "" + c.Kubernetes.LimitRange.DefaultMemoryRequest = "" + c.Kubernetes.LimitRange.MaxCPU = "" + c.Kubernetes.LimitRange.MaxMemory = "" + }) + + err := env.EnsureLimitRange(context.Background()) + g.Assert(err).IsNil() + + lr, _ := client.CoreV1().LimitRanges("pelican").Get(context.Background(), "pelican-wings", metav1.GetOptions{}) + defaultCPU := lr.Spec.Limits[0].Default[corev1.ResourceCPU] + g.Assert(defaultCPU.Cmp(resource.MustParse("4"))).Equal(0) + }) + + g.It("should fail fast on a non-NotFound Get error instead of creating", func() { + client := fake.NewSimpleClientset() + client.PrependReactor("get", "limitranges", func(action k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, errors.New("boom: api server unavailable") + }) + env := &Environment{ + Id: "lr-geterr-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.LimitRange.Enabled = true + c.Kubernetes.LimitRange.DefaultCPULimit = "2" + }) + + err := env.EnsureLimitRange(context.Background()) + g.Assert(err != nil).IsTrue() + }) + }) + + g.Describe("buildResourceQuota", func() { + g.It("should only include non-empty fields", func() { + cfg := &config.KubeResourceQuota{ + CPULimit: "4", + MemoryLimit: "", + MaxPods: 10, + MaxPVCs: 0, + } + rq, err := buildResourceQuota("test-ns", cfg) + g.Assert(err).IsNil() + g.Assert(rq.Namespace).Equal("test-ns") + + _, hasCPU := rq.Spec.Hard[corev1.ResourceLimitsCPU] + g.Assert(hasCPU).IsTrue() + + _, hasMem := rq.Spec.Hard[corev1.ResourceLimitsMemory] + g.Assert(hasMem).IsFalse() + + _, hasPods := rq.Spec.Hard[corev1.ResourcePods] + g.Assert(hasPods).IsTrue() + + _, hasPVCs := rq.Spec.Hard[corev1.ResourcePersistentVolumeClaims] + g.Assert(hasPVCs).IsFalse() + }) + }) + + g.Describe("buildLimitRange", func() { + g.It("should only include non-empty fields", func() { + cfg := &config.KubeLimitRange{ + DefaultCPULimit: "2", + DefaultMemoryLimit: "", + MaxCPU: "8", + MaxMemory: "", + } + lr, err := buildLimitRange("test-ns", cfg) + g.Assert(err).IsNil() + g.Assert(lr.Namespace).Equal("test-ns") + g.Assert(len(lr.Spec.Limits)).Equal(1) + + item := lr.Spec.Limits[0] + _, hasCPU := item.Default[corev1.ResourceCPU] + g.Assert(hasCPU).IsTrue() + + _, hasMem := item.Default[corev1.ResourceMemory] + g.Assert(hasMem).IsFalse() + + _, hasMaxCPU := item.Max[corev1.ResourceCPU] + g.Assert(hasMaxCPU).IsTrue() + + _, hasMaxMem := item.Max[corev1.ResourceMemory] + g.Assert(hasMaxMem).IsFalse() + }) + + g.It("should return an error for an invalid quantity instead of panicking", func() { + lr, err := buildLimitRange("test-ns", &config.KubeLimitRange{DefaultCPULimit: "not-a-quantity"}) + g.Assert(err != nil).IsTrue() + g.Assert(lr == nil).IsTrue() + }) + }) + + g.Describe("buildResourceQuota invalid input", func() { + g.It("should return an error for an invalid quantity instead of panicking", func() { + rq, err := buildResourceQuota("test-ns", &config.KubeResourceQuota{CPULimit: "not-a-quantity"}) + g.Assert(err != nil).IsTrue() + g.Assert(rq == nil).IsTrue() + }) + }) + }) +} diff --git a/environment/kubernetes/stats.go b/environment/kubernetes/stats.go new file mode 100644 index 00000000..a4d90cc9 --- /dev/null +++ b/environment/kubernetes/stats.go @@ -0,0 +1,408 @@ +package kubernetes + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + + "emperror.dev/errors" + + "github.com/pelican/wings/environment" +) + +// statsRequestTimeout bounds each individual metrics/stats API call so a slow +// or hung endpoint cannot stall the polling loop until the next tick. +const statsRequestTimeout = 3 * time.Second + +// podStats holds parsed resource metrics for a pod's "server" container. +type podStats struct { + cpuPercent float64 + memoryBytes int64 + rxBytes uint64 + txBytes uint64 +} + +// Uptime returns the uptime of the server Pod in milliseconds. If the Pod is +// not running, returns 0. +func (e *Environment) Uptime(ctx context.Context) (int64, error) { + pod, err := e.getPod(ctx) + if err != nil { + return 0, errors.Wrap(err, "environment/kubernetes: could not get pod") + } + if !isPodRunning(pod) { + return 0, nil + } + if pod.Status.StartTime == nil { + return 0, nil + } + return time.Since(pod.Status.StartTime.Time).Milliseconds(), nil +} + +// withStatsTimeout invokes fn with a child context bounded by +// statsRequestTimeout so a single slow metrics/stats call cannot stall polling. +func withStatsTimeout[T any](ctx context.Context, fn func(context.Context) (T, error)) (T, error) { + tctx, cancel := context.WithTimeout(ctx, statsRequestTimeout) + defer cancel() + return fn(tctx) +} + +// pollResources periodically fetches resource usage and publishes resource +// events. It tries the Kubernetes Metrics API first (requires metrics-server) +// and falls back to the kubelet stats/summary API (always available). +func (e *Environment) pollResources(ctx context.Context) error { + if e.st.Load() == environment.ProcessOfflineState { + return errors.New("cannot enable resource polling on a stopped server") + } + + e.log().Info("starting resource polling for pod") + defer e.log().Debug("stopped resource polling for pod") + + uptime, err := e.Uptime(ctx) + if err != nil { + e.log().WithField("error", err).Warn("failed to calculate pod uptime") + } + + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + + lastCheck := time.Now() + loggedSource := false + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + if e.st.Load() == environment.ProcessOfflineState { + return nil + } + + now := time.Now() + uptime += now.Sub(lastCheck).Milliseconds() + lastCheck = now + + st := environment.Stats{ + Uptime: uptime, + } + + var stats *podStats + + // Prefer the metrics API and fall back to kubelet stats. The metrics + // API is retried on every tick (the fallback is not sticky) so a + // transiently-unavailable metrics-server is picked back up once it + // recovers. Each call is bounded by statsRequestTimeout. + usingKubelet := false + stats, err = withStatsTimeout(ctx, e.getMetricsAPIStats) + if err != nil { + usingKubelet = true + stats, err = withStatsTimeout(ctx, e.getKubeletPodStats) + } + + if stats != nil { + if !loggedSource { + src := "metrics API (metrics-server)" + if usingKubelet { + src = "kubelet stats/summary" + } + e.log().WithField("source", src).Info("collecting pod resource metrics") + loggedSource = true + } + st.Memory = uint64(stats.memoryBytes) + st.CpuAbsolute = stats.cpuPercent + st.Network = environment.NetworkStats{ + RxBytes: stats.rxBytes, + TxBytes: stats.txBytes, + } + } else if !loggedSource { + e.log().WithField("error", err).Warn("pod resource metrics unavailable from both metrics API and kubelet") + loggedSource = true + } + + // Get memory limit from pod spec. + pod, err := withStatsTimeout(ctx, e.getPod) + if err == nil && pod != nil { + for _, c := range pod.Spec.Containers { + if c.Name == "server" { + if mem, ok := c.Resources.Limits["memory"]; ok { + st.MemoryLimit = uint64(mem.Value()) + } + } + } + } + + e.Events().Publish(environment.ResourceEvent, st) + } + } +} + +// --------------------------------------------------------------------------- +// Source 1: Kubernetes Metrics API (requires metrics-server) +// --------------------------------------------------------------------------- + +// getMetricsAPIStats fetches container metrics from the Kubernetes Metrics API +// (metrics.k8s.io/v1beta1). Returns an error if the API is unavailable. +func (e *Environment) getMetricsAPIStats(ctx context.Context) (*podStats, error) { + result := e.client.CoreV1().RESTClient().Get(). + AbsPath("/apis/metrics.k8s.io/v1beta1"). + Resource("pods"). + Namespace(e.namespace()). + Name(e.Id). + Do(ctx) + + if result.Error() != nil { + return nil, result.Error() + } + + raw, err := result.Raw() + if err != nil { + return nil, err + } + + return parseMetricsAPIPodStats(raw) +} + +// podMetricsResponse represents the relevant fields of the Kubernetes Metrics +// API PodMetrics response (metrics.k8s.io/v1beta1). +type podMetricsResponse struct { + Containers []containerMetricsEntry `json:"containers"` +} + +// containerMetricsEntry represents a single container's usage in the Metrics +// API response. +type containerMetricsEntry struct { + Name string `json:"name"` + Usage map[string]string `json:"usage"` +} + +// parseMetricsAPIPodStats extracts the "server" container's CPU/memory from a +// PodMetrics JSON response. Network stats are not available from this API. +func parseMetricsAPIPodStats(raw []byte) (*podStats, error) { + var resp podMetricsResponse + if err := json.Unmarshal(raw, &resp); err != nil { + return nil, errors.Wrap(err, "environment/kubernetes: failed to parse pod metrics") + } + + for _, c := range resp.Containers { + if c.Name != "server" { + continue + } + + s := &podStats{} + + if memStr, ok := c.Usage["memory"]; ok { + s.memoryBytes = parseResourceQuantity(memStr) + } + if cpuStr, ok := c.Usage["cpu"]; ok { + s.cpuPercent = parseCPUToPercent(cpuStr) + } + + return s, nil + } + + return nil, nil +} + +// --------------------------------------------------------------------------- +// Source 2: Kubelet stats/summary API (always available, no metrics-server) +// --------------------------------------------------------------------------- + +// getKubeletPodStats fetches resource metrics from the kubelet stats/summary +// endpoint via the K8s API server proxy. This works on any cluster without +// requiring metrics-server. +func (e *Environment) getKubeletPodStats(ctx context.Context) (*podStats, error) { + pod, err := e.getPod(ctx) + if err != nil { + return nil, errors.Wrap(err, "failed to get pod for kubelet stats") + } + if pod.Spec.NodeName == "" { + return nil, errors.New("pod has no node assignment yet") + } + + result := e.client.CoreV1().RESTClient().Get(). + AbsPath("/api/v1/nodes", pod.Spec.NodeName, "proxy", "stats", "summary"). + Do(ctx) + + if result.Error() != nil { + return nil, errors.Wrap(result.Error(), "kubelet stats/summary request failed") + } + + raw, err := result.Raw() + if err != nil { + return nil, errors.Wrap(err, "failed to read kubelet stats response") + } + + return parseKubeletPodStats(raw, e.Id, e.namespace()) +} + +// kubelet stats/summary response types (only the fields we need). +type kubeletStatsSummary struct { + Pods []kubeletPodStats `json:"pods"` +} + +type kubeletPodStats struct { + PodRef struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + } `json:"podRef"` + Containers []kubeletContainerStats `json:"containers"` + Network *kubeletNetworkStats `json:"network,omitempty"` +} + +type kubeletContainerStats struct { + Name string `json:"name"` + CPU *kubeletCPUStats `json:"cpu,omitempty"` + Memory *kubeletMemoryStats `json:"memory,omitempty"` +} + +type kubeletCPUStats struct { + UsageNanoCores *uint64 `json:"usageNanoCores,omitempty"` +} + +type kubeletMemoryStats struct { + WorkingSetBytes *uint64 `json:"workingSetBytes,omitempty"` +} + +type kubeletNetworkStats struct { + RxBytes *uint64 `json:"rxBytes,omitempty"` + TxBytes *uint64 `json:"txBytes,omitempty"` +} + +// parseKubeletPodStats finds our pod in a kubelet stats/summary response and +// extracts CPU, memory, and network metrics for the "server" container. +func parseKubeletPodStats(raw []byte, podName, namespace string) (*podStats, error) { + var summary kubeletStatsSummary + if err := json.Unmarshal(raw, &summary); err != nil { + return nil, errors.Wrap(err, "failed to parse kubelet stats summary") + } + + for _, p := range summary.Pods { + if p.PodRef.Name != podName || p.PodRef.Namespace != namespace { + continue + } + + s := &podStats{} + + for _, c := range p.Containers { + if c.Name != "server" { + continue + } + if c.CPU != nil && c.CPU.UsageNanoCores != nil { + s.cpuPercent = float64(*c.CPU.UsageNanoCores) / 1e9 * 100 + } + if c.Memory != nil && c.Memory.WorkingSetBytes != nil { + s.memoryBytes = int64(*c.Memory.WorkingSetBytes) + } + } + + if p.Network != nil { + if p.Network.RxBytes != nil { + s.rxBytes = *p.Network.RxBytes + } + if p.Network.TxBytes != nil { + s.txBytes = *p.Network.TxBytes + } + } + + return s, nil + } + + return nil, fmt.Errorf("pod %s/%s not found in kubelet stats", namespace, podName) +} + +// --------------------------------------------------------------------------- +// Parsing helpers +// --------------------------------------------------------------------------- + +// parseResourceQuantity parses a Kubernetes resource quantity string for memory +// and returns the value in bytes. Supports suffixes: Ki, Mi, Gi, Ti, k, M, G, T, +// or plain bytes. +func parseResourceQuantity(s string) int64 { + s = strings.TrimSpace(s) + if s == "" { + return 0 + } + + // Binary suffixes (Ki, Mi, Gi, Ti). + binarySuffixes := map[string]int64{ + "Ki": 1024, + "Mi": 1024 * 1024, + "Gi": 1024 * 1024 * 1024, + "Ti": 1024 * 1024 * 1024 * 1024, + } + for suffix, multiplier := range binarySuffixes { + if strings.HasSuffix(s, suffix) { + val, err := strconv.ParseInt(strings.TrimSuffix(s, suffix), 10, 64) + if err != nil { + return 0 + } + return val * multiplier + } + } + + // Decimal suffixes (k, M, G, T). + decimalSuffixes := map[string]int64{ + "T": 1000000000000, + "G": 1000000000, + "M": 1000000, + "k": 1000, + } + for suffix, multiplier := range decimalSuffixes { + if strings.HasSuffix(s, suffix) { + val, err := strconv.ParseInt(strings.TrimSuffix(s, suffix), 10, 64) + if err != nil { + return 0 + } + return val * multiplier + } + } + + // Plain bytes (no suffix). + val, err := strconv.ParseInt(s, 10, 64) + if err != nil { + return 0 + } + return val +} + +// parseCPUToPercent converts a Kubernetes CPU quantity string to a percentage +// of a single core. Examples: "250m" → 25.0, "1" → 100.0, "0.5" → 50.0, +// "1500m" → 150.0, "100n" → 0.0001. +func parseCPUToPercent(s string) float64 { + s = strings.TrimSpace(s) + if s == "" { + return 0 + } + + // Nanocores suffix "n" (e.g. "250000000n" = 0.25 cores). + if strings.HasSuffix(s, "n") { + val, err := strconv.ParseFloat(strings.TrimSuffix(s, "n"), 64) + if err != nil { + return 0 + } + return (val / 1000000000) * 100 + } + + // Millicores suffix "m" (e.g. "250m" = 0.25 cores). + if strings.HasSuffix(s, "m") { + val, err := strconv.ParseFloat(strings.TrimSuffix(s, "m"), 64) + if err != nil { + return 0 + } + return (val / 1000) * 100 + } + + // Whole/decimal cores (e.g. "1" or "0.5"). + val, err := strconv.ParseFloat(s, 64) + if err != nil { + return 0 + } + return val * 100 +} + +// podLabelSelector returns a label selector for this server's Pod. +func (e *Environment) podLabelSelector() string { + return "pelican.dev/server-id=" + e.Id +} diff --git a/environment/kubernetes/stats_test.go b/environment/kubernetes/stats_test.go new file mode 100644 index 00000000..1de812e6 --- /dev/null +++ b/environment/kubernetes/stats_test.go @@ -0,0 +1,327 @@ +package kubernetes + +import ( + "encoding/json" + "testing" + + . "github.com/franela/goblin" +) + +func mustMarshal(v interface{}) []byte { + b, err := json.Marshal(v) + if err != nil { + panic(err) + } + return b +} + +func TestStats(t *testing.T) { + g := Goblin(t) + + g.Describe("parseMetricsAPIPodStats", func() { + g.It("should parse a valid PodMetrics response", func() { + raw := []byte(`{ + "kind": "PodMetrics", + "metadata": {"name": "test-pod", "namespace": "pelican"}, + "containers": [ + { + "name": "server", + "usage": { + "cpu": "250m", + "memory": "134217728" + } + } + ] + }`) + + metrics, err := parseMetricsAPIPodStats(raw) + g.Assert(err).IsNil() + g.Assert(metrics).IsNotNil() + g.Assert(metrics.memoryBytes).Equal(int64(134217728)) + g.Assert(metrics.cpuPercent).Equal(25.0) + }) + + g.It("should handle nanocores CPU format", func() { + raw := []byte(`{ + "containers": [ + { + "name": "server", + "usage": { + "cpu": "500000000n", + "memory": "67108864" + } + } + ] + }`) + + metrics, err := parseMetricsAPIPodStats(raw) + g.Assert(err).IsNil() + g.Assert(metrics.cpuPercent).Equal(50.0) + }) + + g.It("should handle whole core CPU format", func() { + raw := []byte(`{ + "containers": [ + { + "name": "server", + "usage": { + "cpu": "2", + "memory": "0" + } + } + ] + }`) + + metrics, err := parseMetricsAPIPodStats(raw) + g.Assert(err).IsNil() + g.Assert(metrics.cpuPercent).Equal(200.0) + }) + + g.It("should handle memory with Ki suffix", func() { + raw := []byte(`{ + "containers": [ + { + "name": "server", + "usage": { + "cpu": "0", + "memory": "131072Ki" + } + } + ] + }`) + + metrics, err := parseMetricsAPIPodStats(raw) + g.Assert(err).IsNil() + g.Assert(metrics.memoryBytes).Equal(int64(131072 * 1024)) + }) + + g.It("should handle memory with Mi suffix", func() { + raw := []byte(`{ + "containers": [ + { + "name": "server", + "usage": { + "cpu": "100m", + "memory": "256Mi" + } + } + ] + }`) + + metrics, err := parseMetricsAPIPodStats(raw) + g.Assert(err).IsNil() + g.Assert(metrics.memoryBytes).Equal(int64(256 * 1024 * 1024)) + }) + + g.It("should return nil when server container is not found", func() { + raw := []byte(`{ + "containers": [ + { + "name": "sidecar", + "usage": { + "cpu": "50m", + "memory": "32Mi" + } + } + ] + }`) + + metrics, err := parseMetricsAPIPodStats(raw) + g.Assert(err).IsNil() + g.Assert(metrics == nil).IsTrue() + }) + + g.It("should return nil for empty containers list", func() { + raw := []byte(`{"containers": []}`) + metrics, err := parseMetricsAPIPodStats(raw) + g.Assert(err).IsNil() + g.Assert(metrics == nil).IsTrue() + }) + + g.It("should return error for invalid JSON", func() { + raw := []byte(`not json at all`) + _, err := parseMetricsAPIPodStats(raw) + g.Assert(err).IsNotNil() + }) + + g.It("should handle multi-container pod and find server", func() { + raw := []byte(`{ + "containers": [ + { + "name": "init-container", + "usage": {"cpu": "10m", "memory": "8Mi"} + }, + { + "name": "server", + "usage": {"cpu": "750m", "memory": "512Mi"} + }, + { + "name": "sidecar-proxy", + "usage": {"cpu": "20m", "memory": "16Mi"} + } + ] + }`) + + metrics, err := parseMetricsAPIPodStats(raw) + g.Assert(err).IsNil() + g.Assert(metrics).IsNotNil() + g.Assert(metrics.cpuPercent).Equal(75.0) + g.Assert(metrics.memoryBytes).Equal(int64(512 * 1024 * 1024)) + }) + }) + + g.Describe("parseKubeletPodStats", func() { + g.It("should parse kubelet stats for the target pod", func() { + cpuNano := uint64(500000000) + memBytes := uint64(268435456) + rxBytes := uint64(1024) + txBytes := uint64(2048) + raw := mustMarshal(kubeletStatsSummary{ + Pods: []kubeletPodStats{ + { + PodRef: struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + }{Name: "other-pod", Namespace: "pelican"}, + Containers: []kubeletContainerStats{ + {Name: "server", CPU: &kubeletCPUStats{UsageNanoCores: &cpuNano}, Memory: &kubeletMemoryStats{WorkingSetBytes: &memBytes}}, + }, + }, + { + PodRef: struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + }{Name: "target-pod", Namespace: "pelican"}, + Containers: []kubeletContainerStats{ + {Name: "server", CPU: &kubeletCPUStats{UsageNanoCores: &cpuNano}, Memory: &kubeletMemoryStats{WorkingSetBytes: &memBytes}}, + }, + Network: &kubeletNetworkStats{RxBytes: &rxBytes, TxBytes: &txBytes}, + }, + }, + }) + + stats, err := parseKubeletPodStats(raw, "target-pod", "pelican") + g.Assert(err).IsNil() + g.Assert(stats).IsNotNil() + g.Assert(stats.cpuPercent).Equal(50.0) + g.Assert(stats.memoryBytes).Equal(int64(268435456)) + g.Assert(stats.rxBytes).Equal(uint64(1024)) + g.Assert(stats.txBytes).Equal(uint64(2048)) + }) + + g.It("should return error when pod not found", func() { + raw := mustMarshal(kubeletStatsSummary{Pods: []kubeletPodStats{}}) + _, err := parseKubeletPodStats(raw, "missing", "pelican") + g.Assert(err).IsNotNil() + }) + + g.It("should handle missing network stats", func() { + cpuNano := uint64(100000000) + memBytes := uint64(1024) + raw := mustMarshal(kubeletStatsSummary{ + Pods: []kubeletPodStats{ + { + PodRef: struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + }{Name: "test", Namespace: "ns"}, + Containers: []kubeletContainerStats{ + {Name: "server", CPU: &kubeletCPUStats{UsageNanoCores: &cpuNano}, Memory: &kubeletMemoryStats{WorkingSetBytes: &memBytes}}, + }, + }, + }, + }) + + stats, err := parseKubeletPodStats(raw, "test", "ns") + g.Assert(err).IsNil() + g.Assert(stats.rxBytes).Equal(uint64(0)) + g.Assert(stats.txBytes).Equal(uint64(0)) + }) + + g.It("should handle nil CPU and memory pointers", func() { + raw := mustMarshal(kubeletStatsSummary{ + Pods: []kubeletPodStats{ + { + PodRef: struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + }{Name: "test", Namespace: "ns"}, + Containers: []kubeletContainerStats{ + {Name: "server"}, + }, + }, + }, + }) + + stats, err := parseKubeletPodStats(raw, "test", "ns") + g.Assert(err).IsNil() + g.Assert(stats.cpuPercent).Equal(0.0) + g.Assert(stats.memoryBytes).Equal(int64(0)) + }) + }) + + g.Describe("parseResourceQuantity", func() { + g.It("should parse plain bytes", func() { + g.Assert(parseResourceQuantity("134217728")).Equal(int64(134217728)) + }) + + g.It("should parse Ki suffix", func() { + g.Assert(parseResourceQuantity("128Ki")).Equal(int64(128 * 1024)) + }) + + g.It("should parse Mi suffix", func() { + g.Assert(parseResourceQuantity("512Mi")).Equal(int64(512 * 1024 * 1024)) + }) + + g.It("should parse Gi suffix", func() { + g.Assert(parseResourceQuantity("2Gi")).Equal(int64(2 * 1024 * 1024 * 1024)) + }) + + g.It("should parse decimal k suffix", func() { + g.Assert(parseResourceQuantity("500k")).Equal(int64(500000)) + }) + + g.It("should parse decimal M suffix", func() { + g.Assert(parseResourceQuantity("100M")).Equal(int64(100000000)) + }) + + g.It("should return 0 for empty string", func() { + g.Assert(parseResourceQuantity("")).Equal(int64(0)) + }) + + g.It("should return 0 for invalid input", func() { + g.Assert(parseResourceQuantity("not-a-number")).Equal(int64(0)) + }) + }) + + g.Describe("parseCPUToPercent", func() { + g.It("should parse millicores", func() { + g.Assert(parseCPUToPercent("250m")).Equal(25.0) + g.Assert(parseCPUToPercent("1000m")).Equal(100.0) + g.Assert(parseCPUToPercent("1500m")).Equal(150.0) + }) + + g.It("should parse nanocores", func() { + g.Assert(parseCPUToPercent("250000000n")).Equal(25.0) + g.Assert(parseCPUToPercent("1000000000n")).Equal(100.0) + }) + + g.It("should parse whole cores", func() { + g.Assert(parseCPUToPercent("1")).Equal(100.0) + g.Assert(parseCPUToPercent("2")).Equal(200.0) + }) + + g.It("should parse decimal cores", func() { + g.Assert(parseCPUToPercent("0.5")).Equal(50.0) + g.Assert(parseCPUToPercent("0.25")).Equal(25.0) + }) + + g.It("should return 0 for empty string", func() { + g.Assert(parseCPUToPercent("")).Equal(0.0) + }) + + g.It("should return 0 for invalid input", func() { + g.Assert(parseCPUToPercent("invalid")).Equal(0.0) + }) + }) +} diff --git a/environment/kubernetes/storage.go b/environment/kubernetes/storage.go new file mode 100644 index 00000000..bf3b82ca --- /dev/null +++ b/environment/kubernetes/storage.go @@ -0,0 +1,175 @@ +package kubernetes + +import ( + "context" + "fmt" + + "emperror.dev/errors" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/pelican/wings/config" +) + +// pvcName returns the PersistentVolumeClaim name for this server. +func (e *Environment) pvcName() string { + return fmt.Sprintf("gs-%s", e.Id) +} + +// EnsurePVC creates the PersistentVolumeClaim for this server if it does not +// already exist. When StorageMode is "hostpath" or DataPVC is set (shared PVC +// mode), this is a no-op. +func (e *Environment) EnsurePVC(ctx context.Context) error { + cfg := config.Get() + if cfg.Kubernetes.StorageMode != config.KubeStoragePVC { + return nil + } + // Shared PVC mode: the Wings data PVC is already mounted. No per-server + // PVC is needed; the server data directory is a subPath of the shared PVC. + if cfg.Kubernetes.DataPVC != "" { + return nil + } + + ns := e.namespace() + name := e.pvcName() + + // Check if PVC already exists. + _, err := e.client.CoreV1().PersistentVolumeClaims(ns).Get(ctx, name, metav1.GetOptions{}) + if err == nil { + return nil // Already exists. + } + if !isNotFound(err) { + return errors.Wrap(err, "environment/kubernetes: failed to check existing PVC") + } + + // Parse storage size. + storageSize, err := resource.ParseQuantity(cfg.Kubernetes.StorageSize) + if err != nil { + return errors.Wrap(err, "environment/kubernetes: invalid storage_size in config") + } + + // Determine access mode. + accessMode := corev1.ReadWriteOnce + if cfg.Kubernetes.StorageAccessMode == "ReadWriteMany" { + accessMode = corev1.ReadWriteMany + } + + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + Labels: map[string]string{ + "app.kubernetes.io/managed-by": "pelican-wings", + "pelican.dev/server-id": e.Id, + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{accessMode}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: storageSize, + }, + }, + }, + } + + // Set storage class if configured. + if cfg.Kubernetes.StorageClass != "" { + pvc.Spec.StorageClassName = &cfg.Kubernetes.StorageClass + } + + e.log().WithField("pvc", name).Info("creating PersistentVolumeClaim for server") + _, err = e.client.CoreV1().PersistentVolumeClaims(ns).Create(ctx, pvc, metav1.CreateOptions{}) + if err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to create PVC") + } + + return nil +} + +// DeletePVC removes the PersistentVolumeClaim for this server. This +// permanently deletes the server's data. When StorageMode is "hostpath" or +// DataPVC is set (shared PVC mode), this is a no-op. +func (e *Environment) DeletePVC(ctx context.Context) error { + cfg := config.Get() + if cfg.Kubernetes.StorageMode != config.KubeStoragePVC { + return nil + } + if cfg.Kubernetes.DataPVC != "" { + return nil + } + + ns := e.namespace() + name := e.pvcName() + + err := e.client.CoreV1().PersistentVolumeClaims(ns).Delete(ctx, name, metav1.DeleteOptions{}) + if err != nil && !isNotFound(err) { + return errors.Wrap(err, "environment/kubernetes: failed to delete PVC") + } + + if err == nil { + e.log().WithField("pvc", name).Info("deleted PersistentVolumeClaim for server") + } + + return nil +} + +// GetPVCStatus returns the current status of the server's PVC, or nil if not +// in PVC mode or if the PVC doesn't exist. +func (e *Environment) GetPVCStatus(ctx context.Context) (*corev1.PersistentVolumeClaimPhase, error) { + cfg := config.Get() + if cfg.Kubernetes.StorageMode != config.KubeStoragePVC { + return nil, nil + } + + ns := e.namespace() + name := e.pvcName() + + pvc, err := e.client.CoreV1().PersistentVolumeClaims(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + if isNotFound(err) { + return nil, nil + } + return nil, errors.Wrap(err, "environment/kubernetes: failed to get PVC status") + } + + return &pvc.Status.Phase, nil +} + +// ResizePVC updates the storage request on the PVC to the given size. The +// underlying StorageClass must support volume expansion. +func (e *Environment) ResizePVC(ctx context.Context, newSize string) error { + cfg := config.Get() + if cfg.Kubernetes.StorageMode != config.KubeStoragePVC { + return nil + } + // When a shared DataPVC is configured, per-server PVCs are not created, so + // there is nothing to resize (mirrors EnsurePVC/DeletePVC). + if cfg.Kubernetes.DataPVC != "" { + return nil + } + + ns := e.namespace() + name := e.pvcName() + + quantity, err := resource.ParseQuantity(newSize) + if err != nil { + return errors.Wrap(err, "environment/kubernetes: invalid size for PVC resize") + } + + pvc, err := e.client.CoreV1().PersistentVolumeClaims(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to get PVC for resize") + } + + pvc.Spec.Resources.Requests[corev1.ResourceStorage] = quantity + + _, err = e.client.CoreV1().PersistentVolumeClaims(ns).Update(ctx, pvc, metav1.UpdateOptions{}) + if err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to resize PVC") + } + + e.log().WithField("pvc", name).WithField("new_size", newSize).Info("resized PersistentVolumeClaim") + return nil +} diff --git a/environment/kubernetes/storage_test.go b/environment/kubernetes/storage_test.go new file mode 100644 index 00000000..0bf1b1bd --- /dev/null +++ b/environment/kubernetes/storage_test.go @@ -0,0 +1,389 @@ +package kubernetes + +import ( + "context" + "testing" + + . "github.com/franela/goblin" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + + "github.com/pelican/wings/config" + "github.com/pelican/wings/environment" + "github.com/pelican/wings/system" +) + +func TestStorage(t *testing.T) { + g := Goblin(t) + + g.Describe("PVC lifecycle", func() { + g.Describe("pvcName", func() { + g.It("should return correct PVC name", func() { + env := &Environment{Id: "abc-123-def"} + g.Assert(env.pvcName()).Equal("gs-abc-123-def") + }) + }) + + g.Describe("EnsurePVC", func() { + g.It("should be a no-op in hostpath mode", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "hostpath-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.StorageMode = config.KubeStorageHostPath + c.Kubernetes.Namespace = "pelican" + }) + + err := env.EnsurePVC(context.Background()) + g.Assert(err).IsNil() + + // No PVC should exist. + pvcs, _ := client.CoreV1().PersistentVolumeClaims("pelican").List(context.Background(), metav1.ListOptions{}) + g.Assert(len(pvcs.Items)).Equal(0) + }) + + g.It("should create a PVC in pvc mode", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "pvc-create-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.StorageMode = config.KubeStoragePVC + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.StorageSize = "20Gi" + c.Kubernetes.StorageClass = "fast-ssd" + c.Kubernetes.StorageAccessMode = "ReadWriteOnce" + }) + + err := env.EnsurePVC(context.Background()) + g.Assert(err).IsNil() + + pvc, err := client.CoreV1().PersistentVolumeClaims("pelican").Get(context.Background(), "gs-pvc-create-uuid", metav1.GetOptions{}) + g.Assert(err).IsNil() + g.Assert(pvc.Name).Equal("gs-pvc-create-uuid") + g.Assert(pvc.Labels["pelican.dev/server-id"]).Equal("pvc-create-uuid") + g.Assert(pvc.Spec.AccessModes[0]).Equal(corev1.ReadWriteOnce) + + // Check storage size. + storageReq := pvc.Spec.Resources.Requests[corev1.ResourceStorage] + expected := resource.MustParse("20Gi") + g.Assert(storageReq.Equal(expected)).IsTrue() + + // Check storage class. + g.Assert(*pvc.Spec.StorageClassName).Equal("fast-ssd") + }) + + g.It("should create PVC with ReadWriteMany access mode", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "rwm-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.StorageMode = config.KubeStoragePVC + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.StorageSize = "5Gi" + c.Kubernetes.StorageAccessMode = "ReadWriteMany" + c.Kubernetes.StorageClass = "" + }) + + err := env.EnsurePVC(context.Background()) + g.Assert(err).IsNil() + + pvc, _ := client.CoreV1().PersistentVolumeClaims("pelican").Get(context.Background(), "gs-rwm-uuid", metav1.GetOptions{}) + g.Assert(pvc.Spec.AccessModes[0]).Equal(corev1.ReadWriteMany) + // No storage class set — should be nil. + g.Assert(pvc.Spec.StorageClassName == nil).IsTrue() + }) + + g.It("should not recreate existing PVC", func() { + existingPVC := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gs-existing-uuid", + Namespace: "pelican", + }, + } + client := fake.NewSimpleClientset(existingPVC) + env := &Environment{ + Id: "existing-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.StorageMode = config.KubeStoragePVC + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.StorageSize = "10Gi" + }) + + err := env.EnsurePVC(context.Background()) + g.Assert(err).IsNil() + }) + + g.It("should error on invalid storage size", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "bad-size-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.StorageMode = config.KubeStoragePVC + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.StorageSize = "not-a-quantity" + }) + + err := env.EnsurePVC(context.Background()) + g.Assert(err).IsNotNil() + }) + }) + + g.Describe("DeletePVC", func() { + g.It("should be a no-op in hostpath mode", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "hostpath-del-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.StorageMode = config.KubeStorageHostPath + c.Kubernetes.Namespace = "pelican" + }) + + err := env.DeletePVC(context.Background()) + g.Assert(err).IsNil() + }) + + g.It("should delete an existing PVC", func() { + existingPVC := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gs-del-uuid", + Namespace: "pelican", + }, + } + client := fake.NewSimpleClientset(existingPVC) + env := &Environment{ + Id: "del-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.StorageMode = config.KubeStoragePVC + c.Kubernetes.Namespace = "pelican" + }) + + err := env.DeletePVC(context.Background()) + g.Assert(err).IsNil() + + _, err = client.CoreV1().PersistentVolumeClaims("pelican").Get(context.Background(), "gs-del-uuid", metav1.GetOptions{}) + g.Assert(err).IsNotNil() + }) + + g.It("should not error when PVC does not exist", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "nonexist-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.StorageMode = config.KubeStoragePVC + c.Kubernetes.Namespace = "pelican" + }) + + err := env.DeletePVC(context.Background()) + g.Assert(err).IsNil() + }) + }) + + g.Describe("ResizePVC", func() { + g.It("should be a no-op in hostpath mode", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "resize-hp-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.StorageMode = config.KubeStorageHostPath + c.Kubernetes.Namespace = "pelican" + }) + + err := env.ResizePVC(context.Background(), "50Gi") + g.Assert(err).IsNil() + }) + + g.It("should update PVC storage request", func() { + existingPVC := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gs-resize-uuid", + Namespace: "pelican", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("10Gi"), + }, + }, + }, + } + client := fake.NewSimpleClientset(existingPVC) + env := &Environment{ + Id: "resize-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.StorageMode = config.KubeStoragePVC + c.Kubernetes.Namespace = "pelican" + }) + + err := env.ResizePVC(context.Background(), "50Gi") + g.Assert(err).IsNil() + + pvc, _ := client.CoreV1().PersistentVolumeClaims("pelican").Get(context.Background(), "gs-resize-uuid", metav1.GetOptions{}) + storageReq := pvc.Spec.Resources.Requests[corev1.ResourceStorage] + expected := resource.MustParse("50Gi") + g.Assert(storageReq.Equal(expected)).IsTrue() + }) + + g.It("should error on invalid size", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "bad-resize-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.StorageMode = config.KubeStoragePVC + c.Kubernetes.Namespace = "pelican" + }) + + err := env.ResizePVC(context.Background(), "invalid") + g.Assert(err).IsNotNil() + }) + + g.It("should be a no-op when a shared DataPVC is configured", func() { + existingPVC := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gs-shared-resize-uuid", + Namespace: "pelican", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("10Gi"), + }, + }, + }, + } + client := fake.NewSimpleClientset(existingPVC) + env := &Environment{ + Id: "shared-resize-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.StorageMode = config.KubeStoragePVC + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.DataPVC = "shared-data" + }) + defer config.Update(func(c *config.Configuration) { + c.Kubernetes.DataPVC = "" + }) + + err := env.ResizePVC(context.Background(), "50Gi") + g.Assert(err).IsNil() + + // The per-server PVC must be left untouched at its original size. + pvc, _ := client.CoreV1().PersistentVolumeClaims("pelican").Get(context.Background(), "gs-shared-resize-uuid", metav1.GetOptions{}) + storageReq := pvc.Spec.Resources.Requests[corev1.ResourceStorage] + original := resource.MustParse("10Gi") + g.Assert(storageReq.Equal(original)).IsTrue() + }) + }) + + g.Describe("buildVolumes with PVC mode", func() { + g.It("should use PVC for default mount in pvc mode", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "bv-pvc-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.StorageMode = config.KubeStoragePVC + c.Kubernetes.Namespace = "pelican" + }) + + mounts := []environment.Mount{ + {Default: true, Source: "/data/servers/bv-pvc-uuid", Target: "/home/container"}, + {Default: false, Source: "/shared/plugins", Target: "/plugins", ReadOnly: true}, + } + + volumes, volumeMounts := env.buildVolumes(mounts) + + g.Assert(len(volumes)).Equal(2) + // First volume (default) should use PVC. + g.Assert(volumes[0].Name).Equal("server-data") + g.Assert(volumes[0].PersistentVolumeClaim).IsNotNil() + g.Assert(volumes[0].PersistentVolumeClaim.ClaimName).Equal("gs-bv-pvc-uuid") + g.Assert(volumes[0].HostPath == nil).IsTrue() + + // Second volume (non-default) should still use HostPath. + g.Assert(volumes[1].Name).Equal("mount-1") + g.Assert(volumes[1].HostPath).IsNotNil() + g.Assert(volumes[1].HostPath.Path).Equal("/shared/plugins") + + g.Assert(len(volumeMounts)).Equal(2) + g.Assert(volumeMounts[0].MountPath).Equal("/home/container") + g.Assert(volumeMounts[1].MountPath).Equal("/plugins") + g.Assert(volumeMounts[1].ReadOnly).IsTrue() + }) + + g.It("should use HostPath for default mount in hostpath mode", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "bv-hp-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.StorageMode = config.KubeStorageHostPath + c.Kubernetes.Namespace = "pelican" + }) + + mounts := []environment.Mount{ + {Default: true, Source: "/data/servers/test", Target: "/home/container"}, + } + + volumes, _ := env.buildVolumes(mounts) + g.Assert(volumes[0].HostPath).IsNotNil() + g.Assert(volumes[0].HostPath.Path).Equal("/data/servers/test") + g.Assert(volumes[0].PersistentVolumeClaim == nil).IsTrue() + }) + }) + }) +} diff --git a/environment/kubernetes/testutil_test.go b/environment/kubernetes/testutil_test.go new file mode 100644 index 00000000..c837adfc --- /dev/null +++ b/environment/kubernetes/testutil_test.go @@ -0,0 +1,21 @@ +package kubernetes + +import ( + "os" + "testing" + + "github.com/pelican/wings/config" +) + +func TestMain(m *testing.M) { + // Initialize a minimal config so config.Get() doesn't panic. + c := &config.Configuration{} + c.AuthenticationToken = "test-token-for-testing" + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.NetworkMode = config.KubeNetworkHostPort + c.Kubernetes.NodePortRangeMin = 30000 + c.Kubernetes.NodePortRangeMax = 32767 + config.Set(c) + + os.Exit(m.Run()) +} diff --git a/go.mod b/go.mod index 33821004..60768723 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/pelican/wings -go 1.25.0 +go 1.26.0 require ( emperror.dev/errors v0.8.1 @@ -24,7 +24,7 @@ require ( github.com/go-co-op/gocron/v2 v2.21.2 github.com/goccy/go-json v0.10.6 github.com/google/uuid v1.6.0 - github.com/gorilla/websocket v1.5.3 + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 github.com/iancoleman/strcase v0.3.0 github.com/icza/dyno v0.0.0-20230330125955-09f820a8d9c0 github.com/juju/ratelimit v1.0.2 @@ -51,6 +51,9 @@ require ( gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 gorm.io/gorm v1.31.1 + k8s.io/api v0.36.1 + k8s.io/apimachinery v0.36.1 + k8s.io/client-go v0.36.1 ) require ( @@ -69,26 +72,52 @@ require ( github.com/charmbracelet/x/term v0.2.2 // indirect github.com/clipperhouse/displaywidth v0.11.0 // indirect github.com/clipperhouse/uax29/v2 v2.7.0 // indirect + github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect github.com/goccy/go-yaml v1.19.2 // indirect + github.com/google/gnostic-models v0.7.0 // indirect github.com/hashicorp/go-version v1.9.0 // indirect + github.com/josharian/intern v1.0.0 // indirect github.com/lucasb-eyer/go-colorful v1.4.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.24 // indirect github.com/mikelolasagasti/xz v1.0.1 // indirect github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect + github.com/moby/spdystream v0.5.1 // indirect github.com/moby/sys/mountinfo v0.7.2 // indirect github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.16.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/quic-go/qpack v0.6.0 // indirect github.com/quic-go/quic-go v0.60.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/spf13/afero v1.15.0 // indirect github.com/stangelandcl/ppmd v0.1.1 // indirect github.com/tidwall/match v1.2.0 // indirect + github.com/x448/float16 v0.8.4 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect go.mongodb.org/mongo-driver/v2 v2.7.0 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect + golang.org/x/oauth2 v0.35.0 // indirect + golang.org/x/term v0.44.0 // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + k8s.io/klog/v2 v2.140.0 // indirect + k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect + k8s.io/streaming v0.36.1 // indirect + k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect ) require ( @@ -107,7 +136,7 @@ require ( github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/distribution/reference v0.6.0 // indirect + github.com/distribution/reference v0.6.0 github.com/docker/go-units v0.5.0 // indirect github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -136,10 +165,10 @@ require ( github.com/minio/minlz v1.1.1 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/sys/atomicwriter v0.1.0 // indirect - github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae // indirect + github.com/moby/term v0.5.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/morikuni/aec v1.0.0 // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/morikuni/aec v1.1.0 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect github.com/nwaples/rardecode/v2 v2.2.5 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect @@ -164,7 +193,7 @@ require ( go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect go.opentelemetry.io/otel v1.44.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0 // indirect go.opentelemetry.io/otel/metric v1.44.0 // indirect go.opentelemetry.io/otel/trace v1.44.0 // indirect go.uber.org/multierr v1.11.0 // indirect @@ -174,8 +203,8 @@ require ( golang.org/x/text v0.38.0 // indirect golang.org/x/time v0.15.0 golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect - google.golang.org/protobuf v1.36.11 // indirect - gotest.tools/v3 v3.0.2 // indirect + google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect + gotest.tools/v3 v3.5.2 // indirect modernc.org/libc v1.73.4 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect diff --git a/go.sum b/go.sum index 76375125..36789c86 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ emperror.dev/errors v0.8.1 h1:UavXZ5cSX/4u9iyvH6aDcuGkVjeexUGJ7Ij7G4VfQT0= emperror.dev/errors v0.8.1/go.mod h1:YcRvLPh626Ubn2xqtoprejnA5nFha+TJ+2vew48kWuE= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= @@ -21,6 +21,8 @@ github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo= github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE= github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= @@ -51,6 +53,8 @@ github.com/catppuccin/go v0.3.0 h1:d+0/YicIq+hSTo5oPuRi5kOpqkVA5tAsU6dNhvRu+aY= github.com/catppuccin/go v0.3.0/go.mod h1:8IHJuMGaUUjQM82qBrGNBv7LFq6JI3NnQCF6MOlZjpc= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/bubbles v1.0.0 h1:12J8/ak/uCZEMQ6KU7pcfwceyjLlWsDLAxB5fXonfvc= @@ -94,7 +98,7 @@ github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmC github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= -github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/creasty/defaults v1.8.0 h1:z27FJxCAa0JKt3utc0sCImAEb+spPucmKoOdLHvHYKk= @@ -116,6 +120,8 @@ github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj6 github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= +github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -126,6 +132,8 @@ github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9 github.com/franela/goblin v0.0.0-20211003143422-0a4f594942bf h1:NrF81UtW8gG2LBGkXFQFqlfNnvMt9WdB46sfdJY4oqc= github.com/franela/goblin v0.0.0-20211003143422-0a4f594942bf/go.mod h1:VzmDKDJVZI3aJmnRI9VjAn9nJ8qPPsN1fqzr9dqInIo= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM= github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= github.com/gammazero/deque v1.2.1 h1:9fnQVFCCZ9/NOc7ccTNqzoKd1tCWOqeI05/lPqFPMGQ= @@ -153,6 +161,14 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= @@ -167,7 +183,8 @@ github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -178,10 +195,10 @@ github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= -github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 h1:Wqo399gCIufwto+VfwCSvsnfGpF/w5E9CNxSwbpD6No= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0/go.mod h1:qmOFXW2epJhM0qSnUUYpldc7gVz2KMQwJ/QYCDIa7XU= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= github.com/hashicorp/go-version v1.9.0 h1:CeOIz6k+LoN3qX9Z0tyQrPtiB1DFYRPfCIBtaXPSCnA= github.com/hashicorp/go-version v1.9.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= @@ -200,6 +217,8 @@ github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= @@ -217,6 +236,7 @@ github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -234,6 +254,8 @@ github.com/magefile/mage v1.17.2 h1:fyXVu1eadI8Ap1HCCNgEhJ5McIWiYhLR8uol64ZZc40= github.com/magefile/mage v1.17.2/go.mod h1:Yj51kqllmsgFpvvSzgrZPK9WtluG3kUhFaBUVLo4feA= github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY= @@ -259,27 +281,32 @@ github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4 github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/spdystream v0.5.1 h1:9sNYeYZUcci9R6/w7KDaFWEWeV4LStVG78Mpyq/Zm/Y= +github.com/moby/spdystream v0.5.1/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= -github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae h1:O4SWKdcHVCvYqyDV+9CJA1fcDN2L11Bule0iFy3YlAI= -github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/morikuni/aec v1.1.0 h1:vBBl0pUnvi/Je71dsRrhMBtreIqNMYErSAbEeb8jrXQ= +github.com/morikuni/aec v1.1.0/go.mod h1:xDRgiq/iw5l+zkao76YTKzKttOp2cwPEne25HDkJnBw= github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo= github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/nwaples/rardecode/v2 v2.2.5 h1:L5doqgGfQwI7qADJMqnkrSB86rpPsqQDrHeO0HWa5JY= @@ -344,7 +371,6 @@ github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -393,6 +419,8 @@ github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2W github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY= github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= @@ -407,10 +435,10 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 h1:t6wl9SPayj+c7lEIFgm4ooDBZVb01IhLB4InpomhRw8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0/go.mod h1:iSDOcsnSA5INXzZtwaBPrKp/lWu/V14Dd+llD0oI2EA= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0 h1:Xw8U6u2f8DK2XAkGRFV7BBLENgnTGX9i4rQRxJf+/vs= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0/go.mod h1:6KW1Fm6R/s6Z3PGXwSJN2K4eT6wQB3vXX6CVnYX9NmM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 h1:4YsVu3B8+3qtWYYrsUYgn0OG78pN0rnNPRGX4SbokQI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0/go.mod h1:+wnlSn0mD1ADVMe3v9Z/WIaiz6q6gL2J/ejaAmdmv80= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0 h1:lgh3PiVrRUWMLOVSkQicxzZll5NjF1r+AtsX1XRIHw0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0/go.mod h1:5Cnhth3m/AgOeTgE3ex12pPmiu/gGtZit03kSzx9X7s= go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= @@ -419,8 +447,8 @@ go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRk go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= -go.opentelemetry.io/proto/otlp v1.1.0 h1:2Di21piLrCqJ3U3eXGCTPHE9R8Nh+0uglSnOyxikMeI= -go.opentelemetry.io/proto/otlp v1.1.0/go.mod h1:GpBHCBWiqvVLDqmHZsoMM3C5ySeKTC7ej/RNTae6MdY= +go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= +go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= @@ -429,6 +457,9 @@ go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go4.org v0.0.0-20260112195520-a5071408f32f h1:ziUVAjmTPwQMBmYR1tbdRFJPtTcQUI12fH9QQjfb0Sw= go4.org v0.0.0-20260112195520-a5071408f32f/go.mod h1:ZRJnO5ZI4zAwMFp+dS1+V6J6MSyAowhRqAE+DPa1Xp0= @@ -439,8 +470,8 @@ golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo= +golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= @@ -450,6 +481,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= +golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= @@ -460,7 +493,6 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= @@ -475,7 +507,6 @@ golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= @@ -484,19 +515,23 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217 h1:GvESR9BIyHUahIb0NcTum6itIWtdoglGX+rnGxm2934= -google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls= -google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b h1:Mv8VFug0MP9e5vUxfBcE3vUkV6CImK3cMNMIDFjmzxU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= -google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= -google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= -google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= -google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= +google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw= gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -510,8 +545,22 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/gorm v1.31.1 h1:7CA8FTFz/gRfgqgpeKIBcervUn3xSyPUmr6B2WXJ7kg= gorm.io/gorm v1.31.1/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs= -gotest.tools/v3 v3.0.2 h1:kG1BFyqVHuQoVQiR1bWGnfz/fmHvvuiSPIV7rvl360E= -gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= +k8s.io/api v0.36.1 h1:XbL/EMj8K2aJpJtePmqUyQMsM0D4QI2pvl7YKJ20FTY= +k8s.io/api v0.36.1/go.mod h1:KOWo4ey3TINlXjeHVuwB3i+tXXnu+UcwFBHlI/9dvEo= +k8s.io/apimachinery v0.36.1 h1:G63Gjx2W+q0YD+72Vo8oY0nDnePVwnuzTmmy5ENrVSA= +k8s.io/apimachinery v0.36.1/go.mod h1:ibYOR00vW/I1kzvi5SF0dRuJ52BvKtfvRdOn35GPQ+8= +k8s.io/client-go v0.36.1 h1:FN/K8QIT2CEDt+2WB2HnWrUANZ50AP5GII43/SP2JR0= +k8s.io/client-go v0.36.1/go.mod h1:s6rAnCtTGYDQnpNjEhSaISV+2O8jwruZ6m3QOYBFbtU= +k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= +k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= +k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a h1:xCeOEAOoGYl2jnJoHkC3hkbPJgdATINPMAxaynU2Ovg= +k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= +k8s.io/streaming v0.36.1 h1:L+K68n4Gg940BGNNYtUBvL1WTLL0YnKT3s+P1MNAmR4= +k8s.io/streaming v0.36.1/go.mod h1:z6fV3D+NVkoeqRMtWwlUZK6U17SY/LqNzOxWL6GyR/s= +k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU= +k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= modernc.org/cc/v4 v4.28.4 h1:Hd/4Es+MBj+/7hSdZaisNyu6bv3V0Dp2MdllyfqaH+c= modernc.org/cc/v4 v4.28.4/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= modernc.org/ccgo/v4 v4.34.4 h1:OVnSOWQjVKOYkFxoHYB+qQmSHK5gqMqARM+K9DpR/Ws= @@ -540,3 +589,11 @@ modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2 h1:kwVWMx5yS1CrnFWA/2QHyRVJ8jM6dBA80uLmm0wJkk8= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/kubernetes/README.md b/kubernetes/README.md new file mode 100644 index 00000000..9b113ce4 --- /dev/null +++ b/kubernetes/README.md @@ -0,0 +1,135 @@ +# Kubernetes RBAC Configuration + +This directory contains the required Kubernetes RBAC manifests for running Wings +in Kubernetes mode. These resources grant Wings the minimum permissions needed to +manage game server workloads as Pods within a namespace. + +## Quick Start + +```bash +# Apply all RBAC resources +kubectl apply -f kubernetes/ + +# Verify +kubectl get serviceaccount pelican-wings -n pelican +kubectl get role pelican-wings -n pelican +kubectl get rolebinding pelican-wings -n pelican +kubectl get clusterrole pelican-wings-metrics +kubectl get clusterrolebinding pelican-wings-metrics +``` + +## Architecture + +Wings requires two levels of RBAC: + +1. **Namespace-scoped (Role + RoleBinding)** — Manages Pods, Services, Jobs, and + PVCs within the `pelican` namespace. +2. **Cluster-scoped (ClusterRole + ClusterRoleBinding)** — Reads Pod metrics from + the `metrics.k8s.io` API (which is cluster-scoped). + +## Required Permissions + +### Namespace-scoped (pelican namespace) + +| Resource | Verbs | Purpose | +|---------------------------|--------------------------------|------------------------------------------| +| pods | get, create, delete, list, watch | Game server Pod lifecycle | +| pods/log | get | Stream server console output | +| pods/attach | create | Interactive console (SPDY attach) | +| services | get, create, update, delete, list | NodePort/LoadBalancer Service management | +| jobs | get, create, delete | Egg installation scripts | +| configmaps | get, create, update, delete | Install script + identity files (multi-node) | +| resourcequotas | get, create, update | Namespace resource limits (if enabled) | +| limitranges | get, create, update | Container default limits (if enabled) | +| persistentvolumeclaims | get, create, update, delete, list | PVC storage lifecycle (if enabled) | + +### Cluster-scoped + +| Resource | Verbs | Purpose | +|---------------------------------------|-------|--------------------------------------------------------| +| pods (metrics.k8s.io/v1beta1) | get | CPU/memory usage polling | +| nodes | get | Read node addresses for the IP-allocation endpoint | +| nodes/proxy *(optional)* | get | Kubelet stats fallback when metrics-server is absent | + +> `nodes/proxy` is a broad, cluster-wide permission and is therefore **not** +> granted by `clusterrole-metrics.yaml`. Apply `clusterrole-metrics-kubelet.yaml` +> only if you do not run metrics-server and need the kubelet stats fallback. + +## Files + +- `namespace.yaml` — Namespace definition +- `serviceaccount.yaml` — ServiceAccount for Wings Pods +- `role.yaml` — Namespace-scoped Role with required permissions +- `rolebinding.yaml` — Binds the Role to the ServiceAccount +- `clusterrole-metrics.yaml` — Cluster-scoped access to the metrics API + node addresses +- `clusterrolebinding-metrics.yaml` — Binds the ClusterRole to the ServiceAccount +- `clusterrole-metrics-kubelet.yaml` — **Optional** `nodes/proxy` ClusterRole + binding for the kubelet stats fallback + +## Configuration + +After applying these manifests, configure Wings to use the ServiceAccount: + +```yaml +# config.yml +kubernetes: + enabled: true + namespace: pelican + service_account: pelican-wings +``` + +If Wings runs **inside** the cluster as a Pod, assign the ServiceAccount directly +to the Wings Deployment/Pod. If Wings runs **outside** the cluster, create a +kubeconfig that authenticates as the ServiceAccount (or use a token). + +## Customization + +### Different namespace + +Replace `pelican` with your namespace in all manifests: + +```bash +sed -i 's/namespace: pelican/namespace: my-namespace/g' kubernetes/*.yaml +``` + +### Disable PVC permissions + +If you only use `storage_mode: hostpath`, you can remove the +`persistentvolumeclaims` resource from `role.yaml`. + +### Image pulling + +By default, game server Pods and installation Jobs use `imagePullPolicy: Always` +for remote images so updated tags are re-pulled instead of reusing a stale node +cache (matching the Docker backend); `~`-prefixed local images are never pulled. +For air-gapped clusters, pin the policy: + +```yaml +# config.yml +kubernetes: + image_pull_policy: IfNotPresent # or "Never" +``` + +### Disable metrics + +If you don't have metrics-server installed, you can skip the ClusterRole and +ClusterRoleBinding. Wings will gracefully degrade (no CPU/memory stats). + +### Multiple Wings instances + +If you run multiple Wings nodes targeting different namespaces, create a Role and +RoleBinding per namespace, but share the ClusterRole/ClusterRoleBinding (it's +namespace-independent). Give each instance its own ServiceAccount and use a +distinct `ClusterRoleBinding` subject per ServiceAccount so the cluster-scoped +metrics permission is granted to every Wings ServiceAccount that needs it. + +Do **not** point multiple Wings nodes at the same namespace: they share Pod/ +Service/Job names derived from server UUIDs and the namespaced ResourceQuota/ +LimitRange (`pelican-wings`), so concurrent reconciliation would conflict. + +### Helm-managed installs + +The Helm chart under `chart/pelican-wings` renders all of these resources (and a +config **Secret**) for you. When `serviceAccount.create=false` you must set +`serviceAccount.name` explicitly — the chart refuses to bind its RBAC to the +namespace `default` ServiceAccount. Enable the kubelet fallback with +`rbac.kubeletMetricsFallback=true`. diff --git a/kubernetes/clusterrole-metrics-kubelet.yaml b/kubernetes/clusterrole-metrics-kubelet.yaml new file mode 100644 index 00000000..c95eb551 --- /dev/null +++ b/kubernetes/clusterrole-metrics-kubelet.yaml @@ -0,0 +1,39 @@ +# OPTIONAL kubelet stats/summary fallback for resource metrics. +# +# Only apply this if you do NOT run metrics-server and want Wings to read CPU/ +# memory usage directly from the kubelet. nodes/proxy is a broad, cluster-wide +# permission, so it is kept out of the default clusterrole-metrics.yaml and must +# be opted into explicitly: +# +# kubectl apply -f clusterrole-metrics-kubelet.yaml +# +# Adjust the ServiceAccount name/namespace below to match your install. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: pelican-wings-metrics-kubelet + labels: + app.kubernetes.io/managed-by: pelican-wings + app.kubernetes.io/component: wings +rules: + # Proxy to the kubelet stats/summary API for resource metrics when + # metrics-server is not installed. + - apiGroups: [""] + resources: ["nodes/proxy"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: pelican-wings-metrics-kubelet + labels: + app.kubernetes.io/managed-by: pelican-wings + app.kubernetes.io/component: wings +subjects: + - kind: ServiceAccount + name: pelican-wings + namespace: pelican +roleRef: + kind: ClusterRole + name: pelican-wings-metrics-kubelet + apiGroup: rbac.authorization.k8s.io diff --git a/kubernetes/clusterrole-metrics.yaml b/kubernetes/clusterrole-metrics.yaml new file mode 100644 index 00000000..82624ffd --- /dev/null +++ b/kubernetes/clusterrole-metrics.yaml @@ -0,0 +1,23 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: pelican-wings-metrics + labels: + app.kubernetes.io/managed-by: pelican-wings + app.kubernetes.io/component: wings +rules: + # Read Pod metrics from the Kubernetes Metrics API (metrics.k8s.io/v1beta1). + # Required for CPU/memory usage reporting. If metrics-server is not installed, + # this ClusterRole is unused and Wings will gracefully degrade. + - apiGroups: ["metrics.k8s.io"] + resources: ["pods"] + verbs: ["get"] + # Read Node addresses for IP allocation endpoint. Required so Wings can + # report the node's external/internal IPs instead of the Pod's cluster IP. + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] + # NOTE: The kubelet stats/summary fallback (nodes/proxy) is intentionally NOT + # granted here because it is a broad, cluster-wide permission. If you do not + # run metrics-server and need that fallback, additionally apply + # clusterrole-metrics-kubelet.yaml. diff --git a/kubernetes/clusterrolebinding-metrics.yaml b/kubernetes/clusterrolebinding-metrics.yaml new file mode 100644 index 00000000..34d3b50c --- /dev/null +++ b/kubernetes/clusterrolebinding-metrics.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: pelican-wings-metrics + labels: + app.kubernetes.io/managed-by: pelican-wings + app.kubernetes.io/component: wings +subjects: + - kind: ServiceAccount + name: pelican-wings + namespace: pelican +roleRef: + kind: ClusterRole + name: pelican-wings-metrics + apiGroup: rbac.authorization.k8s.io diff --git a/kubernetes/namespace.yaml b/kubernetes/namespace.yaml new file mode 100644 index 00000000..8c578382 --- /dev/null +++ b/kubernetes/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: pelican + labels: + app.kubernetes.io/managed-by: pelican-wings diff --git a/kubernetes/role.yaml b/kubernetes/role.yaml new file mode 100644 index 00000000..1e67d038 --- /dev/null +++ b/kubernetes/role.yaml @@ -0,0 +1,56 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: pelican-wings + namespace: pelican + labels: + app.kubernetes.io/managed-by: pelican-wings + app.kubernetes.io/component: wings +rules: + # Game server Pod lifecycle: create, get (status checks), delete, list + # (installer pod lookup by label). + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "create", "delete", "list", "watch"] + + # Stream console output from game server and installer Pods. + - apiGroups: [""] + resources: ["pods/log"] + verbs: ["get"] + + # Interactive console attach via SPDY (stdin/stdout/stderr). + - apiGroups: [""] + resources: ["pods/attach"] + verbs: ["create"] + + # Service management for exposing game server ports (NodePort or LoadBalancer). + # List is needed for discovering LoadBalancer external IPs. + - apiGroups: [""] + resources: ["services"] + verbs: ["get", "create", "update", "delete", "list"] + + # Egg installation Jobs: create, get (poll status), delete (cleanup). + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get", "create", "delete"] + + # ConfigMaps for install scripts and identity files: create, get, update, delete. + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "create", "update", "delete"] + + # ResourceQuota management (when resource_quota.enabled: true). + - apiGroups: [""] + resources: ["resourcequotas"] + verbs: ["get", "create", "update"] + + # LimitRange management (when limit_range.enabled: true). + - apiGroups: [""] + resources: ["limitranges"] + verbs: ["get", "create", "update"] + + # PersistentVolumeClaim lifecycle (when storage_mode: pvc). + # Remove this rule if you only use storage_mode: hostpath. + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "create", "update", "delete", "list"] diff --git a/kubernetes/rolebinding.yaml b/kubernetes/rolebinding.yaml new file mode 100644 index 00000000..cdc010d7 --- /dev/null +++ b/kubernetes/rolebinding.yaml @@ -0,0 +1,16 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: pelican-wings + namespace: pelican + labels: + app.kubernetes.io/managed-by: pelican-wings + app.kubernetes.io/component: wings +subjects: + - kind: ServiceAccount + name: pelican-wings + namespace: pelican +roleRef: + kind: Role + name: pelican-wings + apiGroup: rbac.authorization.k8s.io diff --git a/kubernetes/serviceaccount.yaml b/kubernetes/serviceaccount.yaml new file mode 100644 index 00000000..08ea60ca --- /dev/null +++ b/kubernetes/serviceaccount.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: pelican-wings + namespace: pelican + labels: + app.kubernetes.io/managed-by: pelican-wings + app.kubernetes.io/component: wings diff --git a/router/router_system.go b/router/router_system.go index c27eb289..843f4313 100644 --- a/router/router_system.go +++ b/router/router_system.go @@ -12,6 +12,7 @@ import ( "github.com/gin-gonic/gin" "github.com/pelican/wings/config" + "github.com/pelican/wings/environment/kubernetes" "github.com/pelican/wings/internal/diagnostics" "github.com/pelican/wings/router/middleware" "github.com/pelican/wings/router/tokens" @@ -22,7 +23,7 @@ import ( // Returns information about the system that wings is running on. func getSystemInformation(c *gin.Context) { - i, err := system.GetSystemInformation() + i, err := system.GetSystemInformationWithOptions(config.Get().Kubernetes.Enabled) if err != nil { middleware.CaptureAndAbort(c, err) return @@ -89,20 +90,61 @@ func getDiagnostics(c *gin.Context) { // Returns list of host machine IP addresses func getSystemIps(c *gin.Context) { - interfaces, err := system.GetSystemIps() - if err != nil { - middleware.CaptureAndAbort(c, err) - return - } + cfg := config.Get() + var interfaces []string + + if cfg.Kubernetes.Enabled { + ctx := c.Request.Context() + + // In LoadBalancer mode, collect IPs from existing LB Services. + if cfg.Kubernetes.NetworkMode == config.KubeNetworkLoadBalancer { + lbIPs, err := kubernetes.GetLoadBalancerIPs(ctx) + if err != nil { + log.WithField("error", err).Warn("failed to query LoadBalancer IPs") + } else { + interfaces = append(interfaces, lbIPs...) + } + } + + // Always try to get node IPs as a base (needed for NodePort, useful + // as fallback for LoadBalancer before any servers exist). + nodeIPs, err := kubernetes.GetNodeIPs(ctx) + if err != nil { + log.WithField("error", err).Warn("failed to query node IPs") + } else { + for _, ip := range nodeIPs { + if !slices.Contains(interfaces, ip) { + interfaces = append(interfaces, ip) + } + } + } - // Append config defined ips as well - for i := range config.Get().Docker.SystemIps { - targetIp := config.Get().Docker.SystemIps[i] - if slices.Contains(interfaces, targetIp) { - continue + // Append user-configured static IPs. + for _, ip := range cfg.Kubernetes.SystemIPs { + if !slices.Contains(interfaces, ip) { + interfaces = append(interfaces, ip) + } } - interfaces = append(interfaces, targetIp) + // If IP discovery failed across the board, surface an error instead of + // returning a misleading empty-but-successful response that the Panel + // would treat as "no assignable IPs". + if len(interfaces) == 0 { + middleware.CaptureAndAbort(c, errors.New("failed to discover any assignable IP addresses in kubernetes mode; check node/LoadBalancer RBAC or configure kubernetes.system_ips")) + return + } + } else { + ips, err := system.GetSystemIps() + if err != nil { + middleware.CaptureAndAbort(c, err) + return + } + interfaces = append(interfaces, ips...) + for _, ip := range cfg.Docker.SystemIps { + if !slices.Contains(interfaces, ip) { + interfaces = append(interfaces, ip) + } + } } c.JSON(http.StatusOK, &system.IpAddresses{IpAddresses: interfaces}) diff --git a/server/install.go b/server/install.go index e1237b8e..376cd963 100644 --- a/server/install.go +++ b/server/install.go @@ -103,6 +103,12 @@ func (s *Server) internalInstall() error { if err != nil { return err } + + // Route to the Kubernetes-based installer when K8s is enabled. + if config.Get().Kubernetes.Enabled { + return s.internalInstallKubernetes(&script) + } + p, err := NewInstallationProcess(s, &script) if err != nil { return err diff --git a/server/install_kubernetes.go b/server/install_kubernetes.go new file mode 100644 index 00000000..03740c72 --- /dev/null +++ b/server/install_kubernetes.go @@ -0,0 +1,61 @@ +package server + +import ( + "path/filepath" + + "emperror.dev/errors" + + "github.com/pelican/wings/config" + "github.com/pelican/wings/environment/kubernetes" + "github.com/pelican/wings/remote" + "github.com/pelican/wings/system" +) + +// internalInstallKubernetes runs the installation process using a Kubernetes +// Job instead of a Docker container. This is called when K8s mode is enabled. +func (s *Server) internalInstallKubernetes(script *remote.InstallationScript) error { + env, ok := s.Environment.(*kubernetes.Environment) + if !ok { + return errors.New("install: kubernetes enabled but environment is not kubernetes") + } + + tmpDir := filepath.Join(config.Get().System.TmpDirectory, s.ID()) + + ip := kubernetes.NewInstallerProcess( + env, + script, + s.GetEnvironmentVariables(), + s.Filesystem().Path(), + tmpDir, + s.Sink(system.InstallSink), + ) + + if !s.installing.SwapIf(true) { + return errors.New("install: cannot obtain installation lock") + } + defer s.installing.Store(false) + + s.Log().Info("beginning kubernetes job-based installation process for server") + s.Events().Publish(DaemonMessageEvent, "Starting installation process via Kubernetes Job, this could take a few minutes...") + + // Write the install script to disk for mounting into the Job Pod. + if err := ip.WriteInstallScript(s.Context()); err != nil { + return errors.WithMessage(err, "install: failed to write installation script") + } + + // Run the Job and wait for completion. + if err := ip.Run(s.Context()); err != nil { + _ = ip.Cleanup(s.Context()) + return errors.WithMessage(err, "install: kubernetes installer job failed") + } + + s.Events().Publish(DaemonMessageEvent, "Installation process completed.") + s.Log().Info("completed kubernetes job-based installation process for server") + + // Cleanup the Job and temp files. + if err := ip.Cleanup(s.Context()); err != nil { + s.Log().WithField("error", err).Warn("failed to clean up installer job resources") + } + + return nil +} diff --git a/server/manager.go b/server/manager.go index ca8f33aa..a61ece88 100644 --- a/server/manager.go +++ b/server/manager.go @@ -14,9 +14,11 @@ import ( "github.com/apex/log" "github.com/gammazero/workerpool" "github.com/goccy/go-json" + "github.com/pelican/wings/config" "github.com/pelican/wings/environment" "github.com/pelican/wings/environment/docker" + "github.com/pelican/wings/environment/kubernetes" "github.com/pelican/wings/remote" "github.com/pelican/wings/server/filesystem" "github.com/pelican/wings/server/filesystem/quotas" @@ -208,9 +210,6 @@ func (m *Manager) InitServer(data remote.ServerConfigurationResponse) (*Server, } } - // Right now we only support a Docker based environment, so I'm going to hard code - // this logic in. When we're ready to support other environment we'll need to make - // some modifications here, obviously. settings := environment.Settings{ Mounts: s.Mounts(), Allocations: s.cfg.Allocations, @@ -219,15 +218,30 @@ func (m *Manager) InitServer(data remote.ServerConfigurationResponse) (*Server, } envCfg := environment.NewConfiguration(settings, s.GetEnvironmentVariables()) - meta := docker.Metadata{ - Image: s.Config().Container.Image, - } - if env, err := docker.New(s.ID(), &meta, envCfg); err != nil { - return nil, err + if config.Get().Kubernetes.Enabled { + meta := kubernetes.Metadata{ + Image: s.Config().Container.Image, + } + if pc := s.ProcessConfiguration(); pc != nil { + meta.Stop = pc.Stop + } + if env, err := kubernetes.New(s.ID(), &meta, envCfg); err != nil { + return nil, err + } else { + s.Environment = env + s.StartEventListeners() + } } else { - s.Environment = env - s.StartEventListeners() + meta := docker.Metadata{ + Image: s.Config().Container.Image, + } + if env, err := docker.New(s.ID(), &meta, envCfg); err != nil { + return nil, err + } else { + s.Environment = env + s.StartEventListeners() + } } // If the server's data directory exists, force disk usage calculation. diff --git a/server/update.go b/server/update.go index 6a168df6..ef4a2e78 100644 --- a/server/update.go +++ b/server/update.go @@ -4,6 +4,7 @@ import ( "time" "github.com/pelican/wings/environment/docker" + "github.com/pelican/wings/environment/kubernetes" "github.com/pelican/wings/environment" ) @@ -39,6 +40,14 @@ func (s *Server) SyncWithEnvironment() { e.SetStopConfiguration(s.ProcessConfiguration().Stop) } + // The Kubernetes environment likewise needs the configured image and stop + // configuration so command-based stops and image updates take effect. + if e, ok := s.Environment.(*kubernetes.Environment); ok { + s.Log().Debug("syncing stop configuration with configured kubernetes environment") + e.SetImage(cfg.Container.Image) + e.SetStopConfiguration(s.ProcessConfiguration().Stop) + } + // If build limits are changed, environment variables also change. Plus, any modifications to // the startup command also need to be properly propagated to this environment. s.Environment.Config().SetEnvironmentVariables(s.GetEnvironmentVariables()) diff --git a/system/system.go b/system/system.go index 50ec088a..a28b771d 100644 --- a/system/system.go +++ b/system/system.go @@ -7,7 +7,7 @@ import ( "net" "runtime" "strings" - + "golang.org/x/sys/unix" "github.com/docker/docker/api/types/filters" @@ -104,11 +104,52 @@ type DockerDiskUsage struct { } func GetSystemInformation() (*Information, error) { + return GetSystemInformationWithOptions(false) +} + +func GetSystemInformationWithOptions(kubernetesMode bool) (*Information, error) { k, err := kernel.GetKernelVersion() if err != nil { return nil, err } + if kubernetesMode { + // In Kubernetes mode the daemon frequently runs on minimal images that + // lack /etc/os-release; treat it as best-effort and fall back to the + // runtime OS rather than failing the whole system info request. + release, err := osrelease.Read() + if err != nil { + release = map[string]string{} + } + + v, err := mem.VirtualMemory() + if err != nil { + return nil, err + } + + var osName string + if release["PRETTY_NAME"] != "" { + osName = release["PRETTY_NAME"] + } else if release["NAME"] != "" { + osName = release["NAME"] + } else { + osName = runtime.GOOS + } + + return &Information{ + Version: Version, + Docker: DockerInformation{}, + System: System{ + Architecture: runtime.GOARCH, + CPUThreads: runtime.NumCPU(), + MemoryBytes: int64(v.Total), + KernelVersion: k.String(), + OS: osName, + OSType: runtime.GOOS, + }, + }, nil + } + version, info, err := GetDockerInfo(context.Background()) if err != nil { return nil, err @@ -181,9 +222,13 @@ func GetSystemIps() ([]string, error) { } for _, addr := range iface_addrs { ipNet, valid := addr.(*net.IPNet) - if valid && !ipNet.IP.IsLoopback() && (len(ipNet.IP) == net.IPv6len && !ipNet.IP.IsLinkLocalUnicast()) { - ip_addrs = append(ip_addrs, ipNet.IP.String()) + if !valid || ipNet.IP.IsLoopback() { + continue + } + if ipNet.IP.IsLinkLocalUnicast() || ipNet.IP.IsLinkLocalMulticast() { + continue } + ip_addrs = append(ip_addrs, ipNet.IP.String()) } return ip_addrs, nil }