From a10346f7da00079de1f4958a83619a626aa9d3b1 Mon Sep 17 00:00:00 2001 From: Exonical Date: Tue, 2 Jun 2026 06:10:09 +0000 Subject: [PATCH 1/6] feat: add Kubernetes environment (run game servers as Pods) Adds an opt-in Kubernetes backend alongside Docker, gated behind kubernetes.enabled (default false). A new environment/kubernetes package implements the ProcessEnvironment interface: Pod lifecycle/power, SPDY attach/exec console, hostport/nodeport/loadbalancer networking, hostpath/PVC storage, optional ResourceQuota/LimitRange, metrics-API stats, and egg installation via a Job + ConfigMap. Environment selection is gated in server/manager.go and server/install.go; system info and /api/system/ips become backend-aware in system/system.go and router/router_system.go. Docker behavior is unchanged when kubernetes.enabled is false. Ships raw RBAC manifests under kubernetes/ and a Helm chart under chart/pelican-wings/ for deploying Wings in-cluster, plus a helm-lint workflow. The Kubernetes client libraries (k8s.io/* v0.36.1) require Go >= 1.26.0, so the go directive is bumped accordingly. --- .github/workflows/helm-lint.yaml | 26 + chart/pelican-wings/Chart.yaml | 16 + chart/pelican-wings/README.md | 116 +++ chart/pelican-wings/templates/NOTES.txt | 22 + chart/pelican-wings/templates/_helpers.tpl | 60 ++ .../templates/clusterrole-metrics.yaml | 20 + .../templates/clusterrolebinding-metrics.yaml | 16 + chart/pelican-wings/templates/configmap.yaml | 76 ++ chart/pelican-wings/templates/deployment.yaml | 121 ++++ chart/pelican-wings/templates/limitrange.yaml | 39 + chart/pelican-wings/templates/namespace.yaml | 8 + chart/pelican-wings/templates/pvc.yaml | 22 + .../templates/resourcequota.yaml | 32 + chart/pelican-wings/templates/role.yaml | 37 + .../pelican-wings/templates/rolebinding.yaml | 17 + chart/pelican-wings/templates/service.yaml | 20 + .../templates/serviceaccount.yaml | 13 + chart/pelican-wings/values.yaml | 222 ++++++ config/config.go | 5 + config/config_kubernetes.go | 229 ++++++ environment/kubernetes/api.go | 42 ++ environment/kubernetes/container.go | 577 +++++++++++++++ environment/kubernetes/container_test.go | 52 ++ environment/kubernetes/environment.go | 209 ++++++ environment/kubernetes/environment_test.go | 567 +++++++++++++++ environment/kubernetes/identity.go | 158 ++++ environment/kubernetes/installer.go | 389 ++++++++++ environment/kubernetes/installer_test.go | 366 ++++++++++ environment/kubernetes/network.go | 367 ++++++++++ environment/kubernetes/network_test.go | 476 ++++++++++++ environment/kubernetes/node.go | 77 ++ environment/kubernetes/power.go | 282 ++++++++ environment/kubernetes/power_test.go | 77 ++ environment/kubernetes/quota.go | 161 +++++ environment/kubernetes/quota_test.go | 311 ++++++++ environment/kubernetes/stats.go | 403 +++++++++++ environment/kubernetes/stats_test.go | 327 +++++++++ environment/kubernetes/storage.go | 170 +++++ environment/kubernetes/storage_test.go | 349 +++++++++ environment/kubernetes/testutil_test.go | 21 + go.mod | 190 +++-- go.sum | 679 +++++++----------- kubernetes/README.md | 114 +++ kubernetes/clusterrole-metrics.yaml | 24 + kubernetes/clusterrolebinding-metrics.yaml | 15 + kubernetes/namespace.yaml | 6 + kubernetes/role.yaml | 56 ++ kubernetes/rolebinding.yaml | 16 + kubernetes/serviceaccount.yaml | 8 + router/router_system.go | 58 +- server/install.go | 6 + server/install_kubernetes.go | 61 ++ server/manager.go | 30 +- system/system.go | 45 +- 54 files changed, 7292 insertions(+), 514 deletions(-) create mode 100644 .github/workflows/helm-lint.yaml create mode 100644 chart/pelican-wings/Chart.yaml create mode 100644 chart/pelican-wings/README.md create mode 100644 chart/pelican-wings/templates/NOTES.txt create mode 100644 chart/pelican-wings/templates/_helpers.tpl create mode 100644 chart/pelican-wings/templates/clusterrole-metrics.yaml create mode 100644 chart/pelican-wings/templates/clusterrolebinding-metrics.yaml create mode 100644 chart/pelican-wings/templates/configmap.yaml create mode 100644 chart/pelican-wings/templates/deployment.yaml create mode 100644 chart/pelican-wings/templates/limitrange.yaml create mode 100644 chart/pelican-wings/templates/namespace.yaml create mode 100644 chart/pelican-wings/templates/pvc.yaml create mode 100644 chart/pelican-wings/templates/resourcequota.yaml create mode 100644 chart/pelican-wings/templates/role.yaml create mode 100644 chart/pelican-wings/templates/rolebinding.yaml create mode 100644 chart/pelican-wings/templates/service.yaml create mode 100644 chart/pelican-wings/templates/serviceaccount.yaml create mode 100644 chart/pelican-wings/values.yaml create mode 100644 config/config_kubernetes.go create mode 100644 environment/kubernetes/api.go create mode 100644 environment/kubernetes/container.go create mode 100644 environment/kubernetes/container_test.go create mode 100644 environment/kubernetes/environment.go create mode 100644 environment/kubernetes/environment_test.go create mode 100644 environment/kubernetes/identity.go create mode 100644 environment/kubernetes/installer.go create mode 100644 environment/kubernetes/installer_test.go create mode 100644 environment/kubernetes/network.go create mode 100644 environment/kubernetes/network_test.go create mode 100644 environment/kubernetes/node.go create mode 100644 environment/kubernetes/power.go create mode 100644 environment/kubernetes/power_test.go create mode 100644 environment/kubernetes/quota.go create mode 100644 environment/kubernetes/quota_test.go create mode 100644 environment/kubernetes/stats.go create mode 100644 environment/kubernetes/stats_test.go create mode 100644 environment/kubernetes/storage.go create mode 100644 environment/kubernetes/storage_test.go create mode 100644 environment/kubernetes/testutil_test.go create mode 100644 kubernetes/README.md create mode 100644 kubernetes/clusterrole-metrics.yaml create mode 100644 kubernetes/clusterrolebinding-metrics.yaml create mode 100644 kubernetes/namespace.yaml create mode 100644 kubernetes/role.yaml create mode 100644 kubernetes/rolebinding.yaml create mode 100644 kubernetes/serviceaccount.yaml create mode 100644 server/install_kubernetes.go diff --git a/.github/workflows/helm-lint.yaml b/.github/workflows/helm-lint.yaml new file mode 100644 index 00000000..1ba2e7ad --- /dev/null +++ b/.github/workflows/helm-lint.yaml @@ -0,0 +1,26 @@ +name: Helm Lint + +on: + push: + branches: [main] + paths: + - "chart/**" + - ".github/workflows/helm-lint.yaml" + pull_request: + paths: + - "chart/**" + - ".github/workflows/helm-lint.yaml" + +permissions: + contents: read + +jobs: + lint-helm: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: azure/setup-helm@v4 + + - name: Lint Helm chart + run: helm lint chart/pelican-wings diff --git a/chart/pelican-wings/Chart.yaml b/chart/pelican-wings/Chart.yaml new file mode 100644 index 00000000..0f28fdaa --- /dev/null +++ b/chart/pelican-wings/Chart.yaml @@ -0,0 +1,16 @@ +apiVersion: v2 +name: pelican-wings +description: Helm chart for Pelican Wings — the game server management daemon with Kubernetes support +type: application +version: 0.1.0 +appVersion: "1.0.0" +keywords: + - pelican + - wings + - game-server + - kubernetes +maintainers: + - name: Exonical + url: https://github.com/Exonical +sources: + - https://github.com/pelican-dev/wings diff --git a/chart/pelican-wings/README.md b/chart/pelican-wings/README.md new file mode 100644 index 00000000..49f636bb --- /dev/null +++ b/chart/pelican-wings/README.md @@ -0,0 +1,116 @@ +# Pelican Wings Helm Chart + +Deploys [Pelican Wings](https://github.com/pelican-dev/wings) — the game server +management daemon — into a Kubernetes cluster with full RBAC, storage, and +networking support. + +## Prerequisites + +- Kubernetes 1.34+ +- Helm 3.x +- A running [Pelican Panel](https://github.com/pelican-dev/panel) instance + +## Quick Start + +```bash +# Add your node credentials from the Panel +helm install wings ./chart/pelican-wings \ + --set wings.panelUrl=https://panel.example.com \ + --set wings.token=YOUR_TOKEN \ + --set wings.tokenId=YOUR_TOKEN_ID \ + --set wings.uuid=YOUR_NODE_UUID +``` + +## Configuration + +See [values.yaml](values.yaml) for the full list of configurable values. + +### Key Values + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `wings.panelUrl` | Panel URL | `https://panel.example.com` | +| `wings.token` | Panel authentication token | `""` | +| `wings.tokenId` | Panel token ID | `""` | +| `wings.uuid` | Node UUID from Panel | `""` | +| `wings.kubernetes.networkMode` | Port exposure: `hostport` or `nodeport` | `nodeport` | +| `wings.kubernetes.storageMode` | Storage: `hostpath` or `pvc` | `pvc` | +| `wings.kubernetes.storageClass` | StorageClass for PVCs | `""` (cluster default) | +| `wings.kubernetes.storageSize` | Default PVC size | `10Gi` | +| `wings.kubernetes.imagePullPolicy` | Pull policy for game server Pods/install Jobs: `Always`, `IfNotPresent`, `Never` | `""` (smart default) | +| `gameNamespace` | Namespace for game server resources | `pelican` | +| `rbac.create` | Create RBAC resources | `true` | +| `serviceAccount.create` | Create ServiceAccount | `true` | + +### Storage + +By default, the chart uses PVC-based storage (`storageMode: pvc`). This creates +a PersistentVolumeClaim per game server, enabling proper data lifecycle +management. + +For single-node setups or testing, you can use HostPath: + +```yaml +wings: + kubernetes: + storageMode: hostpath +``` + +### Networking + +NodePort mode (default) creates a Kubernetes Service per game server, exposing +ports via cluster-assigned NodePorts: + +```yaml +wings: + kubernetes: + networkMode: nodeport + nodeportPreserve: true # Try to use game port as NodePort +``` + +HostPort mode binds game server ports directly to the node: + +```yaml +wings: + kubernetes: + networkMode: hostport +``` + +### Image pulling + +Game server Pods and installation Jobs default to `imagePullPolicy: Always` +for remote images, so updated tags are re-pulled rather than reusing a stale +copy cached on the node (matching the Docker backend). `~`-prefixed local +images are never pulled. Override this for air-gapped clusters: + +```yaml +wings: + kubernetes: + imagePullPolicy: IfNotPresent # or "Never" +``` + +This is independent of `image.pullPolicy`, which applies to the Wings daemon +image itself. + +## What Gets Created + +- **Namespace** — `pelican` (configurable) +- **ServiceAccount** — For Wings and game server Pods +- **Role + RoleBinding** — Namespace-scoped permissions (Pods, Services, Jobs, PVCs) +- **ClusterRole + ClusterRoleBinding** — Metrics API access +- **ConfigMap** — Wings configuration file +- **Deployment** — Wings daemon with health probes +- **Service** — Exposes Wings API within the cluster + +## Uninstalling + +```bash +helm uninstall wings +``` + +Note: PVCs created for game servers are NOT automatically deleted when +uninstalling the chart. Delete them manually if you want to remove all data: + +```bash +kubectl delete pvc -n pelican -l app.kubernetes.io/managed-by=pelican-wings +``` diff --git a/chart/pelican-wings/templates/NOTES.txt b/chart/pelican-wings/templates/NOTES.txt new file mode 100644 index 00000000..e88b9444 --- /dev/null +++ b/chart/pelican-wings/templates/NOTES.txt @@ -0,0 +1,22 @@ +Pelican Wings has been deployed! + +1. Get the Wings API URL: +{{- if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Values.gameNamespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "pelican-wings.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Values.gameNamespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo "Wings API: http://$NODE_IP:$NODE_PORT" +{{- else if contains "ClusterIP" .Values.service.type }} + kubectl port-forward --namespace {{ .Values.gameNamespace }} svc/{{ include "pelican-wings.fullname" . }} {{ .Values.service.port }}:{{ .Values.service.port }} + echo "Wings API: http://127.0.0.1:{{ .Values.service.port }}" +{{- end }} + +2. Configure your Panel to connect to this Wings node using the URL above. + +3. Verify the deployment: + kubectl get pods --namespace {{ .Values.gameNamespace }} -l "{{ include "pelican-wings.selectorLabels" . | replace "\n" "," }}" + +Configuration: + - Game namespace: {{ .Values.gameNamespace }} + - Network mode: {{ .Values.wings.kubernetes.networkMode }} + - Storage mode: {{ .Values.wings.kubernetes.storageMode }} + - Panel URL: {{ .Values.wings.panelUrl }} diff --git a/chart/pelican-wings/templates/_helpers.tpl b/chart/pelican-wings/templates/_helpers.tpl new file mode 100644 index 00000000..b7d93818 --- /dev/null +++ b/chart/pelican-wings/templates/_helpers.tpl @@ -0,0 +1,60 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "pelican-wings.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "pelican-wings.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "pelican-wings.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels. +*/}} +{{- define "pelican-wings.labels" -}} +helm.sh/chart: {{ include "pelican-wings.chart" . }} +{{ include "pelican-wings.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels. +*/}} +{{- define "pelican-wings.selectorLabels" -}} +app.kubernetes.io/name: {{ include "pelican-wings.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use. +*/}} +{{- define "pelican-wings.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "pelican-wings.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/chart/pelican-wings/templates/clusterrole-metrics.yaml b/chart/pelican-wings/templates/clusterrole-metrics.yaml new file mode 100644 index 00000000..aaf02289 --- /dev/null +++ b/chart/pelican-wings/templates/clusterrole-metrics.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "pelican-wings.fullname" . }}-metrics + labels: + {{- include "pelican-wings.labels" . | nindent 4 }} +rules: + - apiGroups: ["metrics.k8s.io"] + resources: ["pods"] + verbs: ["get"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] + # Proxy to kubelet stats/summary API for resource metrics when + # metrics-server is not installed. + - apiGroups: [""] + resources: ["nodes/proxy"] + verbs: ["get"] +{{- end }} diff --git a/chart/pelican-wings/templates/clusterrolebinding-metrics.yaml b/chart/pelican-wings/templates/clusterrolebinding-metrics.yaml new file mode 100644 index 00000000..5679fd59 --- /dev/null +++ b/chart/pelican-wings/templates/clusterrolebinding-metrics.yaml @@ -0,0 +1,16 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "pelican-wings.fullname" . }}-metrics + labels: + {{- include "pelican-wings.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ include "pelican-wings.serviceAccountName" . }} + namespace: {{ .Values.gameNamespace }} +roleRef: + kind: ClusterRole + name: {{ include "pelican-wings.fullname" . }}-metrics + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/chart/pelican-wings/templates/configmap.yaml b/chart/pelican-wings/templates/configmap.yaml new file mode 100644 index 00000000..52867e84 --- /dev/null +++ b/chart/pelican-wings/templates/configmap.yaml @@ -0,0 +1,76 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "pelican-wings.fullname" . }}-config + namespace: {{ .Values.gameNamespace }} + labels: + {{- include "pelican-wings.labels" . | nindent 4 }} +data: + config.yml: | + debug: false + app_name: pelican + uuid: {{ .Values.wings.uuid | quote }} + token_id: {{ .Values.wings.tokenId | quote }} + token: {{ .Values.wings.token | quote }} + remote: {{ .Values.wings.panelUrl | quote }} + api: + host: {{ .Values.wings.api.host | quote }} + port: {{ .Values.wings.api.port }} + sftp: + bind_addr: {{ .Values.wings.sftp.host | quote }} + bind_port: {{ .Values.wings.sftp.port }} + system: + root_directory: {{ .Values.wings.system.rootDirectory | quote }} + log_directory: {{ .Values.wings.system.logDirectory | quote }} + data: {{ .Values.wings.system.data | quote }} + tmp_directory: {{ .Values.wings.system.tmpDirectory | quote }} + kubernetes: + enabled: {{ .Values.wings.kubernetes.enabled }} + namespace: {{ .Values.wings.kubernetes.namespace | quote }} + network_mode: {{ .Values.wings.kubernetes.networkMode | quote }} + {{- if .Values.wings.kubernetes.lbAnnotations }} + lb_annotations: + {{- toYaml .Values.wings.kubernetes.lbAnnotations | nindent 8 }} + {{- end }} + {{- if .Values.wings.kubernetes.lbIPAnnotation }} + lb_ip_annotation: {{ .Values.wings.kubernetes.lbIPAnnotation | quote }} + {{- end }} + {{- if .Values.wings.kubernetes.lbSharingKey }} + lb_sharing_key: {{ .Values.wings.kubernetes.lbSharingKey | quote }} + {{- end }} + storage_mode: {{ .Values.wings.kubernetes.storageMode | quote }} + {{- if .Values.wings.kubernetes.storageClass }} + storage_class: {{ .Values.wings.kubernetes.storageClass | quote }} + {{- end }} + storage_size: {{ .Values.wings.kubernetes.storageSize | quote }} + storage_access_mode: {{ .Values.wings.kubernetes.storageAccessMode | quote }} + nodeport_preserve: {{ .Values.wings.kubernetes.nodeportPreserve }} + nodeport_range_min: {{ .Values.wings.kubernetes.nodeportRangeMin }} + nodeport_range_max: {{ .Values.wings.kubernetes.nodeportRangeMax }} + dns_policy: {{ .Values.wings.kubernetes.dnsPolicy | quote }} + {{- if .Values.wings.kubernetes.nodeName }} + node_name: {{ .Values.wings.kubernetes.nodeName | quote }} + {{- end }} + {{- if .Values.wings.kubernetes.systemIPs }} + system_ips: + {{- toYaml .Values.wings.kubernetes.systemIPs | nindent 8 }} + {{- end }} + {{- if .Values.persistence.enabled }} + data_pvc: {{ .Values.persistence.existingClaim | default (printf "%s-data" (include "pelican-wings.fullname" .)) | quote }} + {{- end }} + service_account: {{ include "pelican-wings.serviceAccountName" . | quote }} + {{- if .Values.wings.kubernetes.nodeSelector }} + node_selector: + {{- toYaml .Values.wings.kubernetes.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.wings.kubernetes.tolerations }} + tolerations: + {{- toYaml .Values.wings.kubernetes.tolerations | nindent 8 }} + {{- end }} + {{- if .Values.wings.kubernetes.imagePullSecrets }} + image_pull_secrets: + {{- toYaml .Values.wings.kubernetes.imagePullSecrets | nindent 8 }} + {{- end }} + {{- if .Values.wings.kubernetes.imagePullPolicy }} + image_pull_policy: {{ .Values.wings.kubernetes.imagePullPolicy | quote }} + {{- end }} diff --git a/chart/pelican-wings/templates/deployment.yaml b/chart/pelican-wings/templates/deployment.yaml new file mode 100644 index 00000000..e7de1499 --- /dev/null +++ b/chart/pelican-wings/templates/deployment.yaml @@ -0,0 +1,121 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "pelican-wings.fullname" . }} + namespace: {{ .Values.gameNamespace }} + labels: + {{- include "pelican-wings.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "pelican-wings.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + labels: + {{- include "pelican-wings.labels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "pelican-wings.serviceAccountName" . }} + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + initContainers: + - name: copy-config + image: busybox:1.36 + command: ["cp", "/config-source/config.yml", "/etc/pelican/config.yml"] + volumeMounts: + - name: config-source + mountPath: /config-source + readOnly: true + - name: config + mountPath: /etc/pelican + containers: + - name: wings + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: api + containerPort: {{ .Values.wings.api.port }} + protocol: TCP + - name: sftp + containerPort: {{ .Values.wings.sftp.port }} + protocol: TCP + livenessProbe: + httpGet: + path: /api/system + port: api + initialDelaySeconds: 10 + periodSeconds: 30 + readinessProbe: + httpGet: + path: /api/system + port: api + initialDelaySeconds: 5 + periodSeconds: 10 + volumeMounts: + - name: config + mountPath: /etc/pelican + {{- if .Values.persistence.enabled }} + - name: data + mountPath: {{ .Values.wings.system.rootDirectory }} + {{- end }} + - name: logs + mountPath: {{ .Values.wings.system.logDirectory }} + - name: tmp + mountPath: {{ .Values.wings.system.tmpDirectory }} + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- with .Values.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: config-source + configMap: + name: {{ include "pelican-wings.fullname" . }}-config + - name: config + emptyDir: {} + {{- if .Values.persistence.enabled }} + - name: data + persistentVolumeClaim: + claimName: {{ .Values.persistence.existingClaim | default (printf "%s-data" (include "pelican-wings.fullname" .)) }} + {{- end }} + - name: logs + emptyDir: {} + - name: tmp + emptyDir: {} + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/chart/pelican-wings/templates/limitrange.yaml b/chart/pelican-wings/templates/limitrange.yaml new file mode 100644 index 00000000..13d03582 --- /dev/null +++ b/chart/pelican-wings/templates/limitrange.yaml @@ -0,0 +1,39 @@ +{{- if .Values.limitRange.enabled }} +apiVersion: v1 +kind: LimitRange +metadata: + name: {{ include "pelican-wings.fullname" . }} + namespace: {{ .Values.gameNamespace }} + labels: + {{- include "pelican-wings.labels" . | nindent 4 }} +spec: + limits: + - type: Container + {{- if or .Values.limitRange.defaultCPULimit .Values.limitRange.defaultMemoryLimit }} + default: + {{- if .Values.limitRange.defaultCPULimit }} + cpu: {{ .Values.limitRange.defaultCPULimit | quote }} + {{- end }} + {{- if .Values.limitRange.defaultMemoryLimit }} + memory: {{ .Values.limitRange.defaultMemoryLimit | quote }} + {{- end }} + {{- end }} + {{- if or .Values.limitRange.defaultCPURequest .Values.limitRange.defaultMemoryRequest }} + defaultRequest: + {{- if .Values.limitRange.defaultCPURequest }} + cpu: {{ .Values.limitRange.defaultCPURequest | quote }} + {{- end }} + {{- if .Values.limitRange.defaultMemoryRequest }} + memory: {{ .Values.limitRange.defaultMemoryRequest | quote }} + {{- end }} + {{- end }} + {{- if or .Values.limitRange.maxCPU .Values.limitRange.maxMemory }} + max: + {{- if .Values.limitRange.maxCPU }} + cpu: {{ .Values.limitRange.maxCPU | quote }} + {{- end }} + {{- if .Values.limitRange.maxMemory }} + memory: {{ .Values.limitRange.maxMemory | quote }} + {{- end }} + {{- end }} +{{- end }} diff --git a/chart/pelican-wings/templates/namespace.yaml b/chart/pelican-wings/templates/namespace.yaml new file mode 100644 index 00000000..264ea787 --- /dev/null +++ b/chart/pelican-wings/templates/namespace.yaml @@ -0,0 +1,8 @@ +{{- if .Values.createGameNamespace }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Values.gameNamespace }} + labels: + {{- include "pelican-wings.labels" . | nindent 4 }} +{{- end }} diff --git a/chart/pelican-wings/templates/pvc.yaml b/chart/pelican-wings/templates/pvc.yaml new file mode 100644 index 00000000..147f3289 --- /dev/null +++ b/chart/pelican-wings/templates/pvc.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "pelican-wings.fullname" . }}-data + namespace: {{ .Values.gameNamespace }} + labels: + {{- include "pelican-wings.labels" . | nindent 4 }} + {{- with .Values.persistence.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + accessModes: + - {{ .Values.persistence.accessMode }} + {{- if .Values.persistence.storageClass }} + storageClassName: {{ .Values.persistence.storageClass | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size }} +{{- end }} diff --git a/chart/pelican-wings/templates/resourcequota.yaml b/chart/pelican-wings/templates/resourcequota.yaml new file mode 100644 index 00000000..8ec70eb4 --- /dev/null +++ b/chart/pelican-wings/templates/resourcequota.yaml @@ -0,0 +1,32 @@ +{{- if .Values.resourceQuota.enabled }} +apiVersion: v1 +kind: ResourceQuota +metadata: + name: {{ include "pelican-wings.fullname" . }} + namespace: {{ .Values.gameNamespace }} + labels: + {{- include "pelican-wings.labels" . | nindent 4 }} +spec: + hard: + {{- if .Values.resourceQuota.cpuLimit }} + limits.cpu: {{ .Values.resourceQuota.cpuLimit | quote }} + {{- end }} + {{- if .Values.resourceQuota.memoryLimit }} + limits.memory: {{ .Values.resourceQuota.memoryLimit | quote }} + {{- end }} + {{- if .Values.resourceQuota.cpuRequest }} + requests.cpu: {{ .Values.resourceQuota.cpuRequest | quote }} + {{- end }} + {{- if .Values.resourceQuota.memoryRequest }} + requests.memory: {{ .Values.resourceQuota.memoryRequest | quote }} + {{- end }} + {{- if .Values.resourceQuota.maxPods }} + pods: {{ .Values.resourceQuota.maxPods | quote }} + {{- end }} + {{- if .Values.resourceQuota.maxPVCs }} + persistentvolumeclaims: {{ .Values.resourceQuota.maxPVCs | quote }} + {{- end }} + {{- if .Values.resourceQuota.maxStorage }} + requests.storage: {{ .Values.resourceQuota.maxStorage | quote }} + {{- end }} +{{- end }} diff --git a/chart/pelican-wings/templates/role.yaml b/chart/pelican-wings/templates/role.yaml new file mode 100644 index 00000000..64ab2cc5 --- /dev/null +++ b/chart/pelican-wings/templates/role.yaml @@ -0,0 +1,37 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "pelican-wings.fullname" . }} + namespace: {{ .Values.gameNamespace }} + labels: + {{- include "pelican-wings.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "create", "delete", "list", "watch"] + - apiGroups: [""] + resources: ["pods/log"] + verbs: ["get"] + - apiGroups: [""] + resources: ["pods/attach"] + verbs: ["create"] + - apiGroups: [""] + resources: ["services"] + verbs: ["get", "create", "update", "delete", "list"] + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get", "create", "delete"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "create", "update", "delete"] + - apiGroups: [""] + resources: ["resourcequotas"] + verbs: ["get", "create", "update"] + - apiGroups: [""] + resources: ["limitranges"] + verbs: ["get", "create", "update"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "create", "update", "delete", "list"] +{{- end }} diff --git a/chart/pelican-wings/templates/rolebinding.yaml b/chart/pelican-wings/templates/rolebinding.yaml new file mode 100644 index 00000000..afd69c61 --- /dev/null +++ b/chart/pelican-wings/templates/rolebinding.yaml @@ -0,0 +1,17 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "pelican-wings.fullname" . }} + namespace: {{ .Values.gameNamespace }} + labels: + {{- include "pelican-wings.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ include "pelican-wings.serviceAccountName" . }} + namespace: {{ .Values.gameNamespace }} +roleRef: + kind: Role + name: {{ include "pelican-wings.fullname" . }} + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/chart/pelican-wings/templates/service.yaml b/chart/pelican-wings/templates/service.yaml new file mode 100644 index 00000000..038eee4b --- /dev/null +++ b/chart/pelican-wings/templates/service.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "pelican-wings.fullname" . }} + namespace: {{ .Values.gameNamespace }} + labels: + {{- include "pelican-wings.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: api + protocol: TCP + name: api + - port: {{ .Values.service.sftpPort }} + targetPort: sftp + protocol: TCP + name: sftp + selector: + {{- include "pelican-wings.selectorLabels" . | nindent 4 }} diff --git a/chart/pelican-wings/templates/serviceaccount.yaml b/chart/pelican-wings/templates/serviceaccount.yaml new file mode 100644 index 00000000..0a74a112 --- /dev/null +++ b/chart/pelican-wings/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "pelican-wings.serviceAccountName" . }} + namespace: {{ .Values.gameNamespace }} + labels: + {{- include "pelican-wings.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/chart/pelican-wings/values.yaml b/chart/pelican-wings/values.yaml new file mode 100644 index 00000000..9d5dea8a --- /dev/null +++ b/chart/pelican-wings/values.yaml @@ -0,0 +1,222 @@ +# Default values for pelican-wings. + +# -- Number of Wings replicas (typically 1 per node via DaemonSet, or 1 Deployment) +replicaCount: 1 + +image: + # -- Wings container image repository + repository: ghcr.io/pelican-dev/wings + # -- Image pull policy + pullPolicy: IfNotPresent + # -- Overrides the image tag (defaults to chart appVersion) + tag: "" + +# -- Image pull secrets for private registries +imagePullSecrets: [] + +# -- Override the release name +nameOverride: "" +# -- Override the full release name +fullnameOverride: "" + +serviceAccount: + # -- Whether to create the ServiceAccount + create: true + # -- Annotations to add to the ServiceAccount + annotations: {} + # -- Name of the ServiceAccount (generated if not set) + name: "" + +rbac: + # -- Whether to create RBAC resources (Role, RoleBinding, ClusterRole, ClusterRoleBinding) + create: true + +# -- Namespace for game server resources (Pods, Services, Jobs, PVCs) +gameNamespace: pelican + +# -- Whether to create the game namespace +createGameNamespace: true + +# Wings configuration +wings: + # -- Panel URL that Wings connects to + panelUrl: "https://panel.example.com" + # -- Authentication token for the Panel API + token: "" + # -- Token ID for Panel authentication + tokenId: "" + # -- Node UUID assigned by the Panel + uuid: "" + + api: + # -- Wings API listen address + host: "0.0.0.0" + # -- Wings API listen port + port: 8080 + + sftp: + # -- SFTP server listen address + host: "0.0.0.0" + # -- SFTP server listen port + port: 2022 + + system: + # -- Root data directory + rootDirectory: /var/lib/pelican + # -- Log directory + logDirectory: /var/log/pelican + # -- Server data directory + data: /var/lib/pelican/volumes + # -- Temp directory for installation scripts + tmpDirectory: /tmp/pelican + + kubernetes: + # -- Enable Kubernetes mode (required for K8s game server scheduling) + enabled: true + # -- Namespace for game server Pods (should match gameNamespace) + namespace: pelican + # -- Network mode: "hostport", "nodeport", or "loadbalancer" + # hostport: binds directly on host (closest to Docker) + # nodeport: shared node IP, auto-assigned external ports (30000-32767) + # loadbalancer: dedicated external IP per server via LB provisioner (Cilium, MetalLB, cloud) + networkMode: nodeport + # -- Annotations applied to LoadBalancer Services (only used when networkMode=loadbalancer) + # Example for Cilium LB-IPAM: + # lbAnnotations: + # io.cilium/lb-ipam-pool: "game-servers" + lbAnnotations: {} + # -- Annotation key Wings sets to the server's allocation IP on each LB Service. + # This pins the LB to the IP selected in the Panel so multiple servers share one IP. + # Cilium: "lbipam.cilium.io/ips" | MetalLB: "metallb.universe.tf/loadBalancerIPs" + lbIPAnnotation: "" + # -- Annotation key for LB IP sharing. Wings sets the value to the allocation IP, + # grouping all servers on the same IP under one sharing key. + # Cilium: "lbipam.cilium.io/sharing-key" | MetalLB: "metallb.universe.tf/allow-shared-ip" + lbSharingKey: "" + # -- Storage mode: "hostpath" or "pvc" + storageMode: pvc + # -- StorageClass for PVCs (empty = cluster default) + storageClass: "" + # -- Default PVC size per server + storageSize: "10Gi" + # -- PVC access mode + storageAccessMode: ReadWriteOnce + # -- Preserve game port as NodePort when in range (nodeport mode only) + nodeportPreserve: false + # -- Lower bound of the Kubernetes NodePort range (nodeport mode only) + nodeportRangeMin: 30000 + # -- Upper bound of the Kubernetes NodePort range (nodeport mode only) + nodeportRangeMax: 32767 + # -- Kubernetes node name for IP discovery (auto-detected from NODE_NAME env var if empty) + nodeName: "" + # -- Additional IPs to advertise in the allocation endpoint (e.g. floating IPs, VIPs) + systemIPs: [] + # -- DNS policy for game server Pods + dnsPolicy: ClusterFirst + # -- Node selector for game server Pods + nodeSelector: {} + # -- Tolerations for game server Pods + tolerations: [] + # -- Image pull secrets for game server Pods + imagePullSecrets: [] + # -- Override pull policy for game server Pods and install Jobs: + # "Always", "IfNotPresent", or "Never". Empty = remote images use Always + # (re-pull updated tags, matching Docker), ~-local images use IfNotPresent. + imagePullPolicy: "" + +# -- Pod-level security context for the Wings Deployment +podSecurityContext: {} + +# -- Container-level security context for Wings +securityContext: {} + +# Wings service configuration +service: + # -- Service type for the Wings API + type: ClusterIP + # -- Wings API port + port: 8080 + # -- SFTP port + sftpPort: 2022 + +# -- Resource requests and limits for the Wings container +resources: {} + # requests: + # cpu: 100m + # memory: 256Mi + # limits: + # cpu: "1" + # memory: 1Gi + +# -- Node selector for the Wings Pod itself +nodeSelector: {} + +# -- Tolerations for the Wings Pod itself +tolerations: [] + +# -- Affinity rules for the Wings Pod +affinity: {} + +# Persistence for the Wings root data directory (/var/lib/pelican). +# This stores server volumes, backups, and Wings state. Without persistence, +# all data is lost when the Wings pod restarts. +persistence: + # -- Enable persistent storage for the Wings data directory + enabled: true + # -- StorageClass for the PVC (empty = cluster default) + storageClass: "" + # -- PVC size + size: 50Gi + # -- Access mode + accessMode: ReadWriteOnce + # -- Use an existing PVC instead of creating one (set to PVC name) + existingClaim: "" + # -- Annotations for the PVC + annotations: {} + +# -- Additional environment variables for the Wings container +extraEnv: [] + +# -- Additional volume mounts for the Wings container +extraVolumeMounts: [] + +# -- Additional volumes for the Wings Pod +extraVolumes: [] + +# Resource Quota for the game server namespace. +# Caps aggregate resource usage across all game server Pods. +resourceQuota: + # -- Whether to create a ResourceQuota in the game namespace + enabled: false + # -- Total CPU limit across all Pods (e.g. "16", "8000m") + cpuLimit: "" + # -- Total memory limit across all Pods (e.g. "32Gi") + memoryLimit: "" + # -- Total CPU request across all Pods (e.g. "8") + cpuRequest: "" + # -- Total memory request across all Pods (e.g. "16Gi") + memoryRequest: "" + # -- Maximum number of Pods + maxPods: "" + # -- Maximum number of PVCs + maxPVCs: "" + # -- Total storage across all PVCs (e.g. "500Gi") + maxStorage: "" + +# Limit Range for the game server namespace. +# Sets default resource limits/requests for containers that don't specify their own. +limitRange: + # -- Whether to create a LimitRange in the game namespace + enabled: false + # -- Default CPU limit per container (e.g. "2", "1000m") + defaultCPULimit: "" + # -- Default memory limit per container (e.g. "4Gi") + defaultMemoryLimit: "" + # -- Default CPU request per container (e.g. "500m") + defaultCPURequest: "" + # -- Default memory request per container (e.g. "1Gi") + defaultMemoryRequest: "" + # -- Maximum CPU any single container can use (e.g. "8") + maxCPU: "" + # -- Maximum memory any single container can use (e.g. "16Gi") + maxMemory: "" diff --git a/config/config.go b/config/config.go index cf0cd38d..bceb5fd7 100644 --- a/config/config.go +++ b/config/config.go @@ -345,6 +345,11 @@ type Configuration struct { System SystemConfiguration `json:"system" yaml:"system"` Docker DockerConfiguration `json:"docker" yaml:"docker"` + // Kubernetes holds configuration for scheduling game server workloads as + // Kubernetes Pods. When Kubernetes.Enabled is true, all servers on this + // node are run in K8s instead of Docker. + Kubernetes KubernetesConfiguration `json:"kubernetes" yaml:"kubernetes"` + // Defines internal throttling configurations for server processes to prevent // someone from running an endless loop that spams data to logs. Throttles ConsoleThrottles diff --git a/config/config_kubernetes.go b/config/config_kubernetes.go new file mode 100644 index 00000000..525eb62e --- /dev/null +++ b/config/config_kubernetes.go @@ -0,0 +1,229 @@ +package config + +// KubernetesStorageMode defines how server data volumes are provisioned. +type KubernetesStorageMode string + +const ( + // KubeStorageHostPath uses HostPath volumes (node-local, default). + KubeStorageHostPath KubernetesStorageMode = "hostpath" + + // KubeStoragePVC uses PersistentVolumeClaims for server data directories. + KubeStoragePVC KubernetesStorageMode = "pvc" +) + +// KubernetesNetworkMode defines how game server ports are exposed. +type KubernetesNetworkMode string + +const ( + // KubeNetworkHostPort uses hostNetwork + hostPort for direct port binding + // (closest to Docker behavior). + KubeNetworkHostPort KubernetesNetworkMode = "hostport" + + // KubeNetworkNodePort uses Kubernetes Services with NodePort to expose game + // server ports via the node's IP with auto-assigned external ports. + KubeNetworkNodePort KubernetesNetworkMode = "nodeport" + + // KubeNetworkLoadBalancer uses Kubernetes Services with type LoadBalancer + // to assign a dedicated external IP per game server. Requires a load + // balancer provisioner (MetalLB, cloud controller, kube-vip, etc.). + KubeNetworkLoadBalancer KubernetesNetworkMode = "loadbalancer" +) + +// KubernetesConfiguration defines the Kubernetes configuration used by the +// daemon when scheduling game server workloads as Pods. +type KubernetesConfiguration struct { + // Enabled controls whether this Wings node uses Kubernetes to run game + // server workloads instead of Docker. + Enabled bool `default:"false" json:"enabled" yaml:"enabled"` + + // Namespace is the Kubernetes namespace where game server Pods and related + // resources are created. + Namespace string `default:"pelican" json:"namespace" yaml:"namespace"` + + // Kubeconfig is the path to a kubeconfig file. If empty, in-cluster + // configuration is used (suitable when Wings itself runs inside K8s). + Kubeconfig string `default:"" json:"kubeconfig" yaml:"kubeconfig"` + + // ImagePullSecrets is a list of Secret names used for pulling container + // images in game server Pods. + ImagePullSecrets []string `json:"image_pull_secrets" yaml:"image_pull_secrets"` + + // ImagePullPolicy overrides the pull policy applied to game server Pods + // and installation Jobs. Valid values are "Always", "IfNotPresent", and + // "Never". When empty, remote images use "Always" (so updated tags are + // re-pulled, matching the Docker backend) and ~-prefixed local images use + // "IfNotPresent". Set to "IfNotPresent" or "Never" for air-gapped clusters. + ImagePullPolicy string `default:"" json:"image_pull_policy" yaml:"image_pull_policy"` + + // ServiceAccount is the name of the Kubernetes ServiceAccount assigned to + // game server Pods. + ServiceAccount string `default:"" json:"service_account" yaml:"service_account"` + + // StorageMode controls how server data volumes are provisioned. + // "hostpath" uses HostPath volumes (default, node-local). + // "pvc" creates PersistentVolumeClaims for each server. + StorageMode KubernetesStorageMode `default:"hostpath" json:"storage_mode" yaml:"storage_mode"` + + // StorageClass is the StorageClass used for PersistentVolumeClaims that + // back server data directories. Leave empty to use the cluster default. + StorageClass string `default:"" json:"storage_class" yaml:"storage_class"` + + // StorageSize is the default size for server PVCs (e.g. "10Gi", "50Gi"). + // Only used when StorageMode is "pvc". + StorageSize string `default:"10Gi" json:"storage_size" yaml:"storage_size"` + + // StorageAccessMode is the access mode for PVCs. Defaults to + // ReadWriteOnce. Options: ReadWriteOnce, ReadWriteMany. + StorageAccessMode string `default:"ReadWriteOnce" json:"storage_access_mode" yaml:"storage_access_mode"` + + // NodeSelector is a set of key-value pairs used for scheduling Pods onto + // specific nodes. + NodeSelector map[string]string `json:"node_selector" yaml:"node_selector"` + + // DNSPolicy sets the DNS policy for game server Pods. + DNSPolicy string `default:"ClusterFirst" json:"dns_policy" yaml:"dns_policy"` + + // NetworkMode controls how game server ports are exposed to clients. + NetworkMode KubernetesNetworkMode `default:"hostport" json:"network_mode" yaml:"network_mode"` + + // NodePortPreserve attempts to use the game server's allocated port as + // the NodePort value (only works if the port falls within the cluster's + // NodePort range). When false, Kubernetes auto-assigns NodePorts. + NodePortPreserve bool `default:"false" json:"nodeport_preserve" yaml:"nodeport_preserve"` + + // NodePortRangeMin is the lower bound of the Kubernetes NodePort range. + // Defaults to 30000 if unset. Used when NodePortPreserve is true. + NodePortRangeMin int32 `default:"30000" json:"nodeport_range_min" yaml:"nodeport_range_min"` + + // NodePortRangeMax is the upper bound of the Kubernetes NodePort range. + // Defaults to 32767 if unset. Used when NodePortPreserve is true. + NodePortRangeMax int32 `default:"32767" json:"nodeport_range_max" yaml:"nodeport_range_max"` + + // LBAnnotations is a map of annotations applied to LoadBalancer Services. + // Use this for provider-specific configuration (e.g. MetalLB address pool, + // cloud LB options). Example: + // lb_annotations: + // metallb.universe.tf/address-pool: "game-servers" + LBAnnotations map[string]string `json:"lb_annotations" yaml:"lb_annotations"` + + // LBIPAnnotation is the annotation key that Wings sets to the server's + // allocation IP on each LoadBalancer Service. This pins the LB to the IP + // the user selected in the Panel, enabling multiple game servers to share + // the same external IP (on different ports). + // + // Common values: + // Cilium: "lbipam.cilium.io/ips" + // MetalLB: "metallb.universe.tf/loadBalancerIPs" + // + // When empty (default), no IP-pinning annotation is added. + LBIPAnnotation string `default:"" json:"lb_ip_annotation" yaml:"lb_ip_annotation"` + + // LBSharingKey is the annotation key used to allow multiple Services to + // share the same external IP. Wings sets this annotation to the allocation + // IP value, grouping all game servers on the same IP under one sharing key. + // + // Common values: + // Cilium: "lbipam.cilium.io/sharing-key" + // MetalLB: "metallb.universe.tf/allow-shared-ip" + // + // When empty (default), no sharing-key annotation is added. + LBSharingKey string `default:"" json:"lb_sharing_key" yaml:"lb_sharing_key"` + + // Tolerations allow game server Pods to be scheduled on tainted nodes. + Tolerations []KubeToleration `json:"tolerations" yaml:"tolerations"` + + // NodeName is the Kubernetes node name where Wings is running. Used to + // query node addresses for the IP allocation endpoint. If empty, Wings + // reads the NODE_NAME environment variable (typically set via the + // Kubernetes downward API). + NodeName string `default:"" json:"node_name" yaml:"node_name"` + + // SystemIPs is a list of additional IP addresses to expose in the + // allocation endpoint. These are appended to the auto-discovered node + // addresses and can be used to advertise public IPs that are not + // directly assigned to the node (e.g. floating IPs, load balancer VIPs). + SystemIPs []string `json:"system_ips" yaml:"system_ips"` + + // DataPVC is the name of a shared PersistentVolumeClaim that Wings and game + // server Pods both mount for server data. When set, game servers use this PVC + // with a subPath instead of creating individual PVCs per server. This enables + // the Panel file browser and SFTP access, since Wings can read/write the same + // storage as the game server Pods. + // + // The PVC must already exist and be mounted into the Wings Pod at the + // configured rootDirectory. Typically this is the Wings data PVC created + // by the Helm chart. + // + // When empty (default), each server gets its own PVC (file browser won't work). + DataPVC string `default:"" json:"data_pvc" yaml:"data_pvc"` + + // ResourceQuota configures namespace-level resource limits. + // When enabled, Wings creates/updates a ResourceQuota in the game server + // namespace to cap aggregate resource usage across all Pods. + ResourceQuota KubeResourceQuota `json:"resource_quota" yaml:"resource_quota"` + + // LimitRange configures default resource limits and requests for Pods + // that do not specify their own. This prevents unbounded resource usage + // from misconfigured servers. + LimitRange KubeLimitRange `json:"limit_range" yaml:"limit_range"` +} + +// KubeResourceQuota defines namespace-level resource quotas. +type KubeResourceQuota struct { + // Enabled controls whether a ResourceQuota is created/enforced. + Enabled bool `default:"false" json:"enabled" yaml:"enabled"` + + // CPULimit is the total CPU limit across all Pods (e.g. "16", "8000m"). + CPULimit string `default:"" json:"cpu_limit" yaml:"cpu_limit"` + + // MemoryLimit is the total memory limit across all Pods (e.g. "32Gi"). + MemoryLimit string `default:"" json:"memory_limit" yaml:"memory_limit"` + + // CPURequest is the total CPU request across all Pods (e.g. "8", "4000m"). + CPURequest string `default:"" json:"cpu_request" yaml:"cpu_request"` + + // MemoryRequest is the total memory request across all Pods (e.g. "16Gi"). + MemoryRequest string `default:"" json:"memory_request" yaml:"memory_request"` + + // MaxPods is the maximum number of Pods allowed in the namespace. + MaxPods int64 `default:"0" json:"max_pods" yaml:"max_pods"` + + // MaxPVCs is the maximum number of PersistentVolumeClaims in the namespace. + MaxPVCs int64 `default:"0" json:"max_pvcs" yaml:"max_pvcs"` + + // MaxStorage is the total storage allowed across all PVCs (e.g. "500Gi"). + MaxStorage string `default:"" json:"max_storage" yaml:"max_storage"` +} + +// KubeLimitRange defines default resource limits and requests for containers. +type KubeLimitRange struct { + // Enabled controls whether a LimitRange is created. + Enabled bool `default:"false" json:"enabled" yaml:"enabled"` + + // DefaultCPULimit is the default CPU limit for containers (e.g. "2", "1000m"). + DefaultCPULimit string `default:"" json:"default_cpu_limit" yaml:"default_cpu_limit"` + + // DefaultMemoryLimit is the default memory limit for containers (e.g. "4Gi"). + DefaultMemoryLimit string `default:"" json:"default_memory_limit" yaml:"default_memory_limit"` + + // DefaultCPURequest is the default CPU request for containers (e.g. "500m"). + DefaultCPURequest string `default:"" json:"default_cpu_request" yaml:"default_cpu_request"` + + // DefaultMemoryRequest is the default memory request for containers (e.g. "1Gi"). + DefaultMemoryRequest string `default:"" json:"default_memory_request" yaml:"default_memory_request"` + + // MaxCPU is the maximum CPU any single container can request (e.g. "4"). + MaxCPU string `default:"" json:"max_cpu" yaml:"max_cpu"` + + // MaxMemory is the maximum memory any single container can request (e.g. "8Gi"). + MaxMemory string `default:"" json:"max_memory" yaml:"max_memory"` +} + +// KubeToleration mirrors corev1.Toleration for YAML configuration. +type KubeToleration struct { + Key string `json:"key" yaml:"key"` + Operator string `default:"Equal" json:"operator" yaml:"operator"` + Value string `json:"value" yaml:"value"` + Effect string `json:"effect" yaml:"effect"` + TolerationSeconds *int64 `json:"toleration_seconds" yaml:"toleration_seconds"` +} diff --git a/environment/kubernetes/api.go b/environment/kubernetes/api.go new file mode 100644 index 00000000..a82ac1d9 --- /dev/null +++ b/environment/kubernetes/api.go @@ -0,0 +1,42 @@ +package kubernetes + +import ( + "sync" + + "emperror.dev/errors" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + + "github.com/pelican-dev/wings/config" +) + +var ( + _konce sync.Once + _client kubernetes.Interface +) + +// Client returns a shared Kubernetes clientset. The client is created once +// and reused for all subsequent calls. It uses in-cluster config when no +// kubeconfig path is specified, falling back to the provided kubeconfig file. +func Client() (kubernetes.Interface, error) { + var err error + _konce.Do(func() { + var cfg *rest.Config + kubeconfig := config.Get().Kubernetes.Kubeconfig + if kubeconfig != "" { + cfg, err = clientcmd.BuildConfigFromFlags("", kubeconfig) + } else { + cfg, err = rest.InClusterConfig() + } + if err != nil { + err = errors.Wrap(err, "environment/kubernetes: failed to build config") + return + } + _client, err = kubernetes.NewForConfig(cfg) + if err != nil { + err = errors.Wrap(err, "environment/kubernetes: failed to create clientset") + } + }) + return _client, err +} diff --git a/environment/kubernetes/container.go b/environment/kubernetes/container.go new file mode 100644 index 00000000..d9b57e3a --- /dev/null +++ b/environment/kubernetes/container.go @@ -0,0 +1,577 @@ +package kubernetes + +import ( + "bufio" + "context" + "fmt" + "io" + "path/filepath" + "strings" + + "emperror.dev/errors" + "github.com/apex/log" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/remotecommand" + + "github.com/pelican-dev/wings/config" + "github.com/pelican-dev/wings/environment" + "github.com/pelican-dev/wings/system" +) + +// resolveImagePullPolicy returns the cleaned image reference (without the ~ +// local-image prefix) and the pull policy to apply. It mirrors the Docker +// backend, which always attempts to pull remote images before starting a +// server so updated tags are picked up, while never pulling ~-prefixed local +// images. A non-empty kubernetes.image_pull_policy config value overrides the +// default (useful for air-gapped clusters). +func resolveImagePullPolicy(image string) (string, corev1.PullPolicy) { + cleaned := strings.TrimPrefix(image, "~") + if override := config.Get().Kubernetes.ImagePullPolicy; override != "" { + return cleaned, corev1.PullPolicy(override) + } + if strings.HasPrefix(image, "~") { + return cleaned, corev1.PullIfNotPresent + } + return cleaned, corev1.PullAlways +} + +// Create builds and creates the Pod for this game server in Kubernetes. +// If the Pod already exists this is a no-op. +func (e *Environment) Create() error { + ctx := context.Background() + + // If the Pod already exists, return immediately. + if exists, _ := e.Exists(); exists { + return nil + } + + cfg := config.Get() + limits := e.Configuration.Limits() + evs := e.Configuration.EnvironmentVariables() + mounts := e.Configuration.Mounts() + + // Build environment variables for the container. + var envVars []corev1.EnvVar + for _, ev := range evs { + parts := strings.SplitN(ev, "=", 2) + if len(parts) == 2 { + envVars = append(envVars, corev1.EnvVar{ + Name: parts[0], + Value: parts[1], + }) + } + } + + // Build resource requirements from server limits. + resources := e.buildResources(limits) + + // Build volume mounts and volumes. In K8s mode, identity files (passwd, + // group, machine-id) are served from a ConfigMap instead of hostPath. + identityMounts, regularMounts := e.splitIdentityMounts(mounts) + volumes, volumeMounts := e.buildVolumes(regularMounts) + + if len(identityMounts) > 0 { + idVol, idMounts, err := e.ensureIdentityConfigMap(ctx, identityMounts) + if err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to create identity ConfigMap") + } + volumes = append(volumes, idVol) + volumeMounts = append(volumeMounts, idMounts...) + } + + // Determine image and pull policy (mirrors the Docker backend). + image, pullPolicy := resolveImagePullPolicy(e.meta.Image) + + // Build container ports from allocations. + containerPorts := e.buildContainerPorts() + + // Labels for identification. + labels := map[string]string{ + "app.kubernetes.io/managed-by": "pelican-wings", + "pelican.dev/server-id": e.Id, + "pelican.dev/container-type": "server_process", + } + + // Merge user-provided labels. + for k, v := range e.Configuration.Labels() { + labels[k] = v + } + + // Determine the UID/GID for the pelican user. + uid := int64(config.Get().System.User.Uid) + gid := int64(config.Get().System.User.Gid) + + // Build the Pod spec. + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: e.Id, + Namespace: e.namespace(), + Labels: labels, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + SecurityContext: &corev1.PodSecurityContext{ + RunAsUser: &uid, + RunAsGroup: &gid, + FSGroup: &gid, + }, + InitContainers: []corev1.Container{ + { + Name: "set-permissions", + Image: "busybox:1.36", + Command: []string{"sh", "-c", fmt.Sprintf("chown -R %d:%d /home/container && chmod -R 755 /home/container", uid, gid)}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "server-data", + MountPath: "/home/container", + SubPath: e.serverDataSubPathIfShared(), + }, + }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: func() *int64 { v := int64(0); return &v }(), + RunAsGroup: func() *int64 { v := int64(0); return &v }(), + }, + }, + }, + Containers: []corev1.Container{ + { + Name: "server", + Image: image, + Env: envVars, + Resources: resources, + Ports: containerPorts, + VolumeMounts: volumeMounts, + Stdin: true, + TTY: true, + ImagePullPolicy: pullPolicy, + }, + }, + Volumes: volumes, + }, + } + + // Apply network mode configuration. + if cfg.Kubernetes.NetworkMode == config.KubeNetworkHostPort { + pod.Spec.HostNetwork = false // hostPort is set on containerPorts + } + + // Apply DNS policy. + if cfg.Kubernetes.DNSPolicy != "" { + pod.Spec.DNSPolicy = corev1.DNSPolicy(cfg.Kubernetes.DNSPolicy) + } + + // Apply node selector. + if len(cfg.Kubernetes.NodeSelector) > 0 { + pod.Spec.NodeSelector = cfg.Kubernetes.NodeSelector + } + + // Apply service account. + if cfg.Kubernetes.ServiceAccount != "" { + pod.Spec.ServiceAccountName = cfg.Kubernetes.ServiceAccount + } + + // Apply image pull secrets. + for _, secret := range cfg.Kubernetes.ImagePullSecrets { + pod.Spec.ImagePullSecrets = append(pod.Spec.ImagePullSecrets, corev1.LocalObjectReference{ + Name: secret, + }) + } + + // Apply tolerations. + for _, t := range cfg.Kubernetes.Tolerations { + toleration := corev1.Toleration{ + Key: t.Key, + Operator: corev1.TolerationOperator(t.Operator), + Value: t.Value, + Effect: corev1.TaintEffect(t.Effect), + } + if t.TolerationSeconds != nil { + toleration.TolerationSeconds = t.TolerationSeconds + } + pod.Spec.Tolerations = append(pod.Spec.Tolerations, toleration) + } + + // Ensure namespace-level resource constraints are applied. + if err := e.EnsureResourceQuota(ctx); err != nil { + e.log().WithField("error", err).Warn("failed to ensure ResourceQuota") + } + if err := e.EnsureLimitRange(ctx); err != nil { + e.log().WithField("error", err).Warn("failed to ensure LimitRange") + } + + // Ensure the PVC exists if using PVC storage mode. + if err := e.EnsurePVC(ctx); err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to ensure PVC") + } + + e.log().WithField("image", image).Info("creating pod for server") + + _, err := e.client.CoreV1().Pods(e.namespace()).Create(ctx, pod, metav1.CreateOptions{}) + if err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to create pod") + } + + // Create or update the NodePort Service if using NodePort network mode. + if err := e.EnsureService(ctx); err != nil { + e.log().WithField("error", err).Warn("failed to create/update NodePort service") + } + + return nil +} + +// Destroy removes the Pod and associated Service from Kubernetes. +func (e *Environment) Destroy() error { + e.SetState(environment.ProcessStoppingState) + + ctx := context.Background() + + gracePeriod := int64(0) + err := e.client.CoreV1().Pods(e.namespace()).Delete( + ctx, + e.Id, + metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod}, + ) + + // Clean up the associated NodePort Service. + if svcErr := e.DeleteService(ctx); svcErr != nil { + e.log().WithField("error", svcErr).Warn("failed to delete NodePort service during destroy") + } + + // Clean up the PVC if using PVC storage mode. + if pvcErr := e.DeletePVC(ctx); pvcErr != nil { + e.log().WithField("error", pvcErr).Warn("failed to delete PVC during destroy") + } + + // Clean up the identity ConfigMap. + if cmErr := e.deleteIdentityConfigMap(ctx); cmErr != nil { + e.log().WithField("error", cmErr).Warn("failed to delete identity ConfigMap during destroy") + } + + e.SetState(environment.ProcessOfflineState) + + if err != nil && !isNotFound(err) { + return errors.Wrap(err, "environment/kubernetes: failed to delete pod") + } + + return nil +} + +// Attach connects to the running Pod's stdin/stdout and begins streaming +// output to the log callback. This should be called before the server starts +// producing output. +func (e *Environment) Attach(ctx context.Context) error { + if e.IsAttached() { + return nil + } + + restCfg, err := e.getRESTConfig() + if err != nil { + return errors.WrapIf(err, "environment/kubernetes: failed to get REST config") + } + + req := e.client.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(e.Id). + Namespace(e.namespace()). + SubResource("attach"). + VersionedParams(&corev1.PodAttachOptions{ + Container: "server", + Stdin: true, + Stdout: true, + Stderr: true, + TTY: true, + }, scheme.ParameterCodec) + + exec, err := remotecommand.NewSPDYExecutor(restCfg, "POST", req.URL()) + if err != nil { + return errors.WrapIf(err, "environment/kubernetes: failed to create SPDY executor") + } + + // Create a pipe for stdin. + stdinReader, stdinWriter := io.Pipe() + e.setStream(stdinWriter) + + go func() { + defer e.setStream(nil) + defer func() { + e.SetState(environment.ProcessOfflineState) + }() + + // Start resource polling. + pollCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go func() { + if err := e.pollResources(pollCtx); err != nil { + if !errors.Is(err, context.Canceled) { + e.log().WithField("error", err).Error("error during resource polling") + } + } + }() + + // Stream stdout/stderr to the log callback via a pipe. + stdoutReader, stdoutWriter := io.Pipe() + + go func() { + if err := system.ScanReader(stdoutReader, func(v []byte) { + e.logCallbackMx.Lock() + defer e.logCallbackMx.Unlock() + if e.logCallback != nil { + e.logCallback(v) + } + }); err != nil && err != io.EOF { + log.WithField("error", err).WithField("pod_id", e.Id).Warn("error processing output in console") + } + }() + + streamErr := exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdin: stdinReader, + Stdout: stdoutWriter, + Stderr: stdoutWriter, + Tty: true, + }) + stdoutWriter.Close() + stdinReader.Close() + + if streamErr != nil && !errors.Is(streamErr, context.Canceled) { + e.log().WithField("error", streamErr).Warn("pod attach stream ended") + } + }() + + return nil +} + +// SendCommand writes a command string to the attached Pod's stdin. +func (e *Environment) SendCommand(c string) error { + if !e.IsAttached() { + return errors.New("environment/kubernetes: not attached to pod") + } + + e.mu.RLock() + defer e.mu.RUnlock() + + // If this is the stop command, mark the server as stopping. + if e.meta.Stop.Type == "command" && c == e.meta.Stop.Value { + e.SetState(environment.ProcessStoppingState) + } + + _, err := e.stream.Write([]byte(c + "\n")) + return errors.Wrap(err, "environment/kubernetes: could not write to pod stream") +} + +// Readlog reads the last N lines of log output from the Pod. +func (e *Environment) Readlog(lines int) ([]string, error) { + tailLines := int64(lines) + opts := &corev1.PodLogOptions{ + Container: "server", + TailLines: &tailLines, + } + + req := e.client.CoreV1().Pods(e.namespace()).GetLogs(e.Id, opts) + stream, err := req.Stream(context.Background()) + if err != nil { + return nil, errors.WithStack(err) + } + defer stream.Close() + + var out []string + scanner := bufio.NewScanner(stream) + for scanner.Scan() { + out = append(out, scanner.Text()) + } + + return out, nil +} + +// buildResources converts Wings Limits into Kubernetes resource requirements. +func (e *Environment) buildResources(limits environment.Limits) corev1.ResourceRequirements { + resources := corev1.ResourceRequirements{ + Limits: corev1.ResourceList{}, + Requests: corev1.ResourceList{}, + } + + // Memory: limits.MemoryLimit is in MiB. + if limits.MemoryLimit > 0 { + mem := resource.MustParse(fmt.Sprintf("%dMi", limits.MemoryLimit)) + resources.Limits[corev1.ResourceMemory] = mem + // Request 50% of the limit as a baseline. + memReq := resource.MustParse(fmt.Sprintf("%dMi", limits.MemoryLimit/2)) + resources.Requests[corev1.ResourceMemory] = memReq + } + + // CPU: limits.CpuLimit is a percentage (100 = 1 core). + if limits.CpuLimit > 0 { + cpuMillis := limits.CpuLimit * 10 // 100% = 1000m + cpu := resource.MustParse(fmt.Sprintf("%dm", cpuMillis)) + resources.Limits[corev1.ResourceCPU] = cpu + // Request 25% of the limit. + cpuReqMillis := cpuMillis / 4 + if cpuReqMillis < 50 { + cpuReqMillis = 50 + } + cpuReq := resource.MustParse(fmt.Sprintf("%dm", cpuReqMillis)) + resources.Requests[corev1.ResourceCPU] = cpuReq + } + + return resources +} + +// buildVolumes constructs Kubernetes volumes and mounts from the environment +// mount configuration. When StorageMode is "pvc", the default server-data +// volume uses a PersistentVolumeClaim instead of a HostPath. +// +// When DataPVC is set, the default mount uses the shared Wings data PVC with +// a subPath so that Wings and game server Pods share the same storage. This +// enables the Panel file browser and SFTP to work in K8s mode. +func (e *Environment) buildVolumes(mounts []environment.Mount) ([]corev1.Volume, []corev1.VolumeMount) { + cfg := config.Get() + var volumes []corev1.Volume + var volumeMounts []corev1.VolumeMount + + for i, m := range mounts { + volName := fmt.Sprintf("mount-%d", i) + if m.Default { + volName = "server-data" + } + + var volSource corev1.VolumeSource + var subPath string + + if m.Default && cfg.Kubernetes.DataPVC != "" { + // Shared PVC mode: mount the Wings data PVC with a subPath + // so both Wings and the game server see the same files. + volSource = corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: cfg.Kubernetes.DataPVC, + }, + } + subPath = e.serverDataSubPath() + } else if m.Default && cfg.Kubernetes.StorageMode == config.KubeStoragePVC { + // Per-server PVC mode (original behavior). + volSource = corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: e.pvcName(), + }, + } + } else { + // Use HostPath for non-default mounts or when in hostpath mode. + volSource = corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: m.Source, + }, + } + } + + volumes = append(volumes, corev1.Volume{ + Name: volName, + VolumeSource: volSource, + }) + + vm := corev1.VolumeMount{ + Name: volName, + MountPath: m.Target, + ReadOnly: m.ReadOnly, + } + if subPath != "" { + vm.SubPath = subPath + } + volumeMounts = append(volumeMounts, vm) + } + + return volumes, volumeMounts +} + +// serverDataSubPath returns the subPath within the shared data PVC for this +// server's data directory. This is the relative path from the Wings root +// directory to the server's data directory. +func (e *Environment) serverDataSubPath() string { + cfg := config.Get() + serverPath := filepath.Join(cfg.System.Data, e.Id) + rel, err := filepath.Rel(cfg.System.RootDirectory, serverPath) + if err != nil { + return filepath.Join("volumes", e.Id) + } + return rel +} + +// serverDataSubPathIfShared returns the subPath for shared PVC mode, or an +// empty string if not using shared PVC mode. +func (e *Environment) serverDataSubPathIfShared() string { + if config.Get().Kubernetes.DataPVC != "" { + return e.serverDataSubPath() + } + return "" +} + +// buildContainerPorts creates the container port list from allocations. +func (e *Environment) buildContainerPorts() []corev1.ContainerPort { + cfg := config.Get() + allocs := e.Configuration.Allocations() + var ports []corev1.ContainerPort + + for _, allocPorts := range allocs.Mappings { + for _, port := range allocPorts { + if port < 1 || port > 65535 { + continue + } + + tcpPort := corev1.ContainerPort{ + Name: fmt.Sprintf("tcp-%d", port), + ContainerPort: int32(port), + Protocol: corev1.ProtocolTCP, + } + udpPort := corev1.ContainerPort{ + Name: fmt.Sprintf("udp-%d", port), + ContainerPort: int32(port), + Protocol: corev1.ProtocolUDP, + } + + // If using hostPort mode, bind directly to the host. + if cfg.Kubernetes.NetworkMode == config.KubeNetworkHostPort { + tcpPort.HostPort = int32(port) + udpPort.HostPort = int32(port) + } + + ports = append(ports, tcpPort, udpPort) + } + } + + return ports +} + +// getRESTConfig returns the rest.Config used for exec/attach operations. +func (e *Environment) getRESTConfig() (*rest.Config, error) { + kubeconfig := config.Get().Kubernetes.Kubeconfig + if kubeconfig != "" { + return clientcmd.BuildConfigFromFlags("", kubeconfig) + } + return rest.InClusterConfig() +} + +// getPod fetches the Pod object from the Kubernetes API. +func (e *Environment) getPod(ctx context.Context) (*corev1.Pod, error) { + return e.client.CoreV1().Pods(e.namespace()).Get(ctx, e.Id, metav1.GetOptions{}) +} + +// isNotFound checks if the error is a Kubernetes NotFound error. +func isNotFound(err error) bool { + return strings.Contains(err.Error(), "not found") +} + +// isPodRunning checks if a Pod is in Running phase with a ready container. +func isPodRunning(pod *corev1.Pod) bool { + if pod.Status.Phase != corev1.PodRunning { + return false + } + for _, cs := range pod.Status.ContainerStatuses { + if cs.Name == "server" && cs.State.Running != nil { + return true + } + } + return false +} diff --git a/environment/kubernetes/container_test.go b/environment/kubernetes/container_test.go new file mode 100644 index 00000000..29ff619e --- /dev/null +++ b/environment/kubernetes/container_test.go @@ -0,0 +1,52 @@ +package kubernetes + +import ( + "testing" + + . "github.com/franela/goblin" + corev1 "k8s.io/api/core/v1" + + "github.com/pelican-dev/wings/config" +) + +func TestResolveImagePullPolicy(t *testing.T) { + g := Goblin(t) + + g.Describe("resolveImagePullPolicy", func() { + g.BeforeEach(func() { + config.Update(func(c *config.Configuration) { + c.Kubernetes.ImagePullPolicy = "" + }) + }) + + g.It("always pulls remote images so updated tags are picked up", func() { + image, policy := resolveImagePullPolicy("ghcr.io/pelican-eggs/games:latest") + g.Assert(image).Equal("ghcr.io/pelican-eggs/games:latest") + g.Assert(policy).Equal(corev1.PullAlways) + }) + + g.It("does not pull ~-prefixed local images and strips the prefix", func() { + image, policy := resolveImagePullPolicy("~local/custom:dev") + g.Assert(image).Equal("local/custom:dev") + g.Assert(policy).Equal(corev1.PullIfNotPresent) + }) + + g.It("honors a configured override for remote images", func() { + config.Update(func(c *config.Configuration) { + c.Kubernetes.ImagePullPolicy = "IfNotPresent" + }) + image, policy := resolveImagePullPolicy("ghcr.io/pelican-eggs/games:latest") + g.Assert(image).Equal("ghcr.io/pelican-eggs/games:latest") + g.Assert(policy).Equal(corev1.PullIfNotPresent) + }) + + g.It("honors a configured override and still strips the local prefix", func() { + config.Update(func(c *config.Configuration) { + c.Kubernetes.ImagePullPolicy = "Never" + }) + image, policy := resolveImagePullPolicy("~local/custom:dev") + g.Assert(image).Equal("local/custom:dev") + g.Assert(policy).Equal(corev1.PullNever) + }) + }) +} diff --git a/environment/kubernetes/environment.go b/environment/kubernetes/environment.go new file mode 100644 index 00000000..eef8f418 --- /dev/null +++ b/environment/kubernetes/environment.go @@ -0,0 +1,209 @@ +package kubernetes + +import ( + "context" + "fmt" + "io" + "sync" + + "emperror.dev/errors" + "github.com/apex/log" + "k8s.io/client-go/kubernetes" + + "github.com/pelican-dev/wings/config" + "github.com/pelican-dev/wings/environment" + "github.com/pelican-dev/wings/events" + "github.com/pelican-dev/wings/remote" + "github.com/pelican-dev/wings/system" +) + +// Metadata holds runtime metadata for the Kubernetes environment that can be +// updated on the fly (e.g., image changes from the Panel). +type Metadata struct { + Image string + Stop remote.ProcessStopConfiguration +} + +// Ensure that the Kubernetes environment always implements the full +// ProcessEnvironment interface. +var _ environment.ProcessEnvironment = (*Environment)(nil) + +// Environment is the Kubernetes implementation of ProcessEnvironment. It +// manages game server workloads as Pods within a configured namespace. +type Environment struct { + mu sync.RWMutex + + // Id is the unique server identifier (UUID) used as the Pod name. + Id string + + // Configuration holds the environment settings (limits, mounts, env vars). + Configuration *environment.Configuration + + meta *Metadata + + // client is the Kubernetes clientset. + client kubernetes.Interface + + // stream holds the attach connection to the running Pod's stdin. + stream io.WriteCloser + + emitter *events.Bus + + logCallbackMx sync.Mutex + logCallback func([]byte) + + // st tracks the current process state. + st *system.AtomicString +} + +// New creates a new Kubernetes environment for the given server ID. +func New(id string, m *Metadata, c *environment.Configuration) (*Environment, error) { + cli, err := Client() + if err != nil { + return nil, err + } + + e := &Environment{ + Id: id, + Configuration: c, + meta: m, + client: cli, + st: system.NewAtomicString(environment.ProcessOfflineState), + emitter: events.NewBus(), + } + + return e, nil +} + +func (e *Environment) log() *log.Entry { + return log.WithField("environment", e.Type()).WithField("pod_id", e.Id) +} + +// Type returns the environment type identifier. +func (e *Environment) Type() string { + return "kubernetes" +} + +// Events returns the event bus for this environment. +func (e *Environment) Events() *events.Bus { + return e.emitter +} + +// Config returns the environment configuration. +func (e *Environment) Config() *environment.Configuration { + e.mu.RLock() + defer e.mu.RUnlock() + return e.Configuration +} + +// State returns the current process state string. +func (e *Environment) State() string { + return e.st.Load() +} + +// SetState updates the environment state and publishes a state change event. +func (e *Environment) SetState(state string) { + if state != environment.ProcessOfflineState && + state != environment.ProcessStartingState && + state != environment.ProcessRunningState && + state != environment.ProcessStoppingState { + panic(errors.New(fmt.Sprintf("invalid server state received: %s", state))) + } + + if e.State() != state { + e.st.Store(state) + e.Events().Publish(environment.StateChangeEvent, state) + } +} + +// SetLogCallback sets the callback function for container log output. +func (e *Environment) SetLogCallback(f func([]byte)) { + e.logCallbackMx.Lock() + defer e.logCallbackMx.Unlock() + e.logCallback = f +} + +// SetStopConfiguration updates the stop configuration on the fly. +func (e *Environment) SetStopConfiguration(c remote.ProcessStopConfiguration) { + e.mu.Lock() + e.meta.Stop = c + e.mu.Unlock() +} + +// SetImage updates the container image for the server. +func (e *Environment) SetImage(i string) { + e.mu.Lock() + defer e.mu.Unlock() + e.meta.Image = i +} + +// IsAttached returns whether the environment is currently attached to the Pod. +func (e *Environment) IsAttached() bool { + e.mu.RLock() + defer e.mu.RUnlock() + return e.stream != nil +} + +// setStream updates the attach stream reference. +func (e *Environment) setStream(s io.WriteCloser) { + e.mu.Lock() + e.stream = s + e.mu.Unlock() +} + +// Exists checks whether the Pod for this server exists in the cluster. +func (e *Environment) Exists() (bool, error) { + _, err := e.getPod(context.Background()) + if err != nil { + if isNotFound(err) { + return false, nil + } + return false, err + } + return true, nil +} + +// IsRunning checks whether the Pod is in Running phase. +func (e *Environment) IsRunning(ctx context.Context) (bool, error) { + pod, err := e.getPod(ctx) + if err != nil { + return false, err + } + return isPodRunning(pod), nil +} + +// ExitState returns the exit code and OOM-killed status of the terminated +// container. +func (e *Environment) ExitState() (uint32, bool, error) { + pod, err := e.getPod(context.Background()) + if err != nil { + if isNotFound(err) { + return 1, false, nil + } + return 0, false, errors.WrapIf(err, "environment/kubernetes: failed to get pod") + } + + for _, cs := range pod.Status.ContainerStatuses { + if cs.Name == "server" && cs.State.Terminated != nil { + oom := cs.State.Terminated.Reason == "OOMKilled" + return uint32(cs.State.Terminated.ExitCode), oom, nil + } + } + + return 0, false, nil +} + +// InSituUpdate is a no-op for Kubernetes since Pod resource limits are +// immutable after creation. The Pod must be recreated to apply new limits. +func (e *Environment) InSituUpdate() error { + return nil +} + +// namespace returns the configured Kubernetes namespace. +func (e *Environment) namespace() string { + ns := config.Get().Kubernetes.Namespace + if ns == "" { + return "pelican" + } + return ns +} diff --git a/environment/kubernetes/environment_test.go b/environment/kubernetes/environment_test.go new file mode 100644 index 00000000..433dd9ad --- /dev/null +++ b/environment/kubernetes/environment_test.go @@ -0,0 +1,567 @@ +package kubernetes + +import ( + "context" + "testing" + + . "github.com/franela/goblin" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + + "github.com/pelican-dev/wings/config" + "github.com/pelican-dev/wings/environment" + "github.com/pelican-dev/wings/events" + "github.com/pelican-dev/wings/system" +) + +func TestEnvironment(t *testing.T) { + g := Goblin(t) + + g.Describe("Environment", func() { + g.Describe("Type", func() { + g.It("should return 'kubernetes'", func() { + env := &Environment{st: system.NewAtomicString(environment.ProcessOfflineState)} + g.Assert(env.Type()).Equal("kubernetes") + }) + }) + + g.Describe("State", func() { + g.It("should return the current state", func() { + env := &Environment{ + st: system.NewAtomicString(environment.ProcessOfflineState), + emitter: events.NewBus(), + } + g.Assert(env.State()).Equal(environment.ProcessOfflineState) + }) + + g.It("should update state via SetState", func() { + env := &Environment{ + st: system.NewAtomicString(environment.ProcessOfflineState), + emitter: events.NewBus(), + } + env.SetState(environment.ProcessRunningState) + g.Assert(env.State()).Equal(environment.ProcessRunningState) + }) + + g.It("should panic on invalid state", func() { + env := &Environment{ + st: system.NewAtomicString(environment.ProcessOfflineState), + emitter: events.NewBus(), + } + panicked := false + func() { + defer func() { + if r := recover(); r != nil { + panicked = true + } + }() + env.SetState("bogus") + }() + g.Assert(panicked).IsTrue() + }) + }) + + g.Describe("Exists", func() { + g.It("should return true when Pod exists", func() { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-uuid", + Namespace: "pelican", + }, + } + client := fake.NewSimpleClientset(pod) + env := &Environment{ + Id: "test-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + }) + + exists, err := env.Exists() + g.Assert(err).IsNil() + g.Assert(exists).IsTrue() + }) + + g.It("should return false when Pod does not exist", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "nonexistent-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + }) + + exists, err := env.Exists() + g.Assert(err).IsNil() + g.Assert(exists).IsFalse() + }) + }) + + g.Describe("IsRunning", func() { + g.It("should return true when Pod is in Running phase", func() { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-uuid", + Namespace: "pelican", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "server", + State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}, + }, + }, + }, + } + client := fake.NewSimpleClientset(pod) + env := &Environment{ + Id: "test-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + }) + + running, err := env.IsRunning(context.Background()) + g.Assert(err).IsNil() + g.Assert(running).IsTrue() + }) + + g.It("should return false when Pod is Pending", func() { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-uuid", + Namespace: "pelican", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodPending, + }, + } + client := fake.NewSimpleClientset(pod) + env := &Environment{ + Id: "test-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + }) + + running, err := env.IsRunning(context.Background()) + g.Assert(err).IsNil() + g.Assert(running).IsFalse() + }) + }) + + g.Describe("ExitState", func() { + g.It("should return exit code and OOM status from terminated container", func() { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-uuid", + Namespace: "pelican", + }, + Status: corev1.PodStatus{ + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "server", + State: corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ + ExitCode: 137, + Reason: "OOMKilled", + }, + }, + }, + }, + }, + } + client := fake.NewSimpleClientset(pod) + env := &Environment{ + Id: "test-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + }) + + code, oom, err := env.ExitState() + g.Assert(err).IsNil() + g.Assert(code).Equal(uint32(137)) + g.Assert(oom).IsTrue() + }) + + g.It("should return exit code 1 when Pod does not exist", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "nonexistent-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + }) + + code, oom, err := env.ExitState() + g.Assert(err).IsNil() + g.Assert(code).Equal(uint32(1)) + g.Assert(oom).IsFalse() + }) + }) + }) + + g.Describe("Create", func() { + g.It("should create a Pod with correct spec", func() { + client := fake.NewSimpleClientset() + allocs := environment.Allocations{ + Mappings: map[string][]int{"0.0.0.0": {25565}}, + } + settings := environment.Settings{ + Allocations: allocs, + Limits: environment.Limits{ + MemoryLimit: 1024, + CpuLimit: 200, + }, + Mounts: []environment.Mount{ + {Default: true, Source: "/var/lib/pelican/servers/test", Target: "/home/container"}, + }, + } + cfg := environment.NewConfiguration(settings, []string{"JAVA_OPTS=-Xmx512M"}) + env := &Environment{ + Id: "create-test-uuid", + Configuration: cfg, + meta: &Metadata{Image: "itzg/minecraft-server:latest"}, + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + emitter: events.NewBus(), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.NetworkMode = config.KubeNetworkHostPort + }) + + err := env.Create() + g.Assert(err).IsNil() + + pod, err := client.CoreV1().Pods("pelican").Get(context.Background(), "create-test-uuid", metav1.GetOptions{}) + g.Assert(err).IsNil() + g.Assert(pod.Name).Equal("create-test-uuid") + g.Assert(pod.Namespace).Equal("pelican") + + // Verify container. + g.Assert(len(pod.Spec.Containers)).Equal(1) + c := pod.Spec.Containers[0] + g.Assert(c.Name).Equal("server") + g.Assert(c.Image).Equal("itzg/minecraft-server:latest") + g.Assert(c.Stdin).IsTrue() + g.Assert(c.TTY).IsTrue() + + // Verify resource limits. + memLimit := c.Resources.Limits[corev1.ResourceMemory] + g.Assert(memLimit.Equal(resource.MustParse("1024Mi"))).IsTrue() + cpuLimit := c.Resources.Limits[corev1.ResourceCPU] + g.Assert(cpuLimit.Equal(resource.MustParse("2000m"))).IsTrue() + + // Verify environment variables. + g.Assert(len(c.Env) >= 1).IsTrue() + found := false + for _, ev := range c.Env { + if ev.Name == "JAVA_OPTS" && ev.Value == "-Xmx512M" { + found = true + } + } + g.Assert(found).IsTrue() + + // Verify volumes. + g.Assert(len(pod.Spec.Volumes)).Equal(1) + g.Assert(pod.Spec.Volumes[0].Name).Equal("server-data") + g.Assert(pod.Spec.Volumes[0].HostPath.Path).Equal("/var/lib/pelican/servers/test") + + // Verify labels. + g.Assert(pod.Labels["pelican.dev/server-id"]).Equal("create-test-uuid") + g.Assert(pod.Labels["app.kubernetes.io/managed-by"]).Equal("pelican-wings") + }) + + g.It("should not recreate existing Pod", func() { + existingPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "existing-uuid", + Namespace: "pelican", + }, + } + client := fake.NewSimpleClientset(existingPod) + allocs := environment.Allocations{} + settings := environment.Settings{ + Allocations: allocs, + Limits: environment.Limits{MemoryLimit: 512}, + } + cfg := environment.NewConfiguration(settings, nil) + env := &Environment{ + Id: "existing-uuid", + Configuration: cfg, + meta: &Metadata{Image: "nginx"}, + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + emitter: events.NewBus(), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + }) + + err := env.Create() + g.Assert(err).IsNil() + }) + + g.It("should strip ~ prefix from image", func() { + client := fake.NewSimpleClientset() + allocs := environment.Allocations{} + settings := environment.Settings{Allocations: allocs} + cfg := environment.NewConfiguration(settings, nil) + env := &Environment{ + Id: "tilde-test-uuid", + Configuration: cfg, + meta: &Metadata{Image: "~local/myimage:latest"}, + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + emitter: events.NewBus(), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + }) + + err := env.Create() + g.Assert(err).IsNil() + + pod, _ := client.CoreV1().Pods("pelican").Get(context.Background(), "tilde-test-uuid", metav1.GetOptions{}) + g.Assert(pod.Spec.Containers[0].Image).Equal("local/myimage:latest") + }) + }) + + g.Describe("Destroy", func() { + g.It("should delete the Pod and transition to offline state", func() { + existingPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "destroy-uuid", + Namespace: "pelican", + }, + } + client := fake.NewSimpleClientset(existingPod) + env := &Environment{ + Id: "destroy-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessRunningState), + emitter: events.NewBus(), + Configuration: environment.NewConfiguration(environment.Settings{}, nil), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.NetworkMode = config.KubeNetworkHostPort + }) + + err := env.Destroy() + g.Assert(err).IsNil() + g.Assert(env.State()).Equal(environment.ProcessOfflineState) + + // Pod should be gone. + _, err = client.CoreV1().Pods("pelican").Get(context.Background(), "destroy-uuid", metav1.GetOptions{}) + g.Assert(err).IsNotNil() + }) + + g.It("should not error when Pod does not exist", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "nonexistent-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessRunningState), + emitter: events.NewBus(), + Configuration: environment.NewConfiguration(environment.Settings{}, nil), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.NetworkMode = config.KubeNetworkHostPort + }) + + err := env.Destroy() + g.Assert(err).IsNil() + g.Assert(env.State()).Equal(environment.ProcessOfflineState) + }) + + g.It("should also delete the NodePort Service", func() { + existingPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "svc-destroy-uuid", + Namespace: "pelican", + }, + } + existingSvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gs-svc-destroy-uuid", + Namespace: "pelican", + }, + } + client := fake.NewSimpleClientset(existingPod, existingSvc) + env := &Environment{ + Id: "svc-destroy-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessRunningState), + emitter: events.NewBus(), + Configuration: environment.NewConfiguration(environment.Settings{}, nil), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.NetworkMode = config.KubeNetworkNodePort + }) + + err := env.Destroy() + g.Assert(err).IsNil() + + // Both Pod and Service should be gone. + _, err = client.CoreV1().Pods("pelican").Get(context.Background(), "svc-destroy-uuid", metav1.GetOptions{}) + g.Assert(err).IsNotNil() + _, err = client.CoreV1().Services("pelican").Get(context.Background(), "gs-svc-destroy-uuid", metav1.GetOptions{}) + g.Assert(err).IsNotNil() + }) + }) + + g.Describe("buildResources", func() { + g.It("should convert memory and CPU limits correctly", func() { + env := &Environment{} + limits := environment.Limits{ + MemoryLimit: 2048, + CpuLimit: 150, + } + resources := env.buildResources(limits) + + memLimit := resources.Limits[corev1.ResourceMemory] + g.Assert(memLimit.Equal(resource.MustParse("2048Mi"))).IsTrue() + + memReq := resources.Requests[corev1.ResourceMemory] + g.Assert(memReq.Equal(resource.MustParse("1024Mi"))).IsTrue() + + cpuLimit := resources.Limits[corev1.ResourceCPU] + g.Assert(cpuLimit.Equal(resource.MustParse("1500m"))).IsTrue() + + cpuReq := resources.Requests[corev1.ResourceCPU] + g.Assert(cpuReq.Equal(resource.MustParse("375m"))).IsTrue() + }) + + g.It("should set minimum CPU request of 50m", func() { + env := &Environment{} + limits := environment.Limits{ + CpuLimit: 10, // 10% = 100m limit, 25m request would be below 50m + } + resources := env.buildResources(limits) + cpuReq := resources.Requests[corev1.ResourceCPU] + g.Assert(cpuReq.Equal(resource.MustParse("50m"))).IsTrue() + }) + + g.It("should handle zero limits gracefully", func() { + env := &Environment{} + limits := environment.Limits{} + resources := env.buildResources(limits) + g.Assert(len(resources.Limits)).Equal(0) + g.Assert(len(resources.Requests)).Equal(0) + }) + }) + + g.Describe("buildVolumes", func() { + g.It("should create volumes with correct names", func() { + env := &Environment{} + mounts := []environment.Mount{ + {Default: true, Source: "/data/server1", Target: "/home/container", ReadOnly: false}, + {Default: false, Source: "/shared/plugins", Target: "/plugins", ReadOnly: true}, + } + volumes, volumeMounts := env.buildVolumes(mounts) + + g.Assert(len(volumes)).Equal(2) + g.Assert(volumes[0].Name).Equal("server-data") + g.Assert(volumes[0].HostPath.Path).Equal("/data/server1") + g.Assert(volumes[1].Name).Equal("mount-1") + g.Assert(volumes[1].HostPath.Path).Equal("/shared/plugins") + + g.Assert(len(volumeMounts)).Equal(2) + g.Assert(volumeMounts[0].MountPath).Equal("/home/container") + g.Assert(volumeMounts[0].ReadOnly).IsFalse() + g.Assert(volumeMounts[1].MountPath).Equal("/plugins") + g.Assert(volumeMounts[1].ReadOnly).IsTrue() + }) + }) + + g.Describe("buildContainerPorts", func() { + g.It("should create TCP and UDP ports for each allocation", func() { + allocs := environment.Allocations{ + Mappings: map[string][]int{"0.0.0.0": {25565, 25575}}, + } + settings := environment.Settings{Allocations: allocs} + cfg := environment.NewConfiguration(settings, nil) + env := &Environment{Configuration: cfg} + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkHostPort + }) + + ports := env.buildContainerPorts() + g.Assert(len(ports)).Equal(4) // 2 ports × (TCP + UDP) + + // Verify hostPort is set in hostport mode. + for _, p := range ports { + g.Assert(p.HostPort).Equal(p.ContainerPort) + } + }) + + g.It("should not set hostPort in nodeport mode", func() { + allocs := environment.Allocations{ + Mappings: map[string][]int{"0.0.0.0": {25565}}, + } + settings := environment.Settings{Allocations: allocs} + cfg := environment.NewConfiguration(settings, nil) + env := &Environment{Configuration: cfg} + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkNodePort + }) + + ports := env.buildContainerPorts() + g.Assert(len(ports)).Equal(2) + for _, p := range ports { + g.Assert(p.HostPort).Equal(int32(0)) + } + }) + + g.It("should skip invalid ports", func() { + allocs := environment.Allocations{ + Mappings: map[string][]int{"0.0.0.0": {0, -1, 70000, 8080}}, + } + settings := environment.Settings{Allocations: allocs} + cfg := environment.NewConfiguration(settings, nil) + env := &Environment{Configuration: cfg} + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkNodePort + }) + + ports := env.buildContainerPorts() + // Only 8080 is valid, so 2 ports (TCP + UDP). + g.Assert(len(ports)).Equal(2) + }) + }) +} diff --git a/environment/kubernetes/identity.go b/environment/kubernetes/identity.go new file mode 100644 index 00000000..d52e5bb1 --- /dev/null +++ b/environment/kubernetes/identity.go @@ -0,0 +1,158 @@ +package kubernetes + +import ( + "context" + "fmt" + "os" + "strings" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/pelican-dev/wings/config" + "github.com/pelican-dev/wings/environment" +) + +// identityConfigMapName returns the name for the identity ConfigMap. +func (e *Environment) identityConfigMapName() string { + return fmt.Sprintf("gs-%s-identity", e.Id) +} + +// isIdentityMount returns true if the mount targets one of the well-known +// identity file paths (/etc/passwd, /etc/group, /etc/machine-id). +func isIdentityMount(m environment.Mount) bool { + switch m.Target { + case "/etc/passwd", "/etc/group", "/etc/machine-id": + return true + } + return false +} + +// splitIdentityMounts partitions mounts into identity mounts (passwd, group, +// machine-id) and everything else. In Kubernetes mode, identity mounts are +// served from a ConfigMap instead of hostPath. +func (e *Environment) splitIdentityMounts(mounts []environment.Mount) (identity, regular []environment.Mount) { + if !config.Get().Kubernetes.Enabled { + return nil, mounts + } + for _, m := range mounts { + if isIdentityMount(m) { + identity = append(identity, m) + } else { + regular = append(regular, m) + } + } + return +} + +// ensureIdentityConfigMap creates or updates a ConfigMap containing the +// identity files (passwd, group, machine-id) for this game server. It returns +// the Volume and VolumeMounts to be added to the Pod spec. +func (e *Environment) ensureIdentityConfigMap(ctx context.Context, mounts []environment.Mount) (corev1.Volume, []corev1.VolumeMount, error) { + cmName := e.identityConfigMapName() + ns := e.namespace() + + data := make(map[string]string) + var volumeMounts []corev1.VolumeMount + + for _, m := range mounts { + var key, content string + switch m.Target { + case "/etc/passwd": + key = "passwd" + content = e.readFileOrGenerate(m.Source, e.generatePasswd()) + case "/etc/group": + key = "group" + content = e.readFileOrGenerate(m.Source, e.generateGroup()) + case "/etc/machine-id": + key = "machine-id" + content = strings.ReplaceAll(e.Id, "-", "") + default: + continue + } + data[key] = content + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "identity", + MountPath: m.Target, + SubPath: key, + ReadOnly: true, + }) + } + + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cmName, + Namespace: ns, + Labels: map[string]string{ + "app.kubernetes.io/managed-by": "pelican-wings", + "pelican.dev/server-id": e.Id, + "pelican.dev/resource-type": "identity", + }, + }, + Data: data, + } + + existing, err := e.client.CoreV1().ConfigMaps(ns).Get(ctx, cmName, metav1.GetOptions{}) + if err != nil { + if !isNotFound(err) { + return corev1.Volume{}, nil, err + } + _, err = e.client.CoreV1().ConfigMaps(ns).Create(ctx, cm, metav1.CreateOptions{}) + if err != nil { + return corev1.Volume{}, nil, err + } + } else { + existing.Data = data + _, err = e.client.CoreV1().ConfigMaps(ns).Update(ctx, existing, metav1.UpdateOptions{}) + if err != nil { + return corev1.Volume{}, nil, err + } + } + + vol := corev1.Volume{ + Name: "identity", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: cmName, + }, + }, + }, + } + + return vol, volumeMounts, nil +} + +// deleteIdentityConfigMap removes the identity ConfigMap for this server. +func (e *Environment) deleteIdentityConfigMap(ctx context.Context) error { + err := e.client.CoreV1().ConfigMaps(e.namespace()).Delete(ctx, e.identityConfigMapName(), metav1.DeleteOptions{}) + if err != nil && !isNotFound(err) { + return err + } + return nil +} + +// readFileOrGenerate tries to read the file at path. If it doesn't exist +// (common on immutable OSes like Talos), returns the fallback content. +func (e *Environment) readFileOrGenerate(path, fallback string) string { + data, err := os.ReadFile(path) + if err != nil { + return fallback + } + return string(data) +} + +// generatePasswd returns a passwd file matching the format from +// config.ConfigurePasswd(). +func (e *Environment) generatePasswd() string { + cfg := config.Get() + return fmt.Sprintf("container:x:%d:%d::/home/container:/usr/sbin/nologin", + cfg.System.User.Uid, cfg.System.User.Gid) +} + +// generateGroup returns a group file matching the format from +// config.ConfigurePasswd(). +func (e *Environment) generateGroup() string { + cfg := config.Get() + return fmt.Sprintf("container:x:%d:container", cfg.System.User.Gid) +} diff --git a/environment/kubernetes/installer.go b/environment/kubernetes/installer.go new file mode 100644 index 00000000..62c3e7f6 --- /dev/null +++ b/environment/kubernetes/installer.go @@ -0,0 +1,389 @@ +package kubernetes + +import ( + "bufio" + "context" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "emperror.dev/errors" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + "github.com/pelican-dev/wings/config" + "github.com/pelican-dev/wings/remote" + "github.com/pelican-dev/wings/system" +) + +// InstallerProcess handles running server installation scripts as Kubernetes +// Jobs. It mirrors the Docker-based InstallationProcess but uses Jobs instead +// of standalone containers. +type InstallerProcess struct { + ServerID string + Script *remote.InstallationScript + EnvVars []string + ServerPath string + TmpDir string + Sink *system.SinkPool + client kubernetes.Interface + namespace string +} + +// NewInstallerProcess creates a new Kubernetes-based installer process for a +// server. The caller is responsible for writing the install script to tmpDir. +func NewInstallerProcess(env *Environment, script *remote.InstallationScript, envVars []string, serverPath, tmpDir string, sink *system.SinkPool) *InstallerProcess { + return &InstallerProcess{ + ServerID: env.Id, + Script: script, + EnvVars: envVars, + ServerPath: serverPath, + TmpDir: tmpDir, + Sink: sink, + client: env.client, + namespace: env.namespace(), + } +} + +// jobName returns the name for the installation Job. +func (ip *InstallerProcess) jobName() string { + return fmt.Sprintf("%s-installer", ip.ServerID) +} + +// configMapName returns the name for the install script ConfigMap. +func (ip *InstallerProcess) configMapName() string { + return fmt.Sprintf("%s-install-script", ip.ServerID) +} + +// Run executes the full installation process: creates a Job, streams its logs, +// waits for completion, and cleans up. +func (ip *InstallerProcess) Run(ctx context.Context) error { + // Clean up any existing installer Job from a previous run. + _ = ip.client.BatchV1().Jobs(ip.namespace).Delete(ctx, ip.jobName(), metav1.DeleteOptions{ + PropagationPolicy: propagationBackground(), + }) + + // Build environment variables. + var envVars []corev1.EnvVar + for _, ev := range ip.EnvVars { + parts := strings.SplitN(ev, "=", 2) + if len(parts) == 2 { + envVars = append(envVars, corev1.EnvVar{ + Name: parts[0], + Value: parts[1], + }) + } + } + + cfg := config.Get() + + // Determine install image and pull policy (mirrors the Docker backend). + installImage, installPullPolicy := resolveImagePullPolicy(ip.Script.ContainerImage) + + // Build the Job spec. + backoffLimit := int32(0) + ttl := int32(300) // Auto-clean Job after 5 minutes. + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: ip.jobName(), + Namespace: ip.namespace, + Labels: map[string]string{ + "app.kubernetes.io/managed-by": "pelican-wings", + "pelican.dev/server-id": ip.ServerID, + "pelican.dev/container-type": "server_installer", + }, + }, + Spec: batchv1.JobSpec{ + BackoffLimit: &backoffLimit, + TTLSecondsAfterFinished: &ttl, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app.kubernetes.io/managed-by": "pelican-wings", + "pelican.dev/server-id": ip.ServerID, + "pelican.dev/container-type": "server_installer", + "job-name": ip.jobName(), + }, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + Containers: []corev1.Container{ + { + Name: "installer", + Image: installImage, + Command: []string{ip.Script.Entrypoint, "/mnt/install/install.sh"}, + Env: envVars, + VolumeMounts: []corev1.VolumeMount{ + ip.buildServerDataMount(), + { + Name: "install-script", + MountPath: "/mnt/install", + }, + }, + ImagePullPolicy: installPullPolicy, + }, + }, + Volumes: ip.buildJobVolumes(cfg), + }, + }, + }, + } + + // Apply node selector from config. + if len(cfg.Kubernetes.NodeSelector) > 0 { + job.Spec.Template.Spec.NodeSelector = cfg.Kubernetes.NodeSelector + } + + // Apply service account. + if cfg.Kubernetes.ServiceAccount != "" { + job.Spec.Template.Spec.ServiceAccountName = cfg.Kubernetes.ServiceAccount + } + + // Apply image pull secrets. + for _, secret := range cfg.Kubernetes.ImagePullSecrets { + job.Spec.Template.Spec.ImagePullSecrets = append( + job.Spec.Template.Spec.ImagePullSecrets, + corev1.LocalObjectReference{Name: secret}, + ) + } + + // Apply tolerations. + for _, t := range cfg.Kubernetes.Tolerations { + toleration := corev1.Toleration{ + Key: t.Key, + Operator: corev1.TolerationOperator(t.Operator), + Value: t.Value, + Effect: corev1.TaintEffect(t.Effect), + } + if t.TolerationSeconds != nil { + toleration.TolerationSeconds = t.TolerationSeconds + } + job.Spec.Template.Spec.Tolerations = append(job.Spec.Template.Spec.Tolerations, toleration) + } + + // Create the Job. + if _, err := ip.client.BatchV1().Jobs(ip.namespace).Create(ctx, job, metav1.CreateOptions{}); err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to create installer job") + } + + // Stream logs in the background. + go ip.streamJobLogs(ctx) + + // Wait for the Job to complete. + if err := ip.waitForJob(ctx); err != nil { + return err + } + + return nil +} + +// streamJobLogs finds the Pod created by the Job and streams its logs to the +// install sink. +func (ip *InstallerProcess) streamJobLogs(ctx context.Context) { + // Wait a moment for the Pod to be created by the Job controller. + time.Sleep(2 * time.Second) + + for i := 0; i < 30; i++ { + pods, err := ip.client.CoreV1().Pods(ip.namespace).List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("job-name=%s", ip.jobName()), + }) + if err != nil || len(pods.Items) == 0 { + time.Sleep(2 * time.Second) + continue + } + + pod := pods.Items[0] + // Wait for the pod to be running or completed. + if pod.Status.Phase == corev1.PodPending { + time.Sleep(2 * time.Second) + continue + } + + follow := pod.Status.Phase == corev1.PodRunning + stream, err := ip.client.CoreV1().Pods(ip.namespace).GetLogs(pod.Name, &corev1.PodLogOptions{ + Container: "installer", + Follow: follow, + }).Stream(ctx) + if err != nil { + return + } + defer stream.Close() + + scanner := bufio.NewScanner(stream) + for scanner.Scan() { + line := scanner.Text() + if ip.Sink != nil { + ip.Sink.Push([]byte(line)) + } + } + return + } +} + +// waitForJob polls the Job until it succeeds, fails, or the context is +// canceled. +func (ip *InstallerProcess) waitForJob(ctx context.Context) error { + timeout := 30 * time.Minute + deadline := time.After(timeout) + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-deadline: + return errors.New("environment/kubernetes: installer job timed out after 30 minutes") + case <-ticker.C: + job, err := ip.client.BatchV1().Jobs(ip.namespace).Get(ctx, ip.jobName(), metav1.GetOptions{}) + if err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to get installer job status") + } + + // Check for completion. + if job.Status.Succeeded > 0 { + return nil + } + + // Check for failure. + if job.Status.Failed > 0 { + return errors.New("environment/kubernetes: installer job failed") + } + } + } +} + +// Cleanup removes the installer Job, ConfigMap, and associated resources. +func (ip *InstallerProcess) Cleanup(ctx context.Context) error { + err := ip.client.BatchV1().Jobs(ip.namespace).Delete(ctx, ip.jobName(), metav1.DeleteOptions{ + PropagationPolicy: propagationBackground(), + }) + if err != nil && !isNotFound(err) { + return errors.Wrap(err, "environment/kubernetes: failed to delete installer job") + } + + // Delete the install script ConfigMap. + cmErr := ip.client.CoreV1().ConfigMaps(ip.namespace).Delete(ctx, ip.configMapName(), metav1.DeleteOptions{}) + if cmErr != nil && !isNotFound(cmErr) { + return errors.Wrap(cmErr, "environment/kubernetes: failed to delete install script configmap") + } + + // Remove temporary install script directory (legacy cleanup). + if ip.TmpDir != "" { + os.RemoveAll(ip.TmpDir) + } + + return nil +} + +// WriteInstallScript creates a ConfigMap containing the installation script. +// The ConfigMap is mounted into the Job Pod at /mnt/install, eliminating the +// need for a shared filesystem between Wings and the Job Pod. +func (ip *InstallerProcess) WriteInstallScript() error { + content := strings.ReplaceAll(ip.Script.Script, "\r\n", "\n") + + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: ip.configMapName(), + Namespace: ip.namespace, + Labels: map[string]string{ + "app.kubernetes.io/managed-by": "pelican-wings", + "pelican.dev/server-id": ip.ServerID, + "pelican.dev/container-type": "server_installer", + }, + }, + Data: map[string]string{ + "install.sh": content, + }, + } + + ctx := context.Background() + + // Delete any existing ConfigMap from a previous run. + _ = ip.client.CoreV1().ConfigMaps(ip.namespace).Delete(ctx, ip.configMapName(), metav1.DeleteOptions{}) + + _, err := ip.client.CoreV1().ConfigMaps(ip.namespace).Create(ctx, cm, metav1.CreateOptions{}) + if err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to create install script configmap") + } + return nil +} + +// buildJobVolumes constructs the volume list for the installer Job. When +// DataPVC is set, the shared Wings data PVC is used. Otherwise falls back to +// per-server PVC (StorageMode "pvc") or HostPath. The install script is always +// mounted from a ConfigMap. +func (ip *InstallerProcess) buildJobVolumes(cfg *config.Configuration) []corev1.Volume { + var serverDataSource corev1.VolumeSource + if cfg.Kubernetes.DataPVC != "" { + serverDataSource = corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: cfg.Kubernetes.DataPVC, + }, + } + } else if cfg.Kubernetes.StorageMode == config.KubeStoragePVC { + serverDataSource = corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: fmt.Sprintf("gs-%s", ip.ServerID), + }, + } + } else { + serverDataSource = corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: ip.ServerPath, + }, + } + } + + // Script volume uses ConfigMap — works across multi-node clusters. + defaultMode := int32(0755) + + return []corev1.Volume{ + { + Name: "server-data", + VolumeSource: serverDataSource, + }, + { + Name: "install-script", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: ip.configMapName(), + }, + DefaultMode: &defaultMode, + }, + }, + }, + } +} + +// buildServerDataMount returns the VolumeMount for the server-data volume. +// When DataPVC is set, the mount includes a subPath to the server's data +// directory within the shared PVC. +func (ip *InstallerProcess) buildServerDataMount() corev1.VolumeMount { + vm := corev1.VolumeMount{ + Name: "server-data", + MountPath: "/mnt/server", + } + cfg := config.Get() + if cfg.Kubernetes.DataPVC != "" { + serverPath := filepath.Join(cfg.System.Data, ip.ServerID) + rel, err := filepath.Rel(cfg.System.RootDirectory, serverPath) + if err != nil { + vm.SubPath = filepath.Join("volumes", ip.ServerID) + } else { + vm.SubPath = rel + } + } + return vm +} + +// propagationBackground returns a pointer to the Background propagation policy. +func propagationBackground() *metav1.DeletionPropagation { + p := metav1.DeletePropagationBackground + return &p +} diff --git a/environment/kubernetes/installer_test.go b/environment/kubernetes/installer_test.go new file mode 100644 index 00000000..e4631ea3 --- /dev/null +++ b/environment/kubernetes/installer_test.go @@ -0,0 +1,366 @@ +package kubernetes + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + . "github.com/franela/goblin" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + + "github.com/pelican-dev/wings/config" + "github.com/pelican-dev/wings/environment" + "github.com/pelican-dev/wings/remote" + "github.com/pelican-dev/wings/system" +) + +func int32Ptr(i int32) *int32 { return &i } + +func TestInstaller(t *testing.T) { + g := Goblin(t) + + g.Describe("InstallerProcess", func() { + g.Describe("jobName", func() { + g.It("should return correct job name", func() { + ip := &InstallerProcess{ServerID: "abc-123-def"} + g.Assert(ip.jobName()).Equal("abc-123-def-installer") + }) + }) + + g.Describe("configMapName", func() { + g.It("should return correct configmap name", func() { + ip := &InstallerProcess{ServerID: "abc-123-def"} + g.Assert(ip.configMapName()).Equal("abc-123-def-install-script") + }) + }) + + g.Describe("WriteInstallScript", func() { + g.It("should create a ConfigMap with the script content", func() { + client := fake.NewSimpleClientset() + ip := &InstallerProcess{ + ServerID: "write-cm-uuid", + client: client, + namespace: "pelican", + Script: &remote.InstallationScript{ + Script: "#!/bin/bash\necho hello\r\necho world", + }, + } + + err := ip.WriteInstallScript() + g.Assert(err).IsNil() + + cm, err := client.CoreV1().ConfigMaps("pelican").Get(context.Background(), "write-cm-uuid-install-script", metav1.GetOptions{}) + g.Assert(err).IsNil() + // Should replace \r\n with \n. + g.Assert(cm.Data["install.sh"]).Equal("#!/bin/bash\necho hello\necho world") + g.Assert(cm.Labels["pelican.dev/server-id"]).Equal("write-cm-uuid") + }) + + g.It("should replace existing ConfigMap on re-run", func() { + existingCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rewrite-uuid-install-script", + Namespace: "pelican", + }, + Data: map[string]string{"install.sh": "old script"}, + } + client := fake.NewSimpleClientset(existingCM) + ip := &InstallerProcess{ + ServerID: "rewrite-uuid", + client: client, + namespace: "pelican", + Script: &remote.InstallationScript{ + Script: "new script", + }, + } + + err := ip.WriteInstallScript() + g.Assert(err).IsNil() + + cm, err := client.CoreV1().ConfigMaps("pelican").Get(context.Background(), "rewrite-uuid-install-script", metav1.GetOptions{}) + g.Assert(err).IsNil() + g.Assert(cm.Data["install.sh"]).Equal("new script") + }) + }) + + g.Describe("Run", func() { + g.It("should create a Job with correct spec", func() { + client := fake.NewSimpleClientset() + ip := &InstallerProcess{ + ServerID: "test-server-uuid", + client: client, + namespace: "pelican", + ServerPath: "/var/lib/pelican/servers/test-server-uuid", + TmpDir: "/tmp/test-server-uuid", + EnvVars: []string{"SERVER_MEMORY=1024", "SERVER_IP=0.0.0.0"}, + Script: &remote.InstallationScript{ + ContainerImage: "ghcr.io/pelican/installers:alpine", + Entrypoint: "bash", + Script: "echo installing", + }, + Sink: system.NewSinkPool(), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.NodeSelector = map[string]string{"role": "game"} + c.Kubernetes.ServiceAccount = "pelican-wings" + }) + + // Run will create the Job but waitForJob will keep polling. + // We'll cancel the context to stop it. + ctx, cancel := context.WithCancel(context.Background()) + + // Run in goroutine since it will block waiting for Job completion. + errCh := make(chan error, 1) + go func() { + errCh <- ip.Run(ctx) + }() + + // Poll until Job exists or timeout. + var job *batchv1.Job + for i := 0; i < 50; i++ { + var err error + job, err = client.BatchV1().Jobs("pelican").Get(context.Background(), "test-server-uuid-installer", metav1.GetOptions{}) + if err == nil { + break + } + time.Sleep(10 * time.Millisecond) + } + g.Assert(job).IsNotNil() + + // Verify Job metadata. + g.Assert(job.Name).Equal("test-server-uuid-installer") + g.Assert(job.Namespace).Equal("pelican") + g.Assert(job.Labels["pelican.dev/server-id"]).Equal("test-server-uuid") + g.Assert(job.Labels["pelican.dev/container-type"]).Equal("server_installer") + + // Verify Pod template spec. + podSpec := job.Spec.Template.Spec + g.Assert(podSpec.RestartPolicy).Equal(corev1.RestartPolicyNever) + g.Assert(len(podSpec.Containers)).Equal(1) + + container := podSpec.Containers[0] + g.Assert(container.Name).Equal("installer") + g.Assert(container.Image).Equal("ghcr.io/pelican/installers:alpine") + g.Assert(container.Command).Equal([]string{"bash", "/mnt/install/install.sh"}) + + // Verify env vars. + g.Assert(len(container.Env)).Equal(2) + g.Assert(container.Env[0].Name).Equal("SERVER_MEMORY") + g.Assert(container.Env[0].Value).Equal("1024") + g.Assert(container.Env[1].Name).Equal("SERVER_IP") + g.Assert(container.Env[1].Value).Equal("0.0.0.0") + + // Verify volume mounts. + g.Assert(len(container.VolumeMounts)).Equal(2) + g.Assert(container.VolumeMounts[0].MountPath).Equal("/mnt/server") + g.Assert(container.VolumeMounts[1].MountPath).Equal("/mnt/install") + + // Verify volumes. + g.Assert(len(podSpec.Volumes)).Equal(2) + g.Assert(podSpec.Volumes[0].HostPath.Path).Equal("/var/lib/pelican/servers/test-server-uuid") + // Install script now uses ConfigMap. + g.Assert(podSpec.Volumes[1].ConfigMap).IsNotNil() + g.Assert(podSpec.Volumes[1].ConfigMap.Name).Equal("test-server-uuid-install-script") + g.Assert(*podSpec.Volumes[1].ConfigMap.DefaultMode).Equal(int32(0755)) + + // Verify node selector. + g.Assert(podSpec.NodeSelector["role"]).Equal("game") + + // Verify service account. + g.Assert(podSpec.ServiceAccountName).Equal("pelican-wings") + + // Verify backoff limit. + g.Assert(*job.Spec.BackoffLimit).Equal(int32(0)) + + // Cancel to stop waitForJob. + cancel() + <-errCh + }) + + g.It("should succeed when Job completes successfully", func() { + // Pre-create a Job that is already succeeded. + completedJob := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: "completed-uuid-installer", + Namespace: "pelican", + }, + Status: batchv1.JobStatus{ + Succeeded: 1, + }, + } + client := fake.NewSimpleClientset(completedJob) + ip := &InstallerProcess{ + ServerID: "completed-uuid", + client: client, + namespace: "pelican", + EnvVars: []string{}, + Script: &remote.InstallationScript{ + ContainerImage: "alpine", + Entrypoint: "sh", + Script: "echo done", + }, + Sink: system.NewSinkPool(), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + }) + + // The fake client will return the existing Job (already succeeded) + // when Run queries for it after creating a new one. + // However the fake client won't auto-set status. Let's test Cleanup instead. + err := ip.Cleanup(context.Background()) + g.Assert(err).IsNil() + + // Verify the job was deleted. + _, err = client.BatchV1().Jobs("pelican").Get(context.Background(), "completed-uuid-installer", metav1.GetOptions{}) + g.Assert(err).IsNotNil() + }) + }) + + g.Describe("Cleanup", func() { + g.It("should delete the Job and ConfigMap", func() { + existingJob := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cleanup-uuid-installer", + Namespace: "pelican", + }, + } + existingCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cleanup-uuid-install-script", + Namespace: "pelican", + }, + } + client := fake.NewSimpleClientset(existingJob, existingCM) + ip := &InstallerProcess{ + ServerID: "cleanup-uuid", + client: client, + namespace: "pelican", + } + + err := ip.Cleanup(context.Background()) + g.Assert(err).IsNil() + + _, err = client.BatchV1().Jobs("pelican").Get(context.Background(), "cleanup-uuid-installer", metav1.GetOptions{}) + g.Assert(err).IsNotNil() + + _, err = client.CoreV1().ConfigMaps("pelican").Get(context.Background(), "cleanup-uuid-install-script", metav1.GetOptions{}) + g.Assert(err).IsNotNil() + }) + + g.It("should not error when Job and ConfigMap do not exist", func() { + client := fake.NewSimpleClientset() + ip := &InstallerProcess{ + ServerID: "nonexistent-uuid", + client: client, + namespace: "pelican", + } + + err := ip.Cleanup(context.Background()) + g.Assert(err).IsNil() + }) + + g.It("should remove temp directory", func() { + tmpDir := filepath.Join(os.TempDir(), "test-installer-cleanup") + os.MkdirAll(tmpDir, 0o700) + os.WriteFile(filepath.Join(tmpDir, "install.sh"), []byte("echo hi"), 0o644) + + client := fake.NewSimpleClientset() + ip := &InstallerProcess{ + ServerID: "tmpdir-uuid", + client: client, + namespace: "pelican", + TmpDir: tmpDir, + } + + err := ip.Cleanup(context.Background()) + g.Assert(err).IsNil() + + _, err = os.Stat(tmpDir) + g.Assert(os.IsNotExist(err)).IsTrue() + }) + }) + + g.Describe("buildJobVolumes", func() { + g.It("should use HostPath for server-data in hostpath mode and ConfigMap for install script", func() { + ip := &InstallerProcess{ + ServerID: "vol-hp-uuid", + ServerPath: "/data/servers/vol-hp-uuid", + } + config.Update(func(c *config.Configuration) { + c.Kubernetes.StorageMode = config.KubeStorageHostPath + }) + cfg := config.Get() + volumes := ip.buildJobVolumes(cfg) + g.Assert(len(volumes)).Equal(2) + g.Assert(volumes[0].Name).Equal("server-data") + g.Assert(volumes[0].HostPath).IsNotNil() + g.Assert(volumes[0].HostPath.Path).Equal("/data/servers/vol-hp-uuid") + g.Assert(volumes[0].PersistentVolumeClaim == nil).IsTrue() + // Install script uses ConfigMap. + g.Assert(volumes[1].Name).Equal("install-script") + g.Assert(volumes[1].ConfigMap).IsNotNil() + g.Assert(volumes[1].ConfigMap.Name).Equal("vol-hp-uuid-install-script") + g.Assert(*volumes[1].ConfigMap.DefaultMode).Equal(int32(0755)) + }) + + g.It("should use PVC for server-data in pvc mode and ConfigMap for install script", func() { + ip := &InstallerProcess{ + ServerID: "vol-pvc-uuid", + ServerPath: "/data/servers/vol-pvc-uuid", + } + config.Update(func(c *config.Configuration) { + c.Kubernetes.StorageMode = config.KubeStoragePVC + }) + cfg := config.Get() + volumes := ip.buildJobVolumes(cfg) + g.Assert(len(volumes)).Equal(2) + g.Assert(volumes[0].Name).Equal("server-data") + g.Assert(volumes[0].PersistentVolumeClaim).IsNotNil() + g.Assert(volumes[0].PersistentVolumeClaim.ClaimName).Equal("gs-vol-pvc-uuid") + // Install script uses ConfigMap. + g.Assert(volumes[1].ConfigMap).IsNotNil() + g.Assert(volumes[1].ConfigMap.Name).Equal("vol-pvc-uuid-install-script") + }) + }) + + g.Describe("NewInstallerProcess", func() { + g.It("should initialize all fields from Environment", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "init-test-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + }) + + script := &remote.InstallationScript{ + ContainerImage: "alpine:latest", + Entrypoint: "sh", + Script: "echo test", + } + sink := system.NewSinkPool() + + ip := NewInstallerProcess(env, script, []string{"FOO=bar"}, "/data/servers/test", "/tmp/install", sink) + + g.Assert(ip.ServerID).Equal("init-test-uuid") + g.Assert(ip.namespace).Equal("pelican") + g.Assert(ip.Script.ContainerImage).Equal("alpine:latest") + g.Assert(ip.EnvVars[0]).Equal("FOO=bar") + g.Assert(ip.ServerPath).Equal("/data/servers/test") + g.Assert(ip.TmpDir).Equal("/tmp/install") + }) + }) + }) +} diff --git a/environment/kubernetes/network.go b/environment/kubernetes/network.go new file mode 100644 index 00000000..6068906e --- /dev/null +++ b/environment/kubernetes/network.go @@ -0,0 +1,367 @@ +package kubernetes + +import ( + "context" + "fmt" + "strings" + "time" + + "emperror.dev/errors" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + + "github.com/pelican-dev/wings/config" +) + +// serviceName returns the Kubernetes Service name for this server. +func (e *Environment) serviceName() string { + return fmt.Sprintf("gs-%s", e.Id) +} + +// EnsureService creates or updates the Kubernetes Service that exposes game +// server ports. In NodePort mode, ports are exposed via auto-assigned NodePorts. +// In LoadBalancer mode, each server gets a dedicated external IP with 1:1 port +// mapping. In HostPort mode this is a no-op. +func (e *Environment) EnsureService(ctx context.Context) error { + cfg := config.Get() + if cfg.Kubernetes.NetworkMode == config.KubeNetworkHostPort { + return nil + } + + allocs := e.Configuration.Allocations() + if len(allocs.Mappings) == 0 { + return nil + } + + svcName := e.serviceName() + ns := e.namespace() + + isLB := cfg.Kubernetes.NetworkMode == config.KubeNetworkLoadBalancer + + // Build the Service port list from allocations. A port may appear under + // multiple allocation IPs, but the Service only needs it once. + seenPorts := make(map[int]bool) + var servicePorts []corev1.ServicePort + for _, ports := range allocs.Mappings { + for _, port := range ports { + if port < 1 || port > 65535 || seenPorts[port] { + continue + } + seenPorts[port] = true + + tcp := corev1.ServicePort{ + Name: portName("tcp", port), + Protocol: corev1.ProtocolTCP, + Port: int32(port), + TargetPort: intstr.FromInt32(int32(port)), + } + udp := corev1.ServicePort{ + Name: portName("udp", port), + Protocol: corev1.ProtocolUDP, + Port: int32(port), + TargetPort: intstr.FromInt32(int32(port)), + } + + if !isLB { + tcp.NodePort = e.resolveNodePort(cfg, port) + udp.NodePort = e.resolveNodePort(cfg, port) + } + + servicePorts = append(servicePorts, tcp, udp) + } + } + + if len(servicePorts) == 0 { + return nil + } + + // Labels to match the Pod. + selector := map[string]string{ + "pelican.dev/server-id": e.Id, + } + + labels := map[string]string{ + "app.kubernetes.io/managed-by": "pelican-wings", + "pelican.dev/server-id": e.Id, + "pelican.dev/resource-type": "service", + } + + svcType := corev1.ServiceTypeNodePort + var annotations map[string]string + if isLB { + svcType = corev1.ServiceTypeLoadBalancer + annotations = make(map[string]string) + for k, v := range cfg.Kubernetes.LBAnnotations { + annotations[k] = v + } + + // Auto-set IP-pinning and sharing-key annotations from the + // server's allocation IP so the LB is bound to the IP the user + // selected in the Panel. + if allocIP := e.allocationIP(); allocIP != "" { + if cfg.Kubernetes.LBIPAnnotation != "" { + annotations[cfg.Kubernetes.LBIPAnnotation] = allocIP + } + if cfg.Kubernetes.LBSharingKey != "" { + annotations[cfg.Kubernetes.LBSharingKey] = allocIP + } + } + } + + desired := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: svcName, + Namespace: ns, + Labels: labels, + Annotations: annotations, + }, + Spec: corev1.ServiceSpec{ + Type: svcType, + Selector: selector, + Ports: servicePorts, + }, + } + + // Check if the Service already exists. + existing, err := e.client.CoreV1().Services(ns).Get(ctx, svcName, metav1.GetOptions{}) + if err != nil { + if !isNotFound(err) { + return errors.Wrap(err, "environment/kubernetes: failed to get service") + } + // Service does not exist; create it. + e.log().WithField("service", svcName).Infof("creating %s service for server", svcType) + _, err = e.client.CoreV1().Services(ns).Create(ctx, desired, metav1.CreateOptions{}) + if err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to create service") + } + return nil + } + + // Service exists; update it with the desired spec while preserving + // existing NodePort assignments where possible. + existing.Spec.Selector = desired.Spec.Selector + existing.Spec.Ports = mergeServicePorts(existing.Spec.Ports, desired.Spec.Ports) + existing.Labels = labels + + e.log().WithField("service", svcName).Infof("updating %s service for server", svcType) + if isLB { + existing.Annotations = annotations + } + _, err = e.client.CoreV1().Services(ns).Update(ctx, existing, metav1.UpdateOptions{}) + if err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to update service") + } + + return nil +} + +// DeleteService removes the Kubernetes Service associated with this server. +func (e *Environment) DeleteService(ctx context.Context) error { + cfg := config.Get() + if cfg.Kubernetes.NetworkMode == config.KubeNetworkHostPort { + return nil + } + + svcName := e.serviceName() + ns := e.namespace() + + err := e.client.CoreV1().Services(ns).Delete(ctx, svcName, metav1.DeleteOptions{}) + if err != nil && !isNotFound(err) { + return errors.Wrap(err, "environment/kubernetes: failed to delete service") + } + + e.log().WithField("service", svcName).Debug("deleted service for server") + return nil +} + +// GetServiceNodePorts returns a map of container port → assigned NodePort for +// this server's Service. Returns nil if no Service exists or we're in HostPort +// mode. +func (e *Environment) GetServiceNodePorts(ctx context.Context) (map[int32]int32, error) { + cfg := config.Get() + if cfg.Kubernetes.NetworkMode == config.KubeNetworkHostPort { + return nil, nil + } + + svc, err := e.client.CoreV1().Services(e.namespace()).Get(ctx, e.serviceName(), metav1.GetOptions{}) + if err != nil { + if isNotFound(err) { + return nil, nil + } + return nil, errors.Wrap(err, "environment/kubernetes: failed to get service") + } + + result := make(map[int32]int32) + for _, p := range svc.Spec.Ports { + if p.NodePort > 0 { + result[p.Port] = p.NodePort + } + } + return result, nil +} + +// GetServiceExternalIP returns the external IP assigned to this server's +// LoadBalancer Service. Returns an empty string if no IP is assigned yet, +// or if the network mode is not LoadBalancer. +func (e *Environment) GetServiceExternalIP(ctx context.Context) (string, error) { + cfg := config.Get() + if cfg.Kubernetes.NetworkMode != config.KubeNetworkLoadBalancer { + return "", nil + } + + svc, err := e.client.CoreV1().Services(e.namespace()).Get(ctx, e.serviceName(), metav1.GetOptions{}) + if err != nil { + if isNotFound(err) { + return "", nil + } + return "", errors.Wrap(err, "environment/kubernetes: failed to get service") + } + + for _, ingress := range svc.Status.LoadBalancer.Ingress { + if ingress.IP != "" { + return ingress.IP, nil + } + if ingress.Hostname != "" { + return ingress.Hostname, nil + } + } + return "", nil +} + +// WaitForLoadBalancerIP polls the Service until an external IP is assigned by +// the load balancer provisioner, or until the timeout (2 minutes) is reached. +func (e *Environment) WaitForLoadBalancerIP(ctx context.Context) (string, error) { + cfg := config.Get() + if cfg.Kubernetes.NetworkMode != config.KubeNetworkLoadBalancer { + return "", nil + } + + timeout := 2 * time.Minute + interval := 3 * time.Second + deadline := time.Now().Add(timeout) + + for time.Now().Before(deadline) { + ip, err := e.GetServiceExternalIP(ctx) + if err != nil { + return "", err + } + if ip != "" { + e.log().WithField("external_ip", ip).Info("load balancer IP assigned") + return ip, nil + } + + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-time.After(interval): + } + } + + e.log().Warn("timed out waiting for load balancer IP assignment") + return "", nil +} + +// resolveNodePort determines the NodePort to request for a given game server +// port. If PreserveNodePorts is enabled and the port falls within the +// Kubernetes NodePort range (default 30000-32767), use it directly. Otherwise, +// return 0 to let Kubernetes assign automatically. +func (e *Environment) resolveNodePort(cfg *config.Configuration, port int) int32 { + if !cfg.Kubernetes.NodePortPreserve { + return 0 + } + // Only request a specific NodePort if the port is within the valid range. + min := int(cfg.Kubernetes.NodePortRangeMin) + max := int(cfg.Kubernetes.NodePortRangeMax) + if min == 0 { + min = 30000 + } + if max == 0 { + max = 32767 + } + if port >= min && port <= max { + return int32(port) + } + return 0 +} + +// mergeServicePorts merges desired ports into existing ports, preserving +// assigned NodePorts for ports that haven't changed. +func mergeServicePorts(existing, desired []corev1.ServicePort) []corev1.ServicePort { + existingByKey := make(map[string]corev1.ServicePort) + for _, p := range existing { + key := fmt.Sprintf("%s/%s", p.Name, p.Protocol) + existingByKey[key] = p + } + + var merged []corev1.ServicePort + for _, d := range desired { + key := fmt.Sprintf("%s/%s", d.Name, d.Protocol) + if ex, ok := existingByKey[key]; ok { + // Preserve the existing NodePort if the target hasn't changed and + // we didn't request a specific one. + if d.NodePort == 0 && ex.NodePort > 0 && ex.TargetPort.IntValue() == d.TargetPort.IntValue() { + d.NodePort = ex.NodePort + } + } + merged = append(merged, d) + } + + return merged +} + +// sanitizePortName ensures a Kubernetes Service port name is valid (lowercase, +// alphanumeric, hyphens, max 15 chars, must start/end with alphanumeric). +func sanitizePortName(name string) string { + name = strings.ToLower(name) + name = strings.ReplaceAll(name, ".", "-") + name = strings.ReplaceAll(name, ":", "-") + + // Remove invalid characters. + var cleaned []byte + for i, c := range []byte(name) { + if (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || (c == '-' && i > 0) { + cleaned = append(cleaned, c) + } + } + name = string(cleaned) + + // Trim trailing hyphens. + name = strings.TrimRight(name, "-") + + // Max 15 characters for port names. + if len(name) > 15 { + name = name[:15] + } + + // Must not end with hyphen after truncation. + name = strings.TrimRight(name, "-") + + if name == "" { + name = "port" + } + + return name +} + +// portName builds a unique, K8s-valid Service port name. The format is +// "{proto}-{port}" (e.g. "tcp-27015"), which fits within the 15-char limit +// and avoids collisions that occur when long IP strings are truncated. +func portName(proto string, port int) string { + return sanitizePortName(fmt.Sprintf("%s-%d", proto, port)) +} + +// allocationIP returns the server's default allocation IP if it is a usable +// public/external address. Returns empty for 0.0.0.0, 127.0.0.1, or when +// no default allocation is configured. +func (e *Environment) allocationIP() string { + allocs := e.Configuration.Allocations() + if allocs.DefaultMapping == nil { + return "" + } + ip := allocs.DefaultMapping.Ip + if ip == "" || ip == "0.0.0.0" || ip == "127.0.0.1" { + return "" + } + return ip +} diff --git a/environment/kubernetes/network_test.go b/environment/kubernetes/network_test.go new file mode 100644 index 00000000..216ff58b --- /dev/null +++ b/environment/kubernetes/network_test.go @@ -0,0 +1,476 @@ +package kubernetes + +import ( + "context" + "testing" + + . "github.com/franela/goblin" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + + "github.com/pelican-dev/wings/config" + "github.com/pelican-dev/wings/environment" + "github.com/pelican-dev/wings/events" + "github.com/pelican-dev/wings/system" +) + +func newTestEnv(client *fake.Clientset, allocs environment.Allocations) *Environment { + settings := environment.Settings{ + Allocations: allocs, + Limits: environment.Limits{ + MemoryLimit: 512, + CpuLimit: 100, + }, + } + cfg := environment.NewConfiguration(settings, []string{"FOO=bar"}) + + return &Environment{ + Id: "test-server-uuid", + Configuration: cfg, + meta: &Metadata{Image: "nginx:latest"}, + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + emitter: events.NewBus(), + } +} + +func TestNetwork(t *testing.T) { + g := Goblin(t) + + g.Describe("EnsureService", func() { + g.It("should be a no-op in hostport mode", func() { + client := fake.NewSimpleClientset() + allocs := environment.Allocations{ + Mappings: map[string][]int{"0.0.0.0": {25565}}, + } + env := newTestEnv(client, allocs) + + // Set hostport mode. + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkHostPort + c.Kubernetes.Namespace = "pelican" + }) + + err := env.EnsureService(context.Background()) + g.Assert(err).IsNil() + + // No Service should be created. + svcs, _ := client.CoreV1().Services("pelican").List(context.Background(), metav1.ListOptions{}) + g.Assert(len(svcs.Items)).Equal(0) + }) + + g.It("should create a NodePort service in nodeport mode", func() { + client := fake.NewSimpleClientset() + allocs := environment.Allocations{ + Mappings: map[string][]int{"0.0.0.0": {25565, 25575}}, + } + env := newTestEnv(client, allocs) + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkNodePort + c.Kubernetes.Namespace = "pelican" + }) + + err := env.EnsureService(context.Background()) + g.Assert(err).IsNil() + + svc, err := client.CoreV1().Services("pelican").Get(context.Background(), "gs-test-server-uuid", metav1.GetOptions{}) + g.Assert(err).IsNil() + g.Assert(svc.Spec.Type).Equal(corev1.ServiceTypeNodePort) + // Should have TCP+UDP for each port = 4 ports total. + g.Assert(len(svc.Spec.Ports)).Equal(4) + + // Verify selector targets our Pod. + g.Assert(svc.Spec.Selector["pelican.dev/server-id"]).Equal("test-server-uuid") + + // Verify labels. + g.Assert(svc.Labels["pelican.dev/server-id"]).Equal("test-server-uuid") + g.Assert(svc.Labels["pelican.dev/resource-type"]).Equal("service") + }) + + g.It("should update an existing service", func() { + // Pre-create a service with one port. + existingSvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gs-test-server-uuid", + Namespace: "pelican", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeNodePort, + Ports: []corev1.ServicePort{ + { + Name: "tcp-0-0-0-0-255", + Protocol: corev1.ProtocolTCP, + Port: 25565, + NodePort: 31000, + }, + }, + Selector: map[string]string{"pelican.dev/server-id": "test-server-uuid"}, + }, + } + client := fake.NewSimpleClientset(existingSvc) + allocs := environment.Allocations{ + Mappings: map[string][]int{"0.0.0.0": {25565, 25575}}, + } + env := newTestEnv(client, allocs) + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkNodePort + c.Kubernetes.Namespace = "pelican" + }) + + err := env.EnsureService(context.Background()) + g.Assert(err).IsNil() + + svc, err := client.CoreV1().Services("pelican").Get(context.Background(), "gs-test-server-uuid", metav1.GetOptions{}) + g.Assert(err).IsNil() + // Should now have 4 ports (2 for each allocation, TCP+UDP). + g.Assert(len(svc.Spec.Ports)).Equal(4) + }) + + g.It("should be a no-op with empty allocations", func() { + client := fake.NewSimpleClientset() + allocs := environment.Allocations{ + Mappings: map[string][]int{}, + } + env := newTestEnv(client, allocs) + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkNodePort + c.Kubernetes.Namespace = "pelican" + }) + + err := env.EnsureService(context.Background()) + g.Assert(err).IsNil() + + svcs, _ := client.CoreV1().Services("pelican").List(context.Background(), metav1.ListOptions{}) + g.Assert(len(svcs.Items)).Equal(0) + }) + + g.It("should auto-set LB IP and sharing-key annotations from allocation IP", func() { + client := fake.NewSimpleClientset() + allocs := environment.Allocations{ + DefaultMapping: &environment.DefaultAllocationMapping{ + Ip: "23.227.184.222", + Port: 27015, + }, + Mappings: map[string][]int{"23.227.184.222": {27015}}, + } + env := newTestEnv(client, allocs) + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkLoadBalancer + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.LBAnnotations = map[string]string{ + "io.cilium/lb-ipam-pool": "game-servers", + } + c.Kubernetes.LBIPAnnotation = "lbipam.cilium.io/ips" + c.Kubernetes.LBSharingKey = "lbipam.cilium.io/sharing-key" + }) + + err := env.EnsureService(context.Background()) + g.Assert(err).IsNil() + + svc, err := client.CoreV1().Services("pelican").Get(context.Background(), "gs-test-server-uuid", metav1.GetOptions{}) + g.Assert(err).IsNil() + g.Assert(svc.Spec.Type).Equal(corev1.ServiceTypeLoadBalancer) + g.Assert(svc.Annotations["io.cilium/lb-ipam-pool"]).Equal("game-servers") + g.Assert(svc.Annotations["lbipam.cilium.io/ips"]).Equal("23.227.184.222") + g.Assert(svc.Annotations["lbipam.cilium.io/sharing-key"]).Equal("23.227.184.222") + }) + + g.It("should not set IP annotations when allocation IP is 0.0.0.0", func() { + client := fake.NewSimpleClientset() + allocs := environment.Allocations{ + DefaultMapping: &environment.DefaultAllocationMapping{ + Ip: "0.0.0.0", + Port: 27015, + }, + Mappings: map[string][]int{"0.0.0.0": {27015}}, + } + env := newTestEnv(client, allocs) + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkLoadBalancer + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.LBIPAnnotation = "lbipam.cilium.io/ips" + c.Kubernetes.LBSharingKey = "lbipam.cilium.io/sharing-key" + }) + + err := env.EnsureService(context.Background()) + g.Assert(err).IsNil() + + svc, err := client.CoreV1().Services("pelican").Get(context.Background(), "gs-test-server-uuid", metav1.GetOptions{}) + g.Assert(err).IsNil() + _, hasIP := svc.Annotations["lbipam.cilium.io/ips"] + g.Assert(hasIP).IsFalse() + _, hasKey := svc.Annotations["lbipam.cilium.io/sharing-key"] + g.Assert(hasKey).IsFalse() + }) + + g.It("should not set IP annotations when config keys are empty", func() { + client := fake.NewSimpleClientset() + allocs := environment.Allocations{ + DefaultMapping: &environment.DefaultAllocationMapping{ + Ip: "23.227.184.222", + Port: 27015, + }, + Mappings: map[string][]int{"23.227.184.222": {27015}}, + } + env := newTestEnv(client, allocs) + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkLoadBalancer + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.LBIPAnnotation = "" + c.Kubernetes.LBSharingKey = "" + }) + + err := env.EnsureService(context.Background()) + g.Assert(err).IsNil() + + svc, err := client.CoreV1().Services("pelican").Get(context.Background(), "gs-test-server-uuid", metav1.GetOptions{}) + g.Assert(err).IsNil() + _, hasIP := svc.Annotations["lbipam.cilium.io/ips"] + g.Assert(hasIP).IsFalse() + }) + + g.It("should remove stale IP annotations when allocation IP becomes invalid", func() { + existingSvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gs-test-server-uuid", + Namespace: "pelican", + Annotations: map[string]string{ + "io.cilium/lb-ipam-pool": "game-servers", + "lbipam.cilium.io/ips": "23.227.184.222", + "lbipam.cilium.io/sharing-key": "23.227.184.222", + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + Ports: []corev1.ServicePort{ + {Name: "tcp-27015", Protocol: corev1.ProtocolTCP, Port: 27015}, + {Name: "udp-27015", Protocol: corev1.ProtocolUDP, Port: 27015}, + }, + Selector: map[string]string{"pelican.dev/server-id": "test-server-uuid"}, + }, + } + client := fake.NewSimpleClientset(existingSvc) + allocs := environment.Allocations{ + DefaultMapping: &environment.DefaultAllocationMapping{ + Ip: "0.0.0.0", + Port: 27015, + }, + Mappings: map[string][]int{"0.0.0.0": {27015}}, + } + env := newTestEnv(client, allocs) + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkLoadBalancer + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.LBAnnotations = map[string]string{ + "io.cilium/lb-ipam-pool": "game-servers", + } + c.Kubernetes.LBIPAnnotation = "lbipam.cilium.io/ips" + c.Kubernetes.LBSharingKey = "lbipam.cilium.io/sharing-key" + }) + + err := env.EnsureService(context.Background()) + g.Assert(err).IsNil() + + svc, err := client.CoreV1().Services("pelican").Get(context.Background(), "gs-test-server-uuid", metav1.GetOptions{}) + g.Assert(err).IsNil() + // Pool annotation should remain. + g.Assert(svc.Annotations["io.cilium/lb-ipam-pool"]).Equal("game-servers") + // IP-pinning annotations should be removed. + _, hasIP := svc.Annotations["lbipam.cilium.io/ips"] + g.Assert(hasIP).IsFalse() + _, hasKey := svc.Annotations["lbipam.cilium.io/sharing-key"] + g.Assert(hasKey).IsFalse() + }) + }) + + g.Describe("DeleteService", func() { + g.It("should delete an existing service", func() { + existingSvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gs-test-server-uuid", + Namespace: "pelican", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeNodePort, + }, + } + client := fake.NewSimpleClientset(existingSvc) + allocs := environment.Allocations{} + env := newTestEnv(client, allocs) + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkNodePort + c.Kubernetes.Namespace = "pelican" + }) + + err := env.DeleteService(context.Background()) + g.Assert(err).IsNil() + + svcs, _ := client.CoreV1().Services("pelican").List(context.Background(), metav1.ListOptions{}) + g.Assert(len(svcs.Items)).Equal(0) + }) + + g.It("should not error when service does not exist", func() { + client := fake.NewSimpleClientset() + allocs := environment.Allocations{} + env := newTestEnv(client, allocs) + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkNodePort + c.Kubernetes.Namespace = "pelican" + }) + + err := env.DeleteService(context.Background()) + g.Assert(err).IsNil() + }) + + g.It("should be a no-op in hostport mode", func() { + existingSvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gs-test-server-uuid", + Namespace: "pelican", + }, + } + client := fake.NewSimpleClientset(existingSvc) + allocs := environment.Allocations{} + env := newTestEnv(client, allocs) + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkHostPort + c.Kubernetes.Namespace = "pelican" + }) + + err := env.DeleteService(context.Background()) + g.Assert(err).IsNil() + + // Service should still exist since delete is no-op in hostport mode. + svcs, _ := client.CoreV1().Services("pelican").List(context.Background(), metav1.ListOptions{}) + g.Assert(len(svcs.Items)).Equal(1) + }) + }) + + g.Describe("GetServiceNodePorts", func() { + g.It("should return assigned NodePorts", func() { + existingSvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gs-test-server-uuid", + Namespace: "pelican", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeNodePort, + Ports: []corev1.ServicePort{ + {Name: "tcp-25565", Port: 25565, NodePort: 31234, Protocol: corev1.ProtocolTCP}, + {Name: "udp-25575", Port: 25575, NodePort: 31235, Protocol: corev1.ProtocolUDP}, + }, + }, + } + client := fake.NewSimpleClientset(existingSvc) + allocs := environment.Allocations{} + env := newTestEnv(client, allocs) + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkNodePort + c.Kubernetes.Namespace = "pelican" + }) + + ports, err := env.GetServiceNodePorts(context.Background()) + g.Assert(err).IsNil() + g.Assert(ports[25565]).Equal(int32(31234)) + g.Assert(ports[25575]).Equal(int32(31235)) + }) + + g.It("should return nil in hostport mode", func() { + client := fake.NewSimpleClientset() + allocs := environment.Allocations{} + env := newTestEnv(client, allocs) + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkHostPort + }) + + ports, err := env.GetServiceNodePorts(context.Background()) + g.Assert(err).IsNil() + g.Assert(ports == nil).IsTrue() + }) + }) + + g.Describe("sanitizePortName", func() { + g.It("should lowercase and replace dots with hyphens", func() { + g.Assert(sanitizePortName("TCP-192.168.1.1-25565")).Equal("tcp-192-168-1-1") + }) + + g.It("should truncate to 15 characters", func() { + result := sanitizePortName("very-long-port-name-that-exceeds") + g.Assert(len(result) <= 15).IsTrue() + }) + + g.It("should not start or end with a hyphen", func() { + result := sanitizePortName("-invalid-name-") + g.Assert(result[0] != '-').IsTrue() + g.Assert(result[len(result)-1] != '-').IsTrue() + }) + + g.It("should return 'port' for empty input", func() { + g.Assert(sanitizePortName("")).Equal("port") + }) + }) + + g.Describe("portName", func() { + g.It("should produce unique names for different ports", func() { + g.Assert(portName("tcp", 27015)).Equal("tcp-27015") + g.Assert(portName("tcp", 27016)).Equal("tcp-27016") + g.Assert(portName("udp", 27015)).Equal("udp-27015") + }) + + g.It("should fit within 15-char limit for 5-digit ports", func() { + name := portName("tcp", 65535) + g.Assert(len(name) <= 15).IsTrue() + g.Assert(name).Equal("tcp-65535") + }) + }) + + g.Describe("mergeServicePorts", func() { + g.It("should preserve existing NodePorts for unchanged ports", func() { + existing := []corev1.ServicePort{ + {Name: "tcp-25565", Protocol: corev1.ProtocolTCP, Port: 25565, NodePort: 31000}, + } + desired := []corev1.ServicePort{ + {Name: "tcp-25565", Protocol: corev1.ProtocolTCP, Port: 25565, NodePort: 0}, + } + merged := mergeServicePorts(existing, desired) + g.Assert(len(merged)).Equal(1) + g.Assert(merged[0].NodePort).Equal(int32(31000)) + }) + + g.It("should use desired NodePort when explicitly set", func() { + existing := []corev1.ServicePort{ + {Name: "tcp-25565", Protocol: corev1.ProtocolTCP, Port: 25565, NodePort: 31000}, + } + desired := []corev1.ServicePort{ + {Name: "tcp-25565", Protocol: corev1.ProtocolTCP, Port: 25565, NodePort: 31500}, + } + merged := mergeServicePorts(existing, desired) + g.Assert(merged[0].NodePort).Equal(int32(31500)) + }) + + g.It("should add new ports", func() { + existing := []corev1.ServicePort{ + {Name: "tcp-25565", Protocol: corev1.ProtocolTCP, Port: 25565, NodePort: 31000}, + } + desired := []corev1.ServicePort{ + {Name: "tcp-25565", Protocol: corev1.ProtocolTCP, Port: 25565, NodePort: 0}, + {Name: "tcp-25575", Protocol: corev1.ProtocolTCP, Port: 25575, NodePort: 0}, + } + merged := mergeServicePorts(existing, desired) + g.Assert(len(merged)).Equal(2) + }) + }) +} diff --git a/environment/kubernetes/node.go b/environment/kubernetes/node.go new file mode 100644 index 00000000..534257ba --- /dev/null +++ b/environment/kubernetes/node.go @@ -0,0 +1,77 @@ +package kubernetes + +import ( + "context" + "os" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/pelican-dev/wings/config" +) + +// GetNodeIPs returns the IP addresses of the Kubernetes node where Wings is +// running. It queries the node's status.addresses for ExternalIP and +// InternalIP entries. The node name is read from the config or the NODE_NAME +// environment variable (set via the Kubernetes downward API). +func GetNodeIPs(ctx context.Context) ([]string, error) { + nodeName := config.Get().Kubernetes.NodeName + if nodeName == "" { + nodeName = os.Getenv("NODE_NAME") + } + if nodeName == "" { + return nil, nil + } + + c, err := Client() + if err != nil { + return nil, err + } + + node, err := c.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + var ips []string + for _, addr := range node.Status.Addresses { + if addr.Type == corev1.NodeExternalIP || addr.Type == corev1.NodeInternalIP { + ips = append(ips, addr.Address) + } + } + return ips, nil +} + +// GetLoadBalancerIPs returns the external IPs assigned to all game server +// LoadBalancer Services in the configured namespace. This is used by the +// allocation endpoint to show available IPs when in LoadBalancer mode. +func GetLoadBalancerIPs(ctx context.Context) ([]string, error) { + cfg := config.Get() + c, err := Client() + if err != nil { + return nil, err + } + + svcs, err := c.CoreV1().Services(cfg.Kubernetes.Namespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/managed-by=pelican-wings,pelican.dev/resource-type=service", + }) + if err != nil { + return nil, err + } + + seen := make(map[string]bool) + var ips []string + for _, svc := range svcs.Items { + for _, ingress := range svc.Status.LoadBalancer.Ingress { + addr := ingress.IP + if addr == "" { + addr = ingress.Hostname + } + if addr != "" && !seen[addr] { + seen[addr] = true + ips = append(ips, addr) + } + } + } + return ips, nil +} diff --git a/environment/kubernetes/power.go b/environment/kubernetes/power.go new file mode 100644 index 00000000..7964405c --- /dev/null +++ b/environment/kubernetes/power.go @@ -0,0 +1,282 @@ +package kubernetes + +import ( + "context" + "strings" + "time" + + "emperror.dev/errors" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" + + "github.com/pelican-dev/wings/environment" + "github.com/pelican-dev/wings/remote" +) + +// OnBeforeStart is called before the server starts. It ensures the Pod is in a +// clean state by deleting any existing Pod and recreating it with the latest +// configuration from the Panel. +func (e *Environment) OnBeforeStart(ctx context.Context) error { + // Delete any existing Pod to ensure fresh config is applied. + gracePeriod := int64(0) + _ = e.client.CoreV1().Pods(e.namespace()).Delete(ctx, e.Id, metav1.DeleteOptions{ + GracePeriodSeconds: &gracePeriod, + }) + + // Wait briefly for deletion to propagate. + e.waitForPodDeletion(ctx, 10*time.Second) + + // Create the Pod with current configuration. + if err := e.Create(); err != nil { + return err + } + + return nil +} + +// Start boots the server by creating the Pod (if needed) and attaching to it. +// Since Kubernetes Pods start running immediately upon creation, this mainly +// ensures we're attached to capture output. +func (e *Environment) Start(ctx context.Context) error { + sawError := false + + defer func() { + if sawError { + e.SetState(environment.ProcessStoppingState) + e.SetState(environment.ProcessOfflineState) + } + }() + + // Check if Pod already exists and is running. + if running, _ := e.IsRunning(ctx); running { + e.SetState(environment.ProcessRunningState) + return e.Attach(ctx) + } + + e.SetState(environment.ProcessStartingState) + sawError = true + + // Run pre-start to ensure the Pod exists with fresh configuration. + if err := e.OnBeforeStart(ctx); err != nil { + return errors.WrapIf(err, "environment/kubernetes: failed to run pre-boot process") + } + + // Wait for the Pod to reach Running phase. + waitCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + + if err := e.waitForPodRunning(waitCtx); err != nil { + return errors.WrapIf(err, "environment/kubernetes: pod did not reach running state") + } + + // Attach to the Pod to stream output. + if err := e.Attach(ctx); err != nil { + return errors.WrapIf(err, "environment/kubernetes: failed to attach to pod") + } + + e.SetState(environment.ProcessRunningState) + sawError = false + return nil +} + +// Stop sends the configured stop command or deletes the Pod with a grace +// period to allow graceful shutdown. +func (e *Environment) Stop(ctx context.Context) error { + e.mu.RLock() + s := e.meta.Stop + e.mu.RUnlock() + + // If using a signal-based stop, terminate with that signal. + if s.Type == "" || s.Type == remote.ProcessStopSignal { + return e.Terminate(ctx, "SIGTERM") + } + + if e.st.Load() != environment.ProcessOfflineState { + e.SetState(environment.ProcessStoppingState) + } + + // If using a command-based stop and we're attached, send the command. + if e.IsAttached() && s.Type == remote.ProcessStopCommand { + return e.SendCommand(s.Value) + } + + // Default: delete the Pod with a 30-second grace period. + gracePeriod := int64(30) + err := e.client.CoreV1().Pods(e.namespace()).Delete(ctx, e.Id, metav1.DeleteOptions{ + GracePeriodSeconds: &gracePeriod, + }) + if err != nil && !isNotFound(err) { + return errors.Wrap(err, "environment/kubernetes: failed to stop pod") + } + + return nil +} + +// WaitForStop attempts to gracefully stop the server and waits for the Pod to +// terminate. If the timeout is reached and terminate is true, the Pod is +// forcefully deleted. +func (e *Environment) WaitForStop(ctx context.Context, duration time.Duration, terminate bool) error { + // If the Pod is already gone, or exists but is no longer running, the server + // is effectively stopped and there is nothing to wait for. A stopped Pod is + // not automatically removed, so proceeding into waitForPodDeletion would block + // for the full duration (holding the power lock) waiting for a deletion that + // never happens. This is what gets hit when a stop/restart is issued against a + // server that is already offline. + if pod, err := e.getPod(ctx); err != nil { + if isNotFound(err) { + e.markOffline() + return nil + } + // Fall through on unexpected errors and attempt the normal stop flow. + } else if !isPodRunning(pod) { + e.markOffline() + return nil + } + + tctx, cancel := context.WithTimeout(context.Background(), duration) + defer cancel() + + go func() { + select { + case <-ctx.Done(): + cancel() + case <-tctx.Done(): + } + }() + + // Send the stop command/signal. + if err := e.Stop(tctx); err != nil { + if terminate && errors.Is(err, context.DeadlineExceeded) { + return e.Terminate(ctx, "SIGKILL") + } + return err + } + + // Wait for the Pod to be gone. + if err := e.waitForPodDeletion(tctx, duration); err != nil { + if terminate { + e.log().Warn("pod did not terminate in time, forcing deletion") + return e.Terminate(ctx, "SIGKILL") + } + return err + } + + return nil +} + +// Terminate forcefully stops the Pod by deleting it with a zero grace period. +func (e *Environment) Terminate(ctx context.Context, signal string) error { + _ = signal // K8s doesn't support arbitrary signals; we just force-delete. + + pod, err := e.getPod(ctx) + if err != nil { + if isNotFound(err) { + return nil + } + return errors.WithStack(err) + } + + if !isPodRunning(pod) { + e.markOffline() + return nil + } + + e.SetState(environment.ProcessStoppingState) + + gracePeriod := int64(0) + err = e.client.CoreV1().Pods(e.namespace()).Delete(ctx, e.Id, metav1.DeleteOptions{ + GracePeriodSeconds: &gracePeriod, + }) + if err != nil && !isNotFound(err) { + return errors.WithStack(err) + } + + e.SetState(environment.ProcessOfflineState) + return nil +} + +// markOffline transitions the environment to the offline state, first passing +// through the stopping state so that crash detection is not triggered. It is a +// no-op if the environment already considers itself offline. +func (e *Environment) markOffline() { + if e.st.Load() != environment.ProcessOfflineState { + e.SetState(environment.ProcessStoppingState) + e.SetState(environment.ProcessOfflineState) + } +} + +// waitForPodRunning blocks until the Pod reaches Running phase or the context +// is canceled. +func (e *Environment) waitForPodRunning(ctx context.Context) error { + // First check if already running. + if running, _ := e.IsRunning(ctx); running { + return nil + } + + watcher, err := e.client.CoreV1().Pods(e.namespace()).Watch(ctx, metav1.ListOptions{ + FieldSelector: "metadata.name=" + e.Id, + }) + if err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to watch pod") + } + defer watcher.Stop() + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case event, ok := <-watcher.ResultChan(): + if !ok { + return errors.New("environment/kubernetes: watch channel closed") + } + if event.Type == watch.Modified || event.Type == watch.Added { + pod, ok := event.Object.(*corev1.Pod) + if !ok { + continue + } + if isPodRunning(pod) { + return nil + } + // Check for failure. + if pod.Status.Phase == corev1.PodFailed || pod.Status.Phase == corev1.PodSucceeded { + return errors.New("environment/kubernetes: pod terminated before reaching running state") + } + // Check for container crashes during startup. + for _, cs := range pod.Status.ContainerStatuses { + if cs.State.Waiting != nil && strings.Contains(cs.State.Waiting.Reason, "CrashLoopBackOff") { + return errors.New("environment/kubernetes: container in CrashLoopBackOff") + } + if cs.State.Waiting != nil && strings.Contains(cs.State.Waiting.Reason, "ErrImagePull") { + return errors.New("environment/kubernetes: failed to pull container image") + } + if cs.State.Waiting != nil && strings.Contains(cs.State.Waiting.Reason, "ImagePullBackOff") { + return errors.New("environment/kubernetes: image pull backoff") + } + } + } + } + } +} + +// waitForPodDeletion blocks until the Pod is fully deleted or the timeout +// elapses. +func (e *Environment) waitForPodDeletion(ctx context.Context, timeout time.Duration) error { + dctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-dctx.Done(): + return dctx.Err() + case <-ticker.C: + _, err := e.getPod(dctx) + if err != nil && isNotFound(err) { + return nil + } + } + } +} diff --git a/environment/kubernetes/power_test.go b/environment/kubernetes/power_test.go new file mode 100644 index 00000000..e40b3712 --- /dev/null +++ b/environment/kubernetes/power_test.go @@ -0,0 +1,77 @@ +package kubernetes + +import ( + "context" + "testing" + "time" + + . "github.com/franela/goblin" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + + "github.com/pelican-dev/wings/config" + "github.com/pelican-dev/wings/environment" + "github.com/pelican-dev/wings/events" + "github.com/pelican-dev/wings/system" +) + +func TestWaitForStop(t *testing.T) { + g := Goblin(t) + + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + }) + + newEnv := func(state string, objects ...runtime.Object) *Environment { + return &Environment{ + Id: "test-uuid", + meta: &Metadata{}, + client: fake.NewSimpleClientset(objects...), + st: system.NewAtomicString(state), + emitter: events.NewBus(), + } + } + + g.Describe("WaitForStop when the server is already offline", func() { + g.It("returns immediately when the Pod does not exist", func() { + env := newEnv(environment.ProcessOfflineState) + + done := make(chan error, 1) + go func() { + // A long duration would previously block here for the full period. + done <- env.WaitForStop(context.Background(), time.Minute, true) + }() + + select { + case err := <-done: + g.Assert(err).IsNil() + case <-time.After(2 * time.Second): + g.Fail("WaitForStop blocked when the Pod was absent") + } + }) + + g.It("returns immediately when the Pod exists but is not running", func() { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "test-uuid", Namespace: "pelican"}, + Status: corev1.PodStatus{Phase: corev1.PodSucceeded}, + } + // Start from a non-offline state to prove the helper drives it offline. + env := newEnv(environment.ProcessRunningState, pod) + + done := make(chan error, 1) + go func() { + done <- env.WaitForStop(context.Background(), time.Minute, true) + }() + + select { + case err := <-done: + g.Assert(err).IsNil() + g.Assert(env.State()).Equal(environment.ProcessOfflineState) + case <-time.After(2 * time.Second): + g.Fail("WaitForStop blocked when the Pod was not running") + } + }) + }) +} diff --git a/environment/kubernetes/quota.go b/environment/kubernetes/quota.go new file mode 100644 index 00000000..68998370 --- /dev/null +++ b/environment/kubernetes/quota.go @@ -0,0 +1,161 @@ +package kubernetes + +import ( + "context" + + "emperror.dev/errors" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/pelican-dev/wings/config" +) + +const ( + quotaName = "pelican-wings" + limitRangeName = "pelican-wings" +) + +// EnsureResourceQuota creates or updates the ResourceQuota in the game server +// namespace. If resource_quota is not enabled in config, this is a no-op. +func (e *Environment) EnsureResourceQuota(ctx context.Context) error { + cfg := config.Get() + if !cfg.Kubernetes.ResourceQuota.Enabled { + return nil + } + + ns := e.namespace() + quota := buildResourceQuota(ns, &cfg.Kubernetes.ResourceQuota) + + existing, err := e.client.CoreV1().ResourceQuotas(ns).Get(ctx, quotaName, metav1.GetOptions{}) + if err == nil { + // Update existing. + existing.Spec = quota.Spec + _, err = e.client.CoreV1().ResourceQuotas(ns).Update(ctx, existing, metav1.UpdateOptions{}) + if err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to update ResourceQuota") + } + return nil + } + + _, err = e.client.CoreV1().ResourceQuotas(ns).Create(ctx, quota, metav1.CreateOptions{}) + if err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to create ResourceQuota") + } + + e.log().Info("created ResourceQuota for namespace") + return nil +} + +// EnsureLimitRange creates or updates the LimitRange in the game server +// namespace. If limit_range is not enabled in config, this is a no-op. +func (e *Environment) EnsureLimitRange(ctx context.Context) error { + cfg := config.Get() + if !cfg.Kubernetes.LimitRange.Enabled { + return nil + } + + ns := e.namespace() + lr := buildLimitRange(ns, &cfg.Kubernetes.LimitRange) + + existing, err := e.client.CoreV1().LimitRanges(ns).Get(ctx, limitRangeName, metav1.GetOptions{}) + if err == nil { + // Update existing. + existing.Spec = lr.Spec + _, err = e.client.CoreV1().LimitRanges(ns).Update(ctx, existing, metav1.UpdateOptions{}) + if err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to update LimitRange") + } + return nil + } + + _, err = e.client.CoreV1().LimitRanges(ns).Create(ctx, lr, metav1.CreateOptions{}) + if err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to create LimitRange") + } + + e.log().Info("created LimitRange for namespace") + return nil +} + +// buildResourceQuota constructs the ResourceQuota spec from config values. +func buildResourceQuota(namespace string, cfg *config.KubeResourceQuota) *corev1.ResourceQuota { + hard := corev1.ResourceList{} + + if cfg.CPULimit != "" { + hard[corev1.ResourceLimitsCPU] = resource.MustParse(cfg.CPULimit) + } + if cfg.MemoryLimit != "" { + hard[corev1.ResourceLimitsMemory] = resource.MustParse(cfg.MemoryLimit) + } + if cfg.CPURequest != "" { + hard[corev1.ResourceRequestsCPU] = resource.MustParse(cfg.CPURequest) + } + if cfg.MemoryRequest != "" { + hard[corev1.ResourceRequestsMemory] = resource.MustParse(cfg.MemoryRequest) + } + if cfg.MaxPods > 0 { + hard[corev1.ResourcePods] = *resource.NewQuantity(cfg.MaxPods, resource.DecimalSI) + } + if cfg.MaxPVCs > 0 { + hard[corev1.ResourcePersistentVolumeClaims] = *resource.NewQuantity(cfg.MaxPVCs, resource.DecimalSI) + } + if cfg.MaxStorage != "" { + hard[corev1.ResourceRequestsStorage] = resource.MustParse(cfg.MaxStorage) + } + + return &corev1.ResourceQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: quotaName, + Namespace: namespace, + Labels: map[string]string{ + "app.kubernetes.io/managed-by": "pelican-wings", + }, + }, + Spec: corev1.ResourceQuotaSpec{ + Hard: hard, + }, + } +} + +// buildLimitRange constructs the LimitRange spec from config values. +func buildLimitRange(namespace string, cfg *config.KubeLimitRange) *corev1.LimitRange { + containerLimit := corev1.LimitRangeItem{ + Type: corev1.LimitTypeContainer, + Default: corev1.ResourceList{}, + DefaultRequest: corev1.ResourceList{}, + Max: corev1.ResourceList{}, + } + + if cfg.DefaultCPULimit != "" { + containerLimit.Default[corev1.ResourceCPU] = resource.MustParse(cfg.DefaultCPULimit) + } + if cfg.DefaultMemoryLimit != "" { + containerLimit.Default[corev1.ResourceMemory] = resource.MustParse(cfg.DefaultMemoryLimit) + } + if cfg.DefaultCPURequest != "" { + containerLimit.DefaultRequest[corev1.ResourceCPU] = resource.MustParse(cfg.DefaultCPURequest) + } + if cfg.DefaultMemoryRequest != "" { + containerLimit.DefaultRequest[corev1.ResourceMemory] = resource.MustParse(cfg.DefaultMemoryRequest) + } + if cfg.MaxCPU != "" { + containerLimit.Max[corev1.ResourceCPU] = resource.MustParse(cfg.MaxCPU) + } + if cfg.MaxMemory != "" { + containerLimit.Max[corev1.ResourceMemory] = resource.MustParse(cfg.MaxMemory) + } + + return &corev1.LimitRange{ + ObjectMeta: metav1.ObjectMeta{ + Name: limitRangeName, + Namespace: namespace, + Labels: map[string]string{ + "app.kubernetes.io/managed-by": "pelican-wings", + }, + }, + Spec: corev1.LimitRangeSpec{ + Limits: []corev1.LimitRangeItem{containerLimit}, + }, + } +} diff --git a/environment/kubernetes/quota_test.go b/environment/kubernetes/quota_test.go new file mode 100644 index 00000000..b36328e6 --- /dev/null +++ b/environment/kubernetes/quota_test.go @@ -0,0 +1,311 @@ +package kubernetes + +import ( + "context" + "testing" + + . "github.com/franela/goblin" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + + "github.com/pelican-dev/wings/config" + "github.com/pelican-dev/wings/environment" + "github.com/pelican-dev/wings/system" +) + +func TestQuota(t *testing.T) { + g := Goblin(t) + + g.Describe("ResourceQuota", func() { + g.Describe("EnsureResourceQuota", func() { + g.It("should be a no-op when disabled", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "quota-noop-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.ResourceQuota.Enabled = false + }) + + err := env.EnsureResourceQuota(context.Background()) + g.Assert(err).IsNil() + + quotas, _ := client.CoreV1().ResourceQuotas("pelican").List(context.Background(), metav1.ListOptions{}) + g.Assert(len(quotas.Items)).Equal(0) + }) + + g.It("should create a ResourceQuota with CPU and memory limits", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "quota-create-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.ResourceQuota.Enabled = true + c.Kubernetes.ResourceQuota.CPULimit = "16" + c.Kubernetes.ResourceQuota.MemoryLimit = "32Gi" + c.Kubernetes.ResourceQuota.CPURequest = "8" + c.Kubernetes.ResourceQuota.MemoryRequest = "16Gi" + c.Kubernetes.ResourceQuota.MaxPods = 20 + }) + + err := env.EnsureResourceQuota(context.Background()) + g.Assert(err).IsNil() + + rq, err := client.CoreV1().ResourceQuotas("pelican").Get(context.Background(), "pelican-wings", metav1.GetOptions{}) + g.Assert(err).IsNil() + g.Assert(rq.Name).Equal("pelican-wings") + g.Assert(rq.Labels["app.kubernetes.io/managed-by"]).Equal("pelican-wings") + + cpuLimit := rq.Spec.Hard[corev1.ResourceLimitsCPU] + g.Assert(cpuLimit.Cmp(resource.MustParse("16"))).Equal(0) + + memLimit := rq.Spec.Hard[corev1.ResourceLimitsMemory] + g.Assert(memLimit.Cmp(resource.MustParse("32Gi"))).Equal(0) + + pods := rq.Spec.Hard[corev1.ResourcePods] + g.Assert(pods.Value()).Equal(int64(20)) + }) + + g.It("should create ResourceQuota with storage limits", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "quota-storage-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.ResourceQuota.Enabled = true + c.Kubernetes.ResourceQuota.MaxPVCs = 50 + c.Kubernetes.ResourceQuota.MaxStorage = "500Gi" + c.Kubernetes.ResourceQuota.CPULimit = "" + c.Kubernetes.ResourceQuota.MemoryLimit = "" + c.Kubernetes.ResourceQuota.CPURequest = "" + c.Kubernetes.ResourceQuota.MemoryRequest = "" + c.Kubernetes.ResourceQuota.MaxPods = 0 + }) + + err := env.EnsureResourceQuota(context.Background()) + g.Assert(err).IsNil() + + rq, err := client.CoreV1().ResourceQuotas("pelican").Get(context.Background(), "pelican-wings", metav1.GetOptions{}) + g.Assert(err).IsNil() + + pvcs := rq.Spec.Hard[corev1.ResourcePersistentVolumeClaims] + g.Assert(pvcs.Value()).Equal(int64(50)) + + storage := rq.Spec.Hard[corev1.ResourceRequestsStorage] + g.Assert(storage.Cmp(resource.MustParse("500Gi"))).Equal(0) + }) + + g.It("should update an existing ResourceQuota", func() { + existingRQ := &corev1.ResourceQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pelican-wings", + Namespace: "pelican", + }, + Spec: corev1.ResourceQuotaSpec{ + Hard: corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("8"), + }, + }, + } + client := fake.NewSimpleClientset(existingRQ) + env := &Environment{ + Id: "quota-update-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.ResourceQuota.Enabled = true + c.Kubernetes.ResourceQuota.CPULimit = "32" + c.Kubernetes.ResourceQuota.MemoryLimit = "64Gi" + c.Kubernetes.ResourceQuota.CPURequest = "" + c.Kubernetes.ResourceQuota.MemoryRequest = "" + c.Kubernetes.ResourceQuota.MaxPods = 0 + c.Kubernetes.ResourceQuota.MaxPVCs = 0 + c.Kubernetes.ResourceQuota.MaxStorage = "" + }) + + err := env.EnsureResourceQuota(context.Background()) + g.Assert(err).IsNil() + + rq, _ := client.CoreV1().ResourceQuotas("pelican").Get(context.Background(), "pelican-wings", metav1.GetOptions{}) + cpuLimit := rq.Spec.Hard[corev1.ResourceLimitsCPU] + g.Assert(cpuLimit.Cmp(resource.MustParse("32"))).Equal(0) + + memLimit := rq.Spec.Hard[corev1.ResourceLimitsMemory] + g.Assert(memLimit.Cmp(resource.MustParse("64Gi"))).Equal(0) + }) + }) + + g.Describe("EnsureLimitRange", func() { + g.It("should be a no-op when disabled", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "lr-noop-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.LimitRange.Enabled = false + }) + + err := env.EnsureLimitRange(context.Background()) + g.Assert(err).IsNil() + + lrs, _ := client.CoreV1().LimitRanges("pelican").List(context.Background(), metav1.ListOptions{}) + g.Assert(len(lrs.Items)).Equal(0) + }) + + g.It("should create a LimitRange with defaults and max", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "lr-create-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.LimitRange.Enabled = true + c.Kubernetes.LimitRange.DefaultCPULimit = "2" + c.Kubernetes.LimitRange.DefaultMemoryLimit = "4Gi" + c.Kubernetes.LimitRange.DefaultCPURequest = "500m" + c.Kubernetes.LimitRange.DefaultMemoryRequest = "1Gi" + c.Kubernetes.LimitRange.MaxCPU = "8" + c.Kubernetes.LimitRange.MaxMemory = "16Gi" + }) + + err := env.EnsureLimitRange(context.Background()) + g.Assert(err).IsNil() + + lr, err := client.CoreV1().LimitRanges("pelican").Get(context.Background(), "pelican-wings", metav1.GetOptions{}) + g.Assert(err).IsNil() + g.Assert(lr.Name).Equal("pelican-wings") + g.Assert(len(lr.Spec.Limits)).Equal(1) + + item := lr.Spec.Limits[0] + g.Assert(item.Type).Equal(corev1.LimitTypeContainer) + + defaultCPU := item.Default[corev1.ResourceCPU] + g.Assert(defaultCPU.Cmp(resource.MustParse("2"))).Equal(0) + + defaultMem := item.Default[corev1.ResourceMemory] + g.Assert(defaultMem.Cmp(resource.MustParse("4Gi"))).Equal(0) + + reqCPU := item.DefaultRequest[corev1.ResourceCPU] + g.Assert(reqCPU.Cmp(resource.MustParse("500m"))).Equal(0) + + maxCPU := item.Max[corev1.ResourceCPU] + g.Assert(maxCPU.Cmp(resource.MustParse("8"))).Equal(0) + + maxMem := item.Max[corev1.ResourceMemory] + g.Assert(maxMem.Cmp(resource.MustParse("16Gi"))).Equal(0) + }) + + g.It("should update an existing LimitRange", func() { + existingLR := &corev1.LimitRange{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pelican-wings", + Namespace: "pelican", + }, + Spec: corev1.LimitRangeSpec{ + Limits: []corev1.LimitRangeItem{ + { + Type: corev1.LimitTypeContainer, + Default: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + }, + }, + }, + }, + } + client := fake.NewSimpleClientset(existingLR) + env := &Environment{ + Id: "lr-update-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.LimitRange.Enabled = true + c.Kubernetes.LimitRange.DefaultCPULimit = "4" + c.Kubernetes.LimitRange.DefaultMemoryLimit = "8Gi" + c.Kubernetes.LimitRange.DefaultCPURequest = "" + c.Kubernetes.LimitRange.DefaultMemoryRequest = "" + c.Kubernetes.LimitRange.MaxCPU = "" + c.Kubernetes.LimitRange.MaxMemory = "" + }) + + err := env.EnsureLimitRange(context.Background()) + g.Assert(err).IsNil() + + lr, _ := client.CoreV1().LimitRanges("pelican").Get(context.Background(), "pelican-wings", metav1.GetOptions{}) + defaultCPU := lr.Spec.Limits[0].Default[corev1.ResourceCPU] + g.Assert(defaultCPU.Cmp(resource.MustParse("4"))).Equal(0) + }) + }) + + g.Describe("buildResourceQuota", func() { + g.It("should only include non-empty fields", func() { + cfg := &config.KubeResourceQuota{ + CPULimit: "4", + MemoryLimit: "", + MaxPods: 10, + MaxPVCs: 0, + } + rq := buildResourceQuota("test-ns", cfg) + g.Assert(rq.Namespace).Equal("test-ns") + + _, hasCPU := rq.Spec.Hard[corev1.ResourceLimitsCPU] + g.Assert(hasCPU).IsTrue() + + _, hasMem := rq.Spec.Hard[corev1.ResourceLimitsMemory] + g.Assert(hasMem).IsFalse() + + _, hasPods := rq.Spec.Hard[corev1.ResourcePods] + g.Assert(hasPods).IsTrue() + + _, hasPVCs := rq.Spec.Hard[corev1.ResourcePersistentVolumeClaims] + g.Assert(hasPVCs).IsFalse() + }) + }) + + g.Describe("buildLimitRange", func() { + g.It("should only include non-empty fields", func() { + cfg := &config.KubeLimitRange{ + DefaultCPULimit: "2", + DefaultMemoryLimit: "", + MaxCPU: "8", + MaxMemory: "", + } + lr := buildLimitRange("test-ns", cfg) + g.Assert(lr.Namespace).Equal("test-ns") + g.Assert(len(lr.Spec.Limits)).Equal(1) + + item := lr.Spec.Limits[0] + _, hasCPU := item.Default[corev1.ResourceCPU] + g.Assert(hasCPU).IsTrue() + + _, hasMem := item.Default[corev1.ResourceMemory] + g.Assert(hasMem).IsFalse() + + _, hasMaxCPU := item.Max[corev1.ResourceCPU] + g.Assert(hasMaxCPU).IsTrue() + + _, hasMaxMem := item.Max[corev1.ResourceMemory] + g.Assert(hasMaxMem).IsFalse() + }) + }) + }) +} diff --git a/environment/kubernetes/stats.go b/environment/kubernetes/stats.go new file mode 100644 index 00000000..4ce29482 --- /dev/null +++ b/environment/kubernetes/stats.go @@ -0,0 +1,403 @@ +package kubernetes + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + + "emperror.dev/errors" + + "github.com/pelican-dev/wings/environment" +) + +// podStats holds parsed resource metrics for a pod's "server" container. +type podStats struct { + cpuPercent float64 + memoryBytes int64 + rxBytes uint64 + txBytes uint64 +} + +// Uptime returns the uptime of the server Pod in milliseconds. If the Pod is +// not running, returns 0. +func (e *Environment) Uptime(ctx context.Context) (int64, error) { + pod, err := e.getPod(ctx) + if err != nil { + return 0, errors.Wrap(err, "environment/kubernetes: could not get pod") + } + if !isPodRunning(pod) { + return 0, nil + } + if pod.Status.StartTime == nil { + return 0, nil + } + return time.Since(pod.Status.StartTime.Time).Milliseconds(), nil +} + +// pollResources periodically fetches resource usage and publishes resource +// events. It tries the Kubernetes Metrics API first (requires metrics-server) +// and falls back to the kubelet stats/summary API (always available). +func (e *Environment) pollResources(ctx context.Context) error { + if e.st.Load() == environment.ProcessOfflineState { + return errors.New("cannot enable resource polling on a stopped server") + } + + e.log().Info("starting resource polling for pod") + defer e.log().Debug("stopped resource polling for pod") + + uptime, err := e.Uptime(ctx) + if err != nil { + e.log().WithField("error", err).Warn("failed to calculate pod uptime") + } + + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + + lastCheck := time.Now() + useKubelet := false + loggedSource := false + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + if e.st.Load() == environment.ProcessOfflineState { + return nil + } + + now := time.Now() + uptime += now.Sub(lastCheck).Milliseconds() + lastCheck = now + + st := environment.Stats{ + Uptime: uptime, + } + + var stats *podStats + + if !useKubelet { + stats, err = e.getMetricsAPIStats(ctx) + if err != nil { + if !loggedSource { + e.log().WithField("error", err).Info("metrics API unavailable, falling back to kubelet stats") + } + useKubelet = true + } + } + + if useKubelet { + stats, err = e.getKubeletPodStats(ctx) + if err != nil { + if !loggedSource { + e.log().WithField("error", err).Warn("kubelet stats also unavailable, resource metrics will not be reported") + loggedSource = true + } + } + } + + if stats != nil { + if !loggedSource { + src := "metrics API (metrics-server)" + if useKubelet { + src = "kubelet stats/summary" + } + e.log().WithField("source", src).Info("collecting pod resource metrics") + loggedSource = true + } + st.Memory = uint64(stats.memoryBytes) + st.CpuAbsolute = stats.cpuPercent + st.Network = environment.NetworkStats{ + RxBytes: stats.rxBytes, + TxBytes: stats.txBytes, + } + } + + // Get memory limit from pod spec. + pod, err := e.getPod(ctx) + if err == nil && pod != nil { + for _, c := range pod.Spec.Containers { + if c.Name == "server" { + if mem, ok := c.Resources.Limits["memory"]; ok { + st.MemoryLimit = uint64(mem.Value()) + } + } + } + } + + e.Events().Publish(environment.ResourceEvent, st) + } + } +} + +// --------------------------------------------------------------------------- +// Source 1: Kubernetes Metrics API (requires metrics-server) +// --------------------------------------------------------------------------- + +// getMetricsAPIStats fetches container metrics from the Kubernetes Metrics API +// (metrics.k8s.io/v1beta1). Returns an error if the API is unavailable. +func (e *Environment) getMetricsAPIStats(ctx context.Context) (*podStats, error) { + result := e.client.CoreV1().RESTClient().Get(). + AbsPath("/apis/metrics.k8s.io/v1beta1"). + Resource("pods"). + Namespace(e.namespace()). + Name(e.Id). + Do(ctx) + + if result.Error() != nil { + return nil, result.Error() + } + + raw, err := result.Raw() + if err != nil { + return nil, err + } + + return parseMetricsAPIPodStats(raw) +} + +// podMetricsResponse represents the relevant fields of the Kubernetes Metrics +// API PodMetrics response (metrics.k8s.io/v1beta1). +type podMetricsResponse struct { + Containers []containerMetricsEntry `json:"containers"` +} + +// containerMetricsEntry represents a single container's usage in the Metrics +// API response. +type containerMetricsEntry struct { + Name string `json:"name"` + Usage map[string]string `json:"usage"` +} + +// parseMetricsAPIPodStats extracts the "server" container's CPU/memory from a +// PodMetrics JSON response. Network stats are not available from this API. +func parseMetricsAPIPodStats(raw []byte) (*podStats, error) { + var resp podMetricsResponse + if err := json.Unmarshal(raw, &resp); err != nil { + return nil, errors.Wrap(err, "environment/kubernetes: failed to parse pod metrics") + } + + for _, c := range resp.Containers { + if c.Name != "server" { + continue + } + + s := &podStats{} + + if memStr, ok := c.Usage["memory"]; ok { + s.memoryBytes = parseResourceQuantity(memStr) + } + if cpuStr, ok := c.Usage["cpu"]; ok { + s.cpuPercent = parseCPUToPercent(cpuStr) + } + + return s, nil + } + + return nil, nil +} + +// --------------------------------------------------------------------------- +// Source 2: Kubelet stats/summary API (always available, no metrics-server) +// --------------------------------------------------------------------------- + +// getKubeletPodStats fetches resource metrics from the kubelet stats/summary +// endpoint via the K8s API server proxy. This works on any cluster without +// requiring metrics-server. +func (e *Environment) getKubeletPodStats(ctx context.Context) (*podStats, error) { + pod, err := e.getPod(ctx) + if err != nil { + return nil, errors.Wrap(err, "failed to get pod for kubelet stats") + } + if pod.Spec.NodeName == "" { + return nil, errors.New("pod has no node assignment yet") + } + + result := e.client.CoreV1().RESTClient().Get(). + AbsPath("/api/v1/nodes", pod.Spec.NodeName, "proxy", "stats", "summary"). + Do(ctx) + + if result.Error() != nil { + return nil, errors.Wrap(result.Error(), "kubelet stats/summary request failed") + } + + raw, err := result.Raw() + if err != nil { + return nil, errors.Wrap(err, "failed to read kubelet stats response") + } + + return parseKubeletPodStats(raw, e.Id, e.namespace()) +} + +// kubelet stats/summary response types (only the fields we need). +type kubeletStatsSummary struct { + Pods []kubeletPodStats `json:"pods"` +} + +type kubeletPodStats struct { + PodRef struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + } `json:"podRef"` + Containers []kubeletContainerStats `json:"containers"` + Network *kubeletNetworkStats `json:"network,omitempty"` +} + +type kubeletContainerStats struct { + Name string `json:"name"` + CPU *kubeletCPUStats `json:"cpu,omitempty"` + Memory *kubeletMemoryStats `json:"memory,omitempty"` +} + +type kubeletCPUStats struct { + UsageNanoCores *uint64 `json:"usageNanoCores,omitempty"` +} + +type kubeletMemoryStats struct { + WorkingSetBytes *uint64 `json:"workingSetBytes,omitempty"` +} + +type kubeletNetworkStats struct { + RxBytes *uint64 `json:"rxBytes,omitempty"` + TxBytes *uint64 `json:"txBytes,omitempty"` +} + +// parseKubeletPodStats finds our pod in a kubelet stats/summary response and +// extracts CPU, memory, and network metrics for the "server" container. +func parseKubeletPodStats(raw []byte, podName, namespace string) (*podStats, error) { + var summary kubeletStatsSummary + if err := json.Unmarshal(raw, &summary); err != nil { + return nil, errors.Wrap(err, "failed to parse kubelet stats summary") + } + + for _, p := range summary.Pods { + if p.PodRef.Name != podName || p.PodRef.Namespace != namespace { + continue + } + + s := &podStats{} + + for _, c := range p.Containers { + if c.Name != "server" { + continue + } + if c.CPU != nil && c.CPU.UsageNanoCores != nil { + s.cpuPercent = float64(*c.CPU.UsageNanoCores) / 1e9 * 100 + } + if c.Memory != nil && c.Memory.WorkingSetBytes != nil { + s.memoryBytes = int64(*c.Memory.WorkingSetBytes) + } + } + + if p.Network != nil { + if p.Network.RxBytes != nil { + s.rxBytes = *p.Network.RxBytes + } + if p.Network.TxBytes != nil { + s.txBytes = *p.Network.TxBytes + } + } + + return s, nil + } + + return nil, fmt.Errorf("pod %s/%s not found in kubelet stats", namespace, podName) +} + +// --------------------------------------------------------------------------- +// Parsing helpers +// --------------------------------------------------------------------------- + +// parseResourceQuantity parses a Kubernetes resource quantity string for memory +// and returns the value in bytes. Supports suffixes: Ki, Mi, Gi, Ti, k, M, G, T, +// or plain bytes. +func parseResourceQuantity(s string) int64 { + s = strings.TrimSpace(s) + if s == "" { + return 0 + } + + // Binary suffixes (Ki, Mi, Gi, Ti). + binarySuffixes := map[string]int64{ + "Ki": 1024, + "Mi": 1024 * 1024, + "Gi": 1024 * 1024 * 1024, + "Ti": 1024 * 1024 * 1024 * 1024, + } + for suffix, multiplier := range binarySuffixes { + if strings.HasSuffix(s, suffix) { + val, err := strconv.ParseInt(strings.TrimSuffix(s, suffix), 10, 64) + if err != nil { + return 0 + } + return val * multiplier + } + } + + // Decimal suffixes (k, M, G, T). + decimalSuffixes := map[string]int64{ + "T": 1000000000000, + "G": 1000000000, + "M": 1000000, + "k": 1000, + } + for suffix, multiplier := range decimalSuffixes { + if strings.HasSuffix(s, suffix) { + val, err := strconv.ParseInt(strings.TrimSuffix(s, suffix), 10, 64) + if err != nil { + return 0 + } + return val * multiplier + } + } + + // Plain bytes (no suffix). + val, err := strconv.ParseInt(s, 10, 64) + if err != nil { + return 0 + } + return val +} + +// parseCPUToPercent converts a Kubernetes CPU quantity string to a percentage +// of a single core. Examples: "250m" → 25.0, "1" → 100.0, "0.5" → 50.0, +// "1500m" → 150.0, "100n" → 0.0001. +func parseCPUToPercent(s string) float64 { + s = strings.TrimSpace(s) + if s == "" { + return 0 + } + + // Nanocores suffix "n" (e.g. "250000000n" = 0.25 cores). + if strings.HasSuffix(s, "n") { + val, err := strconv.ParseFloat(strings.TrimSuffix(s, "n"), 64) + if err != nil { + return 0 + } + return (val / 1000000000) * 100 + } + + // Millicores suffix "m" (e.g. "250m" = 0.25 cores). + if strings.HasSuffix(s, "m") { + val, err := strconv.ParseFloat(strings.TrimSuffix(s, "m"), 64) + if err != nil { + return 0 + } + return (val / 1000) * 100 + } + + // Whole/decimal cores (e.g. "1" or "0.5"). + val, err := strconv.ParseFloat(s, 64) + if err != nil { + return 0 + } + return val * 100 +} + +// podLabelSelector returns a label selector for this server's Pod. +func (e *Environment) podLabelSelector() string { + return "pelican.dev/server-id=" + e.Id +} diff --git a/environment/kubernetes/stats_test.go b/environment/kubernetes/stats_test.go new file mode 100644 index 00000000..1de812e6 --- /dev/null +++ b/environment/kubernetes/stats_test.go @@ -0,0 +1,327 @@ +package kubernetes + +import ( + "encoding/json" + "testing" + + . "github.com/franela/goblin" +) + +func mustMarshal(v interface{}) []byte { + b, err := json.Marshal(v) + if err != nil { + panic(err) + } + return b +} + +func TestStats(t *testing.T) { + g := Goblin(t) + + g.Describe("parseMetricsAPIPodStats", func() { + g.It("should parse a valid PodMetrics response", func() { + raw := []byte(`{ + "kind": "PodMetrics", + "metadata": {"name": "test-pod", "namespace": "pelican"}, + "containers": [ + { + "name": "server", + "usage": { + "cpu": "250m", + "memory": "134217728" + } + } + ] + }`) + + metrics, err := parseMetricsAPIPodStats(raw) + g.Assert(err).IsNil() + g.Assert(metrics).IsNotNil() + g.Assert(metrics.memoryBytes).Equal(int64(134217728)) + g.Assert(metrics.cpuPercent).Equal(25.0) + }) + + g.It("should handle nanocores CPU format", func() { + raw := []byte(`{ + "containers": [ + { + "name": "server", + "usage": { + "cpu": "500000000n", + "memory": "67108864" + } + } + ] + }`) + + metrics, err := parseMetricsAPIPodStats(raw) + g.Assert(err).IsNil() + g.Assert(metrics.cpuPercent).Equal(50.0) + }) + + g.It("should handle whole core CPU format", func() { + raw := []byte(`{ + "containers": [ + { + "name": "server", + "usage": { + "cpu": "2", + "memory": "0" + } + } + ] + }`) + + metrics, err := parseMetricsAPIPodStats(raw) + g.Assert(err).IsNil() + g.Assert(metrics.cpuPercent).Equal(200.0) + }) + + g.It("should handle memory with Ki suffix", func() { + raw := []byte(`{ + "containers": [ + { + "name": "server", + "usage": { + "cpu": "0", + "memory": "131072Ki" + } + } + ] + }`) + + metrics, err := parseMetricsAPIPodStats(raw) + g.Assert(err).IsNil() + g.Assert(metrics.memoryBytes).Equal(int64(131072 * 1024)) + }) + + g.It("should handle memory with Mi suffix", func() { + raw := []byte(`{ + "containers": [ + { + "name": "server", + "usage": { + "cpu": "100m", + "memory": "256Mi" + } + } + ] + }`) + + metrics, err := parseMetricsAPIPodStats(raw) + g.Assert(err).IsNil() + g.Assert(metrics.memoryBytes).Equal(int64(256 * 1024 * 1024)) + }) + + g.It("should return nil when server container is not found", func() { + raw := []byte(`{ + "containers": [ + { + "name": "sidecar", + "usage": { + "cpu": "50m", + "memory": "32Mi" + } + } + ] + }`) + + metrics, err := parseMetricsAPIPodStats(raw) + g.Assert(err).IsNil() + g.Assert(metrics == nil).IsTrue() + }) + + g.It("should return nil for empty containers list", func() { + raw := []byte(`{"containers": []}`) + metrics, err := parseMetricsAPIPodStats(raw) + g.Assert(err).IsNil() + g.Assert(metrics == nil).IsTrue() + }) + + g.It("should return error for invalid JSON", func() { + raw := []byte(`not json at all`) + _, err := parseMetricsAPIPodStats(raw) + g.Assert(err).IsNotNil() + }) + + g.It("should handle multi-container pod and find server", func() { + raw := []byte(`{ + "containers": [ + { + "name": "init-container", + "usage": {"cpu": "10m", "memory": "8Mi"} + }, + { + "name": "server", + "usage": {"cpu": "750m", "memory": "512Mi"} + }, + { + "name": "sidecar-proxy", + "usage": {"cpu": "20m", "memory": "16Mi"} + } + ] + }`) + + metrics, err := parseMetricsAPIPodStats(raw) + g.Assert(err).IsNil() + g.Assert(metrics).IsNotNil() + g.Assert(metrics.cpuPercent).Equal(75.0) + g.Assert(metrics.memoryBytes).Equal(int64(512 * 1024 * 1024)) + }) + }) + + g.Describe("parseKubeletPodStats", func() { + g.It("should parse kubelet stats for the target pod", func() { + cpuNano := uint64(500000000) + memBytes := uint64(268435456) + rxBytes := uint64(1024) + txBytes := uint64(2048) + raw := mustMarshal(kubeletStatsSummary{ + Pods: []kubeletPodStats{ + { + PodRef: struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + }{Name: "other-pod", Namespace: "pelican"}, + Containers: []kubeletContainerStats{ + {Name: "server", CPU: &kubeletCPUStats{UsageNanoCores: &cpuNano}, Memory: &kubeletMemoryStats{WorkingSetBytes: &memBytes}}, + }, + }, + { + PodRef: struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + }{Name: "target-pod", Namespace: "pelican"}, + Containers: []kubeletContainerStats{ + {Name: "server", CPU: &kubeletCPUStats{UsageNanoCores: &cpuNano}, Memory: &kubeletMemoryStats{WorkingSetBytes: &memBytes}}, + }, + Network: &kubeletNetworkStats{RxBytes: &rxBytes, TxBytes: &txBytes}, + }, + }, + }) + + stats, err := parseKubeletPodStats(raw, "target-pod", "pelican") + g.Assert(err).IsNil() + g.Assert(stats).IsNotNil() + g.Assert(stats.cpuPercent).Equal(50.0) + g.Assert(stats.memoryBytes).Equal(int64(268435456)) + g.Assert(stats.rxBytes).Equal(uint64(1024)) + g.Assert(stats.txBytes).Equal(uint64(2048)) + }) + + g.It("should return error when pod not found", func() { + raw := mustMarshal(kubeletStatsSummary{Pods: []kubeletPodStats{}}) + _, err := parseKubeletPodStats(raw, "missing", "pelican") + g.Assert(err).IsNotNil() + }) + + g.It("should handle missing network stats", func() { + cpuNano := uint64(100000000) + memBytes := uint64(1024) + raw := mustMarshal(kubeletStatsSummary{ + Pods: []kubeletPodStats{ + { + PodRef: struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + }{Name: "test", Namespace: "ns"}, + Containers: []kubeletContainerStats{ + {Name: "server", CPU: &kubeletCPUStats{UsageNanoCores: &cpuNano}, Memory: &kubeletMemoryStats{WorkingSetBytes: &memBytes}}, + }, + }, + }, + }) + + stats, err := parseKubeletPodStats(raw, "test", "ns") + g.Assert(err).IsNil() + g.Assert(stats.rxBytes).Equal(uint64(0)) + g.Assert(stats.txBytes).Equal(uint64(0)) + }) + + g.It("should handle nil CPU and memory pointers", func() { + raw := mustMarshal(kubeletStatsSummary{ + Pods: []kubeletPodStats{ + { + PodRef: struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + }{Name: "test", Namespace: "ns"}, + Containers: []kubeletContainerStats{ + {Name: "server"}, + }, + }, + }, + }) + + stats, err := parseKubeletPodStats(raw, "test", "ns") + g.Assert(err).IsNil() + g.Assert(stats.cpuPercent).Equal(0.0) + g.Assert(stats.memoryBytes).Equal(int64(0)) + }) + }) + + g.Describe("parseResourceQuantity", func() { + g.It("should parse plain bytes", func() { + g.Assert(parseResourceQuantity("134217728")).Equal(int64(134217728)) + }) + + g.It("should parse Ki suffix", func() { + g.Assert(parseResourceQuantity("128Ki")).Equal(int64(128 * 1024)) + }) + + g.It("should parse Mi suffix", func() { + g.Assert(parseResourceQuantity("512Mi")).Equal(int64(512 * 1024 * 1024)) + }) + + g.It("should parse Gi suffix", func() { + g.Assert(parseResourceQuantity("2Gi")).Equal(int64(2 * 1024 * 1024 * 1024)) + }) + + g.It("should parse decimal k suffix", func() { + g.Assert(parseResourceQuantity("500k")).Equal(int64(500000)) + }) + + g.It("should parse decimal M suffix", func() { + g.Assert(parseResourceQuantity("100M")).Equal(int64(100000000)) + }) + + g.It("should return 0 for empty string", func() { + g.Assert(parseResourceQuantity("")).Equal(int64(0)) + }) + + g.It("should return 0 for invalid input", func() { + g.Assert(parseResourceQuantity("not-a-number")).Equal(int64(0)) + }) + }) + + g.Describe("parseCPUToPercent", func() { + g.It("should parse millicores", func() { + g.Assert(parseCPUToPercent("250m")).Equal(25.0) + g.Assert(parseCPUToPercent("1000m")).Equal(100.0) + g.Assert(parseCPUToPercent("1500m")).Equal(150.0) + }) + + g.It("should parse nanocores", func() { + g.Assert(parseCPUToPercent("250000000n")).Equal(25.0) + g.Assert(parseCPUToPercent("1000000000n")).Equal(100.0) + }) + + g.It("should parse whole cores", func() { + g.Assert(parseCPUToPercent("1")).Equal(100.0) + g.Assert(parseCPUToPercent("2")).Equal(200.0) + }) + + g.It("should parse decimal cores", func() { + g.Assert(parseCPUToPercent("0.5")).Equal(50.0) + g.Assert(parseCPUToPercent("0.25")).Equal(25.0) + }) + + g.It("should return 0 for empty string", func() { + g.Assert(parseCPUToPercent("")).Equal(0.0) + }) + + g.It("should return 0 for invalid input", func() { + g.Assert(parseCPUToPercent("invalid")).Equal(0.0) + }) + }) +} diff --git a/environment/kubernetes/storage.go b/environment/kubernetes/storage.go new file mode 100644 index 00000000..0b725a78 --- /dev/null +++ b/environment/kubernetes/storage.go @@ -0,0 +1,170 @@ +package kubernetes + +import ( + "context" + "fmt" + + "emperror.dev/errors" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/pelican-dev/wings/config" +) + +// pvcName returns the PersistentVolumeClaim name for this server. +func (e *Environment) pvcName() string { + return fmt.Sprintf("gs-%s", e.Id) +} + +// EnsurePVC creates the PersistentVolumeClaim for this server if it does not +// already exist. When StorageMode is "hostpath" or DataPVC is set (shared PVC +// mode), this is a no-op. +func (e *Environment) EnsurePVC(ctx context.Context) error { + cfg := config.Get() + if cfg.Kubernetes.StorageMode != config.KubeStoragePVC { + return nil + } + // Shared PVC mode: the Wings data PVC is already mounted. No per-server + // PVC is needed; the server data directory is a subPath of the shared PVC. + if cfg.Kubernetes.DataPVC != "" { + return nil + } + + ns := e.namespace() + name := e.pvcName() + + // Check if PVC already exists. + _, err := e.client.CoreV1().PersistentVolumeClaims(ns).Get(ctx, name, metav1.GetOptions{}) + if err == nil { + return nil // Already exists. + } + if !isNotFound(err) { + return errors.Wrap(err, "environment/kubernetes: failed to check existing PVC") + } + + // Parse storage size. + storageSize, err := resource.ParseQuantity(cfg.Kubernetes.StorageSize) + if err != nil { + return errors.Wrap(err, "environment/kubernetes: invalid storage_size in config") + } + + // Determine access mode. + accessMode := corev1.ReadWriteOnce + if cfg.Kubernetes.StorageAccessMode == "ReadWriteMany" { + accessMode = corev1.ReadWriteMany + } + + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + Labels: map[string]string{ + "app.kubernetes.io/managed-by": "pelican-wings", + "pelican.dev/server-id": e.Id, + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{accessMode}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: storageSize, + }, + }, + }, + } + + // Set storage class if configured. + if cfg.Kubernetes.StorageClass != "" { + pvc.Spec.StorageClassName = &cfg.Kubernetes.StorageClass + } + + e.log().WithField("pvc", name).Info("creating PersistentVolumeClaim for server") + _, err = e.client.CoreV1().PersistentVolumeClaims(ns).Create(ctx, pvc, metav1.CreateOptions{}) + if err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to create PVC") + } + + return nil +} + +// DeletePVC removes the PersistentVolumeClaim for this server. This +// permanently deletes the server's data. When StorageMode is "hostpath" or +// DataPVC is set (shared PVC mode), this is a no-op. +func (e *Environment) DeletePVC(ctx context.Context) error { + cfg := config.Get() + if cfg.Kubernetes.StorageMode != config.KubeStoragePVC { + return nil + } + if cfg.Kubernetes.DataPVC != "" { + return nil + } + + ns := e.namespace() + name := e.pvcName() + + err := e.client.CoreV1().PersistentVolumeClaims(ns).Delete(ctx, name, metav1.DeleteOptions{}) + if err != nil && !isNotFound(err) { + return errors.Wrap(err, "environment/kubernetes: failed to delete PVC") + } + + if err == nil { + e.log().WithField("pvc", name).Info("deleted PersistentVolumeClaim for server") + } + + return nil +} + +// GetPVCStatus returns the current status of the server's PVC, or nil if not +// in PVC mode or if the PVC doesn't exist. +func (e *Environment) GetPVCStatus(ctx context.Context) (*corev1.PersistentVolumeClaimPhase, error) { + cfg := config.Get() + if cfg.Kubernetes.StorageMode != config.KubeStoragePVC { + return nil, nil + } + + ns := e.namespace() + name := e.pvcName() + + pvc, err := e.client.CoreV1().PersistentVolumeClaims(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + if isNotFound(err) { + return nil, nil + } + return nil, errors.Wrap(err, "environment/kubernetes: failed to get PVC status") + } + + return &pvc.Status.Phase, nil +} + +// ResizePVC updates the storage request on the PVC to the given size. The +// underlying StorageClass must support volume expansion. +func (e *Environment) ResizePVC(ctx context.Context, newSize string) error { + cfg := config.Get() + if cfg.Kubernetes.StorageMode != config.KubeStoragePVC { + return nil + } + + ns := e.namespace() + name := e.pvcName() + + quantity, err := resource.ParseQuantity(newSize) + if err != nil { + return errors.Wrap(err, "environment/kubernetes: invalid size for PVC resize") + } + + pvc, err := e.client.CoreV1().PersistentVolumeClaims(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to get PVC for resize") + } + + pvc.Spec.Resources.Requests[corev1.ResourceStorage] = quantity + + _, err = e.client.CoreV1().PersistentVolumeClaims(ns).Update(ctx, pvc, metav1.UpdateOptions{}) + if err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to resize PVC") + } + + e.log().WithField("pvc", name).WithField("new_size", newSize).Info("resized PersistentVolumeClaim") + return nil +} diff --git a/environment/kubernetes/storage_test.go b/environment/kubernetes/storage_test.go new file mode 100644 index 00000000..80581a46 --- /dev/null +++ b/environment/kubernetes/storage_test.go @@ -0,0 +1,349 @@ +package kubernetes + +import ( + "context" + "testing" + + . "github.com/franela/goblin" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + + "github.com/pelican-dev/wings/config" + "github.com/pelican-dev/wings/environment" + "github.com/pelican-dev/wings/system" +) + +func TestStorage(t *testing.T) { + g := Goblin(t) + + g.Describe("PVC lifecycle", func() { + g.Describe("pvcName", func() { + g.It("should return correct PVC name", func() { + env := &Environment{Id: "abc-123-def"} + g.Assert(env.pvcName()).Equal("gs-abc-123-def") + }) + }) + + g.Describe("EnsurePVC", func() { + g.It("should be a no-op in hostpath mode", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "hostpath-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.StorageMode = config.KubeStorageHostPath + c.Kubernetes.Namespace = "pelican" + }) + + err := env.EnsurePVC(context.Background()) + g.Assert(err).IsNil() + + // No PVC should exist. + pvcs, _ := client.CoreV1().PersistentVolumeClaims("pelican").List(context.Background(), metav1.ListOptions{}) + g.Assert(len(pvcs.Items)).Equal(0) + }) + + g.It("should create a PVC in pvc mode", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "pvc-create-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.StorageMode = config.KubeStoragePVC + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.StorageSize = "20Gi" + c.Kubernetes.StorageClass = "fast-ssd" + c.Kubernetes.StorageAccessMode = "ReadWriteOnce" + }) + + err := env.EnsurePVC(context.Background()) + g.Assert(err).IsNil() + + pvc, err := client.CoreV1().PersistentVolumeClaims("pelican").Get(context.Background(), "gs-pvc-create-uuid", metav1.GetOptions{}) + g.Assert(err).IsNil() + g.Assert(pvc.Name).Equal("gs-pvc-create-uuid") + g.Assert(pvc.Labels["pelican.dev/server-id"]).Equal("pvc-create-uuid") + g.Assert(pvc.Spec.AccessModes[0]).Equal(corev1.ReadWriteOnce) + + // Check storage size. + storageReq := pvc.Spec.Resources.Requests[corev1.ResourceStorage] + expected := resource.MustParse("20Gi") + g.Assert(storageReq.Equal(expected)).IsTrue() + + // Check storage class. + g.Assert(*pvc.Spec.StorageClassName).Equal("fast-ssd") + }) + + g.It("should create PVC with ReadWriteMany access mode", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "rwm-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.StorageMode = config.KubeStoragePVC + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.StorageSize = "5Gi" + c.Kubernetes.StorageAccessMode = "ReadWriteMany" + c.Kubernetes.StorageClass = "" + }) + + err := env.EnsurePVC(context.Background()) + g.Assert(err).IsNil() + + pvc, _ := client.CoreV1().PersistentVolumeClaims("pelican").Get(context.Background(), "gs-rwm-uuid", metav1.GetOptions{}) + g.Assert(pvc.Spec.AccessModes[0]).Equal(corev1.ReadWriteMany) + // No storage class set — should be nil. + g.Assert(pvc.Spec.StorageClassName == nil).IsTrue() + }) + + g.It("should not recreate existing PVC", func() { + existingPVC := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gs-existing-uuid", + Namespace: "pelican", + }, + } + client := fake.NewSimpleClientset(existingPVC) + env := &Environment{ + Id: "existing-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.StorageMode = config.KubeStoragePVC + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.StorageSize = "10Gi" + }) + + err := env.EnsurePVC(context.Background()) + g.Assert(err).IsNil() + }) + + g.It("should error on invalid storage size", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "bad-size-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.StorageMode = config.KubeStoragePVC + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.StorageSize = "not-a-quantity" + }) + + err := env.EnsurePVC(context.Background()) + g.Assert(err).IsNotNil() + }) + }) + + g.Describe("DeletePVC", func() { + g.It("should be a no-op in hostpath mode", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "hostpath-del-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.StorageMode = config.KubeStorageHostPath + c.Kubernetes.Namespace = "pelican" + }) + + err := env.DeletePVC(context.Background()) + g.Assert(err).IsNil() + }) + + g.It("should delete an existing PVC", func() { + existingPVC := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gs-del-uuid", + Namespace: "pelican", + }, + } + client := fake.NewSimpleClientset(existingPVC) + env := &Environment{ + Id: "del-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.StorageMode = config.KubeStoragePVC + c.Kubernetes.Namespace = "pelican" + }) + + err := env.DeletePVC(context.Background()) + g.Assert(err).IsNil() + + _, err = client.CoreV1().PersistentVolumeClaims("pelican").Get(context.Background(), "gs-del-uuid", metav1.GetOptions{}) + g.Assert(err).IsNotNil() + }) + + g.It("should not error when PVC does not exist", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "nonexist-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.StorageMode = config.KubeStoragePVC + c.Kubernetes.Namespace = "pelican" + }) + + err := env.DeletePVC(context.Background()) + g.Assert(err).IsNil() + }) + }) + + g.Describe("ResizePVC", func() { + g.It("should be a no-op in hostpath mode", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "resize-hp-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.StorageMode = config.KubeStorageHostPath + c.Kubernetes.Namespace = "pelican" + }) + + err := env.ResizePVC(context.Background(), "50Gi") + g.Assert(err).IsNil() + }) + + g.It("should update PVC storage request", func() { + existingPVC := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gs-resize-uuid", + Namespace: "pelican", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("10Gi"), + }, + }, + }, + } + client := fake.NewSimpleClientset(existingPVC) + env := &Environment{ + Id: "resize-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.StorageMode = config.KubeStoragePVC + c.Kubernetes.Namespace = "pelican" + }) + + err := env.ResizePVC(context.Background(), "50Gi") + g.Assert(err).IsNil() + + pvc, _ := client.CoreV1().PersistentVolumeClaims("pelican").Get(context.Background(), "gs-resize-uuid", metav1.GetOptions{}) + storageReq := pvc.Spec.Resources.Requests[corev1.ResourceStorage] + expected := resource.MustParse("50Gi") + g.Assert(storageReq.Equal(expected)).IsTrue() + }) + + g.It("should error on invalid size", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "bad-resize-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.StorageMode = config.KubeStoragePVC + c.Kubernetes.Namespace = "pelican" + }) + + err := env.ResizePVC(context.Background(), "invalid") + g.Assert(err).IsNotNil() + }) + }) + + g.Describe("buildVolumes with PVC mode", func() { + g.It("should use PVC for default mount in pvc mode", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "bv-pvc-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.StorageMode = config.KubeStoragePVC + c.Kubernetes.Namespace = "pelican" + }) + + mounts := []environment.Mount{ + {Default: true, Source: "/data/servers/bv-pvc-uuid", Target: "/home/container"}, + {Default: false, Source: "/shared/plugins", Target: "/plugins", ReadOnly: true}, + } + + volumes, volumeMounts := env.buildVolumes(mounts) + + g.Assert(len(volumes)).Equal(2) + // First volume (default) should use PVC. + g.Assert(volumes[0].Name).Equal("server-data") + g.Assert(volumes[0].PersistentVolumeClaim).IsNotNil() + g.Assert(volumes[0].PersistentVolumeClaim.ClaimName).Equal("gs-bv-pvc-uuid") + g.Assert(volumes[0].HostPath == nil).IsTrue() + + // Second volume (non-default) should still use HostPath. + g.Assert(volumes[1].Name).Equal("mount-1") + g.Assert(volumes[1].HostPath).IsNotNil() + g.Assert(volumes[1].HostPath.Path).Equal("/shared/plugins") + + g.Assert(len(volumeMounts)).Equal(2) + g.Assert(volumeMounts[0].MountPath).Equal("/home/container") + g.Assert(volumeMounts[1].MountPath).Equal("/plugins") + g.Assert(volumeMounts[1].ReadOnly).IsTrue() + }) + + g.It("should use HostPath for default mount in hostpath mode", func() { + client := fake.NewSimpleClientset() + env := &Environment{ + Id: "bv-hp-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.StorageMode = config.KubeStorageHostPath + c.Kubernetes.Namespace = "pelican" + }) + + mounts := []environment.Mount{ + {Default: true, Source: "/data/servers/test", Target: "/home/container"}, + } + + volumes, _ := env.buildVolumes(mounts) + g.Assert(volumes[0].HostPath).IsNotNil() + g.Assert(volumes[0].HostPath.Path).Equal("/data/servers/test") + g.Assert(volumes[0].PersistentVolumeClaim == nil).IsTrue() + }) + }) + }) +} diff --git a/environment/kubernetes/testutil_test.go b/environment/kubernetes/testutil_test.go new file mode 100644 index 00000000..26ee28b3 --- /dev/null +++ b/environment/kubernetes/testutil_test.go @@ -0,0 +1,21 @@ +package kubernetes + +import ( + "os" + "testing" + + "github.com/pelican-dev/wings/config" +) + +func TestMain(m *testing.M) { + // Initialize a minimal config so config.Get() doesn't panic. + c := &config.Configuration{} + c.AuthenticationToken = "test-token-for-testing" + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.NetworkMode = config.KubeNetworkHostPort + c.Kubernetes.NodePortRangeMin = 30000 + c.Kubernetes.NodePortRangeMax = 32767 + config.Set(c) + + os.Exit(m.Run()) +} diff --git a/go.mod b/go.mod index a0ff6df6..c3d412f1 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/pelican-dev/wings -go 1.25.0 +go 1.26.0 require ( emperror.dev/errors v0.8.1 @@ -9,11 +9,11 @@ require ( github.com/apex/log v1.9.0 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 github.com/beevik/etree v1.6.0 - github.com/buger/jsonparser v1.1.2 + github.com/buger/jsonparser v1.2.0 github.com/cenkalti/backoff/v4 v4.3.0 github.com/creasty/defaults v1.8.0 github.com/docker/docker v28.5.2+incompatible - github.com/docker/go-connections v0.6.0 + github.com/docker/go-connections v0.7.0 github.com/fatih/color v1.19.0 github.com/franela/goblin v0.0.0-20211003143422-0a4f594942bf github.com/gabriel-vasile/mimetype v1.4.13 @@ -21,14 +21,14 @@ require ( github.com/gbrlsnchs/jwt/v3 v3.0.1 github.com/gin-gonic/gin v1.12.0 github.com/glebarez/sqlite v1.11.0 - github.com/go-co-op/gocron/v2 v2.19.1 + github.com/go-co-op/gocron/v2 v2.21.2 github.com/goccy/go-json v0.10.6 github.com/google/uuid v1.6.0 - github.com/gorilla/websocket v1.5.3 + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 github.com/iancoleman/strcase v0.3.0 github.com/icza/dyno v0.0.0-20230330125955-09f820a8d9c0 github.com/juju/ratelimit v1.0.2 - github.com/klauspost/compress v1.18.5 + github.com/klauspost/compress v1.18.6 github.com/klauspost/pgzip v1.2.6 github.com/magiconair/properties v1.8.10 github.com/mattn/go-colorable v0.1.14 @@ -40,80 +40,114 @@ require ( github.com/shirou/gopsutil/v3 v3.24.5 github.com/spf13/cobra v1.10.2 github.com/stretchr/testify v1.11.1 - github.com/tidwall/gjson v1.18.0 - github.com/tidwall/pretty v1.2.0 + github.com/tidwall/gjson v1.19.0 + github.com/tidwall/pretty v1.2.1 github.com/tidwall/sjson v1.2.5 - golang.org/x/crypto v0.49.0 + golang.org/x/crypto v0.52.0 golang.org/x/sync v0.20.0 - golang.org/x/sys v0.42.0 - gopkg.in/ini.v1 v1.67.1 + golang.org/x/sys v0.45.0 + gopkg.in/ini.v1 v1.67.2 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 gorm.io/gorm v1.31.1 + k8s.io/api v0.36.1 + k8s.io/apimachinery v0.36.1 + k8s.io/client-go v0.36.1 ) require ( github.com/atotto/clipboard v0.1.4 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect - github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/gopkg v0.1.4 // indirect github.com/catppuccin/go v0.3.0 // indirect - github.com/charmbracelet/bubbles v0.21.0 // indirect - github.com/charmbracelet/bubbletea v1.3.4 // indirect - github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/charmbracelet/bubbles v1.0.0 // indirect + github.com/charmbracelet/bubbletea v1.3.10 // indirect + github.com/charmbracelet/colorprofile v0.4.3 // indirect github.com/charmbracelet/lipgloss v1.1.0 // indirect - github.com/charmbracelet/x/ansi v0.8.0 // indirect - github.com/charmbracelet/x/cellbuf v0.0.13 // indirect - github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0 // indirect - github.com/charmbracelet/x/term v0.2.1 // indirect + github.com/charmbracelet/x/ansi v0.11.7 // indirect + github.com/charmbracelet/x/cellbuf v0.0.15 // indirect + github.com/charmbracelet/x/exp/strings v0.1.0 // indirect + github.com/charmbracelet/x/term v0.2.2 // indirect + github.com/clipperhouse/displaywidth v0.11.0 // indirect + github.com/clipperhouse/uax29/v2 v2.7.0 // indirect + github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect github.com/goccy/go-yaml v1.19.2 // indirect - github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/lucasb-eyer/go-colorful v1.4.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-localereader v0.0.1 // indirect - github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/mattn/go-runewidth v0.0.23 // indirect github.com/mikelolasagasti/xz v1.0.1 // indirect github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect + github.com/moby/spdystream v0.5.1 // indirect github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.16.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/quic-go/qpack v0.6.0 // indirect - github.com/quic-go/quic-go v0.59.0 // indirect + github.com/quic-go/quic-go v0.59.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/spf13/afero v1.15.0 // indirect - github.com/tidwall/match v1.1.1 // indirect + github.com/stangelandcl/ppmd v0.1.1 // indirect + github.com/tidwall/match v1.2.0 // indirect + github.com/x448/float16 v0.8.4 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect - go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect + go.mongodb.org/mongo-driver/v2 v2.6.0 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect + golang.org/x/oauth2 v0.35.0 // indirect + golang.org/x/term v0.43.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260511170946-3700d4141b60 // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + k8s.io/klog/v2 v2.140.0 // indirect + k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect + k8s.io/streaming v0.36.1 // indirect + k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect ) require ( - github.com/Microsoft/go-winio v0.6.1 // indirect - github.com/Microsoft/hcsshim v0.12.2 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/Microsoft/hcsshim v0.14.1 // indirect github.com/STARRY-S/zip v0.2.3 // indirect - github.com/andybalholm/brotli v1.2.0 // indirect + github.com/andybalholm/brotli v1.2.1 // indirect github.com/bodgit/plumbing v1.3.0 // indirect - github.com/bodgit/sevenzip v1.6.1 // indirect + github.com/bodgit/sevenzip v1.6.4 // indirect github.com/bodgit/windows v1.0.1 // indirect - github.com/bytedance/sonic v1.15.0 // indirect - github.com/bytedance/sonic/loader v0.5.0 // indirect - github.com/charmbracelet/huh v0.7.0 - github.com/cloudwego/base64x v0.1.6 // indirect - github.com/containerd/errdefs v0.3.0 // indirect + github.com/bytedance/sonic v1.15.1 // indirect + github.com/bytedance/sonic/loader v0.5.1 // indirect + github.com/charmbracelet/huh v1.0.0 + github.com/cloudwego/base64x v0.1.7 // indirect + github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/gammazero/deque v1.2.1 // indirect - github.com/gin-contrib/sse v1.1.0 // indirect + github.com/gin-contrib/sse v1.1.1 // indirect github.com/glebarez/go-sqlite v1.22.0 // indirect - github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-ole/go-ole v1.2.6 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.30.1 // indirect + github.com/go-playground/validator/v10 v10.30.2 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect @@ -123,58 +157,56 @@ require ( github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/kr/fs v0.1.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect - github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect - github.com/magefile/mage v1.15.0 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect - github.com/minio/minlz v1.0.1 // indirect + github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e // indirect + github.com/magefile/mage v1.17.2 // indirect + github.com/mattn/go-isatty v0.0.22 // indirect + github.com/minio/minlz v1.1.1 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/sys/atomicwriter v0.1.0 // indirect - github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae // indirect + github.com/moby/term v0.5.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/morikuni/aec v1.0.0 // indirect - github.com/ncruces/go-strftime v0.1.9 // indirect - github.com/nwaples/rardecode/v2 v2.2.0 // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/morikuni/aec v1.1.0 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/nwaples/rardecode/v2 v2.2.2 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0 // indirect - github.com/pelletier/go-toml/v2 v2.2.4 - github.com/pierrec/lz4/v4 v4.1.22 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/pelletier/go-toml/v2 v2.3.1 + github.com/pierrec/lz4/v4 v4.1.26 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/robfig/cron/v3 v3.0.1 // indirect - github.com/shoenig/go-m1cpu v0.1.6 // indirect - github.com/sirupsen/logrus v1.9.3 // indirect + github.com/shoenig/go-m1cpu v0.2.1 // indirect + github.com/sirupsen/logrus v1.9.4 // indirect github.com/sorairolake/lzip-go v0.3.8 // indirect - github.com/spf13/pflag v1.0.9 // indirect - github.com/tklauser/go-sysconf v0.3.12 // indirect - github.com/tklauser/numcpus v0.6.1 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/tklauser/go-sysconf v0.4.0 // indirect + github.com/tklauser/numcpus v0.12.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.3.1 // indirect github.com/ulikunitz/xz v0.5.15 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect - go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.50.0 // indirect - go.opentelemetry.io/otel v1.34.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0 // indirect - go.opentelemetry.io/otel/metric v1.34.0 // indirect - go.opentelemetry.io/otel/sdk v1.24.0 // indirect - go.opentelemetry.io/otel/trace v1.34.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect + go.opentelemetry.io/otel v1.43.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/trace v1.43.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/arch v0.22.0 // indirect - golang.org/x/mod v0.33.0 // indirect - golang.org/x/net v0.51.0 // indirect - golang.org/x/text v0.35.0 // indirect - golang.org/x/time v0.0.0-20220922220347-f3bd1da661af - golang.org/x/tools v0.42.0 // indirect - golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect - google.golang.org/protobuf v1.36.10 // indirect - gotest.tools/v3 v3.0.2 // indirect - modernc.org/libc v1.49.3 // indirect - modernc.org/mathutil v1.6.0 // indirect - modernc.org/memory v1.8.0 // indirect - modernc.org/sqlite v1.29.6 // indirect + go4.org v0.0.0-20260112195520-a5071408f32f // indirect + golang.org/x/arch v0.27.0 // indirect + golang.org/x/net v0.55.0 // indirect + golang.org/x/text v0.37.0 // indirect + golang.org/x/time v0.15.0 + golang.org/x/tools v0.45.0 // indirect + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260523011958-0a33c5d7ca68 // indirect + google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect + gotest.tools/v3 v3.5.2 // indirect + modernc.org/libc v1.72.3 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect + modernc.org/sqlite v1.50.1 // indirect ) diff --git a/go.sum b/go.sum index bcd9c167..2ef35e99 100644 --- a/go.sum +++ b/go.sum @@ -1,45 +1,28 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= emperror.dev/errors v0.8.1 h1:UavXZ5cSX/4u9iyvH6aDcuGkVjeexUGJ7Ij7G4VfQT0= emperror.dev/errors v0.8.1/go.mod h1:YcRvLPh626Ubn2xqtoprejnA5nFha+TJ+2vew48kWuE= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= -github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= -github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/Microsoft/hcsshim v0.12.2 h1:AcXy+yfRvrx20g9v7qYaJv5Rh+8GaHOS6b8G6Wx/nKs= -github.com/Microsoft/hcsshim v0.12.2/go.mod h1:RZV12pcHCXQ42XnlQ3pz6FZfmrC1C+R4gaOHhRNML1g= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/Microsoft/hcsshim v0.14.1 h1:CMuB3fqQVfPdhyXhUqYdUmPUIOhJkmghCx3dJet8Cqs= +github.com/Microsoft/hcsshim v0.14.1/go.mod h1:VnzvPLyWUhxiPVsJ31P6XadxCcTogTguBFDy/1GR/OM= github.com/NYTimes/logrotate v1.0.0 h1:6jFGbon6jOtpy3t3kwZZKS4Gdmf1C/Wv5J4ll4Xn5yk= github.com/NYTimes/logrotate v1.0.0/go.mod h1:GxNz1cSw1c6t99PXoZlw+nm90H6cyQyrH66pjVv7x88= github.com/STARRY-S/zip v0.2.3 h1:luE4dMvRPDOWQdeDdUxUoZkzUIpTccdKdhHHsQJ1fm4= github.com/STARRY-S/zip v0.2.3/go.mod h1:lqJ9JdeRipyOQJrYSOtpNAiaesFO6zVDsE8GIGFaoSk= github.com/acobaugh/osrelease v0.1.0 h1:Yb59HQDGGNhCj4suHaFQQfBps5wyoKLSSX/J/+UifRE= github.com/acobaugh/osrelease v0.1.0/go.mod h1:4bFEs0MtgHNHBrmHCt67gNisnabCRAlzdVasCEGHTWY= -github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= -github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro= +github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA= github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo= github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE= github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= @@ -48,84 +31,88 @@ github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= -github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8= -github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA= +github.com/aymanbagabas/go-udiff v0.3.1 h1:LV+qyBQ2pqe0u42ZsUEtPiCaUoqgA9gYRDs3vj1nolY= +github.com/aymanbagabas/go-udiff v0.3.1/go.mod h1:G0fsKmG+P6ylD0r6N/KgQD/nWzgfnl8ZBcNLgcbrw8E= github.com/beevik/etree v1.6.0 h1:u8Kwy8pp9D9XeITj2Z0XtA5qqZEmtJtuXZRQi+j03eE= github.com/beevik/etree v1.6.0/go.mod h1:bh4zJxiIr62SOf9pRzN7UUYaEDa9HEKafK25+sLc0Gc= github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU= github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs= -github.com/bodgit/sevenzip v1.6.1 h1:kikg2pUMYC9ljU7W9SaqHXhym5HyKm8/M/jd31fYan4= -github.com/bodgit/sevenzip v1.6.1/go.mod h1:GVoYQbEVbOGT8n2pfqCIMRUaRjQ8F9oSqoBEqZh5fQ8= +github.com/bodgit/sevenzip v1.6.4 h1:iHiVJfxbrB6RF4X+snI2MpVgNBKmVfGaTqZGNlMQIU0= +github.com/bodgit/sevenzip v1.6.4/go.mod h1:ZtNi5KNgHXeXg1G7WiF0IWSuFE2eG6lt/cTGlvuirO0= github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4= github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM= -github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk= -github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= -github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= -github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= -github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= -github.com/bytedance/sonic v1.15.0/go.mod h1:tFkWrPz0/CUCLEF4ri4UkHekCIcdnkqXw9VduqpJh0k= -github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiDd5VjlgE= -github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= +github.com/buger/jsonparser v1.2.0 h1:4EFcvK1kD4jyj6YqNK6skK6w+y7FHHBR+XBCtxwu/6g= +github.com/buger/jsonparser v1.2.0/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/bytedance/gopkg v0.1.4 h1:oZnQwnX82KAIWb7033bEwtxvTqXcYMxDBaQxo5JJHWM= +github.com/bytedance/gopkg v0.1.4/go.mod h1:v1zWfPm21Fb+OsyXN2VAHdL6TBb2L88anLQgdyje6R4= +github.com/bytedance/sonic v1.15.1 h1:nJD5PmM0vY7J8CT6MxoqbVAAMhkSmV2HgRAUrrpLoOw= +github.com/bytedance/sonic v1.15.1/go.mod h1:mT2NbXunuaEbnZ+mRIX/vYqKISmgEuHFDI4UzmKx2SA= +github.com/bytedance/sonic/loader v0.5.1 h1:Ygpfa9zwRCCKSlrp5bBP/b/Xzc3VxsAW+5NIYXrOOpI= +github.com/bytedance/sonic/loader v0.5.1/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= github.com/catppuccin/go v0.3.0 h1:d+0/YicIq+hSTo5oPuRi5kOpqkVA5tAsU6dNhvRu+aY= github.com/catppuccin/go v0.3.0/go.mod h1:8IHJuMGaUUjQM82qBrGNBv7LFq6JI3NnQCF6MOlZjpc= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/charmbracelet/bubbles v0.21.0 h1:9TdC97SdRVg/1aaXNVWfFH3nnLAwOXr8Fn6u6mfQdFs= -github.com/charmbracelet/bubbles v0.21.0/go.mod h1:HF+v6QUR4HkEpz62dx7ym2xc71/KBHg+zKwJtMw+qtg= -github.com/charmbracelet/bubbletea v1.3.4 h1:kCg7B+jSCFPLYRA52SDZjr51kG/fMUEoPoZrkaDHyoI= -github.com/charmbracelet/bubbletea v1.3.4/go.mod h1:dtcUCyCGEX3g9tosuYiut3MXgY/Jsv9nKVdibKKRRXo= -github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc h1:4pZI35227imm7yK2bGPcfpFEmuY1gc2YSTShr4iJBfs= -github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc/go.mod h1:X4/0JoqgTIPSFcRA/P6INZzIuyqdFY5rm8tb41s9okk= -github.com/charmbracelet/huh v0.7.0 h1:W8S1uyGETgj9Tuda3/JdVkc3x7DBLZYPZc4c+/rnRdc= -github.com/charmbracelet/huh v0.7.0/go.mod h1:UGC3DZHlgOKHvHC07a5vHag41zzhpPFj34U92sOmyuk= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/charmbracelet/bubbles v1.0.0 h1:12J8/ak/uCZEMQ6KU7pcfwceyjLlWsDLAxB5fXonfvc= +github.com/charmbracelet/bubbles v1.0.0/go.mod h1:9d/Zd5GdnauMI5ivUIVisuEm3ave1XwXtD1ckyV6r3E= +github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw= +github.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4= +github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex9t5KX76i20Q= +github.com/charmbracelet/colorprofile v0.4.3/go.mod h1:/zT4BhpD5aGFpqQQqw7a+VtHCzu+zrQtt1zhMt9mR4Q= +github.com/charmbracelet/huh v1.0.0 h1:wOnedH8G4qzJbmhftTqrpppyqHakl/zbbNdXIWJyIxw= +github.com/charmbracelet/huh v1.0.0/go.mod h1:5YVc+SlZ1IhQALxRPpkGwwEKftN/+OlJlnJYlDRFqN4= github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY= github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30= -github.com/charmbracelet/x/ansi v0.8.0 h1:9GTq3xq9caJW8ZrBTe0LIe2fvfLR/bYXKTx2llXn7xE= -github.com/charmbracelet/x/ansi v0.8.0/go.mod h1:wdYl/ONOLHLIVmQaxbIYEC/cRKOQyjTkowiI4blgS9Q= -github.com/charmbracelet/x/cellbuf v0.0.13 h1:/KBBKHuVRbq1lYx5BzEHBAFBP8VcQzJejZ/IA3iR28k= -github.com/charmbracelet/x/cellbuf v0.0.13/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs= +github.com/charmbracelet/x/ansi v0.11.7 h1:kzv1kJvjg2S3r9KHo8hDdHFQLEqn4RBCb39dAYC84jI= +github.com/charmbracelet/x/ansi v0.11.7/go.mod h1:9qGpnAVYz+8ACONkZBUWPtL7lulP9No6p1epAihUZwQ= +github.com/charmbracelet/x/cellbuf v0.0.15 h1:ur3pZy0o6z/R7EylET877CBxaiE1Sp1GMxoFPAIztPI= +github.com/charmbracelet/x/cellbuf v0.0.15/go.mod h1:J1YVbR7MUuEGIFPCaaZ96KDl5NoS0DAWkskup+mOY+Q= github.com/charmbracelet/x/conpty v0.1.0 h1:4zc8KaIcbiL4mghEON8D72agYtSeIgq8FSThSPQIb+U= github.com/charmbracelet/x/conpty v0.1.0/go.mod h1:rMFsDJoDwVmiYM10aD4bH2XiRgwI7NYJtQgl5yskjEQ= github.com/charmbracelet/x/errors v0.0.0-20240508181413-e8d8b6e2de86 h1:JSt3B+U9iqk37QUU2Rvb6DSBYRLtWqFqfxf8l5hOZUA= github.com/charmbracelet/x/errors v0.0.0-20240508181413-e8d8b6e2de86/go.mod h1:2P0UgXMEa6TsToMSuFqKFQR+fZTO9CNGUNokkPatT/0= github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91 h1:payRxjMjKgx2PaCWLZ4p3ro9y97+TVLZNaRZgJwSVDQ= github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= -github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0 h1:qko3AQ4gK1MTS/de7F5hPGx6/k1u0w4TeYmBFwzYVP4= -github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0/go.mod h1:pBhA0ybfXv6hDjQUZ7hk1lVxBiUbupdw5R31yPUViVQ= -github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ= -github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= +github.com/charmbracelet/x/exp/strings v0.1.0 h1:i69S2XI7uG1u4NLGeJPSYU++Nmjvpo9nwd6aoEm7gkA= +github.com/charmbracelet/x/exp/strings v0.1.0/go.mod h1:/ehtMPNh9K4odGFkqYJKpIYyePhdp1hLBRvyY4bWkH8= +github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk= +github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI= github.com/charmbracelet/x/termios v0.1.1 h1:o3Q2bT8eqzGnGPOYheoYS8eEleT5ZVNYNy8JawjaNZY= github.com/charmbracelet/x/termios v0.1.1/go.mod h1:rB7fnv1TgOPOyyKRJ9o+AsTU/vK5WHJ2ivHeut/Pcwo= github.com/charmbracelet/x/xpty v0.1.2 h1:Pqmu4TEJ8KeA9uSkISKMU3f+C1F6OGBn8ABuGlqCbtI= github.com/charmbracelet/x/xpty v0.1.2/go.mod h1:XK2Z0id5rtLWcpeNiMYBccNNBrP2IJnzHI0Lq13Xzq4= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= -github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= -github.com/containerd/errdefs v0.3.0 h1:FSZgGOeK4yuT/+DnF07/Olde/q4KBoMsaamhXxIMDp4= -github.com/containerd/errdefs v0.3.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSEFgwIwO+UVM8= +github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0= +github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= +github.com/cloudwego/base64x v0.1.7 h1:NppS+Fgzg5ovhn4NkUXaDT3x9jldgH5ToMCqzBSi2zI= +github.com/cloudwego/base64x v0.1.7/go.mod h1:Cu1PV9zfrSf7ET2tIbWbbEy7jO7HHJ13q4X2SQ8aWYg= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= -github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/creasty/defaults v1.8.0 h1:z27FJxCAa0JKt3utc0sCImAEb+spPucmKoOdLHvHYKk= github.com/creasty/defaults v1.8.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM= github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= -github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= +github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c= +github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4= @@ -133,8 +120,8 @@ github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj6 github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= +github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -145,6 +132,8 @@ github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSw github.com/franela/goblin v0.0.0-20211003143422-0a4f594942bf h1:NrF81UtW8gG2LBGkXFQFqlfNnvMt9WdB46sfdJY4oqc= github.com/franela/goblin v0.0.0-20211003143422-0a4f594942bf/go.mod h1:VzmDKDJVZI3aJmnRI9VjAn9nJ8qPPsN1fqzr9dqInIo= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM= github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= github.com/gammazero/deque v1.2.1 h1:9fnQVFCCZ9/NOc7ccTNqzoKd1tCWOqeI05/lPqFPMGQ= @@ -153,85 +142,68 @@ github.com/gammazero/workerpool v1.2.1 h1:MEDvUJsNYGuCvl1RwIXNKu2YtQtHqCSF9XWF04 github.com/gammazero/workerpool v1.2.1/go.mod h1:E32GVRUanF4d6QtRmdss3AScgaDkIyrvPtgRQUWgmx4= github.com/gbrlsnchs/jwt/v3 v3.0.1 h1:lbUmgAKpxnClrKloyIwpxm4OuWeDl5wLk52G91ODPw4= github.com/gbrlsnchs/jwt/v3 v3.0.1/go.mod h1:AncDcjXz18xetI3A6STfXq2w+LuTx8pQ8bGEwRN8zVM= -github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= -github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= +github.com/gin-contrib/sse v1.1.1 h1:uGYpNwTacv5R68bSGMapo62iLTRa9l5zxGCps4hK6ko= +github.com/gin-contrib/sse v1.1.1/go.mod h1:QXzuVkA0YO7o/gun03UI1Q+FTI8ZV/n5t03kIQAI89s= github.com/gin-gonic/gin v1.12.0 h1:b3YAbrZtnf8N//yjKeU2+MQsh2mY5htkZidOM7O0wG8= github.com/gin-gonic/gin v1.12.0/go.mod h1:VxccKfsSllpKshkBWgVgRniFFAzFb9csfngsqANjnLc= github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ= github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc= github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw= github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ= -github.com/go-co-op/gocron/v2 v2.19.1 h1:B4iLeA0NB/2iO3EKQ7NfKn5KsQgZfjb2fkvoZJU3yBI= -github.com/go-co-op/gocron/v2 v2.19.1/go.mod h1:5lEiCKk1oVJV39Zg7/YG10OnaVrDAV5GGR6O0663k6U= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-co-op/gocron/v2 v2.21.2 h1:bD8/YwkojYHgXFr3iEulL148KBdTbKVxUZzFKpXcdbY= +github.com/go-co-op/gocron/v2 v2.21.2/go.mod h1:5lEiCKk1oVJV39Zg7/YG10OnaVrDAV5GGR6O0663k6U= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w= -github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM= +github.com/go-playground/validator/v10 v10.30.2 h1:JiFIMtSSHb2/XBUbWM4i/MpeQm9ZK2xqPNk8vgvu5JQ= +github.com/go-playground/validator/v10 v10.30.2/go.mod h1:mAf2pIOVXjTEBrwUMGKkCWKKPs9NheYGabeB04txQSc= github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= -github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= -github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 h1:Wqo399gCIufwto+VfwCSvsnfGpF/w5E9CNxSwbpD6No= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0/go.mod h1:qmOFXW2epJhM0qSnUUYpldc7gVz2KMQwJ/QYCDIa7XU= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI= github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/icza/dyno v0.0.0-20230330125955-09f820a8d9c0 h1:nHoRIX8iXob3Y2kdt9KsjyIb7iApSvb3vgsd93xb5Ow= github.com/icza/dyno v0.0.0-20230330125955-09f820a8d9c0/go.mod h1:c1tRKs5Tx7E2+uHGSyyncziFjvGpgv4H2HrqXeUQ/Uk= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -243,17 +215,16 @@ github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/juju/ratelimit v1.0.2 h1:sRxmtRiajbvrcLQT7S+JbqU0ntsb9W2yhSdNN8tWfaI= github.com/juju/ratelimit v1.0.2/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= -github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= @@ -262,8 +233,8 @@ github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQ github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -272,113 +243,117 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= -github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= -github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= -github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= -github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4= +github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e h1:Q6MvJtQK/iRcRtzAscm/zF23XxJlbECiGPyRicsX+Ak= +github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= github.com/magefile/mage v1.9.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= -github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= -github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/magefile/mage v1.17.2 h1:fyXVu1eadI8Ap1HCCNgEhJ5McIWiYhLR8uol64ZZc40= +github.com/magefile/mage v1.17.2/go.mod h1:Yj51kqllmsgFpvvSzgrZPK9WtluG3kUhFaBUVLo4feA= github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= +github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= -github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= -github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw= +github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mholt/archives v0.1.5 h1:Fh2hl1j7VEhc6DZs2DLMgiBNChUux154a1G+2esNvzQ= github.com/mholt/archives v0.1.5/go.mod h1:3TPMmBLPsgszL+1As5zECTuKwKvIfj6YcwWPpeTAXF4= github.com/mikelolasagasti/xz v1.0.1 h1:Q2F2jX0RYJUG3+WsM+FJknv+6eVjsjXNDV0KJXZzkD0= github.com/mikelolasagasti/xz v1.0.1/go.mod h1:muAirjiOUxPRXwm9HdDtB3uoRPrGnL85XHtokL9Hcgc= -github.com/minio/minlz v1.0.1 h1:OUZUzXcib8diiX+JYxyRLIdomyZYzHct6EShOKtQY2A= -github.com/minio/minlz v1.0.1/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec= +github.com/minio/minlz v1.1.1 h1:OGmft1V6AnI/Wme332U6bhG54nxEan+VFgkD7lat4KM= +github.com/minio/minlz v1.1.1/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/spdystream v0.5.1 h1:9sNYeYZUcci9R6/w7KDaFWEWeV4LStVG78Mpyq/Zm/Y= +github.com/moby/spdystream v0.5.1/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= -github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae h1:O4SWKdcHVCvYqyDV+9CJA1fcDN2L11Bule0iFy3YlAI= -github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/morikuni/aec v1.1.0 h1:vBBl0pUnvi/Je71dsRrhMBtreIqNMYErSAbEeb8jrXQ= +github.com/morikuni/aec v1.1.0/go.mod h1:xDRgiq/iw5l+zkao76YTKzKttOp2cwPEne25HDkJnBw= github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo= github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= -github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= -github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/nwaples/rardecode/v2 v2.2.0 h1:4ufPGHiNe1rYJxYfehALLjup4Ls3ck42CWwjKiOqu0A= -github.com/nwaples/rardecode/v2 v2.2.0/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/nwaples/rardecode/v2 v2.2.2 h1:/5oL8dzYivRM/tqX9VcTSWfbpwcbwKG1QtSJr3b3KcU= +github.com/nwaples/rardecode/v2 v2.2.2/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= -github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= -github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= -github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= -github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= -github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pelletier/go-toml/v2 v2.3.1 h1:MYEvvGnQjeNkRF1qUuGolNtNExTDwct51yp7olPtrEc= +github.com/pelletier/go-toml/v2 v2.3.1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= +github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.10 h1:+5FbKNTe5Z9aspU88DPIKJ9z2KZoaGCu6Sr6kKR/5mU= github.com/pkg/sftp v1.13.10/go.mod h1:bJ1a7uDhrX/4OII+agvy28lzRvQrmIQuaHrcI1HbeGA= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= -github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= -github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= -github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= +github.com/quic-go/quic-go v0.59.1 h1:0Gmua0HW1Tv7ANR7hUYwRyD0MG5OJfgvYSZasGZzBic= +github.com/quic-go/quic-go v0.59.1/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI= github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI= github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk= -github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= -github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= -github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= -github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/shoenig/go-m1cpu v0.2.1 h1:yqRB4fvOge2+FyRXFkXqsyMoqPazv14Yyy+iyccT2E4= +github.com/shoenig/go-m1cpu v0.2.1/go.mod h1:KkDOw6m3ZJQAPHbrzkZki4hnx+pDRR1Lo+ldA56wD5w= +github.com/shoenig/test v1.7.0 h1:eWcHtTXa6QLnBvm0jgEabMRN/uJ4DMV3M8xUGgRkZmk= +github.com/shoenig/test v1.7.0/go.mod h1:UxJ6u/x2v/TNs/LoLxBNJRV9DiwBBKYxXSyczsBHFoI= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= @@ -388,18 +363,18 @@ github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stangelandcl/ppmd v0.1.1 h1:c25QazhlWUn5nmR1QOzafKhQxBicAr7GGCKER2aJ8H8= +github.com/stangelandcl/ppmd v0.1.1/go.mod h1:Rrv7M+/2P5jYr/GMLhBl7Ug3uJ1bUiVzr5LbbaV6xgY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -408,12 +383,14 @@ github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= -github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/gjson v1.19.0 h1:xwxm7n691Uf3u5OFjzngavjGTh55KX5q/9w9xHW88JU= +github.com/tidwall/gjson v1.19.0/go.mod h1:V37/opeE/JbLUOfH0QTXiNez2l0RUjYUhpT4szFQAfc= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= -github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/match v1.2.0 h1:0pt8FlkOwjN2fPt4bIl4BoNxb98gGHN2ObFEDkrfZnM= +github.com/tidwall/match v1.2.0/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= @@ -423,10 +400,10 @@ github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= -github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= -github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= -github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= -github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU= +github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI= +github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4= +github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY= @@ -434,37 +411,36 @@ github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2W github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY= github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF3xaE= -go.mongodb.org/mongo-driver/v2 v2.5.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.50.0 h1:cEPbyTSEHlQR89XVlyo78gqluF8Y3oMeBkXGWzQsfXY= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.50.0/go.mod h1:DKdbWcT4GH1D0Y3Sqt/PFXt2naRKDWtU+eE6oLdFNA8= -go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= -go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 h1:t6wl9SPayj+c7lEIFgm4ooDBZVb01IhLB4InpomhRw8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0/go.mod h1:iSDOcsnSA5INXzZtwaBPrKp/lWu/V14Dd+llD0oI2EA= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0 h1:Xw8U6u2f8DK2XAkGRFV7BBLENgnTGX9i4rQRxJf+/vs= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0/go.mod h1:6KW1Fm6R/s6Z3PGXwSJN2K4eT6wQB3vXX6CVnYX9NmM= -go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= -go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= -go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= -go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= -go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= -go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= -go.opentelemetry.io/proto/otlp v1.1.0 h1:2Di21piLrCqJ3U3eXGCTPHE9R8Nh+0uglSnOyxikMeI= -go.opentelemetry.io/proto/otlp v1.1.0/go.mod h1:GpBHCBWiqvVLDqmHZsoMM3C5ySeKTC7ej/RNTae6MdY= +go.mongodb.org/mongo-driver/v2 v2.6.0 h1:b9sJOYrkmt4l8bY43ZenFBcPlhYIjaOfYHLtbB/5qi8= +go.mongodb.org/mongo-driver/v2 v2.6.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 h1:CqXxU8VOmDefoh0+ztfGaymYbhdB/tT3zs79QaZTNGY= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0/go.mod h1:BuhAPThV8PBHBvg8ZzZ/Ok3idOdhWIodywz2xEcRbJo= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJKlCZufyRzPzlQhUIWVmfltrXuGyfjREgGP3UUjc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= +go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= @@ -473,215 +449,82 @@ go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= -go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= -golang.org/x/arch v0.22.0 h1:c/Zle32i5ttqRXjdLyyHZESLD/bB90DCU1g9l/0YBDI= -golang.org/x/arch v0.22.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= +go4.org v0.0.0-20260112195520-a5071408f32f h1:ziUVAjmTPwQMBmYR1tbdRFJPtTcQUI12fH9QQjfb0Sw= +go4.org v0.0.0-20260112195520-a5071408f32f/go.mod h1:ZRJnO5ZI4zAwMFp+dS1+V6J6MSyAowhRqAE+DPa1Xp0= +golang.org/x/arch v0.27.0 h1:0WNVcR8u9yFz8j5FvdHpgwNp3FS5U4guYdzHwEiGjoU= +golang.org/x/arch v0.27.0/go.mod h1:0X+GdSIP+kL5wPmpK7sdkEVTt2XoYP0cSjQSbZBwOi8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= -golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= +golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= +golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo= +golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= +golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= -golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= +golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= -golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= -golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= +golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= -golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y= -golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= -golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ= -google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 h1:wKguEg1hsxI2/L3hUYrpo1RVi48K+uTyzKqprwLXsb8= -google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142/go.mod h1:d6be+8HhtEtucleCbxpPW9PA9XwISACu8nvpPqF0BVo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.67.0 h1:IdH9y6PF5MPSdAntIcpjQ+tXO41pcQsfZV2RxtQgVcw= -google.golang.org/grpc v1.67.0/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= -google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= -google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +google.golang.org/genproto/googleapis/api v0.0.0-20260523011958-0a33c5d7ca68 h1:WVVw1Nl19li0fMX++FJ3ye1z9+S1N35QODDy5qpnaXw= +google.golang.org/genproto/googleapis/api v0.0.0-20260523011958-0a33c5d7ca68/go.mod h1:1dCETSCY2YKZNXQE3h4fun3TYwF5p8jejRKZgfWAgAY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260511170946-3700d4141b60 h1:seT2EwLWM78plQ7wcDfuWBc/4FAEAXDDiaSol4ku4qo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260511170946-3700d4141b60/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= +google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/ini.v1 v1.67.1 h1:tVBILHy0R6e4wkYOn3XmiITt/hEVH4TFMYvAX2Ytz6k= -gopkg.in/ini.v1 v1.67.1/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.67.2 h1:JtOSMb9OuaCZKr7h5D/h6iii14sK0hLbplTc6frx4Ss= +gopkg.in/ini.v1 v1.67.2/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -693,37 +536,55 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/gorm v1.31.1 h1:7CA8FTFz/gRfgqgpeKIBcervUn3xSyPUmr6B2WXJ7kg= gorm.io/gorm v1.31.1/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs= -gotest.tools/v3 v3.0.2 h1:kG1BFyqVHuQoVQiR1bWGnfz/fmHvvuiSPIV7rvl360E= -gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -modernc.org/cc/v4 v4.20.0 h1:45Or8mQfbUqJOG9WaxvlFYOAQO0lQ5RvqBcFCXngjxk= -modernc.org/cc/v4 v4.20.0/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= -modernc.org/ccgo/v4 v4.16.0 h1:ofwORa6vx2FMm0916/CkZjpFPSR70VwTjUCe2Eg5BnA= -modernc.org/ccgo/v4 v4.16.0/go.mod h1:dkNyWIjFrVIZ68DTo36vHK+6/ShBn4ysU61So6PIqCI= -modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= -modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= -modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= -modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= -modernc.org/libc v1.49.3 h1:j2MRCRdwJI2ls/sGbeSk0t2bypOG/uvPZUsGQFDulqg= -modernc.org/libc v1.49.3/go.mod h1:yMZuGkn7pXbKfoT/M35gFJOAEdSKdxL0q64sF7KqCDo= -modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= -modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= -modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= -modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= -modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= -modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= -modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= -modernc.org/sqlite v1.29.6 h1:0lOXGrycJPptfHDuohfYgNqoe4hu+gYuN/pKgY5XjS4= -modernc.org/sqlite v1.29.6/go.mod h1:S02dvcmm7TnTRvGhv8IGYyLnIt7AS2KPaB1F/71p75U= -modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= -modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= +k8s.io/api v0.36.1 h1:XbL/EMj8K2aJpJtePmqUyQMsM0D4QI2pvl7YKJ20FTY= +k8s.io/api v0.36.1/go.mod h1:KOWo4ey3TINlXjeHVuwB3i+tXXnu+UcwFBHlI/9dvEo= +k8s.io/apimachinery v0.36.1 h1:G63Gjx2W+q0YD+72Vo8oY0nDnePVwnuzTmmy5ENrVSA= +k8s.io/apimachinery v0.36.1/go.mod h1:ibYOR00vW/I1kzvi5SF0dRuJ52BvKtfvRdOn35GPQ+8= +k8s.io/client-go v0.36.1 h1:FN/K8QIT2CEDt+2WB2HnWrUANZ50AP5GII43/SP2JR0= +k8s.io/client-go v0.36.1/go.mod h1:s6rAnCtTGYDQnpNjEhSaISV+2O8jwruZ6m3QOYBFbtU= +k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= +k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= +k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a h1:xCeOEAOoGYl2jnJoHkC3hkbPJgdATINPMAxaynU2Ovg= +k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= +k8s.io/streaming v0.36.1 h1:L+K68n4Gg940BGNNYtUBvL1WTLL0YnKT3s+P1MNAmR4= +k8s.io/streaming v0.36.1/go.mod h1:z6fV3D+NVkoeqRMtWwlUZK6U17SY/LqNzOxWL6GyR/s= +k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU= +k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= +modernc.org/cc/v4 v4.28.2 h1:3tQ0lf2ADtoby2EtSP+J7IE2SHwEJdP8ioR59wx7XpY= +modernc.org/cc/v4 v4.28.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.0 h1:yRLPFZieg532OT4rp4JFNIVcquwalMX26G95WQDqwCQ= +modernc.org/ccgo/v4 v4.34.0/go.mod h1:AS5WYMyBakQ+fhsHhtP8mWB82KTGPkNNJDGfGQCe0/A= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.2 h1:ZtDCnhonXSZexk/AYsegNRV1lJGgaNZJuKjJSWKyEqo= +modernc.org/gc/v3 v3.1.2/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.72.3 h1:ZnDF4tXn4NBXFutMMQC4vtbTFSXhhKzR73fv0beZEAU= +modernc.org/libc v1.72.3/go.mod h1:dn0dZNnnn1clLyvRxLxYExxiKRZIRENOfqQ8XEeg4Qs= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.50.1 h1:l+cQvn0sd0zJJtfygGHuQJ5AjlrwXmWPw4KP3ZMwr9w= +modernc.org/sqlite v1.50.1/go.mod h1:tcNzv5p84E0skkmJn038y+hWJbLQXQqEnQfeh5r2JLM= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2 h1:kwVWMx5yS1CrnFWA/2QHyRVJ8jM6dBA80uLmm0wJkk8= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/kubernetes/README.md b/kubernetes/README.md new file mode 100644 index 00000000..3be47fec --- /dev/null +++ b/kubernetes/README.md @@ -0,0 +1,114 @@ +# Kubernetes RBAC Configuration + +This directory contains the required Kubernetes RBAC manifests for running Wings +in Kubernetes mode. These resources grant Wings the minimum permissions needed to +manage game server workloads as Pods within a namespace. + +## Quick Start + +```bash +# Apply all RBAC resources +kubectl apply -f kubernetes/ + +# Verify +kubectl get serviceaccount pelican-wings -n pelican +kubectl get role pelican-wings -n pelican +kubectl get rolebinding pelican-wings -n pelican +kubectl get clusterrole pelican-wings-metrics +kubectl get clusterrolebinding pelican-wings-metrics +``` + +## Architecture + +Wings requires two levels of RBAC: + +1. **Namespace-scoped (Role + RoleBinding)** — Manages Pods, Services, Jobs, and + PVCs within the `pelican` namespace. +2. **Cluster-scoped (ClusterRole + ClusterRoleBinding)** — Reads Pod metrics from + the `metrics.k8s.io` API (which is cluster-scoped). + +## Required Permissions + +### Namespace-scoped (pelican namespace) + +| Resource | Verbs | Purpose | +|---------------------------|--------------------------------|------------------------------------------| +| pods | get, create, delete, list | Game server Pod lifecycle | +| pods/log | get | Stream server console output | +| pods/attach | create | Interactive console (SPDY attach) | +| services | get, create, update, delete | NodePort Service management | +| jobs | get, create, delete | Egg installation scripts | +| configmaps | get, create, delete | Install script storage (multi-node) | +| resourcequotas | get, create, update | Namespace resource limits (if enabled) | +| limitranges | get, create, update | Container default limits (if enabled) | +| persistentvolumeclaims | get, create, update, delete, list | PVC storage lifecycle (if enabled) | + +### Cluster-scoped + +| Resource | Verbs | Purpose | +|---------------------------------------|-------|---------------------------------| +| pods (metrics.k8s.io/v1beta1) | get | CPU/memory usage polling | + +## Files + +- `namespace.yaml` — Namespace definition +- `serviceaccount.yaml` — ServiceAccount for Wings Pods +- `role.yaml` — Namespace-scoped Role with required permissions +- `rolebinding.yaml` — Binds the Role to the ServiceAccount +- `clusterrole-metrics.yaml` — Cluster-scoped access to metrics API +- `clusterrolebinding-metrics.yaml` — Binds the ClusterRole to the ServiceAccount + +## Configuration + +After applying these manifests, configure Wings to use the ServiceAccount: + +```yaml +# config.yml +kubernetes: + enabled: true + namespace: pelican + service_account: pelican-wings +``` + +If Wings runs **inside** the cluster as a Pod, assign the ServiceAccount directly +to the Wings Deployment/Pod. If Wings runs **outside** the cluster, create a +kubeconfig that authenticates as the ServiceAccount (or use a token). + +## Customization + +### Different namespace + +Replace `pelican` with your namespace in all manifests: + +```bash +sed -i 's/namespace: pelican/namespace: my-namespace/g' kubernetes/*.yaml +``` + +### Disable PVC permissions + +If you only use `storage_mode: hostpath`, you can remove the +`persistentvolumeclaims` resource from `role.yaml`. + +### Image pulling + +By default, game server Pods and installation Jobs use `imagePullPolicy: Always` +for remote images so updated tags are re-pulled instead of reusing a stale node +cache (matching the Docker backend); `~`-prefixed local images are never pulled. +For air-gapped clusters, pin the policy: + +```yaml +# config.yml +kubernetes: + image_pull_policy: IfNotPresent # or "Never" +``` + +### Disable metrics + +If you don't have metrics-server installed, you can skip the ClusterRole and +ClusterRoleBinding. Wings will gracefully degrade (no CPU/memory stats). + +### Multiple Wings instances + +If you run multiple Wings nodes targeting different namespaces, create a Role and +RoleBinding per namespace, but share the ClusterRole/ClusterRoleBinding (it's +namespace-independent). diff --git a/kubernetes/clusterrole-metrics.yaml b/kubernetes/clusterrole-metrics.yaml new file mode 100644 index 00000000..88b461c8 --- /dev/null +++ b/kubernetes/clusterrole-metrics.yaml @@ -0,0 +1,24 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: pelican-wings-metrics + labels: + app.kubernetes.io/managed-by: pelican-wings + app.kubernetes.io/component: wings +rules: + # Read Pod metrics from the Kubernetes Metrics API (metrics.k8s.io/v1beta1). + # Required for CPU/memory usage reporting. If metrics-server is not installed, + # this ClusterRole is unused and Wings will gracefully degrade. + - apiGroups: ["metrics.k8s.io"] + resources: ["pods"] + verbs: ["get"] + # Read Node addresses for IP allocation endpoint. Required so Wings can + # report the node's external/internal IPs instead of the Pod's cluster IP. + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] + # Proxy to kubelet stats/summary API for resource metrics when + # metrics-server is not installed. + - apiGroups: [""] + resources: ["nodes/proxy"] + verbs: ["get"] diff --git a/kubernetes/clusterrolebinding-metrics.yaml b/kubernetes/clusterrolebinding-metrics.yaml new file mode 100644 index 00000000..34d3b50c --- /dev/null +++ b/kubernetes/clusterrolebinding-metrics.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: pelican-wings-metrics + labels: + app.kubernetes.io/managed-by: pelican-wings + app.kubernetes.io/component: wings +subjects: + - kind: ServiceAccount + name: pelican-wings + namespace: pelican +roleRef: + kind: ClusterRole + name: pelican-wings-metrics + apiGroup: rbac.authorization.k8s.io diff --git a/kubernetes/namespace.yaml b/kubernetes/namespace.yaml new file mode 100644 index 00000000..8c578382 --- /dev/null +++ b/kubernetes/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: pelican + labels: + app.kubernetes.io/managed-by: pelican-wings diff --git a/kubernetes/role.yaml b/kubernetes/role.yaml new file mode 100644 index 00000000..1e67d038 --- /dev/null +++ b/kubernetes/role.yaml @@ -0,0 +1,56 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: pelican-wings + namespace: pelican + labels: + app.kubernetes.io/managed-by: pelican-wings + app.kubernetes.io/component: wings +rules: + # Game server Pod lifecycle: create, get (status checks), delete, list + # (installer pod lookup by label). + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "create", "delete", "list", "watch"] + + # Stream console output from game server and installer Pods. + - apiGroups: [""] + resources: ["pods/log"] + verbs: ["get"] + + # Interactive console attach via SPDY (stdin/stdout/stderr). + - apiGroups: [""] + resources: ["pods/attach"] + verbs: ["create"] + + # Service management for exposing game server ports (NodePort or LoadBalancer). + # List is needed for discovering LoadBalancer external IPs. + - apiGroups: [""] + resources: ["services"] + verbs: ["get", "create", "update", "delete", "list"] + + # Egg installation Jobs: create, get (poll status), delete (cleanup). + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get", "create", "delete"] + + # ConfigMaps for install scripts and identity files: create, get, update, delete. + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "create", "update", "delete"] + + # ResourceQuota management (when resource_quota.enabled: true). + - apiGroups: [""] + resources: ["resourcequotas"] + verbs: ["get", "create", "update"] + + # LimitRange management (when limit_range.enabled: true). + - apiGroups: [""] + resources: ["limitranges"] + verbs: ["get", "create", "update"] + + # PersistentVolumeClaim lifecycle (when storage_mode: pvc). + # Remove this rule if you only use storage_mode: hostpath. + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "create", "update", "delete", "list"] diff --git a/kubernetes/rolebinding.yaml b/kubernetes/rolebinding.yaml new file mode 100644 index 00000000..cdc010d7 --- /dev/null +++ b/kubernetes/rolebinding.yaml @@ -0,0 +1,16 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: pelican-wings + namespace: pelican + labels: + app.kubernetes.io/managed-by: pelican-wings + app.kubernetes.io/component: wings +subjects: + - kind: ServiceAccount + name: pelican-wings + namespace: pelican +roleRef: + kind: Role + name: pelican-wings + apiGroup: rbac.authorization.k8s.io diff --git a/kubernetes/serviceaccount.yaml b/kubernetes/serviceaccount.yaml new file mode 100644 index 00000000..08ea60ca --- /dev/null +++ b/kubernetes/serviceaccount.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: pelican-wings + namespace: pelican + labels: + app.kubernetes.io/managed-by: pelican-wings + app.kubernetes.io/component: wings diff --git a/router/router_system.go b/router/router_system.go index cdc5bd58..ed502a7c 100644 --- a/router/router_system.go +++ b/router/router_system.go @@ -12,6 +12,7 @@ import ( "github.com/gin-gonic/gin" "github.com/pelican-dev/wings/config" + "github.com/pelican-dev/wings/environment/kubernetes" "github.com/pelican-dev/wings/internal/diagnostics" "github.com/pelican-dev/wings/router/middleware" "github.com/pelican-dev/wings/router/tokens" @@ -22,7 +23,7 @@ import ( // Returns information about the system that wings is running on. func getSystemInformation(c *gin.Context) { - i, err := system.GetSystemInformation() + i, err := system.GetSystemInformationWithOptions(config.Get().Kubernetes.Enabled) if err != nil { middleware.CaptureAndAbort(c, err) return @@ -89,20 +90,53 @@ func getDiagnostics(c *gin.Context) { // Returns list of host machine IP addresses func getSystemIps(c *gin.Context) { - interfaces, err := system.GetSystemIps() - if err != nil { - middleware.CaptureAndAbort(c, err) - return - } + cfg := config.Get() + var interfaces []string + + if cfg.Kubernetes.Enabled { + ctx := c.Request.Context() + + // In LoadBalancer mode, collect IPs from existing LB Services. + if cfg.Kubernetes.NetworkMode == config.KubeNetworkLoadBalancer { + lbIPs, err := kubernetes.GetLoadBalancerIPs(ctx) + if err != nil { + log.WithField("error", err).Warn("failed to query LoadBalancer IPs") + } else { + interfaces = append(interfaces, lbIPs...) + } + } - // Append config defined ips as well - for i := range config.Get().Docker.SystemIps { - targetIp := config.Get().Docker.SystemIps[i] - if slices.Contains(interfaces, targetIp) { - continue + // Always try to get node IPs as a base (needed for NodePort, useful + // as fallback for LoadBalancer before any servers exist). + nodeIPs, err := kubernetes.GetNodeIPs(ctx) + if err != nil { + log.WithField("error", err).Warn("failed to query node IPs") + } else { + for _, ip := range nodeIPs { + if !slices.Contains(interfaces, ip) { + interfaces = append(interfaces, ip) + } + } } - interfaces = append(interfaces, targetIp) + // Append user-configured static IPs. + for _, ip := range cfg.Kubernetes.SystemIPs { + if !slices.Contains(interfaces, ip) { + interfaces = append(interfaces, ip) + } + } + } else { + ips, err := system.GetSystemIps() + if err != nil { + middleware.CaptureAndAbort(c, err) + return + } + interfaces = append(interfaces, ips...) + for _, ip := range cfg.Docker.SystemIps { + if !slices.Contains(interfaces, ip) { + interfaces = append(interfaces, ip) + } + } } c.JSON(http.StatusOK, &system.IpAddresses{IpAddresses: interfaces}) diff --git a/server/install.go b/server/install.go index 8307517e..deda915e 100644 --- a/server/install.go +++ b/server/install.go @@ -103,6 +103,12 @@ func (s *Server) internalInstall() error { if err != nil { return err } + + // Route to the Kubernetes-based installer when K8s is enabled. + if config.Get().Kubernetes.Enabled { + return s.internalInstallKubernetes(&script) + } + p, err := NewInstallationProcess(s, &script) if err != nil { return err diff --git a/server/install_kubernetes.go b/server/install_kubernetes.go new file mode 100644 index 00000000..63c480eb --- /dev/null +++ b/server/install_kubernetes.go @@ -0,0 +1,61 @@ +package server + +import ( + "path/filepath" + + "emperror.dev/errors" + + "github.com/pelican-dev/wings/config" + "github.com/pelican-dev/wings/environment/kubernetes" + "github.com/pelican-dev/wings/remote" + "github.com/pelican-dev/wings/system" +) + +// internalInstallKubernetes runs the installation process using a Kubernetes +// Job instead of a Docker container. This is called when K8s mode is enabled. +func (s *Server) internalInstallKubernetes(script *remote.InstallationScript) error { + env, ok := s.Environment.(*kubernetes.Environment) + if !ok { + return errors.New("install: kubernetes enabled but environment is not kubernetes") + } + + tmpDir := filepath.Join(config.Get().System.TmpDirectory, s.ID()) + + ip := kubernetes.NewInstallerProcess( + env, + script, + s.GetEnvironmentVariables(), + s.Filesystem().Path(), + tmpDir, + s.Sink(system.InstallSink), + ) + + if !s.installing.SwapIf(true) { + return errors.New("install: cannot obtain installation lock") + } + defer s.installing.Store(false) + + s.Log().Info("beginning kubernetes job-based installation process for server") + s.Events().Publish(DaemonMessageEvent, "Starting installation process via Kubernetes Job, this could take a few minutes...") + + // Write the install script to disk for mounting into the Job Pod. + if err := ip.WriteInstallScript(); err != nil { + return errors.WithMessage(err, "install: failed to write installation script") + } + + // Run the Job and wait for completion. + if err := ip.Run(s.Context()); err != nil { + _ = ip.Cleanup(s.Context()) + return errors.WithMessage(err, "install: kubernetes installer job failed") + } + + s.Events().Publish(DaemonMessageEvent, "Installation process completed.") + s.Log().Info("completed kubernetes job-based installation process for server") + + // Cleanup the Job and temp files. + if err := ip.Cleanup(s.Context()); err != nil { + s.Log().WithField("error", err).Warn("failed to clean up installer job resources") + } + + return nil +} diff --git a/server/manager.go b/server/manager.go index 55dd645e..2263f1ad 100644 --- a/server/manager.go +++ b/server/manager.go @@ -18,6 +18,7 @@ import ( "github.com/pelican-dev/wings/config" "github.com/pelican-dev/wings/environment" "github.com/pelican-dev/wings/environment/docker" + "github.com/pelican-dev/wings/environment/kubernetes" "github.com/pelican-dev/wings/remote" "github.com/pelican-dev/wings/server/filesystem" ) @@ -201,9 +202,6 @@ func (m *Manager) InitServer(data remote.ServerConfigurationResponse) (*Server, return nil, errors.WithStackIf(err) } - // Right now we only support a Docker based environment, so I'm going to hard code - // this logic in. When we're ready to support other environment we'll need to make - // some modifications here, obviously. settings := environment.Settings{ Mounts: s.Mounts(), Allocations: s.cfg.Allocations, @@ -212,15 +210,27 @@ func (m *Manager) InitServer(data remote.ServerConfigurationResponse) (*Server, } envCfg := environment.NewConfiguration(settings, s.GetEnvironmentVariables()) - meta := docker.Metadata{ - Image: s.Config().Container.Image, - } - if env, err := docker.New(s.ID(), &meta, envCfg); err != nil { - return nil, err + if config.Get().Kubernetes.Enabled { + meta := kubernetes.Metadata{ + Image: s.Config().Container.Image, + } + if env, err := kubernetes.New(s.ID(), &meta, envCfg); err != nil { + return nil, err + } else { + s.Environment = env + s.StartEventListeners() + } } else { - s.Environment = env - s.StartEventListeners() + meta := docker.Metadata{ + Image: s.Config().Container.Image, + } + if env, err := docker.New(s.ID(), &meta, envCfg); err != nil { + return nil, err + } else { + s.Environment = env + s.StartEventListeners() + } } // If the server's data directory exists, force disk usage calculation. diff --git a/system/system.go b/system/system.go index 87ffc56b..8204c442 100644 --- a/system/system.go +++ b/system/system.go @@ -101,17 +101,50 @@ type DockerDiskUsage struct { } func GetSystemInformation() (*Information, error) { + return GetSystemInformationWithOptions(false) +} + +func GetSystemInformationWithOptions(kubernetesMode bool) (*Information, error) { k, err := kernel.GetKernelVersion() if err != nil { return nil, err } - version, info, err := GetDockerInfo(context.Background()) + release, err := osrelease.Read() if err != nil { return nil, err } - release, err := osrelease.Read() + if kubernetesMode { + v, err := mem.VirtualMemory() + if err != nil { + return nil, err + } + + var osName string + if release["PRETTY_NAME"] != "" { + osName = release["PRETTY_NAME"] + } else if release["NAME"] != "" { + osName = release["NAME"] + } else { + osName = runtime.GOOS + } + + return &Information{ + Version: Version, + Docker: DockerInformation{}, + System: System{ + Architecture: runtime.GOARCH, + CPUThreads: runtime.NumCPU(), + MemoryBytes: int64(v.Total), + KernelVersion: k.String(), + OS: osName, + OSType: runtime.GOOS, + }, + }, nil + } + + version, info, err := GetDockerInfo(context.Background()) if err != nil { return nil, err } @@ -175,9 +208,13 @@ func GetSystemIps() ([]string, error) { } for _, addr := range iface_addrs { ipNet, valid := addr.(*net.IPNet) - if valid && !ipNet.IP.IsLoopback() && (len(ipNet.IP) == net.IPv6len && !ipNet.IP.IsLinkLocalUnicast()) { - ip_addrs = append(ip_addrs, ipNet.IP.String()) + if !valid || ipNet.IP.IsLoopback() { + continue + } + if ipNet.IP.IsLinkLocalUnicast() || ipNet.IP.IsLinkLocalMulticast() { + continue } + ip_addrs = append(ip_addrs, ipNet.IP.String()) } return ip_addrs, nil } From eaf734e21a1248f1af5a10c5821f8308b1a3a2d1 Mon Sep 17 00:00:00 2001 From: Exonical Date: Fri, 19 Jun 2026 20:55:09 -0700 Subject: [PATCH 2/6] chore(ci): Update GitHub Actions to use latest Helm setup --- .github/workflows/helm-lint.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/helm-lint.yaml b/.github/workflows/helm-lint.yaml index 1ba2e7ad..e9d58d2b 100644 --- a/.github/workflows/helm-lint.yaml +++ b/.github/workflows/helm-lint.yaml @@ -18,9 +18,9 @@ jobs: lint-helm: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - - uses: azure/setup-helm@v4 + - uses: azure/setup-helm@v5 - name: Lint Helm chart run: helm lint chart/pelican-wings From 3be8dda1e91e96b3176d5f966359993fe971a74e Mon Sep 17 00:00:00 2001 From: Exonical Date: Tue, 2 Jun 2026 06:10:09 +0000 Subject: [PATCH 3/6] fix(k8s): harden client init, attach write, and quota parsing Address three robustness issues in the Kubernetes backend: - api.go: Client() stored the init error in a function-local variable, so after a failed first call the sync.Once never re-ran and subsequent calls returned (nil, nil), masking the failure. Persist the error in a package variable and return it on every call. - container.go: SendCommand() checked IsAttached() before taking the lock that guards e.stream, so the stream could be cleared in between and cause a nil-pointer write. Nil-check e.stream under the read lock instead. - quota.go: buildResourceQuota/buildLimitRange used resource.MustParse on user-configurable quantity strings, panicking the process on invalid values. Parse with resource.ParseQuantity and propagate errors through EnsureResourceQuota/EnsureLimitRange. --- environment/kubernetes/api.go | 18 ++--- environment/kubernetes/container.go | 10 ++- environment/kubernetes/container_test.go | 2 + environment/kubernetes/quota.go | 94 ++++++++++++++---------- environment/kubernetes/quota_test.go | 22 +++++- 5 files changed, 94 insertions(+), 52 deletions(-) diff --git a/environment/kubernetes/api.go b/environment/kubernetes/api.go index a82ac1d9..4c961144 100644 --- a/environment/kubernetes/api.go +++ b/environment/kubernetes/api.go @@ -14,29 +14,29 @@ import ( var ( _konce sync.Once _client kubernetes.Interface + _kerr error ) // Client returns a shared Kubernetes clientset. The client is created once // and reused for all subsequent calls. It uses in-cluster config when no // kubeconfig path is specified, falling back to the provided kubeconfig file. func Client() (kubernetes.Interface, error) { - var err error _konce.Do(func() { var cfg *rest.Config kubeconfig := config.Get().Kubernetes.Kubeconfig if kubeconfig != "" { - cfg, err = clientcmd.BuildConfigFromFlags("", kubeconfig) + cfg, _kerr = clientcmd.BuildConfigFromFlags("", kubeconfig) } else { - cfg, err = rest.InClusterConfig() + cfg, _kerr = rest.InClusterConfig() } - if err != nil { - err = errors.Wrap(err, "environment/kubernetes: failed to build config") + if _kerr != nil { + _kerr = errors.Wrap(_kerr, "environment/kubernetes: failed to build config") return } - _client, err = kubernetes.NewForConfig(cfg) - if err != nil { - err = errors.Wrap(err, "environment/kubernetes: failed to create clientset") + _client, _kerr = kubernetes.NewForConfig(cfg) + if _kerr != nil { + _kerr = errors.Wrap(_kerr, "environment/kubernetes: failed to create clientset") } }) - return _client, err + return _client, _kerr } diff --git a/environment/kubernetes/container.go b/environment/kubernetes/container.go index d9b57e3a..a1a716ee 100644 --- a/environment/kubernetes/container.go +++ b/environment/kubernetes/container.go @@ -348,13 +348,15 @@ func (e *Environment) Attach(ctx context.Context) error { // SendCommand writes a command string to the attached Pod's stdin. func (e *Environment) SendCommand(c string) error { - if !e.IsAttached() { - return errors.New("environment/kubernetes: not attached to pod") - } - e.mu.RLock() defer e.mu.RUnlock() + // Check the stream under the lock so it cannot be cleared between an + // attachment check and the write below. + if e.stream == nil { + return errors.New("environment/kubernetes: not attached to pod") + } + // If this is the stop command, mark the server as stopping. if e.meta.Stop.Type == "command" && c == e.meta.Stop.Value { e.SetState(environment.ProcessStoppingState) diff --git a/environment/kubernetes/container_test.go b/environment/kubernetes/container_test.go index 29ff619e..90d35fa9 100644 --- a/environment/kubernetes/container_test.go +++ b/environment/kubernetes/container_test.go @@ -9,6 +9,8 @@ import ( "github.com/pelican-dev/wings/config" ) +// TestResolveImagePullPolicy verifies pull-policy resolution for remote +// images, ~-prefixed local images, and configured overrides. func TestResolveImagePullPolicy(t *testing.T) { g := Goblin(t) diff --git a/environment/kubernetes/quota.go b/environment/kubernetes/quota.go index 68998370..4d297a38 100644 --- a/environment/kubernetes/quota.go +++ b/environment/kubernetes/quota.go @@ -25,7 +25,10 @@ func (e *Environment) EnsureResourceQuota(ctx context.Context) error { } ns := e.namespace() - quota := buildResourceQuota(ns, &cfg.Kubernetes.ResourceQuota) + quota, err := buildResourceQuota(ns, &cfg.Kubernetes.ResourceQuota) + if err != nil { + return err + } existing, err := e.client.CoreV1().ResourceQuotas(ns).Get(ctx, quotaName, metav1.GetOptions{}) if err == nil { @@ -56,7 +59,10 @@ func (e *Environment) EnsureLimitRange(ctx context.Context) error { } ns := e.namespace() - lr := buildLimitRange(ns, &cfg.Kubernetes.LimitRange) + lr, err := buildLimitRange(ns, &cfg.Kubernetes.LimitRange) + if err != nil { + return err + } existing, err := e.client.CoreV1().LimitRanges(ns).Get(ctx, limitRangeName, metav1.GetOptions{}) if err == nil { @@ -78,21 +84,39 @@ func (e *Environment) EnsureLimitRange(ctx context.Context) error { return nil } +// setQuantity parses a quantity string into the resource list under key when +// the value is non-empty. Unlike resource.MustParse it returns an error for +// invalid (user-configurable) values instead of panicking the process. +func setQuantity(list corev1.ResourceList, key corev1.ResourceName, field, value string) error { + if value == "" { + return nil + } + q, err := resource.ParseQuantity(value) + if err != nil { + return errors.Wrapf(err, "environment/kubernetes: invalid quantity %q for %s", value, field) + } + list[key] = q + return nil +} + // buildResourceQuota constructs the ResourceQuota spec from config values. -func buildResourceQuota(namespace string, cfg *config.KubeResourceQuota) *corev1.ResourceQuota { +func buildResourceQuota(namespace string, cfg *config.KubeResourceQuota) (*corev1.ResourceQuota, error) { hard := corev1.ResourceList{} - if cfg.CPULimit != "" { - hard[corev1.ResourceLimitsCPU] = resource.MustParse(cfg.CPULimit) - } - if cfg.MemoryLimit != "" { - hard[corev1.ResourceLimitsMemory] = resource.MustParse(cfg.MemoryLimit) - } - if cfg.CPURequest != "" { - hard[corev1.ResourceRequestsCPU] = resource.MustParse(cfg.CPURequest) - } - if cfg.MemoryRequest != "" { - hard[corev1.ResourceRequestsMemory] = resource.MustParse(cfg.MemoryRequest) + for _, q := range []struct { + key corev1.ResourceName + field string + value string + }{ + {corev1.ResourceLimitsCPU, "cpu_limit", cfg.CPULimit}, + {corev1.ResourceLimitsMemory, "memory_limit", cfg.MemoryLimit}, + {corev1.ResourceRequestsCPU, "cpu_request", cfg.CPURequest}, + {corev1.ResourceRequestsMemory, "memory_request", cfg.MemoryRequest}, + {corev1.ResourceRequestsStorage, "max_storage", cfg.MaxStorage}, + } { + if err := setQuantity(hard, q.key, q.field, q.value); err != nil { + return nil, err + } } if cfg.MaxPods > 0 { hard[corev1.ResourcePods] = *resource.NewQuantity(cfg.MaxPods, resource.DecimalSI) @@ -100,9 +124,6 @@ func buildResourceQuota(namespace string, cfg *config.KubeResourceQuota) *corev1 if cfg.MaxPVCs > 0 { hard[corev1.ResourcePersistentVolumeClaims] = *resource.NewQuantity(cfg.MaxPVCs, resource.DecimalSI) } - if cfg.MaxStorage != "" { - hard[corev1.ResourceRequestsStorage] = resource.MustParse(cfg.MaxStorage) - } return &corev1.ResourceQuota{ ObjectMeta: metav1.ObjectMeta{ @@ -115,11 +136,11 @@ func buildResourceQuota(namespace string, cfg *config.KubeResourceQuota) *corev1 Spec: corev1.ResourceQuotaSpec{ Hard: hard, }, - } + }, nil } // buildLimitRange constructs the LimitRange spec from config values. -func buildLimitRange(namespace string, cfg *config.KubeLimitRange) *corev1.LimitRange { +func buildLimitRange(namespace string, cfg *config.KubeLimitRange) (*corev1.LimitRange, error) { containerLimit := corev1.LimitRangeItem{ Type: corev1.LimitTypeContainer, Default: corev1.ResourceList{}, @@ -127,23 +148,22 @@ func buildLimitRange(namespace string, cfg *config.KubeLimitRange) *corev1.Limit Max: corev1.ResourceList{}, } - if cfg.DefaultCPULimit != "" { - containerLimit.Default[corev1.ResourceCPU] = resource.MustParse(cfg.DefaultCPULimit) - } - if cfg.DefaultMemoryLimit != "" { - containerLimit.Default[corev1.ResourceMemory] = resource.MustParse(cfg.DefaultMemoryLimit) - } - if cfg.DefaultCPURequest != "" { - containerLimit.DefaultRequest[corev1.ResourceCPU] = resource.MustParse(cfg.DefaultCPURequest) - } - if cfg.DefaultMemoryRequest != "" { - containerLimit.DefaultRequest[corev1.ResourceMemory] = resource.MustParse(cfg.DefaultMemoryRequest) - } - if cfg.MaxCPU != "" { - containerLimit.Max[corev1.ResourceCPU] = resource.MustParse(cfg.MaxCPU) - } - if cfg.MaxMemory != "" { - containerLimit.Max[corev1.ResourceMemory] = resource.MustParse(cfg.MaxMemory) + for _, q := range []struct { + list corev1.ResourceList + key corev1.ResourceName + field string + value string + }{ + {containerLimit.Default, corev1.ResourceCPU, "default_cpu_limit", cfg.DefaultCPULimit}, + {containerLimit.Default, corev1.ResourceMemory, "default_memory_limit", cfg.DefaultMemoryLimit}, + {containerLimit.DefaultRequest, corev1.ResourceCPU, "default_cpu_request", cfg.DefaultCPURequest}, + {containerLimit.DefaultRequest, corev1.ResourceMemory, "default_memory_request", cfg.DefaultMemoryRequest}, + {containerLimit.Max, corev1.ResourceCPU, "max_cpu", cfg.MaxCPU}, + {containerLimit.Max, corev1.ResourceMemory, "max_memory", cfg.MaxMemory}, + } { + if err := setQuantity(q.list, q.key, q.field, q.value); err != nil { + return nil, err + } } return &corev1.LimitRange{ @@ -157,5 +177,5 @@ func buildLimitRange(namespace string, cfg *config.KubeLimitRange) *corev1.Limit Spec: corev1.LimitRangeSpec{ Limits: []corev1.LimitRangeItem{containerLimit}, }, - } + }, nil } diff --git a/environment/kubernetes/quota_test.go b/environment/kubernetes/quota_test.go index b36328e6..8c1ca151 100644 --- a/environment/kubernetes/quota_test.go +++ b/environment/kubernetes/quota_test.go @@ -15,6 +15,8 @@ import ( "github.com/pelican-dev/wings/system" ) +// TestQuota covers building and reconciling ResourceQuota and LimitRange +// objects from configuration, including invalid-quantity error handling. func TestQuota(t *testing.T) { g := Goblin(t) @@ -264,7 +266,8 @@ func TestQuota(t *testing.T) { MaxPods: 10, MaxPVCs: 0, } - rq := buildResourceQuota("test-ns", cfg) + rq, err := buildResourceQuota("test-ns", cfg) + g.Assert(err).IsNil() g.Assert(rq.Namespace).Equal("test-ns") _, hasCPU := rq.Spec.Hard[corev1.ResourceLimitsCPU] @@ -289,7 +292,8 @@ func TestQuota(t *testing.T) { MaxCPU: "8", MaxMemory: "", } - lr := buildLimitRange("test-ns", cfg) + lr, err := buildLimitRange("test-ns", cfg) + g.Assert(err).IsNil() g.Assert(lr.Namespace).Equal("test-ns") g.Assert(len(lr.Spec.Limits)).Equal(1) @@ -306,6 +310,20 @@ func TestQuota(t *testing.T) { _, hasMaxMem := item.Max[corev1.ResourceMemory] g.Assert(hasMaxMem).IsFalse() }) + + g.It("should return an error for an invalid quantity instead of panicking", func() { + lr, err := buildLimitRange("test-ns", &config.KubeLimitRange{DefaultCPULimit: "not-a-quantity"}) + g.Assert(err != nil).IsTrue() + g.Assert(lr == nil).IsTrue() + }) + }) + + g.Describe("buildResourceQuota invalid input", func() { + g.It("should return an error for an invalid quantity instead of panicking", func() { + rq, err := buildResourceQuota("test-ns", &config.KubeResourceQuota{CPULimit: "not-a-quantity"}) + g.Assert(err != nil).IsTrue() + g.Assert(rq == nil).IsTrue() + }) }) }) } From 7531e217e4d6ecded2a7da5fb7ed8fe14eeaffa5 Mon Sep 17 00:00:00 2001 From: Exonical Date: Sat, 20 Jun 2026 05:19:00 +0000 Subject: [PATCH 4/6] fix(k8s): address CodeRabbit review findings Harden the Kubernetes environment and Helm chart per the full CodeRabbit review: environment/kubernetes: - container.go: surface Exists() errors in Create(), order Destroy() offline state after the delete check, use apierrors.IsNotFound, and deduplicate container ports shared across allocation IPs. - power.go: handle delete/wait errors in OnBeforeStart, give signal-based stops a graceful SIGTERM grace period (immediate for SIGKILL), and treat a terminal (non-running) Pod as stopped so restart-while-off does not hang. - installer.go: propagate caller context into WriteInstallScript, use foreground propagation + wait when recreating the installer Job, and reject SubPath values that escape the root directory. - quota.go: fail fast on non-NotFound Get errors instead of falling through to Create. - storage.go: ResizePVC is a no-op when a shared DataPVC is configured. - network.go: reconcile Service type and annotations on a networkMode switch. - stats.go: bound each metrics/stats request with a timeout and retry the metrics API every tick instead of sticking to the kubelet fallback. server/router/system: - update.go: sync image and stop configuration to the Kubernetes environment. - router_system.go: return an error when no assignable IPs are discovered. - system.go: treat a missing os-release as best-effort in Kubernetes mode. chart/pelican-wings: - Render credentials into a Secret instead of a ConfigMap. - Guard wings.kubernetes.namespace against gameNamespace drift. - Fail rendering when ResourceQuota/LimitRange are enabled but empty. - Require an explicit serviceAccount.name when not creating one. - Gate the broad nodes/proxy kubelet fallback behind rbac.kubeletMetricsFallback. docs/manifests: - Split the optional nodes/proxy ClusterRole into clusterrole-metrics-kubelet.yaml. - Expand the chart and kubernetes RBAC READMEs (credentials, networkMode, multi-instance guidance, RBAC tables). Add regression tests for port deduplication, Service type reconciliation, ResizePVC with a shared DataPVC, and quota Get-failure fail-fast. --- chart/pelican-wings/README.md | 28 +++++++- chart/pelican-wings/templates/_helpers.tpl | 4 +- .../templates/clusterrole-metrics.yaml | 5 +- chart/pelican-wings/templates/configmap.yaml | 10 ++- chart/pelican-wings/templates/deployment.yaml | 4 +- chart/pelican-wings/templates/limitrange.yaml | 3 + .../templates/resourcequota.yaml | 3 + chart/pelican-wings/values.yaml | 5 ++ environment/kubernetes/container.go | 20 ++++-- environment/kubernetes/environment_test.go | 20 ++++++ environment/kubernetes/installer.go | 52 ++++++++++++--- environment/kubernetes/installer_test.go | 4 +- environment/kubernetes/network.go | 9 +-- environment/kubernetes/network_test.go | 36 +++++++++++ environment/kubernetes/power.go | 64 +++++++++++++++---- environment/kubernetes/quota.go | 6 ++ environment/kubernetes/quota_test.go | 43 +++++++++++++ environment/kubernetes/stats.go | 47 ++++++++------ environment/kubernetes/storage.go | 5 ++ environment/kubernetes/storage_test.go | 40 ++++++++++++ kubernetes/README.md | 31 +++++++-- kubernetes/clusterrole-metrics-kubelet.yaml | 39 +++++++++++ kubernetes/clusterrole-metrics.yaml | 9 ++- router/router_system.go | 8 +++ server/install_kubernetes.go | 2 +- server/update.go | 9 +++ system/system.go | 8 ++- 27 files changed, 439 insertions(+), 75 deletions(-) create mode 100644 kubernetes/clusterrole-metrics-kubelet.yaml diff --git a/chart/pelican-wings/README.md b/chart/pelican-wings/README.md index 49f636bb..d2692e21 100644 --- a/chart/pelican-wings/README.md +++ b/chart/pelican-wings/README.md @@ -33,14 +33,26 @@ See [values.yaml](values.yaml) for the full list of configurable values. | `wings.token` | Panel authentication token | `""` | | `wings.tokenId` | Panel token ID | `""` | | `wings.uuid` | Node UUID from Panel | `""` | -| `wings.kubernetes.networkMode` | Port exposure: `hostport` or `nodeport` | `nodeport` | +| `wings.kubernetes.networkMode` | Port exposure: `hostport`, `nodeport`, or `loadbalancer` | `nodeport` | | `wings.kubernetes.storageMode` | Storage: `hostpath` or `pvc` | `pvc` | | `wings.kubernetes.storageClass` | StorageClass for PVCs | `""` (cluster default) | | `wings.kubernetes.storageSize` | Default PVC size | `10Gi` | | `wings.kubernetes.imagePullPolicy` | Pull policy for game server Pods/install Jobs: `Always`, `IfNotPresent`, `Never` | `""` (smart default) | | `gameNamespace` | Namespace for game server resources | `pelican` | | `rbac.create` | Create RBAC resources | `true` | +| `rbac.kubeletMetricsFallback` | Grant cluster-wide `nodes/proxy` for the kubelet stats fallback (broad permission; prefer metrics-server) | `false` | | `serviceAccount.create` | Create ServiceAccount | `true` | +| `serviceAccount.name` | ServiceAccount name (**required** when `serviceAccount.create=false`) | `""` | + +> **Namespace:** Wings schedules game-server workloads into `gameNamespace`, and +> the chart creates the namespaced RBAC there. `wings.kubernetes.namespace` is +> therefore derived from `gameNamespace`; if you set it explicitly it must match +> `gameNamespace` or the chart will fail to render. + +> **Credentials:** `wings.token`, `wings.tokenId`, and `wings.uuid` are rendered +> into a Kubernetes **Secret** (not a ConfigMap). Supply them via a private +> values file or `--set`, e.g. `helm install ... -f my-creds.yaml`, and keep that +> file out of version control. ### Storage @@ -76,6 +88,16 @@ wings: networkMode: hostport ``` +LoadBalancer mode provisions a `Service` of type `LoadBalancer` per game server +(for use with MetalLB, Cilium LB-IPAM, or a cloud LB). LB IP/sharing-key +annotations can be auto-populated from the allocation IP: + +```yaml +wings: + kubernetes: + networkMode: loadbalancer +``` + ### Image pulling Game server Pods and installation Jobs default to `imagePullPolicy: Always` @@ -97,8 +119,8 @@ image itself. - **Namespace** — `pelican` (configurable) - **ServiceAccount** — For Wings and game server Pods - **Role + RoleBinding** — Namespace-scoped permissions (Pods, Services, Jobs, PVCs) -- **ClusterRole + ClusterRoleBinding** — Metrics API access -- **ConfigMap** — Wings configuration file +- **ClusterRole + ClusterRoleBinding** — Metrics API access (`nodes/proxy` only when `rbac.kubeletMetricsFallback=true`) +- **Secret** — Wings configuration file (contains Panel token) - **Deployment** — Wings daemon with health probes - **Service** — Exposes Wings API within the cluster diff --git a/chart/pelican-wings/templates/_helpers.tpl b/chart/pelican-wings/templates/_helpers.tpl index b7d93818..dbde1679 100644 --- a/chart/pelican-wings/templates/_helpers.tpl +++ b/chart/pelican-wings/templates/_helpers.tpl @@ -54,7 +54,9 @@ Create the name of the service account to use. {{- define "pelican-wings.serviceAccountName" -}} {{- if .Values.serviceAccount.create }} {{- default (include "pelican-wings.fullname" .) .Values.serviceAccount.name }} +{{- else if .Values.serviceAccount.name }} +{{- .Values.serviceAccount.name }} {{- else }} -{{- default "default" .Values.serviceAccount.name }} +{{- fail "serviceAccount.name must be set when serviceAccount.create is false: binding RBAC to the namespace 'default' ServiceAccount would grant Wings' permissions to every workload using it." }} {{- end }} {{- end }} diff --git a/chart/pelican-wings/templates/clusterrole-metrics.yaml b/chart/pelican-wings/templates/clusterrole-metrics.yaml index aaf02289..399f0bbb 100644 --- a/chart/pelican-wings/templates/clusterrole-metrics.yaml +++ b/chart/pelican-wings/templates/clusterrole-metrics.yaml @@ -12,9 +12,12 @@ rules: - apiGroups: [""] resources: ["nodes"] verbs: ["get"] + {{- if .Values.rbac.kubeletMetricsFallback }} # Proxy to kubelet stats/summary API for resource metrics when - # metrics-server is not installed. + # metrics-server is not installed. Opt-in: this is a broad cluster-scoped + # permission (rbac.kubeletMetricsFallback). - apiGroups: [""] resources: ["nodes/proxy"] verbs: ["get"] + {{- end }} {{- end }} diff --git a/chart/pelican-wings/templates/configmap.yaml b/chart/pelican-wings/templates/configmap.yaml index 52867e84..a332fae0 100644 --- a/chart/pelican-wings/templates/configmap.yaml +++ b/chart/pelican-wings/templates/configmap.yaml @@ -1,11 +1,15 @@ +{{- if and .Values.wings.kubernetes.namespace (ne .Values.wings.kubernetes.namespace .Values.gameNamespace) }} +{{- fail "wings.kubernetes.namespace must equal gameNamespace: the chart's namespaced RBAC is created in gameNamespace, so Wings must schedule game-server workloads there too. Leave wings.kubernetes.namespace empty or set it equal to gameNamespace." }} +{{- end }} apiVersion: v1 -kind: ConfigMap +kind: Secret metadata: name: {{ include "pelican-wings.fullname" . }}-config namespace: {{ .Values.gameNamespace }} labels: {{- include "pelican-wings.labels" . | nindent 4 }} -data: +type: Opaque +stringData: config.yml: | debug: false app_name: pelican @@ -26,7 +30,7 @@ data: tmp_directory: {{ .Values.wings.system.tmpDirectory | quote }} kubernetes: enabled: {{ .Values.wings.kubernetes.enabled }} - namespace: {{ .Values.wings.kubernetes.namespace | quote }} + namespace: {{ .Values.gameNamespace | quote }} network_mode: {{ .Values.wings.kubernetes.networkMode | quote }} {{- if .Values.wings.kubernetes.lbAnnotations }} lb_annotations: diff --git a/chart/pelican-wings/templates/deployment.yaml b/chart/pelican-wings/templates/deployment.yaml index e7de1499..f0bb4944 100644 --- a/chart/pelican-wings/templates/deployment.yaml +++ b/chart/pelican-wings/templates/deployment.yaml @@ -91,8 +91,8 @@ spec: {{- end }} volumes: - name: config-source - configMap: - name: {{ include "pelican-wings.fullname" . }}-config + secret: + secretName: {{ include "pelican-wings.fullname" . }}-config - name: config emptyDir: {} {{- if .Values.persistence.enabled }} diff --git a/chart/pelican-wings/templates/limitrange.yaml b/chart/pelican-wings/templates/limitrange.yaml index 13d03582..20243f37 100644 --- a/chart/pelican-wings/templates/limitrange.yaml +++ b/chart/pelican-wings/templates/limitrange.yaml @@ -1,4 +1,7 @@ {{- if .Values.limitRange.enabled }} +{{- if not (or .Values.limitRange.defaultCPULimit .Values.limitRange.defaultMemoryLimit .Values.limitRange.defaultCPURequest .Values.limitRange.defaultMemoryRequest .Values.limitRange.maxCPU .Values.limitRange.maxMemory) }} +{{- fail "limitRange.enabled is true but no limits are set; configure at least one of defaultCPULimit, defaultMemoryLimit, defaultCPURequest, defaultMemoryRequest, maxCPU or maxMemory (an empty LimitRange has no effect)." }} +{{- end }} apiVersion: v1 kind: LimitRange metadata: diff --git a/chart/pelican-wings/templates/resourcequota.yaml b/chart/pelican-wings/templates/resourcequota.yaml index 8ec70eb4..4ac5877f 100644 --- a/chart/pelican-wings/templates/resourcequota.yaml +++ b/chart/pelican-wings/templates/resourcequota.yaml @@ -1,4 +1,7 @@ {{- if .Values.resourceQuota.enabled }} +{{- if not (or .Values.resourceQuota.cpuLimit .Values.resourceQuota.memoryLimit .Values.resourceQuota.cpuRequest .Values.resourceQuota.memoryRequest .Values.resourceQuota.maxPods .Values.resourceQuota.maxPVCs .Values.resourceQuota.maxStorage) }} +{{- fail "resourceQuota.enabled is true but no quota fields are set; configure at least one of cpuLimit, memoryLimit, cpuRequest, memoryRequest, maxPods, maxPVCs or maxStorage (an empty ResourceQuota would block all Pods from scheduling)." }} +{{- end }} apiVersion: v1 kind: ResourceQuota metadata: diff --git a/chart/pelican-wings/values.yaml b/chart/pelican-wings/values.yaml index 9d5dea8a..f7492b42 100644 --- a/chart/pelican-wings/values.yaml +++ b/chart/pelican-wings/values.yaml @@ -30,6 +30,11 @@ serviceAccount: rbac: # -- Whether to create RBAC resources (Role, RoleBinding, ClusterRole, ClusterRoleBinding) create: true + # -- Grant the cluster-wide nodes/proxy permission so Wings can read resource + # metrics directly from the kubelet stats/summary API when metrics-server is + # not installed. This is a broad, cluster-scoped permission; leave it off and + # install metrics-server unless you specifically need the kubelet fallback. + kubeletMetricsFallback: false # -- Namespace for game server resources (Pods, Services, Jobs, PVCs) gameNamespace: pelican diff --git a/environment/kubernetes/container.go b/environment/kubernetes/container.go index a1a716ee..0e23c278 100644 --- a/environment/kubernetes/container.go +++ b/environment/kubernetes/container.go @@ -11,6 +11,7 @@ import ( "emperror.dev/errors" "github.com/apex/log" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" @@ -46,7 +47,11 @@ func (e *Environment) Create() error { ctx := context.Background() // If the Pod already exists, return immediately. - if exists, _ := e.Exists(); exists { + exists, err := e.Exists() + if err != nil { + return errors.Wrap(err, "environment/kubernetes: failed to check pod existence") + } + if exists { return nil } @@ -211,7 +216,7 @@ func (e *Environment) Create() error { e.log().WithField("image", image).Info("creating pod for server") - _, err := e.client.CoreV1().Pods(e.namespace()).Create(ctx, pod, metav1.CreateOptions{}) + _, err = e.client.CoreV1().Pods(e.namespace()).Create(ctx, pod, metav1.CreateOptions{}) if err != nil { return errors.Wrap(err, "environment/kubernetes: failed to create pod") } @@ -252,12 +257,12 @@ func (e *Environment) Destroy() error { e.log().WithField("error", cmErr).Warn("failed to delete identity ConfigMap during destroy") } - e.SetState(environment.ProcessOfflineState) - if err != nil && !isNotFound(err) { return errors.Wrap(err, "environment/kubernetes: failed to delete pod") } + e.SetState(environment.ProcessOfflineState) + return nil } @@ -515,12 +520,17 @@ func (e *Environment) buildContainerPorts() []corev1.ContainerPort { cfg := config.Get() allocs := e.Configuration.Allocations() var ports []corev1.ContainerPort + seen := make(map[int]struct{}) for _, allocPorts := range allocs.Mappings { for _, port := range allocPorts { if port < 1 || port > 65535 { continue } + if _, ok := seen[port]; ok { + continue + } + seen[port] = struct{}{} tcpPort := corev1.ContainerPort{ Name: fmt.Sprintf("tcp-%d", port), @@ -562,7 +572,7 @@ func (e *Environment) getPod(ctx context.Context) (*corev1.Pod, error) { // isNotFound checks if the error is a Kubernetes NotFound error. func isNotFound(err error) bool { - return strings.Contains(err.Error(), "not found") + return apierrors.IsNotFound(err) } // isPodRunning checks if a Pod is in Running phase with a ready container. diff --git a/environment/kubernetes/environment_test.go b/environment/kubernetes/environment_test.go index 433dd9ad..bdc30375 100644 --- a/environment/kubernetes/environment_test.go +++ b/environment/kubernetes/environment_test.go @@ -563,5 +563,25 @@ func TestEnvironment(t *testing.T) { // Only 8080 is valid, so 2 ports (TCP + UDP). g.Assert(len(ports)).Equal(2) }) + + g.It("should deduplicate ports shared across allocation IPs", func() { + allocs := environment.Allocations{ + Mappings: map[string][]int{ + "1.1.1.1": {25565}, + "2.2.2.2": {25565}, + }, + } + settings := environment.Settings{Allocations: allocs} + cfg := environment.NewConfiguration(settings, nil) + env := &Environment{Configuration: cfg} + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkNodePort + }) + + ports := env.buildContainerPorts() + // 25565 appears under two IPs but must yield a single TCP + UDP pair. + g.Assert(len(ports)).Equal(2) + }) }) } diff --git a/environment/kubernetes/installer.go b/environment/kubernetes/installer.go index 62c3e7f6..17759efd 100644 --- a/environment/kubernetes/installer.go +++ b/environment/kubernetes/installer.go @@ -62,10 +62,17 @@ func (ip *InstallerProcess) configMapName() string { // Run executes the full installation process: creates a Job, streams its logs, // waits for completion, and cleans up. func (ip *InstallerProcess) Run(ctx context.Context) error { - // Clean up any existing installer Job from a previous run. - _ = ip.client.BatchV1().Jobs(ip.namespace).Delete(ctx, ip.jobName(), metav1.DeleteOptions{ - PropagationPolicy: propagationBackground(), - }) + // Clean up any existing installer Job from a previous run. Use foreground + // propagation and wait for the Job to disappear so the Create below does not + // race a still-terminating Job and fail with AlreadyExists. + if err := ip.client.BatchV1().Jobs(ip.namespace).Delete(ctx, ip.jobName(), metav1.DeleteOptions{ + PropagationPolicy: propagationForeground(), + }); err != nil && !isNotFound(err) { + return errors.Wrap(err, "environment/kubernetes: failed to delete existing installer job") + } + if err := ip.waitForJobDeletion(ctx, 30*time.Second); err != nil { + return errors.Wrap(err, "environment/kubernetes: timed out waiting for old installer job deletion") + } // Build environment variables. var envVars []corev1.EnvVar @@ -283,7 +290,7 @@ func (ip *InstallerProcess) Cleanup(ctx context.Context) error { // WriteInstallScript creates a ConfigMap containing the installation script. // The ConfigMap is mounted into the Job Pod at /mnt/install, eliminating the // need for a shared filesystem between Wings and the Job Pod. -func (ip *InstallerProcess) WriteInstallScript() error { +func (ip *InstallerProcess) WriteInstallScript(ctx context.Context) error { content := strings.ReplaceAll(ip.Script.Script, "\r\n", "\n") cm := &corev1.ConfigMap{ @@ -301,8 +308,6 @@ func (ip *InstallerProcess) WriteInstallScript() error { }, } - ctx := context.Background() - // Delete any existing ConfigMap from a previous run. _ = ip.client.CoreV1().ConfigMaps(ip.namespace).Delete(ctx, ip.configMapName(), metav1.DeleteOptions{}) @@ -373,7 +378,10 @@ func (ip *InstallerProcess) buildServerDataMount() corev1.VolumeMount { if cfg.Kubernetes.DataPVC != "" { serverPath := filepath.Join(cfg.System.Data, ip.ServerID) rel, err := filepath.Rel(cfg.System.RootDirectory, serverPath) - if err != nil { + // filepath.Rel can yield a path escaping the volume root ("..") when + // System.Data is outside RootDirectory; such a SubPath is rejected by + // Kubernetes, so fall back to the default layout in that case. + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { vm.SubPath = filepath.Join("volumes", ip.ServerID) } else { vm.SubPath = rel @@ -382,6 +390,34 @@ func (ip *InstallerProcess) buildServerDataMount() corev1.VolumeMount { return vm } +// waitForJobDeletion blocks until the installer Job is fully deleted or the +// timeout elapses. +func (ip *InstallerProcess) waitForJobDeletion(ctx context.Context, timeout time.Duration) error { + dctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + for { + _, err := ip.client.BatchV1().Jobs(ip.namespace).Get(dctx, ip.jobName(), metav1.GetOptions{}) + if err != nil && isNotFound(err) { + return nil + } + select { + case <-dctx.Done(): + return dctx.Err() + case <-ticker.C: + } + } +} + +// propagationForeground returns a pointer to the Foreground propagation policy. +func propagationForeground() *metav1.DeletionPropagation { + p := metav1.DeletePropagationForeground + return &p +} + // propagationBackground returns a pointer to the Background propagation policy. func propagationBackground() *metav1.DeletionPropagation { p := metav1.DeletePropagationBackground diff --git a/environment/kubernetes/installer_test.go b/environment/kubernetes/installer_test.go index e4631ea3..83514564 100644 --- a/environment/kubernetes/installer_test.go +++ b/environment/kubernetes/installer_test.go @@ -51,7 +51,7 @@ func TestInstaller(t *testing.T) { }, } - err := ip.WriteInstallScript() + err := ip.WriteInstallScript(context.Background()) g.Assert(err).IsNil() cm, err := client.CoreV1().ConfigMaps("pelican").Get(context.Background(), "write-cm-uuid-install-script", metav1.GetOptions{}) @@ -79,7 +79,7 @@ func TestInstaller(t *testing.T) { }, } - err := ip.WriteInstallScript() + err := ip.WriteInstallScript(context.Background()) g.Assert(err).IsNil() cm, err := client.CoreV1().ConfigMaps("pelican").Get(context.Background(), "rewrite-uuid-install-script", metav1.GetOptions{}) diff --git a/environment/kubernetes/network.go b/environment/kubernetes/network.go index 6068906e..7d881819 100644 --- a/environment/kubernetes/network.go +++ b/environment/kubernetes/network.go @@ -139,15 +139,16 @@ func (e *Environment) EnsureService(ctx context.Context) error { } // Service exists; update it with the desired spec while preserving - // existing NodePort assignments where possible. + // existing NodePort assignments where possible. Type and annotations must + // also be reconciled so a networkMode switch (e.g. NodePort <-> LoadBalancer) + // is applied and stale LB annotations are cleared. + existing.Spec.Type = desired.Spec.Type existing.Spec.Selector = desired.Spec.Selector existing.Spec.Ports = mergeServicePorts(existing.Spec.Ports, desired.Spec.Ports) existing.Labels = labels + existing.Annotations = annotations e.log().WithField("service", svcName).Infof("updating %s service for server", svcType) - if isLB { - existing.Annotations = annotations - } _, err = e.client.CoreV1().Services(ns).Update(ctx, existing, metav1.UpdateOptions{}) if err != nil { return errors.Wrap(err, "environment/kubernetes: failed to update service") diff --git a/environment/kubernetes/network_test.go b/environment/kubernetes/network_test.go index 216ff58b..4928b0b0 100644 --- a/environment/kubernetes/network_test.go +++ b/environment/kubernetes/network_test.go @@ -129,6 +129,42 @@ func TestNetwork(t *testing.T) { g.Assert(len(svc.Spec.Ports)).Equal(4) }) + g.It("should reconcile Service type and annotations when network mode changes", func() { + // Pre-create a LoadBalancer Service with stale LB annotations. + existingSvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gs-test-server-uuid", + Namespace: "pelican", + Annotations: map[string]string{"lbipam.cilium.io/ips": "10.0.0.1"}, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + Selector: map[string]string{"pelican.dev/server-id": "test-server-uuid"}, + }, + } + client := fake.NewSimpleClientset(existingSvc) + allocs := environment.Allocations{ + Mappings: map[string][]int{"0.0.0.0": {25565}}, + } + env := newTestEnv(client, allocs) + + // Switch to NodePort mode. + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkNodePort + c.Kubernetes.Namespace = "pelican" + }) + + err := env.EnsureService(context.Background()) + g.Assert(err).IsNil() + + svc, err := client.CoreV1().Services("pelican").Get(context.Background(), "gs-test-server-uuid", metav1.GetOptions{}) + g.Assert(err).IsNil() + // Type must be updated to NodePort and the stale LB annotation cleared. + g.Assert(svc.Spec.Type).Equal(corev1.ServiceTypeNodePort) + _, hasStale := svc.Annotations["lbipam.cilium.io/ips"] + g.Assert(hasStale).IsFalse() + }) + g.It("should be a no-op with empty allocations", func() { client := fake.NewSimpleClientset() allocs := environment.Allocations{ diff --git a/environment/kubernetes/power.go b/environment/kubernetes/power.go index 7964405c..1f5c30af 100644 --- a/environment/kubernetes/power.go +++ b/environment/kubernetes/power.go @@ -20,12 +20,17 @@ import ( func (e *Environment) OnBeforeStart(ctx context.Context) error { // Delete any existing Pod to ensure fresh config is applied. gracePeriod := int64(0) - _ = e.client.CoreV1().Pods(e.namespace()).Delete(ctx, e.Id, metav1.DeleteOptions{ + if err := e.client.CoreV1().Pods(e.namespace()).Delete(ctx, e.Id, metav1.DeleteOptions{ GracePeriodSeconds: &gracePeriod, - }) + }); err != nil && !isNotFound(err) { + return errors.Wrap(err, "environment/kubernetes: failed to delete existing pod before start") + } - // Wait briefly for deletion to propagate. - e.waitForPodDeletion(ctx, 10*time.Second) + // Wait for the old Pod to be fully removed so the recreate below does not + // no-op against a stale Pod. + if err := e.waitForPodDeletion(ctx, 10*time.Second); err != nil { + return errors.Wrap(err, "environment/kubernetes: timed out waiting for old pod deletion") + } // Create the Pod with current configuration. if err := e.Create(); err != nil { @@ -87,22 +92,23 @@ func (e *Environment) Stop(ctx context.Context) error { s := e.meta.Stop e.mu.RUnlock() - // If using a signal-based stop, terminate with that signal. - if s.Type == "" || s.Type == remote.ProcessStopSignal { - return e.Terminate(ctx, "SIGTERM") - } - if e.st.Load() != environment.ProcessOfflineState { e.SetState(environment.ProcessStoppingState) } // If using a command-based stop and we're attached, send the command. - if e.IsAttached() && s.Type == remote.ProcessStopCommand { + if s.Type == remote.ProcessStopCommand && e.IsAttached() { return e.SendCommand(s.Value) } - // Default: delete the Pod with a 30-second grace period. + // Otherwise (including signal-based stops) delete the Pod gracefully so the + // kubelet sends SIGTERM to PID 1 and the process can shut down within the + // grace period before being force-killed. An explicit SIGKILL maps to an + // immediate force-delete. gracePeriod := int64(30) + if strings.EqualFold(s.Value, "SIGKILL") { + gracePeriod = 0 + } err := e.client.CoreV1().Pods(e.namespace()).Delete(ctx, e.Id, metav1.DeleteOptions{ GracePeriodSeconds: &gracePeriod, }) @@ -153,8 +159,10 @@ func (e *Environment) WaitForStop(ctx context.Context, duration time.Duration, t return err } - // Wait for the Pod to be gone. - if err := e.waitForPodDeletion(tctx, duration); err != nil { + // Wait for the Pod to be gone or to stop running. A command-based stop + // leaves the Pod in a terminal Succeeded/Failed phase without deleting it, + // so waiting only for deletion would block until the timeout. + if err := e.waitForPodStoppedOrDeleted(tctx, duration); err != nil { if terminate { e.log().Warn("pod did not terminate in time, forcing deletion") return e.Terminate(ctx, "SIGKILL") @@ -259,6 +267,36 @@ func (e *Environment) waitForPodRunning(ctx context.Context) error { } } +// waitForPodStoppedOrDeleted blocks until the Pod is deleted or has stopped +// running (a terminal Succeeded/Failed phase), whichever happens first, or the +// timeout elapses. +func (e *Environment) waitForPodStoppedOrDeleted(ctx context.Context, timeout time.Duration) error { + dctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-dctx.Done(): + return dctx.Err() + case <-ticker.C: + pod, err := e.getPod(dctx) + if err != nil { + if isNotFound(err) { + return nil + } + continue + } + if !isPodRunning(pod) { + e.markOffline() + return nil + } + } + } +} + // waitForPodDeletion blocks until the Pod is fully deleted or the timeout // elapses. func (e *Environment) waitForPodDeletion(ctx context.Context, timeout time.Duration) error { diff --git a/environment/kubernetes/quota.go b/environment/kubernetes/quota.go index 4d297a38..5ea01425 100644 --- a/environment/kubernetes/quota.go +++ b/environment/kubernetes/quota.go @@ -31,6 +31,9 @@ func (e *Environment) EnsureResourceQuota(ctx context.Context) error { } existing, err := e.client.CoreV1().ResourceQuotas(ns).Get(ctx, quotaName, metav1.GetOptions{}) + if err != nil && !isNotFound(err) { + return errors.Wrap(err, "environment/kubernetes: failed to get ResourceQuota") + } if err == nil { // Update existing. existing.Spec = quota.Spec @@ -65,6 +68,9 @@ func (e *Environment) EnsureLimitRange(ctx context.Context) error { } existing, err := e.client.CoreV1().LimitRanges(ns).Get(ctx, limitRangeName, metav1.GetOptions{}) + if err != nil && !isNotFound(err) { + return errors.Wrap(err, "environment/kubernetes: failed to get LimitRange") + } if err == nil { // Update existing. existing.Spec = lr.Spec diff --git a/environment/kubernetes/quota_test.go b/environment/kubernetes/quota_test.go index 8c1ca151..c5db9417 100644 --- a/environment/kubernetes/quota_test.go +++ b/environment/kubernetes/quota_test.go @@ -2,13 +2,16 @@ package kubernetes import ( "context" + "errors" "testing" . "github.com/franela/goblin" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" "github.com/pelican-dev/wings/config" "github.com/pelican-dev/wings/environment" @@ -148,6 +151,26 @@ func TestQuota(t *testing.T) { memLimit := rq.Spec.Hard[corev1.ResourceLimitsMemory] g.Assert(memLimit.Cmp(resource.MustParse("64Gi"))).Equal(0) }) + + g.It("should fail fast on a non-NotFound Get error instead of creating", func() { + client := fake.NewSimpleClientset() + client.PrependReactor("get", "resourcequotas", func(action k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, errors.New("boom: api server unavailable") + }) + env := &Environment{ + Id: "quota-geterr-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.ResourceQuota.Enabled = true + c.Kubernetes.ResourceQuota.CPULimit = "16" + }) + + err := env.EnsureResourceQuota(context.Background()) + g.Assert(err != nil).IsTrue() + }) }) g.Describe("EnsureLimitRange", func() { @@ -256,6 +279,26 @@ func TestQuota(t *testing.T) { defaultCPU := lr.Spec.Limits[0].Default[corev1.ResourceCPU] g.Assert(defaultCPU.Cmp(resource.MustParse("4"))).Equal(0) }) + + g.It("should fail fast on a non-NotFound Get error instead of creating", func() { + client := fake.NewSimpleClientset() + client.PrependReactor("get", "limitranges", func(action k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, errors.New("boom: api server unavailable") + }) + env := &Environment{ + Id: "lr-geterr-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + config.Update(func(c *config.Configuration) { + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.LimitRange.Enabled = true + c.Kubernetes.LimitRange.DefaultCPULimit = "2" + }) + + err := env.EnsureLimitRange(context.Background()) + g.Assert(err != nil).IsTrue() + }) }) g.Describe("buildResourceQuota", func() { diff --git a/environment/kubernetes/stats.go b/environment/kubernetes/stats.go index 4ce29482..63c9e501 100644 --- a/environment/kubernetes/stats.go +++ b/environment/kubernetes/stats.go @@ -13,6 +13,10 @@ import ( "github.com/pelican-dev/wings/environment" ) +// statsRequestTimeout bounds each individual metrics/stats API call so a slow +// or hung endpoint cannot stall the polling loop until the next tick. +const statsRequestTimeout = 3 * time.Second + // podStats holds parsed resource metrics for a pod's "server" container. type podStats struct { cpuPercent float64 @@ -37,6 +41,14 @@ func (e *Environment) Uptime(ctx context.Context) (int64, error) { return time.Since(pod.Status.StartTime.Time).Milliseconds(), nil } +// withStatsTimeout invokes fn with a child context bounded by +// statsRequestTimeout so a single slow metrics/stats call cannot stall polling. +func withStatsTimeout[T any](ctx context.Context, fn func(context.Context) (T, error)) (T, error) { + tctx, cancel := context.WithTimeout(ctx, statsRequestTimeout) + defer cancel() + return fn(tctx) +} + // pollResources periodically fetches resource usage and publishes resource // events. It tries the Kubernetes Metrics API first (requires metrics-server) // and falls back to the kubelet stats/summary API (always available). @@ -57,7 +69,6 @@ func (e *Environment) pollResources(ctx context.Context) error { defer ticker.Stop() lastCheck := time.Now() - useKubelet := false loggedSource := false for { @@ -79,30 +90,21 @@ func (e *Environment) pollResources(ctx context.Context) error { var stats *podStats - if !useKubelet { - stats, err = e.getMetricsAPIStats(ctx) - if err != nil { - if !loggedSource { - e.log().WithField("error", err).Info("metrics API unavailable, falling back to kubelet stats") - } - useKubelet = true - } - } - - if useKubelet { - stats, err = e.getKubeletPodStats(ctx) - if err != nil { - if !loggedSource { - e.log().WithField("error", err).Warn("kubelet stats also unavailable, resource metrics will not be reported") - loggedSource = true - } - } + // Prefer the metrics API and fall back to kubelet stats. The metrics + // API is retried on every tick (the fallback is not sticky) so a + // transiently-unavailable metrics-server is picked back up once it + // recovers. Each call is bounded by statsRequestTimeout. + usingKubelet := false + stats, err = withStatsTimeout(ctx, e.getMetricsAPIStats) + if err != nil { + usingKubelet = true + stats, err = withStatsTimeout(ctx, e.getKubeletPodStats) } if stats != nil { if !loggedSource { src := "metrics API (metrics-server)" - if useKubelet { + if usingKubelet { src = "kubelet stats/summary" } e.log().WithField("source", src).Info("collecting pod resource metrics") @@ -114,10 +116,13 @@ func (e *Environment) pollResources(ctx context.Context) error { RxBytes: stats.rxBytes, TxBytes: stats.txBytes, } + } else if !loggedSource { + e.log().WithField("error", err).Warn("pod resource metrics unavailable from both metrics API and kubelet") + loggedSource = true } // Get memory limit from pod spec. - pod, err := e.getPod(ctx) + pod, err := withStatsTimeout(ctx, e.getPod) if err == nil && pod != nil { for _, c := range pod.Spec.Containers { if c.Name == "server" { diff --git a/environment/kubernetes/storage.go b/environment/kubernetes/storage.go index 0b725a78..bd671692 100644 --- a/environment/kubernetes/storage.go +++ b/environment/kubernetes/storage.go @@ -144,6 +144,11 @@ func (e *Environment) ResizePVC(ctx context.Context, newSize string) error { if cfg.Kubernetes.StorageMode != config.KubeStoragePVC { return nil } + // When a shared DataPVC is configured, per-server PVCs are not created, so + // there is nothing to resize (mirrors EnsurePVC/DeletePVC). + if cfg.Kubernetes.DataPVC != "" { + return nil + } ns := e.namespace() name := e.pvcName() diff --git a/environment/kubernetes/storage_test.go b/environment/kubernetes/storage_test.go index 80581a46..65ca8575 100644 --- a/environment/kubernetes/storage_test.go +++ b/environment/kubernetes/storage_test.go @@ -281,6 +281,46 @@ func TestStorage(t *testing.T) { err := env.ResizePVC(context.Background(), "invalid") g.Assert(err).IsNotNil() }) + + g.It("should be a no-op when a shared DataPVC is configured", func() { + existingPVC := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gs-shared-resize-uuid", + Namespace: "pelican", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("10Gi"), + }, + }, + }, + } + client := fake.NewSimpleClientset(existingPVC) + env := &Environment{ + Id: "shared-resize-uuid", + client: client, + st: system.NewAtomicString(environment.ProcessOfflineState), + } + + config.Update(func(c *config.Configuration) { + c.Kubernetes.StorageMode = config.KubeStoragePVC + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.DataPVC = "shared-data" + }) + defer config.Update(func(c *config.Configuration) { + c.Kubernetes.DataPVC = "" + }) + + err := env.ResizePVC(context.Background(), "50Gi") + g.Assert(err).IsNil() + + // The per-server PVC must be left untouched at its original size. + pvc, _ := client.CoreV1().PersistentVolumeClaims("pelican").Get(context.Background(), "gs-shared-resize-uuid", metav1.GetOptions{}) + storageReq := pvc.Spec.Resources.Requests[corev1.ResourceStorage] + original := resource.MustParse("10Gi") + g.Assert(storageReq.Equal(original)).IsTrue() + }) }) g.Describe("buildVolumes with PVC mode", func() { diff --git a/kubernetes/README.md b/kubernetes/README.md index 3be47fec..a967d38f 100644 --- a/kubernetes/README.md +++ b/kubernetes/README.md @@ -45,9 +45,15 @@ Wings requires two levels of RBAC: ### Cluster-scoped -| Resource | Verbs | Purpose | -|---------------------------------------|-------|---------------------------------| -| pods (metrics.k8s.io/v1beta1) | get | CPU/memory usage polling | +| Resource | Verbs | Purpose | +|---------------------------------------|-------|--------------------------------------------------------| +| pods (metrics.k8s.io/v1beta1) | get | CPU/memory usage polling | +| nodes | get | Read node addresses for the IP-allocation endpoint | +| nodes/proxy *(optional)* | get | Kubelet stats fallback when metrics-server is absent | + +> `nodes/proxy` is a broad, cluster-wide permission and is therefore **not** +> granted by `clusterrole-metrics.yaml`. Apply `clusterrole-metrics-kubelet.yaml` +> only if you do not run metrics-server and need the kubelet stats fallback. ## Files @@ -55,8 +61,9 @@ Wings requires two levels of RBAC: - `serviceaccount.yaml` — ServiceAccount for Wings Pods - `role.yaml` — Namespace-scoped Role with required permissions - `rolebinding.yaml` — Binds the Role to the ServiceAccount -- `clusterrole-metrics.yaml` — Cluster-scoped access to metrics API +- `clusterrole-metrics.yaml` — Cluster-scoped access to the metrics API + node addresses - `clusterrolebinding-metrics.yaml` — Binds the ClusterRole to the ServiceAccount +- `clusterrole-metrics-kubelet.yaml` — **Optional** `nodes/proxy` ClusterRole + binding for the kubelet stats fallback ## Configuration @@ -111,4 +118,18 @@ ClusterRoleBinding. Wings will gracefully degrade (no CPU/memory stats). If you run multiple Wings nodes targeting different namespaces, create a Role and RoleBinding per namespace, but share the ClusterRole/ClusterRoleBinding (it's -namespace-independent). +namespace-independent). Give each instance its own ServiceAccount and use a +distinct `ClusterRoleBinding` subject per ServiceAccount so the cluster-scoped +metrics permission is granted to every Wings ServiceAccount that needs it. + +Do **not** point multiple Wings nodes at the same namespace: they share Pod/ +Service/Job names derived from server UUIDs and the namespaced ResourceQuota/ +LimitRange (`pelican-wings`), so concurrent reconciliation would conflict. + +### Helm-managed installs + +The Helm chart under `chart/pelican-wings` renders all of these resources (and a +config **Secret**) for you. When `serviceAccount.create=false` you must set +`serviceAccount.name` explicitly — the chart refuses to bind its RBAC to the +namespace `default` ServiceAccount. Enable the kubelet fallback with +`rbac.kubeletMetricsFallback=true`. diff --git a/kubernetes/clusterrole-metrics-kubelet.yaml b/kubernetes/clusterrole-metrics-kubelet.yaml new file mode 100644 index 00000000..c95eb551 --- /dev/null +++ b/kubernetes/clusterrole-metrics-kubelet.yaml @@ -0,0 +1,39 @@ +# OPTIONAL kubelet stats/summary fallback for resource metrics. +# +# Only apply this if you do NOT run metrics-server and want Wings to read CPU/ +# memory usage directly from the kubelet. nodes/proxy is a broad, cluster-wide +# permission, so it is kept out of the default clusterrole-metrics.yaml and must +# be opted into explicitly: +# +# kubectl apply -f clusterrole-metrics-kubelet.yaml +# +# Adjust the ServiceAccount name/namespace below to match your install. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: pelican-wings-metrics-kubelet + labels: + app.kubernetes.io/managed-by: pelican-wings + app.kubernetes.io/component: wings +rules: + # Proxy to the kubelet stats/summary API for resource metrics when + # metrics-server is not installed. + - apiGroups: [""] + resources: ["nodes/proxy"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: pelican-wings-metrics-kubelet + labels: + app.kubernetes.io/managed-by: pelican-wings + app.kubernetes.io/component: wings +subjects: + - kind: ServiceAccount + name: pelican-wings + namespace: pelican +roleRef: + kind: ClusterRole + name: pelican-wings-metrics-kubelet + apiGroup: rbac.authorization.k8s.io diff --git a/kubernetes/clusterrole-metrics.yaml b/kubernetes/clusterrole-metrics.yaml index 88b461c8..82624ffd 100644 --- a/kubernetes/clusterrole-metrics.yaml +++ b/kubernetes/clusterrole-metrics.yaml @@ -17,8 +17,7 @@ rules: - apiGroups: [""] resources: ["nodes"] verbs: ["get"] - # Proxy to kubelet stats/summary API for resource metrics when - # metrics-server is not installed. - - apiGroups: [""] - resources: ["nodes/proxy"] - verbs: ["get"] + # NOTE: The kubelet stats/summary fallback (nodes/proxy) is intentionally NOT + # granted here because it is a broad, cluster-wide permission. If you do not + # run metrics-server and need that fallback, additionally apply + # clusterrole-metrics-kubelet.yaml. diff --git a/router/router_system.go b/router/router_system.go index ed502a7c..10e32e3d 100644 --- a/router/router_system.go +++ b/router/router_system.go @@ -125,6 +125,14 @@ func getSystemIps(c *gin.Context) { interfaces = append(interfaces, ip) } } + + // If IP discovery failed across the board, surface an error instead of + // returning a misleading empty-but-successful response that the Panel + // would treat as "no assignable IPs". + if len(interfaces) == 0 { + middleware.CaptureAndAbort(c, errors.New("failed to discover any assignable IP addresses in kubernetes mode; check node/LoadBalancer RBAC or configure kubernetes.system_ips")) + return + } } else { ips, err := system.GetSystemIps() if err != nil { diff --git a/server/install_kubernetes.go b/server/install_kubernetes.go index 63c480eb..c2a010fa 100644 --- a/server/install_kubernetes.go +++ b/server/install_kubernetes.go @@ -39,7 +39,7 @@ func (s *Server) internalInstallKubernetes(script *remote.InstallationScript) er s.Events().Publish(DaemonMessageEvent, "Starting installation process via Kubernetes Job, this could take a few minutes...") // Write the install script to disk for mounting into the Job Pod. - if err := ip.WriteInstallScript(); err != nil { + if err := ip.WriteInstallScript(s.Context()); err != nil { return errors.WithMessage(err, "install: failed to write installation script") } diff --git a/server/update.go b/server/update.go index 1bf603f3..b3dccee0 100644 --- a/server/update.go +++ b/server/update.go @@ -4,6 +4,7 @@ import ( "time" "github.com/pelican-dev/wings/environment/docker" + "github.com/pelican-dev/wings/environment/kubernetes" "github.com/pelican-dev/wings/environment" ) @@ -39,6 +40,14 @@ func (s *Server) SyncWithEnvironment() { e.SetStopConfiguration(s.ProcessConfiguration().Stop) } + // The Kubernetes environment likewise needs the configured image and stop + // configuration so command-based stops and image updates take effect. + if e, ok := s.Environment.(*kubernetes.Environment); ok { + s.Log().Debug("syncing stop configuration with configured kubernetes environment") + e.SetImage(cfg.Container.Image) + e.SetStopConfiguration(s.ProcessConfiguration().Stop) + } + // If build limits are changed, environment variables also change. Plus, any modifications to // the startup command also need to be properly propagated to this environment. s.Environment.Config().SetEnvironmentVariables(s.GetEnvironmentVariables()) diff --git a/system/system.go b/system/system.go index 8204c442..29a9f36f 100644 --- a/system/system.go +++ b/system/system.go @@ -112,7 +112,13 @@ func GetSystemInformationWithOptions(kubernetesMode bool) (*Information, error) release, err := osrelease.Read() if err != nil { - return nil, err + // In Kubernetes mode the daemon frequently runs on minimal images that + // lack /etc/os-release; treat it as best-effort and fall back to the + // runtime OS rather than failing the whole system info request. + if !kubernetesMode { + return nil, err + } + release = map[string]string{} } if kubernetesMode { From 63651758cf6a1c9beaaf77c412daa945e09fc186 Mon Sep 17 00:00:00 2001 From: Exonical Date: Sat, 20 Jun 2026 05:19:25 +0000 Subject: [PATCH 5/6] ci(helm-lint): pin actions to commit SHAs and disable persisted credentials --- .github/workflows/helm-lint.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/helm-lint.yaml b/.github/workflows/helm-lint.yaml index e9d58d2b..89cb2ee5 100644 --- a/.github/workflows/helm-lint.yaml +++ b/.github/workflows/helm-lint.yaml @@ -18,9 +18,11 @@ jobs: lint-helm: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false - - uses: azure/setup-helm@v5 + - uses: azure/setup-helm@f0accbfd55e3332a28f721b8202b1016cecf90d5 # v5 - name: Lint Helm chart run: helm lint chart/pelican-wings From 8e5c3039e43f7adf64d346f990736ef835724660 Mon Sep 17 00:00:00 2001 From: Exonical Date: Sat, 20 Jun 2026 05:29:21 +0000 Subject: [PATCH 6/6] fix(k8s): seed stop config in manager, sync RBAC docs, harden install test - server/manager.go: initialize Kubernetes Metadata with the server stop configuration so command/signal stop behavior is correct from first boot. - kubernetes/README.md: sync the namespace-scoped RBAC verb table with role.yaml (pods watch, services list, configmaps update). - chart README: use a local values file for credentials in the quick-start instead of leaking them via --set. - installer_test.go: assert the Run cancellation path returns context.Canceled. --- chart/pelican-wings/README.md | 20 ++++++++++++++------ environment/kubernetes/installer_test.go | 5 +++-- kubernetes/README.md | 6 +++--- server/manager.go | 3 +++ 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/chart/pelican-wings/README.md b/chart/pelican-wings/README.md index d2692e21..8b55e0e9 100644 --- a/chart/pelican-wings/README.md +++ b/chart/pelican-wings/README.md @@ -12,13 +12,21 @@ networking support. ## Quick Start +Put your node credentials from the Panel in a local values file (kept out of +version control) rather than on the command line, where `--set` would leak them +into shell history and process listings: + +```yaml +# values.local.yaml (do not commit) +wings: + panelUrl: https://panel.example.com + token: YOUR_TOKEN + tokenId: YOUR_TOKEN_ID + uuid: YOUR_NODE_UUID +``` + ```bash -# Add your node credentials from the Panel -helm install wings ./chart/pelican-wings \ - --set wings.panelUrl=https://panel.example.com \ - --set wings.token=YOUR_TOKEN \ - --set wings.tokenId=YOUR_TOKEN_ID \ - --set wings.uuid=YOUR_NODE_UUID +helm install wings ./chart/pelican-wings -f values.local.yaml ``` ## Configuration diff --git a/environment/kubernetes/installer_test.go b/environment/kubernetes/installer_test.go index 83514564..029f373d 100644 --- a/environment/kubernetes/installer_test.go +++ b/environment/kubernetes/installer_test.go @@ -2,6 +2,7 @@ package kubernetes import ( "context" + "errors" "os" "path/filepath" "testing" @@ -179,9 +180,9 @@ func TestInstaller(t *testing.T) { // Verify backoff limit. g.Assert(*job.Spec.BackoffLimit).Equal(int32(0)) - // Cancel to stop waitForJob. + // Cancel to stop waitForJob and assert the cancellation propagates. cancel() - <-errCh + g.Assert(errors.Is(<-errCh, context.Canceled)).IsTrue() }) g.It("should succeed when Job completes successfully", func() { diff --git a/kubernetes/README.md b/kubernetes/README.md index a967d38f..9b113ce4 100644 --- a/kubernetes/README.md +++ b/kubernetes/README.md @@ -33,12 +33,12 @@ Wings requires two levels of RBAC: | Resource | Verbs | Purpose | |---------------------------|--------------------------------|------------------------------------------| -| pods | get, create, delete, list | Game server Pod lifecycle | +| pods | get, create, delete, list, watch | Game server Pod lifecycle | | pods/log | get | Stream server console output | | pods/attach | create | Interactive console (SPDY attach) | -| services | get, create, update, delete | NodePort Service management | +| services | get, create, update, delete, list | NodePort/LoadBalancer Service management | | jobs | get, create, delete | Egg installation scripts | -| configmaps | get, create, delete | Install script storage (multi-node) | +| configmaps | get, create, update, delete | Install script + identity files (multi-node) | | resourcequotas | get, create, update | Namespace resource limits (if enabled) | | limitranges | get, create, update | Container default limits (if enabled) | | persistentvolumeclaims | get, create, update, delete, list | PVC storage lifecycle (if enabled) | diff --git a/server/manager.go b/server/manager.go index 2263f1ad..78a97f8d 100644 --- a/server/manager.go +++ b/server/manager.go @@ -215,6 +215,9 @@ func (m *Manager) InitServer(data remote.ServerConfigurationResponse) (*Server, meta := kubernetes.Metadata{ Image: s.Config().Container.Image, } + if pc := s.ProcessConfiguration(); pc != nil { + meta.Stop = pc.Stop + } if env, err := kubernetes.New(s.ID(), &meta, envCfg); err != nil { return nil, err } else {