diff --git a/charts/ome-alfred/Chart.yaml b/charts/ome-alfred/Chart.yaml new file mode 100644 index 000000000..6ed7053ae --- /dev/null +++ b/charts/ome-alfred/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: ome-alfred +description: Alfred, the OME GPU cluster caretaker (OEP-0008) — observes the physical GPU layer and recommends corrective migrations; recommend-only by default. +type: application +version: 0.1.0 +appVersion: "0.1.0" diff --git a/charts/ome-alfred/templates/NOTES.txt b/charts/ome-alfred/templates/NOTES.txt new file mode 100644 index 000000000..88c5a7345 --- /dev/null +++ b/charts/ome-alfred/templates/NOTES.txt @@ -0,0 +1,19 @@ +{{- if .Values.enabled }} +Alfred is deployed in {{ .Release.Namespace }} (mode: {{ .Values.alfredConfig.mode }}). + +Watch what it observes and recommends: + + # Snapshot-derived gauges (any replica): + kubectl -n {{ .Release.Namespace }} port-forward svc/ome-alfred-metrics {{ .Values.metrics.port }}:{{ .Values.metrics.port }} + curl -s localhost:{{ .Values.metrics.port }}/metrics | grep '^alfred_' + + # Recommendations: + kubectl get events -A --field-selector reason=RecommendationWithheld + kubectl -n {{ .Release.Namespace }} get cm {{ include "ome-alfred.recommendationsName" . }} -o jsonpath='{.data.last-cycle\.json}' + +Policy configuration hot-reloads from the {{ .Values.configMapName }} ConfigMap; +an edit that fails validation is rejected and the last-known-good stays in +force (see the PolicyReloadFailed events on the ConfigMap if edits do not land). +{{- else }} +ome-alfred is disabled (enabled: false); nothing was installed. +{{- end }} diff --git a/charts/ome-alfred/templates/_helpers.tpl b/charts/ome-alfred/templates/_helpers.tpl new file mode 100644 index 000000000..4a391ccda --- /dev/null +++ b/charts/ome-alfred/templates/_helpers.tpl @@ -0,0 +1,37 @@ +{{/* +Common labels. The control-plane label is the stable selector key and must +never change; the rest follow chart conventions. +*/}} +{{- define "ome-alfred.labels" -}} +control-plane: ome-alfred +app.kubernetes.io/component: "ome-alfred" +app.kubernetes.io/name: ome-alfred +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | quote }} +{{- end }} + +{{/* +Full image URL with optional global hub prefix — the same contract as +ome-resources' ome.imageWithHub: if the repository already contains '/', the +hub is ignored. +*/}} +{{- define "ome-alfred.image" -}} +{{- $hub := .Values.global.hub }} +{{- $repo := .Values.image.repository }} +{{- $tag := .Values.image.tag | toString }} +{{- if and $hub (not (contains "/" $repo)) -}} +{{- printf "%s/%s:%s" $hub $repo $tag -}} +{{- else -}} +{{- printf "%s:%s" $repo $tag -}} +{{- end -}} +{{- end }} + +{{/* +Name of the recommendations ConfigMap — single source of truth is the policy +config, so the pre-created ConfigMap and the RBAC write grant can never +drift from what the reporter writes to. +*/}} +{{- define "ome-alfred.recommendationsName" -}} +{{- .Values.alfredConfig.recommendationsConfigMapName | default "alfred-recommendations" -}} +{{- end }} diff --git a/charts/ome-alfred/templates/configmap.yaml b/charts/ome-alfred/templates/configmap.yaml new file mode 100644 index 000000000..18917a6b3 --- /dev/null +++ b/charts/ome-alfred/templates/configmap.yaml @@ -0,0 +1,26 @@ +{{- if .Values.enabled }} +# Alfred's policy configuration (OEP-0008 §Configuration). Alfred watches +# this ConfigMap and hot-reloads it; a config that fails schema validation is +# rejected and the last-known-good stays in force. +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.configMapName }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "ome-alfred.labels" . | nindent 4 }} +data: + config.yaml: | + {{- .Values.alfredConfig | toYaml | nindent 4 }} +--- +# Pre-created because Alfred's RBAC lets it update but never create a +# ConfigMap; the reporter writes its recommendations here. Data written by +# Alfred is not managed by this chart and survives upgrades. +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "ome-alfred.recommendationsName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "ome-alfred.labels" . | nindent 4 }} +{{- end }} diff --git a/charts/ome-alfred/templates/deployment.yaml b/charts/ome-alfred/templates/deployment.yaml new file mode 100644 index 000000000..4b545fc6c --- /dev/null +++ b/charts/ome-alfred/templates/deployment.yaml @@ -0,0 +1,105 @@ +{{- if .Values.enabled }} +# Alfred, the GPU cluster caretaker (OEP-0008). Every replica runs the +# read-only observation loop; only the leader decides and dispatches. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ome-alfred + namespace: {{ .Release.Namespace }} + labels: + {{- include "ome-alfred.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + control-plane: ome-alfred + template: + metadata: + labels: + {{- include "ome-alfred.labels" . | nindent 8 }} + annotations: + kubectl.kubernetes.io/default-container: alfred + prometheus.io/scrape: "true" + prometheus.io/port: {{ .Values.metrics.port | quote }} + prometheus.io/path: "/metrics" + # A config change takes effect via hot reload, no restart needed; + # this checksum only restarts pods when the rendered document + # changes AND the pod template is being applied anyway. + checksum/alfred-config: {{ .Values.alfredConfig | toYaml | sha256sum }} + spec: + serviceAccountName: ome-alfred + {{- with .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: alfred + image: {{ include "ome-alfred.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - "--metrics-bind-address=:{{ .Values.metrics.port }}" + - "--health-probe-bind-address=:{{ .Values.healthPort }}" + - "--leader-elect=true" + - "--config-name={{ .Values.configMapName }}" + - "--config-key=config.yaml" + env: + # Alfred reads its namespace from the downward API: it holds the + # config ConfigMap and the leader-election Lease. + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + # Labels alfred_leader_status so the leader is identifiable. + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + ports: + - name: metrics + containerPort: {{ .Values.metrics.port }} + protocol: TCP + - name: health + containerPort: {{ .Values.healthPort }} + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: health + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /readyz + port: health + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 10 + resources: + {{- toYaml .Values.resources | nindent 10 }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + terminationGracePeriodSeconds: 10 +{{- end }} diff --git a/charts/ome-alfred/templates/rbac.yaml b/charts/ome-alfred/templates/rbac.yaml new file mode 100644 index 000000000..999a91b51 --- /dev/null +++ b/charts/ome-alfred/templates/rbac.yaml @@ -0,0 +1,121 @@ +{{- if .Values.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ome-alfred + namespace: {{ .Release.Namespace }} + labels: + {{- include "ome-alfred.labels" . | nindent 4 }} +--- +# Alfred's cluster-scoped permissions are read-only plus two narrow writes +# (OEP-0008 §RBAC): no node writes, no pod writes, no pods/eviction. Every +# pod-level action is executed by the controller that owns the workload. +# Anything confined to Alfred's own namespace — its ConfigMaps, its +# leader-election Lease — lives in the namespaced Role instead. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ome-alfred + labels: + {{- include "ome-alfred.labels" . | nindent 4 }} +rules: + # Nodes (read-only) — no cordon, no drain, no patch. + - apiGroups: [ "" ] + resources: [ "nodes" ] + verbs: [ "get", "list", "watch" ] + + # Pods (read-only) — candidate selection reads placement; eviction is the + # owning controller's job. + - apiGroups: [ "" ] + resources: [ "pods" ] + verbs: [ "get", "list", "watch" ] + + # PVCs / PVs (read-only) — PVC-backed model topology in the snapshot. + - apiGroups: [ "" ] + resources: [ "persistentvolumeclaims", "persistentvolumes" ] + verbs: [ "get", "list", "watch" ] + + # Events — recommendations, migrations, evacuation and repair signals, on + # workloads and nodes in any namespace. + - apiGroups: [ "" ] + resources: [ "events" ] + verbs: [ "create", "patch" ] + + # OME CRDs (read). + - apiGroups: [ "ome.io" ] + resources: + - inferenceservices + - inferenceservices/status + - servingruntimes + - clusterservingruntimes + - basemodels + - clusterbasemodels + verbs: [ "get", "list", "watch" ] + + # The single executable contract: migration-request annotations on + # InferenceServices, written only in execute mode. + - apiGroups: [ "ome.io" ] + resources: [ "inferenceservices" ] + verbs: [ "patch" ] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ome-alfred + labels: + {{- include "ome-alfred.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ome-alfred +subjects: +- kind: ServiceAccount + name: ome-alfred + namespace: {{ .Release.Namespace }} +--- +# Everything Alfred touches only inside its own namespace. These rules are +# deliberately not in the ClusterRole: a resourceNames restriction cannot +# narrow a cluster-wide list/watch, so granting ConfigMap reads there would +# expose every ConfigMap in the cluster. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ome-alfred + namespace: {{ .Release.Namespace }} + labels: + {{- include "ome-alfred.labels" . | nindent 4 }} +rules: + # ConfigMaps: read for policy loading — Alfred's informer cache is scoped + # to this namespace — and write only to the two pre-created Alfred-owned + # ConfigMaps, which it never creates at runtime. + - apiGroups: [ "" ] + resources: [ "configmaps" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "" ] + resources: [ "configmaps" ] + verbs: [ "update", "patch", "delete" ] + resourceNames: + - {{ .Values.configMapName }} + - {{ include "ome-alfred.recommendationsName" . }} + + # Leader election — the alfred.ome.io Lease lives in Alfred's namespace. + - apiGroups: [ "coordination.k8s.io" ] + resources: [ "leases" ] + verbs: [ "create", "get", "update" ] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ome-alfred + namespace: {{ .Release.Namespace }} + labels: + {{- include "ome-alfred.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ome-alfred +subjects: +- kind: ServiceAccount + name: ome-alfred + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/charts/ome-alfred/templates/service.yaml b/charts/ome-alfred/templates/service.yaml new file mode 100644 index 000000000..5330e55dc --- /dev/null +++ b/charts/ome-alfred/templates/service.yaml @@ -0,0 +1,18 @@ +{{- if .Values.enabled }} +# Metrics endpoint: every replica observes, so scraping any pod works. +apiVersion: v1 +kind: Service +metadata: + name: ome-alfred-metrics + namespace: {{ .Release.Namespace }} + labels: + {{- include "ome-alfred.labels" . | nindent 4 }} +spec: + selector: + control-plane: ome-alfred + ports: + - name: metrics + port: {{ .Values.metrics.port }} + targetPort: metrics + protocol: TCP +{{- end }} diff --git a/charts/ome-alfred/templates/servicemonitor.yaml b/charts/ome-alfred/templates/servicemonitor.yaml new file mode 100644 index 000000000..17d3e67d0 --- /dev/null +++ b/charts/ome-alfred/templates/servicemonitor.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.enabled .Values.metrics.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: ome-alfred + namespace: {{ .Release.Namespace }} + labels: + {{- include "ome-alfred.labels" . | nindent 4 }} + {{- with .Values.metrics.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + control-plane: ome-alfred + endpoints: + - port: metrics + path: /metrics + interval: {{ .Values.metrics.serviceMonitor.interval }} + scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }} +{{- end }} diff --git a/charts/ome-alfred/values.yaml b/charts/ome-alfred/values.yaml new file mode 100644 index 000000000..7f13750ba --- /dev/null +++ b/charts/ome-alfred/values.yaml @@ -0,0 +1,137 @@ +# Whether this chart renders anything: lets ome-alfred ship disabled as a +# dependency of an umbrella chart (the modelAgent.enabled pattern). +enabled: true + +global: + # Image hub prefixed to the repository unless the repository already + # contains a '/' (the same helper contract as ome-resources). + hub: "ghcr.io/moirai-internal" + imagePullSecrets: [] + +# Production runs three replicas: every replica runs the read-only +# observation loop (scrape any pod), only the leader decides. +replicaCount: 3 + +image: + repository: alfred + # CI publishes no released alfred tag yet; pin this to a digest once + # alfred joins the image matrix (build one with `make push-alfred-image`). + tag: latest + pullPolicy: Always + +resources: + requests: + cpu: 100m + memory: 256Mi + # The cluster-wide pod cache dominates Alfred's memory; raise the memory + # limit on clusters with many thousands of pods. + limits: + cpu: "1" + memory: 1Gi + +nodeSelector: {} +tolerations: [] +affinity: {} + +metrics: + port: 8080 + serviceMonitor: + # Requires the Prometheus Operator CRDs. + enabled: false + additionalLabels: {} + interval: 30s + scrapeTimeout: 10s + +healthPort: 8081 + +# Name of the policy ConfigMap: the --config-name flag, the rendered +# ConfigMap, and the namespaced RBAC write grant all follow this value. +configMapName: alfred-config + +# Alfred's policy configuration (OEP-0008 §Configuration), rendered verbatim +# into 's config.yaml and hot-reloaded by every replica. A +# config that fails schema validation is rejected at reload and the +# last-known-good stays in force. +alfredConfig: + schemaVersion: 1 + + # recommend-only | execute. recommend-only never dispatches a migration: + # policies, arbitration and reporting all run, only the dispatcher is off. + mode: recommend-only + + # Loop cadence. earlyTickOn advances the next decision tick on the named + # events; it never interrupts a pass ([] disables). + decisionLoopInterval: 5m + observationLoopInterval: 30s + earlyTickOn: [NodeConditionChange] + + policies: + defragmentation: + enabled: true + fragmentationThreshold: 0.25 + # conservative | balanced | aggressive — a scoring knob, never a + # safety bypass. + aggressiveness: balanced + scoring: + # Within-node demand sizes, ascending. + sizeLadder: [1, 2, 4, 8] + # 0: pure observed demand; 1: pure prior. + demandBlendLambda: 0.3 + # Static prior over demand sizes; keys must be ladder sizes and + # weights must sum to 1. + sizePrior: + "1": 0.1 + "2": 0.1 + "4": 0.2 + "8": 0.6 + pendingUrgencyTauMinutes: 30 + nodeHealth: + enabled: true + aggressiveness: balanced + # Consumed from existing node conditions; Alfred never detects health. + triggerConditions: [GpuUnhealthy] + signalOnly: false + healthCooldownFloorMinutes: 5 + nodeSuspicionWindowMinutes: 30 + + # Per-workload defaults; alfred.ome.io/* annotations on an + # InferenceService win over these. + defaultMovable: true + recentPlacementCooldownMinutes: 10 + perWorkloadCooldownMinutes: 30 + perNodeCooldownMinutes: 10 + + # Execution surfaces — all annotation-mediated; owning controllers execute. + rawDeploymentMigrationEnabled: true + omenativeMigrationEnabled: true + lwsRecommendationsEnabled: true + + # Output. The recommendations ConfigMap is pre-created by this chart; + # Alfred updates it but never creates it. + recommendationsConfigMapEnabled: true + recommendationsConfigMapName: alfred-recommendations + + # Global safety bounds, applied once by the arbiter across every policy. + maxInFlightMigrations: 3 + maxMigrationsPerHour: 10 + emergencyPendingAgeMinutes: 10 + + # Weekly UTC windows gating defrag dispatch; empty means no window + # applies. Node-health evacuation deliberately overrides them. + # maintenanceWindows: + # - days: [Sat, Sun] + # start: "02:00" + # end: "06:00" + maintenanceWindows: [] + + spotPolicy: + avoidAsTarget: true + preferAsSource: true + preemptibleLabels: + - node.kubernetes.io/preemptible + - cloud.google.com/gke-preemptible + + allowCrossTenantOptimization: true + + logLevel: info + structuredLogging: true diff --git a/pkg/alfred/config/shipped_defaults_test.go b/pkg/alfred/config/shipped_defaults_test.go new file mode 100644 index 000000000..a57fb2914 --- /dev/null +++ b/pkg/alfred/config/shipped_defaults_test.go @@ -0,0 +1,69 @@ +package config + +import ( + "os" + "strings" + "testing" + + "sigs.k8s.io/yaml" +) + +// TestShippedDefaultConfigsAreValid is the golden check on every config.yaml +// this repo ships: each must pass Alfred's own schema validation and keep the +// safe default — recommend-only — so an install can never start acting +// because a default drifted. +func TestShippedDefaultConfigsAreValid(t *testing.T) { + shipped := map[string]func(t *testing.T) []byte{ + "charts/ome-alfred values.alfredConfig": func(t *testing.T) []byte { + raw, err := os.ReadFile("../../../charts/ome-alfred/values.yaml") + if err != nil { + t.Fatal(err) + } + var values struct { + AlfredConfig map[string]interface{} `json:"alfredConfig"` + } + if err := yaml.Unmarshal(raw, &values); err != nil { + t.Fatal(err) + } + doc, err := yaml.Marshal(values.AlfredConfig) + if err != nil { + t.Fatal(err) + } + return doc + }, + "config/alfred configmap": func(t *testing.T) []byte { + raw, err := os.ReadFile("../../../config/alfred/configmap.yaml") + if err != nil { + t.Fatal(err) + } + // The manifest is multi-document; alfred-config comes first. + first := strings.SplitN(string(raw), "\n---", 2)[0] + var cm struct { + Data map[string]string `json:"data"` + } + if err := yaml.Unmarshal([]byte(first), &cm); err != nil { + t.Fatal(err) + } + doc, ok := cm.Data["config.yaml"] + if !ok { + t.Fatal("config.yaml key missing from alfred-config manifest") + } + return []byte(doc) + }, + } + + for name, extract := range shipped { + t.Run(name, func(t *testing.T) { + cfg, err := Load(extract(t)) + if err != nil { + t.Fatalf("shipped default rejected by Alfred's own validation: %v", err) + } + if cfg.Mode != ModeRecommendOnly { + t.Fatalf("shipped default mode = %q; the safe default is %q", cfg.Mode, ModeRecommendOnly) + } + if !*cfg.Policies.Defragmentation.Enabled { + t.Fatal("shipped default should enable defragmentation (recommend-only makes it safe)") + } + }) + } +}