Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions pkg/controller/v1beta1/workload/types/conditions.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package types

// ConditionType / ConditionReason are workload-internal identifiers
// stamped on metav1.Condition entries. Values match the legacy
// omenative/status strings byte-for-byte so operator dashboards keep
// matching.
type ConditionType string

func (t ConditionType) String() string { return string(t) }

type ConditionReason string

func (r ConditionReason) String() string { return string(r) }

const (
// ConditionGangSchedulingUnavailable is True when the Component
// has at least one multi-pod Instance but the scheduler-plugins
// PodGroup CRD is missing. The reconciler still creates pods —
// gang scheduling is a soft requirement so workloads proceed
// without blocking — but partial-gang placement is possible and
// the runtime may hang.
ConditionGangSchedulingUnavailable ConditionType = "GangSchedulingUnavailable"
)

const (
// ReasonPodGroupCRDNotInstalled stamps the
// GangSchedulingUnavailable condition when the
// scheduler-plugins PodGroup CRD is missing.
ReasonPodGroupCRDNotInstalled ConditionReason = "PodGroupCRDNotInstalled"
// ReasonGangSchedulingAvailable stamps Status=False (CRD present,
// or Component is single-pod).
ReasonGangSchedulingAvailable ConditionReason = "GangSchedulingAvailable"
)
117 changes: 117 additions & 0 deletions pkg/controller/v1beta1/workload/types/deps.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
package types

import (
"context"
"time"

corev1 "k8s.io/api/core/v1"
"k8s.io/client-go/tools/record"
"k8s.io/utils/clock"
"sigs.k8s.io/controller-runtime/pkg/client"
)

// ObservationReader is a watch-backed (cached) reader; it may lag the
// API server. Safe wherever a later live read or optimistic-concurrency
// write re-validates the observation.
type ObservationReader = client.Reader

// AuthoritativeReader reads live from the API server. Required where
// cache lag is a correctness bug: revision bookkeeping, audit ledger,
// EndpointSlice drain checks, gang topology proofs.
type AuthoritativeReader = client.Reader

// Deps is the manager-scoped wiring the workload reconciler needs.
// Shared across every workload the same controller manages (unlike
// the per-workload ReconcileInput).
type Deps struct {
// Client is the controller-runtime cached client used for the
// vast majority of read/write operations.
Client client.Client

// APIReader is the AuthoritativeReader role (see type docs).
// Used for revision bookkeeping and immutable topology observations where
// stale cache contents could cause a collision or split a live gang. When
// nil the cached Client is used.
APIReader client.Reader

// Recorder emits K8s Events at lifecycle transitions. nil-safe:
// when unset, event helpers are no-ops.
Recorder record.EventRecorder

// Expectations is the create/delete bookkeeping cache used to
// avoid re-issuing batches before the controller-runtime watch
// has confirmed prior writes. Adapters wire the same instance
// into the Pod event handler so observed create / delete events
// release expectations immediately. nil falls back to
// DefaultExpectations.
Expectations *Expectations

// RenderHook is an optional per-pod template mutator invoked
// after composing the canonical pod (name, hostname, labels,
// controller env, owner ref) but before c.Create. The IR
// reconciler (the sole workload.Reconcile caller) wires this to
// coordination.InjectPeerEnv via core.ISVCRenderHook so
// OME_<PEER>_ENDPOINT vars land on every container.
//
// The hook MUST be idempotent — Render may be invoked twice in
// the same reconcile (e.g., cache-lag retry after AlreadyExists).
RenderHook RenderHook

// EnsureGangPodGroup, when set, creates the scheduler-plugins
// PodGroup for one multi-pod surge Instance just before its pods are
// created. The gang-surge op needs PodGroup-before-pods to hold even
// in the window before the surge index lands in the plan the
// top-level EnsurePodGroups keys off (else a gang scheduler rejects
// the surge pods with "PodGroup not found"). The callback lives on
// Deps rather than as a direct call so the workload/ops package stays
// free of the workload/podgroup dependency (ops is imported by
// podgroup's test deps; a direct edge would close a cycle). The gang
// package wires it; nil callback / single-pod Instance / absent CRD
// all no-op. Idempotent.
EnsureGangPodGroup EnsureGangPodGroupFn

// Clock supplies wall-clock time for deadlines and status
// timestamps. nil falls back to the real clock — inject a fake
// (k8s.io/utils/clock/testing) for deterministic boundary tests.
Clock clock.Clock
}

// RenderHook is the optional adapter-specific render-time mutator.
// Receives the pod under construction; may mutate any field except
// Name/Namespace/OwnerReferences (workload-controlled).
type RenderHook func(pod *corev1.Pod, runnerName string, ordinal int32, revisionHash string)

// EnsureGangPodGroupFn ensures the PodGroup for one multi-pod surge Instance
// and returns the effective topology key selected for its live pods. See
// Deps.EnsureGangPodGroup.
type EnsureGangPodGroupFn func(ctx context.Context, input ReconcileInput, plan ComponentPlan, inst InstancePlan) (string, error)

// Reader returns the live API reader when APIReader is set, otherwise
// the cached Client. Use for reads that must not observe a stale
// cache.
func (d *Deps) Reader() client.Reader {
if d.APIReader != nil {
return d.APIReader
}
return d.Client
}

// ExpectationsCache returns the per-controller cache threaded through
// Deps when set, otherwise the DefaultExpectations singleton.
func (d *Deps) ExpectationsCache() *Expectations {
if d.Expectations != nil {
return d.Expectations
}
return DefaultExpectations
}

// Now returns the injected clock's time, or time.Now() when no clock
// is wired. Lifecycle code holding a Deps or ReconcileInput should read
// time through Now, not time.Now(); subpackages without a seam
// (audit, podreadiness, gang) still read real time.
func (d *Deps) Now() time.Time {
if d.Clock != nil {
return d.Clock.Now()
}
return time.Now()
}
142 changes: 142 additions & 0 deletions pkg/controller/v1beta1/workload/types/events.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
package types

// EventReason is the workload-internal event-reason identifier the ops
// state machines stamp on K8s Events. Values match the legacy
// omenative/status reason strings byte-for-byte so existing operator
// dashboards and `kubectl describe` output keep matching.
type EventReason string

func (r EventReason) String() string { return string(r) }

const (
// Create / scale-up (workload/ops/create.go).
EventReasonInstanceCreated EventReason = "InstanceCreated"
EventReasonInstanceReady EventReason = "InstanceReady"

// In-place update (workload/ops/update.go).
EventReasonInPlaceUpdateStarted EventReason = "InPlaceUpdateStarted"
EventReasonInPlaceUpdateCompleted EventReason = "InPlaceUpdateCompleted"
EventReasonInPlaceUpdateNotPossible EventReason = "InPlaceUpdateNotPossible"

// Recreate / surge update (workload/ops/update.go).
EventReasonRecreateUpdateStarted EventReason = "RecreateUpdateStarted"
EventReasonRecreateUpdateCompleted EventReason = "RecreateUpdateCompleted"

// Restart (workload/ops/restart.go).
EventReasonRestartTriggered EventReason = "RestartTriggered"
EventReasonRestartCompleted EventReason = "RestartCompleted"

// EventReasonFoundOrphan fires when Restart or recreate-Update
// finds a pod under the OMENative selector but missing the
// ome.io/instance-incarnation label. The reconciler refuses to
// delete it; the operator must re-classify or remove the pod
// manually.
EventReasonFoundOrphan EventReason = "FoundOrphan"

// EventReasonSupersededWreckageCleaned fires when the corrective-
// edit cleanup deletes pods keyed to a superseded revision — debris
// a failed rollout left behind that no revision-diff trigger could
// reach.
EventReasonSupersededWreckageCleaned EventReason = "SupersededWreckageCleaned"

// EventReasonAutoMigrationTriggered fires when the deadline
// disposition records a relocation directive (terminal AutoRecover
// ledger entry) for a stuck Instance — its rebuild will be steered
// off the recorded node. Value matches the legacy omenative
// detector's reason string.
EventReasonAutoMigrationTriggered EventReason = "AutoMigrationTriggered"

// EventReasonAutoMigrationCapReached fires exactly once per budget
// fill — when the disposition records the relocation directive that
// exhausts the (component, instance) AutoRecover budget
// (lifecycle.autoMigrate.maxAttempts). Subsequent over-budget
// dispositions dispose terminal silently. Operator intervention (or
// an instance reaching Ready, which prunes its records) is required
// before relocation resumes.
EventReasonAutoMigrationCapReached EventReason = "AutoMigrationCapReached"

// EventReasonInstanceFailed fires when an escalation backstop (the
// stuck-pod fast path or the deadline disposition) stamps an
// Instance Phase=Failed. Emitted by adapters through
// ReconcileInput.WarnInstanceFailed.
EventReasonInstanceFailed EventReason = "InstanceFailed"

// EventReasonRetryHeld fires once, at the RetryBlock transition into
// State=Held — same-target update retries exhausted; a corrected
// revision (or raised retry limits) is required. Emitted by adapters
// through ReconcileInput.WarnRetryHeld.
EventReasonRetryHeld EventReason = "RetryHeld"

// EventReasonRetryBlockReleased fires when the operator release
// annotation (ome.io/release-held-revision) removes a Held
// RetryBlock — the manual exit from the terminal Held state. Names
// the released revision and that the removal was operator-requested.
EventReasonRetryBlockReleased EventReason = "RetryBlockReleased"

// EventReasonRetryBlockReleaseSkipped fires when the release
// annotation names no releasable block — no RetryBlock exists for
// the requested revision, or the matched block is not State=Held.
// The annotation is still consumed; the event explains why nothing
// changed.
EventReasonRetryBlockReleaseSkipped EventReason = "RetryBlockReleaseSkipped"

// EventReasonPodForceDeleted fires when scale-down escalation
// force-deletes (grace 0, UID-preconditioned) a Terminating pod
// overdue past its own deletion deadline on a node that provably
// cannot acknowledge the termination (gone, or unreachable-tainted /
// NotReady beyond the configured threshold). Names the pod, node,
// evidence branch, and overdue duration.
EventReasonPodForceDeleted EventReason = "PodForceDeleted"

// EventReasonPodDeleteBlockedByFinalizer fires (once per pod UID)
// when a Terminating pod is overdue past its deletion deadline but
// pinned by foreign finalizers. Report-only: OME never strips
// another controller's finalizer, so the teardown stays blocked
// until the finalizer owner resolves it.
EventReasonPodDeleteBlockedByFinalizer EventReason = "PodDeleteBlockedByFinalizer"

// Migration (workload/ops/migrate.go + the IR accept pass).
EventReasonMigrationRequestAccepted EventReason = "MigrationRequestAccepted"
EventReasonMigrationRequestRejected EventReason = "MigrationRequestRejected"
// EventReasonUnsupportedSchemaVersion fires when a migration-request
// annotation carries a schemaVersion the controller doesn't
// understand. Kept distinct from MigrationRequestRejected so
// dashboards can alert on requester/controller version skew.
EventReasonUnsupportedSchemaVersion EventReason = "UnsupportedSchemaVersion"
EventReasonMigrationCompleted EventReason = "MigrationCompleted"
// EventReasonMigrationExpired fires when a non-terminal Manual
// migration record passes its Deadline: the record is closed
// Failed, the pair's Migrate ops are cleared, the surge is torn
// down by the ordinary scale-down batch pipeline, and the source
// phase is restored from observation.
EventReasonMigrationExpired EventReason = "MigrationExpired"
EventReasonRateLimited EventReason = "RateLimited"
EventReasonMigrationFromNodeMismatch EventReason = "MigrationFromNodeMismatch"
EventReasonMigrationNodeAffinityConflict EventReason = "MigrationNodeAffinityConflict"

// EventReasonMaybeNoGangScheduler is a soft Warning fired the first
// time a multi-pod Instance's PodGroup is created under a pod
// template whose `spec.schedulerName` is unset or equals the
// upstream default ("default-scheduler"). A stock kube-scheduler
// does NOT read scheduling.x-k8s.io/v1alpha1 PodGroup objects, so
// the gang contract degrades to per-pod scheduling silently.
// Operators install scheduler-plugins as a secondary scheduler
// (`scheduler-plugins-scheduler`) or as a default-scheduler plugin
// (in which case the warning is a false positive the controller
// can't detect from inside the cluster). Dedup'd per (owner,
// Component) per process.
EventReasonMaybeNoGangScheduler EventReason = "MaybeNoGangScheduler"

// EventReasonGangSplitRisk is a soft Warning fired the first time a
// multi-node gang WORKER pod is created with no co-location
// podAffinity at all — neither an OME-injected topologyKey term nor a
// user-declared one. Such a gang may schedule across separate
// network / NVLink / TPU topology domains, which breaks the
// tightly-coupled collectives a multi-node runtime needs (NCCL/RCCL/
// NIXL all-reduce, multi-host TPU sessions). The operator sets
// engine.topologyKey / decoder.topologyKey (e.g. a NVLink/RDMA domain
// label, or the GKE TPU topology label) or declares a worker
// podAffinity. Advisory only — never blocks the create. Dedup'd per
// (owner, Component) per process.
EventReasonGangSplitRisk EventReason = "GangSplitRisk"
)
Loading
Loading