CloudNativePG: incident workflows, guarded operations and guided recovery

.design-sync/previews/AlertBanner.tsx

@@ -49,3 +49,17 @@ export function CustomIcon() {
     </div>
   )
 }
+
+export function WithAction() {
+  return (
+    <div style={wrap}>
+      <AlertBanner
+        variant="warning"
+        title="The operator is not reconciling: status below may be stale"
+        message="The operator Deployment has no ready Pod."
+        action={<button type="button" className="text-xs font-medium text-accent-text hover:underline">Open Operator →</button>}
+        className="px-3 py-2"
+      />
+    </div>
+  )
+}

CLAUDE.md

@@ -25,6 +25,7 @@ Not everything is in this file. The following files contain critical details tha
 | Adding or modifying **HTTP endpoints** | `internal/server/server.go` — all routes are defined here — **plus** the handler's doc comments (why the route is gated the way it is lives there; copy the gate of the closest sibling only after reading it) and the integration's section in [docs/integrations.md](docs/integrations.md) |
 | Adding or modifying **CLI flags** | `cmd/explorer/main.go` — flag definitions and defaults |
 | Adding a **new CRD integration** (renderer, topology, discovery) | [docs/INTEGRATION_GUIDE.md](docs/INTEGRATION_GUIDE.md) — full checklist with collision gotchas |
+| Building a **workspace integration** (several related CRDs with their own screens, like `/cnpg` or `/capacity`) | [docs/INTEGRATION_GUIDE.md](docs/INTEGRATION_GUIDE.md#3-workspace-integrations) — the shared server, UI and action pieces to import, and what is not shared yet; [DESIGN.md](DESIGN.md#unknown-partial-and-denied-values) — how unknown, partial and denied values read |
 | Working on the **CloudNativePG workspace** (`/cnpg`) | [docs/cnpg.md](docs/cnpg.md) — destinations, navigation (drawer trail, return label, `ctx` guard) and the certainty table: which source each fact comes from and what it reads when unknown. Data from `/api/cnpg/workspace` (per-kind coverage); derivations in `packages/k8s-ui/src/components/cnpg/workspace.ts` + `relations.ts`; screens in `web/src/components/cnpg/` |
 | Working on **local per-cluster integration settings** (Metrics, Argo CD, Cost in `~/.radar/clusters.json`) | [docs/configuration.md](docs/configuration.md#local-integration-connections) — store `internal/config/profiles.go`, resolve/update `internal/connections`, activation `internal/connectionruntime`, routes `GET/PUT /api/integrations/connections`. In local mode the older `PUT /api/integrations/{prometheus,argocd,cost}` return 409 |
 | Working on **GitOps** (Argo CD / Flux detail pages, operations, Terminating lifecycle, drift, per-resource health, remote destinations) | [docs/gitops.md](docs/gitops.md) — detail-page tabs, operation semantics, the Terminating severity ramp, nested navigation, single-cluster scope. Engine in `pkg/gitops/`, handlers `internal/server/gitops_handlers.go` |
@@ -108,7 +109,7 @@ After `make <name>-demo`, run `kubectl config use-context kind-radar-<name>-demo
 | GitOps | `make gitops-demo` | Argo CD / Flux UI. `-drift` induces live OutOfSync |
 | Kyverno | `make kyverno-demo` | Policy renderers, report-family selection, admission attribution. Scenarios `openreports` / `modern-only` |
 | Velero | `make velero-demo` | Backup/restore surfaces — all 13 Backup phases at once. `-live` for states the controller actually produced |
-| CloudNativePG | `make cnpg-demo` | CNPG renderers/badges. `-live` for real failovers; fixtures have strict ordering constraints |
+| CloudNativePG | `make cnpg-demo` | CNPG renderers/badges. `-live` for real failovers; `-runtime` adds real backups/restore, load, lock chain + Prometheus; fixtures have strict ordering constraints |
 | Beyla | `make beyla-demo` | `internal/traffic/beyla.go` — which labels exist depends on Beyla config, not code. Modes `attrs` / `no-network` |
 | Cilium | `make cilium-demo` | `internal/traffic/hubble.go` — every Hubble connection lane. Modes `tls` / `netpol` / `install-radar` |
 | Kubecost | `make kubecost-demo` | Kubecost 3 current costs — real allocation/assets, local port-forward and in-cluster Service DNS. Modes `query` / `install-radar` / `radar-smoke` |
@@ -139,6 +140,7 @@ After `make <name>-demo`, run `kubectl config use-context kind-radar-<name>-demo
 - Helm: `/api/helm/releases/...`
 - Workloads: `/api/workloads/{kind}/{ns}/{name}/...` (logs, restart, scale, revisions, rollback, images, history). `revisions`/`rollback` accept Deployment, StatefulSet, DaemonSet and **Rollout**, gated by `rollbackableWorkloadKinds` in `server.go`; `images` accepts the same four through a separate map, `workloadImageRoots` in `pkg/k8score/workload_images.go` (a new kind needs both), uses compare-and-swap JSON Patch, and follows a Rollout's `workloadRef`. `history` is the workload's own timeline and never includes sibling workloads — scope rules live in `workload_history.go` and `pkg/timeline`'s `ResourceScope`
 - Argo Rollouts: `/api/rollouts/{ns}/{name}/{abort,retry,promote,promote-full,skip-step}` (POST) + `/capabilities` (GET); rollback/history deliberately live on the `/workloads` routes. The status verbs patch the `rollouts/status` subresource, so capabilities SAR `rollouts` **and** `rollouts/status` separately — `patch rollouts` does not imply `patch rollouts/status`. Promotion waits for the controller to observe the current pod template, so `promote-full` can return **503** (`ErrControllerNotCaughtUp`) and is safe to retry; for a `workloadRef` Rollout the caller must be able to `get` the referenced workload. Engine `pkg/rollouts`, handlers `internal/server/rollouts_handlers.go`
+- GitOps write evidence: `POST /api/gitops/write-evidence` `{kind, group, namespace, name, paths[], owner?}` reads the target (and its GitOps owner) directly as the caller and returns, per field path, whether it appears in last-applied, which field managers own it, and whether an ignore rule covers it, plus the owner's sync policy — never raw managedFields/last-applied. Classification into none/info/may-revert/will-revert is the pure `evaluateGitOpsWriteGuard` in k8s-ui (`utils/gitops-write-guard.ts`); every write dialog shows it via `GitOpsWriteWarning` + `web/src/hooks/useGitOpsWriteGuard.ts`. Copy never promises a change won't be reverted.
 - GitOps controller actions: `/api/argo/applications/...` (sync, refresh, terminate, suspend, resume, rollback, selective-sync), `/api/flux/{kind}/...` (reconcile, suspend, resume, sync-with-source)
 - Argo CD API integration: `PUT /api/integrations/argocd` (non-local config only — local mode uses `/api/integrations/connections`; URL/token, probe-before-persist, token preserved across GET-redaction round-trips); `/api/argo/applications/{ns}/{name}/resource-diff` (Git-rendered desired vs live via argocd-server managed-resources; dual RBAC gate + structural Secret redaction; see docs/gitops.md)
 - GitOps detail data: `/api/gitops/{tree,insights,destination}/{kind}/{ns}/{name}`. `destination` says where a remote Application or `spec.kubeConfig` Flux object deploys; the object and any kubeconfig Secret are read as the caller, and full server URLs never leave the server
@@ -153,7 +155,7 @@ After `make <name>-demo`, run `kubectl config use-context kind-radar-<name>-demo
 - RBAC reverse-lookup: `/api/rbac/subject/{kind}/{namespace}/{name}` (ServiceAccount, plus `usedByPods`) and `/api/rbac/subject/{kind}/{name}` (User/Group) — direct + group-inherited bindings and flattened effective rules; `/api/rbac/role/{kind}/{namespace}/{name}` (`_` for a ClusterRole's namespace) — the bindings that reference it; `/api/rbac/namespace/{namespace}` — backs the Namespace RBAC section (group-only ClusterRoleBindings deliberately excluded); `/api/rbac/whoami` — `SelfSubjectRulesReview` pass-through. All gate on `list rolebindings` AND `list clusterrolebindings`: **403 when either is denied, never a silent partial view**
 - Policy (Kyverno): `/api/policy/resource/{kind}/{ns}/{name}` (one resource's findings), `/api/policy/policies/{policy}` (every resource one policy recorded an outcome for). Report families are authorized **per subject scope** (`policyreports` cluster-wide ≠ `clusterpolicyreports`); findings from an unreadable family are dropped from lists AND counts, with the withheld count reported; `counts` describe the cluster while subject lists are capped and view-filtered. `/api/policy/policies/{policy}/queued` reads Kyverno's `UpdateRequest`s cluster-wide, gated on `list updaterequests`
 - Velero: `/api/velero/backupstoragelocations/{ns}/{name}/backups` (what a location holds; gated on `list backups`); `POST /api/velero/{backups|restores}/{ns}/{name}/messages` (a run's warnings/errors via a `DownloadRequest` — impersonated; needs a running Velero controller and object storage reachable from Radar, and reports which one failed)
-- CloudNativePG: `/api/cnpg/...` — the workspace, operator, catalog reverse-lookups, Cluster logs and activity. Routes, gates and coverage states are listed in [docs/cnpg.md](docs/cnpg.md#api); each is gated on the caller's own access, and a partial answer says what it withheld
+- CloudNativePG: `/api/cnpg/...` — the workspace, operator status and diagnosis, catalog reverse-lookups, and per Cluster: runtime, storage, HA facts, history, logs, activity, sessions, restore, report bundle and actions (plus Pooler and ScheduledBackup actions). Routes, gates and coverage states are listed in [docs/cnpg.md](docs/cnpg.md#api); each is gated on the caller's own access, a partial answer says what it withheld, and every write binds the facts the user reviewed
 
 ## Key Patterns
 

DESIGN.md

@@ -100,6 +100,7 @@ Standard Tailwind type scale. No custom sizes or tracking. Use Tailwind utilitie
 | `.btn-brand` | Primary CTAs — brand-colored bg, white text, 10px radius |
 | `.btn-brand-muted` | Secondary brand actions — dimmed brand bg, white text |
 | `.btn-brand-toggle` | Toggle buttons — 50% brand bg, primary text |
+| `.btn-secondary` | Secondary actions beside a `.btn-brand` — bordered surface bg, primary text, 10px radius |
 
 Hover/disabled states are built into the classes. For non-brand buttons, use shadcn/ui `<Button>` variants.
 
@@ -172,6 +173,17 @@ Use CSS classes from `components.css` for status cells in table rows:
 | `.border-r-subtle` | Right border |
 | `.border-t-subtle` | Top border |
 
+### Unknown, partial and denied values
+Radar shows only what the cluster reports, and says where it came from. These rules apply to every surface that reads several sources at once (workspace integrations such as Capacity and CloudNativePG, multi-source detail pages):
+
+- **Unavailable ≠ zero.** A value Radar could not read (no access, not installed, not cached, the request failed) renders as unread, naming why — never as `0`, "none" or a healthy colour. A missing grant is named exactly (`GrantText`, `formatGrant`).
+- **Partial ≠ exact.** A count or total over data read only in part is a lower bound: `≥N` (`CertaintyGlyph`, `SidebarCategoryDestination.countLowerBound`), and a zero over partial data is unknown, not none.
+- **Unread is listed, not left out.** A summary line names what it could not read ("not read: Pods, zones"); a combined tone is never calmer than a part that was not read (`worseTone` ranks `unknown` above `healthy`).
+- **Recorded ≠ observed.** A value copied from a status field, an annotation or a declaration says so; a value matched to its subject by name rather than by identity says that too.
+- **Facts keep their rows.** `FactRow` renders the unread text in place; never hide a row because its value is missing (unlike `Property`, which hides empty values).
+
+The shared pieces live in `packages/k8s-ui/src/components/facts/` (facts, certainty, GitOps manager), `components/problems/` (problems with their sources), `components/ui/FoldSection.tsx` (section headings and folded sections) and `web/src/components/workspace/` (workspace screen layout and tables); each integration's own doc lists which source each value comes from and how it reads when unknown.
+
 ## 5. Layout Principles
 
 ### Spacing

Makefile

@@ -278,6 +278,16 @@ cnpg-demo-status:
 cnpg-demo-live:
 	./scripts/cnpg-demo.sh live
 
+# Runtime fixtures on the EXISTING CNPG demo cluster (operator thawed if frozen):
+# MinIO + real plugin WAL archiving and backups, a restored cluster, a Pooler,
+# pgbench load, a blocked lock chain and a Prometheus Radar auto-discovers.
+# `./scripts/cnpg-demo.sh runtime-lag on|off` induces replica lag.
+cnpg-demo-runtime:
+	./scripts/cnpg-demo.sh runtime
+
+cnpg-demo-runtime-down:
+	./scripts/cnpg-demo.sh runtime-down
+
 # Grafana Beyla on kind: eBPF loaded, a minimal Prometheus scraping it, and two
 # conversations to observe. Which labels Beyla exports depends on configuration —
 # dst_port and transport are off by default, direction is on and doubles every
@@ -523,6 +533,7 @@ help:
 	@echo "  make rollouts-demo    - Argo Rollouts progression fixtures"
 	@echo "  make cnpg-demo        - Frozen CNPG rendering fixtures"
 	@echo "  make cnpg-demo-live   - CNPG fixtures with the operator running"
+	@echo "  make cnpg-demo-runtime - CNPG runtime: backups/restore, load, locks, Prometheus"
 	@echo "  make velero-demo      - Velero fixtures, all 13 backup phases at once"
 	@echo "  make velero-demo-live - Velero with real object storage; states produced by the controller"
 	@echo "  make beyla-demo       - Grafana Beyla eBPF traffic fixtures"

README.md

@@ -362,7 +362,7 @@ View TLS certificate details and expiry dates across all namespaces — catch ex
 
 ### GitOps
 
-Monitor, diagnose, and manage FluxCD and ArgoCD resources from a dedicated GitOps workspace.
+Monitor, diagnose, and manage FluxCD and ArgoCD resources in one place.
 
 <p align="center">
   <img src="docs/screenshots/gitops-view.png" alt="GitOps fleet view" width="800">
@@ -548,7 +548,7 @@ Upgrade impact also gets list-only access to CSIStorageCapacities, FlowSchemas,
 | **Strimzi** | [KafkaConnector failure evidence](docs/integrations.md#strimzi-kafka-connectors) (connector/task status) |
 | **Velero** | Backup, Restore, Schedule, BackupStorageLocation, VolumeSnapshotLocation |
 | **External Secrets** | ExternalSecret, ClusterExternalSecret, SecretStore, ClusterSecretStore |
-| **CloudNativePG** | Cluster, Backup, ScheduledBackup, Pooler, Database, Publication, Subscription, ImageCatalog, ClusterImageCatalog, ObjectStore — plus a [workspace](docs/cnpg.md) for fleet, protection and declaration triage |
+| **CloudNativePG** | Cluster, Backup, ScheduledBackup, Pooler, Database, Publication, Subscription, ImageCatalog, ClusterImageCatalog, ObjectStore — plus [dedicated views](docs/cnpg.md) for fleet, protection and declaration triage |
 | **Crossplane** | Managed Resources (any provider), Composite Resources, Claims, Provider, ProviderConfig, Function, Configuration, Composition, CompositionRevision, XRD |
 | **Kyverno** | Policy, ClusterPolicy, PolicyReport, ClusterPolicyReport |
 | **Sealed Secrets** | SealedSecret |

deploy/helm/radar/files/integration-read-baseline.yaml

@@ -613,7 +613,7 @@ entries:
     reason: "Includes spec.externalClusters[].connectionParameters; inline connection settings (potentially passwords) are intentionally visible, like Helm values."
     source: "https://cloudnative-pg.io/docs/1.28/logical_replication/"
   - group: "postgresql.cnpg.io"
-    resources: ["backups","databases","imagecatalogs","poolers","publications","scheduledbackups","subscriptions"]
+    resources: ["backups","databaseroles","databases","failoverquorums","imagecatalogs","poolers","publications","scheduledbackups","subscriptions"]
     scope: Namespaced
     collection: "cloudnativePg"
     decision: grant

deploy/helm/radar/templates/cloud-rbac-cluster-read.yaml

@@ -12,6 +12,11 @@ bindings to the radar:* groups — deliberately NOT via `aggregate-to-view` labe
 which would widen the shared built-in `view`/`edit` roles cluster-wide for every
 other subject bound to them (mirrors the radar-helm add-on pattern).
 
+Leases are namespaced but read here like infrastructure: they carry only a
+holder identity and renew times (controller leader election, node heartbeats,
+CloudNativePG primary election). `edit` and `admin` already include them, so
+only the viewer tier gains anything.
+
 No Secrets and no RBAC objects here — those stay gated behind the tier roles
 (view excludes Secrets; RBAC visibility is rbac.viewRBAC) so this is a pure
 infrastructure-read widening.
@@ -69,6 +74,9 @@ rules:
   - apiGroups: ["scheduling.k8s.io"]
     resources: ["priorityclasses"]
     verbs: ["get", "list", "watch"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "list", "watch"]
   - apiGroups: ["certificates.k8s.io"]
     resources: ["clustertrustbundles"]
     verbs: ["get", "list", "watch"]

deploy/helm/radar/tests/cloud_rbac_cluster_read_test.yaml

@@ -43,6 +43,12 @@ tests:
             apiGroups: ["scheduling.k8s.io"]
             resources: ["priorityclasses"]
             verbs: ["get", "list", "watch"]
+      - contains:
+          path: rules
+          content:
+            apiGroups: ["coordination.k8s.io"]
+            resources: ["leases"]
+            verbs: ["get", "list", "watch"]
       - contains:
           path: rules
           content: