feat: audit Kubernetes Velero backups

CHANGELOG.md

@@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Added
+- Optional multi-cluster Kubernetes/K3s auditing with Velero, PVC, workload,
+  and node protection checks plus read-only Doctor RBAC preflight.
 - New check: `dashboard-registration-drift`
   - Detects Docker containers with published ports that are not registered in Homepage dashboard
   - Finding codes: 

README.md

@@ -62,6 +62,8 @@ infrastructure *actually is*, and prints a RED/YELLOW/GREEN report per service:
 - **Off-site freshness & capacity** — every scheduled rclone sync job actually
   ran, succeeded, and is recent (a job that never ran is RED, not invisible);
   the destination isn't silently full.
+- **Kubernetes / K3s** — Velero coverage and freshness, PVC protection,
+  deployment availability, and node readiness.
 - **Disk health** — SMART status on the hypervisors.
 - **Suppression hygiene** — accepted-risk entries are first-class and fail loud:
   expired or dead suppressions become findings themselves, never silent.
@@ -106,6 +108,7 @@ have. You need:
 | `sqliteBackupDirs` | recursive `find` for `*-wal`/`*-shm` | read access to the backup folder |
 | `pbsOffsite` *(legacy — prefer `offsiteJobs`)* | `tail` the sync log, `rclone about` | read the log; the host's rclone remote must authenticate |
 | `pbsMaintenance` | `pct exec <CT> -- proxmox-backup-manager` (GC, verify + sync jobs, datastore list); `ls` of `host/<id>` snapshot dirs for `hostBackups` | root on the PVE host that runs the PBS container |
+| `kubernetesClusters` | read-only `kubectl get` for cluster, workload, Velero, PodVolumeBackup, and CSI snapshot resources | SSH access to the configured `kubectl` prefix; for an LXC prefix, root on the PVE host for `pct exec` |
 | `smartHosts` | `smartctl -H` | root (raw device access); smartmontools installed |
 
 Targets are assumed Linux-ish with standard tools (GNU `find`, `awk`, `tail`).

docs/modules/ROOT/examples/restoreguard.sample.json

@@ -207,6 +207,32 @@
     }
   ],
 
+
+  // Dashboard registration drift: verifies running Docker containers with published
+  // ports are registered in the Homepage dashboard (services.yaml ConfigMap on k3s).
+  // Catches the "forgot to add to the dashboard" failure after deploying a new service.
+  // k3sMasterAlias: SSH alias for the k3s node that hosts Homepage (ConfigMap fetch).
+  // dashboardHostAliases: Docker hosts to scan for running containers.
+  "dashboardDrift": {
+    "k3sMasterAlias": "myk3smaster",
+    "dashboardHostAliases": ["mydockerhost", "myotherhost"]
+  },
+
+  // Kubernetes / K3s clusters. This is config-file-only in the v1 wizard.
+  // kubectl is a command PREFIX executed on alias (for sshd-less LXC use
+  // e.g. "pct exec 601 -- k3s kubectl"). veleroNamespace: null explicitly
+  // disables Velero checks; omission defaults to "velero".
+  "kubernetesClusters": [
+    {
+      "name": "homelab k3s",
+      "alias": "mypve",
+      "kubectl": "kubectl",
+      "veleroNamespace": "velero",
+      "maxBackupAgeHours": 26,
+      "checkWorkloads": true
+    }
+  ],
+
   // Where each audit DELIVERS its JSON report (the same document --json prints).
   // The destinations live in their OWN self-contained file so a SECOND tool — e.g.
   // HCC — can read the SAME file, connect to the same folder/bucket/DB, and pull the

docs/modules/ROOT/pages/checks.adoc

@@ -170,6 +170,21 @@ in real labs: the replica keeps its old snapshots and *looks* fine.
 | `cloudsync/not-off-box` | YELLOW | a top-level dataset is covered by no cloud-sync task — its data never leaves the box
 |===
 
+== Kubernetes / K3s (`kubernetesClusters`)
+
+[cols="2,1,4"]
+|===
+| `k8s/unreachable` | RED | a configured cluster could not be discovered; the report is partial
+| `k8s-backup/velero-missing` | RED | Velero's Backup resource type is absent
+| `k8s-backup/no-backups` / `k8s-backup/no-completed` | RED | Velero is installed but has no backups or no successful backup
+| `k8s-backup/stale` | RED | newest Completed backup exceeds `maxBackupAgeHours`
+| `k8s-backup/failed` | RED | the latest unsuccessful backup is Failed, PartiallyFailed, or FailedValidation
+| `k8s-backup/no-schedule` | YELLOW | backups exist but no Velero Schedule is configured
+| `k8s-pvc/unbound` / `k8s-pvc/unprotected` | RED | a PVC is not Bound, or a Bound PVC lacks a completed Velero pod-volume backup or Ready CSI snapshot in the latest completed backup
+| `k8s-workload/unavailable` | YELLOW | a deployment has fewer available than desired replicas
+| `k8s-node/not-ready` | RED | a cluster node is NotReady
+|===
+
 == Storage & disks (`pveNodes`, `trueNas`, `smartHosts`)
 
 [cols="2,1,4"]

docs/modules/ROOT/pages/configuration.adoc

@@ -66,6 +66,31 @@ A list — one entry per dump job you run. Dump files are expected to be named
 
 *Host needs:* read access to the dump directory.
 
+== `kubernetesClusters` — Kubernetes / K3s
+
+[source,json]
+----
+"kubernetesClusters": [
+  { "name": "homelab k3s", "alias": "mypve",
+    "kubectl": "pct exec 601 -- k3s kubectl", "veleroNamespace": "velero",
+    "maxBackupAgeHours": 26, "checkWorkloads": true }
+]
+----
+
+Each cluster has a unique `name`, an SSH `alias`, and a non-empty `kubectl`
+command prefix. `veleroNamespace` omitted defaults to `velero`; set it explicitly
+to `null` to disable Velero requirements. `maxBackupAgeHours` must be positive.
+The provider runs read-only node, deployment, PVC, pod, Velero backup, schedule,
+pod-volume-backup, and optional CSI volume-snapshot queries. Each Bound PVC needs a
+completed Velero pod-volume backup or Ready CSI snapshot from the latest successful
+backup; a Backup resource alone is not data-protection evidence.
+Workload checks can be disabled with `checkWorkloads: false`; do not
+disable both workload checks and Velero, because that would audit no surface.
+Kubernetes is not included in the v1 setup wizard; configure it by hand.
+
+*Host needs:* SSH access to a machine with the configured `kubectl` command prefix;
+for an LXC prefix, root access on the PVE host to run `pct exec`.
+
 == `pveNodes` — Proxmox image backups
 
 [source,json]

docs/modules/ROOT/pages/index.adoc

@@ -70,6 +70,8 @@ xref:configuration.adoc[configuration reference]):
   host: the snapshot job still runs, and the replica is still catching up.
 * *TrueNAS SCALE* — ZFS snapshot freshness, cloud-sync task health, datasets
   that never leave the box, pool health and scrub age.
+* *Kubernetes / K3s* — Velero coverage and freshness, PVC protection,
+  deployment availability, and node readiness.
 * *Off-site copies* — every scheduled rclone sync job ran, succeeded, and is
   recent; the destination has room. A job that never ran is a finding, not a gap.
 * *Disks* — SMART health on the machines that own the physical disks.

restoreguard.sample.json

@@ -218,6 +218,21 @@
     "dashboardHostAliases": ["mydockerhost", "myotherhost"]
   },
 
+  // Kubernetes / K3s clusters. This is config-file-only in the v1 wizard.
+  // kubectl is a command PREFIX executed on alias (for sshd-less LXC use
+  // e.g. "pct exec 601 -- k3s kubectl"). veleroNamespace: null explicitly
+  // disables Velero checks; omission defaults to "velero".
+  "kubernetesClusters": [
+    {
+      "name": "homelab k3s",
+      "alias": "mypve",
+      "kubectl": "kubectl",
+      "veleroNamespace": "velero",
+      "maxBackupAgeHours": 26,
+      "checkWorkloads": true
+    }
+  ],
+
   // Where each audit DELIVERS its JSON report (the same document --json prints).
   // The destinations live in their OWN self-contained file so a SECOND tool — e.g.
   // HCC — can read the SAME file, connect to the same folder/bucket/DB, and pull the

src/RestoreGuard.Checks/KubernetesCheck.cs

@@ -0,0 +1,105 @@
+using RestoreGuard.Core;
+using RestoreGuard.Core.Model;
+using RestoreGuard.Providers.Kubernetes;
+
+namespace RestoreGuard.Checks;
+
+public sealed record KubernetesExpectation(string Name, string Alias, TimeSpan MaxBackupAge, bool CheckWorkloads = true,
+    bool VeleroRequired = true, string? ProviderError = null)
+{
+    public string ClusterService => $"{Name} cluster";
+}
+
+/// <summary>Deterministic Kubernetes posture and Velero backup checks.</summary>
+public sealed class KubernetesCheck(
+    IReadOnlyList<KubernetesState> states,
+    IReadOnlyList<KubernetesExpectation> expectations) : ICheck
+{
+    public string RuleId => "k8s";
+
+    public IEnumerable<Finding> Evaluate(LabInventory inventory)
+    {
+        foreach (var expected in expectations)
+        {
+            var state = states.FirstOrDefault(s => s.ClusterIdentity == expected.Name);
+            if (state is null)
+            {
+                yield return new Finding("k8s/unreachable", Severity.Red, expected.ClusterService, expected.Name,
+                    $"Kubernetes discovery through '{expected.Alias}' did not complete: {expected.ProviderError ?? "no provider result"}",
+                    "Run `restoreguard doctor` for this cluster, then fix kubectl reachability or RBAC before trusting the audit.");
+                continue;
+            }
+            var backups = inventory.Backups.Where(b => b.Tier == BackupTier.KubernetesBackup
+                    && b.TargetService == $"{expected.Name} velero")
+                .OrderByDescending(b => b.Timestamp).ToList();
+
+            if (expected.VeleroRequired)
+            {
+                if (!state.VeleroInstalled)
+                    yield return new Finding("k8s-backup/velero-missing", Severity.Red, expected.ClusterService, expected.Name,
+                        $"Cluster '{expected.Name}' has no readable Velero Backup resource type.",
+                        "Install Velero or set veleroNamespace to null if this cluster intentionally has no Velero protection.");
+                else if (backups.Count == 0)
+                    yield return new Finding("k8s-backup/no-backups", Severity.Red, expected.ClusterService, expected.Name,
+                        $"Velero is installed for cluster '{expected.Name}', but no Backup resources were found.",
+                        "Create a Velero Schedule and run its first backup; also verify that it includes persistent-volume data.");
+                else
+                {
+                    var latestCompleted = backups.Where(b => string.Equals(b.Status, "Completed", StringComparison.OrdinalIgnoreCase))
+                        .OrderByDescending(b => b.Timestamp).FirstOrDefault();
+                    var latestBackup = backups.OrderByDescending(b => b.Timestamp).First();
+                    if (string.Equals(latestBackup.Status, "Failed", StringComparison.OrdinalIgnoreCase)
+                        || string.Equals(latestBackup.Status, "PartiallyFailed", StringComparison.OrdinalIgnoreCase)
+                        || string.Equals(latestBackup.Status, "FailedValidation", StringComparison.OrdinalIgnoreCase))
+                        yield return new Finding("k8s-backup/failed", Severity.Red, expected.ClusterService, expected.Name,
+                            $"Latest Velero backup is '{latestBackup.Location}' ({latestBackup.Status}).",
+                            "Run `velero backup describe` and inspect the backup logs; partial failures often indicate a PVC snapshot or node-agent problem.");
+                    if (latestCompleted is null)
+                        yield return new Finding("k8s-backup/no-completed", Severity.Red, expected.ClusterService, expected.Name,
+                            "Velero Backup resources exist, but none has completed successfully.",
+                            "Fix the latest backup failure and verify a Completed backup before treating the cluster as protected.");
+                    else if (inventory.CapturedAt - latestCompleted.Timestamp > expected.MaxBackupAge)
+                        yield return new Finding("k8s-backup/stale", Severity.Red, expected.ClusterService, expected.Name,
+                            $"Latest Completed Velero backup '{latestCompleted.Location}' is {(inventory.CapturedAt - latestCompleted.Timestamp).TotalHours:F0}h old (limit {expected.MaxBackupAge.TotalHours:F0}h).",
+                            "Check the Velero Schedule and its last run: velero backup logs <name>.");
+
+                    if (state.Schedules.Count == 0)
+                        yield return new Finding("k8s-backup/no-schedule", Severity.Yellow, expected.ClusterService, expected.Name,
+                            "Velero backups exist but no Schedule resources were found.",
+                            "Create a Velero Schedule so coverage does not depend on a human.");
+                    if (latestCompleted is not null)
+                    {
+                        var protectedClaims = state.PodVolumeBackups
+                            .Where(pvb => pvb.BackupName == latestCompleted.Location && pvb.Completed && pvb.PersistentVolumeClaim is not null)
+                            .Select(pvb => $"{pvb.PodNamespace}/{pvb.PersistentVolumeClaim}")
+                            .Concat(state.VolumeSnapshots.Where(snapshot => snapshot.BackupName == latestCompleted.Location && snapshot.ReadyToUse)
+                                .Select(snapshot => $"{snapshot.Namespace}/{snapshot.ClaimName}"))
+                            .ToHashSet(StringComparer.Ordinal);
+                        foreach (var pvc in state.PersistentVolumeClaims.Where(pvc =>
+                                     string.Equals(pvc.Phase, "Bound", StringComparison.OrdinalIgnoreCase)
+                                     && !protectedClaims.Contains($"{pvc.Namespace}/{pvc.Name}")))
+                        {
+                            yield return new Finding("k8s-pvc/unprotected", Severity.Red, $"{pvc.Namespace}/{pvc.Name}", expected.Name,
+                                $"Bound PVC '{pvc.Namespace}/{pvc.Name}' has no completed Velero pod-volume backup in latest Completed backup '{latestCompleted.Location}'.",
+                                "Enable and verify a Velero file-system backup or CSI snapshot for this PVC, then restore a test PVC before treating it as protected.");
+                        }
+                    }
+                }
+            }
+
+            if (!expected.CheckWorkloads) continue;
+            foreach (var pvc in state.PersistentVolumeClaims.Where(p => !string.Equals(p.Phase, "Bound", StringComparison.OrdinalIgnoreCase)))
+                yield return new Finding("k8s-pvc/unbound", Severity.Red, $"{pvc.Namespace}/{pvc.Name}", expected.Name,
+                    $"PVC '{pvc.Namespace}/{pvc.Name}' is {pvc.Phase}, not Bound.",
+                    "Inspect the PVC events and storage class; restore protection cannot rely on an unbound volume.");
+            foreach (var service in inventory.Services.Where(s => s.Host == expected.Name && s.Kind == ServiceKind.K8sWorkload && s.State == "degraded"))
+                yield return new Finding("k8s-workload/unavailable", Severity.Yellow, service.Name, expected.Name,
+                    $"Deployment '{service.Name}' has fewer available replicas than desired.",
+                    "Run kubectl describe deployment and kubectl rollout status to investigate.");
+            foreach (var service in inventory.Services.Where(s => s.Host == expected.Name && s.Kind == ServiceKind.K8sNode && s.State == "NotReady"))
+                yield return new Finding("k8s-node/not-ready", Severity.Red, service.Name, expected.Name,
+                    $"Node '{service.Name}' is NotReady.",
+                    "Inspect node conditions and kubelet health before trusting cluster backups.");
+        }
+    }
+}

src/RestoreGuard.Checks/RestoreGuard.Checks.csproj

@@ -2,6 +2,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\RestoreGuard.Core\RestoreGuard.Core.csproj" />
+    <ProjectReference Include="..\RestoreGuard.Providers\RestoreGuard.Providers.csproj" />
   </ItemGroup>
 
   <PropertyGroup>

src/RestoreGuard.Cli/AuditRunner.cs

@@ -11,6 +11,7 @@
 using RestoreGuard.Providers.Pve;
 using RestoreGuard.Providers.Smart;
 using RestoreGuard.Providers.TrueNas;
+using RestoreGuard.Providers.Kubernetes;
 
 namespace RestoreGuard.Cli;
 
@@ -22,6 +23,7 @@ public static async Task<int> RunAsync(RestoreGuardConfig config, string configD
         var suppressions = config.LoadSuppressions(configDir);
 
         var docker = new DockerProvider(ssh);
+        var kubernetes = new KubernetesProvider(ssh, Progress);
         var dbDump = new DbDumpProvider(ssh);
         var pve = new PveProvider(ssh);
 
@@ -39,6 +41,9 @@ public static async Task<int> RunAsync(RestoreGuardConfig config, string configD
         var dockerTasks = config.DockerHosts
             .Select(h => Track("docker", h.Alias, docker.GetServicesAsync(h)))
             .ToList();
+        var kubernetesTasks = (config.KubernetesClusters ?? [])
+            .Select(c => Track("k8s", c.Alias, kubernetes.GetClusterAsync(c)))
+            .ToList();
         var pveTasks = (config.PveNodes ?? [])
             .Select(n => Track("pve", n.Alias, pve.GetNodeAsync(n)))
             .ToList();
@@ -89,7 +94,9 @@ public static async Task<int> RunAsync(RestoreGuardConfig config, string configD
                     dd.DashboardHostAliases.Select(a => new DockerHostConfig(a)).ToList()))
             : Task.FromResult<(string, DashboardProvider.DashboardProbeResult?, string?)>(("", null, null));
 
-        Progress($"auditing: {probes.Count} probe(s) across the lab, in parallel (Ctrl+C stops and reports what finished)...");
+        var kubernetesSurfaceCount = (config.KubernetesClusters ?? []).Sum(cluster => KubernetesProvider.EnabledSurfaces(cluster).Count);
+        var displayedProbeCount = probes.Count - kubernetesTasks.Count + kubernetesSurfaceCount;
+        Progress($"auditing: {displayedProbeCount} probe(s) across the lab, in parallel (Ctrl+C stops and reports what finished)...");
 
         var discovery = Stopwatch.StartNew();
         var all = Task.WhenAll(probes.Select(p => p.Task));
@@ -106,12 +113,29 @@ public static async Task<int> RunAsync(RestoreGuardConfig config, string configD
         var artifacts = new List<BackupArtifact>();
         var providerErrors = new List<string>();
 
+        // A cluster-level service makes every cluster-wide Kubernetes finding a stable
+        // suppression target even when its provider fails before yielding any inventory.
+        services.AddRange((config.KubernetesClusters ?? []).Select(cluster =>
+            new Service($"{cluster.Name} cluster", cluster.Name, ServiceKind.K8sCluster, "unknown", null, [], null)));
+
         foreach (var (host, result, error) in dockerTasks.Select(t => t.Result))
         {
             if (result is not null) services.AddRange(result);
             if (error is not null) providerErrors.Add($"{host}: {error}");
         }
 
+        var kubernetesStates = new List<KubernetesState>();
+        foreach (var (host, result, error) in kubernetesTasks.Select(t => t.Result))
+        {
+            if (result is not null)
+            {
+                services.AddRange(result.Services);
+                artifacts.AddRange(result.Backups);
+                kubernetesStates.Add(result.State);
+            }
+            if (error is not null) providerErrors.Add($"{host}: {error}");
+        }
+
         // PBS snapshots join against guests from ALL nodes (shared datastore, colliding
         // vmids), so collect every node inventory before assembling artifacts.
         var guests = new List<PveGuest>();
@@ -283,6 +307,18 @@ public static async Task<int> RunAsync(RestoreGuardConfig config, string configD
         {
             checks.Add(new DashboardRegistrationDriftCheck());
         }
+        if (config.KubernetesClusters is { Count: > 0 } clusters)
+        {
+            var expectations = clusters.Zip(kubernetesTasks, (cluster, task) =>
+            {
+                var result = task.Result;
+                return new KubernetesExpectation(cluster.Name, cluster.Alias,
+                    TimeSpan.FromHours(cluster.MaxBackupAgeHours), cluster.CheckWorkloads,
+                    cluster.VeleroNamespace is not null, result.Item3);
+            }).ToList();
+            checks.Add(new KubernetesCheck(kubernetesStates,
+                expectations));
+        }
 
         var report = new CheckEngine(checks).Run(inventory, suppressions, DateTimeOffset.UtcNow);
         var sinks = ReportPublisher.BuildSinks(config, configDir);