diff --git a/CHANGELOG.md b/CHANGELOG.md index fd8653d..14ecd98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- Optional multi-cluster Kubernetes/K3s auditing with Velero, PVC, workload, + and node protection checks plus read-only Doctor RBAC preflight. - New check: `dashboard-registration-drift` - Detects Docker containers with published ports that are not registered in Homepage dashboard - Finding codes: diff --git a/README.md b/README.md index 48fcd76..e480095 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,8 @@ infrastructure *actually is*, and prints a RED/YELLOW/GREEN report per service: - **Off-site freshness & capacity** — every scheduled rclone sync job actually ran, succeeded, and is recent (a job that never ran is RED, not invisible); the destination isn't silently full. +- **Kubernetes / K3s** — Velero coverage and freshness, PVC protection, + deployment availability, and node readiness. - **Disk health** — SMART status on the hypervisors. - **Suppression hygiene** — accepted-risk entries are first-class and fail loud: expired or dead suppressions become findings themselves, never silent. @@ -106,6 +108,7 @@ have. You need: | `sqliteBackupDirs` | recursive `find` for `*-wal`/`*-shm` | read access to the backup folder | | `pbsOffsite` *(legacy — prefer `offsiteJobs`)* | `tail` the sync log, `rclone about` | read the log; the host's rclone remote must authenticate | | `pbsMaintenance` | `pct exec -- proxmox-backup-manager` (GC, verify + sync jobs, datastore list); `ls` of `host/` snapshot dirs for `hostBackups` | root on the PVE host that runs the PBS container | +| `kubernetesClusters` | read-only `kubectl get` for cluster, workload, Velero, PodVolumeBackup, and CSI snapshot resources | SSH access to the configured `kubectl` prefix; for an LXC prefix, root on the PVE host for `pct exec` | | `smartHosts` | `smartctl -H` | root (raw device access); smartmontools installed | Targets are assumed Linux-ish with standard tools (GNU `find`, `awk`, `tail`). diff --git a/docs/modules/ROOT/examples/restoreguard.sample.json b/docs/modules/ROOT/examples/restoreguard.sample.json index 53d9b75..c44e353 100644 --- a/docs/modules/ROOT/examples/restoreguard.sample.json +++ b/docs/modules/ROOT/examples/restoreguard.sample.json @@ -207,6 +207,32 @@ } ], + + // Dashboard registration drift: verifies running Docker containers with published + // ports are registered in the Homepage dashboard (services.yaml ConfigMap on k3s). + // Catches the "forgot to add to the dashboard" failure after deploying a new service. + // k3sMasterAlias: SSH alias for the k3s node that hosts Homepage (ConfigMap fetch). + // dashboardHostAliases: Docker hosts to scan for running containers. + "dashboardDrift": { + "k3sMasterAlias": "myk3smaster", + "dashboardHostAliases": ["mydockerhost", "myotherhost"] + }, + + // Kubernetes / K3s clusters. This is config-file-only in the v1 wizard. + // kubectl is a command PREFIX executed on alias (for sshd-less LXC use + // e.g. "pct exec 601 -- k3s kubectl"). veleroNamespace: null explicitly + // disables Velero checks; omission defaults to "velero". + "kubernetesClusters": [ + { + "name": "homelab k3s", + "alias": "mypve", + "kubectl": "kubectl", + "veleroNamespace": "velero", + "maxBackupAgeHours": 26, + "checkWorkloads": true + } + ], + // Where each audit DELIVERS its JSON report (the same document --json prints). // The destinations live in their OWN self-contained file so a SECOND tool — e.g. // HCC — can read the SAME file, connect to the same folder/bucket/DB, and pull the diff --git a/docs/modules/ROOT/pages/checks.adoc b/docs/modules/ROOT/pages/checks.adoc index 60e7df8..da200a9 100644 --- a/docs/modules/ROOT/pages/checks.adoc +++ b/docs/modules/ROOT/pages/checks.adoc @@ -170,6 +170,21 @@ in real labs: the replica keeps its old snapshots and *looks* fine. | `cloudsync/not-off-box` | YELLOW | a top-level dataset is covered by no cloud-sync task — its data never leaves the box |=== +== Kubernetes / K3s (`kubernetesClusters`) + +[cols="2,1,4"] +|=== +| `k8s/unreachable` | RED | a configured cluster could not be discovered; the report is partial +| `k8s-backup/velero-missing` | RED | Velero's Backup resource type is absent +| `k8s-backup/no-backups` / `k8s-backup/no-completed` | RED | Velero is installed but has no backups or no successful backup +| `k8s-backup/stale` | RED | newest Completed backup exceeds `maxBackupAgeHours` +| `k8s-backup/failed` | RED | the latest unsuccessful backup is Failed, PartiallyFailed, or FailedValidation +| `k8s-backup/no-schedule` | YELLOW | backups exist but no Velero Schedule is configured +| `k8s-pvc/unbound` / `k8s-pvc/unprotected` | RED | a PVC is not Bound, or a Bound PVC lacks a completed Velero pod-volume backup or Ready CSI snapshot in the latest completed backup +| `k8s-workload/unavailable` | YELLOW | a deployment has fewer available than desired replicas +| `k8s-node/not-ready` | RED | a cluster node is NotReady +|=== + == Storage & disks (`pveNodes`, `trueNas`, `smartHosts`) [cols="2,1,4"] diff --git a/docs/modules/ROOT/pages/configuration.adoc b/docs/modules/ROOT/pages/configuration.adoc index 253060b..971e50b 100644 --- a/docs/modules/ROOT/pages/configuration.adoc +++ b/docs/modules/ROOT/pages/configuration.adoc @@ -66,6 +66,31 @@ A list — one entry per dump job you run. Dump files are expected to be named *Host needs:* read access to the dump directory. +== `kubernetesClusters` — Kubernetes / K3s + +[source,json] +---- +"kubernetesClusters": [ + { "name": "homelab k3s", "alias": "mypve", + "kubectl": "pct exec 601 -- k3s kubectl", "veleroNamespace": "velero", + "maxBackupAgeHours": 26, "checkWorkloads": true } +] +---- + +Each cluster has a unique `name`, an SSH `alias`, and a non-empty `kubectl` +command prefix. `veleroNamespace` omitted defaults to `velero`; set it explicitly +to `null` to disable Velero requirements. `maxBackupAgeHours` must be positive. +The provider runs read-only node, deployment, PVC, pod, Velero backup, schedule, +pod-volume-backup, and optional CSI volume-snapshot queries. Each Bound PVC needs a +completed Velero pod-volume backup or Ready CSI snapshot from the latest successful +backup; a Backup resource alone is not data-protection evidence. +Workload checks can be disabled with `checkWorkloads: false`; do not +disable both workload checks and Velero, because that would audit no surface. +Kubernetes is not included in the v1 setup wizard; configure it by hand. + +*Host needs:* SSH access to a machine with the configured `kubectl` command prefix; +for an LXC prefix, root access on the PVE host to run `pct exec`. + == `pveNodes` — Proxmox image backups [source,json] diff --git a/docs/modules/ROOT/pages/index.adoc b/docs/modules/ROOT/pages/index.adoc index 2941ed4..f9c81e9 100644 --- a/docs/modules/ROOT/pages/index.adoc +++ b/docs/modules/ROOT/pages/index.adoc @@ -70,6 +70,8 @@ xref:configuration.adoc[configuration reference]): host: the snapshot job still runs, and the replica is still catching up. * *TrueNAS SCALE* — ZFS snapshot freshness, cloud-sync task health, datasets that never leave the box, pool health and scrub age. +* *Kubernetes / K3s* — Velero coverage and freshness, PVC protection, + deployment availability, and node readiness. * *Off-site copies* — every scheduled rclone sync job ran, succeeded, and is recent; the destination has room. A job that never ran is a finding, not a gap. * *Disks* — SMART health on the machines that own the physical disks. diff --git a/restoreguard.sample.json b/restoreguard.sample.json index 7847310..c44e353 100644 --- a/restoreguard.sample.json +++ b/restoreguard.sample.json @@ -218,6 +218,21 @@ "dashboardHostAliases": ["mydockerhost", "myotherhost"] }, + // Kubernetes / K3s clusters. This is config-file-only in the v1 wizard. + // kubectl is a command PREFIX executed on alias (for sshd-less LXC use + // e.g. "pct exec 601 -- k3s kubectl"). veleroNamespace: null explicitly + // disables Velero checks; omission defaults to "velero". + "kubernetesClusters": [ + { + "name": "homelab k3s", + "alias": "mypve", + "kubectl": "kubectl", + "veleroNamespace": "velero", + "maxBackupAgeHours": 26, + "checkWorkloads": true + } + ], + // Where each audit DELIVERS its JSON report (the same document --json prints). // The destinations live in their OWN self-contained file so a SECOND tool — e.g. // HCC — can read the SAME file, connect to the same folder/bucket/DB, and pull the diff --git a/src/RestoreGuard.Checks/KubernetesCheck.cs b/src/RestoreGuard.Checks/KubernetesCheck.cs new file mode 100644 index 0000000..4ec05d0 --- /dev/null +++ b/src/RestoreGuard.Checks/KubernetesCheck.cs @@ -0,0 +1,105 @@ +using RestoreGuard.Core; +using RestoreGuard.Core.Model; +using RestoreGuard.Providers.Kubernetes; + +namespace RestoreGuard.Checks; + +public sealed record KubernetesExpectation(string Name, string Alias, TimeSpan MaxBackupAge, bool CheckWorkloads = true, + bool VeleroRequired = true, string? ProviderError = null) +{ + public string ClusterService => $"{Name} cluster"; +} + +/// Deterministic Kubernetes posture and Velero backup checks. +public sealed class KubernetesCheck( + IReadOnlyList states, + IReadOnlyList expectations) : ICheck +{ + public string RuleId => "k8s"; + + public IEnumerable Evaluate(LabInventory inventory) + { + foreach (var expected in expectations) + { + var state = states.FirstOrDefault(s => s.ClusterIdentity == expected.Name); + if (state is null) + { + yield return new Finding("k8s/unreachable", Severity.Red, expected.ClusterService, expected.Name, + $"Kubernetes discovery through '{expected.Alias}' did not complete: {expected.ProviderError ?? "no provider result"}", + "Run `restoreguard doctor` for this cluster, then fix kubectl reachability or RBAC before trusting the audit."); + continue; + } + var backups = inventory.Backups.Where(b => b.Tier == BackupTier.KubernetesBackup + && b.TargetService == $"{expected.Name} velero") + .OrderByDescending(b => b.Timestamp).ToList(); + + if (expected.VeleroRequired) + { + if (!state.VeleroInstalled) + yield return new Finding("k8s-backup/velero-missing", Severity.Red, expected.ClusterService, expected.Name, + $"Cluster '{expected.Name}' has no readable Velero Backup resource type.", + "Install Velero or set veleroNamespace to null if this cluster intentionally has no Velero protection."); + else if (backups.Count == 0) + yield return new Finding("k8s-backup/no-backups", Severity.Red, expected.ClusterService, expected.Name, + $"Velero is installed for cluster '{expected.Name}', but no Backup resources were found.", + "Create a Velero Schedule and run its first backup; also verify that it includes persistent-volume data."); + else + { + var latestCompleted = backups.Where(b => string.Equals(b.Status, "Completed", StringComparison.OrdinalIgnoreCase)) + .OrderByDescending(b => b.Timestamp).FirstOrDefault(); + var latestBackup = backups.OrderByDescending(b => b.Timestamp).First(); + if (string.Equals(latestBackup.Status, "Failed", StringComparison.OrdinalIgnoreCase) + || string.Equals(latestBackup.Status, "PartiallyFailed", StringComparison.OrdinalIgnoreCase) + || string.Equals(latestBackup.Status, "FailedValidation", StringComparison.OrdinalIgnoreCase)) + yield return new Finding("k8s-backup/failed", Severity.Red, expected.ClusterService, expected.Name, + $"Latest Velero backup is '{latestBackup.Location}' ({latestBackup.Status}).", + "Run `velero backup describe` and inspect the backup logs; partial failures often indicate a PVC snapshot or node-agent problem."); + if (latestCompleted is null) + yield return new Finding("k8s-backup/no-completed", Severity.Red, expected.ClusterService, expected.Name, + "Velero Backup resources exist, but none has completed successfully.", + "Fix the latest backup failure and verify a Completed backup before treating the cluster as protected."); + else if (inventory.CapturedAt - latestCompleted.Timestamp > expected.MaxBackupAge) + yield return new Finding("k8s-backup/stale", Severity.Red, expected.ClusterService, expected.Name, + $"Latest Completed Velero backup '{latestCompleted.Location}' is {(inventory.CapturedAt - latestCompleted.Timestamp).TotalHours:F0}h old (limit {expected.MaxBackupAge.TotalHours:F0}h).", + "Check the Velero Schedule and its last run: velero backup logs ."); + + if (state.Schedules.Count == 0) + yield return new Finding("k8s-backup/no-schedule", Severity.Yellow, expected.ClusterService, expected.Name, + "Velero backups exist but no Schedule resources were found.", + "Create a Velero Schedule so coverage does not depend on a human."); + if (latestCompleted is not null) + { + var protectedClaims = state.PodVolumeBackups + .Where(pvb => pvb.BackupName == latestCompleted.Location && pvb.Completed && pvb.PersistentVolumeClaim is not null) + .Select(pvb => $"{pvb.PodNamespace}/{pvb.PersistentVolumeClaim}") + .Concat(state.VolumeSnapshots.Where(snapshot => snapshot.BackupName == latestCompleted.Location && snapshot.ReadyToUse) + .Select(snapshot => $"{snapshot.Namespace}/{snapshot.ClaimName}")) + .ToHashSet(StringComparer.Ordinal); + foreach (var pvc in state.PersistentVolumeClaims.Where(pvc => + string.Equals(pvc.Phase, "Bound", StringComparison.OrdinalIgnoreCase) + && !protectedClaims.Contains($"{pvc.Namespace}/{pvc.Name}"))) + { + yield return new Finding("k8s-pvc/unprotected", Severity.Red, $"{pvc.Namespace}/{pvc.Name}", expected.Name, + $"Bound PVC '{pvc.Namespace}/{pvc.Name}' has no completed Velero pod-volume backup in latest Completed backup '{latestCompleted.Location}'.", + "Enable and verify a Velero file-system backup or CSI snapshot for this PVC, then restore a test PVC before treating it as protected."); + } + } + } + } + + if (!expected.CheckWorkloads) continue; + foreach (var pvc in state.PersistentVolumeClaims.Where(p => !string.Equals(p.Phase, "Bound", StringComparison.OrdinalIgnoreCase))) + yield return new Finding("k8s-pvc/unbound", Severity.Red, $"{pvc.Namespace}/{pvc.Name}", expected.Name, + $"PVC '{pvc.Namespace}/{pvc.Name}' is {pvc.Phase}, not Bound.", + "Inspect the PVC events and storage class; restore protection cannot rely on an unbound volume."); + foreach (var service in inventory.Services.Where(s => s.Host == expected.Name && s.Kind == ServiceKind.K8sWorkload && s.State == "degraded")) + yield return new Finding("k8s-workload/unavailable", Severity.Yellow, service.Name, expected.Name, + $"Deployment '{service.Name}' has fewer available replicas than desired.", + "Run kubectl describe deployment and kubectl rollout status to investigate."); + foreach (var service in inventory.Services.Where(s => s.Host == expected.Name && s.Kind == ServiceKind.K8sNode && s.State == "NotReady")) + yield return new Finding("k8s-node/not-ready", Severity.Red, service.Name, expected.Name, + $"Node '{service.Name}' is NotReady.", + "Inspect node conditions and kubelet health before trusting cluster backups."); + } + } +} diff --git a/src/RestoreGuard.Checks/RestoreGuard.Checks.csproj b/src/RestoreGuard.Checks/RestoreGuard.Checks.csproj index ad0aaee..c93e61a 100644 --- a/src/RestoreGuard.Checks/RestoreGuard.Checks.csproj +++ b/src/RestoreGuard.Checks/RestoreGuard.Checks.csproj @@ -2,6 +2,7 @@ + diff --git a/src/RestoreGuard.Cli/AuditRunner.cs b/src/RestoreGuard.Cli/AuditRunner.cs index 5397999..43d54c5 100644 --- a/src/RestoreGuard.Cli/AuditRunner.cs +++ b/src/RestoreGuard.Cli/AuditRunner.cs @@ -11,6 +11,7 @@ using RestoreGuard.Providers.Pve; using RestoreGuard.Providers.Smart; using RestoreGuard.Providers.TrueNas; +using RestoreGuard.Providers.Kubernetes; namespace RestoreGuard.Cli; @@ -22,6 +23,7 @@ public static async Task RunAsync(RestoreGuardConfig config, string configD var suppressions = config.LoadSuppressions(configDir); var docker = new DockerProvider(ssh); + var kubernetes = new KubernetesProvider(ssh, Progress); var dbDump = new DbDumpProvider(ssh); var pve = new PveProvider(ssh); @@ -39,6 +41,9 @@ public static async Task RunAsync(RestoreGuardConfig config, string configD var dockerTasks = config.DockerHosts .Select(h => Track("docker", h.Alias, docker.GetServicesAsync(h))) .ToList(); + var kubernetesTasks = (config.KubernetesClusters ?? []) + .Select(c => Track("k8s", c.Alias, kubernetes.GetClusterAsync(c))) + .ToList(); var pveTasks = (config.PveNodes ?? []) .Select(n => Track("pve", n.Alias, pve.GetNodeAsync(n))) .ToList(); @@ -89,7 +94,9 @@ public static async Task RunAsync(RestoreGuardConfig config, string configD dd.DashboardHostAliases.Select(a => new DockerHostConfig(a)).ToList())) : Task.FromResult<(string, DashboardProvider.DashboardProbeResult?, string?)>(("", null, null)); - Progress($"auditing: {probes.Count} probe(s) across the lab, in parallel (Ctrl+C stops and reports what finished)..."); + var kubernetesSurfaceCount = (config.KubernetesClusters ?? []).Sum(cluster => KubernetesProvider.EnabledSurfaces(cluster).Count); + var displayedProbeCount = probes.Count - kubernetesTasks.Count + kubernetesSurfaceCount; + Progress($"auditing: {displayedProbeCount} probe(s) across the lab, in parallel (Ctrl+C stops and reports what finished)..."); var discovery = Stopwatch.StartNew(); var all = Task.WhenAll(probes.Select(p => p.Task)); @@ -106,12 +113,29 @@ public static async Task RunAsync(RestoreGuardConfig config, string configD var artifacts = new List(); var providerErrors = new List(); + // A cluster-level service makes every cluster-wide Kubernetes finding a stable + // suppression target even when its provider fails before yielding any inventory. + services.AddRange((config.KubernetesClusters ?? []).Select(cluster => + new Service($"{cluster.Name} cluster", cluster.Name, ServiceKind.K8sCluster, "unknown", null, [], null))); + foreach (var (host, result, error) in dockerTasks.Select(t => t.Result)) { if (result is not null) services.AddRange(result); if (error is not null) providerErrors.Add($"{host}: {error}"); } + var kubernetesStates = new List(); + foreach (var (host, result, error) in kubernetesTasks.Select(t => t.Result)) + { + if (result is not null) + { + services.AddRange(result.Services); + artifacts.AddRange(result.Backups); + kubernetesStates.Add(result.State); + } + if (error is not null) providerErrors.Add($"{host}: {error}"); + } + // PBS snapshots join against guests from ALL nodes (shared datastore, colliding // vmids), so collect every node inventory before assembling artifacts. var guests = new List(); @@ -283,6 +307,18 @@ public static async Task RunAsync(RestoreGuardConfig config, string configD { checks.Add(new DashboardRegistrationDriftCheck()); } + if (config.KubernetesClusters is { Count: > 0 } clusters) + { + var expectations = clusters.Zip(kubernetesTasks, (cluster, task) => + { + var result = task.Result; + return new KubernetesExpectation(cluster.Name, cluster.Alias, + TimeSpan.FromHours(cluster.MaxBackupAgeHours), cluster.CheckWorkloads, + cluster.VeleroNamespace is not null, result.Item3); + }).ToList(); + checks.Add(new KubernetesCheck(kubernetesStates, + expectations)); + } var report = new CheckEngine(checks).Run(inventory, suppressions, DateTimeOffset.UtcNow); var sinks = ReportPublisher.BuildSinks(config, configDir); diff --git a/src/RestoreGuard.Cli/Doctor.cs b/src/RestoreGuard.Cli/Doctor.cs index aeff724..053c4bf 100644 --- a/src/RestoreGuard.Cli/Doctor.cs +++ b/src/RestoreGuard.Cli/Doctor.cs @@ -4,7 +4,7 @@ namespace RestoreGuard.Cli; /// One preflight probe: a cheap read-only command that succeeds iff the /// host grants what the corresponding audit surface needs. -public sealed record DoctorProbe(string Host, string Area, string Command, string Requirement); +public sealed record DoctorProbe(string Host, string Area, string Command, string Requirement, string? ExpectedStdOut = null); /// /// `restoreguard --doctor`: verifies every prerequisite of the configured surfaces @@ -24,6 +24,50 @@ public static IReadOnlyList BuildProbes(RestoreGuardConfig config) "SSH user can reach the Docker daemon and has the compose v2 plugin (>= 2.17 for `config --format json`)")); } + foreach (var cluster in config.KubernetesClusters ?? []) + { + probes.Add(new DoctorProbe(cluster.Alias, "k8s", + $"{cluster.Kubectl} get --raw=/readyz > /dev/null", + $"Kubernetes API reachable through '{cluster.Kubectl}' on {cluster.Alias}")); + if (cluster.VeleroNamespace is not null) + { + probes.Add(new DoctorProbe(cluster.Alias, "k8s", + $"{cluster.Kubectl} auth can-i list persistentvolumeclaims --all-namespaces", + "RBAC allows reading Kubernetes PVCs for Velero data-protection coverage", "yes")); + probes.Add(new DoctorProbe(cluster.Alias, "k8s", + $"{cluster.Kubectl} auth can-i list backups.velero.io -n '{cluster.VeleroNamespace}'", + $"RBAC allows reading Velero backups in namespace '{cluster.VeleroNamespace}'", + "yes")); + probes.Add(new DoctorProbe(cluster.Alias, "k8s", + $"{cluster.Kubectl} auth can-i list schedules.velero.io -n '{cluster.VeleroNamespace}'", + $"RBAC allows reading Velero schedules in namespace '{cluster.VeleroNamespace}'", + "yes")); + probes.Add(new DoctorProbe(cluster.Alias, "k8s", + $"{cluster.Kubectl} auth can-i list podvolumebackups.velero.io -n '{cluster.VeleroNamespace}'", + $"RBAC allows reading Velero pod-volume backups in namespace '{cluster.VeleroNamespace}'", + "yes")); + probes.Add(new DoctorProbe(cluster.Alias, "k8s", + $"{cluster.Kubectl} auth can-i list pods --all-namespaces", + "RBAC allows mapping pod-volume backups to their PVCs", "yes")); + probes.Add(new DoctorProbe(cluster.Alias, "k8s", + $"if {cluster.Kubectl} api-resources --api-group=snapshot.storage.k8s.io --no-headers | grep -q '^volumesnapshots'; then [ \"$({cluster.Kubectl} auth can-i list volumesnapshots.snapshot.storage.k8s.io --all-namespaces)\" = yes ]; fi", + "CSI volume-snapshot API is absent, or RBAC allows reading it when installed")); + } + if (cluster.CheckWorkloads) + { + probes.Add(new DoctorProbe(cluster.Alias, "k8s", + $"{cluster.Kubectl} auth can-i list nodes", + "RBAC allows reading Kubernetes nodes", "yes")); + probes.Add(new DoctorProbe(cluster.Alias, "k8s", + $"{cluster.Kubectl} auth can-i list deployments.apps --all-namespaces", + "RBAC allows reading Kubernetes deployments", "yes")); + if (cluster.VeleroNamespace is null) + probes.Add(new DoctorProbe(cluster.Alias, "k8s", + $"{cluster.Kubectl} auth can-i list persistentvolumeclaims --all-namespaces", + "RBAC allows reading Kubernetes PVCs", "yes")); + } + } + foreach (var db in config.LogicalDbBackups ?? []) { probes.Add(new DoctorProbe(db.Host, "db-dumps", @@ -168,7 +212,9 @@ public static async Task RunAsync(RestoreGuardConfig config, ISshProvider s try { var r = await ssh.RunAsync(p.Host, p.Command); - return (Probe: p, Ok: r.ExitCode == 0, Detail: r.StdErr.Trim()); + var outputOk = p.ExpectedStdOut is null || string.Equals(r.StdOut.Trim(), p.ExpectedStdOut, StringComparison.Ordinal); + var detail = outputOk ? r.StdErr.Trim() : $"expected stdout '{p.ExpectedStdOut}', got '{r.StdOut.Trim()}'"; + return (Probe: p, Ok: r.ExitCode == 0 && outputOk, Detail: detail); } catch (Exception ex) { diff --git a/src/RestoreGuard.Cli/RestoreGuardConfig.cs b/src/RestoreGuard.Cli/RestoreGuardConfig.cs index b09b405..8235fbf 100644 --- a/src/RestoreGuard.Cli/RestoreGuardConfig.cs +++ b/src/RestoreGuard.Cli/RestoreGuardConfig.cs @@ -2,6 +2,7 @@ using RestoreGuard.Core; using RestoreGuard.Providers.Docker; using RestoreGuard.Providers.Pve; +using RestoreGuard.Providers.Kubernetes; namespace RestoreGuard.Cli; @@ -26,7 +27,8 @@ public sealed record RestoreGuardConfig( // section still works but is the legacy form; the wizard writes the file. string? ReportingFile = null, ReportingConfig? Reporting = null, - DashboardDriftCliConfig? DashboardDrift = null) + DashboardDriftCliConfig? DashboardDrift = null, + IReadOnlyList? KubernetesClusters = null) { private static readonly JsonSerializerOptions JsonOptions = new() { @@ -194,11 +196,32 @@ public IReadOnlyList Validate() } } + var kubernetesNames = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var (cluster, i) in (KubernetesClusters ?? []).Select((c, i) => (c, i))) + { + if (string.IsNullOrWhiteSpace(cluster.Name)) errors.Add($"kubernetesClusters[{i}].name is empty."); + else if (!kubernetesNames.Add(cluster.Name)) errors.Add($"kubernetesClusters[{i}].name '{cluster.Name}' is duplicated."); + if (string.IsNullOrWhiteSpace(cluster.Alias)) errors.Add($"kubernetesClusters[{i}].alias is empty."); + if (string.IsNullOrWhiteSpace(cluster.Kubectl)) errors.Add($"kubernetesClusters[{i}].kubectl is empty."); + if (cluster.VeleroNamespace is { } ns && !IsKubernetesNamespace(ns)) + errors.Add($"kubernetesClusters[{i}].veleroNamespace '{ns}' is not a valid Kubernetes namespace."); + if (cluster.VeleroNamespace is null && !cluster.CheckWorkloads) + errors.Add($"kubernetesClusters[{i}] disables both Velero and workload checks, so it would audit nothing."); + if (cluster.MaxBackupAgeHours <= 0) errors.Add($"kubernetesClusters[{i}].maxBackupAgeHours must be positive."); + } + Reporting?.Validate(errors); return errors; } + private static bool IsKubernetesNamespace(string value) + { + if (value.Length is < 1 or > 63 || value[0] == '-' || value[^1] == '-') + return false; + return value.All(c => (c is >= 'a' and <= 'z') || char.IsAsciiDigit(c) || c == '-'); + } + public IReadOnlyList LoadSuppressions(string configDir) { if (SuppressionsFile is null) diff --git a/src/RestoreGuard.Core/Model/BackupArtifact.cs b/src/RestoreGuard.Core/Model/BackupArtifact.cs index 455cfac..98fd716 100644 --- a/src/RestoreGuard.Core/Model/BackupArtifact.cs +++ b/src/RestoreGuard.Core/Model/BackupArtifact.cs @@ -9,6 +9,7 @@ public enum BackupTier ZfsSnapshot, CloudSync, FileBackup, + KubernetesBackup, } /// diff --git a/src/RestoreGuard.Core/Model/Service.cs b/src/RestoreGuard.Core/Model/Service.cs index 12fc233..220bc81 100644 --- a/src/RestoreGuard.Core/Model/Service.cs +++ b/src/RestoreGuard.Core/Model/Service.cs @@ -6,6 +6,10 @@ public enum ServiceKind Vm, Lxc, Stack, + K8sNode, + K8sWorkload, + K8sCluster, + K8sPersistentVolumeClaim, } /// A mount as either declared (compose config) or live (container inspect). diff --git a/src/RestoreGuard.Providers/Docker/DockerProvider.cs b/src/RestoreGuard.Providers/Docker/DockerProvider.cs index 8d443da..132ba0b 100644 --- a/src/RestoreGuard.Providers/Docker/DockerProvider.cs +++ b/src/RestoreGuard.Providers/Docker/DockerProvider.cs @@ -77,4 +77,8 @@ public async Task> GetServicesAsync(DockerHostConfig host private static string Truncate(string s) => s.Length <= 300 ? s.Trim() : s[..300].Trim() + "…"; } -public sealed class ProviderException(string message) : Exception(message); +public class ProviderException : Exception +{ + public ProviderException(string message) : base(message) { } + public ProviderException(string message, Exception innerException) : base(message, innerException) { } +} diff --git a/src/RestoreGuard.Providers/Kubernetes/KubectlParser.cs b/src/RestoreGuard.Providers/Kubernetes/KubectlParser.cs new file mode 100644 index 0000000..f3ee7e1 --- /dev/null +++ b/src/RestoreGuard.Providers/Kubernetes/KubectlParser.cs @@ -0,0 +1,118 @@ +using System.Text.Json; +using RestoreGuard.Core.Model; + +namespace RestoreGuard.Providers.Kubernetes; + +public static class KubectlParser +{ + public static IReadOnlyList ParseNodes(string json) + { + using var doc = JsonDocument.Parse(json); + return Items(doc).Select(x => new KubernetesNode( + x.GetProperty("metadata").GetProperty("name").GetString() ?? "", + x.GetProperty("status").TryGetProperty("conditions", out var conditions) && conditions.EnumerateArray() + .Where(c => c.TryGetProperty("type", out var type) && type.GetString() == "Ready") + .Any(c => c.TryGetProperty("status", out var state) && state.GetString() == "True") ? "Ready" : "NotReady", + x.GetProperty("metadata").TryGetProperty("uid", out var uid) ? uid.GetString() : null)).ToList(); + } + + public static IReadOnlyList ParseDeployments(string json) + { + using var doc = JsonDocument.Parse(json); + return Items(doc).Select(x => new KubernetesDeployment( + x.GetProperty("metadata").GetProperty("namespace").GetString() ?? "", + x.GetProperty("metadata").GetProperty("name").GetString() ?? "", + Number(x, "spec", "replicas", defaultValue: 1), Number(x, "status", "availableReplicas"))).ToList(); + } + + public static IReadOnlyList ParsePersistentVolumeClaims(string json) + { + using var doc = JsonDocument.Parse(json); + return Items(doc).Select(x => new KubernetesPersistentVolumeClaim( + x.GetProperty("metadata").GetProperty("namespace").GetString() ?? "", + x.GetProperty("metadata").GetProperty("name").GetString() ?? "", + x.GetProperty("status").TryGetProperty("phase", out var phase) ? phase.GetString() ?? "Unknown" : "Unknown", + x.GetProperty("spec").TryGetProperty("volumeName", out var volume) ? volume.GetString() : null)).ToList(); + } + + public static IReadOnlyList ParseSchedules(string json, string ns) + { + using var doc = JsonDocument.Parse(json); + return Items(doc).Select(x => new KubernetesSchedule(ns, x.GetProperty("metadata").GetProperty("name").GetString() ?? "", + x.GetProperty("status").TryGetProperty("lastBackup", out var last) ? last.GetString() : null)).ToList(); + } + + public static IReadOnlyList ParseBackups(string json, string clusterIdentity) + { + using var doc = JsonDocument.Parse(json); + return Items(doc).Select(x => { + var status = x.TryGetProperty("status", out var s) ? s : default; + var timestamp = ReadDate(status, "completionTimestamp") ?? ReadDate(x.GetProperty("metadata"), "creationTimestamp") ?? DateTimeOffset.MinValue; + var phase = status.ValueKind == JsonValueKind.Object && status.TryGetProperty("phase", out var p) ? p.GetString() : null; + return new BackupArtifact(BackupTier.KubernetesBackup, $"{clusterIdentity} velero", + x.GetProperty("metadata").GetProperty("name").GetString() ?? "", timestamp, 0, "velero", false, phase); + }).ToList(); + } + + public static IReadOnlyList ParsePodVolumeBackups(string json, string fallbackNamespace) + { + using var doc = JsonDocument.Parse(json); + return Items(doc).Select(x => + { + var metadata = x.GetProperty("metadata"); + var spec = x.TryGetProperty("spec", out var s) ? s : default; + var status = x.TryGetProperty("status", out var st) ? st : default; + var backupName = Label(metadata, "velero.io/backup-name") + ?? String(spec, "backupName") + ?? ""; + var pod = spec.ValueKind == JsonValueKind.Object && spec.TryGetProperty("pod", out var podValue) + ? podValue : default; + return new KubernetesPodVolumeBackup( + backupName, + String(pod, "namespace") ?? (metadata.TryGetProperty("namespace", out var ns) ? ns.GetString() ?? fallbackNamespace : fallbackNamespace), + String(pod, "name") ?? "", + String(spec, "volume") ?? "", + String(status, "phase")); + }).ToList(); + } + + public static IReadOnlyList ParsePodVolumeClaims(string json) + { + using var doc = JsonDocument.Parse(json); + return Items(doc).SelectMany(pod => + { + var metadata = pod.GetProperty("metadata"); + var podNamespace = metadata.GetProperty("namespace").GetString() ?? ""; + var podName = metadata.GetProperty("name").GetString() ?? ""; + if (!pod.TryGetProperty("spec", out var spec) || !spec.TryGetProperty("volumes", out var volumes)) return []; + return volumes.EnumerateArray().Where(volume => volume.TryGetProperty("persistentVolumeClaim", out _)).Select(volume => + new KubernetesPodVolumeClaim(podNamespace, podName, + volume.GetProperty("name").GetString() ?? "", + volume.GetProperty("persistentVolumeClaim").GetProperty("claimName").GetString() ?? "")); + }).ToList(); + } + + public static IReadOnlyList ParseVolumeSnapshots(string json) + { + using var doc = JsonDocument.Parse(json); + return Items(doc).Select(snapshot => + { + var metadata = snapshot.GetProperty("metadata"); + var source = snapshot.GetProperty("spec").GetProperty("source"); + var status = snapshot.TryGetProperty("status", out var candidate) ? candidate : default; + return new KubernetesVolumeSnapshot( + Label(metadata, "velero.io/backup-name") ?? "", + metadata.GetProperty("namespace").GetString() ?? "", + String(source, "persistentVolumeClaimName") ?? "", + status.ValueKind == JsonValueKind.Object && status.TryGetProperty("readyToUse", out var ready) && ready.ValueKind == JsonValueKind.True); + }).ToList(); + } + + private static IEnumerable Items(JsonDocument doc) => doc.RootElement.GetProperty("items").EnumerateArray(); + private static int Number(JsonElement root, string parent, string property, int defaultValue = 0) => root.TryGetProperty(parent, out var p) ? Number(p, property, defaultValue) : defaultValue; + private static int Number(JsonElement root, string property, int defaultValue = 0) => root.ValueKind == JsonValueKind.Object && root.TryGetProperty(property, out var value) && value.TryGetInt32(out var n) ? n : defaultValue; + private static string? String(JsonElement root, string property) => root.ValueKind == JsonValueKind.Object && root.TryGetProperty(property, out var value) ? value.GetString() : null; + private static string? Label(JsonElement metadata, string label) => metadata.TryGetProperty("labels", out var labels) + && labels.ValueKind == JsonValueKind.Object && labels.TryGetProperty(label, out var value) ? value.GetString() : null; + private static DateTimeOffset? ReadDate(JsonElement root, string property) => root.ValueKind == JsonValueKind.Object && root.TryGetProperty(property, out var value) && DateTimeOffset.TryParse(value.GetString(), out var date) ? date : null; +} diff --git a/src/RestoreGuard.Providers/Kubernetes/KubernetesProvider.cs b/src/RestoreGuard.Providers/Kubernetes/KubernetesProvider.cs new file mode 100644 index 0000000..211df8b --- /dev/null +++ b/src/RestoreGuard.Providers/Kubernetes/KubernetesProvider.cs @@ -0,0 +1,237 @@ +using System.Text.Json; +using RestoreGuard.Core.Model; +using RestoreGuard.Providers.Docker; + +namespace RestoreGuard.Providers.Kubernetes; + +public sealed record KubernetesClusterConfig( + string Name, + string Alias, + string Kubectl = "kubectl", + string? VeleroNamespace = "velero", + double MaxBackupAgeHours = 26, + bool CheckWorkloads = true); + +public sealed record KubernetesNode(string Name, string State, string? Uid = null); +public sealed record KubernetesDeployment(string Namespace, string Name, int DesiredReplicas, int AvailableReplicas); +public sealed record KubernetesPersistentVolumeClaim(string Namespace, string Name, string Phase, string? VolumeName = null); +public sealed record KubernetesSchedule(string Namespace, string Name, string? LastBackup = null); +/// Read-only evidence that Velero's file-system backup path captured a pod volume. +public sealed record KubernetesPodVolumeBackup( + string BackupName, + string PodNamespace, + string PodName, + string Volume, + string? Phase, + string? PersistentVolumeClaim = null) +{ + public bool Completed => string.Equals(Phase, "Completed", StringComparison.OrdinalIgnoreCase); +} + +public sealed record KubernetesPodVolumeClaim(string Namespace, string PodName, string Volume, string ClaimName); +public sealed record KubernetesVolumeSnapshot(string BackupName, string Namespace, string ClaimName, bool ReadyToUse); + +/// Discovery output which is deliberately separate from the generic inventory model. +public sealed record KubernetesState( + string ClusterIdentity, + IReadOnlyList PersistentVolumeClaims, + IReadOnlyList Schedules, + bool VeleroInstalled, + IReadOnlyList PodVolumeBackups, + IReadOnlyList VolumeSnapshots, + DateTimeOffset CapturedAt); + +public sealed record KubernetesInventory( + IReadOnlyList Services, + IReadOnlyList Backups, + KubernetesState State); + +public enum KubernetesProviderErrorKind { Forbidden, Authentication, Transport, Malformed, Command } + +public sealed class KubernetesProviderException : ProviderException +{ + public KubernetesProviderException(KubernetesProviderErrorKind kind, string message) + : base(message) => Kind = kind; + + public KubernetesProviderException(KubernetesProviderErrorKind kind, string message, Exception inner) + : base(message, inner) => Kind = kind; + + public KubernetesProviderErrorKind Kind { get; } +} + +public sealed class KubernetesProvider(ISshProvider ssh, Action? progress = null) +{ + public static IReadOnlyList EnabledSurfaces(KubernetesClusterConfig config) + { + var surfaces = new List(); + if (config.CheckWorkloads) surfaces.AddRange(["nodes", "deployments"]); + if (config.CheckWorkloads || config.VeleroNamespace is not null) surfaces.Add("persistent-volume-claims"); + if (config.VeleroNamespace is not null) surfaces.AddRange(["velero-backups", "velero-schedules", "pod-volume-backups", "pods", "csi-volume-snapshots"]); + return surfaces; + } + + public async Task GetClusterAsync( + KubernetesClusterConfig config, CancellationToken ct = default) + { + var services = new List(); + var claims = new List(); + var schedules = new List(); + var backups = new List(); + var podVolumeBackups = new List(); + var volumeSnapshots = new List(); + var veleroInstalled = false; + + if (config.CheckWorkloads) + { + var nodesJson = await RunJsonAsync(config, "get nodes -o json", ct); + foreach (var node in Parse("nodes", () => KubectlParser.ParseNodes(nodesJson))) + services.Add(new Service(node.Name, config.Name, ServiceKind.K8sNode, node.State, null, [], null)); + + var deploymentsJson = await RunJsonAsync(config, "get deploy -A -o json", ct); + services.AddRange(Parse("deployments", () => KubectlParser.ParseDeployments(deploymentsJson)) + .Select(d => new Service($"{d.Namespace}/{d.Name}", config.Name, ServiceKind.K8sWorkload, + d.AvailableReplicas >= d.DesiredReplicas ? "available" : "degraded", null, [], null))); + } + + // Bound claims are data-protection inputs whenever Velero is enabled, even if + // workload availability checks are disabled. Do not claim coverage without them. + if (config.CheckWorkloads || config.VeleroNamespace is not null) + { + var claimsJson = await RunJsonAsync(config, "get pvc -A -o json", ct); + claims.AddRange(Parse("persistent volume claims", () => KubectlParser.ParsePersistentVolumeClaims(claimsJson))); + services.AddRange(claims.Select(pvc => new Service($"{pvc.Namespace}/{pvc.Name}", config.Name, + ServiceKind.K8sPersistentVolumeClaim, pvc.Phase, null, [], null))); + } + + if (config.VeleroNamespace is not null) + { + var backup = await RunVeleroAsync(config, $"get backups.velero.io -n {Quote(config.VeleroNamespace)} -o json", ct); + if (backup.MissingResource) + veleroInstalled = false; + else + { + veleroInstalled = true; + backups.AddRange(Parse("Velero backups", () => KubectlParser.ParseBackups(backup.Json!, config.Name))); + } + + if (veleroInstalled) + { + var schedule = await RunVeleroAsync(config, $"get schedules.velero.io -n {Quote(config.VeleroNamespace)} -o json", ct); + if (!schedule.MissingResource) + schedules.AddRange(Parse("Velero schedules", () => KubectlParser.ParseSchedules(schedule.Json!, config.VeleroNamespace))); + + var pvb = await RunVeleroAsync(config, $"get podvolumebackups.velero.io -n {Quote(config.VeleroNamespace)} -o json", ct); + if (!pvb.MissingResource) + { + var podsJson = await RunJsonAsync(config, "get pods -A -o json", ct); + var podClaims = Parse("pods", () => KubectlParser.ParsePodVolumeClaims(podsJson)); + podVolumeBackups.AddRange(Parse("Velero pod-volume backups", () => KubectlParser.ParsePodVolumeBackups(pvb.Json!, config.VeleroNamespace)) + .Select(pvbBackup => pvbBackup with + { + PersistentVolumeClaim = podClaims.FirstOrDefault(reference => + reference.Namespace == pvbBackup.PodNamespace + && reference.PodName == pvbBackup.PodName + && reference.Volume == pvbBackup.Volume)?.ClaimName, + })); + } + + var snapshots = await RunOptionalJsonAsync(config, "get volumesnapshots.snapshot.storage.k8s.io -A -o json", ct); + if (snapshots is not null) + volumeSnapshots.AddRange(Parse("CSI volume snapshots", () => KubectlParser.ParseVolumeSnapshots(snapshots))); + } + } + + return new KubernetesInventory(services, backups, + new KubernetesState(config.Name, claims, schedules, veleroInstalled, podVolumeBackups, volumeSnapshots, DateTimeOffset.UtcNow)); + } + + public Task GetAsync(KubernetesClusterConfig config, CancellationToken ct = default) => + GetClusterAsync(config, ct); + + private async Task RunJsonAsync(KubernetesClusterConfig config, string args, CancellationToken ct) + { + var command = $"{config.Kubectl} {args}"; + progress?.Invoke($" ... [k8s] {config.Alias}: {Surface(args)}"); + SshResult result; + try { result = await ssh.RunAsync(config.Alias, command, ct); } + catch (Exception ex) when (ex is not OperationCanceledException) + { throw new KubernetesProviderException(KubernetesProviderErrorKind.Transport, $"'{command}' on {config.Alias} failed.", ex); } + if (result.ExitCode != 0) throw Classify(command, result.StdErr); + try { using var _ = JsonDocument.Parse(result.StdOut); } + catch (JsonException ex) { throw new KubernetesProviderException(KubernetesProviderErrorKind.Malformed, $"'{command}' returned malformed JSON.", ex); } + return result.StdOut; + } + + private async Task<(string? Json, bool MissingResource)> RunVeleroAsync(KubernetesClusterConfig config, string args, CancellationToken ct) + { + var command = $"{config.Kubectl} {args}"; + progress?.Invoke($" ... [k8s] {config.Alias}: {Surface(args)}"); + SshResult result; + try { result = await ssh.RunAsync(config.Alias, command, ct); } + catch (Exception ex) when (ex is not OperationCanceledException) + { throw new KubernetesProviderException(KubernetesProviderErrorKind.Transport, $"'{command}' on {config.Alias} failed.", ex); } + if (result.ExitCode != 0) + { + var error = result.StdErr.Trim(); + if (IsMissingResource(error)) return (null, true); + throw Classify(command, error); + } + try { using var _ = JsonDocument.Parse(result.StdOut); } + catch (JsonException ex) { throw new KubernetesProviderException(KubernetesProviderErrorKind.Malformed, $"'{command}' returned malformed JSON.", ex); } + return (result.StdOut, false); + } + + private async Task RunOptionalJsonAsync(KubernetesClusterConfig config, string args, CancellationToken ct) + { + var command = $"{config.Kubectl} {args}"; + progress?.Invoke($" ... [k8s] {config.Alias}: {Surface(args)}"); + SshResult result; + try { result = await ssh.RunAsync(config.Alias, command, ct); } + catch (Exception ex) when (ex is not OperationCanceledException) + { throw new KubernetesProviderException(KubernetesProviderErrorKind.Transport, $"'{command}' on {config.Alias} failed.", ex); } + if (result.ExitCode != 0) + { + if (IsMissingResource(result.StdErr)) return null; + throw Classify(command, result.StdErr); + } + try { using var _ = JsonDocument.Parse(result.StdOut); } + catch (JsonException ex) { throw new KubernetesProviderException(KubernetesProviderErrorKind.Malformed, $"'{command}' returned malformed JSON.", ex); } + return result.StdOut; + } + + private static bool IsMissingResource(string error) => + error.Contains("the server doesn't have a resource type", StringComparison.OrdinalIgnoreCase) + || error.Contains("no matches for kind", StringComparison.OrdinalIgnoreCase); + + private static T Parse(string surface, Func parser) + { + try { return parser(); } + catch (Exception ex) when (ex is InvalidOperationException or KeyNotFoundException) + { throw new KubernetesProviderException(KubernetesProviderErrorKind.Malformed, $"Kubernetes {surface} JSON has an unexpected shape.", ex); } + } + + private static KubernetesProviderException Classify(string command, string stderr) + { + var text = string.IsNullOrWhiteSpace(stderr) ? "command failed" : stderr.Trim().Split('\n')[0]; + var kind = text.Contains("forbidden", StringComparison.OrdinalIgnoreCase) || text.Contains("403", StringComparison.OrdinalIgnoreCase) + ? KubernetesProviderErrorKind.Forbidden + : text.Contains("unauthorized", StringComparison.OrdinalIgnoreCase) || text.Contains("401", StringComparison.OrdinalIgnoreCase) + ? KubernetesProviderErrorKind.Authentication : KubernetesProviderErrorKind.Command; + return new KubernetesProviderException(kind, $"'{command}' failed: {text}"); + } + + private static string Quote(string value) => "'" + value.Replace("'", "'\\''") + "'"; + + private static string Surface(string args) => args switch + { + var value when value.StartsWith("get nodes", StringComparison.Ordinal) => "nodes", + var value when value.StartsWith("get deploy", StringComparison.Ordinal) => "deployments", + var value when value.StartsWith("get pvc", StringComparison.Ordinal) => "persistent-volume-claims", + var value when value.StartsWith("get backups.velero.io", StringComparison.Ordinal) => "velero-backups", + var value when value.StartsWith("get schedules.velero.io", StringComparison.Ordinal) => "velero-schedules", + var value when value.StartsWith("get podvolumebackups.velero.io", StringComparison.Ordinal) => "pod-volume-backups", + var value when value.StartsWith("get pods", StringComparison.Ordinal) => "pods", + var value when value.StartsWith("get volumesnapshots.snapshot.storage.k8s.io", StringComparison.Ordinal) => "csi-volume-snapshots", + _ => args, + }; +} diff --git a/tests/RestoreGuard.Tests/AuditRunnerTests.cs b/tests/RestoreGuard.Tests/AuditRunnerTests.cs index 18bbdd1..c0108e2 100644 --- a/tests/RestoreGuard.Tests/AuditRunnerTests.cs +++ b/tests/RestoreGuard.Tests/AuditRunnerTests.cs @@ -2,6 +2,7 @@ using RestoreGuard.Cli; using RestoreGuard.Providers; using RestoreGuard.Providers.Docker; +using RestoreGuard.Providers.Kubernetes; namespace RestoreGuard.Tests; @@ -84,6 +85,23 @@ public async Task PartialFailure_ProgressOnStderr_CleanJsonOnStdout_ExitCode1() Assert.True(File.Exists(Path.Combine(_dir.FullName, "default-reports", "latest.json"))); } + [Fact] + public async Task KubernetesProviderFailure_IsPartialAndTrackedByClusterAlias() + { + var config = new RestoreGuardConfig([], null, null, 26, null, null, null, null, null, null, + KubernetesClusters: [new KubernetesClusterConfig("cluster-a", "deadhost")]); + + var (exit, stdout, progress) = await RunAsync(config, _dir.FullName); + + Assert.Equal(1, exit); + using var report = JsonDocument.Parse(stdout); + Assert.True(report.RootElement.GetProperty("partial").GetBoolean()); + Assert.Contains(report.RootElement.GetProperty("providerErrors").EnumerateArray().Select(e => e.GetString()), + e => e!.Contains("deadhost")); + Assert.Contains("auditing: 8 probe(s)", progress); + Assert.Contains("FAIL [k8s] deadhost", progress); + } + [Fact] public async Task ConfiguredFolderSink_ReceivesTheExactStdoutReport() { diff --git a/tests/RestoreGuard.Tests/DoctorTests.cs b/tests/RestoreGuard.Tests/DoctorTests.cs index c7e69d2..830077b 100644 --- a/tests/RestoreGuard.Tests/DoctorTests.cs +++ b/tests/RestoreGuard.Tests/DoctorTests.cs @@ -1,9 +1,12 @@ using RestoreGuard.Cli; +using RestoreGuard.Providers; using RestoreGuard.Providers.Docker; using RestoreGuard.Providers.Pve; +using RestoreGuard.Providers.Kubernetes; namespace RestoreGuard.Tests; +[Collection("reports-env")] public class DoctorTests { private static RestoreGuardConfig FullConfig() => new( @@ -114,4 +117,81 @@ public void EmptyConfigYieldsNoProbes() var config = new RestoreGuardConfig([], null, null, 0, null, null, null, null, null, null); Assert.Empty(Doctor.BuildProbes(config)); } + + [Fact] + public void KubernetesDoctor_ProbesEveryEnabledQueryAndRequiresYes() + { + var config = new RestoreGuardConfig([], null, null, 26, null, null, null, null, null, null, + KubernetesClusters: [new KubernetesClusterConfig("cluster-a", "pve99")]); + var probes = Doctor.BuildProbes(config).Where(p => p.Area == "k8s").ToList(); + + Assert.Equal(9, probes.Count); + Assert.Contains(probes, p => p.Command == "kubectl get --raw=/readyz > /dev/null"); + Assert.All(probes.Where(p => p.Command.Contains("auth can-i") && !p.Command.Contains("volumesnapshots.snapshot.storage.k8s.io")), p => Assert.Equal("yes", p.ExpectedStdOut)); + Assert.Contains(probes, p => p.Command.Contains("list backups.velero.io -n 'velero'")); + Assert.Contains(probes, p => p.Command.Contains("list schedules.velero.io -n 'velero'")); + Assert.Contains(probes, p => p.Command.Contains("list podvolumebackups.velero.io -n 'velero'")); + Assert.Contains(probes, p => p.Command.Contains("list pods --all-namespaces")); + Assert.Contains(probes, p => p.Command.Contains("api-resources --api-group=snapshot.storage.k8s.io") + && p.Command.Contains("list volumesnapshots.snapshot.storage.k8s.io --all-namespaces") + && p.ExpectedStdOut is null); + Assert.Contains(probes, p => p.Command.Contains("list deployments.apps --all-namespaces")); + Assert.Contains(probes, p => p.Command.Contains("list persistentvolumeclaims --all-namespaces")); + } + + [Fact] + public void KubernetesDoctor_DisabledVeleroAndWorkloadsHasNoAssociatedQueries() + { + var config = new RestoreGuardConfig([], null, null, 26, null, null, null, null, null, null, + KubernetesClusters: [new KubernetesClusterConfig("cluster-a", "pve99", VeleroNamespace: null, CheckWorkloads: false)]); + var probes = Doctor.BuildProbes(config).Where(p => p.Area == "k8s").ToList(); + + Assert.Single(probes); + Assert.DoesNotContain(probes, p => p.Command.Contains("auth can-i")); + } + + [Fact] + public void KubernetesDoctor_VeleroOnlyStillPreflightsPvcCoverage() + { + var config = new RestoreGuardConfig([], null, null, 26, null, null, null, null, null, null, + KubernetesClusters: [new KubernetesClusterConfig("cluster-a", "pve99", CheckWorkloads: false)]); + var probes = Doctor.BuildProbes(config).Where(probe => probe.Area == "k8s").ToList(); + + Assert.Equal(7, probes.Count); + Assert.Contains(probes, probe => probe.Command.Contains("list persistentvolumeclaims --all-namespaces")); + Assert.DoesNotContain(probes, probe => probe.Command.Contains("list nodes")); + Assert.DoesNotContain(probes, probe => probe.Command.Contains("list deployments.apps")); + } + + [Fact] + public async Task KubernetesDoctor_RejectsCanIOutputOtherThanYes() + { + var reportDir = Directory.CreateTempSubdirectory("rg-doctor-test"); + var config = new RestoreGuardConfig([], null, null, 26, null, null, null, null, null, null, + Reporting: new ReportingConfig(new FolderSinkConfig(reportDir.FullName)), + KubernetesClusters: [new KubernetesClusterConfig("cluster-a", "pve99")]); + var output = new StringWriter(); + var original = Console.Out; + try + { + Console.SetOut(output); + var exit = await Doctor.RunAsync(config, new DoctorSsh(), reportDir.FullName); + Assert.Equal(2, exit); + } + finally + { + Console.SetOut(original); + reportDir.Delete(recursive: true); + } + + Assert.Contains("expected stdout 'yes', got 'no'", output.ToString()); + } + + private sealed class DoctorSsh : ISshProvider + { + public Task RunAsync(string alias, string command, CancellationToken ct = default) => + Task.FromResult(command.Contains("auth can-i", StringComparison.Ordinal) + ? new SshResult(0, "no\n", "") + : new SshResult(0, "", "")); + } } diff --git a/tests/RestoreGuard.Tests/Fixtures/k8s-deployments.json b/tests/RestoreGuard.Tests/Fixtures/k8s-deployments.json new file mode 100644 index 0000000..269568f --- /dev/null +++ b/tests/RestoreGuard.Tests/Fixtures/k8s-deployments.json @@ -0,0 +1 @@ +{"apiVersion":"apps/v1","items":[{"metadata":{"namespace":"apps","name":"api"},"spec":{"replicas":2},"status":{"availableReplicas":2}},{"metadata":{"namespace":"apps","name":"worker"},"spec":{"replicas":1},"status":{"availableReplicas":0}}]} diff --git a/tests/RestoreGuard.Tests/Fixtures/k8s-no-velero.stderr b/tests/RestoreGuard.Tests/Fixtures/k8s-no-velero.stderr new file mode 100644 index 0000000..1822793 --- /dev/null +++ b/tests/RestoreGuard.Tests/Fixtures/k8s-no-velero.stderr @@ -0,0 +1 @@ +error: the server doesn't have a resource type "backups" diff --git a/tests/RestoreGuard.Tests/Fixtures/k8s-nodes.json b/tests/RestoreGuard.Tests/Fixtures/k8s-nodes.json new file mode 100644 index 0000000..058dbee --- /dev/null +++ b/tests/RestoreGuard.Tests/Fixtures/k8s-nodes.json @@ -0,0 +1 @@ +{"apiVersion":"v1","items":[{"metadata":{"name":"rg-node-a","uid":"node-a"},"status":{"conditions":[{"type":"Ready","status":"True"}]}},{"metadata":{"name":"rg-node-b","uid":"node-b"},"status":{"conditions":[{"type":"Ready","status":"False"}]}}]} diff --git a/tests/RestoreGuard.Tests/Fixtures/k8s-pods.json b/tests/RestoreGuard.Tests/Fixtures/k8s-pods.json new file mode 100644 index 0000000..14f57d1 --- /dev/null +++ b/tests/RestoreGuard.Tests/Fixtures/k8s-pods.json @@ -0,0 +1,13 @@ +{ + "items": [ + { + "metadata": { "namespace": "apps", "name": "web-0" }, + "spec": { + "volumes": [ + { "name": "data", "persistentVolumeClaim": { "claimName": "data" } }, + { "name": "config", "configMap": { "name": "web-config" } } + ] + } + } + ] +} diff --git a/tests/RestoreGuard.Tests/Fixtures/k8s-podvolumebackups.json b/tests/RestoreGuard.Tests/Fixtures/k8s-podvolumebackups.json new file mode 100644 index 0000000..94d10f7 --- /dev/null +++ b/tests/RestoreGuard.Tests/Fixtures/k8s-podvolumebackups.json @@ -0,0 +1,28 @@ +{ + "items": [ + { + "metadata": { + "name": "pvb-web-data", + "namespace": "velero", + "labels": { "velero.io/backup-name": "nightly-20260825" } + }, + "spec": { + "pod": { "namespace": "apps", "name": "web-0" }, + "volume": "data" + }, + "status": { "phase": "Completed" } + }, + { + "metadata": { + "name": "pvb-old-data", + "namespace": "velero", + "labels": { "velero.io/backup-name": "failed-20260824" } + }, + "spec": { + "pod": { "namespace": "apps", "name": "web-0" }, + "volume": "data" + }, + "status": { "phase": "Failed" } + } + ] +} diff --git a/tests/RestoreGuard.Tests/Fixtures/k8s-pvcs.json b/tests/RestoreGuard.Tests/Fixtures/k8s-pvcs.json new file mode 100644 index 0000000..6641ae6 --- /dev/null +++ b/tests/RestoreGuard.Tests/Fixtures/k8s-pvcs.json @@ -0,0 +1 @@ +{"apiVersion":"v1","items":[{"metadata":{"namespace":"apps","name":"data"},"spec":{"volumeName":"pvc-1"},"status":{"phase":"Bound"}},{"metadata":{"namespace":"apps","name":"broken"},"spec":{},"status":{"phase":"Pending"}}]} diff --git a/tests/RestoreGuard.Tests/Fixtures/k8s-schedules.json b/tests/RestoreGuard.Tests/Fixtures/k8s-schedules.json new file mode 100644 index 0000000..cac6cc8 --- /dev/null +++ b/tests/RestoreGuard.Tests/Fixtures/k8s-schedules.json @@ -0,0 +1 @@ +{"apiVersion":"velero.io/v1","items":[{"metadata":{"namespace":"velero","name":"nightly"},"status":{"lastBackup":"2026-08-25T00:10:00Z"}}]} diff --git a/tests/RestoreGuard.Tests/Fixtures/k8s-velero.json b/tests/RestoreGuard.Tests/Fixtures/k8s-velero.json new file mode 100644 index 0000000..e1e54f8 --- /dev/null +++ b/tests/RestoreGuard.Tests/Fixtures/k8s-velero.json @@ -0,0 +1 @@ +{"apiVersion":"velero.io/v1","items":[{"metadata":{"namespace":"velero","name":"nightly-20260825","creationTimestamp":"2026-08-25T00:00:00Z"},"status":{"phase":"Completed","completionTimestamp":"2026-08-25T00:10:00Z","volumeSnapshotsAttempted":2,"volumeSnapshotsCompleted":2}},{"metadata":{"namespace":"velero","name":"failed-20260824","creationTimestamp":"2026-08-24T00:00:00Z"},"status":{"phase":"Failed","volumeSnapshotsAttempted":1,"volumeSnapshotsCompleted":0}}]} diff --git a/tests/RestoreGuard.Tests/Fixtures/k8s-volumesnapshots.json b/tests/RestoreGuard.Tests/Fixtures/k8s-volumesnapshots.json new file mode 100644 index 0000000..c0f57ad --- /dev/null +++ b/tests/RestoreGuard.Tests/Fixtures/k8s-volumesnapshots.json @@ -0,0 +1,13 @@ +{ + "items": [ + { + "metadata": { + "namespace": "apps", + "name": "data-snapshot", + "labels": { "velero.io/backup-name": "nightly-20260825" } + }, + "spec": { "source": { "persistentVolumeClaimName": "data" } }, + "status": { "readyToUse": true } + } + ] +} diff --git a/tests/RestoreGuard.Tests/KubernetesCheckTests.cs b/tests/RestoreGuard.Tests/KubernetesCheckTests.cs new file mode 100644 index 0000000..0342c78 --- /dev/null +++ b/tests/RestoreGuard.Tests/KubernetesCheckTests.cs @@ -0,0 +1,177 @@ +using RestoreGuard.Checks; +using RestoreGuard.Core; +using RestoreGuard.Core.Model; +using RestoreGuard.Providers.Kubernetes; + +namespace RestoreGuard.Tests; + +public class KubernetesCheckTests +{ + private static readonly DateTimeOffset Now = new(2026, 8, 25, 12, 0, 0, TimeSpan.Zero); + + private static LabInventory Inventory(IEnumerable? backups = null, IEnumerable? services = null) => + new(Now, services == null ? [] : services.ToList(), backups == null ? [] : backups.ToList(), []); + + private static KubernetesState State(bool installed = true, + IReadOnlyList? pvcs = null, + IReadOnlyList? schedules = null, + IReadOnlyList? podVolumeBackups = null, + IReadOnlyList? volumeSnapshots = null) => + new("cluster-a", pvcs ?? [], schedules ?? [], installed, podVolumeBackups ?? [], volumeSnapshots ?? [], Now); + + private static BackupArtifact Backup(string name, string phase, DateTimeOffset timestamp) => + new(BackupTier.KubernetesBackup, "cluster-a velero", name, timestamp, 0, "velero", false, phase); + + private static List Findings(KubernetesState state, LabInventory inventory, bool velero = true, bool workloads = true) => + new KubernetesCheck([state], [new KubernetesExpectation("cluster-a", "pve99", TimeSpan.FromHours(26), workloads, velero)]) + .Evaluate(inventory).ToList(); + + [Fact] + public void MissingProviderStateIsUnreachableAndDoesNotInventFindings() + { + var findings = new KubernetesCheck([], [new KubernetesExpectation("cluster-a", "pve99", TimeSpan.FromHours(26))]) + .Evaluate(Inventory()).ToList(); + Assert.Contains(findings, finding => finding.RuleId == "k8s/unreachable" && finding.Service == "cluster-a cluster" && finding.Host == "cluster-a"); + } + + [Fact] + public void MissingVeleroAndZeroBackupsAreDistinguished() + { + Assert.Contains(Findings(State(installed: false), Inventory()), f => f.RuleId == "k8s-backup/velero-missing"); + Assert.Contains(Findings(State(), Inventory(), velero: true), f => f.RuleId == "k8s-backup/no-backups"); + } + + [Fact] + public void InProgressOnlyDoesNotPretendToBeCompletedOrStale() + { + var inventory = Inventory([Backup("running", "InProgress", Now.AddHours(-1))]); + var findings = Findings(State(schedules: [new KubernetesSchedule("velero", "nightly")]), inventory); + Assert.DoesNotContain(findings, f => f.RuleId == "k8s-backup/stale"); + Assert.DoesNotContain(findings, f => f.RuleId == "k8s-backup/failed"); + Assert.Contains(findings, f => f.RuleId == "k8s-backup/no-completed"); + } + + [Fact] + public void CompletedStaleAndFailedPhasesHaveStableIdentities() + { + var inventory = Inventory([ + Backup("stale", "Completed", Now.AddHours(-30)), + Backup("failed", "PartiallyFailed", Now.AddHours(-2))]); + var findings = Findings(State(schedules: [new KubernetesSchedule("velero", "nightly")]), inventory); + + Assert.Contains(findings, f => (f.RuleId, f.Service, f.Host) == ("k8s-backup/stale", "cluster-a cluster", "cluster-a")); + Assert.Contains(findings, f => (f.RuleId, f.Service, f.Host) == ("k8s-backup/failed", "cluster-a cluster", "cluster-a")); + } + + [Fact] + public void NewerCompletedBackupClearsAnOlderFailure() + { + var inventory = Inventory([ + Backup("failed", "Failed", Now.AddHours(-2)), + Backup("recovered", "Completed", Now.AddHours(-1))]); + var findings = Findings(State(schedules: [new KubernetesSchedule("velero", "nightly")]), inventory); + + Assert.DoesNotContain(findings, finding => finding.RuleId == "k8s-backup/failed"); + } + + [Fact] + public void NewerCompletedBackupClearsAnOlderPodVolumeBackupFailure() + { + var inventory = Inventory([ + Backup("snapshot-failed", "Completed", Now.AddHours(-2)), + Backup("recovered", "Completed", Now.AddHours(-1))]); + var state = State( + pvcs: [new KubernetesPersistentVolumeClaim("apps", "data", "Bound")], + schedules: [new KubernetesSchedule("velero", "nightly")], + podVolumeBackups: + [ + new KubernetesPodVolumeBackup("snapshot-failed", "apps", "web-0", "data", "Failed", "data"), + new KubernetesPodVolumeBackup("recovered", "apps", "web-0", "data", "Completed", "data"), + ]); + + Assert.DoesNotContain(Findings(state, inventory), finding => finding.RuleId == "k8s-pvc/unprotected"); + } + + [Fact] + public void MissingScheduleIsYellowAndMissingPodVolumeBackupIsRed() + { + var inventory = Inventory([Backup("ok", "Completed", Now.AddHours(-1))]); + var findings = Findings(State( + pvcs: [new KubernetesPersistentVolumeClaim("apps", "data", "Bound")], + podVolumeBackups: []), inventory); + Assert.Contains(findings, f => f.RuleId == "k8s-backup/no-schedule" && f.Severity == Severity.Yellow); + Assert.Contains(findings, f => f.RuleId == "k8s-pvc/unprotected" && f.Service == "apps/data"); + } + + [Fact] + public void CompletedPodVolumeBackupProtectsBoundPvcWhenSnapshotCountersAreAbsent() + { + var inventory = Inventory([Backup("ok", "Completed", Now.AddHours(-1))]); + var state = State( + pvcs: [new KubernetesPersistentVolumeClaim("apps", "data", "Bound")], + schedules: [new KubernetesSchedule("velero", "nightly")], + podVolumeBackups: [new KubernetesPodVolumeBackup("ok", "apps", "web-0", "data", "Completed", "data")]); + + Assert.DoesNotContain(Findings(state, inventory), finding => finding.RuleId == "k8s-pvc/unprotected"); + } + + [Fact] + public void ReadyCsiSnapshotProtectsBoundPvcWhenPodVolumeBackupIsAbsent() + { + var inventory = Inventory([Backup("ok", "Completed", Now.AddHours(-1))]); + var state = State( + pvcs: [new KubernetesPersistentVolumeClaim("apps", "data", "Bound")], + schedules: [new KubernetesSchedule("velero", "nightly")], + volumeSnapshots: [new KubernetesVolumeSnapshot("ok", "apps", "data", true)]); + + Assert.DoesNotContain(Findings(state, inventory), finding => finding.RuleId == "k8s-pvc/unprotected"); + } + + [Fact] + public void EachBoundPvcNeedsItsOwnCompletedPodVolumeBackup() + { + var inventory = Inventory([Backup("ok", "Completed", Now.AddHours(-1))]); + var state = State( + pvcs: + [ + new KubernetesPersistentVolumeClaim("apps", "data", "Bound"), + new KubernetesPersistentVolumeClaim("apps", "uploads", "Bound"), + ], + podVolumeBackups: [new KubernetesPodVolumeBackup("ok", "apps", "web-0", "data", "Completed", "data")]); + + var unprotected = Findings(state, inventory).Where(finding => finding.RuleId == "k8s-pvc/unprotected").ToList(); + Assert.Single(unprotected); + Assert.Equal("apps/uploads", unprotected[0].Service); + } + + [Fact] + public void WorkloadChecksCoverPvcDeploymentAndNodeButCanBeDisabled() + { + var state = State(pvcs: [new("apps", "data", "Pending")]); + var services = new[] + { + new Service("apps/web", "cluster-a", ServiceKind.K8sWorkload, "degraded", null, [], null), + new Service("node-1", "cluster-a", ServiceKind.K8sNode, "NotReady", null, [], null), + }; + var findings = Findings(state, Inventory(services: services)); + Assert.Contains(findings, f => f.RuleId == "k8s-pvc/unbound" && f.Service == "apps/data"); + Assert.Contains(findings, f => f.RuleId == "k8s-workload/unavailable" && f.Service == "apps/web"); + Assert.Contains(findings, f => f.RuleId == "k8s-node/not-ready" && f.Service == "node-1"); + Assert.DoesNotContain(Findings(state, Inventory(services: services), velero: false, workloads: false), + finding => finding.RuleId.StartsWith("k8s-", StringComparison.Ordinal)); + } + + [Fact] + public void BackupFindingIdentityCanBeSuppressedWithoutADeadTarget() + { + var inventory = Inventory(services: [new Service("cluster-a cluster", "cluster-a", ServiceKind.K8sCluster, "unknown", null, [], null)]); + var check = new KubernetesCheck([State(installed: false)], + [new KubernetesExpectation("cluster-a", "pve99", TimeSpan.FromHours(26))]); + var suppression = new Suppression("cluster-a", "cluster-a cluster", "k8s-backup/velero-missing", "cluster is intentionally disposable", new(2026, 8, 25)); + var report = new CheckEngine([check]).Run(inventory, [suppression], Now); + + Assert.DoesNotContain(report.Findings, finding => finding.RuleId == "k8s-backup/velero-missing"); + Assert.Contains(report.SuppressedFindings, f => f.RuleId == "k8s-backup/velero-missing" && f.Service == "cluster-a cluster" && f.Host == "cluster-a"); + Assert.DoesNotContain(report.Findings, f => f.RuleId == "suppression/unknown-target"); + } +} diff --git a/tests/RestoreGuard.Tests/KubernetesConfigTests.cs b/tests/RestoreGuard.Tests/KubernetesConfigTests.cs new file mode 100644 index 0000000..c33fd94 --- /dev/null +++ b/tests/RestoreGuard.Tests/KubernetesConfigTests.cs @@ -0,0 +1,49 @@ +using RestoreGuard.Cli; +using RestoreGuard.Providers.Kubernetes; + +namespace RestoreGuard.Tests; + +public class KubernetesConfigTests +{ + private static RestoreGuardConfig Config(params KubernetesClusterConfig[] clusters) => + new([], null, null, 26, null, null, null, null, null, null, + KubernetesClusters: clusters); + + [Fact] + public void ValidClusterDefaultsVeleroAndHasNoValidationErrors() + { + var errors = Config(new KubernetesClusterConfig("cluster-a", "pve99", Kubectl: "kubectl")).Validate(); + Assert.Empty(errors); + } + + [Fact] + public void BlankAliasPrefixNameAndNonPositiveAgeAreActionable() + { + var errors = Config(new KubernetesClusterConfig("", " ", Kubectl: "", MaxBackupAgeHours: 0)).Validate(); + Assert.Contains(errors, e => e.Contains("kubernetesClusters[0].name is empty")); + Assert.Contains(errors, e => e.Contains("kubernetesClusters[0].alias is empty")); + Assert.Contains(errors, e => e.Contains("kubernetesClusters[0].kubectl is empty")); + Assert.Contains(errors, e => e.Contains("maxBackupAgeHours must be positive")); + } + + [Fact] + public void ClusterNamesMustBeUnique() + { + var errors = Config(new KubernetesClusterConfig("same", "one"), new KubernetesClusterConfig("same", "two")).Validate(); + Assert.Contains(errors, e => e.Contains("kubernetesClusters[1].name 'same' is duplicated")); + } + + [Fact] + public void VeleroNamespaceMustBeSafeForTheRemoteCommand() + { + var errors = Config(new KubernetesClusterConfig("cluster-a", "pve99", VeleroNamespace: "bad namespace")).Validate(); + Assert.Contains(errors, error => error.Contains("veleroNamespace 'bad namespace' is not a valid Kubernetes namespace")); + } + + [Fact] + public void ExplicitNullCannotDisableEveryKubernetesSurface() + { + var config = Config(new KubernetesClusterConfig("cluster-a", "pve99", VeleroNamespace: null, CheckWorkloads: false)); + Assert.Contains(config.Validate(), error => error.Contains("disables both Velero and workload checks")); + } +} diff --git a/tests/RestoreGuard.Tests/KubernetesProviderTests.cs b/tests/RestoreGuard.Tests/KubernetesProviderTests.cs new file mode 100644 index 0000000..5d3e231 --- /dev/null +++ b/tests/RestoreGuard.Tests/KubernetesProviderTests.cs @@ -0,0 +1,145 @@ +using RestoreGuard.Core.Model; +using RestoreGuard.Providers; +using RestoreGuard.Providers.Kubernetes; + +namespace RestoreGuard.Tests; + +public class KubernetesProviderTests +{ + [Fact] + public void ParsesClusterSurfacesAndKeepsVeleroSizeUnknown() + { + var nodes = KubectlParser.ParseNodes(Fixtures.Read("k8s-nodes.json")); + Assert.Equal(["Ready", "NotReady"], nodes.Select(n => n.State)); + Assert.Equal(2, KubectlParser.ParseDeployments(Fixtures.Read("k8s-deployments.json")).Count); + Assert.Equal("Pending", KubectlParser.ParsePersistentVolumeClaims(Fixtures.Read("k8s-pvcs.json"))[1].Phase); + var backup = Assert.Single(KubectlParser.ParseBackups(Fixtures.Read("k8s-velero.json"), "cluster-a"), b => b.Status == "Completed"); + Assert.Equal(BackupTier.KubernetesBackup, backup.Tier); + Assert.Equal(0, backup.SizeBytes); + var podVolumeBackups = KubectlParser.ParsePodVolumeBackups(Fixtures.Read("k8s-podvolumebackups.json"), "velero"); + Assert.Equal(2, podVolumeBackups.Count); + Assert.Contains(podVolumeBackups, backup => backup.BackupName == "nightly-20260825" && backup.Completed); + Assert.Contains(podVolumeBackups, backup => backup.BackupName == "failed-20260824" && !backup.Completed); + Assert.Contains(KubectlParser.ParsePodVolumeClaims(Fixtures.Read("k8s-pods.json")), + reference => reference.Namespace == "apps" && reference.PodName == "web-0" && reference.Volume == "data" && reference.ClaimName == "data"); + Assert.Contains(KubectlParser.ParseVolumeSnapshots(Fixtures.Read("k8s-volumesnapshots.json")), + snapshot => snapshot.BackupName == "nightly-20260825" && snapshot.Namespace == "apps" && snapshot.ClaimName == "data" && snapshot.ReadyToUse); + } + + [Fact] + public void DeploymentWithoutSpecReplicasDefaultsToOne() + { + var deployments = KubectlParser.ParseDeployments(""" + { "items": [{ "metadata": { "namespace": "apps", "name": "web" }, "spec": {}, "status": {} }] } + """); + + Assert.Equal(1, Assert.Single(deployments).DesiredReplicas); + } + + [Fact] + public async Task NullVeleroAndDisabledWorkloadsMakeNoSurfaceCalls() + { + var ssh = new RecordingSsh(); + var result = await new KubernetesProvider(ssh).GetClusterAsync(new("cluster-b", "target", VeleroNamespace: null, CheckWorkloads: false)); + Assert.Empty(result.Services); + Assert.Empty(ssh.Commands); + } + + [Fact] + public async Task MissingVeleroResourceIsCoverageStateButForbiddenAndMalformedResponsesAreProviderErrors() + { + var missing = new ScriptedSsh(command => command.Contains("get pvc -A") + ? new SshResult(0, "{\"items\":[]}", "") + : new SshResult(1, "", Fixtures.Read("k8s-no-velero.stderr"))); + var missingResult = await new KubernetesProvider(missing).GetAsync(new("cluster-a", "target", CheckWorkloads: false)); + Assert.False(missingResult.State.VeleroInstalled); + Assert.Equal(2, missing.Commands.Count); + + var forbidden = new ScriptedSsh(command => command.Contains("get pvc -A") + ? new SshResult(0, "{\"items\":[]}", "") + : new SshResult(1, "", "Error from server (Forbidden): backups.velero.io is forbidden")); + var forbiddenError = await Assert.ThrowsAsync(() => + new KubernetesProvider(forbidden).GetAsync(new("cluster-a", "target", CheckWorkloads: false))); + Assert.Equal(KubernetesProviderErrorKind.Forbidden, forbiddenError.Kind); + + var malformed = new ScriptedSsh(command => command.Contains("get pvc -A") + ? new SshResult(0, "{\"items\":[]}", "") + : new SshResult(0, "not-json", "")); + var malformedError = await Assert.ThrowsAsync(() => + new KubernetesProvider(malformed).GetAsync(new("cluster-a", "target", CheckWorkloads: false))); + Assert.Equal(KubernetesProviderErrorKind.Malformed, malformedError.Kind); + + var wrongShape = new ScriptedSsh(command => command.Contains("get pvc -A") + ? new SshResult(0, "{\"items\":[]}", "") + : new SshResult(0, "{\"items\":{}}", "")); + var wrongShapeError = await Assert.ThrowsAsync(() => + new KubernetesProvider(wrongShape).GetAsync(new("cluster-a", "target", CheckWorkloads: false))); + Assert.Equal(KubernetesProviderErrorKind.Malformed, wrongShapeError.Kind); + } + + [Fact] + public async Task VeleroQueriesIncludeSchedulesAndPodVolumeBackups() + { + var ssh = new ScriptedSsh(command => command switch + { + var value when value.Contains("get pvc -A") => new SshResult(0, Fixtures.Read("k8s-pvcs.json"), ""), + var value when value.Contains("podvolumebackups.velero.io") => new SshResult(0, Fixtures.Read("k8s-podvolumebackups.json"), ""), + var value when value.Contains("get pods -A") => new SshResult(0, Fixtures.Read("k8s-pods.json"), ""), + var value when value.Contains("volumesnapshots.snapshot.storage.k8s.io") => new SshResult(0, Fixtures.Read("k8s-volumesnapshots.json"), ""), + var value when value.Contains("backups.velero.io") => new SshResult(0, Fixtures.Read("k8s-velero.json"), ""), + var value when value.Contains("schedules.velero.io") => new SshResult(0, Fixtures.Read("k8s-schedules.json"), ""), + _ => throw new InvalidOperationException($"Unexpected command: {command}"), + }); + + var ticker = new List(); + var result = await new KubernetesProvider(ssh, ticker.Add).GetAsync(new("cluster-a", "target", CheckWorkloads: false)); + + Assert.True(result.State.VeleroInstalled); + Assert.Equal(2, result.State.PodVolumeBackups.Count); + Assert.Single(result.State.VolumeSnapshots); + Assert.Equal("data", result.State.PodVolumeBackups.Single(backup => backup.BackupName == "nightly-20260825").PersistentVolumeClaim); + Assert.Contains(ssh.Commands, command => command.Contains("schedules.velero.io")); + Assert.Contains(ssh.Commands, command => command.Contains("podvolumebackups.velero.io")); + Assert.Contains(ssh.Commands, command => command.Contains("get pods -A")); + Assert.Contains(ssh.Commands, command => command.Contains("volumesnapshots.snapshot.storage.k8s.io")); + Assert.Equal(["persistent-volume-claims", "velero-backups", "velero-schedules", "pod-volume-backups", "pods", "csi-volume-snapshots"], + ticker.Select(message => message[(message.LastIndexOf(": ", StringComparison.Ordinal) + 2)..]).ToList()); + } + + [Fact] + public async Task MissingOptionalCsiSnapshotResourceDoesNotHideCompletedPodVolumeBackup() + { + var ssh = new ScriptedSsh(command => command switch + { + var value when value.Contains("get pvc -A") => new SshResult(0, Fixtures.Read("k8s-pvcs.json"), ""), + var value when value.Contains("podvolumebackups.velero.io") => new SshResult(0, Fixtures.Read("k8s-podvolumebackups.json"), ""), + var value when value.Contains("backups.velero.io") => new SshResult(0, Fixtures.Read("k8s-velero.json"), ""), + var value when value.Contains("schedules.velero.io") => new SshResult(0, Fixtures.Read("k8s-schedules.json"), ""), + var value when value.Contains("get pods -A") => new SshResult(0, Fixtures.Read("k8s-pods.json"), ""), + var value when value.Contains("volumesnapshots.snapshot.storage.k8s.io") => new SshResult(1, "", "the server doesn't have a resource type \"volumesnapshots\""), + _ => throw new InvalidOperationException($"Unexpected command: {command}"), + }); + + var result = await new KubernetesProvider(ssh).GetAsync(new("cluster-a", "target", CheckWorkloads: false)); + + Assert.Empty(result.State.VolumeSnapshots); + Assert.Contains(result.State.PodVolumeBackups, backup => backup.BackupName == "nightly-20260825" && backup.PersistentVolumeClaim == "data"); + } + + private sealed class RecordingSsh : ISshProvider + { + public List Commands { get; } = []; + public Task RunAsync(string alias, string command, CancellationToken ct = default) + { Commands.Add(command); return Task.FromResult(new SshResult(0, "{\"items\":[]}", "")); } + } + + private sealed class ScriptedSsh(Func respond) : ISshProvider + { + public List Commands { get; } = []; + public Task RunAsync(string alias, string command, CancellationToken ct = default) + { + Commands.Add(command); + return Task.FromResult(respond(command)); + } + } +}