diff --git a/docs/features.md b/docs/features.md index 064bd30..3d0fb28 100644 --- a/docs/features.md +++ b/docs/features.md @@ -662,6 +662,19 @@ right - so a download or an upload is one keystroke rather than a hand-written through right now, so the confirmation says so. Deleting pods is a mutation like any other: blocked in read-only mode, matched by the `pvc-explore` guardrail, recorded in `:journal`. +- **Missing tools trigger built-in recovery.** If the first listing fails + because `sh`, `ls`, or `head` is missing, sofka tries up to 16 other running + containers that mount the same part of the claim. Read-only restrictions + remain in force. If none works, it offers a helper pod and asks before + creating it. Read-only mode and the `pvc-explore` guardrail still apply. + For an in-use `ReadWriteOnce` claim, the helper is scheduled on the consumer's + node. An occupied `ReadWriteOncePod` claim cannot use a second pod. Recovery + reports that restriction without creating a helper. A static `subPath` is + preserved; a `subPathExpr` mount cannot recover automatically because its + boundary cannot be determined from the pod specification. Permission errors, + connection failures, and invalid paths retain their specific messages. + Canceling recovery leaves the original error in the browser. Reopen the claim + to start a new recovery attempt. - **Navigation is confined to the mount.** `⌫` stops at the mount point, and every listing verifies with `pwd -P` that it actually landed inside the volume - so a symlink on the volume pointing at `/` is refused rather than @@ -729,7 +742,7 @@ right - so a download or an upload is one keystroke rather than a hand-written than after. Listings are read with `ls -A -l` over `kubectl exec`, so the pod's image needs -a shell and `ls`; transfers additionally need `tar`, as `kubectl cp` always +a shell, `ls`, and `head`; transfers additionally need `tar`, as `kubectl cp` always does. An entry `ls` cannot stat still appears, with an unknown size and a warning, rather than blanking the whole directory. The helper-pod image and lifetime are configurable: diff --git a/docs/keys.md b/docs/keys.md index f1aff56..99d901d 100644 --- a/docs/keys.md +++ b/docs/keys.md @@ -297,6 +297,11 @@ Interactive actions (`e`, `s` for shell, `a`) suspend the TUI and shell out to `kubectl`. Delete, scale, restart, set-image, suspend, resume, reconcile, and port-forward go through the kube API (or a backgrounded process) directly. +If the first PVC listing fails because required tools are missing, sofka tries +other suitable containers, then offers a helper pod. Accept or cancel the +existing confirmation dialog. Volume access restrictions, read-only mode, and +guardrails still apply. Canceling keeps the original error visible. + ## Plugin commands | Command | Action | diff --git a/src/app/lifecycle.rs b/src/app/lifecycle.rs index 1d3a8eb..a9f6385 100644 --- a/src/app/lifecycle.rs +++ b/src/app/lifecycle.rs @@ -1697,6 +1697,17 @@ impl App { self.discard_pvc_target(namespace, context, result); } } + Msg::PvcRecovery { + generation, + run, + result, + } => { + if generation == self.generation { + self.handle_pvc_recovery(run, result); + } else if run == self.pvc.run { + self.pvc.loading = false; + } + } Msg::PvcListing { generation, run, diff --git a/src/app/pvcexplore.rs b/src/app/pvcexplore.rs index 2dc11e9..6bbd432 100644 --- a/src/app/pvcexplore.rs +++ b/src/app/pvcexplore.rs @@ -85,6 +85,15 @@ pub struct PvcExplore { /// Bumped on every navigation so a slow listing for a directory the user /// has already left is discarded instead of replacing the current one. pub run: u64, + pub(super) recovery: Option, + listed: bool, +} + +pub(super) struct RecoveryState { + pub error: String, + pub original: Mount, + candidates: std::collections::VecDeque, + helper: Option>, } impl Default for PvcExplore { @@ -111,6 +120,8 @@ impl Default for PvcExplore { focus: Pane::Remote, shell_pending: false, run: 0, + recovery: None, + listed: false, } } } @@ -285,7 +296,12 @@ impl App { return; } match result { - Err(e) => self.set_claimed_status(status, e, true), + Err(e) => { + self.set_claimed_status(status, e.clone(), true); + if self.pvc.recovery.is_some() { + self.pvc_recovery_error(&e); + } + } Ok(Some(mount)) => { self.clear_claimed_status(status); self.enter_pvc_target(namespace, mount); @@ -337,7 +353,9 @@ impl App { } } PvcIntent::Browse => { - self.set_return_mode(); + if !self.pvc.active { + self.set_return_mode(); + } self.pvc.active = true; self.pvc.focus = Pane::Remote; // Seeded, not left empty: a first listing that fails restores @@ -345,7 +363,7 @@ impl App { // make `⌫` and `r` both misbehave. self.pvc.displayed_path = path.clone(); self.pvc.remote.clear(); - self.pvc.remote_error = None; + self.pvc.remote_error = self.pvc.recovery.as_ref().map(|r| r.error.clone()); self.pvc.truncated = false; self.mode = Mode::PvcExplore; self.reload_local(); @@ -361,7 +379,7 @@ impl App { fn offer_pvc_helper(&mut self) { if self.readonly { self.flash_warn(&format!( - "nothing mounts {} — browsing it needs a helper pod, which read-only mode blocks", + "browsing {} needs a helper pod, which read-only mode blocks", self.pvc.claim )); return; @@ -390,8 +408,14 @@ impl App { return; }; let image = self.pvc_cfg.image.clone(); - let label = - format!("Nothing mounts {claim}. Create a temporary {image} pod in {ns} to mount it?"); + let label = if let Some(recovery) = &self.pvc.recovery { + format!( + "{}\n\nBrowse with helper pod? Create a temporary {image} pod in {ns} for {claim}.", + recovery.error + ) + } else { + format!("Nothing mounts {claim}. Create a temporary {image} pod in {ns} to mount it?") + }; self.begin_guarded( ConfirmAction::PvcHelper { ns, @@ -407,11 +431,28 @@ impl App { /// Create the helper pod and wait for it to run. `generateName` means two /// sessions browsing the same claim never collide on a name. pub(super) fn create_pvc_helper(&mut self, ns: String, claim: String, intent: PvcIntent) { + if self.deny_readonly() + || self + .guard( + "pvc-explore", + "persistentvolumeclaims", + &[(claim.clone(), ns.clone())], + ConfirmLevel::None, + ) + .is_none() + { + if self.pvc.recovery.is_some() { + let reason = self.flash.clone(); + self.pvc_recovery_error(&reason); + } + return; + } self.pvc.intent = intent; self.pvc.run += 1; let run = self.pvc.run; let ttl = self.pvc_ttl_secs(); - let manifest = pvc::helper_pod(&claim, &self.pvc_cfg.image, ttl); + let mut manifest = pvc::helper_pod(&claim, &self.pvc_cfg.image, ttl); + let original = self.pvc.recovery.as_ref().map(|r| r.original.clone()); self.note_action("pvc-explore helper pod", format!("{claim} in {ns}")); let status = self.claim_status(format!("starting a helper pod for {claim}…")); let context = self.cluster.context.clone(); @@ -419,7 +460,14 @@ impl App { let tx = self.tx.clone(); let genr = self.generation; tokio::spawn(async move { - let result = start_helper(client, &ns, manifest).await; + let result = async { + if let Some(original) = original { + let plan = load_recovery_plan(client.clone(), &ns, &claim, &original).await?; + pvc::apply_helper_options(&mut manifest, &plan.helper?); + } + start_helper(client, &ns, manifest).await + } + .await; let _ = tx .send(Msg::PvcTarget { generation: genr, @@ -433,6 +481,98 @@ impl App { }); } + fn start_pvc_recovery(&mut self, error: String) { + let Some(original) = self.pvc.mount.clone() else { + return; + }; + self.pvc.remote_error = Some(error.clone()); + self.pvc.loading = true; + self.pvc.recovery = Some(RecoveryState { + error, + original: original.clone(), + candidates: Default::default(), + helper: None, + }); + self.pvc.run += 1; + let run = self.pvc.run; + let generation = self.generation; + let client = self.cluster.client.clone(); + let namespace = self.pvc.namespace.clone(); + let claim = self.pvc.claim.clone(); + let tx = self.tx.clone(); + tokio::spawn(async move { + let result = load_recovery_plan(client, &namespace, &claim, &original).await; + let _ = tx + .send(Msg::PvcRecovery { + generation, + run, + result, + }) + .await; + }); + } + + pub(super) fn handle_pvc_recovery( + &mut self, + run: u64, + result: Result, + ) { + if run != self.pvc.run || !self.pvc.active { + return; + } + self.pvc.loading = false; + if self.mode != Mode::PvcExplore { + self.pvc_recovery_error("Recovery canceled because another view or dialog is open."); + return; + } + match result { + Ok(plan) => { + let Some(recovery) = &mut self.pvc.recovery else { + return; + }; + recovery.candidates = plan.candidates.into(); + recovery.helper = Some(plan.helper); + self.next_pvc_candidate(); + } + Err(error) => self.pvc_recovery_error(&error), + } + } + + fn next_pvc_candidate(&mut self) { + let Some(recovery) = &mut self.pvc.recovery else { + return; + }; + if let Some(mount) = recovery.candidates.pop_front() { + self.enter_pvc_target(self.pvc.namespace.clone(), mount); + return; + } + let helper = recovery.helper.take(); + self.pvc.loading = false; + match helper { + Some(Ok(_)) => { + self.offer_pvc_helper(); + if self.mode == Mode::PvcExplore { + let reason = self.flash.clone(); + self.pvc_recovery_error(&reason); + } + } + Some(Err(error)) => self.pvc_recovery_error(&error), + None => self.pvc_recovery_error( + "No further recovery targets are available. Reopen the claim to retry.", + ), + } + } + + fn pvc_recovery_error(&mut self, reason: &str) { + let message = match &self.pvc.recovery { + Some(recovery) => format!("{}\n\n{reason}", recovery.error), + None => reason.to_owned(), + }; + self.pvc.loading = false; + self.pvc.remote_error = Some(message.clone()); + self.flash_warn(&message); + } + /// `[pvc_explore] ttl`, already validated at load; a value that slipped /// through falls back to the default rather than creating a pod that never /// expires. @@ -489,6 +629,8 @@ impl App { self.pvc.remote_path = path.clone(); match result { Ok((listing, warn)) => { + self.pvc.listed = true; + self.pvc.recovery = None; self.pvc.truncated = listing.truncated; // Re-listing the same directory keeps the cursor; stepping // into a new one starts at the top, unless we stepped *out* of @@ -517,6 +659,26 @@ impl App { // directory, so leaving the title pointing somewhere else would // mislabel them. Err(e) => { + if !self.pvc.listed + && self.mode == Mode::PvcExplore + && self + .pvc + .mount + .as_ref() + .is_some_and(|m| !m.helper && path == m.path) + && pvc::missing_listing_tools(&e) + { + if self.pvc.recovery.is_some() { + self.next_pvc_candidate(); + } else { + self.start_pvc_recovery(e); + } + return; + } + if self.pvc.recovery.is_some() { + self.pvc_recovery_error(&e); + return; + } self.pvc.remote_path = self.pvc.displayed_path.clone(); self.pvc.want_select = None; // Only when the failure is about the directory still on @@ -846,6 +1008,8 @@ impl App { self.pvc.remote_path.clear(); self.pvc.displayed_path.clear(); self.pvc.want_select = None; + self.pvc.recovery = None; + self.pvc.listed = false; } /// Delete the helper pod, if this session created one. Best effort and @@ -1203,10 +1367,47 @@ fn pod_resource() -> kube::discovery::ApiResource { kube::discovery::ApiResource::erase::(&()) } +async fn load_recovery_plan( + client: Client, + ns: &str, + claim: &str, + original: &Mount, +) -> Result { + let read = async { + let pods: Api = Api::namespaced_with(client.clone(), ns, &pod_resource()); + let pods = pods + .list(&ListParams::default()) + .await + .map_err(|e| format!("Cannot list recovery pods: {e}"))?; + let resource = kube::discovery::ApiResource::erase::< + k8s_openapi::api::core::v1::PersistentVolumeClaim, + >(&()); + let claims: Api = Api::namespaced_with(client, ns, &resource); + let claim = claims + .get(claim) + .await + .map_err(|e| format!("Cannot check volume access modes: {e}"))?; + Ok(pvc::recovery_plan(&pods.items, &claim, original)) + }; + tokio::time::timeout(LIST_TIMEOUT, read) + .await + .map_err(|_| "Recovery checks timed out.".to_owned())? +} + /// Create the helper pod and poll until it is `Running` (or fails), returning /// the mount to browse it through. async fn start_helper(client: Client, ns: &str, manifest: Value) -> Result { let spec: Pod = serde_json::from_value(manifest).map_err(|e| e.to_string())?; + let volume_mount = spec + .spec + .as_ref() + .and_then(|s| s.containers.first()) + .and_then(|c| c.volume_mounts.as_ref()) + .and_then(|m| m.first()); + let read_only = volume_mount.and_then(|m| m.read_only).unwrap_or(false); + let sub_path = volume_mount + .and_then(|m| m.sub_path.clone()) + .unwrap_or_default(); let pods: Api = Api::namespaced(client, ns); let created = pods .create(&PostParams::default(), &spec) @@ -1223,7 +1424,8 @@ async fn start_helper(client: Client, ns: &str, manifest: Value) -> Result, + helper: Result, +) { + app.handle_msg(Msg::PvcRecovery { + generation: app.generation, + run: app.pvc.run, + result: Ok(crate::pvcexplore::RecoveryPlan { candidates, helper }), + }); +} + +fn recovery_helper() -> crate::pvcexplore::HelperOptions { + crate::pvcexplore::HelperOptions { + node: Some("worker-a".into()), + sub_path: String::new(), + read_only: false, + } +} + +#[tokio::test] +async fn pvc_missing_tools_tries_another_container_before_offering_a_helper() { + let (mut app, _rx) = app_with_pvc("Bound"); + app.handle_key(press(KeyCode::Char('x'))).unwrap(); + resolve_pvc(&mut app, Ok(Some(pvc_mount()))); + fail_pvc_tools(&mut app, "missing browsing tool: ls"); + assert!(app.pvc.loading); + let alternate = crate::pvcexplore::Mount { + container: "tools".into(), + path: "/data".into(), + ..pvc_mount() + }; + pvc_recovery_plan(&mut app, vec![alternate.clone()], Ok(recovery_helper())); + assert_eq!(app.pvc.mount, Some(alternate)); + assert_eq!(app.mode, Mode::PvcExplore); + list_pvc( + &mut app, + "/data", + vec![pvc_entry("folder", crate::pvcexplore::EntryKind::Dir, 0)], + ); + assert!(app.pvc.remote_error.is_none()); + app.handle_key(press(KeyCode::Enter)).unwrap(); + fail_pvc_tools(&mut app, "permission denied"); + assert_eq!(app.pvc.remote_path, "/data"); + assert_eq!(app.pvc.remote[0].name, "folder"); + assert!(app.pvc.recovery.is_none()); + app.handle_key(press(KeyCode::Esc)).unwrap(); + assert_eq!(app.mode, Mode::Table); + assert!(!app.pvc.active); +} + +#[tokio::test] +async fn pvc_missing_tools_offers_helper_once_and_cancel_keeps_the_error() { + let (mut app, _rx) = app_with_pvc("Bound"); + app.handle_key(press(KeyCode::Char('x'))).unwrap(); + resolve_pvc(&mut app, Ok(Some(pvc_mount()))); + fail_pvc_tools(&mut app, "missing browsing tool: ls"); + pvc_recovery_plan( + &mut app, + vec![crate::pvcexplore::Mount { + container: "tools".into(), + ..pvc_mount() + }], + Ok(recovery_helper()), + ); + assert_eq!(app.mode, Mode::PvcExplore); + fail_pvc_tools(&mut app, "missing browsing tool: head"); + assert_eq!(app.mode, Mode::Confirm); + assert!(app.confirm_label.contains("Browse with helper pod")); + assert!(app.confirm_label.contains(&app.pvc_cfg.image)); + assert!(app.confirm_label.contains("default")); + app.handle_key(press(KeyCode::Esc)).unwrap(); + assert_eq!(app.mode, Mode::PvcExplore); + assert!( + app.pvc + .remote_error + .as_ref() + .unwrap() + .contains("missing browsing tool") + ); + assert!(!app.pvc.mount.as_ref().unwrap().helper); + app.handle_key(press(KeyCode::Char('r'))).unwrap(); + fail_pvc_tools(&mut app, "missing browsing tool: ls"); + assert_eq!(app.mode, Mode::PvcExplore); + assert!( + app.pvc + .remote_error + .as_ref() + .unwrap() + .contains("No further recovery") + ); +} + +#[tokio::test] +async fn pvc_recovery_obeys_readonly_guardrails_and_exclusive_access() { + for restriction in ["readonly", "guardrail", "exclusive"] { + let (mut app, _rx) = app_with_pvc("Bound"); + app.handle_key(press(KeyCode::Char('x'))).unwrap(); + resolve_pvc(&mut app, Ok(Some(pvc_mount()))); + fail_pvc_tools(&mut app, "missing browsing tool: ls"); + if restriction == "readonly" { + app.readonly = true; + } + if restriction == "guardrail" { + app.guardrails = vec![crate::config::Guardrail { + actions: vec!["pvc-explore".into()], + deny: true, + ..Default::default() + }]; + } + let helper = if restriction == "exclusive" { + Err("ReadWriteOncePod is already in use".into()) + } else { + Ok(recovery_helper()) + }; + pvc_recovery_plan(&mut app, vec![], helper); + assert_eq!(app.mode, Mode::PvcExplore); + let error = app.pvc.remote_error.as_ref().unwrap(); + assert!(error.contains("missing browsing tool: ls")); + assert!( + error.contains(match restriction { + "readonly" => "read-only", + "guardrail" => "guardrail", + _ => "ReadWriteOncePod", + }), + "{error}" + ); + assert!(app.pending.is_none()); + let mut terminal = + ratatui::Terminal::new(ratatui::backend::TestBackend::new(120, 30)).unwrap(); + terminal.draw(|f| crate::ui::draw(f, &mut app)).unwrap(); + let screen: String = terminal + .backend() + .buffer() + .content + .iter() + .map(|cell| cell.symbol()) + .collect(); + assert!(screen.contains("missing browsing tool: ls"), "{screen}"); + assert!( + screen.contains(match restriction { + "readonly" => "read-only", + "guardrail" => "guardrail", + _ => "ReadWriteOncePod", + }), + "{screen}" + ); + } +} + +#[tokio::test] +async fn pvc_recovery_does_not_retry_permissions_or_stale_results() { + for error in [ + "permission denied", + "connection refused", + "not a directory", + "listing failed (exit 127)", + ] { + let (mut app, _rx) = app_with_pvc("Bound"); + app.handle_key(press(KeyCode::Char('x'))).unwrap(); + resolve_pvc(&mut app, Ok(Some(pvc_mount()))); + fail_pvc_tools(&mut app, error); + assert!(app.pvc.recovery.is_none()); + assert_eq!(app.mode, Mode::PvcExplore); + assert_eq!(app.pvc.remote_error.as_deref(), Some(error)); + } + let (mut app, _rx) = app_with_pvc("Bound"); + app.handle_key(press(KeyCode::Char('x'))).unwrap(); + resolve_pvc(&mut app, Ok(Some(pvc_mount()))); + fail_pvc_tools(&mut app, "missing browsing tool: ls"); + let run = app.pvc.run; + app.handle_key(press(KeyCode::Esc)).unwrap(); + app.handle_msg(Msg::PvcRecovery { + generation: app.generation, + run, + result: Ok(crate::pvcexplore::RecoveryPlan { + candidates: vec![], + helper: Ok(recovery_helper()), + }), + }); + assert_eq!(app.mode, Mode::Table); + assert!(!app.pvc.active); +} + +#[tokio::test] +async fn pvc_recovery_stops_on_non_tool_error_in_an_alternate_container() { + let (mut app, _rx) = app_with_pvc("Bound"); + app.handle_key(press(KeyCode::Char('x'))).unwrap(); + resolve_pvc(&mut app, Ok(Some(pvc_mount()))); + fail_pvc_tools(&mut app, "missing browsing tool: ls"); + pvc_recovery_plan( + &mut app, + vec![crate::pvcexplore::Mount { + container: "tools".into(), + ..pvc_mount() + }], + Ok(recovery_helper()), + ); + fail_pvc_tools(&mut app, "permission denied"); + assert_eq!(app.mode, Mode::PvcExplore); + let error = app.pvc.remote_error.as_ref().unwrap(); + assert!(error.contains("missing browsing tool: ls") && error.contains("permission denied")); +} + +#[tokio::test] +async fn pvc_helper_rechecks_access_after_confirmation_and_cleans_up() { + use http_body_util::BodyExt; + for exclusive_after_offer in [false, true] { + let (mut app, mut rx) = app_with_pvc("Bound"); + let requests = Arc::new(std::sync::Mutex::new(Vec::<(String, String, Value)>::new())); + let recorded = requests.clone(); + let exclusive = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let access = exclusive.clone(); + app.cluster.client = kube::Client::new( + tower::service_fn(move |request: http::Request| { + let recorded = recorded.clone(); + let access = access.clone(); + async move { + let method = request.method().to_string(); + let path = request.uri().path().to_owned(); + let bytes = request.into_body().collect().await.unwrap().to_bytes(); + let body: Value = serde_json::from_slice(&bytes).unwrap_or(Value::Null); + recorded + .lock() + .unwrap() + .push((method.clone(), path.clone(), body)); + let response = if path.ends_with("/persistentvolumeclaims/data") { + json!({"apiVersion":"v1", "kind":"PersistentVolumeClaim", "metadata":{"name":"data"}, + "spec":{"accessModes":[if access.load(std::sync::atomic::Ordering::SeqCst) {"ReadWriteOncePod"} else {"ReadWriteOnce"}]}}) + } else if path.ends_with("/pods") && method == "GET" { + json!({"apiVersion":"v1", "kind":"PodList", "metadata":{}, "items":[{ + "apiVersion":"v1", "kind":"Pod", "metadata":{"name":"api-0", "namespace":"default"}, + "spec":{"nodeName":"worker-a", "volumes":[{"name":"data", "persistentVolumeClaim":{"claimName":"data"}}], + "containers":[{"name":"app", "volumeMounts":[{"name":"data", "mountPath":"/srv", "subPath":"tenant", "readOnly":true}]}]}, + "status":{"phase":"Running", "containerStatuses":[{"name":"app", "state":{"running":{}}}]}}]}) + } else { + json!({"apiVersion":"v1", "kind":"Pod", "metadata":{"name":"sofka-pvc-explore-test", "namespace":"default"}, "status":{"phase":"Running"}}) + }; + Ok::<_, std::convert::Infallible>(http::Response::new( + http_body_util::Full::new(hyper::body::Bytes::from(response.to_string())), + )) + } + }), + "default", + ); + app.handle_key(press(KeyCode::Char('x'))).unwrap(); + resolve_pvc( + &mut app, + Ok(Some(crate::pvcexplore::Mount { + sub_path: Some("tenant".into()), + read_only: true, + ..pvc_mount() + })), + ); + fail_pvc_tools(&mut app, "missing browsing tool: ls"); + tokio::time::timeout(Duration::from_secs(3), async { + while app.mode != Mode::Confirm { + app.handle_msg(rx.recv().await.unwrap()); + } + }) + .await + .unwrap(); + assert!(!requests.lock().unwrap().iter().any(|r| r.0 == "POST")); + exclusive.store(exclusive_after_offer, std::sync::atomic::Ordering::SeqCst); + app.handle_key(press(KeyCode::Char('y'))).unwrap(); + tokio::time::timeout(Duration::from_secs(3), async { + loop { + let message = rx.recv().await.unwrap(); + let completed = + matches!(&message, Msg::PvcTarget { run, .. } if *run == app.pvc.run); + app.handle_msg(message); + if completed { + break; + } + } + }) + .await + .unwrap(); + if exclusive_after_offer { + assert!(!requests.lock().unwrap().iter().any(|r| r.0 == "POST")); + let error = app.pvc.remote_error.as_ref().unwrap(); + assert!(error.contains("ReadWriteOncePod") && error.contains("missing browsing tool")); + } else { + let created = requests + .lock() + .unwrap() + .iter() + .find(|r| r.0 == "POST") + .unwrap() + .2 + .clone(); + assert_eq!( + created.pointer("/spec/containers/0/volumeMounts/0/subPath"), + Some(&json!("tenant")) + ); + assert_eq!( + created.pointer("/spec/containers/0/volumeMounts/0/readOnly"), + Some(&json!(true)) + ); + assert!(created.pointer("/spec/affinity/nodeAffinity").is_some()); + let mount = app.pvc.mount.as_ref().unwrap(); + assert!(mount.helper && mount.read_only); + app.handle_key(press(KeyCode::Esc)).unwrap(); + tokio::time::timeout(Duration::from_secs(3), async { + while !requests + .lock() + .unwrap() + .iter() + .any(|r| r.0 == "DELETE" && r.1.ends_with("/sofka-pvc-explore-test")) + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + } + } +} + /// A PVC view with one bound claim selected. PVCs aren't in `Cluster::fake`'s /// standing registry, so the fixture declares the kind itself. fn app_with_pvc(phase: &str) -> (App, Receiver) { @@ -19029,6 +19357,7 @@ fn pvc_mount() -> crate::pvcexplore::Mount { pod: "api-0".into(), container: "app".into(), path: "/srv".into(), + sub_path: Some(String::new()), read_only: false, helper: false, } @@ -19931,6 +20260,7 @@ async fn a_shell_from_a_pvc_row_keeps_its_pod_until_the_shell_returns() { pod: "sofka-pvc-explore-abc".into(), container: "explore".into(), path: "/pvc".into(), + sub_path: Some(String::new()), read_only: false, helper: true, }; @@ -19953,6 +20283,7 @@ async fn a_shell_from_inside_the_browser_keeps_the_browser_and_its_pod() { pod: "sofka-pvc-explore-abc".into(), container: "explore".into(), path: "/pvc".into(), + sub_path: Some(String::new()), read_only: false, helper: true, }; @@ -20095,6 +20426,7 @@ async fn a_helper_pod_that_lands_after_a_generation_bump_is_not_stranded() { pod: "sofka-pvc-explore-xyz".into(), container: "explore".into(), path: "/pvc".into(), + sub_path: Some(String::new()), read_only: false, helper: true, }; @@ -20144,6 +20476,7 @@ async fn a_helper_pod_from_another_cluster_is_left_alone() { pod: "sofka-pvc-explore-xyz".into(), container: "explore".into(), path: "/pvc".into(), + sub_path: Some(String::new()), read_only: false, helper: true, })), @@ -20289,6 +20622,7 @@ async fn escaping_a_pvc_shell_dialog_with_the_palette_releases_its_pod() { pod: "sofka-pvc-explore-xyz".into(), container: "explore".into(), path: "/pvc".into(), + sub_path: Some(String::new()), read_only: false, helper: true, })), diff --git a/src/pvcexplore.rs b/src/pvcexplore.rs index bdd9025..ebf989d 100644 --- a/src/pvcexplore.rs +++ b/src/pvcexplore.rs @@ -152,6 +152,9 @@ pub fn list_probe(root: &str) -> ListingProbe { script: format!( r#"[ -n "$1" ] && [ -n "$2" ] || exit {EXIT_NOT_A_DIRECTORY} unset TIME_STYLE QUOTING_STYLE BLOCK_SIZE LS_BLOCK_SIZE +for tool in ls head; do + command -v "$tool" >/dev/null 2>&1 || {{ echo "missing browsing tool: $tool" >&2; exit 127; }} +done root=$(cd -- "$2" 2>/dev/null && pwd -P) || exit {EXIT_NOT_A_DIRECTORY} cd -- "$1" 2>/dev/null || exit {EXIT_NOT_A_DIRECTORY} case "$(pwd -P)/" in "${{root%/}}/"*) ;; *) exit {EXIT_OUTSIDE_MOUNT} ;; esac @@ -359,6 +362,8 @@ pub struct Mount { pub pod: String, pub container: String, pub path: String, + /// None means that subPathExpr cannot be resolved from the pod specification. + pub sub_path: Option, /// The mount is `readOnly` in the pod spec: writes will fail, so an upload /// is refused up front instead of failing halfway through a `kubectl cp`. pub read_only: bool, @@ -373,25 +378,18 @@ pub struct Mount { /// `None` means nothing running mounts the claim — the caller's cue to offer a /// helper pod ([`helper_pod`]). pub fn find_mount(pods: &[DynamicObject], claim: &str) -> Option { - let mut best: Option = None; + find_mounts(pods, claim).into_iter().next() +} + +pub fn find_mounts(pods: &[DynamicObject], claim: &str) -> Vec { + let mut mounts = Vec::new(); for pod in pods { - if phase(pod) != "Running" { - continue; + if phase(pod) == "Running" && pod.metadata.deletion_timestamp.is_none() { + mounts.extend(mounts_in(pod, claim)); } - // A pod on its way out will take the exec with it, and its volume is - // about to be released. - if pod.metadata.deletion_timestamp.is_some() { - continue; - } - let Some(mount) = mount_in(pod, claim) else { - continue; - }; - if !mount.read_only { - return Some(mount); - } - best.get_or_insert(mount); } - best + mounts.sort_by_key(|m| m.read_only); + mounts } /// Names of the pod's containers that are in the `running` state right now, @@ -422,87 +420,203 @@ fn phase(pod: &DynamicObject) -> &str { .unwrap_or_default() } -/// The first container in `pod` that mounts `claim`, with its mount path. -fn mount_in(pod: &DynamicObject, claim: &str) -> Option { - let spec = pod.data.get("spec")?; - // Volume names are unique within a pod, so the claim resolves to at most - // one of them. - let volume = spec - .get("volumes")? - .as_array()? - .iter() - .find(|v| { - v.get("persistentVolumeClaim") - .and_then(|p| p.get("claimName")) +fn mounts_in(pod: &DynamicObject, claim: &str) -> Vec { + let Some(spec) = pod.data.get("spec") else { + return Vec::new(); + }; + let volumes: Vec<_> = spec + .get("volumes") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter(|v| { + v.pointer("/persistentVolumeClaim/claimName") .and_then(Value::as_str) == Some(claim) }) - .and_then(|v| v.get("name")) - .and_then(Value::as_str)?; - - let mut best: Option = None; + .collect(); let running = running_containers(pod); - // Sidecars (init containers with `restartPolicy: Always`, GA since 1.29) - // and debug containers run alongside the app and mount the same volumes; - // skipping them would report a claim as unmounted and offer a helper pod - // for a volume that is already attached — which, for ReadWriteOnce, would - // then never schedule. - // - // `status.phase == "Running"` is not enough on its own to exec into any - // one of them: a pod in CrashLoopBackOff is `Running` with nothing to - // enter, and a *completed* init container is in the spec forever. Picking - // either would report the claim as reachable and suppress the helper-pod - // offer, leaving the user in a dead end — so each candidate is checked - // against its own status. - let candidates = [ - ("containers", false), - ("initContainers", true), - ("ephemeralContainers", false), - ] - .into_iter() - .filter_map(|(key, sidecar_only)| Some((spec.get(key)?.as_array()?, sidecar_only))) - .flat_map(|(list, sidecar_only)| list.iter().map(move |c| (c, sidecar_only))); - for (c, sidecar_only) in candidates { - let Some(name) = c.get("name").and_then(Value::as_str) else { - continue; - }; - if !running.contains(name) { - continue; - } - // A plain init container that happens to be running right now is - // mid-initialisation and about to exit; only a native sidecar stays. - if sidecar_only && c.get("restartPolicy").and_then(Value::as_str) != Some("Always") { - continue; - } - let mounts = c - .get("volumeMounts") + let mut mounts = Vec::new(); + for key in ["containers", "initContainers", "ephemeralContainers"] { + for c in spec + .get(key) .and_then(Value::as_array) - .map(Vec::as_slice) - .unwrap_or_default(); - for m in mounts { - if m.get("name").and_then(Value::as_str) != Some(volume) { - continue; - } - let Some(path) = m.get("mountPath").and_then(Value::as_str) else { + .into_iter() + .flatten() + { + let Some(name) = c.get("name").and_then(Value::as_str) else { continue; }; - // A subPath mount shows only part of the volume, but it is still - // the only view that container has of it — browsing it is correct, - // and the alternative is refusing to browse at all. - let mount = Mount { - pod: pod.metadata.name.clone().unwrap_or_default(), - container: name.to_string(), - path: path.to_string(), - read_only: m.get("readOnly").and_then(Value::as_bool).unwrap_or(false), - helper: false, - }; - if !mount.read_only { - return Some(mount); + if !running.contains(name) + || (key == "initContainers" + && c.get("restartPolicy").and_then(Value::as_str) != Some("Always")) + { + continue; + } + for m in c + .get("volumeMounts") + .and_then(Value::as_array) + .into_iter() + .flatten() + { + let Some(volume) = volumes.iter().find(|v| v.get("name") == m.get("name")) else { + continue; + }; + let Some(path) = m.get("mountPath").and_then(Value::as_str) else { + continue; + }; + mounts.push(Mount { + pod: pod.metadata.name.clone().unwrap_or_default(), + container: name.to_owned(), + path: path.to_owned(), + sub_path: if m + .get("subPathExpr") + .and_then(Value::as_str) + .is_some_and(|s| !s.is_empty()) + { + None + } else { + Some( + m.get("subPath") + .and_then(Value::as_str) + .unwrap_or_default() + .to_owned(), + ) + }, + read_only: m.get("readOnly").and_then(Value::as_bool).unwrap_or(false) + || volume + .pointer("/persistentVolumeClaim/readOnly") + .and_then(Value::as_bool) + .unwrap_or(false), + helper: false, + }); } - best.get_or_insert(mount); } } - best + mounts +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HelperOptions { + pub node: Option, + pub sub_path: String, + pub read_only: bool, +} + +#[derive(Debug)] +pub struct RecoveryPlan { + pub candidates: Vec, + pub helper: Result, +} + +pub fn recovery_plan( + pods: &[DynamicObject], + claim: &DynamicObject, + original: &Mount, +) -> RecoveryPlan { + let name = claim.metadata.name.as_deref().unwrap_or_default(); + let candidates = find_mounts(pods, name) + .into_iter() + .filter(|m| { + (m.pod != original.pod || m.container != original.container || m.path != original.path) + && original.sub_path.is_some() + && m.sub_path == original.sub_path + }) + .take(16) + .map(|mut m| { + m.read_only |= original.read_only; + m + }) + .collect(); + RecoveryPlan { + candidates, + helper: helper_options(pods, claim, original), + } +} + +pub fn helper_options( + pods: &[DynamicObject], + claim: &DynamicObject, + original: &Mount, +) -> Result { + let Some(sub_path) = &original.sub_path else { + return Err("Cannot recover a subPathExpr mount without changing its boundaries.".into()); + }; + let name = claim.metadata.name.as_deref().unwrap_or_default(); + let consumers: Vec<_> = pods + .iter() + .filter(|p| !matches!(phase(p), "Succeeded" | "Failed")) + .filter(|p| { + p.data + .pointer("/spec/volumes") + .and_then(Value::as_array) + .into_iter() + .flatten() + .any(|v| { + v.pointer("/persistentVolumeClaim/claimName") + .and_then(Value::as_str) + == Some(name) + }) + }) + .collect(); + let modes: Vec<_> = claim + .data + .pointer("/spec/accessModes") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(Value::as_str) + .collect(); + if modes.is_empty() { + return Err("Cannot determine the volume access modes. No helper was created.".into()); + } + if modes.contains(&"ReadWriteOncePod") && !consumers.is_empty() { + return Err( + "ReadWriteOncePod is already in use. A second pod cannot mount this claim.".into(), + ); + } + let mut node = None; + if modes.contains(&"ReadWriteOnce") && !consumers.is_empty() { + let nodes: std::collections::BTreeSet<_> = consumers + .iter() + .filter_map(|p| p.data.pointer("/spec/nodeName").and_then(Value::as_str)) + .filter(|n| !n.is_empty()) + .collect(); + if nodes.len() != 1 { + return Err("Cannot select one consumer node for this ReadWriteOnce claim.".into()); + } + node = nodes.first().map(|s| (*s).to_owned()); + } + Ok(HelperOptions { + node, + sub_path: sub_path.clone(), + read_only: original.read_only || modes == ["ReadOnlyMany"], + }) +} + +pub fn apply_helper_options(manifest: &mut Value, options: &HelperOptions) { + if let Some(node) = &options.node { + manifest["spec"]["affinity"] = json!({"nodeAffinity": {"requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [{"matchFields": [{"key":"metadata.name", "operator":"In", "values":[node]}]}] + }}}); + } + manifest["spec"]["containers"][0]["volumeMounts"][0]["readOnly"] = json!(options.read_only); + manifest["spec"]["volumes"][0]["persistentVolumeClaim"]["readOnly"] = json!(options.read_only); + if !options.sub_path.is_empty() { + manifest["spec"]["containers"][0]["volumeMounts"][0]["subPath"] = json!(options.sub_path); + } +} + +pub fn missing_listing_tools(error: &str) -> bool { + error.lines().any(|line| { + let line = line.to_ascii_lowercase(); + (["sh", "/bin/sh"] + .iter() + .any(|tool| line.contains(&format!("exec: \"{tool}\""))) + && (line.contains("executable file not found") + || line.contains("no such file or directory"))) + || line.starts_with("missing browsing tool: ") + }) } /// The helper pod sofka creates when nothing mounts the claim: one sleeping @@ -2285,3 +2399,147 @@ mod tests { std::fs::remove_dir_all(&dir).ok(); } } + +#[cfg(test)] +mod recovery_tests { + use super::*; + + fn consumer() -> DynamicObject { + serde_json::from_value(json!({ + "apiVersion":"v1", "kind":"Pod", "metadata":{"name":"app", "namespace":"default"}, + "spec":{"nodeName":"worker-a", "volumes":[{"name":"data", "persistentVolumeClaim":{"claimName":"data"}}], + "containers":[ + {"name":"app", "volumeMounts":[{"name":"data", "mountPath":"/data", "subPath":"tenant", "readOnly":true}]}, + {"name":"tools", "volumeMounts":[{"name":"data", "mountPath":"/tools", "subPath":"tenant"}]}, + {"name":"broad", "volumeMounts":[{"name":"data", "mountPath":"/all"}]}]}, + "status":{"phase":"Running", "containerStatuses":[ + {"name":"app", "state":{"running":{}}}, + {"name":"tools", "state":{"running":{}}}, + {"name":"broad", "state":{"running":{}}}]} + })).unwrap() + } + + fn claim(mode: &str) -> DynamicObject { + serde_json::from_value(json!({"apiVersion":"v1", "kind":"PersistentVolumeClaim", + "metadata":{"name":"data"}, "spec":{"accessModes":[mode]}, "status":{"phase":"Bound"}})) + .unwrap() + } + + fn original(pod: &DynamicObject) -> Mount { + find_mounts(std::slice::from_ref(pod), "data") + .into_iter() + .find(|m| m.container == "app") + .unwrap() + } + + #[test] + fn recovery_preserves_subpath_and_readonly_and_uses_consumer_node() { + let pod = consumer(); + let original = original(&pod); + let plan = recovery_plan(&[pod], &claim("ReadWriteOnce"), &original); + assert_eq!(plan.candidates.len(), 1); + assert_eq!(plan.candidates[0].container, "tools"); + assert!(plan.candidates[0].read_only); + let options = plan.helper.unwrap(); + assert_eq!(options.node.as_deref(), Some("worker-a")); + assert_eq!(options.sub_path, "tenant"); + assert!(options.read_only); + let mut manifest = helper_pod("data", "busybox:1.37", 900); + apply_helper_options(&mut manifest, &options); + assert_eq!(manifest.pointer("/spec/affinity/nodeAffinity/requiredDuringSchedulingIgnoredDuringExecution/nodeSelectorTerms/0/matchFields/0/values/0"), Some(&json!("worker-a"))); + assert_eq!( + manifest.pointer("/spec/containers/0/volumeMounts/0/subPath"), + Some(&json!("tenant")) + ); + assert_eq!( + manifest.pointer("/spec/containers/0/volumeMounts/0/readOnly"), + Some(&json!(true)) + ); + assert_eq!( + manifest.pointer("/spec/volumes/0/persistentVolumeClaim/readOnly"), + Some(&json!(true)) + ); + } + + #[test] + fn recovery_blocks_occupied_rwop_but_allows_existing_container() { + let pod = consumer(); + let original = original(&pod); + let plan = recovery_plan(&[pod], &claim("ReadWriteOncePod"), &original); + assert_eq!(plan.candidates.len(), 1); + assert!(plan.helper.unwrap_err().contains("ReadWriteOncePod")); + assert!(helper_options(&[], &claim("ReadWriteOncePod"), &original).is_ok()); + } + + #[test] + fn recovery_refuses_unknown_subpath_or_ambiguous_rwo_node() { + let mut pod = consumer(); + let mut original = original(&pod); + original.sub_path = None; + let plan = recovery_plan(&[pod.clone()], &claim("ReadWriteMany"), &original); + assert!(plan.candidates.is_empty()); + assert!(plan.helper.unwrap_err().contains("subPathExpr")); + original.sub_path = Some("tenant".into()); + pod.data["spec"].as_object_mut().unwrap().remove("nodeName"); + assert!(helper_options(&[pod.clone()], &claim("ReadWriteOnce"), &original).is_err()); + pod.data["spec"]["nodeName"] = json!("worker-b"); + assert!(helper_options(&[pod, consumer()], &claim("ReadWriteOnce"), &original).is_err()); + } + + #[test] + fn recovery_limits_candidates_and_honors_claim_mount_readonly() { + let mut pod = consumer(); + pod.data["spec"]["volumes"][0]["persistentVolumeClaim"]["readOnly"] = json!(true); + assert!( + find_mounts(&[pod.clone()], "data") + .iter() + .all(|m| m.read_only) + ); + let original = original(&pod); + let pods: Vec<_> = (0..30) + .map(|i| { + let mut p = pod.clone(); + p.metadata.name = Some(format!("pod-{i}")); + p + }) + .collect(); + assert_eq!( + recovery_plan(&pods, &claim("ReadWriteMany"), &original) + .candidates + .len(), + 16 + ); + } + + #[test] + fn missing_tool_checks_are_specific_and_report_before_listing() { + for error in [ + "exec: \"sh\": executable file not found in $PATH", + "missing browsing tool: ls", + "missing browsing tool: head", + ] { + assert!(missing_listing_tools(error), "{error}"); + } + for error in [ + "permission denied", + "connection refused", + "listing failed (exit 127)", + "ls: file not found", + ] { + assert!(!missing_listing_tools(error), "{error}"); + } + let probe = list_probe("/"); + let out = std::process::Command::new("/bin/sh") + .args(["-c", &probe.script, "sh", "/", "/"]) + .env("PATH", "/sofka-test-no-tools") + .output() + .unwrap(); + let result = interpret_listing( + &probe.nonce, + out.status.code(), + &String::from_utf8_lossy(&out.stdout), + &String::from_utf8_lossy(&out.stderr), + ); + assert_eq!(result.unwrap_err(), "missing browsing tool: ls"); + } +} diff --git a/src/store.rs b/src/store.rs index f5c64a3..2724cca 100644 --- a/src/store.rs +++ b/src/store.rs @@ -335,6 +335,11 @@ pub enum Msg { claim: StatusClaim, result: Result, String>, }, + PvcRecovery { + generation: u64, + run: u64, + result: Result, + }, /// One directory listing for the remote pane of the PVC browser. PvcListing { generation: u64, diff --git a/src/ui.rs b/src/ui.rs index a2e3267..eeb6949 100644 --- a/src/ui.rs +++ b/src/ui.rs @@ -4754,10 +4754,14 @@ fn pvc_pane_items( use crate::pvcexplore::EntryKind; if let Some(e) = error { - return vec![ListItem::new(Line::from(Span::styled( - e.to_string(), - Style::default().fg(theme::red()), - )))]; + let lines: Vec<_> = Text::from(e.to_string()) + .lines + .into_iter() + .flat_map(|line| wrap_line(line, usize::from(width.saturating_sub(4)).max(1))) + .collect(); + return vec![ListItem::new( + Text::from(lines).style(Style::default().fg(theme::red())), + )]; } if entries.is_empty() { return vec![ListItem::new(Line::from(Span::styled(