From d41c1bd48df9b3d7f84f1602bc99c7f3e8ba1f87 Mon Sep 17 00:00:00 2001 From: florian Date: Tue, 21 Jul 2026 16:12:22 +0200 Subject: [PATCH 1/2] fix: prevent UI freezes from dead tunnels, unbounded dir sizes, Ollama polls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Severe hangs, all with the same shape: a synchronous Tauri command aimed at a peer that will never answer, blocking until ssh's own keepalive gives up (~45s) — and the file tree issues one per visible folder. VPN drop gating - vpnStatus.refresh now distinguishes a tunnel that died on its own from one torn down deliberately (every deliberate teardown forgets the config first, so only an unplanned death reaches the new branch). - A dropped tunnel clears the status of the projects that were holding it, which flips useRemoteBlocked so probes are never dispatched rather than each paying the timeout. Scoped to the tunnel's holders: a project that never claimed the config reaches its host another way. - The pooled backend session is deliberately left alone — reaping it is the pooled reader's job, and a teardown issued here would itself be a call over the dead connection. Directory sizes (src/lib/dirSizeGuard.ts) - Folder sizes are a best-effort display aid, so they are safe to bound aggressively: timeout, concurrency cap, and a per-project circuit breaker that opens after repeated failures. Ollama status (src/lib/ollamaStatus.ts) - Collapse the per-component polls into one app-wide timer. Also: run-host preference handling for shell/python runs, and backend project/git/openvpn updates. Tests: DirSizeGuard, VpnDropGating, ShellScriptRun. Co-Authored-By: Claude Opus 4.8 --- Cargo.lock | 2 +- src-tauri/src/commands/git.rs | 14 ++ src-tauri/src/commands/projects.rs | 50 +++++++ src-tauri/src/lib.rs | 1 + src-tauri/src/schema/project.rs | 9 ++ src-tauri/src/services/openvpn.rs | 50 ++++++- src/__tests__/DirSizeGuard.test.ts | 145 +++++++++++++++++++++ src/__tests__/ShellScriptRun.test.ts | 40 ++++++ src/__tests__/VpnDropGating.test.ts | 115 ++++++++++++++++ src/components/embed/FileViewerPane.tsx | 47 +++---- src/components/files/FileTree.tsx | 76 ++++++++--- src/components/files/ProjectFilesView.tsx | 12 +- src/components/header/VpnIndicator.tsx | 17 ++- src/components/layout/LocalModelMenu.tsx | 29 +---- src/lib/dirSizeGuard.ts | 152 ++++++++++++++++++++++ src/lib/ollamaStatus.ts | 122 +++++++++++++++++ src/lib/pythonRun.ts | 21 ++- src/lib/shellScriptRun.ts | 35 ++++- src/stores/projects.ts | 12 ++ src/stores/runHostPref.ts | 70 ++++++---- src/stores/tabs.ts | 30 ++++- src/stores/vpnStatus.ts | 42 ++++++ src/types/index.ts | 6 + 23 files changed, 979 insertions(+), 118 deletions(-) create mode 100644 src/__tests__/DirSizeGuard.test.ts create mode 100644 src/__tests__/VpnDropGating.test.ts create mode 100644 src/lib/dirSizeGuard.ts create mode 100644 src/lib/ollamaStatus.ts diff --git a/Cargo.lock b/Cargo.lock index 9d76107..7f719a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1129,7 +1129,7 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "eldrun" -version = "0.1.31" +version = "0.1.32" dependencies = [ "arboard", "base64 0.22.1", diff --git a/src-tauri/src/commands/git.rs b/src-tauri/src/commands/git.rs index b9ffada..c25a68c 100644 --- a/src-tauri/src/commands/git.rs +++ b/src-tauri/src/commands/git.rs @@ -396,6 +396,20 @@ fn git_file_statuses_blocking( } } + // When `rel_path` is itself wholly ignored, EVERY entry under it is ignored — + // but git's porcelain only ever emits *file* lines, so a child that is an + // empty directory (or a tree of only empty directories, e.g. freshly scaffolded + // experiment-output folders) produces no line at all and would silently drop + // its "ignored" marker, landing in the regular section. git also can't be + // coaxed into listing such dirs: it collapses a wholly-ignored subtree to its + // topmost ignored ancestor and never enumerates empty descendants. Emit a + // sentinel under the reserved key "." (never a real listing entry) so the + // frontend can default every child of this directory to ignored, letting any + // explicit per-child status above (a force-added tracked file) still win. + if wholly_ignored { + map.insert(".".to_string(), "ignored".to_string()); + } + Ok(map) } diff --git a/src-tauri/src/commands/projects.rs b/src-tauri/src/commands/projects.rs index 471b5da..8dfb155 100644 --- a/src-tauri/src/commands/projects.rs +++ b/src-tauri/src/commands/projects.rs @@ -1017,6 +1017,56 @@ fn patch_compute_hosts( Ok(hosts) } +/// Persist which machine shells launched from this project run on — the choice +/// made in the `RunHostPicker`. Mirrors `set_project_python`: it writes BOTH the +/// `projects.json` entry's `extra["run_host"]` (what the frontend seeds its live +/// preference store from on load) and the `project.json` `run_host` field (keeps +/// it with the project). `location` is a `TabLocation` string (`"local"` / +/// `"remote"` / `"host:"`); `None` or empty clears it (back to the primary +/// default). Returns the stored value. +#[tauri::command] +pub fn set_project_run_host( + project_id: String, + location: Option, +) -> Result, String> { + let value = location + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()); + + let list_path = storage::state_dir().join("projects.json"); + let mut list: ProjectsList = if list_path.exists() { + storage::read_json(&list_path).map_err(|e| e.to_string())? + } else { + Vec::new() + }; + let entry = list + .iter_mut() + .find(|p| p.id == project_id) + .ok_or_else(|| format!("project '{project_id}' not found"))?; + + match &value { + Some(v) => { + entry + .extra + .insert("run_host".into(), serde_json::Value::String(v.clone())); + } + None => { + entry.extra.remove("run_host"); + } + } + let local_file = entry.local_file.clone(); + storage::write_json(&list_path, &list).map_err(|e| e.to_string())?; + + let proj_path = PathBuf::from(&local_file); + if proj_path.exists() { + if let Ok(mut project) = storage::read_json::(&proj_path) { + project.run_host = value.clone(); + storage::write_json(&proj_path, &project).map_err(|e| e.to_string())?; + } + } + Ok(value) +} + /// Add an extra "worker" host to a remote project (the pill's "Add machine"). /// Mints a stable `id` when the incoming spec has none, then appends it. Returns /// the full updated host list. The primary (`Project.remote`) is untouched. diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 8aeb19a..264efe1 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -601,6 +601,7 @@ pub fn run() { commands::projects::add_compute_host, commands::projects::remove_compute_host, commands::projects::patch_compute_host, + commands::projects::set_project_run_host, commands::projects::set_project_categories, commands::projects::set_project_git_disabled, commands::projects::save_tab_layout, diff --git a/src-tauri/src/schema/project.rs b/src-tauri/src/schema/project.rs index 4765726..43c5040 100644 --- a/src-tauri/src/schema/project.rs +++ b/src-tauri/src/schema/project.rs @@ -339,6 +339,15 @@ pub struct Project { /// entry's `extra["python_interpreter"]`. #[serde(skip_serializing_if = "Option::is_none")] pub python_interpreter: Option, + /// Which machine shells launched from this project run on — the choice made in + /// the `RunHostPicker`, persisted so it survives a relaunch (a Run/Debug or a + /// new shell tab lands on it instead of the primary). A `TabLocation` string: + /// `"local"` (the mirror), `"remote"` (the primary host), or `"host:"` (a + /// worker). Absent = the shell default (the primary). Mirrored into the + /// `projects.json` entry's `extra["run_host"]`, which is what the frontend + /// seeds its live preference store from on load. + #[serde(skip_serializing_if = "Option::is_none")] + pub run_host: Option, #[serde(flatten)] pub extra: HashMap, } diff --git a/src-tauri/src/services/openvpn.rs b/src-tauri/src/services/openvpn.rs index 515b95d..da5805f 100644 --- a/src-tauri/src/services/openvpn.rs +++ b/src-tauri/src/services/openvpn.rs @@ -507,7 +507,7 @@ pub fn openvpn_available() -> bool { /// encrypted private key. /// /// Shape: `--config [--auth-user-pass ] [--askpass ] --auth-nocache -/// --writepid --connect-timeout 20 --connect-retry-max 3 +/// --writepid --connect-timeout 20 --persist-tun /// --verb 3 --mute 0`. pub fn openvpn_args( config: &str, @@ -535,8 +535,27 @@ pub fn openvpn_args( pidfile.to_string_lossy().into_owned(), "--connect-timeout".to_string(), "20".to_string(), - "--connect-retry-max".to_string(), - "3".to_string(), + // NO `--connect-retry-max`. That option is *not* an initial-connect + // guard — it caps reconnect attempts for the tunnel's whole lifetime, so + // the `3` this used to pass meant any mid-session blip (a rekey failure, + // a suspended laptop, a server restart) burned the three retries and + // OpenVPN **exited for good**, leaving the machine's routing changed and + // every SSH/SFTP session on top of it stalled against a peer that would + // never answer. Unlimited (the OpenVPN default) is what makes a tunnel + // survive a blip. The *initial* connect is still bounded — from the + // outside, by `CONNECT_TIMEOUT` + the `child.kill()` on its Err — so a + // doomed connect cannot retry forever behind the user's back. + // + // `--persist-tun` keeps the tun device and its routes across those + // reconnects, which is what lets a connection on top of the tunnel ride + // one out instead of breaking. It also makes the failure mode + // fail-*closed*: while OpenVPN is re-handshaking, traffic is black-holed + // rather than leaking out the physical default route. That is the right + // trade for a VPN, and it is only safe now that the tunnel no longer + // gives up permanently (above) and that a tunnel dying is *detected* + // rather than left showing a green lamp (`stores/vpnStatus`'s drop + // reconcile). + "--persist-tun".to_string(), // Readiness is detected by watching OpenVPN's output for READY_MARKER, // so logging must be deterministic no matter what the config says: a // config's `mute N` suppresses consecutive same-category lines — the @@ -2543,6 +2562,31 @@ mod tests { assert!(args.iter().any(|a| a == "--auth-nocache")); } + /// A tunnel must survive a blip. `--connect-retry-max` caps reconnects for + /// the tunnel's whole *lifetime*, not just the first connect, so passing it + /// meant a momentary drop exhausted the retries and OpenVPN exited for good + /// — leaving the machine's routing changed and every SSH/SFTP session on top + /// stalled against a peer that would never answer. It must stay absent, with + /// `--persist-tun` holding the device and routes across a reconnect. + #[test] + fn openvpn_args_let_a_dropped_tunnel_reconnect_forever() { + let args = openvpn_args( + "/home/u/work.ovpn", + None, + Some(Path::new("/run/eldrun/openvpn/x.pass")), + Path::new("/run/eldrun/openvpn/x.pid"), + ) + .unwrap(); + assert!( + !args.iter().any(|a| a == "--connect-retry-max"), + "a retry cap makes a blip kill the tunnel permanently: {args:?}" + ); + assert!(args.iter().any(|a| a == "--persist-tun")); + // The initial connect stays bounded by CONNECT_TIMEOUT, not by openvpn. + let ti = args.iter().position(|a| a == "--connect-timeout").unwrap(); + assert_eq!(args[ti + 1], "20"); + } + /// Readiness is detected by scanning OpenVPN's output for the /// "Initialization Sequence Completed" marker, so the argv must pin the /// logging knobs a config could otherwise sabotage it with: a `mute 16` diff --git a/src/__tests__/DirSizeGuard.test.ts b/src/__tests__/DirSizeGuard.test.ts new file mode 100644 index 0000000..a5676d8 --- /dev/null +++ b/src/__tests__/DirSizeGuard.test.ts @@ -0,0 +1,145 @@ +/** + * The file tree's folder sizes must never be able to freeze the app. + * + * They are a display aid — the tree renders fine without them — but for a remote + * project each one is a `du` over SSH, and the tree fires one per visible folder + * at once. When the network under the project goes away (a dropped VPN tunnel, + * a suspended laptop) those calls do not *fail*: a black-holed socket returns no + * error, so each blocks until ssh's keepalive gives up ~45s later, holding a + * backend blocking-pool thread the whole time. Opening one folder of thirty + * subfolders then meant thirty stalled threads and a frozen UI. + * + * These lock the three limits that prevent that, and — just as importantly — the + * cases where the guard must stay out of the way. + */ +import { describe, it, expect, beforeEach, vi, afterEach } from "vitest"; + +import { + DirSizeUnavailable, + FAILURES_TO_OPEN, + MAX_CONCURRENT, + OPEN_MS, + TIMEOUT_MS, + guardedDirSize, + isHostTimeout, + resetDirSizeGuard, +} from "../lib/dirSizeGuard"; + +const PROJECT = "/home/u/eldrun/projects/demo"; + +/** A call that never settles — what a black-holed SSH round trip looks like. */ +const hangs = () => new Promise(() => {}); + +beforeEach(() => { + resetDirSizeGuard(); + vi.useFakeTimers(); +}); + +afterEach(() => { + vi.useRealTimers(); + resetDirSizeGuard(); +}); + +describe("timeout", () => { + it("gives up on a call that never answers instead of hanging forever", async () => { + const settled = vi.fn(); + const p = guardedDirSize(PROJECT, hangs).then(settled, settled); + await vi.advanceTimersByTimeAsync(TIMEOUT_MS - 1); + expect(settled).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(2); + await p; + expect(settled).toHaveBeenCalledOnce(); + expect(isHostTimeout(settled.mock.calls[0][0])).toBe(true); + }); + + it("passes a prompt answer straight through", async () => { + await expect(guardedDirSize(PROJECT, () => Promise.resolve(4096))).resolves.toBe(4096); + }); +}); + +describe("concurrency cap", () => { + it("holds calls beyond the cap back rather than dispatching them all at once", async () => { + const started = vi.fn(); + for (let i = 0; i < MAX_CONCURRENT + 3; i += 1) { + void guardedDirSize(PROJECT, () => { + started(); + return hangs(); + }).catch(() => {}); + } + await vi.advanceTimersByTimeAsync(0); + // The point: a wide folder must not fire one call per subfolder into a + // network that isn't answering. + expect(started).toHaveBeenCalledTimes(MAX_CONCURRENT); + }); + + it("hands a freed slot to the next waiter", async () => { + const started = vi.fn(); + for (let i = 0; i < MAX_CONCURRENT + 1; i += 1) { + void guardedDirSize(PROJECT, () => { + started(); + return hangs(); + }).catch(() => {}); + } + await vi.advanceTimersByTimeAsync(0); + expect(started).toHaveBeenCalledTimes(MAX_CONCURRENT); + // First in flight times out → its slot goes to the queued call. + await vi.advanceTimersByTimeAsync(TIMEOUT_MS + 1); + expect(started).toHaveBeenCalledTimes(MAX_CONCURRENT + 1); + }); +}); + +describe("circuit breaker", () => { + /** Time out `n` calls back to back. */ + async function timeOut(n: number) { + for (let i = 0; i < n; i += 1) { + const p = guardedDirSize(PROJECT, hangs).catch((e) => e); + await vi.advanceTimersByTimeAsync(TIMEOUT_MS + 1); + await p; + } + } + + it("refuses instantly once the host has repeatedly failed to answer", async () => { + await timeOut(FAILURES_TO_OPEN); + // This is the whole point: the 30th folder costs nothing at all, instead of + // each one paying the timeout in turn. + const run = vi.fn(hangs); + await expect(guardedDirSize(PROJECT, run)).rejects.toBeInstanceOf(DirSizeUnavailable); + expect(run).not.toHaveBeenCalled(); + }); + + it("does not open on failures that are not timeouts", async () => { + // An unreadable folder answered fine — it says nothing about the host, so it + // must not push the project toward a trip. + for (let i = 0; i < FAILURES_TO_OPEN + 2; i += 1) { + await guardedDirSize(PROJECT, () => Promise.reject(new Error("permission denied"))).catch( + () => {}, + ); + } + await expect(guardedDirSize(PROJECT, () => Promise.resolve(7))).resolves.toBe(7); + }); + + it("lets a retry through once the open window elapses", async () => { + await timeOut(FAILURES_TO_OPEN); + await vi.advanceTimersByTimeAsync(OPEN_MS + 1); + await expect(guardedDirSize(PROJECT, () => Promise.resolve(11))).resolves.toBe(11); + }); + + it("closes again on the first success, so one good answer restores the tree", async () => { + await timeOut(FAILURES_TO_OPEN - 1); + await expect(guardedDirSize(PROJECT, () => Promise.resolve(1))).resolves.toBe(1); + // The earlier timeouts are forgiven — the next timeout starts a fresh count + // rather than tripping the breaker immediately. + const p = guardedDirSize(PROJECT, hangs).catch((e) => e); + await vi.advanceTimersByTimeAsync(TIMEOUT_MS + 1); + await p; + const run = vi.fn(() => Promise.resolve(2)); + await expect(guardedDirSize(PROJECT, run)).resolves.toBe(2); + expect(run).toHaveBeenCalledOnce(); + }); + + it("is per project — one dead host does not stop another project's tree", async () => { + await timeOut(FAILURES_TO_OPEN); + await expect(guardedDirSize(PROJECT, hangs)).rejects.toBeInstanceOf(DirSizeUnavailable); + await expect(guardedDirSize("/home/u/other", () => Promise.resolve(99))).resolves.toBe(99); + }); +}); diff --git a/src/__tests__/ShellScriptRun.test.ts b/src/__tests__/ShellScriptRun.test.ts index 6ea5006..4ee1d49 100644 --- a/src/__tests__/ShellScriptRun.test.ts +++ b/src/__tests__/ShellScriptRun.test.ts @@ -51,6 +51,46 @@ describe("shell script run planning", () => { }); }); + it("runs on the chosen worker machine when a run-host preference is set", () => { + const plan = shellScriptRunPlan({ + project: remoteProject, + treeRoot: "/state/simplegnn", + syncSource: "remote", + scriptPath: "/home/alice/simplegnn/train.sh", + interp: "bash", + runHostPref: "host:mlai20", + }); + + // The chosen machine wins over the browsed side; the script path stays + // project-relative so it resolves against that host's own project root (the + // backend re-cds into the worker's remote_path). + expect(plan).toMatchObject({ + cwd: "/home/alice/simplegnn", + scriptRel: "train.sh", + initialInput: "bash 'train.sh'", + location: "host:mlai20", + }); + }); + + it("sends a mirror-browsed script to the chosen remote machine", () => { + // Browsing the local mirror but with a worker chosen: the run still lands on + // the worker, and scriptRel (project-relative) resolves there. + const plan = shellScriptRunPlan({ + project: remoteProject, + treeRoot: "/state/simplegnn/mirror", + syncSource: "local", + scriptPath: "/state/simplegnn/mirror/train.sh", + interp: "bash", + runHostPref: "host:mlai20", + }); + + expect(plan).toMatchObject({ + cwd: "/home/alice/simplegnn", + scriptRel: "train.sh", + location: "host:mlai20", + }); + }); + it("refuses to build bash with an empty script path", () => { expect( shellScriptRunPlan({ diff --git a/src/__tests__/VpnDropGating.test.ts b/src/__tests__/VpnDropGating.test.ts new file mode 100644 index 0000000..6b85f52 --- /dev/null +++ b/src/__tests__/VpnDropGating.test.ts @@ -0,0 +1,115 @@ +/** + * A tunnel that dies on its own must be *noticed*, and noticing it must gate the + * SSH/SFTP work belonging to the projects that were riding it. + * + * This is the failure this exists for: OpenVPN exits (it used to do so + * permanently after three failed reconnects — see `openvpn_args`), the machine's + * routing is left pointing into a tunnel with nothing behind it, and every + * pooled-SSH call a remote project makes now aims at a peer that will never + * answer. Those calls are synchronous Tauri commands, and a black-holed socket + * yields no error, so each blocks ~45s until ssh's keepalive gives up. The file + * tree issues one per visible folder. The app froze. + * + * `refresh()` is the only thing that reconciles against the backend, so the drop + * has to be detected *there* — and it has to be distinguishable from a + * deliberate disconnect, which must not be reported as a drop. + */ +import { describe, it, expect, beforeEach, vi } from "vitest"; + +import { invoke } from "@tauri-apps/api/core"; + +vi.mock("@tauri-apps/api/core", () => ({ invoke: vi.fn(() => Promise.resolve()) })); + +import { useVpnStatusStore, markVpnConnected, disconnectVpnTunnel } from "../stores/vpnStatus"; +import { useRemoteStatusStore } from "../stores/remoteStatus"; + +const invokeMock = vi.mocked(invoke); +const CONFIG = "/store/office.ovpn"; +const PROJECT = "p-remote"; + +/** What the backend reports as live on the next reconcile. */ +function backendReports(configs: string[]) { + invokeMock.mockImplementation((cmd: string) => { + if (cmd === "openvpn_active") return Promise.resolve(configs); + return Promise.resolve(); + }); +} + +beforeEach(() => { + invokeMock.mockReset(); + useVpnStatusStore.setState({ byConfig: {}, holders: {} }); + useRemoteStatusStore.setState({ byProject: {}, byHost: {} }); +}); + +describe("a tunnel that dies unasked", () => { + beforeEach(() => { + markVpnConnected(PROJECT, CONFIG); + useRemoteStatusStore.getState().setSsh(PROJECT, "connected"); + }); + + it("is detected by the reconcile and clears the holder's remote status", async () => { + backendReports([]); // the tunnel is simply gone + await useVpnStatusStore.getState().refresh(); + + expect(useVpnStatusStore.getState().byConfig[CONFIG]).toBeUndefined(); + // The gate: with the project's status cleared, `useRemoteBlocked` stops the + // SFTP/git probes being dispatched at all, rather than each paying ~45s. + expect(useRemoteStatusStore.getState().byProject[PROJECT]).toBeUndefined(); + }); + + it("releases the claims, so a later reconnect is not seen as already held", async () => { + backendReports([]); + await useVpnStatusStore.getState().refresh(); + expect(useVpnStatusStore.getState().holders[CONFIG]).toBeUndefined(); + }); + + it("leaves a project that never claimed the tunnel alone", async () => { + // It reaches its host by some other route; this tunnel is none of its business. + useRemoteStatusStore.getState().setSsh("p-other", "connected"); + backendReports([]); + await useVpnStatusStore.getState().refresh(); + expect(useRemoteStatusStore.getState().byProject["p-other"]?.ssh).toBe("connected"); + }); +}); + +describe("everything that is not a drop", () => { + it("does not fire for a tunnel that is still up", async () => { + markVpnConnected(PROJECT, CONFIG); + useRemoteStatusStore.getState().setSsh(PROJECT, "connected"); + backendReports([CONFIG]); + await useVpnStatusStore.getState().refresh(); + expect(useRemoteStatusStore.getState().byProject[PROJECT]?.ssh).toBe("connected"); + }); + + it("does not fire for a deliberate disconnect", async () => { + markVpnConnected(PROJECT, CONFIG); + useRemoteStatusStore.getState().setSsh(PROJECT, "connected"); + // A UI teardown forgets the config first, so the next reconcile sees neither + // side and must not mistake it for a death. + disconnectVpnTunnel(CONFIG); + useRemoteStatusStore.getState().setSsh(PROJECT, "connected"); + backendReports([]); + await useVpnStatusStore.getState().refresh(); + expect(useRemoteStatusStore.getState().byProject[PROJECT]?.ssh).toBe("connected"); + }); + + it("does not fire for a tunnel still mid-handshake", async () => { + useVpnStatusStore.getState().setState(CONFIG, "connecting"); + useVpnStatusStore.getState().acquire(CONFIG, PROJECT); + useRemoteStatusStore.getState().setSsh(PROJECT, "connecting"); + backendReports([]); // not in the registry yet — that is normal, not a death + await useVpnStatusStore.getState().refresh(); + expect(useVpnStatusStore.getState().byConfig[CONFIG]).toBe("connecting"); + expect(useRemoteStatusStore.getState().byProject[PROJECT]?.ssh).toBe("connecting"); + }); + + it("does nothing when the backend cannot be reached at all", async () => { + markVpnConnected(PROJECT, CONFIG); + useRemoteStatusStore.getState().setSsh(PROJECT, "connected"); + invokeMock.mockImplementation(() => Promise.reject(new Error("backend gone"))); + await useVpnStatusStore.getState().refresh(); + // No answer is not evidence the tunnel died — the state must be left alone. + expect(useVpnStatusStore.getState().byConfig[CONFIG]).toBe("connected"); + expect(useRemoteStatusStore.getState().byProject[PROJECT]?.ssh).toBe("connected"); + }); +}); diff --git a/src/components/embed/FileViewerPane.tsx b/src/components/embed/FileViewerPane.tsx index 5fb47da..5b831c6 100644 --- a/src/components/embed/FileViewerPane.tsx +++ b/src/components/embed/FileViewerPane.tsx @@ -30,6 +30,7 @@ import { PresentationOverlay } from "./PresentationOverlay"; import { renderMarkdown } from "../../lib/viewers/markdown"; import { enrichMarkdownDom } from "../../lib/viewers/markdownEnrich"; import { highlight, languageForPath, escapeHtml } from "../../lib/viewers/highlight"; +import { useOllamaStatus } from "../../lib/ollamaStatus"; import { printDocument, printHtmlBody, @@ -64,6 +65,7 @@ import { snapBreakpointLine, } from "../../lib/viewers/python"; import { debugPythonFile, runCwd, runPythonFile, placeForFocused } from "../../lib/pythonRun"; +import { RunHostPicker } from "../tabs/TabLocalityBadges"; import { isSlurmScript, parseSbatchDirectives, @@ -4143,30 +4145,16 @@ function useTabAiPrefs(tabKey: string | undefined, type: InternalViewer): TabAiP * Both AI-assist features the controls expose (autocomplete + grammar) run only * against a resident model, so the controls hide themselves entirely when none * is loaded. Mirrors the lamp logic in `LocalModelMenu`: `ollama_status` is - * `"loaded"` iff `/api/ps` reports a resident model. Polled on the same 5s - * cadence as the 🧠 menu so the controls appear/disappear within a few seconds - * of a model being warmed or unloaded out of band. + * `"loaded"` iff `/api/ps` reports a resident model. + * + * Rides the app-wide shared poller (`lib/ollamaStatus`) rather than owning a + * timer. This hook runs **per editable viewer tab**, so a private 5s interval + * made the `/api/ps` request rate a function of how many tabs happened to be + * open — asking the same machine-wide question N times over. One timer now + * serves every surface, and they all flip on the same observation. */ function useLocalModelLoaded(): boolean { - const [loaded, setLoaded] = useState(false); - useEffect(() => { - let cancelled = false; - const check = () => - invoke<"stopped" | "idle" | "loaded">("ollama_status") - .then((s) => { - if (!cancelled) setLoaded(s === "loaded"); - }) - .catch(() => { - if (!cancelled) setLoaded(false); - }); - void check(); - const id = window.setInterval(check, 5000); - return () => { - cancelled = true; - window.clearInterval(id); - }; - }, []); - return loaded; + return useOllamaStatus() === "loaded"; } /** @@ -5253,6 +5241,21 @@ function TextView({ /> )} + {/* Which machine a Run/Debug lands on — shown right next to the Run button + so the choice is co-located with running, and keyed by THIS viewer's + `projectId`, the exact scope `onRun` reads the preference back under (a + picker in the file panel is keyed by the *active* project, which can + differ from the viewed file's project → the choice would be dropped and + the run would fall back to the primary). Only for a remote project with + extra worker machines; a lone-primary project has no machine to pick. */} + {showEditor && pyRun && isRemoteProject && projectId && + (project?.compute_hosts?.length ?? 0) > 0 && ( + + )} {showEditor && pyRun && ( { + if (gitStatuses["."] !== "ignored") return gitStatuses; + const m: GitStatusMap = {}; + for (const e of entries) m[e.name] = gitStatuses[e.name] ?? "ignored"; + return m; + }, [gitStatuses, entries]); + // The three on-screen sections in render order (regular, then the collapsible // scaffold + gitignored groups). Shared by the renderer and the selection // click handler so a shift-range spans exactly what the user sees. Mirrors the @@ -567,10 +584,10 @@ export function FileTree({ const isRoot = !relPath && separateScaffold; const nonStandard = isRoot ? entries.filter((e) => !STANDARD_PROJECT_FILES.has(e.name)) : entries; const standard = isRoot ? entries.filter((e) => STANDARD_PROJECT_FILES.has(e.name)) : []; - const regular = nonStandard.filter((e) => gitStatuses[e.name] !== "ignored"); - const gitignored = nonStandard.filter((e) => gitStatuses[e.name] === "ignored"); + const regular = nonStandard.filter((e) => displayStatuses[e.name] !== "ignored"); + const gitignored = nonStandard.filter((e) => displayStatuses[e.name] === "ignored"); return { regular, standard, gitignored }; - }, [entries, relPath, separateScaffold, gitStatuses]); + }, [entries, relPath, separateScaffold, displayStatuses]); // Flat list of visible row paths in on-screen order — the axis a shift-range // spans. Collapsed sections contribute no rows (they aren't rendered). @@ -664,7 +681,13 @@ export function FileTree({ pending.forEach((e) => requestedSizes.current.add(e.path)); const gen = sizeGeneration.current; for (const e of pending) { - invoke("dir_size", { projectDir, relPath: relForEntry(e), excluded: scanExcludedList }) + guardedDirSize(projectDir, () => + invoke("dir_size", { + projectDir, + relPath: relForEntry(e), + excluded: scanExcludedList, + }), + ) .then((bytes) => { if (sizeGeneration.current === gen) setDirSizes((m) => ({ ...m, [e.path]: bytes })); }) @@ -693,22 +716,34 @@ export function FileTree({ pending.forEach((e) => requestedSizes.current.add(e.path)); const gen = sizeGeneration.current; for (const e of pending) { - invoke<{ total: number; ignored: number }>("dir_size_breakdown", { - projectDir, - relPath: relForEntry(e), - excluded: scanExcludedList, - }) + guardedDirSize(projectDir, () => + invoke<{ total: number; ignored: number }>("dir_size_breakdown", { + projectDir, + relPath: relForEntry(e), + excluded: scanExcludedList, + }), + ) .then(({ total, ignored }) => { if (sizeGeneration.current !== gen) return; setDirSizes((m) => ({ ...m, [e.path]: total })); if (ignored > 0) setDirIgnoredBytes((m) => ({ ...m, [e.path]: ignored })); }) - .catch(() => { - // Fall back to the plain total rather than showing no size at all — - // `dir_size_breakdown` can fail for reasons `dir_size` wouldn't (a - // backend that hasn't picked up this new command yet, no `git` on - // PATH), and a folder's size is more important than its ignored split. - invoke("dir_size", { projectDir, relPath: relForEntry(e), excluded: scanExcludedList }) + .catch((err) => { + // The fallback is for a `dir_size_breakdown` that failed for a reason + // `dir_size` wouldn't (a backend that hasn't picked up this newer + // command, no `git` on PATH) — there, a folder's size matters more than + // its ignored split. It is NOT for a host that stopped answering: a + // timed-out or breaker-refused call says nothing about the *command*, + // so retrying with the other one would just queue a second doomed call + // per folder — twice the pile-up the guard exists to prevent. + if (err instanceof DirSizeUnavailable || isHostTimeout(err)) return; + guardedDirSize(projectDir, () => + invoke("dir_size", { + projectDir, + relPath: relForEntry(e), + excluded: scanExcludedList, + }), + ) .then((bytes) => { if (sizeGeneration.current === gen) setDirSizes((m) => ({ ...m, [e.path]: bytes })); }) @@ -2151,6 +2186,11 @@ export function FileTree({ syncSource, scriptPath: entry.path, interp, + // Run on the machine chosen in the RunHostPicker (the same preference the + // Python Run honours), so "pick machine X ⇒ all shells run on X". + runHostPref: projectId + ? useRunHostPrefStore.getState().byProject[projectId] + : undefined, }); if (!plan) { setError(`Run failed: ${entry.path} is outside the current project tree`); @@ -2570,7 +2610,7 @@ export function FileTree({ const { regular, standard, gitignored } = sections; function renderEntry(e: FileEntry, isScaffold = false, isGitignored = false) { - const status = isGitignored ? undefined : gitStatuses[e.name]; + const status = isGitignored ? undefined : displayStatuses[e.name]; const sizeClass = !e.is_dir ? sizeCategory(e.size) : ""; // A folder's headline number is its non-ignored weight — the part // that's actually "yours" — with the full total + ignored split @@ -2997,7 +3037,7 @@ export function FileTree({ ); } - const status = gitStatuses[entry.name]; + const status = displayStatuses[entry.name]; const isTex = !entry.is_dir && entry.extension === ".tex"; const isZip = !entry.is_dir && entry.extension === ".zip"; const entryRel = relForEntry(entry); @@ -3344,7 +3384,7 @@ export function FileTree({ {/* Same rule as the row: no ignored/total split for a folder that is itself ignored — "412 MB of 412 MB ignored" says nothing. */} {tooltip.entry.is_dir - && gitStatuses[tooltip.entry.name] !== "ignored" + && displayStatuses[tooltip.entry.name] !== "ignored" && dirIgnoredBytes[tooltip.entry.path] !== undefined && (
Ignored diff --git a/src/components/files/ProjectFilesView.tsx b/src/components/files/ProjectFilesView.tsx index aabef9b..389ee29 100644 --- a/src/components/files/ProjectFilesView.tsx +++ b/src/components/files/ProjectFilesView.tsx @@ -11,7 +11,6 @@ import { useRemoteBlocked, } from "./ProjectFilesPane"; import { RunHostPicker } from "../tabs/TabLocalityBadges"; -import { runHostPickerApplies } from "../../stores/runHostPref"; import { ProjectFilesSettingsDialog, useProjectFileFilters } from "./ProjectFilesSettings"; import { useImportDrop } from "./importDrop"; import { logoutRemote } from "../../stores/projects"; @@ -789,10 +788,11 @@ export function ProjectFilesView({ {/* Run-host picker — which machine scripts/shells launched from this project run on (primary or a worker), distinct from the source - switch's read side. Shown only while the switch is on Remote (no - machine axis on Local), and hidden once a worker keeps its own synced - code copy — then the run target is the tab's picked locality. */} - {source === "remote" && runHostPickerApplies(project.compute_hosts) && ( + switch's read side. Shown whenever the switch is on Remote (no machine + axis on Local), including a multi-machine project with a synced-code + worker: a Python Run opens a fresh tab, so this project-wide picker is + the only control that can send that run to a worker. */} + {source === "remote" && ( - {source === "remote" && runHostPickerApplies(project.compute_hosts) && ( + {source === "remote" && ( { void refresh(); const onFocus = () => void refresh(); window.addEventListener("focus", onFocus); - return () => window.removeEventListener("focus", onFocus); + const id = window.setInterval(() => void refresh(), 10_000); + return () => { + window.removeEventListener("focus", onFocus); + window.clearInterval(id); + }; }, [refresh]); // The stored `.ovpn` list is only needed once the menu is open — and with it, which diff --git a/src/components/layout/LocalModelMenu.tsx b/src/components/layout/LocalModelMenu.tsx index de45241..e3d3dfd 100644 --- a/src/components/layout/LocalModelMenu.tsx +++ b/src/components/layout/LocalModelMenu.tsx @@ -3,6 +3,7 @@ import { invoke } from "@tauri-apps/api/core"; import { listen } from "@tauri-apps/api/event"; import { useSettingsStore } from "../../stores/settings"; import { useEnergySaver, saverInterval } from "../../stores/power"; +import { useOllamaStatus } from "../../lib/ollamaStatus"; import { formatBytes, gpuAdapterTooltip, @@ -62,7 +63,12 @@ export function LocalModelMenu() { // Three-state Ollama health for the status lamp: "stopped" (server down, red), // "idle" (server up, no model in memory, yellow), "loaded" (a model is loaded // in memory, green). - const [status, setStatus] = useState<"stopped" | "idle" | "loaded">("stopped"); + // Once Ollama is installed, the server's health is polled so the button shows a + // live lamp without the user opening the menu. The poll itself is the app-wide + // shared one (`lib/ollamaStatus`) — it is a machine-wide fact, and the file + // viewer asks the same question per open tab, so a timer here as well meant the + // same `/api/ps` round trip several times over. + const status = useOllamaStatus(installed, saverInterval(5000, energySaver)); const [open, setOpen] = useState(false); // Every installed model (from list_ollama_models_detailed). Resident ones are // selectable as the active local model; the rest can be loaded into memory. @@ -163,27 +169,6 @@ export function LocalModelMenu() { // eslint-disable-next-line react-hooks/exhaustive-deps }, []); - // Once Ollama is installed, poll the server's health so the button can show a - // live stopped/idle/loaded lamp without the user opening the menu. - useEffect(() => { - if (!installed) { - setStatus("stopped"); - return; - } - let cancelled = false; - const check = () => - invoke<"stopped" | "idle" | "loaded">("ollama_status") - .then((s) => { - if (!cancelled) setStatus(s); - }) - .catch(() => {}); - void check(); - const id = window.setInterval(check, saverInterval(5000, energySaver)); - return () => { - cancelled = true; - window.clearInterval(id); - }; - }, [installed, energySaver]); // The GPU's own memory, polled only while the menu is open: the question this // menu raises is "will the next model fit?", which each model's `size_vram` diff --git a/src/lib/dirSizeGuard.ts b/src/lib/dirSizeGuard.ts new file mode 100644 index 0000000..a7d1abc --- /dev/null +++ b/src/lib/dirSizeGuard.ts @@ -0,0 +1,152 @@ +/** + * The guard around the file tree's per-folder size calls. + * + * Folder sizes are a **best-effort display aid** — the tree renders without them + * and simply shows no size when one doesn't arrive. That is what makes them safe + * to bound aggressively, and bounding them matters because of where they run: for + * a remote project `dir_size`/`dir_size_breakdown` are a `du` over SSH, and the + * tree fires **one per visible folder, all at once**. + * + * When the network under a remote project goes away — a dropped VPN tunnel, a + * suspended laptop — those calls do not fail. A black-holed socket produces no + * error, so each one blocks until ssh's own keepalive gives up (~45 s), and each + * blocked call is holding a backend blocking-pool thread the whole time. Opening + * one folder of thirty subfolders then means thirty stalled threads and a UI with + * nothing to render, which is exactly how a lost tunnel turned into a frozen app. + * + * Three limits, each aimed at a different part of that: + * + * - **Timeout** — a size that hasn't arrived in {@link TIMEOUT_MS} is treated as + * one that isn't coming. This does *not* cancel the backend call (a Tauri + * command in flight cannot be recalled); it frees the caller, which is the + * part that was stuck. + * - **Concurrency cap** — at most {@link MAX_CONCURRENT} in flight per project, + * so a wide folder queues instead of dispatching thirty calls into a network + * that is not answering. + * - **Circuit breaker** — {@link FAILURES_TO_OPEN} consecutive timeouts mean the + * host, not the folder, is the problem. The breaker then opens for + * {@link OPEN_MS} and every further request for that project is refused + * *immediately*, so the other twenty-nine folders cost nothing at all rather + * than each paying the timeout in turn. One success closes it again. + * + * Keyed by `projectDir`, because "is this host answering?" is a property of the + * project's remote, not of any one folder. A local project's walk never times out + * in practice, so the breaker stays shut and this is pure overhead-free + * pass-through for it. + */ + +/** Longest a folder size may take before it is written off. */ +export const TIMEOUT_MS = 15_000; +/** Most size calls in flight at once, per project. */ +export const MAX_CONCURRENT = 4; +/** Consecutive timeouts that mean the host is gone, not the folder slow. */ +export const FAILURES_TO_OPEN = 3; +/** How long the breaker stays open before one request is let through to retry. */ +export const OPEN_MS = 30_000; + +/** Thrown when the breaker refuses a call outright. Callers treat it like any + * other failure — the size is simply left unresolved. */ +export class DirSizeUnavailable extends Error { + constructor(reason: string) { + super(reason); + this.name = "DirSizeUnavailable"; + } +} + +interface ProjectGate { + /** Consecutive timeouts since the last success. */ + failures: number; + /** Epoch ms until which the breaker is open; 0 when shut. */ + openUntil: number; + /** Calls currently in flight. */ + active: number; + /** Waiters parked on the concurrency cap, released FIFO. */ + queue: Array<() => void>; +} + +const gates = new Map(); + +function gateFor(projectDir: string): ProjectGate { + let g = gates.get(projectDir); + if (!g) { + g = { failures: 0, openUntil: 0, active: 0, queue: [] }; + gates.set(projectDir, g); + } + return g; +} + +function acquire(g: ProjectGate): Promise { + if (g.active < MAX_CONCURRENT) { + g.active += 1; + return Promise.resolve(); + } + return new Promise((resolve) => g.queue.push(resolve)); +} + +function release(g: ProjectGate): void { + const next = g.queue.shift(); + if (next) { + // Hand the slot straight to the next waiter — `active` never drops. + next(); + return; + } + g.active -= 1; +} + +/** + * Run one folder-size call under the timeout, the concurrency cap and the + * breaker. Rejects (with {@link DirSizeUnavailable} when the breaker is open, or + * a timeout error) rather than hanging; the caller leaves the size unresolved. + * + * `run` is a thunk, not a promise, so a queued call is not *started* until it has + * a slot — passing an already-running promise would defeat the cap. + */ +export async function guardedDirSize(projectDir: string, run: () => Promise): Promise { + const g = gateFor(projectDir); + if (g.openUntil > Date.now()) { + throw new DirSizeUnavailable("folder sizes paused: the project's host is not responding"); + } + await acquire(g); + let timer: number | undefined; + try { + const result = await Promise.race([ + run(), + new Promise((_, reject) => { + timer = window.setTimeout( + () => reject(new Error(`folder size timed out after ${TIMEOUT_MS}ms`)), + TIMEOUT_MS, + ); + }), + ]); + // Any answer at all proves the host is talking — shut the breaker. + g.failures = 0; + g.openUntil = 0; + return result; + } catch (e) { + // Only a *timeout* is evidence about the host. A call that came back with an + // error (an unreadable folder, a backend that lacks the command) answered + // fine, and must not push the project toward a breaker trip. + if (!(e instanceof DirSizeUnavailable) && timer !== undefined && isHostTimeout(e)) { + g.failures += 1; + if (g.failures >= FAILURES_TO_OPEN) g.openUntil = Date.now() + OPEN_MS; + } + throw e; + } finally { + if (timer !== undefined) window.clearTimeout(timer); + release(g); + } +} + +/** Whether a rejection means *the host stopped answering* rather than the call + * itself failing — the distinction that decides whether a retry could ever + * help. Only this guard's own timeout qualifies. */ +export function isHostTimeout(e: unknown): boolean { + return e instanceof Error && e.message.includes("timed out"); +} + +/** Drop a project's gate — on project close, so a stale open breaker can't + * outlive the tree that tripped it. Exported for tests too. */ +export function resetDirSizeGuard(projectDir?: string): void { + if (projectDir === undefined) gates.clear(); + else gates.delete(projectDir); +} diff --git a/src/lib/ollamaStatus.ts b/src/lib/ollamaStatus.ts new file mode 100644 index 0000000..35d9e15 --- /dev/null +++ b/src/lib/ollamaStatus.ts @@ -0,0 +1,122 @@ +import { useEffect, useState } from "react"; +import { invoke } from "@tauri-apps/api/core"; + +/** + * **The** poll of Ollama's `stopped | idle | loaded` state — one timer for the + * whole app, however many components ask. + * + * `ollama_status` is a `GET /api/ps` round trip, and it had two independent + * callers on 5 s timers: the header's 🧠 `LocalModelMenu` lamp, and + * `useLocalModelLoaded` in the file viewer — the second of which is **per + * editable viewer tab**, so the request rate grew with the number of open tabs + * rather than staying a property of the app. The observable result was a steady + * stream of paired `/api/ps` hits a couple of seconds apart, forever, asking the + * same question and getting the same answer. + * + * The state is genuinely global — whether a model is resident in Ollama's memory + * is a fact about the machine, not about a tab — so a single shared poller is not + * just cheaper, it is the more correct model: every surface now flips on the same + * observation instead of each discovering it up to 5 s apart. + * + * Shape: subscribers register an interval they'd be happy with, the timer runs at + * the **shortest** one asked for, and it only exists while at least one component + * is subscribed. A tick that lands while a request is still in flight is skipped + * rather than queued, so a slow or wedged Ollama can never accumulate a backlog. + */ +export type OllamaStatus = "stopped" | "idle" | "loaded"; + +/** Default cadence, matching what each caller polled at individually before. */ +export const DEFAULT_INTERVAL_MS = 5000; + +interface Subscriber { + notify: (s: OllamaStatus) => void; + intervalMs: number; +} + +const subscribers = new Set(); +let timer: number | undefined; +let timerInterval = 0; +let inFlight = false; +/** Last observed value, replayed to a new subscriber so it renders the known + * state immediately instead of flashing `stopped` until the next tick. */ +let current: OllamaStatus = "stopped"; + +async function poll(): Promise { + if (inFlight) return; // a wedged server must not build a queue of retries + inFlight = true; + try { + const next = await invoke("ollama_status"); + current = next; + for (const s of subscribers) s.notify(next); + } catch { + // Unreachable backend — report the same thing an unreachable Ollama does. + current = "stopped"; + for (const s of subscribers) s.notify("stopped"); + } finally { + inFlight = false; + } +} + +/** (Re)start the shared timer at the shortest interval any subscriber wants, or + * stop it entirely when the last one leaves. */ +function retime(): void { + if (subscribers.size === 0) { + if (timer !== undefined) window.clearInterval(timer); + timer = undefined; + timerInterval = 0; + return; + } + const wanted = Math.min(...[...subscribers].map((s) => s.intervalMs)); + if (timer !== undefined && timerInterval === wanted) return; + if (timer !== undefined) window.clearInterval(timer); + timerInterval = wanted; + timer = window.setInterval(() => void poll(), wanted); +} + +/** + * Subscribe to the shared Ollama status. + * + * `enabled` false unsubscribes (and reports `"stopped"`), so a caller gated on + * something else — the menu's "is Ollama even installed?" check — contributes no + * polling at all while it has nothing to show. + */ +export function useOllamaStatus(enabled = true, intervalMs = DEFAULT_INTERVAL_MS): OllamaStatus { + const [status, setStatus] = useState(enabled ? current : "stopped"); + + useEffect(() => { + if (!enabled) { + setStatus("stopped"); + return; + } + let cancelled = false; + const sub: Subscriber = { + notify: (s) => { + if (!cancelled) setStatus(s); + }, + intervalMs, + }; + subscribers.add(sub); + retime(); + // Seat from the last known value, then ask straight away — a component that + // mounts between ticks must not wait a whole interval for its first answer. + setStatus(current); + void poll(); + return () => { + cancelled = true; + subscribers.delete(sub); + retime(); + }; + }, [enabled, intervalMs]); + + return status; +} + +/** Reset the module between tests (no component is subscribed by then). */ +export function resetOllamaStatusPoller(): void { + subscribers.clear(); + if (timer !== undefined) window.clearInterval(timer); + timer = undefined; + timerInterval = 0; + inFlight = false; + current = "stopped"; +} diff --git a/src/lib/pythonRun.ts b/src/lib/pythonRun.ts index 285bc69..9d5b6df 100644 --- a/src/lib/pythonRun.ts +++ b/src/lib/pythonRun.ts @@ -24,8 +24,7 @@ import { invoke } from "@tauri-apps/api/core"; import { basename, dirname } from "./paths"; import { useTabsStore, type TabEntry } from "../stores/tabs"; -import { useProjectsStore } from "../stores/projects"; -import { useRunHostPrefStore, runHostPickerApplies } from "../stores/runHostPref"; +import { useRunHostPrefStore } from "../stores/runHostPref"; /** How a run/debug tab is inserted into the layout. Given the built (keyless) * tab, place it and return the created entry — or null when it streamed the tab @@ -194,15 +193,15 @@ export function openPythonTab(opts: { if (prior) store.removeTab(prior.key); // Which machine to run on: the project's run-host preference (set from the file - // viewer's picker), keyed by the owning project. `scope` is the project id for a - // project tab ("root" has no pref). Honoured only while the picker applies — - // once a worker keeps its own synced code copy the picker is hidden and the run - // target is the tab's picked locality, so we ignore any stale pref and omit - // `location` (⇒ the shell default, the primary on a remote project). - const project = useProjectsStore.getState().projects.find((p) => p.id === scope); - const runHost = runHostPickerApplies(project?.compute_hosts) - ? useRunHostPrefStore.getState().byProject[scope] - : undefined; + // viewer's `RunHostPicker`), keyed by the owning project. `scope` is the project + // id for a project tab ("root" has no pref). Honoured for EVERY remote project, + // including a genuinely multi-machine one with a synced-code worker — a Python + // Run creates a fresh tab, so the per-tab locality badge can't pre-target it and + // the project-wide picker is the only control that can send a run to a worker. + // Unset ⇒ `location` is omitted (⇒ the shell default, the primary). A pref naming + // a since-removed worker is harmless: CenterPanel/`wrap_pty_options` fall the tab + // back to the primary. + const runHost = useRunHostPrefStore.getState().byProject[scope]; const tab: Omit = { label: pyTabLabel(mode, file), cmd: "", // the host's default shell diff --git a/src/lib/shellScriptRun.ts b/src/lib/shellScriptRun.ts index d1b15b4..a87ccc3 100644 --- a/src/lib/shellScriptRun.ts +++ b/src/lib/shellScriptRun.ts @@ -1,5 +1,5 @@ import { relativePathWithin } from "./paths"; -import type { TabLocation } from "../stores/tabs"; +import { isRemoteLocation, type TabLocation } from "../stores/tabs"; import type { ProjectEntry } from "../types"; export type ScriptShell = "bash" | "zsh" | "fish" | "ksh" | "powershell" | "cmd"; @@ -63,24 +63,45 @@ export function scriptRelFromRoot(root: string, absPath: string): string | null * user is browsing. Mount-free remote listings return host paths, while the * tree's `projectDir` is the local state dir; using that dir to relativize a * host path produced `bash ''`. Use the host `remote_path` for remote-source - * rows, and explicitly pin the tab locality to match the selected side. */ + * rows to compute the script's project-relative path. + * + * WHERE the script runs is the project's run-host preference (`runHostPref`, the + * machine chosen in the `RunHostPicker`) when set — so "pick machine X ⇒ shells + * run on X" holds for a script Run exactly as for a Python Run — falling back to + * the browsed side. `scriptRel` is project-relative, so it resolves against the + * chosen host's own project root (the backend re-cds into the target host's + * `remote_path`) or the local mirror, whichever the run location names — the two + * sides mirror the same tree. */ export function shellScriptRunPlan(opts: { project: ProjectEntry | null | undefined; treeRoot: string; syncSource?: "remote" | "local"; scriptPath: string; interp: ScriptShell; + /** The project's run-host preference (`useRunHostPrefStore`), if any. */ + runHostPref?: TabLocation; }): ShellScriptRunPlan | null { const remote = opts.project?.remote; const isRemoteProject = !!remote; - const runsRemote = isRemoteProject && opts.syncSource !== "local"; - const root = runsRemote ? remote.remote_path : opts.treeRoot; - const scriptRel = scriptRelFromRoot(root, opts.scriptPath); + const browsedRemote = isRemoteProject && opts.syncSource !== "local"; + // The script's project-relative path, computed from the side it was browsed on + // (that is the side `scriptPath` belongs to). + const browsedRoot = browsedRemote ? remote.remote_path : opts.treeRoot; + const scriptRel = scriptRelFromRoot(browsedRoot, opts.scriptPath); if (!scriptRel) return null; + // The run location: the chosen machine wins; unset ⇒ the browsed side. + const location: TabLocation | undefined = isRemoteProject + ? (opts.runHostPref ?? (browsedRemote ? "remote" : "local")) + : undefined; + // The tab cwd must match the run side so `scriptRel` resolves: the host project + // root for a remote run (the backend re-cds into the *target* host's remote_path + // anyway), the local mirror for a local run. + const runsRemote = location ? isRemoteLocation(location) : false; + const cwd = runsRemote && remote ? remote.remote_path : opts.treeRoot; return { - cwd: root, + cwd, scriptRel, initialInput: shellRunCommand(opts.interp, scriptRel), - location: isRemoteProject ? (runsRemote ? "remote" : "local") : undefined, + location, }; } diff --git a/src/stores/projects.ts b/src/stores/projects.ts index 5fd8789..9cbcbd5 100644 --- a/src/stores/projects.ts +++ b/src/stores/projects.ts @@ -19,8 +19,10 @@ import { useTabsStore, type SavedLayoutTree, type TabKind, + type TabLocation, type ViewerState, } from "./tabs"; +import { useRunHostPrefStore } from "./runHostPref"; import { type AgentMode } from "../components/tabs/agentModes"; import { useTimerStore } from "./timer"; import { useVpnPromptStore } from "./vpnPrompt"; @@ -690,6 +692,16 @@ export const useProjectsStore = create((set, get) => ({ activeId, rightPanelFolderByProject, }); + // Re-hydrate the run-host preference (which machine shells run on) from each + // project's persisted `run_host`, so a choice made in a previous session still + // sends this session's Run/Debug and "+" shells to that machine. Merge-only, so + // a change made this session before a reload is never clobbered. + useRunHostPrefStore.getState().seed( + projects.map((p) => ({ + projectId: p.id, + location: p.run_host as TabLocation | undefined, + })), + ); // Fire-and-forget: sniff each local repo's `origin` host so pills can badge // a hosting provider (GitHub/GitLab) and the hover can show the git address, // even when the repo was pushed outside Eldrun's Publish flow. Host-only, no diff --git a/src/stores/runHostPref.ts b/src/stores/runHostPref.ts index 7f1aa83..bb0f92b 100644 --- a/src/stores/runHostPref.ts +++ b/src/stores/runHostPref.ts @@ -1,38 +1,58 @@ import { create } from "zustand"; -import type { LocalityHost, TabLocation } from "./tabs"; +import { invoke } from "@tauri-apps/api/core"; +import type { TabLocation } from "./tabs"; /** - * Whether the per-project run-host picker applies at all. It is meant for the - * simple case (a lone primary, or shared-fs workers that are just one shared - * tree). The moment a worker keeps its OWN **synced code copy** the project is - * genuinely multi-machine, and the run target is chosen **per tab** (the locality - * badge) — so the picker is hidden and a Run/Debug falls back to the tab's - * locality (the primary by default) rather than a project-wide preference. - * Shared-fs workers (`shared_fs`) don't trip this — they leave the picker in place. - */ -export function runHostPickerApplies(computeHosts?: LocalityHost[]): boolean { - return !(computeHosts ?? []).some((h) => !h.shared_fs && h.sync_code !== false); -} - -/** - * Ephemeral, non-persisted "which machine should scripts and shells launched - * from this project's file view run on" preference, keyed by project id - * (`docs/multi_host_remote_plan.md`). Set from the file viewer's run-host picker - * (`RunHostPicker`), read at launch time by `lib/pythonRun` so a Run/Debug lands - * on the chosen host (primary or a worker) instead of the shell default. + * "Which machine should scripts and shells launched from this project run on" + * preference, keyed by project id (`docs/multi_host_remote_plan.md`). Set from the + * `RunHostPicker`, read at launch time by `lib/pythonRun`, `lib/shellScriptRun`, + * and `stores/tabs`' new-shell-tab funnel so a Run/Debug or a "+" shell lands on + * the chosen host (primary or a worker) instead of the shell default. + * + * The picker + this preference apply to EVERY remote project, including a + * genuinely multi-machine one with a synced-code worker: a Run opens a fresh tab, + * so the per-tab locality badge can't pre-target it — the project-wide picker is + * the only control that can send a run to a worker. * - * Unset ⇒ the tab's own kind default (a shell defaults to the primary on a remote - * project) — so a project that never touches the picker behaves exactly as before. - * A value is a `TabLocation` (`"local" | "remote" | "host:"`), the same axis a - * tab's locality badge sets, so the run tab carries it verbatim. + * This live store is the read cache; the value is **persisted** per project (in + * `project.json`'s `run_host`, mirrored into `projects.json`) so the choice + * survives a relaunch. `set` writes through to the backend; `seed` re-hydrates the + * cache from the loaded projects on startup (see `stores/projects` `load`). Unset ⇒ + * the tab's own kind default (a shell defaults to the primary on a remote project), + * so a project that never touches the picker behaves exactly as before. A value is + * a `TabLocation` (`"local" | "remote" | "host:"`), the same axis a tab's + * locality badge sets, so the run tab carries it verbatim. */ interface RunHostPrefStore { byProject: Record; + /** Set (and persist) the run host for a project. The store updates immediately; + * the disk write is fire-and-forget (a failure only means it won't survive a + * relaunch — never worth blocking the click or the run). */ set: (projectId: string, location: TabLocation) => void; + /** Re-hydrate the cache from the persisted per-project values on load. Merges + * (never clobbers a choice already made this session), so it is safe to call on + * every projects reload. */ + seed: (entries: { projectId: string; location: TabLocation | undefined }[]) => void; } export const useRunHostPrefStore = create((set) => ({ byProject: {}, - set: (projectId, location) => - set((s) => ({ byProject: { ...s.byProject, [projectId]: location } })), + set: (projectId, location) => { + set((s) => ({ byProject: { ...s.byProject, [projectId]: location } })); + void invoke("set_project_run_host", { projectId, location }).catch((e) => + console.error(`persist run host for ${projectId} failed`, e), + ); + }, + seed: (entries) => + set((s) => { + const byProject = { ...s.byProject }; + for (const { projectId, location } of entries) { + // A choice already made this session wins over the persisted one (they + // can only differ if the user just changed it before this reload). + if (location && byProject[projectId] === undefined) { + byProject[projectId] = location; + } + } + return { byProject }; + }), })); diff --git a/src/stores/tabs.ts b/src/stores/tabs.ts index 7f1fcdb..fa069de 100644 --- a/src/stores/tabs.ts +++ b/src/stores/tabs.ts @@ -9,6 +9,7 @@ import { METRIC, agentMetricLeaf, sub } from "../lib/usageMetrics"; import { useLinkRoutingStore } from "./linkRouting"; import { bumpUsage } from "./usage"; import { newTmuxSessionName } from "../lib/tmuxSession"; +import { useRunHostPrefStore } from "./runHostPref"; /** A shell tab gets a stable persisted tmux session name at creation (TODO #85), * so a persistent remote run reattaches after a relaunch instead of forking a @@ -21,6 +22,24 @@ function withTmuxSession(tab: Omit): Omit { return tab; } +/** A new SHELL tab launched from a project inherits that project's run-host + * preference (the machine chosen in the `RunHostPicker`) as its `location`, so + * "pick machine X ⇒ ALL shells run on X" — the "+" → Shell tab, not just a + * Python/script Run. Scoped tightly: only a `shell` tab, only when it did NOT + * already pin a `location` (a script/tmux/SLURM/git-resolve tab sets its own and + * must keep it), and only when the owning scope has a stored preference (a local + * project, the root scope, and a box scope have none → unchanged). Baked in at + * creation because `effectiveTabLocation`'s per-kind default has no project + * context to consult the preference. */ +function withRunHostDefault( + scope: string, + tab: Omit, +): Omit { + if (tab.kind !== "shell" || tab.location !== undefined) return tab; + const pref = useRunHostPrefStore.getState().byProject[scope]; + return pref ? { ...tab, location: pref } : tab; +} + export type TabKind = | "agent" | "local_agent" @@ -1689,7 +1708,10 @@ export const useTabsStore = create((set, get) => ({ addTab: (tab, opts) => { const key = nextKey(tab.kind); // Spread first so a stray `key` on the payload can't shadow the minted one. - const entry: TabEntry = { ...withTmuxSession(tab), key }; + const entry: TabEntry = { + ...withTmuxSession(withRunHostDefault(get().scope, tab)), + key, + }; if (!opts?.seeded) countTabOpen(get().scope, entry); set((s) => { const { tabs, layout, focusedGroupId } = currentScopeState(s); @@ -1722,7 +1744,11 @@ export const useTabsStore = create((set, get) => ({ addTabToScope: (scope, tab) => { const key = nextKey(tab.kind); - const entry: TabEntry = { ...withTmuxSession(tab), key, scope }; + const entry: TabEntry = { + ...withTmuxSession(withRunHostDefault(scope, tab)), + key, + scope, + }; countTabOpen(scope, entry); set((s) => { const tabs = s.tabsByScope[scope] ?? []; diff --git a/src/stores/vpnStatus.ts b/src/stores/vpnStatus.ts index 5da9ed2..31a3358 100644 --- a/src/stores/vpnStatus.ts +++ b/src/stores/vpnStatus.ts @@ -108,6 +108,7 @@ export const useVpnStatusStore = create((set) => ({ refresh: async () => { const active = await invoke("openvpn_active").catch(() => null); if (!active) return; + let dropped: string[] = []; set((s) => { const byConfig: Record = {}; // What the backend says is up... @@ -117,11 +118,52 @@ export const useVpnStatusStore = create((set) => ({ for (const [config, state] of Object.entries(s.byConfig)) { if (state === "connecting" && !(config in byConfig)) byConfig[config] = state; } + // A tunnel we believed was UP that the backend no longer reports has + // **died on its own** — it was not disconnected from the UI, because every + // deliberate teardown (`disconnectVpnTunnel`, `releaseAll`, `setState off`) + // forgets the config here first, so it is already absent by the time the + // next reconcile runs. Only an unplanned death reaches this branch. + dropped = Object.keys(s.byConfig).filter( + (config) => s.byConfig[config] === "connected" && !(config in byConfig), + ); return { byConfig }; }); + for (const config of dropped) { + onTunnelDropped(config); + // Drop the claims too: the tunnel they were holding no longer exists, so + // leaving them behind would make a later reconnect look already-held. + useVpnStatusStore.getState().releaseAll(config); + } }, })); +/** + * React to a tunnel that died without being asked to (see `refresh`). + * + * The point is not the lamp — it is that **every SSH/SFTP call belonging to a + * project that was riding this tunnel is now aimed at a peer that will never + * answer**. Those calls are synchronous Tauri commands over a pooled + * ControlMaster: nothing about a black-holed socket produces an error, so each + * one blocks until ssh's own keepalive gives up (~45 s), and the file tree + * issues one per visible folder. That is the freeze this exists to prevent — + * clearing the project's status flips `useRemoteBlocked`, so the probes are + * never dispatched in the first place rather than each paying the timeout. + * + * Scoped to the tunnel's **holders**: a project that never claimed this config + * reaches its host by some other route and is none of this tunnel's business. + * + * The backend's pooled session is deliberately NOT torn down here. Reaping it is + * already the pooled reader's job (`services::remote::pooled_sftp_host` evicts a + * child whose ssh keepalive killed it), and a teardown issued from this path + * would itself be a call over the dead connection — the exact thing being + * avoided. + */ +function onTunnelDropped(config: string): void { + const holders = useVpnStatusStore.getState().holders[config] ?? []; + const remote = useRemoteStatusStore.getState(); + for (const projectId of holders) remote.clear(projectId); +} + /** True when at least one tunnel is up or coming up — i.e. the machine's routing * is (or is about to be) Eldrun's doing. */ export function anyVpnLive(byConfig: Record): boolean { diff --git a/src/types/index.ts b/src/types/index.ts index 0eb8546..5dabff2 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -463,6 +463,12 @@ export interface ProjectEntry { * environments auto-detect cannot see (a conda env, a Poetry venv outside the * tree, a second venv). Set from the pill's "Python interpreter…" dialog. */ python_interpreter?: string; + /** Which machine shells launched from this project run on — the persisted + * `RunHostPicker` choice (a `TabLocation`: "local" | "remote" | "host:"). + * Seeds the live `useRunHostPrefStore` on load so the choice survives a + * relaunch. Mirrored from project.json's `run_host` into the entry's flattened + * `extra`. Absent = the shell default (the primary). */ + run_host?: string; /** Denormalized inverse of `ProjectBox.member_ids` (the box this pill is in). */ box_id?: string; /** Per-project git-hosting profile URL that overrides the global one. Mirrored From a2c6d38d6d9ad0651a2288f679014517f87d4023 Mon Sep 17 00:00:00 2001 From: florian Date: Tue, 21 Jul 2026 16:12:28 +0200 Subject: [PATCH 2/2] chore: bump version to v0.1.33 --- package.json | 2 +- src-tauri/Cargo.toml | 2 +- src-tauri/tauri.conf.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index a3727b3..c5b149e 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "eldrun", "private": true, - "version": "0.1.32", + "version": "0.1.33", "license": "MIT OR Apache-2.0", "type": "module", "scripts": { diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index e1c5304..933eaa8 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "eldrun" -version = "0.1.32" +version = "0.1.33" edition = "2021" license = "MIT OR Apache-2.0" diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index a72d31e..412c483 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -1,6 +1,6 @@ { "productName": "Eldrun", - "version": "0.1.32", + "version": "0.1.33", "identifier": "io.github.fseiffarth.eldrun", "build": { "frontendDist": "../dist",