diff --git a/gui/web/src/App.tsx b/gui/web/src/App.tsx index 18b6abc..b5f4aff 100644 --- a/gui/web/src/App.tsx +++ b/gui/web/src/App.tsx @@ -24,6 +24,8 @@ const TABS: { id: TabId; label: string }[] = [ export default function App() { const { machines, machine, devices, device, tab, loadingMachines, init, setMachine, setDevice, setTab } = useStore(); + const removeMachine = useStore((s) => s.removeMachine); + const clearMachines = useStore((s) => s.clearMachines); useEffect(() => { void init(); }, [init]); @@ -40,6 +42,29 @@ export default function App() { } }, [device, machines, devices]); // eslint-disable-line react-hooks/exhaustive-deps + // Deletable = real fetched shots (mock demo machines have nothing on disk), scoped + // to the visible device. Only offer delete controls against a live backend. + const deletable = usingLiveBackend() ? visibleMachines.filter((m) => !m.mock) : []; + + async function onDelete(id: string, label: string) { + if (!window.confirm(`Delete ${label} and all its underlying data? This cannot be undone.`)) return; + try { + await removeMachine(id); + } catch (e) { + window.alert(`Delete failed: ${String(e)}`); + } + } + + async function onClearAll() { + if (!window.confirm(`Delete ALL ${deletable.length} fetched shot(s) and their data? This cannot be undone.`)) + return; + try { + await clearMachines(); + } catch (e) { + window.alert(`Clear all failed: ${String(e)}`); + } + } + return (
@@ -68,7 +93,15 @@ export default function App() {
)}
-

Shot / machine

+
+

Shot / machine

+ {deletable.length > 0 && ( + + )} +
{loadingMachines &&
loading…
} {visibleMachines.map((m) => (
setMachine(m.id)} > -
{m.label}
- {m.note &&
{m.note}
} +
+
{m.label}
+ {m.note &&
{m.note}
} +
+ {!m.mock && usingLiveBackend() && ( + + )}
))}
diff --git a/gui/web/src/lib/api.ts b/gui/web/src/lib/api.ts index c07cdb9..0d5718b 100644 --- a/gui/web/src/lib/api.ts +++ b/gui/web/src/lib/api.ts @@ -19,6 +19,7 @@ export interface MachineInfo { device: string; // "DIII-D" | "NSTX-U" | "synthetic" note?: string; synthetic?: boolean; + mock?: boolean; // true = built-in demo machine (no data on disk, not deletable) } /** List available machines/shots. */ @@ -131,6 +132,21 @@ export async function startFetch(body: FetchBody): Promise<{ job_id: string }> { return res.json(); } +/** Delete one shot's underlying data files from the backend. Requires a live + * backend (there is nothing to delete against the static mock). */ +export async function deleteMachine(shot: string): Promise { + if (!API_BASE) throw new Error("set VITE_API_BASE to manage live data"); + const res = await fetch(`${API_BASE}/api/machines/${shot}`, { method: "DELETE" }); + if (!res.ok) throw new Error(`delete failed (${res.status}): ${await res.text()}`); +} + +/** Delete ALL fetched shot data (the "clear all"). Requires a live backend. */ +export async function deleteAllMachines(): Promise { + if (!API_BASE) throw new Error("set VITE_API_BASE to manage live data"); + const res = await fetch(`${API_BASE}/api/machines`, { method: "DELETE" }); + if (!res.ok) throw new Error(`clear failed (${res.status}): ${await res.text()}`); +} + // ── helpers ── function base(): string { return import.meta.env.BASE_URL; // respects vite `base` for sub-path deploys diff --git a/gui/web/src/store.ts b/gui/web/src/store.ts index 7e40038..62b4cb3 100644 --- a/gui/web/src/store.ts +++ b/gui/web/src/store.ts @@ -3,7 +3,14 @@ // rotating views (VISION §6.4). Views read what they need and render nodes from // the API; heavy data stays in the nodes, not here. import { create } from "zustand"; -import { fetchMachines, fetchDevices, type MachineInfo, type DeviceInfo } from "./lib/api"; +import { + deleteAllMachines, + deleteMachine, + fetchDevices, + fetchMachines, + type DeviceInfo, + type MachineInfo, +} from "./lib/api"; export type TabId = "sensors" | "qs" | "rotating"; export type Theme = "dark" | "light"; @@ -71,6 +78,8 @@ interface State { fontScale: number; init: () => Promise; + removeMachine: (id: string) => Promise; + clearMachines: () => Promise; setMachine: (id: string) => void; setDevice: (id: string) => void; setTab: (t: TabId) => void; @@ -107,6 +116,23 @@ export const useStore = create((set) => ({ device: s.device || defaultDevice, })); }, + // Delete one shot's data, then re-list. If the removed shot was selected, fall + // back to the first remaining machine (or none). + async removeMachine(id) { + await deleteMachine(id); + const machines = await fetchMachines(); + set((s) => ({ + machines, + machine: s.machine === id ? (machines[0]?.id ?? null) : s.machine, + })); + }, + // Delete every fetched shot, then re-list and reselect (the backend falls back + // to demo machines when no real data remains). + async clearMachines() { + await deleteAllMachines(); + const machines = await fetchMachines(); + set({ machines, machine: machines[0]?.id ?? null }); + }, setMachine: (id) => set({ machine: id }), setDevice: (id) => set({ device: id }), setTab: (t) => set({ tab: t }), diff --git a/gui/web/src/theme.css b/gui/web/src/theme.css index 174285a..61e439b 100644 --- a/gui/web/src/theme.css +++ b/gui/web/src/theme.css @@ -137,12 +137,22 @@ code, .mono { font-family: "IBM Plex Mono", ui-monospace, monospace; } .rail-section { padding: 12px 14px; border-bottom: 1px solid var(--border); } .rail-section h3 { margin: 0 0 8px; font-size: 0.769em; text-transform: uppercase; letter-spacing: 1px; color: var(--text-dim); font-weight: 600; } +/* shot-list header: title + clear-all */ +.rail-head { display: flex; align-items: baseline; justify-content: space-between; } +.rail-clear { margin: 0 0 8px; background: none; border: none; cursor: pointer; font-size: 10px; text-transform: uppercase; letter-spacing: 1px; font-weight: 600; color: var(--text-dim); padding: 0; } +.rail-clear:hover { color: var(--danger, #e5484d); } + /* shot selector */ -.machine-item { padding: 7px 9px; border-radius: 4px; cursor: pointer; margin-bottom: 4px; border: 1px solid transparent; } +.machine-item { display: flex; align-items: center; gap: 6px; padding: 7px 9px; border-radius: 4px; cursor: pointer; margin-bottom: 4px; border: 1px solid transparent; } .machine-item:hover { background: var(--panel-2); } .machine-item.active { background: var(--panel-2); border-color: var(--accent); } +.machine-main { flex: 1; min-width: 0; } .machine-item .id { font-family: "IBM Plex Mono", monospace; font-weight: 600; } .machine-item .note { color: var(--text-dim); font-size: 0.846em; } +/* per-shot delete (×): dim until the row is hovered, red on hover */ +.machine-del { flex: none; background: none; border: none; cursor: pointer; color: var(--text-dim); font-size: 16px; line-height: 1; padding: 0 2px; opacity: 0; transition: opacity 0.1s; } +.machine-item:hover .machine-del { opacity: 0.7; } +.machine-del:hover { opacity: 1 !important; color: var(--danger, #e5484d); } /* tabs */ .tabbar { display: flex; gap: 4px; margin-bottom: 14px; border-bottom: 1px solid var(--border); } diff --git a/src/magnetics/data/fetch/remote.py b/src/magnetics/data/fetch/remote.py index fc30220..4131096 100644 --- a/src/magnetics/data/fetch/remote.py +++ b/src/magnetics/data/fetch/remote.py @@ -9,8 +9,8 @@ 2. rsync the fetcher (toksearch_fetch.py + magnetics_signals.py) up; 3. run the fetcher there with the cluster env's interpreter directly -- PTDATA is local and `toksearch_d3d` reads it natively (benchmarked ~5-7x faster than - mdsip), then writes a compact (lzf) .h5; - 4. rsync that .h5 back into the local data/datafile/ dir. + mdsip), then writes a compact (lzf) .h5 to node-local /tmp (off the home quota); + 4. rsync that .h5 back into the local data/datafile/ dir, then delete the /tmp copy. Why this beats the laptop mdsthin pull: only the *compressed* .h5 crosses the tunnel (~80 MB) instead of ~370 MB of raw float over mdsip. Measured end-to-end ~21-24s @@ -59,7 +59,14 @@ # fallback when the device omits a field. Explicit args / CLI flags override both. DEFAULT_HOST = "omega.gat.com" # real cluster host (no ~/.ssh/config alias needed) DEFAULT_JUMP = "cybele.gat.com:2039" # gateway ProxyJump to reach the cluster -DEFAULT_DIR = "~/magnetics_fetch" +DEFAULT_DIR = "~/magnetics_fetch" # code sync dir (small: just the package) +# Where the transient .h5 is staged on the cluster before it is copied back. The +# file is written, rsync'd home, and deleted -- it never needs to persist -- so we +# stage it in node-local /tmp instead of the home dir, whose quota an ~80 MB pull +# (accumulating across runs) otherwise blows ([Errno 122] Disk quota exceeded). +# A per-user suffix (`_`) is appended at run time so pulls from different +# users on the shared login node don't collide -- see run_remote. +DEFAULT_OUT_STAGE = "/tmp/magnetics_out" # Invoke the cluster env's interpreter DIRECTLY -- no `module load` / `conda activate` # (its activate.d scripts set nothing PTDATA needs; direct is ~4-5s faster per pull). DEFAULT_PYTHON = "/fusion/projects/codes/conda/omega/envs_public/toksearch_env/bin/python" @@ -94,6 +101,7 @@ def run_remote( password=None, duo=None, remote_dir=DEFAULT_DIR, + out_stage=None, python=None, tmin=None, tmax=None, @@ -115,6 +123,7 @@ def run_remote( ProxyJump).""" remote_dir = _validate_remote_dir(remote_dir) login = cluster_login(device) + out_stage_base = (out_stage or DEFAULT_OUT_STAGE).rstrip("/") host = host or login["host"] or DEFAULT_HOST port = login["port"] # cluster SSH port (22 unless the device overrides it) python = python or login["python"] or DEFAULT_PYTHON @@ -154,9 +163,23 @@ def run(cmd, **kw): "username / password / Duo)." ) - # 2) ensure the remote dir exists, then rsync the fetcher up. - out_remote = f"{remote_dir}/out" - run([*reuse, target, f"mkdir -p {remote_dir} {out_remote}"], check=True) + # Make the /tmp stage per-user so concurrent pulls on the SHARED login node + # don't collide: the username goes in the top-level dir name (created + # directly in sticky-bit /tmp, so each user owns their own entry) rather + # than a subdir under one shared parent (whose owner others can't write to). + # Prefer the known GA username; else ask the cluster -- `id -un` reads the + # name from the UID, so it works even when $USER/$LOGNAME are unset in this + # non-login ssh command shell (the reason `echo $USER` comes back empty). + stage_user = username + if not stage_user: + who = run([*reuse, target, "id -un"], capture_output=True, text=True) + stage_user = who.stdout.strip() if who.returncode == 0 else "" + out_stage = f"{out_stage_base}_{stage_user}" if stage_user else out_stage_base + + # 2) ensure the code sync dir + the /tmp output stage exist, then rsync the + # fetcher up. The .h5 is staged in out_stage (node-local /tmp), off the + # home quota; only the small package lives under remote_dir. + run([*reuse, target, f"mkdir -p {remote_dir} {out_stage}"], check=True) _log(f"Syncing {PKG_NAME} package → {target}:{remote_dir}/ ...") rsh = f"ssh -o ControlPath={sock}" # rsync reuses the master connection run( @@ -173,8 +196,10 @@ def run(cmd, **kw): check=True, ) - # 3) run the pull on the cluster via the env interpreter directly. - remote_out = f"out/shot_{shot}.h5" + # 3) run the pull on the cluster via the env interpreter directly. --out is + # an ABSOLUTE path in the /tmp stage, so it lands there regardless of the + # `cd {remote_dir}` the inner command does for the package imports. + remote_out = f"{out_stage}/shot_{shot}.h5" fetch = [ python, "-m", @@ -209,15 +234,22 @@ def run(cmd, **kw): if run([*reuse, target, inner]).returncode != 0: sys.exit("remote fetch failed.") - # 4) copy the result back. + # 4) copy the result back (remote_out is already an absolute /tmp path). out_dir = Path(local_out_dir) if local_out_dir else LOCAL_OUT out_dir.mkdir(parents=True, exist_ok=True) local_path = out_dir / f"shot_{shot}.h5" _log(f"Copying result → {local_path} ...") run( - ["rsync", "-az", "-e", rsh, f"{target}:{remote_dir}/{remote_out}", str(local_path)], + ["rsync", "-az", "-e", rsh, f"{target}:{remote_out}", str(local_path)], check=True, ) + # Remove the staged copy now that it's home -- /tmp is roomy but shared, so + # don't leave ~80 MB per pull lying around for the next user/run. + run( + [*reuse, target, f"rm -f {shlex.quote(remote_out)}"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) if progress: progress(1.0, f"done: {local_path.name}") _log(f"Saved {local_path}") diff --git a/src/magnetics/data/h5source.py b/src/magnetics/data/h5source.py index 1aa903a..2f80140 100644 --- a/src/magnetics/data/h5source.py +++ b/src/magnetics/data/h5source.py @@ -54,6 +54,50 @@ def refresh() -> None: _shot_index.cache_clear() +def _shot_of(path: Path) -> str | None: + """The shot id (str) a file belongs to, or None if it isn't a shot file.""" + try: + with h5py.File(path, "r") as h5: + shot = str(int(np.asarray(h5.attrs.get("shot", 0)))) + except Exception: + return None + return shot if shot != "0" else None + + +def delete_shot(shot: str | int) -> list[str]: + """Delete EVERY HDF5 file backing `shot` from the data dir; return the paths + removed. A shot can span more than one file (different windows/fmax, or a + bench artifact), so we match on the stored ``shot`` attr rather than a single + indexed path. Refreshes the index afterward. Empty list = nothing matched.""" + target = str(shot) + removed: list[str] = [] + for path in sorted(data_dir().rglob("*.h5")): + if _shot_of(path) == target: + try: + path.unlink() + removed.append(str(path)) + except OSError: + pass # e.g. already gone / permission — skip, keep clearing the rest + refresh() + return removed + + +def delete_all_shots() -> list[str]: + """Delete every shot HDF5 file in the data dir (the "clear all"); return the + paths removed. Only files that parse as shot files (a non-zero ``shot`` attr) + are touched, so unrelated .h5 are left alone. Refreshes the index afterward.""" + removed: list[str] = [] + for path in sorted(data_dir().rglob("*.h5")): + if _shot_of(path) is not None: + try: + path.unlink() + removed.append(str(path)) + except OSError: + pass + refresh() + return removed + + def shot_file(shot: str | int) -> Path: path = _shot_index().get(str(shot)) if path is None: diff --git a/src/magnetics/service/app.py b/src/magnetics/service/app.py index 2f4a8ad..688f1a8 100644 --- a/src/magnetics/service/app.py +++ b/src/magnetics/service/app.py @@ -74,6 +74,28 @@ def machines(): return real if real else mock.MACHINES +@app.delete("/api/machines/{shot}") +def delete_machine(shot: str) -> dict: + """Delete one shot's underlying HDF5 data and drop its cached analysis state. + Returns the removed file paths + the refreshed machine list. 404 if the shot + has no data files (e.g. a mock machine, which has nothing on disk).""" + removed = h5source.delete_shot(shot) + if not removed: + raise HTTPException(404, f"no data files for shot {shot}") + nodes.refresh() # forget the shot's cached STFT / QS state now its data is gone + return {"shot": shot, "removed": removed, "machines": nodes.machines()} + + +@app.delete("/api/machines") +def delete_all_machines() -> dict: + """Delete ALL fetched shot data (the "clear all") and drop cached state. + Returns the removed paths + the (now real-shot-free) machine list.""" + removed = h5source.delete_all_shots() + nodes.refresh() + real = nodes.machines() + return {"removed": removed, "machines": real if real else mock.MACHINES} + + @app.get("/api/devices") def devices(): """List device configs (data/device/*.json) + their sensor-set names, so the GUI diff --git a/tests/test_delete.py b/tests/test_delete.py new file mode 100644 index 0000000..7684877 --- /dev/null +++ b/tests/test_delete.py @@ -0,0 +1,109 @@ +"""Shot deletion: the h5source removers + the DELETE /api/machines routes. + +Runs against an isolated temp data dir (monkeypatched MAGNETICS_DATA_DIR) so it +never touches the session synthetic shots other tests rely on. Each test clears +the cached index at the end so the restored env is re-globbed cleanly. + +Run: uv run python -m pytest tests/test_delete.py -q +""" + +from __future__ import annotations + +import h5py +import numpy as np +from fastapi.testclient import TestClient + +from magnetics.data import h5source +from magnetics.service import nodes +from magnetics.service.app import app + +client = TestClient(app) + + +def _write_shot(path, shot, nchan=2): + """Minimal shot file: a `shot` attr + a couple of channel groups.""" + with h5py.File(path, "w") as h5: + h5.attrs["shot"] = int(shot) + for i in range(nchan): + g = h5.create_group(f"CH{i}") + g.create_dataset("data", data=np.zeros(4, np.float32)) + g["time"] = np.arange(4, dtype=float) + + +def _use_tmp_data_dir(tmp_path, monkeypatch): + monkeypatch.setenv("MAGNETICS_DATA_DIR", str(tmp_path)) + d = tmp_path / "datafile" + d.mkdir() + h5source.refresh() + return d + + +def test_delete_shot_removes_every_file_for_that_shot(tmp_path, monkeypatch): + d = _use_tmp_data_dir(tmp_path, monkeypatch) + # a shot can span two files (e.g. a normal pull + a bench artifact) + _write_shot(d / "shot_555.h5", 555) + _write_shot(d / "bench_full_555.h5", 555) + _write_shot(d / "shot_777.h5", 777) + + removed = h5source.delete_shot(555) + + assert len(removed) == 2 + assert not (d / "shot_555.h5").exists() + assert not (d / "bench_full_555.h5").exists() + assert (d / "shot_777.h5").exists() # a different shot is untouched + ids = {s["id"] for s in h5source.list_shots()} + assert ids == {"777"} + h5source.refresh() + + +def test_delete_shot_no_match_returns_empty(tmp_path, monkeypatch): + d = _use_tmp_data_dir(tmp_path, monkeypatch) + _write_shot(d / "shot_111.h5", 111) + + assert h5source.delete_shot(999) == [] + assert (d / "shot_111.h5").exists() # nothing deleted on a miss + h5source.refresh() + + +def test_delete_all_shots_clears_every_file(tmp_path, monkeypatch): + d = _use_tmp_data_dir(tmp_path, monkeypatch) + _write_shot(d / "shot_111.h5", 111) + _write_shot(d / "shot_222.h5", 222) + + removed = h5source.delete_all_shots() + + assert len(removed) == 2 + assert h5source.list_shots() == [] + assert not any(d.glob("*.h5")) + h5source.refresh() + + +def test_delete_route_then_404(tmp_path, monkeypatch): + d = _use_tmp_data_dir(tmp_path, monkeypatch) + _write_shot(d / "shot_555.h5", 555) + nodes.refresh() + + r = client.delete("/api/machines/555") + assert r.status_code == 200 + body = r.json() + assert body["removed"] and body["shot"] == "555" + assert not (d / "shot_555.h5").exists() + + # second delete: nothing left to remove → 404 + assert client.delete("/api/machines/555").status_code == 404 + h5source.refresh() + nodes.refresh() + + +def test_delete_all_route(tmp_path, monkeypatch): + d = _use_tmp_data_dir(tmp_path, monkeypatch) + _write_shot(d / "shot_1.h5", 111) + _write_shot(d / "shot_2.h5", 222) + nodes.refresh() + + r = client.delete("/api/machines") + assert r.status_code == 200 + assert len(r.json()["removed"]) == 2 + assert not any(d.glob("*.h5")) + h5source.refresh() + nodes.refresh()