From ad1856c742ba61cac09b3e772f0a9a23d9351a7a Mon Sep 17 00:00:00 2001 From: AuroraLHT Date: Wed, 9 Sep 2026 13:58:16 -0400 Subject: [PATCH 1/2] start_server_host: --with-experiment flag; add scripts/create_api_user.py; document both account types The server-host launcher started monitor/storage/detection/api but never the experiment node, so a notebook or the MCP server driving a growth against the production stack had nothing to talk to. Add an opt-in --with-experiment flag (matching start_simulation.sh) that starts it with the same broker credentials, sequenced after storage/detection and before api, with a preflight check for a missing [experiment] config block. Account creation was asymmetric: broker accounts have scripts/apply_broker_ permissions.py, but the API user store (cfg/users.db) was only reachable via the python -m lumi.api.manage module CLI. Add scripts/create_api_user.py, a thin wrapper over that CLI's create-user kept next to the broker script so both account types are discoverable together; it adds --database for targeting a non-default users.db. README gains an "Accounts" section laying out the two independent credential systems (API vs broker) side by side, with the commands for each. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01UyxfY7uJetWk4KxVSEDTpZ --- README.md | 68 +++++++++++++++++++++++---- scripts/create_api_user.py | 90 ++++++++++++++++++++++++++++++++++++ scripts/start_server_host.sh | 37 +++++++++++++-- 3 files changed, 183 insertions(+), 12 deletions(-) create mode 100755 scripts/create_api_user.py diff --git a/README.md b/README.md index 87580fc..af5d789 100644 --- a/README.md +++ b/README.md @@ -140,7 +140,7 @@ fails, rather than leaving a half-dead stack behind: # On the server machine -- monitor, storage, detection, api. The broker lives here. uv sync --extra api --extra storage --extra detection scripts/install_detection_deps.sh # detection host only -scripts/start_server_host.sh +scripts/start_server_host.sh # add --with-experiment for the notebook / MCP driver # On the instrument machine -- pascal and rheed. --host is required: the broker is # on the other machine, and RabbitMQ refuses `guest` off loopback. @@ -150,12 +150,8 @@ scripts/start_instrument_host.sh --host --user --passwor ``` Start the **server host first**: its storage node declares the exchanges the instrument -host's producers publish into. Give the instrument host a real broker account: - -```bash -uv run python scripts/apply_broker_permissions.py --host \ - --user lumi-node --role node --password -``` +host's producers publish into. Give the instrument host a real broker account (see +[Accounts](#accounts) below). `start_server_host.sh` writes to the real HDF5 root and the real user database, and refuses to start with `auth.enabled = false` or the placeholder signing key @@ -196,6 +192,62 @@ The API server reads its broker URL from `LUMI_AMQP_URL` first, falling back to LUMI_AMQP_URL="amqp://guest:guest@:5672/" python nodes/api.py ``` +### Accounts + +There are **two separate credential systems**, and one does not imply the other: + +| | who authenticates | where it lives | created with | +| --- | --- | --- | --- | +| **API account** | the browser console, and anything hitting `POST /auth/login` (which mints the JWT the `/ws` bridge and the MCP HTTP transport check) | SQLite at `auth.database_path` (default `cfg/users.db`) | `python -m lumi.api.manage` (or `scripts/create_api_user.py`) | +| **Broker account** | every node, and a notebook that calls `ExperimentSession.open()` (it connects straight to RabbitMQ, no API in the path) | RabbitMQ's own user list | `scripts/apply_broker_permissions.py` | + +Both take a role of `viewer` / `operator` / `admin` (the broker script also has `node`, +for equipment processes), and the same `lumi.contracts.policy` predicate gates the broker +and the `/ws` bridge, so a role means the same thing on either side. But the accounts are +independent: an `hliang16` in `cfg/users.db` is not an `hliang16` on the broker, and the +passwords need not match. + +**API account** — for logging into the web console, or for the notebook when it goes +through the API rather than the bus. `python -m lumi.api.manage` is the account CLI, and +covers the whole lifecycle: + +```bash +python -m lumi.api.manage create-user hliang16 --role admin # prompts for a password +python -m lumi.api.manage list-users +python -m lumi.api.manage set-password +python -m lumi.api.manage gen-secret # a fresh JWT signing key +``` + +`scripts/create_api_user.py` is a thin wrapper over the same `create-user`, kept next to +`apply_broker_permissions.py` so the two account types are found together. It adds one +thing the module CLI lacks — `--database `, to target a users.db other than the +configured one (the simulation stack's is under `run/simulation/`): + +```bash +uv run python scripts/create_api_user.py agent --role operator --password +uv run python scripts/create_api_user.py alice --admin --database run/simulation/users.db +``` + +**Broker account** — for a node, or a notebook/MCP session that talks to the bus +directly. The permissions are derived from `src/lumi/contracts`, so re-run it after any +contract change: + +```bash +# a node account for the instrument host +uv run python scripts/apply_broker_permissions.py --host \ + --user lumi-node --role node --password + +# an operator account for a notebook driving a growth +uv run python scripts/apply_broker_permissions.py --host \ + --user hliang16 --role operator --password +``` + +It needs the management plugin (`rabbitmq-plugins enable rabbitmq_management`) and an +admin broker login to authenticate with (`--admin-user` / `--admin-pass`, default +`guest`/`guest`, which only works from the broker host itself). `--dry-run` prints the +permissions the role would get without applying them. Once real accounts exist, delete +`guest`. + ## Driving the simulated chamber Two demo scripts, at two different layers. Both need the stack already running, and both @@ -371,7 +423,7 @@ gated by `auth.enabled`, so a dev-mode bypass never opens real equipment control network. Viewer tokens are refused at the door. ```bash -uv run python -m lumi.api.manage create-user agent --role operator +uv run python scripts/create_api_user.py agent --role operator # an API account; see Accounts ``` It serves plain HTTP; put nginx/Caddy in front for TLS (`docs/TODO.md`). `--bind-host` diff --git a/scripts/create_api_user.py b/scripts/create_api_user.py new file mode 100755 index 0000000..a75e5b0 --- /dev/null +++ b/scripts/create_api_user.py @@ -0,0 +1,90 @@ +"""Create an account in the API server's user database. + + uv run python scripts/create_api_user.py hliang16 --role admin + uv run python scripts/create_api_user.py agent --role operator --password + +The account CLI is `python -m lumi.api.manage` (create-user / list-users / +set-password / gen-secret). This script is a thin wrapper over its `create-user`, +kept in scripts/ next to `apply_broker_permissions.py` so the two account types are +found together; it adds `--database` for targeting a users.db other than the +configured one. Reach for the module CLI for anything past creating a user. + +This is the login the browser console and the `/auth/login` endpoint check; it is +stored in the SQLite file at `auth.database_path` (default `cfg/users.db`) and is +**separate** from a RabbitMQ account -- see `scripts/apply_broker_permissions.py` +for those. A notebook that connects straight to the broker needs a broker account, +not one made here. + +The password is prompted for (not echoed) unless `--password` is given. Roles are +viewer / operator / admin; `is_admin` is derived from `role == "admin"`. +""" + +from __future__ import annotations + +import argparse +import asyncio +import getpass +import sys +from pathlib import Path + +from lumi.api.db import USER_ROLES, UserStore +from lumi.path import PROJECT_ROOT + + +async def create(args: argparse.Namespace) -> int: + path: Path | None = None + if args.database: + path = Path(args.database) + if not path.is_absolute(): + path = PROJECT_ROOT / path + + store = UserStore(path) + await store.connect() + try: + if await store.get_user_by_username(args.username) is not None: + print(f"error: user '{args.username}' already exists", file=sys.stderr) + return 1 + + password = args.password or getpass.getpass(f"Password for {args.username}: ") + if not password: + print("error: password may not be empty", file=sys.stderr) + return 1 + + role = "admin" if args.admin else args.role + user = await store.create_user( + username=args.username, + password=password, + full_name=args.full_name, + role=role, + ) + print(f"created {user['role']} '{user['username']}' (id={user['id']}) in {store.path}") + return 0 + finally: + await store.close() + + +def main() -> int: + p = argparse.ArgumentParser( + prog="create_api_user", + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + p.add_argument("username") + p.add_argument("--password", default=None, help="prompted for (hidden) if omitted") + p.add_argument("--full-name", default="") + p.add_argument( + "--role", choices=sorted(USER_ROLES), default="viewer", help="default: viewer" + ) + p.add_argument("--admin", action="store_true", help="shorthand for --role admin") + p.add_argument( + "--database", + default=None, + help="user database path (default: auth.database_path from settings, " + "usually cfg/users.db); relative paths are under the repo root", + ) + args = p.parse_args() + return asyncio.run(create(args)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/start_server_host.sh b/scripts/start_server_host.sh index c4fe359..fb66048 100755 --- a/scripts/start_server_host.sh +++ b/scripts/start_server_host.sh @@ -5,6 +5,7 @@ # monitor presence registry (see --no-monitor) # storage HDF5 recorder, and the node that declares the exchanges # detection RHEED spot detection (see --no-detection) +# experiment PLD growth driver -- off by default (see --with-experiment) # api FastAPI + the /ws bridge the browser connects to # # The instrument-side half (pascal, rheed) runs on the other machine -- @@ -14,7 +15,7 @@ # Usage: # scripts/start_server_host.sh [--host HOST] [--user U] [--password P] # [--root DIR] [--workers N] -# [--no-detection] [--no-monitor] +# [--no-detection] [--no-monitor] [--with-experiment] # [--allow-insecure-auth] [--check] # # --host broker host (default localhost -- the broker runs here) @@ -22,6 +23,10 @@ # over loopback, which is exactly the case this script defaults to) # --root HDF5 output directory (default storage.hdf5_recorder.database_path) # --workers uvicorn workers for the api node (default: api.workers in settings) +# --with-experiment +# also start the experiment node -- the PLD growth driver the +# notebooks and the MCP server talk to. Off by default: the browser +# stack does not need it. # --check run the preflight checks and exit without starting anything # # Unlike start_simulation.sh this writes to the REAL HDF5 root and the REAL user @@ -50,6 +55,7 @@ STORAGE_ROOT="" API_WORKERS="" WITH_DETECTION=1 WITH_MONITOR=1 +WITH_EXPERIMENT=0 ALLOW_INSECURE_AUTH=0 CHECK_ONLY=0 @@ -62,9 +68,10 @@ while [[ $# -gt 0 ]]; do --workers) API_WORKERS="$2"; shift 2 ;; --no-detection) WITH_DETECTION=0; shift ;; --no-monitor) WITH_MONITOR=0; shift ;; + --with-experiment) WITH_EXPERIMENT=1; shift ;; --allow-insecure-auth) ALLOW_INSECURE_AUTH=1; shift ;; --check) CHECK_ONLY=1; shift ;; - -h|--help) sed -n '3,29p' "${BASH_SOURCE[0]}"; exit 0 ;; + -h|--help) sed -n '3,34p' "${BASH_SOURCE[0]}"; exit 0 ;; *) echo "unknown option: $1" >&2; exit 1 ;; esac done @@ -291,6 +298,20 @@ PY fi fi +# ---- experiment ----------------------------------------------------------- +# The node's only extra dep is aiosqlite, already checked above. What is worth +# surfacing here is a config file with no [experiment] block: the node would +# otherwise die on a KeyError deep in its own log. +if [[ $WITH_EXPERIMENT -eq 1 ]]; then + if "$PYTHON" -c 'from lumi.config import settings; settings.experiment.pld_config; settings.experiment.bounds' 2>/dev/null; then + ok "experiment config present ([experiment] in cfg/settings.toml)" + else + fail "experiment: no usable [experiment] block in the config -- cannot start the node." + echo " Copy the [experiment] section from cfg/settings.example.toml, or drop" >&2 + echo " --with-experiment." >&2 + fi +fi + if [[ $FAILED -ne 0 ]]; then echo >&2 echo "preflight failed -- nothing started." >&2 @@ -361,6 +382,13 @@ if [[ $WITH_DETECTION -eq 1 ]]; then start_node detection --host "$RABBITMQ_HOST" --user "$BROKER_USER" --password "$BROKER_PASS" fi +# The growth driver the notebooks and the MCP server talk to. A consumer of +# chamber/rheed/storage, like storage is of rheed/chamber -- it needs them already +# up, which they are by here. Off unless asked for: the browser stack does not use it. +if [[ $WITH_EXPERIMENT -eq 1 ]]; then + start_node experiment --host "$RABBITMQ_HOST" --user "$BROKER_USER" --password "$BROKER_PASS" +fi + # The bridge last, so the browser only reaches a stack whose consumers are up. sleep 2 start_node api @@ -370,8 +398,9 @@ echo "nodes running (Ctrl-C to stop):" for i in "${!NAMES[@]}"; do printf " %-10s pid %s\n" "${NAMES[$i]}" "${PIDS[$i]}" done -[[ $WITH_MONITOR -eq 0 ]] && echo " monitor skipped (--no-monitor: the UI's presence panel stays empty)" -[[ $WITH_DETECTION -eq 0 ]] && echo " detection skipped (--no-detection)" +[[ $WITH_MONITOR -eq 0 ]] && echo " monitor skipped (--no-monitor: the UI's presence panel stays empty)" +[[ $WITH_DETECTION -eq 0 ]] && echo " detection skipped (--no-detection)" +[[ $WITH_EXPERIMENT -eq 0 ]] && echo " experiment skipped (pass --with-experiment for the notebook / MCP driver)" echo API_IP="$(hostname -I 2>/dev/null | awk '{print $1}')" echo "API on http://${API_IP:-}:8000 -- contract $CONTRACT_HASH" From ef3de87c98f2281828ee3ce9d884ec555479b986 Mon Sep 17 00:00:00 2001 From: AuroraLHT Date: Wed, 9 Sep 2026 14:40:21 -0400 Subject: [PATCH 2/2] experiment: let the MI runner accept PASCAL's spurious aborts The PASCAL firmware writes the `Aborted_*` assist file for some MI scripts that actually ran to completion, so `MiCommandRunner.execute` raising MIExecutionFailed on `is_aborted` fails ops whose commands the controller did carry out. A multi-command op like `to_temperature` (Temperature Control / Temperature Ramp / Temperature Set) dies on the first spurious abort and never sends the setpoint. Add `raise_on_abort` (default True -- behaviour unchanged): a constructor arg on MiCommandRunner and a per-call override on execute(), mirroring the existing `timeout` option. When off, an aborted execution is returned with a WARNING instead of raising; `is_stopped` (a deliberate `$stop`) still always raises. Wire it through nodes/experiment.py as `--ignore-mi-abort` and `experiment.mi_ignore_abort` (default false). The physical result must be verified against the chamber log whenever this is on. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01UyxfY7uJetWk4KxVSEDTpZ --- cfg/settings.example.toml | 10 ++++++++ nodes/experiment.py | 13 +++++++++- src/lumi/experiment/mi.py | 34 ++++++++++++++++++++++--- tests/experiment/test_mi_runner.py | 40 ++++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 4 deletions(-) diff --git a/cfg/settings.example.toml b/cfg/settings.example.toml index df32327..366045f 100644 --- a/cfg/settings.example.toml +++ b/cfg/settings.example.toml @@ -502,6 +502,16 @@ growth_db_path = "cfg/growth.db" # the node merely stops listening. mi_command_timeout = 0 +# Fail an op when the chamber reports its MI script `aborted`? Default true. The PASCAL +# firmware writes the `Aborted_*` assist file for some scripts that actually ran to +# completion (a known bug), so an op like `to_temperature` -- which issues several MI +# commands in sequence -- can raise on a spurious abort of an early command and never +# send the rest. Set this true only when the caller verifies the physical result another +# way (the chamber log); an aborted script then looks like it succeeded. A deliberate +# `$stop` still fails the op regardless. Override per run with +# `nodes/experiment.py --ignore-mi-abort`. +mi_ignore_abort = false + # PLDChamberConfiguration -- the chamber geometry a growth is planned against. # Previously copy-pasted into each notebook run and drifted with every physical # realignment; one source of truth now. diff --git a/nodes/experiment.py b/nodes/experiment.py index ab5b045..67cdb9f 100644 --- a/nodes/experiment.py +++ b/nodes/experiment.py @@ -117,8 +117,14 @@ async def main(args: argparse.Namespace) -> None: if mi_timeout <= 0: mi_timeout = None log.warning("MI completion waits are unbounded -- a lost update will hang the op that is waiting") + raise_on_abort = not (args.ignore_mi_abort or bool(settings.get("experiment.mi_ignore_abort", False))) + if not raise_on_abort: + log.warning( + "MI abort reports are being ignored (--ignore-mi-abort / experiment.mi_ignore_abort) -- " + "an aborted script will not fail its op; confirm the physical result against the chamber log" + ) sources = { - "chamber_mi": MiCommandRunner(chamber_mi_client, timeout=mi_timeout), + "chamber_mi": MiCommandRunner(chamber_mi_client, timeout=mi_timeout, raise_on_abort=raise_on_abort), "chamber_log": ChamberLogClient(channel, chamber_x), "chamber_config": ChamberConfigClient(channel, chamber_x), "rheed_camera": RheedCameraClient(channel, rheed_x), @@ -170,6 +176,11 @@ def cli() -> None: parser.add_argument("--mi-timeout", type=float, default=None, help="seconds to wait for an MI script to finish; 0 = wait forever " "(defaults to experiment.mi_command_timeout)") + parser.add_argument("--ignore-mi-abort", action="store_true", + help="do not fail an op when the chamber reports its MI script " + "aborted -- PASCAL raises spurious aborts for scripts that " + "ran. Verify results against the chamber log. Defaults from " + "experiment.mi_ignore_abort.") parser.add_argument("-v", "--verbose", action="store_true") args = parser.parse_args() diff --git a/src/lumi/experiment/mi.py b/src/lumi/experiment/mi.py index d4f5cf6..98a124b 100644 --- a/src/lumi/experiment/mi.py +++ b/src/lumi/experiment/mi.py @@ -73,9 +73,22 @@ class MiCommandRunner: So it is left bounded by default, and the caller opts in. See docs/TODO.md. """ - def __init__(self, client: ChamberMiModeClient, *, timeout: float | None = 30.0) -> None: + def __init__( + self, + client: ChamberMiModeClient, + *, + timeout: float | None = 30.0, + raise_on_abort: bool = True, + ) -> None: self.client = client self.timeout = timeout + #: Whether an execution the chamber reports as `is_aborted` raises + #: MIExecutionFailed. Default on. The PASCAL firmware writes the + #: `Aborted_*` assist file for scripts that actually ran to completion + #: (a known firmware bug), so a caller that verifies the physical result + #: another way -- the chamber log -- can turn this off, runner-wide here + #: or per call. `is_stopped` (a deliberate `$stop`) always raises. + self.raise_on_abort = raise_on_abort self._pending: dict[str, asyncio.Future[MIExecution]] = {} self._subscribed = False @@ -117,16 +130,22 @@ async def execute( commands: str | PascalCommand | PascalScope, *, timeout: float | None | object = _DEFAULT, + raise_on_abort: bool | object = _DEFAULT, ) -> MIExecution: """Submit a command script and wait for it to finish. `timeout` defaults to this runner's; pass a number to override it for one call, or `None` to wait indefinitely (see the class docstring for what that gives up). + `raise_on_abort` defaults to this runner's; pass `False` to accept an execution + the chamber reports as aborted (PASCAL raises spurious aborts -- verify the + physical result against the chamber log when you do this). Raises TimeoutError if the chamber does not report completion in time, - MIExecutionFailed if it reports one that was aborted or stopped. + MIExecutionFailed if it reports one that was stopped, or aborted while + `raise_on_abort` is on. """ deadline = self.timeout if timeout is _DEFAULT else timeout assert deadline is None or isinstance(deadline, (int, float)) + check_abort = self.raise_on_abort if raise_on_abort is _DEFAULT else bool(raise_on_abort) await self.start() text = commands.to_text() if hasattr(commands, "to_text") else str(commands) @@ -148,6 +167,15 @@ async def execute( finally: self._pending.pop(commands_uuid, None) - if execution.is_aborted or execution.is_stopped: + if execution.is_stopped: raise MIExecutionFailed(execution) + if execution.is_aborted: + if check_abort: + raise MIExecutionFailed(execution) + log.warning( + "MI execution %s reported aborted; continuing because raise_on_abort is " + "off -- PASCAL reports spurious aborts for scripts that ran, so confirm " + "the physical result against the chamber log: %s", + execution.commands_uuid, brief(text), + ) return execution diff --git a/tests/experiment/test_mi_runner.py b/tests/experiment/test_mi_runner.py index c3b41e2..94d97e8 100644 --- a/tests/experiment/test_mi_runner.py +++ b/tests/experiment/test_mi_runner.py @@ -141,6 +141,46 @@ async def test_an_unbounded_wait_still_fails_on_an_aborted_execution() -> None: await asyncio.wait_for(task, 1.0) +async def test_raise_on_abort_false_returns_the_aborted_execution(caplog) -> None: + """PASCAL reports spurious aborts; a caller that verifies the result another way can + opt out per call and get the execution back instead of an exception.""" + client = FakeMiClient() + runner = MiCommandRunner(client, timeout=None) + + task = asyncio.create_task(runner.execute("Temperature Ramp 20.0\n", raise_on_abort=False)) + await asyncio.sleep(0.01) + with caplog.at_level(logging.WARNING, logger="lumi.experiment.mi"): + await client.complete(aborted=True) + execution = await asyncio.wait_for(task, 1.0) + + assert execution.is_aborted + assert any("reported aborted" in r.getMessage() for r in caplog.records) + + +async def test_raise_on_abort_can_be_off_runner_wide() -> None: + client = FakeMiClient() + runner = MiCommandRunner(client, timeout=None, raise_on_abort=False) + + task = asyncio.create_task(runner.execute("Temperature Ramp 20.0\n")) + await asyncio.sleep(0.01) + await client.complete(aborted=True) + assert (await asyncio.wait_for(task, 1.0)).is_aborted + assert runner._pending == {} + + +async def test_a_stopped_execution_still_raises_with_raise_on_abort_off() -> None: + """`raise_on_abort=False` covers spurious aborts only -- a deliberate `$stop` is a + real cancellation and must still fail the op.""" + client = FakeMiClient() + runner = MiCommandRunner(client, timeout=None, raise_on_abort=False) + + task = asyncio.create_task(runner.execute("Trigger Laser N=3000 (0) F=10.0\n")) + await asyncio.sleep(0.01) + await client.complete(stopped=True) + with pytest.raises(MIExecutionFailed): + await asyncio.wait_for(task, 1.0) + + async def test_a_special_command_never_waits() -> None: """`$stop`/`$clean` register no execution, so there is nothing to wait for -- with or without a deadline."""