From 5c45d5470de69addb4810849e8145598bdcf4c85 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Sun, 30 Aug 2026 21:59:45 +0200 Subject: [PATCH 001/182] Bound heavy AI workload concurrency (#7) Introduce a process-wide bounded scheduler for memory-heavy ClosedRoom jobs, with deterministic queue capacity, cancellation, shutdown, and metrics. Validated by STRONG remote preflight on exact PR head. --- src/local_asr_server/analysis_jobs.py | 36 +++- src/local_asr_server/runtime/leases.py | 29 ++- .../runtime/workload_arbiter.py | 200 ++++++++++++++++++ src/local_asr_server/server.py | 10 +- src/local_asr_server/transcription_jobs.py | 53 ++++- test/test_workload_arbiter.py | 140 ++++++++++++ 6 files changed, 449 insertions(+), 19 deletions(-) create mode 100644 src/local_asr_server/runtime/workload_arbiter.py create mode 100644 test/test_workload_arbiter.py diff --git a/src/local_asr_server/analysis_jobs.py b/src/local_asr_server/analysis_jobs.py index 4c93c972..68098949 100644 --- a/src/local_asr_server/analysis_jobs.py +++ b/src/local_asr_server/analysis_jobs.py @@ -22,6 +22,11 @@ from local_asr_server.settings import load_settings from local_asr_server.llm import DEFAULT_GEMINI_MODEL from local_asr_server.runtime.models import resolve_local_llm_model_path +from local_asr_server.runtime.workload_arbiter import ( + HeavyWorkloadArbiter, + WorkloadArbiterClosed, + WorkloadQueueFull, +) ANALYSIS_JOB_TYPE = "analysis" @@ -29,9 +34,16 @@ class AnalysisJobManager: """Runs analysis workflows as persistent jobs backed by JobStore.""" - def __init__(self, services: AppServices, store: JobStore) -> None: + def __init__( + self, + services: AppServices, + store: JobStore, + *, + arbiter: HeavyWorkloadArbiter | None = None, + ) -> None: self._services = services self._store = store + self._arbiter = arbiter def create(self, body: AnalysisRequest) -> dict[str, Any]: body = self._with_recording_transcription(body) @@ -89,11 +101,29 @@ def create(self, body: AnalysisRequest) -> dict[str, Any]: } ) - threading.Thread(target=self._run, args=(job_id, run_id, body), daemon=True).start() + if self._arbiter is None: + threading.Thread(target=self._run, args=(job_id, run_id, body), daemon=True).start() + return { + "job_id": job_id, + "analysis_run_id": run_id, + "status": "queued", + } + + try: + self._arbiter.submit( + task_id=job_id, + workload_type=ANALYSIS_JOB_TYPE, + run=lambda: self._run(job_id, run_id, body), + on_cancel=lambda _reason: self._mark_cancelled(job_id, run_id), + ) + status = "queued" + except (WorkloadQueueFull, WorkloadArbiterClosed) as exc: + self._mark_failed(job_id, run_id, str(exc)) + status = "failed" return { "job_id": job_id, "analysis_run_id": run_id, - "status": "queued", + "status": status, } def create_pipeline(self, body: AnalysisPipelineRequest) -> dict[str, Any]: diff --git a/src/local_asr_server/runtime/leases.py b/src/local_asr_server/runtime/leases.py index 03c5fe96..832f9ef0 100644 --- a/src/local_asr_server/runtime/leases.py +++ b/src/local_asr_server/runtime/leases.py @@ -8,7 +8,13 @@ class ModelRuntimeLeaseManager: - """Manages mutual exclusion of memory-heavy ML workloads on Apple Silicon.""" + """Coordinates model phase transitions; it is not a workload scheduler. + + Global admission, queue bounds and mutual exclusion belong to + ``HeavyWorkloadArbiter``. This compatibility hook only records the current + model phase and asks the runtime service manager to release the LLM/VLM + sidecar before ASR/diarization phases when appropriate. + """ _lock = Lock() _active_lease: str | None = None @@ -20,28 +26,29 @@ def set_service_manager(cls, manager: Any) -> None: @classmethod def acquire_lease(cls, lease_type: str) -> None: - """Acquire lease for a specific heavy ML execution type (asr, diarization, vision, llm).""" + """Activate a model phase hook (asr, diarization, vision, llm).""" with cls._lock: - logger.info("[Model Lease] Requesting lease for: %s (current: %s)", lease_type, cls._active_lease) - - # If the requested lease is ASR or Diarization, and the VLM/LLM sidecar is running, - # we shut it down to free up memory before running the new workload. + logger.info("[Model Phase] Activating: %s (current: %s)", lease_type, cls._active_lease) + if lease_type in ("asr", "diarization"): if cls._service_manager: status = cls._service_manager.llm_status() if status.get("status") in ("ready", "running"): - logger.info("[Model Lease] Stopping VLM/LLM sidecar to free up unified memory for %s", lease_type) + logger.info( + "[Model Phase] Stopping VLM/LLM sidecar to free unified memory for %s", + lease_type, + ) try: cls._service_manager.stop_llm() except Exception as e: - logger.warning("[Model Lease] Failed to stop LLM sidecar: %s", e) - + logger.warning("[Model Phase] Failed to stop LLM sidecar: %s", e) + cls._active_lease = lease_type @classmethod def release_lease(cls, lease_type: str) -> None: - """Release the acquired lease.""" + """Clear the phase marker when it still belongs to the caller.""" with cls._lock: if cls._active_lease == lease_type: - logger.info("[Model Lease] Released lease for: %s", lease_type) + logger.info("[Model Phase] Released: %s", lease_type) cls._active_lease = None diff --git a/src/local_asr_server/runtime/workload_arbiter.py b/src/local_asr_server/runtime/workload_arbiter.py new file mode 100644 index 00000000..9b1be326 --- /dev/null +++ b/src/local_asr_server/runtime/workload_arbiter.py @@ -0,0 +1,200 @@ +from __future__ import annotations + +import os +import queue +import threading +import time +from dataclasses import dataclass +from typing import Callable + + +class WorkloadQueueFull(RuntimeError): + """Raised when the bounded heavy-workload queue cannot admit more work.""" + + +class WorkloadArbiterClosed(RuntimeError): + """Raised when work is submitted after shutdown has started.""" + + +@dataclass(frozen=True, slots=True) +class _WorkItem: + task_id: str + workload_type: str + run: Callable[[], None] + on_cancel: Callable[[str], None] | None = None + + +class HeavyWorkloadArbiter: + """Bounded process-wide scheduler for memory-heavy ClosedRoom jobs. + + ClosedRoom owns cross-workload scheduling here. Model-level request admission, + residency and eviction remain owned by local-llm-server. The default of one + active heavy workload protects Apple unified memory until representative + hardware evidence justifies a higher profile. + """ + + DEFAULT_MAX_CONCURRENT = 1 + DEFAULT_QUEUE_CAPACITY = 8 + + def __init__(self, *, max_concurrent: int = DEFAULT_MAX_CONCURRENT, queue_capacity: int = DEFAULT_QUEUE_CAPACITY) -> None: + if max_concurrent < 1: + raise ValueError("max_concurrent must be >= 1") + if queue_capacity < 1: + raise ValueError("queue_capacity must be >= 1") + self.max_concurrent = max_concurrent + self.queue_capacity = queue_capacity + self._queue: queue.Queue[_WorkItem] = queue.Queue(maxsize=queue_capacity) + self._lock = threading.RLock() + self._pending: dict[str, str] = {} + self._active: dict[str, str] = {} + self._cancelled: set[str] = set() + self._closed = False + self._submitted = 0 + self._completed = 0 + self._failed = 0 + self._rejected = 0 + self._cancelled_pending = 0 + self._workers = [ + threading.Thread( + target=self._worker, + name=f"closedroom-heavy-{index + 1}", + daemon=True, + ) + for index in range(max_concurrent) + ] + for worker in self._workers: + worker.start() + + @classmethod + def from_env(cls) -> "HeavyWorkloadArbiter": + return cls( + max_concurrent=_positive_env_int( + "CLOSEDROOM_HEAVY_WORKLOAD_CONCURRENCY", + cls.DEFAULT_MAX_CONCURRENT, + ), + queue_capacity=_positive_env_int( + "CLOSEDROOM_HEAVY_WORKLOAD_QUEUE_CAPACITY", + cls.DEFAULT_QUEUE_CAPACITY, + ), + ) + + def submit( + self, + *, + task_id: str, + workload_type: str, + run: Callable[[], None], + on_cancel: Callable[[str], None] | None = None, + ) -> None: + if not task_id.strip(): + raise ValueError("task_id must be non-empty") + if not workload_type.strip(): + raise ValueError("workload_type must be non-empty") + item = _WorkItem(task_id=task_id, workload_type=workload_type, run=run, on_cancel=on_cancel) + with self._lock: + if self._closed: + raise WorkloadArbiterClosed("heavy-workload arbiter is shutting down") + if task_id in self._pending or task_id in self._active: + raise ValueError(f"task is already scheduled: {task_id}") + try: + self._queue.put_nowait(item) + except queue.Full as exc: + self._rejected += 1 + raise WorkloadQueueFull( + f"heavy-workload queue is full ({self.queue_capacity} pending); retry later" + ) from exc + self._pending[task_id] = workload_type + self._submitted += 1 + + def cancel_pending(self, task_id: str) -> bool: + """Mark queued work for cancellation without interrupting active execution.""" + with self._lock: + if task_id not in self._pending: + return False + self._cancelled.add(task_id) + return True + + def snapshot(self) -> dict[str, object]: + with self._lock: + return { + "max_concurrent": self.max_concurrent, + "queue_capacity": self.queue_capacity, + "queue_depth": len(self._pending), + "active_count": len(self._active), + "pending": dict(self._pending), + "active": dict(self._active), + "submitted": self._submitted, + "completed": self._completed, + "failed": self._failed, + "rejected": self._rejected, + "cancelled_pending": self._cancelled_pending, + "closed": self._closed, + } + + def shutdown(self, *, cancel_pending: bool = True, wait_timeout: float = 2.0) -> None: + """Stop admission, optionally cancel queued work and wait boundedly for workers. + + Running work is not force-killed here because the owning runtime/process + boundary is responsible for safe cancellation. Worker threads are daemon + threads so process shutdown cannot be held indefinitely by a model job. + """ + with self._lock: + self._closed = True + if cancel_pending: + self._cancelled.update(self._pending) + deadline = time.monotonic() + max(0.0, wait_timeout) + for worker in self._workers: + remaining = max(0.0, deadline - time.monotonic()) + worker.join(timeout=remaining) + + def _worker(self) -> None: + while True: + try: + item = self._queue.get(timeout=0.1) + except queue.Empty: + with self._lock: + if self._closed and not self._pending: + return + continue + + cancelled = False + with self._lock: + self._pending.pop(item.task_id, None) + if item.task_id in self._cancelled: + self._cancelled.discard(item.task_id) + self._cancelled_pending += 1 + cancelled = True + else: + self._active[item.task_id] = item.workload_type + + try: + if cancelled: + if item.on_cancel is not None: + item.on_cancel("cancelled_before_start") + continue + item.run() + except Exception: + with self._lock: + self._failed += 1 + # The job owner records user-visible failure. Do not kill the + # scheduler worker because one workload failed. + else: + with self._lock: + self._completed += 1 + finally: + with self._lock: + self._active.pop(item.task_id, None) + self._queue.task_done() + + +def _positive_env_int(name: str, default: int) -> int: + raw = os.environ.get(name) + if raw is None or not raw.strip(): + return default + try: + value = int(raw) + except ValueError as exc: + raise ValueError(f"{name} must be an integer") from exc + if value < 1: + raise ValueError(f"{name} must be >= 1") + return value diff --git a/src/local_asr_server/server.py b/src/local_asr_server/server.py index 249f8685..f0cbb6b4 100644 --- a/src/local_asr_server/server.py +++ b/src/local_asr_server/server.py @@ -23,6 +23,7 @@ from local_asr_server.transcription_jobs import TranscriptionJobManager from local_asr_server.paths import get_static_dir from local_asr_server.runtime.service_manager import RuntimeServiceManager +from local_asr_server.runtime.workload_arbiter import HeavyWorkloadArbiter from local_asr_server.services.transcription_service import TranscriptionService from local_asr_server.transcriber import transcribe_file_sync from local_asr_server.app_logging import configure_application_logging @@ -78,6 +79,8 @@ def create_app( app.state.default_model = default_model capture_manager = NativeCaptureManager() runtime_services = RuntimeServiceManager() + heavy_workloads = HeavyWorkloadArbiter.from_env() + app.state.heavy_workload_arbiter = heavy_workloads from local_asr_server.runtime.leases import ModelRuntimeLeaseManager ModelRuntimeLeaseManager.set_service_manager(runtime_services) transcription_service = TranscriptionService() @@ -148,7 +151,7 @@ def _start_llm_sidecar(): [job["id"] for job in interrupted_jobs if job["type"] == "analysis"], reason="Interrupted by server restart", ) - transcription_jobs = TranscriptionJobManager(job_store) + transcription_jobs = TranscriptionJobManager(job_store, arbiter=heavy_workloads) recording_store = RecordingStore( recordings_dir or Path("~/Recordings/local-asr"), use_settings_dir=recordings_dir is None, @@ -169,7 +172,7 @@ def _start_llm_sidecar(): recordings=recording_store, transcriptions=transcription_store, ) - services.analysis_jobs = AnalysisJobManager(services, job_store) + services.analysis_jobs = AnalysisJobManager(services, job_store, arbiter=heavy_workloads) install_compatibility_aliases(app, services) # Clean up any orphan aggregate devices from previous runs/crashes @@ -206,7 +209,7 @@ async def require_local_auth(request: Request, call_next): def read_index() -> FileResponse: return FileResponse( static_dir / "index.html", - headers={"Cache-Control": "no-store, no-cache, must-revalidate, max-age=0"} + headers={"Cache-Control": "no-store, no-cache, must-revalidate, max-age=0"}, ) # Include routers @@ -223,6 +226,7 @@ def read_index() -> FileResponse: @app.on_event("shutdown") def shutdown_event(): + heavy_workloads.shutdown(cancel_pending=True, wait_timeout=2.0) print("Stopping local LLM sidecar...") logger.info("Stopping local LLM sidecar...") runtime_services.shutdown() diff --git a/src/local_asr_server/transcription_jobs.py b/src/local_asr_server/transcription_jobs.py index f2f58e9f..ed961d05 100644 --- a/src/local_asr_server/transcription_jobs.py +++ b/src/local_asr_server/transcription_jobs.py @@ -9,6 +9,11 @@ from local_asr_server.jobs import JobStore from local_asr_server.jobs.models import TERMINAL_JOB_STATUSES +from local_asr_server.runtime.workload_arbiter import ( + HeavyWorkloadArbiter, + WorkloadArbiterClosed, + WorkloadQueueFull, +) TRANSCRIPTION_JOB_TYPE = "transcription" DIARIZATION_JOB_TYPE = "diarization" @@ -52,10 +57,16 @@ def public(self) -> dict[str, Any]: class TranscriptionJobManager: - def __init__(self, store: JobStore | None = None) -> None: + def __init__( + self, + store: JobStore | None = None, + *, + arbiter: HeavyWorkloadArbiter | None = None, + ) -> None: self._lock = threading.Lock() self._jobs: dict[str, TranscriptionJob] = {} self._store = store + self._arbiter = arbiter def create( self, @@ -88,7 +99,28 @@ def create( job.events.put(job.public()) else: self._emit(job, "queued", 0) - threading.Thread(target=self._run, args=(job, runner), daemon=True).start() + + if self._arbiter is None: + threading.Thread(target=self._run, args=(job, runner), daemon=True).start() + return job.public() + + try: + self._arbiter.submit( + task_id=job.id, + workload_type=job.job_type, + run=lambda: self._run(job, runner), + on_cancel=lambda reason: self._cancel_before_start(job, reason), + ) + except (WorkloadQueueFull, WorkloadArbiterClosed) as exc: + job.error = str(exc)[:2000] + self._emit( + job, + "failed", + job.progress, + "resource_admission", + message="heavy_workload_not_admitted", + event_payload={"reason": job.error}, + ) return job.public() def get(self, job_id: str) -> dict[str, Any] | None: @@ -142,6 +174,8 @@ def cancel(self, job_id: str) -> dict[str, Any] | None: job.cancel_requested = True if self._store is not None: self._store.request_cancel(job.id) + if self._arbiter is not None: + self._arbiter.cancel_pending(job.id) self._emit(job, "cancelling", job.progress, "cancelling") return job.public() @@ -181,6 +215,21 @@ def events_after(self, job_id: str, sequence: int = 0) -> list[dict[str, Any]] | events = self.drain_events(job_id) return events + def _cancel_before_start(self, job: TranscriptionJob, reason: str) -> None: + if job.status in TERMINAL_JOB_STATUSES: + return + job.cancel_requested = True + if self._store is not None: + self._store.request_cancel(job.id) + self._emit( + job, + "cancelled", + job.progress, + "cancelled", + message=reason, + event_payload={"reason": reason}, + ) + def _run(self, job: TranscriptionJob, runner: Callable[[TranscriptionJob], dict[str, Any]]) -> None: try: if job.cancel_requested: diff --git a/test/test_workload_arbiter.py b/test/test_workload_arbiter.py new file mode 100644 index 00000000..570f8416 --- /dev/null +++ b/test/test_workload_arbiter.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +import threading +import time +import unittest + +from local_asr_server.runtime.workload_arbiter import HeavyWorkloadArbiter, WorkloadQueueFull +from local_asr_server.transcription_jobs import TranscriptionJobManager + + +class HeavyWorkloadArbiterTests(unittest.TestCase): + def test_serializes_heavy_work_by_default(self) -> None: + arbiter = HeavyWorkloadArbiter(max_concurrent=1, queue_capacity=2) + first_started = threading.Event() + release_first = threading.Event() + second_started = threading.Event() + order: list[str] = [] + + def first() -> None: + order.append("first-start") + first_started.set() + release_first.wait(timeout=2.0) + order.append("first-end") + + def second() -> None: + order.append("second-start") + second_started.set() + + try: + arbiter.submit(task_id="one", workload_type="transcription", run=first) + self.assertTrue(first_started.wait(timeout=1.0)) + arbiter.submit(task_id="two", workload_type="analysis", run=second) + + snapshot = arbiter.snapshot() + self.assertEqual(snapshot["active_count"], 1) + self.assertEqual(snapshot["queue_depth"], 1) + self.assertFalse(second_started.wait(timeout=0.05)) + + release_first.set() + self.assertTrue(second_started.wait(timeout=1.0)) + deadline = time.monotonic() + 1.0 + while time.monotonic() < deadline and arbiter.snapshot()["completed"] < 2: + time.sleep(0.01) + + self.assertEqual(order, ["first-start", "first-end", "second-start"]) + self.assertEqual(arbiter.snapshot()["completed"], 2) + finally: + release_first.set() + arbiter.shutdown() + + def test_rejects_when_pending_capacity_is_full(self) -> None: + arbiter = HeavyWorkloadArbiter(max_concurrent=1, queue_capacity=1) + first_started = threading.Event() + release_first = threading.Event() + + def first() -> None: + first_started.set() + release_first.wait(timeout=2.0) + + try: + arbiter.submit(task_id="one", workload_type="transcription", run=first) + self.assertTrue(first_started.wait(timeout=1.0)) + arbiter.submit(task_id="two", workload_type="analysis", run=lambda: None) + with self.assertRaises(WorkloadQueueFull): + arbiter.submit(task_id="three", workload_type="vision", run=lambda: None) + self.assertEqual(arbiter.snapshot()["rejected"], 1) + self.assertEqual(arbiter.snapshot()["queue_depth"], 1) + finally: + release_first.set() + arbiter.shutdown() + + def test_cancels_pending_work_without_running_it(self) -> None: + arbiter = HeavyWorkloadArbiter(max_concurrent=1, queue_capacity=2) + first_started = threading.Event() + release_first = threading.Event() + cancelled = threading.Event() + second_ran = threading.Event() + + def first() -> None: + first_started.set() + release_first.wait(timeout=2.0) + + try: + arbiter.submit(task_id="one", workload_type="transcription", run=first) + self.assertTrue(first_started.wait(timeout=1.0)) + arbiter.submit( + task_id="two", + workload_type="analysis", + run=second_ran.set, + on_cancel=lambda _reason: cancelled.set(), + ) + self.assertTrue(arbiter.cancel_pending("two")) + release_first.set() + self.assertTrue(cancelled.wait(timeout=1.0)) + self.assertFalse(second_ran.is_set()) + self.assertEqual(arbiter.snapshot()["cancelled_pending"], 1) + finally: + release_first.set() + arbiter.shutdown() + + def test_transcription_manager_uses_shared_arbiter(self) -> None: + arbiter = HeavyWorkloadArbiter(max_concurrent=1, queue_capacity=2) + manager = TranscriptionJobManager(arbiter=arbiter) + first_started = threading.Event() + release_first = threading.Event() + second_started = threading.Event() + + def first_runner(_job): + first_started.set() + release_first.wait(timeout=2.0) + return {"outcome_status": "completed", "diagnostics": []} + + def second_runner(_job): + second_started.set() + return {"outcome_status": "completed", "diagnostics": []} + + try: + first = manager.create("rec-1", first_runner) + self.assertTrue(first_started.wait(timeout=1.0)) + second = manager.create("rec-2", second_runner) + self.assertFalse(second_started.wait(timeout=0.05)) + self.assertEqual(arbiter.snapshot()["queue_depth"], 1) + + release_first.set() + self.assertTrue(second_started.wait(timeout=1.0)) + deadline = time.monotonic() + 1.0 + while time.monotonic() < deadline: + if manager.get(second["id"])["status"] == "completed": + break + time.sleep(0.01) + + self.assertEqual(manager.get(first["id"])["status"], "completed") + self.assertEqual(manager.get(second["id"])["status"], "completed") + finally: + release_first.set() + arbiter.shutdown() + + +if __name__ == "__main__": + unittest.main() From a92495a7e64a4fbfbfe848fcab6df0ccafbc4b6a Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Sun, 30 Aug 2026 22:18:35 +0200 Subject: [PATCH 002/182] Bound native capture event retention (#9) Coalesce high-frequency volume telemetry, bound discrete capture event/history retention and warnings, and expose saturation counters. STRONG remote preflight passed on exact PR head including arm64 package smoke. --- src/local_asr_server/native_capture.py | 18 ++- .../runtime/capture_events.py | 115 ++++++++++++++++++ test/test_capture_events.py | 73 +++++++++++ test/test_native_capture_event_retention.py | 89 ++++++++++++++ 4 files changed, 292 insertions(+), 3 deletions(-) create mode 100644 src/local_asr_server/runtime/capture_events.py create mode 100644 test/test_capture_events.py create mode 100644 test/test_native_capture_event_retention.py diff --git a/src/local_asr_server/native_capture.py b/src/local_asr_server/native_capture.py index 49fb22e3..25ad94a0 100644 --- a/src/local_asr_server/native_capture.py +++ b/src/local_asr_server/native_capture.py @@ -9,10 +9,16 @@ import threading import time import uuid +from collections import deque from dataclasses import dataclass, field from pathlib import Path from typing import Any +from local_asr_server.runtime.capture_events import ( + BoundedCaptureEventHistory, + CoalescingCaptureEventQueue, +) + from local_asr_server.paths import ( get_native_capture_helper_path, get_ffmpeg_path, @@ -37,13 +43,13 @@ class CaptureSession: output_dir: Path reader_thread: threading.Thread | None = None started_at: float = field(default_factory=time.time) - events: "queue.Queue[dict[str, Any]]" = field(default_factory=queue.Queue) - event_log: list[dict[str, Any]] = field(default_factory=list) + events: CoalescingCaptureEventQueue = field(default_factory=CoalescingCaptureEventQueue) + event_log: BoundedCaptureEventHistory = field(default_factory=BoundedCaptureEventHistory) ready_event: dict[str, Any] | None = None track_ready: dict[str, dict[str, Any]] = field(default_factory=dict) track_written: dict[str, dict[str, Any]] = field(default_factory=dict) last_volume: dict[str, float] = field(default_factory=dict) - warnings: list[dict[str, Any]] = field(default_factory=list) + warnings: deque[dict[str, Any]] = field(default_factory=lambda: deque(maxlen=128)) stopped: bool = False @@ -484,4 +490,10 @@ def _terminate(self, recording_id: str, *, cancel: bool) -> dict[str, Any]: "backend": "native", "status": "cancelled" if cancel else ("interrupted" if was_killed else "stopped"), "events": events, + "event_buffer": { + "queue": session.events.stats(), + "history": session.event_log.stats(), + "retained_warnings": len(session.warnings), + "warning_capacity": session.warnings.maxlen, + }, } diff --git a/src/local_asr_server/runtime/capture_events.py b/src/local_asr_server/runtime/capture_events.py new file mode 100644 index 00000000..0263b382 --- /dev/null +++ b/src/local_asr_server/runtime/capture_events.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import queue +import threading +from collections import OrderedDict, deque +from collections.abc import Iterator +from typing import Any + + +DEFAULT_CAPTURE_EVENT_CAPACITY = 512 +DEFAULT_CAPTURE_HISTORY_CAPACITY = 512 + + +class CoalescingCaptureEventQueue: + """Queue-compatible bounded buffer for native capture events. + + High-frequency volume samples are latest-state telemetry, not durable events. + Keep at most one pending sample per source while lifecycle/warning/error events + use a bounded FIFO. When the discrete FIFO saturates, drop the oldest item and + expose the count rather than growing memory for the lifetime of a meeting. + """ + + def __init__(self, capacity: int = DEFAULT_CAPTURE_EVENT_CAPACITY) -> None: + if capacity < 1: + raise ValueError("capacity must be >= 1") + self.capacity = capacity + self._events: deque[dict[str, Any]] = deque() + self._volumes: OrderedDict[str, dict[str, Any]] = OrderedDict() + self._lock = threading.Lock() + self.dropped_events = 0 + self.coalesced_volume_events = 0 + + def put(self, event: dict[str, Any]) -> None: + event_type = event.get("type") + if event_type == "volume": + source = str(event.get("source") or "unknown") + with self._lock: + if source in self._volumes: + self.coalesced_volume_events += 1 + self._volumes.pop(source, None) + self._volumes[source] = event + return + + with self._lock: + if len(self._events) >= self.capacity: + self._events.popleft() + self.dropped_events += 1 + self._events.append(event) + + def get_nowait(self) -> dict[str, Any]: + with self._lock: + if self._events: + return self._events.popleft() + if self._volumes: + _, event = self._volumes.popitem(last=False) + return event + raise queue.Empty + + def qsize(self) -> int: + with self._lock: + return len(self._events) + len(self._volumes) + + def stats(self) -> dict[str, int]: + with self._lock: + return { + "capacity": self.capacity, + "pending_discrete_events": len(self._events), + "pending_volume_sources": len(self._volumes), + "dropped_events": self.dropped_events, + "coalesced_volume_events": self.coalesced_volume_events, + } + + +class BoundedCaptureEventHistory: + """Bounded diagnostics history that excludes high-rate volume samples.""" + + def __init__(self, capacity: int = DEFAULT_CAPTURE_HISTORY_CAPACITY) -> None: + if capacity < 1: + raise ValueError("capacity must be >= 1") + self.capacity = capacity + self._events: deque[dict[str, Any]] = deque(maxlen=capacity) + self._lock = threading.Lock() + self.dropped_events = 0 + self.ignored_volume_events = 0 + + def append(self, event: dict[str, Any]) -> None: + if event.get("type") == "volume": + with self._lock: + self.ignored_volume_events += 1 + return + with self._lock: + if len(self._events) == self.capacity: + self.dropped_events += 1 + self._events.append(event) + + def __len__(self) -> int: + with self._lock: + return len(self._events) + + def __iter__(self) -> Iterator[dict[str, Any]]: + with self._lock: + snapshot = tuple(self._events) + return iter(snapshot) + + def snapshot(self) -> list[dict[str, Any]]: + return list(self) + + def stats(self) -> dict[str, int]: + with self._lock: + return { + "capacity": self.capacity, + "retained_events": len(self._events), + "dropped_events": self.dropped_events, + "ignored_volume_events": self.ignored_volume_events, + } diff --git a/test/test_capture_events.py b/test/test_capture_events.py new file mode 100644 index 00000000..63d7c5c3 --- /dev/null +++ b/test/test_capture_events.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import queue +import unittest + +from local_asr_server.runtime.capture_events import ( + BoundedCaptureEventHistory, + CoalescingCaptureEventQueue, +) + + +class CaptureEventBufferTests(unittest.TestCase): + def test_volume_events_are_coalesced_per_source(self) -> None: + events = CoalescingCaptureEventQueue(capacity=4) + for index in range(1000): + events.put({"type": "volume", "source": "mic", "db": -60.0 + index}) + events.put({"type": "volume", "source": "system", "db": -50.0 + index}) + + self.assertEqual(events.qsize(), 2) + stats = events.stats() + self.assertEqual(stats["pending_volume_sources"], 2) + self.assertEqual(stats["coalesced_volume_events"], 1998) + + drained = [events.get_nowait(), events.get_nowait()] + by_source = {event["source"]: event for event in drained} + self.assertEqual(by_source["mic"]["db"], 939.0) + self.assertEqual(by_source["system"]["db"], 949.0) + with self.assertRaises(queue.Empty): + events.get_nowait() + + def test_discrete_events_are_bounded_and_drop_oldest(self) -> None: + events = CoalescingCaptureEventQueue(capacity=3) + for index in range(5): + events.put({"type": "warning", "message": f"warning-{index}"}) + + self.assertEqual(events.qsize(), 3) + self.assertEqual(events.stats()["dropped_events"], 2) + self.assertEqual( + [events.get_nowait()["message"] for _ in range(3)], + ["warning-2", "warning-3", "warning-4"], + ) + + def test_discrete_events_are_drained_before_latest_volume(self) -> None: + events = CoalescingCaptureEventQueue(capacity=3) + events.put({"type": "volume", "source": "mic", "db": -20.0}) + events.put({"type": "ready"}) + + self.assertEqual(events.get_nowait()["type"], "ready") + self.assertEqual(events.get_nowait()["type"], "volume") + + def test_history_ignores_volume_and_has_fixed_capacity(self) -> None: + history = BoundedCaptureEventHistory(capacity=2) + for index in range(1000): + history.append({"type": "volume", "source": "mic", "db": float(index)}) + history.append({"type": "ready", "sequence": 1}) + history.append({"type": "warning", "sequence": 2}) + history.append({"type": "stopped", "sequence": 3}) + + self.assertEqual(len(history), 2) + self.assertEqual([event["type"] for event in history], ["warning", "stopped"]) + stats = history.stats() + self.assertEqual(stats["ignored_volume_events"], 1000) + self.assertEqual(stats["dropped_events"], 1) + + def test_capacity_must_be_positive(self) -> None: + with self.assertRaises(ValueError): + CoalescingCaptureEventQueue(capacity=0) + with self.assertRaises(ValueError): + BoundedCaptureEventHistory(capacity=0) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_native_capture_event_retention.py b/test/test_native_capture_event_retention.py new file mode 100644 index 00000000..26ae59bd --- /dev/null +++ b/test/test_native_capture_event_retention.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path +from typing import Any + +from local_asr_server.native_capture import CaptureSession, NativeCaptureManager +from local_asr_server.runtime.capture_events import ( + BoundedCaptureEventHistory, + CoalescingCaptureEventQueue, +) + + +class _FakeStdout: + def __init__(self, lines: list[str]) -> None: + self._lines = lines + self.closed = False + + def __iter__(self): + return iter(self._lines) + + def close(self) -> None: + self.closed = True + + +class _FakeProcess: + def __init__(self, lines: list[str]) -> None: + self.stdout = _FakeStdout(lines) + self.returncode = 0 + + def wait(self, timeout: float | None = None) -> int: + return self.returncode + + +class NativeCaptureEventRetentionTests(unittest.TestCase): + def test_capture_session_defaults_are_bounded(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + session = CaptureSession( + recording_id="rec-1", + mode="both", + process=_FakeProcess([]), # type: ignore[arg-type] + output_dir=Path(temporary), + ) + + self.assertIsInstance(session.events, CoalescingCaptureEventQueue) + self.assertIsInstance(session.event_log, BoundedCaptureEventHistory) + self.assertEqual(session.warnings.maxlen, 128) + + def test_reader_coalesces_volume_and_bounds_warning_retention(self) -> None: + lines: list[str] = [json.dumps({"type": "ready"}) + "\n"] + for index in range(1000): + lines.append(json.dumps({"type": "volume", "source": "mic", "db": float(index)}) + "\n") + lines.append(json.dumps({"type": "volume", "source": "system", "db": float(index + 1)}) + "\n") + for index in range(140): + lines.append(json.dumps({"type": "warning", "source": "backend", "message": f"warning-{index}"}) + "\n") + lines.append(json.dumps({"type": "stopped"}) + "\n") + + process = _FakeProcess(lines) + with tempfile.TemporaryDirectory() as temporary: + session = CaptureSession( + recording_id="rec-2", + mode="both", + process=process, # type: ignore[arg-type] + output_dir=Path(temporary), + ) + manager = NativeCaptureManager(helper_path=Path("/nonexistent")) + manager._read_events(session) + + self.assertTrue(process.stdout.closed) + self.assertTrue(session.stopped) + self.assertEqual(session.last_volume["mic"], 999.0) + self.assertEqual(session.last_volume["system"], 1000.0) + self.assertEqual(len(session.warnings), 128) + self.assertEqual(session.warnings[0]["message"], "warning-12") + + queue_stats = session.events.stats() + self.assertEqual(queue_stats["pending_volume_sources"], 2) + self.assertEqual(queue_stats["coalesced_volume_events"], 1998) + self.assertLessEqual(queue_stats["pending_discrete_events"], queue_stats["capacity"]) + + history_stats = session.event_log.stats() + self.assertEqual(history_stats["ignored_volume_events"], 2000) + self.assertLessEqual(history_stats["retained_events"], history_stats["capacity"]) + + +if __name__ == "__main__": + unittest.main() From 25a7c711787b367941e04d4aab1106a36ba3ee47 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Sun, 30 Aug 2026 22:46:26 +0200 Subject: [PATCH 003/182] Upgrade to source-pinned local LLM runtime (#8) Pin local-llm-server 0.4.0 to validated immutable source revision 53500af8c1e1df1c92937b18928790fded8b437f, move updater logic to reproducible source/release modes, and update compatibility tests. FULL remote preflight passed on exact PR head including arm64 package smoke. --- pyproject.toml | 8 +- scripts/update_local_llm_server.py | 207 +++++++++++++++++------ src/local_asr_server/local_llm_params.py | 66 ++++++-- test/test_update_local_llm_server.py | 62 +++++++ test/test_visual_intelligence_service.py | 19 ++- uv.lock | 26 +-- 6 files changed, 288 insertions(+), 100 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e37d9ced..44b9d3ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,10 +25,10 @@ dependencies = [ # is resolved. "mlx==0.31.2; sys_platform == 'darwin'", "onnxruntime>=1.14.0,<1.24.0", - # Local LLM inference (Nemotron text + Voxtral audio). Pin the existing - # 0.3.8 integration to its published immutable release artifact instead of - # a developer-machine absolute path so clean builds and CI are portable. - "local-llm-server[vision] @ https://github.com/daniele21/local-llm-server/releases/download/v0.3.8/local_llm_server-0.3.8-py3-none-any.whl#sha256=cbd7b7d684658021b17ba04f9e0fddb246ebd65192bb7af581aabb492c456edb", + # Local LLM/VLM control plane. Pin the validated repository source to an + # immutable Git revision so clean builds compile exactly the same runtime + # while still allowing ClosedRoom to consume post-release resource fixes. + "local-llm-server[vision] @ git+https://github.com/daniele21/local-llm-server.git@53500af8c1e1df1c92937b18928790fded8b437f", # Audio preprocessing for Voxtral multimodal analysis "soundfile>=0.12.0", "numpy>=1.24.0", diff --git a/scripts/update_local_llm_server.py b/scripts/update_local_llm_server.py index 6216250f..b64d0511 100755 --- a/scripts/update_local_llm_server.py +++ b/scripts/update_local_llm_server.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 -"""Update the local-llm-server wheel dependency to the latest GitHub semver tag.""" +"""Update the local-llm-server dependency from a reproducible GitHub source.""" from __future__ import annotations import argparse from dataclasses import dataclass +import hashlib import os from pathlib import Path import re @@ -18,11 +19,20 @@ PYPROJECT_PATH = PROJECT_ROOT / "pyproject.toml" LOCK_PATH = PROJECT_ROOT / "uv.lock" DEFAULT_REPOSITORY_URL = "https://github.com/daniele21/local-llm-server.git" +DEFAULT_BRANCH = "main" TAG_PATTERN = re.compile(r"refs/tags/v?(\d+)\.(\d+)\.(\d+)$") DEPENDENCY_PATTERN = re.compile( - r'(?P"local-llm-server(?:\[[A-Za-z0-9_,.-]+\])?\s*@\s*file://)' - r'(?P[^"\n]+)(?P")' + r'(?P"local-llm-server(?:\[[A-Za-z0-9_,.-]+\])?\s*@\s*)' + r'(?P[^"\n]+)(?P")' ) +RELEASE_WHEEL_PATTERN = re.compile( + r"/releases/download/v(?P\d+\.\d+\.\d+)/" + r"local_llm_server-(?P=version)-py3-none-any\.whl(?:#sha256=(?P[0-9a-f]{64}))?$" +) +FILE_WHEEL_PATTERN = re.compile( + r"^file://.+/local_llm_server-(?P\d+\.\d+\.\d+)-py3-none-any\.whl$" +) +GIT_SOURCE_PATTERN = re.compile(r"^git\+(?P.+)@(?P[0-9a-f]{40})$") @dataclass(frozen=True, order=True) @@ -39,6 +49,21 @@ def tag(self) -> str: return f"v{self}" +@dataclass(frozen=True) +class DependencyIdentity: + kind: str + version: Version | None = None + revision: str | None = None + source: str = "" + + def display(self) -> str: + if self.revision: + return f"{self.kind}:{self.revision}" + if self.version: + return f"{self.kind}:{self.version}" + return self.kind + + def latest_version_from_ls_remote(output: str) -> Version: versions = [] for line in output.splitlines(): @@ -50,25 +75,75 @@ def latest_version_from_ls_remote(output: str) -> Version: return max(versions) -def dependency_version(pyproject: str) -> Version | None: +def revision_from_ls_remote(output: str, *, ref: str) -> str: + for line in output.splitlines(): + parts = line.strip().split() + if len(parts) == 2 and parts[1] == ref and re.fullmatch(r"[0-9a-f]{40}", parts[0]): + return parts[0] + raise RuntimeError(f"Impossibile risolvere la revisione Git per {ref}") + + +def dependency_identity(pyproject: str) -> DependencyIdentity: match = DEPENDENCY_PATTERN.search(pyproject) if not match: - raise RuntimeError("Dipendenza file:// di local-llm-server non trovata in pyproject.toml") - wheel = Path(match.group("path")).name - version_match = re.fullmatch(r"local_llm_server-(\d+)\.(\d+)\.(\d+)-py3-none-any\.whl", wheel) - if not version_match: - return None - return Version(*(int(value) for value in version_match.groups())) + raise RuntimeError("Dipendenza local-llm-server non trovata in pyproject.toml") + source = match.group("source") + git_match = GIT_SOURCE_PATTERN.match(source) + if git_match: + return DependencyIdentity(kind="git", revision=git_match.group("revision"), source=source) + release_match = RELEASE_WHEEL_PATTERN.search(source) + if release_match: + version = Version(*(int(value) for value in release_match.group("version").split("."))) + return DependencyIdentity(kind="release", version=version, source=source) + file_match = FILE_WHEEL_PATTERN.match(source) + if file_match: + version = Version(*(int(value) for value in file_match.group("version").split("."))) + return DependencyIdentity(kind="file", version=version, source=source) + return DependencyIdentity(kind="unknown", source=source) -def replace_dependency(pyproject: str, wheel_path: Path) -> str: - replacement = rf'\g{wheel_path.resolve()}\g' - updated, count = DEPENDENCY_PATTERN.subn(replacement, pyproject, count=1) +def dependency_version(pyproject: str) -> Version | None: + return dependency_identity(pyproject).version + + +def replace_dependency_source(pyproject: str, source: str) -> str: + updated, count = DEPENDENCY_PATTERN.subn( + lambda match: f"{match.group('prefix')}{source}{match.group('suffix')}", + pyproject, + count=1, + ) if count != 1: raise RuntimeError("Impossibile aggiornare la dipendenza local-llm-server") return updated +def replace_dependency(pyproject: str, wheel_path: Path) -> str: + """Backward-compatible helper used by older tooling/tests.""" + return replace_dependency_source(pyproject, f"file://{wheel_path.resolve()}") + + +def git_dependency_source(repository_url: str, revision: str) -> str: + if not re.fullmatch(r"[0-9a-f]{40}", revision): + raise ValueError("revision must be a full 40-character Git SHA") + url = repository_url.strip() + if url.startswith("git@github.com:"): + url = "https://github.com/" + url.removeprefix("git@github.com:") + if not url.startswith(("https://", "http://")): + raise RuntimeError(f"Remote Git non supportato per dependency source: {repository_url}") + return f"git+{url}@{revision}" + + +def release_dependency_source(repository_url: str, version: Version, sha256: str) -> str: + if not re.fullmatch(r"[0-9a-f]{64}", sha256): + raise ValueError("sha256 must be a lowercase 64-character digest") + slug = github_repository_slug(repository_url) + wheel_name = f"local_llm_server-{version}-py3-none-any.whl" + return ( + f"https://github.com/{slug}/releases/download/{version.tag}/{wheel_name}" + f"#sha256={sha256}" + ) + + def run(command: list[str], *, cwd: Path | None = None, capture: bool = False) -> str: result = subprocess.run( command, @@ -110,39 +185,37 @@ def github_repository_slug(repository_url: str) -> str: return f"{match.group(1)}/{match.group(2)}" -def ensure_wheel(version: Version, repository_url: str, dist_dir: Path) -> Path: - wheel = dist_dir / f"local_llm_server-{version}-py3-none-any.whl" - if wheel.exists(): - return wheel +def ensure_release_wheel(version: Version, repository_url: str) -> Path: gh = shutil.which("gh") if gh is None: raise RuntimeError("GitHub CLI (gh) non trovato: necessario per scaricare il wheel della release") - with tempfile.TemporaryDirectory(prefix="closedroom-local-llm-wheel-") as temporary: - download_dir = Path(temporary) - run([ - gh, - "release", - "download", - version.tag, - "--repo", - github_repository_slug(repository_url), - "--pattern", - wheel.name, - "--dir", - str(download_dir), - ]) - downloaded = download_dir / wheel.name - if not downloaded.exists(): - raise RuntimeError( - f"La release {version.tag} non pubblica il wheel atteso {wheel.name}" - ) - dist_dir.mkdir(parents=True, exist_ok=True) - temporary_target = wheel.with_suffix(".whl.tmp") - shutil.copy2(downloaded, temporary_target) - os.replace(temporary_target, wheel) + temporary = Path(tempfile.mkdtemp(prefix="closedroom-local-llm-wheel-")) + wheel = temporary / f"local_llm_server-{version}-py3-none-any.whl" + run([ + gh, + "release", + "download", + version.tag, + "--repo", + github_repository_slug(repository_url), + "--pattern", + wheel.name, + "--dir", + str(temporary), + ]) + if not wheel.exists(): + raise RuntimeError(f"La release {version.tag} non pubblica il wheel atteso {wheel.name}") return wheel +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + def update_lock(pyproject_original: bytes, lock_original: bytes | None) -> None: try: run([shutil.which("uv") or "uv", "lock"], cwd=PROJECT_ROOT) @@ -157,41 +230,71 @@ def update_lock(pyproject_original: bytes, lock_original: bytes | None) -> None: def main() -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--check", action="store_true", help="Mostra l'ultima versione senza modificare file") + parser.add_argument("--check", action="store_true", help="Mostra la revisione/versione target senza modificare file") + parser.add_argument( + "--source", + choices=("main", "release"), + default="main", + help="main: pin a exact source SHA and let uv build it; release: pin published wheel + sha256", + ) + parser.add_argument("--branch", default=DEFAULT_BRANCH, help="Branch local-llm-server da risolvere con --source main") + parser.add_argument("--revision", help="SHA esatto da usare con --source main invece di risolvere il branch") parser.add_argument( "--local-repo", type=Path, default=PROJECT_ROOT.parent / "local-llm-server", - help="Worktree locale usato per il remote origin e come destinazione dist/ dei wheel", + help="Worktree locale usato solo per risolvere il remote origin quando disponibile", ) args = parser.parse_args() local_repo = args.local_repo.expanduser().resolve() repository_url = resolve_repository_url(local_repo) - tags = run(["git", "ls-remote", "--tags", "--refs", repository_url], capture=True) - latest = latest_version_from_ls_remote(tags) pyproject_original = PYPROJECT_PATH.read_bytes() - current = dependency_version(pyproject_original.decode("utf-8")) - print(f"local-llm-server: corrente={current or 'sconosciuta'} ultima={latest}") + pyproject_text = pyproject_original.decode("utf-8") + current = dependency_identity(pyproject_text) + + if args.source == "main": + if args.revision: + revision = args.revision.lower() + if not re.fullmatch(r"[0-9a-f]{40}", revision): + raise RuntimeError("--revision deve essere uno SHA Git completo di 40 caratteri") + else: + ref = f"refs/heads/{args.branch}" + remote = run(["git", "ls-remote", repository_url, ref], capture=True) + revision = revision_from_ls_remote(remote, ref=ref) + target_source = git_dependency_source(repository_url, revision) + target_display = f"git:{revision}" + else: + tags = run(["git", "ls-remote", "--tags", "--refs", repository_url], capture=True) + version = latest_version_from_ls_remote(tags) + wheel = ensure_release_wheel(version, repository_url) + try: + target_source = release_dependency_source(repository_url, version, sha256_file(wheel)) + finally: + shutil.rmtree(wheel.parent, ignore_errors=True) + target_display = f"release:{version}" + + print(f"local-llm-server: corrente={current.display()} target={target_display}") if args.check: - return 0 if current == latest else 1 + if args.source == "main": + return 0 if current.kind == "git" and current.revision == revision else 1 + return 0 if current.kind == "release" and current.version == version else 1 - wheel = ensure_wheel(latest, repository_url, local_repo / "dist") - updated = replace_dependency(pyproject_original.decode("utf-8"), wheel) - if updated == pyproject_original.decode("utf-8") and current == latest: + updated = replace_dependency_source(pyproject_text, target_source) + if updated == pyproject_text: print("Nessun aggiornamento necessario") return 0 lock_original = LOCK_PATH.read_bytes() if LOCK_PATH.exists() else None atomic_write(PYPROJECT_PATH, updated.encode("utf-8")) update_lock(pyproject_original, lock_original) - print(f"Aggiornato local-llm-server a {latest}: {wheel}") + print(f"Aggiornato local-llm-server a {target_display}") return 0 if __name__ == "__main__": try: raise SystemExit(main()) - except (OSError, RuntimeError) as error: + except (OSError, RuntimeError, ValueError) as error: print(f"errore: {error}", file=sys.stderr) raise SystemExit(2) diff --git a/src/local_asr_server/local_llm_params.py b/src/local_asr_server/local_llm_params.py index 70256dd9..c019ccd0 100644 --- a/src/local_asr_server/local_llm_params.py +++ b/src/local_asr_server/local_llm_params.py @@ -6,6 +6,7 @@ import json import logging +import os from pathlib import Path from typing import Any @@ -15,6 +16,8 @@ logger = logging.getLogger("uvicorn.error") +LOCAL_LLM_REGISTRY_PATHS_ENV = "LOCAL_LLM_REGISTRY_PATHS" + DEFAULT_LOCAL_LLM_PARAMS: dict[str, Any] = { "models": { "nemotron-nano-4b-q8": { @@ -80,23 +83,54 @@ def load_local_llm_params() -> dict[str, Any]: return _default_params_copy() +def _external_registry_paths_without(adapter_file: Path) -> list[str]: + """Preserve caller-provided external registries while excluding our prior overlay.""" + raw = os.environ.get(LOCAL_LLM_REGISTRY_PATHS_ENV, "") + if not raw.strip(): + return [] + adapter = adapter_file.expanduser().resolve() + retained: list[str] = [] + for value in raw.split(os.pathsep): + value = value.strip() + if not value: + continue + candidate = Path(value).expanduser().resolve() + if candidate == adapter: + continue + retained.append(str(candidate)) + return retained + + def configure_local_llm_server_registry() -> Path: - """Materialize ClosedRoom model overrides as an upstream registry overlay. + """Materialize ClosedRoom overrides through local-llm-server's public API. + + local-llm-server 0.4 exposes generic registry overlays through + ``LOCAL_LLM_REGISTRY_PATHS``. ClosedRoom writes one private adapter file and + appends it to that public environment contract instead of mutating upstream + module globals. Any pre-existing external registry layers are preserved. - ``local-llm-server`` 0.3.8 deliberately owns a YAML model registry, while - ClosedRoom owns the user-facing ``local_llm_params.json`` configuration. - The managed sidecar calls this adapter before invoking the pinned upstream - CLI. The real ``~/.local-llm/models.yaml`` is read as an input and is never - modified. + The upstream user registry keeps the precedence defined by local-llm-server + 0.4. ClosedRoom does not rewrite or monkey-patch that upstream source. """ import local_llm_server.registry as upstream_registry - # A previous invocation in the same process may have pointed the upstream - # module at our generated overlay. Always rebuild from the genuine upstream - # user registry first. - upstream_user_registry = Path.home() / ".local-llm" / "models.yaml" - upstream_registry._USER_REGISTRY = upstream_user_registry - registry = upstream_registry.load_registry() + adapter_file = get_local_llm_params_file().with_name("local_llm_registry.yaml") + external_paths = _external_registry_paths_without(adapter_file) + + # Build the adapter from genuine upstream inputs, excluding an older + # ClosedRoom-generated overlay from a prior invocation in this process. + previous_env = os.environ.get(LOCAL_LLM_REGISTRY_PATHS_ENV) + try: + if external_paths: + os.environ[LOCAL_LLM_REGISTRY_PATHS_ENV] = os.pathsep.join(external_paths) + else: + os.environ.pop(LOCAL_LLM_REGISTRY_PATHS_ENV, None) + registry = upstream_registry.load_registry() + finally: + if previous_env is None: + os.environ.pop(LOCAL_LLM_REGISTRY_PATHS_ENV, None) + else: + os.environ[LOCAL_LLM_REGISTRY_PATHS_ENV] = previous_env models: dict[str, Any] = {} for key, entry in (registry.get("models") or {}).items(): @@ -126,7 +160,6 @@ def configure_local_llm_server_registry() -> Path: "default_model": registry.get("default_model"), "startup_models": startup_models or list(registry.get("startup_models") or []), } - adapter_file = get_local_llm_params_file().with_name("local_llm_registry.yaml") adapter_file.parent.mkdir(parents=True, exist_ok=True) temporary = adapter_file.with_suffix(".yaml.tmp") temporary.write_text( @@ -136,7 +169,8 @@ def configure_local_llm_server_registry() -> Path: temporary.replace(adapter_file) adapter_file.chmod(0o600) - # The CLI imports this same module object afterwards, so pointing it at the - # generated overlay preserves per-model params without touching user files. - upstream_registry._USER_REGISTRY = adapter_file + # The upstream CLI reads this public environment contract when it loads its + # registry. Append our overlay exactly once and preserve existing layers. + combined = [*external_paths, str(adapter_file.resolve())] + os.environ[LOCAL_LLM_REGISTRY_PATHS_ENV] = os.pathsep.join(combined) return adapter_file diff --git a/test/test_update_local_llm_server.py b/test/test_update_local_llm_server.py index 5d97055b..24f45be0 100644 --- a/test/test_update_local_llm_server.py +++ b/test/test_update_local_llm_server.py @@ -24,6 +24,50 @@ def test_selects_latest_stable_semver_tag(self) -> None: ]) self.assertEqual(str(MODULE.latest_version_from_ls_remote(output)), "0.10.0") + def test_resolves_exact_branch_revision(self) -> None: + revision = "a" * 40 + output = f"{revision}\trefs/heads/main\n" + self.assertEqual( + MODULE.revision_from_ls_remote(output, ref="refs/heads/main"), + revision, + ) + with self.assertRaises(RuntimeError): + MODULE.revision_from_ls_remote(output, ref="refs/heads/dev") + + def test_builds_git_dependency_source_from_full_sha(self) -> None: + revision = "b" * 40 + source = MODULE.git_dependency_source( + "https://github.com/daniele21/local-llm-server.git", + revision, + ) + self.assertEqual( + source, + f"git+https://github.com/daniele21/local-llm-server.git@{revision}", + ) + with self.assertRaises(ValueError): + MODULE.git_dependency_source( + "https://github.com/daniele21/local-llm-server.git", + "main", + ) + + def test_replaces_release_dependency_with_exact_git_source(self) -> None: + old = ( + 'dependencies = [\n' + ' "local-llm-server[vision] @ https://github.com/daniele21/local-llm-server/releases/' + 'download/v0.3.8/local_llm_server-0.3.8-py3-none-any.whl#sha256=' + "c" * 64 + '",\n' + ']\n' + ) + revision = "d" * 40 + source = MODULE.git_dependency_source( + "https://github.com/daniele21/local-llm-server.git", + revision, + ) + updated = MODULE.replace_dependency_source(old, source) + identity = MODULE.dependency_identity(updated) + self.assertEqual(identity.kind, "git") + self.assertEqual(identity.revision, revision) + self.assertIsNone(identity.version) + def test_replaces_only_dependency_wheel_path(self) -> None: source = 'dependencies = [\n "local-llm-server[vision] @ file:///old/local_llm_server-0.3.1-py3-none-any.whl",\n]\n' wheel = Path("/tmp/local_llm_server-0.3.8-py3-none-any.whl") @@ -32,6 +76,24 @@ def test_replaces_only_dependency_wheel_path(self) -> None: self.assertIn(f"file://{wheel.resolve()}", updated) self.assertEqual(str(MODULE.dependency_version(updated)), "0.3.8") + def test_builds_release_dependency_with_digest(self) -> None: + version = MODULE.Version(0, 4, 0) + source = MODULE.release_dependency_source( + "https://github.com/daniele21/local-llm-server.git", + version, + "e" * 64, + ) + self.assertEqual( + source, + "https://github.com/daniele21/local-llm-server/releases/download/v0.4.0/" + "local_llm_server-0.4.0-py3-none-any.whl#sha256=" + "e" * 64, + ) + identity = MODULE.dependency_identity( + f'dependencies = [\n "local-llm-server[vision] @ {source}",\n]\n' + ) + self.assertEqual(identity.kind, "release") + self.assertEqual(str(identity.version), "0.4.0") + def test_extracts_github_repository_slug(self) -> None: self.assertEqual( MODULE.github_repository_slug("https://github.com/daniele21/local-llm-server.git"), diff --git a/test/test_visual_intelligence_service.py b/test/test_visual_intelligence_service.py index a7a87a56..4dec5d97 100644 --- a/test/test_visual_intelligence_service.py +++ b/test/test_visual_intelligence_service.py @@ -1380,12 +1380,19 @@ def test_local_llm_server_registry_loads_closedroom_overlay(self) -> None: } }, f) - original_registry = registry_module._USER_REGISTRY - with patch("pathlib.Path.home", return_value=tmp_home), patch.object( - registry_module, "_USER_REGISTRY", original_registry - ): - adapter_path = configure_local_llm_server_registry() - registry = registry_module.load_registry() + previous_registry_paths = os.environ.get("LOCAL_LLM_REGISTRY_PATHS") + try: + with patch("pathlib.Path.home", return_value=tmp_home): + os.environ.pop("LOCAL_LLM_REGISTRY_PATHS", None) + adapter_path = configure_local_llm_server_registry() + configured_paths = os.environ.get("LOCAL_LLM_REGISTRY_PATHS", "").split(os.pathsep) + self.assertEqual(configured_paths, [str(adapter_path.resolve())]) + registry = registry_module.load_registry() + finally: + if previous_registry_paths is None: + os.environ.pop("LOCAL_LLM_REGISTRY_PATHS", None) + else: + os.environ["LOCAL_LLM_REGISTRY_PATHS"] = previous_registry_paths self.assertEqual(adapter_path, config_dir / "local_llm_registry.yaml") self.assertTrue(adapter_path.exists()) diff --git a/uv.lock b/uv.lock index 85405d24..be174c86 100644 --- a/uv.lock +++ b/uv.lock @@ -870,7 +870,7 @@ build = [ [package.metadata] requires-dist = [ { name = "fastapi", specifier = ">=0.115.0" }, - { name = "local-llm-server", extras = ["vision"], url = "https://github.com/daniele21/local-llm-server/releases/download/v0.3.8/local_llm_server-0.3.8-py3-none-any.whl" }, + { name = "local-llm-server", extras = ["vision"], git = "https://github.com/daniele21/local-llm-server.git?rev=53500af8c1e1df1c92937b18928790fded8b437f" }, { name = "mlx", marker = "sys_platform == 'darwin'", specifier = "==0.31.2" }, { name = "mlx-audio", extras = ["stt"] }, { name = "mlx-whisper" }, @@ -889,39 +889,21 @@ provides-extras = ["app", "build", "speechmatics"] [[package]] name = "local-llm-server" -version = "0.3.8" -source = { url = "https://github.com/daniele21/local-llm-server/releases/download/v0.3.8/local_llm_server-0.3.8-py3-none-any.whl" } +version = "0.4.0" +source = { git = "https://github.com/daniele21/local-llm-server.git?rev=53500af8c1e1df1c92937b18928790fded8b437f#53500af8c1e1df1c92937b18928790fded8b437f" } dependencies = [ { name = "fastapi" }, { name = "llama-cpp-python" }, + { name = "python-multipart" }, { name = "pyyaml" }, { name = "uvicorn" }, ] -wheels = [ - { url = "https://github.com/daniele21/local-llm-server/releases/download/v0.3.8/local_llm_server-0.3.8-py3-none-any.whl", hash = "sha256:cbd7b7d684658021b17ba04f9e0fddb246ebd65192bb7af581aabb492c456edb" }, -] [package.optional-dependencies] vision = [ { name = "mlx-vlm" }, ] -[package.metadata] -requires-dist = [ - { name = "fastapi", specifier = ">=0.100.0" }, - { name = "httpx", marker = "extra == 'dev'" }, - { name = "llama-cpp-python", specifier = ">=0.3.0" }, - { name = "mlx-lm", marker = "extra == 'mlx'", specifier = ">=0.31.0" }, - { name = "mlx-vlm", marker = "extra == 'vision'", specifier = ">=0.6.4,<0.7.0" }, - { name = "numpy", marker = "extra == 'audio'", specifier = ">=1.24.0" }, - { name = "pytest", marker = "extra == 'dev'" }, - { name = "pyyaml", specifier = ">=6.0" }, - { name = "ruff", marker = "extra == 'dev'" }, - { name = "soundfile", marker = "extra == 'audio'", specifier = ">=0.12.0" }, - { name = "uvicorn", specifier = ">=0.20.0" }, -] -provides-extras = ["dev", "mlx", "vision", "audio"] - [[package]] name = "macholib" version = "1.16.4" From 6d318a6e61b5240f70ba895884deb7cfd7ad9dd1 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Sun, 30 Aug 2026 22:51:42 +0200 Subject: [PATCH 004/182] Bound browser recording upload backlog (#10) Bound browser fallback upload backlog to 64 MiB / 24 pending chunks with lossless fail-closed finalization and localized saturation messaging. SCOPED remote preflight passed on the exact PR head. --- frontend/src/hooks/useRecorder.ts | 34 ++++++++++- frontend/src/i18n/locales/en.ts | 1 + frontend/src/i18n/locales/it.ts | 1 + frontend/src/utils/browserUploadBacklog.ts | 65 ++++++++++++++++++++++ test/test_frontend_browser_backpressure.py | 37 ++++++++++++ 5 files changed, 136 insertions(+), 2 deletions(-) create mode 100644 frontend/src/utils/browserUploadBacklog.ts create mode 100644 test/test_frontend_browser_backpressure.py diff --git a/frontend/src/hooks/useRecorder.ts b/frontend/src/hooks/useRecorder.ts index 2e5c9dc3..f5e65240 100644 --- a/frontend/src/hooks/useRecorder.ts +++ b/frontend/src/hooks/useRecorder.ts @@ -5,6 +5,7 @@ import { useTranslation } from '../i18n/i18n'; import { useToast } from '../context/ToastContext'; import { useAudioDevices, AudioDevice, AudioRouteStatus } from './useAudioDevices'; import { drawAudioMeterOnCanvas } from '../utils/audioVisualizer'; +import { BrowserUploadBacklog } from '../utils/browserUploadBacklog'; export type { AudioDevice, AudioRouteStatus }; @@ -84,6 +85,8 @@ export function useRecorder(onSaved?: (recording: Recording) => void) { const sessionIdRef = useRef(null); const sequenceRef = useRef>(new Map()); const uploadChainsRef = useRef>>(new Map()); + const browserUploadBacklogRef = useRef(new BrowserUploadBacklog()); + const browserBackpressureTriggeredRef = useRef(false); const startedAtRef = useRef(0); const routeActivatedRef = useRef(false); const captureBackendRef = useRef<'browser' | 'native'>('browser'); @@ -312,6 +315,11 @@ export function useRecorder(onSaved?: (recording: Recording) => void) { setStatusState('error'); setProgressText(t('recording.emptyRecordingWarning')); showToast(t('recording.emptyRecordingWarning'), 'error'); + } else if (browserBackpressureTriggeredRef.current) { + const message = t('recording.uploadBackpressure'); + setStatusText(t('common.error')); + setStatusState('error'); + setProgressText(message); } else { setStatusText(t('recording.saved')); setStatusState('success'); @@ -325,6 +333,8 @@ export function useRecorder(onSaved?: (recording: Recording) => void) { setStatusState('error'); showToast(t('recording.finalizationFailed', { error: error.message }), 'error'); } finally { + browserUploadBacklogRef.current.reset(); + browserBackpressureTriggeredRef.current = false; await restoreAudioRoute(); } }, [t, onSaved, releaseMedia, restoreAudioRoute, showToast]); @@ -730,6 +740,8 @@ export function useRecorder(onSaved?: (recording: Recording) => void) { localStorage.setItem('asr-active-recording-id', session.id); sequenceRef.current = new Map(); uploadChainsRef.current = new Map(); + browserUploadBacklogRef.current.reset(); + browserBackpressureTriggeredRef.current = false; // 5. Start MediaRecorder const recorderInputs: Array<{ trackId: string; stream: MediaStream }> = []; @@ -757,16 +769,34 @@ export function useRecorder(onSaved?: (recording: Recording) => void) { recorder.addEventListener('dataavailable', (event) => { if (!event.data || event.data.size === 0 || !sessionIdRef.current) return; + const blob = event.data; + const backlogSnapshot = browserUploadBacklogRef.current.accept(blob.size); const currentSequence = sequenceRef.current.get(trackId) || 0; sequenceRef.current.set(trackId, currentSequence + 1); const currentChain = uploadChainsRef.current.get(trackId) || Promise.resolve(); - const nextChain = currentChain.then(() => uploadChunk(trackId, event.data, currentSequence)); + const nextChain = currentChain + .then(() => uploadChunk(trackId, blob, currentSequence)) + .finally(() => { + browserUploadBacklogRef.current.release(blob.size); + }); uploadChainsRef.current.set(trackId, nextChain); nextChain.catch((error) => { setStatusText(t('common.error')); setStatusState('error'); showToast(t('recording.chunkSaveFailed', { error: error.message }), 'error'); }); + + if (backlogSnapshot.saturated && !browserBackpressureTriggeredRef.current) { + browserBackpressureTriggeredRef.current = true; + const message = t('recording.uploadBackpressure'); + setStatusText(t('recording.finalizing')); + setStatusState('working'); + setProgressText(message); + showToast(message, 'error'); + queueMicrotask(() => { + void stopRecording(); + }); + } }); recorder.addEventListener('error', (event: any) => { @@ -827,7 +857,7 @@ export function useRecorder(onSaved?: (recording: Recording) => void) { setStatusState('error'); showToast(t('recording.startFailed', { error: error.message }), 'error'); } - }, [t, selectedMicrophone, selectedSystemDevice, captureCapabilities, setCapturePermissions, startAudioMeter, loadDevices, releaseMedia, restoreAudioRoute, showToast]); + }, [t, selectedMicrophone, selectedSystemDevice, captureCapabilities, setCapturePermissions, startAudioMeter, loadDevices, releaseMedia, restoreAudioRoute, showToast, stopRecording]); const toggleTestAudioRoute = async () => { setIsVerifying(true); diff --git a/frontend/src/i18n/locales/en.ts b/frontend/src/i18n/locales/en.ts index 10193235..baed61fc 100644 --- a/frontend/src/i18n/locales/en.ts +++ b/frontend/src/i18n/locales/en.ts @@ -363,6 +363,7 @@ const en = { error: 'Recording error: {error}', startFailed: 'Unable to start recording: {error}', chunkSaveFailed: 'Chunk not saved: {error}', + uploadBackpressure: 'Upload backlog limit reached. ClosedRoom stopped and saved the recording early to protect memory.', finalizationFailed: 'Finalization failed: {error}', emptyRecordingWarning: 'The recording is empty. Please verify that ClosedRoom has Microphone and Screen Recording permissions in macOS System Settings.', routingActive: 'Active routing: {devices}', diff --git a/frontend/src/i18n/locales/it.ts b/frontend/src/i18n/locales/it.ts index f07a44b9..efe5a107 100644 --- a/frontend/src/i18n/locales/it.ts +++ b/frontend/src/i18n/locales/it.ts @@ -363,6 +363,7 @@ const it = { error: 'Errore registrazione: {error}', startFailed: 'Impossibile avviare la registrazione: {error}', chunkSaveFailed: 'Chunk non salvato: {error}', + uploadBackpressure: 'Il backlog di salvataggio ha raggiunto il limite. ClosedRoom ha fermato e salvato in anticipo la registrazione per proteggere la memoria.', finalizationFailed: 'Finalizzazione fallita: {error}', emptyRecordingWarning: 'La registrazione è vuota. Verifica che l\'app ClosedRoom abbia i permessi per Microfono e Registrazione schermo nelle Impostazioni di Sistema di macOS.', routingActive: 'Routing attivo: {devices}', diff --git a/frontend/src/utils/browserUploadBacklog.ts b/frontend/src/utils/browserUploadBacklog.ts new file mode 100644 index 00000000..d9087b9a --- /dev/null +++ b/frontend/src/utils/browserUploadBacklog.ts @@ -0,0 +1,65 @@ +export const DEFAULT_BROWSER_UPLOAD_MAX_PENDING_BYTES = 64 * 1024 * 1024; +export const DEFAULT_BROWSER_UPLOAD_MAX_PENDING_CHUNKS = 24; + +export type BrowserUploadBacklogSnapshot = { + pendingBytes: number; + pendingChunks: number; + maxPendingBytes: number; + maxPendingChunks: number; + saturated: boolean; + highWaterBytes: number; + highWaterChunks: number; +}; + +export class BrowserUploadBacklog { + private pendingBytes = 0; + private pendingChunks = 0; + private highWaterBytes = 0; + private highWaterChunks = 0; + + constructor( + private readonly maxPendingBytes = DEFAULT_BROWSER_UPLOAD_MAX_PENDING_BYTES, + private readonly maxPendingChunks = DEFAULT_BROWSER_UPLOAD_MAX_PENDING_CHUNKS, + ) { + if (maxPendingBytes < 1 || maxPendingChunks < 1) { + throw new Error('Browser upload backlog limits must be positive.'); + } + } + + accept(bytes: number): BrowserUploadBacklogSnapshot { + const normalizedBytes = Math.max(0, Math.floor(bytes)); + this.pendingBytes += normalizedBytes; + this.pendingChunks += 1; + this.highWaterBytes = Math.max(this.highWaterBytes, this.pendingBytes); + this.highWaterChunks = Math.max(this.highWaterChunks, this.pendingChunks); + return this.snapshot(); + } + + release(bytes: number): BrowserUploadBacklogSnapshot { + const normalizedBytes = Math.max(0, Math.floor(bytes)); + this.pendingBytes = Math.max(0, this.pendingBytes - normalizedBytes); + this.pendingChunks = Math.max(0, this.pendingChunks - 1); + return this.snapshot(); + } + + reset(): void { + this.pendingBytes = 0; + this.pendingChunks = 0; + this.highWaterBytes = 0; + this.highWaterChunks = 0; + } + + snapshot(): BrowserUploadBacklogSnapshot { + return { + pendingBytes: this.pendingBytes, + pendingChunks: this.pendingChunks, + maxPendingBytes: this.maxPendingBytes, + maxPendingChunks: this.maxPendingChunks, + saturated: + this.pendingBytes > this.maxPendingBytes || + this.pendingChunks > this.maxPendingChunks, + highWaterBytes: this.highWaterBytes, + highWaterChunks: this.highWaterChunks, + }; + } +} diff --git a/test/test_frontend_browser_backpressure.py b/test/test_frontend_browser_backpressure.py new file mode 100644 index 00000000..b9137d30 --- /dev/null +++ b/test/test_frontend_browser_backpressure.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import unittest +from pathlib import Path + + +ROOT = Path(__file__).parents[1] +HOOK = ROOT / "frontend" / "src" / "hooks" / "useRecorder.ts" +BACKLOG = ROOT / "frontend" / "src" / "utils" / "browserUploadBacklog.ts" + + +class BrowserRecordingBackpressureContractTests(unittest.TestCase): + def test_backlog_has_explicit_byte_and_chunk_budgets(self) -> None: + source = BACKLOG.read_text(encoding="utf-8") + self.assertIn("DEFAULT_BROWSER_UPLOAD_MAX_PENDING_BYTES", source) + self.assertIn("DEFAULT_BROWSER_UPLOAD_MAX_PENDING_CHUNKS", source) + self.assertIn("64 * 1024 * 1024", source) + self.assertIn("= 24", source) + self.assertIn("highWaterBytes", source) + self.assertIn("highWaterChunks", source) + + def test_recorder_fails_closed_without_pausing_or_dropping_chunks(self) -> None: + source = HOOK.read_text(encoding="utf-8") + self.assertIn("BrowserUploadBacklog", source) + self.assertIn("browserUploadBacklogRef", source) + self.assertIn("browserBackpressureTriggeredRef", source) + self.assertIn("backlogSnapshot.saturated", source) + self.assertIn(".finally(() =>", source) + self.assertIn("browserUploadBacklogRef.current.release", source) + self.assertIn("t('recording.uploadBackpressure')", source) + self.assertNotIn("browser upload backlog limit reached", source) + self.assertNotIn("recorder.pause()", source) + self.assertNotIn("event.data.size >", source) + + +if __name__ == "__main__": + unittest.main() From f1ab946d27b2d4e341f1549d581747c12d0cd6e2 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Sun, 30 Aug 2026 23:07:45 +0200 Subject: [PATCH 005/182] Expose bounded runtime resource telemetry (#11) Add on-demand privacy-safe process, sidecar, machine and heavy-workload resource telemetry without background polling. STRONG remote preflight passed on exact PR head, including arm64 packaged-app lifecycle smoke. --- src/local_asr_server/routers/system.py | 18 ++- .../runtime/resource_metrics.py | 148 ++++++++++++++++++ test/test_resource_metrics.py | 93 +++++++++++ test/test_runtime_resource_api.py | 46 ++++++ 4 files changed, 304 insertions(+), 1 deletion(-) create mode 100644 src/local_asr_server/runtime/resource_metrics.py create mode 100644 test/test_resource_metrics.py create mode 100644 test/test_runtime_resource_api.py diff --git a/src/local_asr_server/routers/system.py b/src/local_asr_server/routers/system.py index 22b3fd12..7f3c0c4a 100644 --- a/src/local_asr_server/routers/system.py +++ b/src/local_asr_server/routers/system.py @@ -23,6 +23,7 @@ ) from local_asr_server.asr_provider import asr_catalog from local_asr_server.macos_permissions import accessibility_status +from local_asr_server.runtime.resource_metrics import ResourceMetricsCollector logger = logging.getLogger("uvicorn.error") @@ -81,6 +82,7 @@ def health(request: Request) -> dict: "GET /v1/recordings/{id}/project", "GET /v1/projects", "GET /v1/runtime/status", + "GET /v1/runtime/resources", "GET /v1/runtime/services", "GET /v1/runtime/services/llm", "POST /v1/runtime/services/llm/start", @@ -307,9 +309,23 @@ def get_stats(request: Request): return stats +@router.get("/v1/runtime/resources") +def runtime_resources(request: Request): + runtime = get_services(request.app).runtime + llm = runtime.llm_status() + arbiter = getattr(request.app.state, "heavy_workload_arbiter", None) + return ResourceMetricsCollector().snapshot( + sidecar_pid=llm.get("pid"), + workload_arbiter=arbiter, + ) + + @router.get("/v1/runtime/status") def runtime_status(request: Request): - return get_services(request.app).runtime.status() + return { + **get_services(request.app).runtime.status(), + "resources": runtime_resources(request), + } @router.get("/v1/runtime/services") diff --git a/src/local_asr_server/runtime/resource_metrics.py b/src/local_asr_server/runtime/resource_metrics.py new file mode 100644 index 00000000..74b73653 --- /dev/null +++ b/src/local_asr_server/runtime/resource_metrics.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +import os +import platform +import resource +import subprocess +import time +from collections import Counter +from dataclasses import dataclass +from typing import Any, Protocol + + +class WorkloadSnapshotProvider(Protocol): + def snapshot(self) -> dict[str, object]: ... + + +@dataclass(frozen=True, slots=True) +class ProcessMemorySnapshot: + pid: int + current_rss_bytes: int | None + peak_rss_bytes: int | None + + def public(self) -> dict[str, Any]: + available = self.current_rss_bytes is not None or self.peak_rss_bytes is not None + return { + "status": "available" if available else "unknown", + "pid": self.pid, + "current_rss_bytes": self.current_rss_bytes, + "peak_rss_bytes": self.peak_rss_bytes, + } + + +class ResourceMetricsCollector: + """Collect privacy-safe resource state on demand without background polling. + + Resource telemetry is deliberately best-effort. Unsupported or unavailable + measurements are returned as ``None``/``unknown`` rather than being + represented as zero, which would make pressure diagnostics misleading. + """ + + def snapshot( + self, + *, + sidecar_pid: int | None = None, + workload_arbiter: WorkloadSnapshotProvider | None = None, + ) -> dict[str, Any]: + app_pid = os.getpid() + return { + "captured_at": time.time(), + "app_process": self.process_memory(app_pid, include_peak=True).public(), + "llm_sidecar": ( + self.process_memory(sidecar_pid, include_peak=False).public() + if sidecar_pid is not None + else { + "status": "not_running", + "pid": None, + "current_rss_bytes": None, + "peak_rss_bytes": None, + } + ), + "heavy_workloads": self._workload_snapshot(workload_arbiter), + "machine": self.machine_memory(), + } + + def process_memory(self, pid: int, *, include_peak: bool) -> ProcessMemorySnapshot: + current = self._current_rss_bytes(pid) + peak = self._self_peak_rss_bytes() if include_peak and pid == os.getpid() else None + return ProcessMemorySnapshot(pid=pid, current_rss_bytes=current, peak_rss_bytes=peak) + + @staticmethod + def _workload_snapshot(workload_arbiter: WorkloadSnapshotProvider | None) -> dict[str, Any]: + if workload_arbiter is None: + return { + "status": "unknown", + "max_concurrent": None, + "queue_capacity": None, + "queue_depth": None, + "active_count": None, + "pending_by_type": {}, + "active_by_type": {}, + } + raw = workload_arbiter.snapshot() + pending = raw.get("pending") if isinstance(raw.get("pending"), dict) else {} + active = raw.get("active") if isinstance(raw.get("active"), dict) else {} + return { + "status": "available", + "max_concurrent": raw.get("max_concurrent"), + "queue_capacity": raw.get("queue_capacity"), + "queue_depth": raw.get("queue_depth"), + "active_count": raw.get("active_count"), + "pending_by_type": dict(Counter(str(value) for value in pending.values())), + "active_by_type": dict(Counter(str(value) for value in active.values())), + "submitted": raw.get("submitted"), + "completed": raw.get("completed"), + "failed": raw.get("failed"), + "rejected": raw.get("rejected"), + "cancelled_pending": raw.get("cancelled_pending"), + "closed": raw.get("closed"), + } + + @staticmethod + def _current_rss_bytes(pid: int) -> int | None: + try: + completed = subprocess.run( + ["ps", "-o", "rss=", "-p", str(pid)], + capture_output=True, + text=True, + timeout=1.0, + check=False, + ) + except (OSError, subprocess.SubprocessError): + return None + if completed.returncode != 0: + return None + try: + rss_kib = int(completed.stdout.strip()) + except (TypeError, ValueError): + return None + return rss_kib * 1024 if rss_kib >= 0 else None + + @staticmethod + def _self_peak_rss_bytes() -> int | None: + try: + raw = int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) + except (AttributeError, OSError, TypeError, ValueError): + return None + if raw < 0: + return None + # Darwin reports bytes; Linux and most BSD CI environments report KiB. + return raw if platform.system() == "Darwin" else raw * 1024 + + @staticmethod + def machine_memory() -> dict[str, Any]: + total_bytes: int | None = None + try: + page_size = int(os.sysconf("SC_PAGE_SIZE")) + page_count = int(os.sysconf("SC_PHYS_PAGES")) + if page_size > 0 and page_count > 0: + total_bytes = page_size * page_count + except (AttributeError, OSError, TypeError, ValueError): + total_bytes = None + + return { + "status": "available" if total_bytes is not None else "unknown", + "system": platform.system() or None, + "machine": platform.machine() or None, + "physical_memory_bytes": total_bytes, + } diff --git a/test/test_resource_metrics.py b/test/test_resource_metrics.py new file mode 100644 index 00000000..f691bd7e --- /dev/null +++ b/test/test_resource_metrics.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +import os +import unittest +from types import SimpleNamespace +from unittest.mock import Mock, patch + +from local_asr_server.runtime.resource_metrics import ResourceMetricsCollector + + +class ResourceMetricsCollectorTests(unittest.TestCase): + def test_current_rss_uses_ps_kib_and_converts_to_bytes(self) -> None: + completed = SimpleNamespace(returncode=0, stdout="1234\n") + with patch("local_asr_server.runtime.resource_metrics.subprocess.run", return_value=completed) as run: + value = ResourceMetricsCollector._current_rss_bytes(42) + + self.assertEqual(value, 1234 * 1024) + run.assert_called_once() + + def test_current_rss_is_unknown_when_process_measurement_fails(self) -> None: + with patch( + "local_asr_server.runtime.resource_metrics.subprocess.run", + side_effect=OSError("ps unavailable"), + ): + self.assertIsNone(ResourceMetricsCollector._current_rss_bytes(42)) + + def test_peak_rss_normalizes_darwin_bytes_without_multiplying(self) -> None: + usage = SimpleNamespace(ru_maxrss=987654) + with patch("local_asr_server.runtime.resource_metrics.resource.getrusage", return_value=usage), patch( + "local_asr_server.runtime.resource_metrics.platform.system", return_value="Darwin" + ): + self.assertEqual(ResourceMetricsCollector._self_peak_rss_bytes(), 987654) + + def test_snapshot_never_represents_missing_sidecar_as_zero(self) -> None: + collector = ResourceMetricsCollector() + with patch.object(collector, "process_memory") as process_memory, patch.object( + collector, "machine_memory", return_value={"status": "unknown", "physical_memory_bytes": None} + ): + process_memory.return_value.public.return_value = { + "status": "unknown", + "pid": os.getpid(), + "current_rss_bytes": None, + "peak_rss_bytes": None, + } + snapshot = collector.snapshot(sidecar_pid=None, workload_arbiter=None) + + self.assertEqual(snapshot["llm_sidecar"]["status"], "not_running") + self.assertIsNone(snapshot["llm_sidecar"]["current_rss_bytes"]) + self.assertIsNone(snapshot["heavy_workloads"]["queue_depth"]) + self.assertNotEqual(snapshot["llm_sidecar"]["current_rss_bytes"], 0) + + def test_snapshot_aggregates_arbiter_state_without_job_identifiers(self) -> None: + collector = ResourceMetricsCollector() + arbiter = Mock() + arbiter.snapshot.return_value = { + "max_concurrent": 1, + "queue_capacity": 8, + "queue_depth": 3, + "active_count": 1, + "pending": {"job-2": "analysis", "job-3": "analysis", "job-4": "diarization"}, + "active": {"job-1": "transcription"}, + "submitted": 4, + "completed": 0, + "failed": 0, + "rejected": 0, + "cancelled_pending": 0, + "closed": False, + } + with patch.object(collector, "process_memory") as process_memory, patch.object( + collector, "machine_memory", return_value={"status": "available", "physical_memory_bytes": 16} + ): + process_memory.return_value.public.return_value = { + "status": "available", + "pid": 123, + "current_rss_bytes": 1024, + "peak_rss_bytes": None, + } + snapshot = collector.snapshot(sidecar_pid=123, workload_arbiter=arbiter) + + workloads = snapshot["heavy_workloads"] + self.assertEqual(workloads["queue_depth"], 3) + self.assertEqual(workloads["active_count"], 1) + self.assertEqual(workloads["pending_by_type"], {"analysis": 2, "diarization": 1}) + self.assertEqual(workloads["active_by_type"], {"transcription": 1}) + self.assertNotIn("pending", workloads) + self.assertNotIn("active", workloads) + self.assertNotIn("job-1", str(workloads)) + self.assertEqual(snapshot["llm_sidecar"]["current_rss_bytes"], 1024) + arbiter.snapshot.assert_called_once_with() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_runtime_resource_api.py b/test/test_runtime_resource_api.py new file mode 100644 index 00000000..c1b3ff7a --- /dev/null +++ b/test/test_runtime_resource_api.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import unittest +from types import SimpleNamespace +from unittest.mock import Mock, patch + +from local_asr_server.routers import system + + +class RuntimeResourceApiTests(unittest.TestCase): + def test_resource_snapshot_uses_managed_sidecar_pid_and_process_arbiter(self) -> None: + runtime = Mock() + runtime.llm_status.return_value = {"name": "llm", "pid": 4321} + arbiter = Mock() + request = SimpleNamespace(app=SimpleNamespace(state=SimpleNamespace(heavy_workload_arbiter=arbiter))) + services = SimpleNamespace(runtime=runtime) + + with patch.object(system, "get_services", return_value=services), patch.object( + system.ResourceMetricsCollector, + "snapshot", + return_value={"app_process": {"status": "available"}}, + ) as snapshot: + result = system.runtime_resources(request) + + self.assertEqual(result["app_process"]["status"], "available") + snapshot.assert_called_once_with(sidecar_pid=4321, workload_arbiter=arbiter) + + def test_runtime_status_adds_resources_without_replacing_service_status(self) -> None: + runtime = Mock() + runtime.status.return_value = {"services": {"llm": {"status": "ready"}}} + request = SimpleNamespace(app=SimpleNamespace(state=SimpleNamespace(heavy_workload_arbiter=None))) + services = SimpleNamespace(runtime=runtime) + + with patch.object(system, "get_services", return_value=services), patch.object( + system, + "runtime_resources", + return_value={"app_process": {"status": "unknown"}}, + ): + result = system.runtime_status(request) + + self.assertEqual(result["services"]["llm"]["status"], "ready") + self.assertEqual(result["resources"]["app_process"]["status"], "unknown") + + +if __name__ == "__main__": + unittest.main() From 33b28b21357d07d3e96e0814fb7de0a26381a7c0 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Sun, 30 Aug 2026 23:20:24 +0200 Subject: [PATCH 006/182] Align baseline revision with documentation governance --- .engineering/baseline.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.engineering/baseline.json b/.engineering/baseline.json index 4a3c4174..d716cf7b 100644 --- a/.engineering/baseline.json +++ b/.engineering/baseline.json @@ -2,7 +2,8 @@ "schema_version": 1, "standard": { "source": "daniele21/repo-template-sw", - "version": "0.8.0" + "version": "0.8.0", + "revision": "6677c5349d64ea6d935f1b460d03a47c236821bc" }, "target_level": "L2", "profiles": [ From 3dced321ca7de52ae55533c6040928380b9b4ef6 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Sun, 30 Aug 2026 23:20:57 +0200 Subject: [PATCH 007/182] Add documentation freshness to preflight --- skills/preflight-change/SKILL.md | 80 +++++++++++++++++++++++++------- 1 file changed, 62 insertions(+), 18 deletions(-) diff --git a/skills/preflight-change/SKILL.md b/skills/preflight-change/SKILL.md index 221741d8..c98abaf0 100644 --- a/skills/preflight-change/SKILL.md +++ b/skills/preflight-change/SKILL.md @@ -1,18 +1,22 @@ --- name: preflight-change -description: Establish exact-head automated-validation readiness by resolving material ambiguity, verifying target-base freshness, reviewing the complete diff, selecting validation depth and E2E environment fidelity from blast radius, classifying execution capability and routing every required deterministic gate without turning the user into a test runner. +description: Establish exact-head automated-validation readiness by resolving material ambiguity, verifying target-base freshness, reviewing the complete diff, proving affected durable documentation is current, selecting validation depth and E2E environment fidelity from blast radius, classifying execution capability and routing every required deterministic gate without turning the user into a test runner. --- # Preflight Change -Use this Skill immediately before pushing, opening/updating a PR, or otherwise publishing a change for automated validation. `validate-change` owns the iterative test loop; this Skill owns final publication/readiness, validation-depth selection, E2E environment-fidelity selection and execution routing. +Use this Skill immediately before pushing, opening/updating a PR, or otherwise publishing a change for automated validation. `validate-change` owns the iterative test loop; this Skill owns final publication/readiness, documentation freshness, validation-depth selection, E2E environment-fidelity selection and execution routing. -Read `EXECUTION-CAPABILITY-CONTRACT.md` when the current agent may lack a shell, checkout, SDK or platform toolchain. Read `.engineering/e2e.json` when the change affects a complete workflow or a platform/device/browser/runtime/environment-dependent claim. +Read `EXECUTION-CAPABILITY-CONTRACT.md` when the current agent may lack a shell, checkout, SDK or platform toolchain. Read `.engineering/e2e.json` when the change affects a complete workflow or a platform/device/browser/runtime/environment-dependent claim. Read `docs/README.md` when documentation ownership or README impact is not obvious. The governing rules are: > Validation depth follows blast radius: use the narrowest profile that proves the changed invariants. +> Code and durable documentation ship together: every affected canonical documentation owner must describe the exact-head behavior being published. + +> README identity and README usage are separate owners: do not rewrite stable mission/positioning for a usage-only change, and do not leave stale usage instructions because identity remains valid. + > E2E environment fidelity follows the claim: use the cheapest declared automated environment that represents the material target dimensions, then leave only irreducible fidelity gaps for real-environment confirmation. > CI should confirm locally reproducible deterministic failures when the agent has equivalent execution capability. @@ -63,7 +67,36 @@ Look for: A diff review is a semantic review, not only a formatting pass. -## 4. Select validation depth from blast radius +## 4. Assess documentation impact + +Determine documentation impact from the resulting observable behavior, not merely from which source files changed. Inspect the existing canonical owners before deciding `N/A`. + +At minimum classify: + +- `README_IDENTITY` — title/summary/`Why this exists`, primary audience/outcome and stable positioning; +- `README_USAGE` — prerequisites, setup, run/start, public configuration, public CLI/API/UI usage and copy-paste examples; +- `FEATURE_DOCS` — durable non-obvious feature behavior/constraints/evidence; +- `ARCHITECTURE` — boundaries and ownership; +- `ADR` — material durable decision/rationale; +- `SECURITY_DATA` — trust, privacy, security or data-lifecycle contract; +- `OPERATIONS` — canonical project command/operational semantics; +- `PRODUCT_EXPERIENCE` — adopted design/UX/brand contract when applicable; +- `CURRENT_STATE` — repository-level integrated/blocked/next truth. + +For each owner use `UPDATED` or `N/A`; when impact is plausible but classified `N/A`, state a short reason. + +README rules are deliberately section-specific: + +- changing implementation details, a feature workflow, setup, command syntax, configuration or defaults does **not** by itself justify rewriting project mission/positioning; +- changing the project's core purpose, primary audience or primary outcome requires reviewing README identity; +- any change that makes existing setup/run/use/configuration/examples incomplete, incorrect, removed, newly mandatory or misleading requires `README_USAGE: UPDATED` in the same change; +- a normal feature change may therefore produce `README_IDENTITY: N/A` and `README_USAGE: UPDATED`. + +For feature documentation, update an existing feature owner whenever the behavior it describes changed. Create a new feature document only when durable non-obvious behavior is not sufficiently discoverable from code, public contracts, tests or architecture; do not create one file per trivial feature. + +Publication is blocked when an affected canonical owner is stale. `verify_docs.py` can enforce structure/budgets but cannot prove semantic freshness, so this assessment remains part of diff/preflight review rather than being falsely delegated to a static checker. + +## 5. Select validation depth from blast radius Read `.engineering/commands.json` and use the project-owned selector to choose `auto -> LEAN | SCOPED | STRONG | FULL`. @@ -78,7 +111,7 @@ The selector must report the profile and reason. Unknown executable paths fail s Do not silently downgrade below `auto`. Explicit stronger validation is always allowed. If an attempted fix broadens blast radius — for example by adding a global Gradle or ProGuard change — re-run selection and allow escalation. -## 5. Select E2E journey and environment fidelity +## 6. Select E2E journey and environment fidelity When the selected profile/claim requires E2E, read `.engineering/e2e.json` before classifying executors. @@ -104,7 +137,7 @@ cheapest sufficient automated E2E If a required critical journey has no automated environment, retain its explicit `automation_gap_reason`; do not silently turn an undocumented human test into the primary E2E strategy. -## 6. Classify required gates by execution capability +## 7. Classify required gates by execution capability Use `validate-change`, the selected profile, `.engineering/commands.json` and any selected E2E environments to construct the final matrix. @@ -128,13 +161,13 @@ Do not classify a Gradle/R8/compiler/unit-test gate as `REAL_ENVIRONMENT` merely For an E2E gate, report both dimensions: executor classification and `.engineering/e2e.json` environment ID/fidelity class. -## 7. Execute or route deterministic validation +## 8. Execute or route deterministic validation Run every required `AGENT_LOCAL` gate in the selected validation profile on the exact current head. If all required deterministic gates are `AGENT_LOCAL` and pass, readiness may be `READY_FOR_CI`: remote CI is an independent confirmation environment and should use the same blast-radius profile or a deliberately stronger one. -If one or more required deterministic gates are `REMOTE_AUTOMATED` and all semantic/base/diff plus available local gates pass, readiness is `READY_FOR_REMOTE_PREFLIGHT`. Hand off immediately to `skills/remote-preflight/SKILL.md` and trigger repository-owned automation with the default `auto` profile unless a stronger profile is justified. +If one or more required deterministic gates are `REMOTE_AUTOMATED` and all semantic/base/diff/documentation plus available local gates pass, readiness is `READY_FOR_REMOTE_PREFLIGHT`. Hand off immediately to `skills/remote-preflight/SKILL.md` and trigger repository-owned automation with the default `auto` profile unless a stronger profile is justified. Do **not** ask the user to run an automatable deterministic command solely because the agent lacks a shell, checkout, SDK or toolchain. @@ -142,7 +175,7 @@ If a required deterministic gate is unavailable both locally and through reposit `REAL_ENVIRONMENT` evidence may remain pending after automated validation, but still blocks any stronger claim that depends on it. A target-device/manual run should primarily cover the residual fidelity gap declared for the journey, not act as the first complete workflow execution unless an explicit automation capability gap makes that unavoidable. -## 8. Diagnose failures before editing +## 9. Diagnose failures before editing For every failure, classify it before changing production code: @@ -157,11 +190,11 @@ Then identify the violated invariant and its owner. Fix the owner and add/streng Never delete, suppress, weaken or rewrite a legitimate gate simply to make the branch green unless the owning contract itself is intentionally changed. -If the same gate fails after a repair, stop symptom patching. Re-examine the cause, owner and assumptions and form a new falsifiable hypothesis before editing again. If that exposes material ambiguity, return to section 1 and ask the user. +If the same gate fails again after an attempted fix, stop symptom patching. Re-examine the cause, owner and assumptions and form a new falsifiable hypothesis before editing again. If that exposes material ambiguity, return to section 1 and ask the user. -After every material fix, reconsider the selected validation profile and E2E fidelity because the repair itself may broaden or narrow the blast radius or add a target-environment dependency. +After every material fix, reconsider documentation impact, the selected validation profile and E2E fidelity because the repair itself may change durable behavior, broaden/narrow blast radius or add a target-environment dependency. -## 9. Check command and evidence parity +## 10. Check command and evidence parity Deterministic automation should invoke the same project-owned canonical commands/scripts regardless of whether execution occurs agent-local or remotely. Workflow YAML may orchestrate scope detection, environment setup, caching and evidence, but should not secretly own a divergent formatter/test/build policy. @@ -171,7 +204,7 @@ If a real target-environment run repeatedly discovers ordinary complete-workflow If a remote run executes materially unrelated suites, improve the scope selector rather than accepting full-CI-by-default as permanent overhead. -## 10. Output readiness +## 11. Output readiness Report: @@ -181,6 +214,17 @@ TARGET: @ AMBIGUITY: PASS|FAIL BASE_FRESHNESS: PASS|FAIL FULL_DIFF_REVIEW: PASS|FAIL +DOCUMENTATION_IMPACT: + README_IDENTITY: UPDATED|N/A + README_USAGE: UPDATED|N/A + FEATURE_DOCS: UPDATED|N/A + ARCHITECTURE: UPDATED|N/A + ADR: UPDATED|N/A + SECURITY_DATA: UPDATED|N/A + OPERATIONS: UPDATED|N/A + PRODUCT_EXPERIENCE: UPDATED|N/A + CURRENT_STATE: UPDATED|N/A +DOCS_CURRENT_WITH_IMPLEMENTATION: PASS|FAIL VALIDATION_PROFILE: LEAN|SCOPED|STRONG|FULL PROFILE_REASON: EXECUTION_CAPABILITY: local|mixed|remote-only @@ -199,11 +243,11 @@ READINESS: READY_FOR_CI|READY_FOR_REMOTE_PREFLIGHT|AUTOMATED_PREFLIGHT_CONFIRMED Readiness meanings: -- `READY_FOR_CI` — all deterministic gates required by the selected profile could run agent-local and passed; CI can confirm independently; -- `READY_FOR_REMOTE_PREFLIGHT` — semantic/base/diff checks and all available local gates passed; required deterministic remote gates from the selected profile must now be triggered by the agent; -- `AUTOMATED_PREFLIGHT_CONFIRMED` — every deterministic automated gate required by the selected profile passed on the exact head/base at the required declared E2E fidelity, regardless of execution location; -- `NOT_READY_FOR_AUTOMATED_PREFLIGHT` — a required gate failed, profile/fidelity selection is unsafe, a material ambiguity/base/diff issue remains, or required automation routing is missing. +- `READY_FOR_CI` — documentation is current and all deterministic gates required by the selected profile could run agent-local and passed; CI can confirm independently; +- `READY_FOR_REMOTE_PREFLIGHT` — semantic/base/diff/documentation checks and all available local gates passed; required deterministic remote gates from the selected profile must now be triggered by the agent; +- `AUTOMATED_PREFLIGHT_CONFIRMED` — documentation is current and every deterministic automated gate required by the selected profile passed on the exact head/base at the required declared E2E fidelity, regardless of execution location; +- `NOT_READY_FOR_AUTOMATED_PREFLIGHT` — an affected canonical document is stale, a required gate failed, profile/fidelity selection is unsafe, a material ambiguity/base/diff issue remains, or required automation routing is missing. -Any later edit, rebase/merge/replay, dependency change or material target-base/environment relationship change invalidates the affected evidence and may change the selected profile or fidelity requirement. +Any later edit, rebase/merge/replay, dependency change or material target-base/environment relationship change invalidates the affected evidence and requires rechecking documentation impact as well as applicable validation/fidelity. A known-red draft may be published only when the user explicitly wants a collaboration/investigation artifact. State the known-red condition clearly; do not represent it as automated readiness. From 35942fc9fd7c7e7580e394b9a0157aaf017c2d20 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Sun, 30 Aug 2026 23:21:10 +0200 Subject: [PATCH 008/182] Align workstream finalization with durable docs --- skills/finalize-workstream/SKILL.md | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/skills/finalize-workstream/SKILL.md b/skills/finalize-workstream/SKILL.md index 48677710..b2a273da 100644 --- a/skills/finalize-workstream/SKILL.md +++ b/skills/finalize-workstream/SKILL.md @@ -11,30 +11,41 @@ Implementation plans are working memory. Code/tests/current durable docs are lon Completed plans are deleted by default. +A workstream is not documentation-complete merely because its code and tests are complete. The durable documentation affected by the resulting behavior must describe the system as it exists now. + ## Workflow 1. Read the workstream goal, invariants, DAG, acceptance and validation. 2. Confirm every required slice is `DONE` and no acceptance/evidence claim is unresolved. If real-device/hardware evidence is required but missing, the workstream is not fully complete; keep the relevant state truthful. 3. Inspect the resulting code/contracts/tests rather than trusting the plan's narrative. -4. Extract only knowledge that future maintainers need about the system **as it exists now**: +4. Assess documentation impact from the final observable behavior. Use `docs/README.md` when ownership is unclear. +5. Extract only knowledge that future maintainers/users need about the system **as it exists now**: + - core project purpose, primary audience or primary outcome -> README identity sections; + - setup, prerequisites, run/start, public configuration, public CLI/API/UI usage or examples -> README usage sections; - architecture/ownership changes -> `docs/architecture.md`; - - durable non-obvious feature behavior -> `docs/features/`; + - durable non-obvious feature behavior -> existing/new `docs/features/` owner; - material design decision/rationale -> ADR; + - security/trust/data-lifecycle contract -> `SECURITY.md` and/or owning architecture/feature doc; - operational procedure -> existing/new runbook only when genuinely recurring; + - canonical command semantics -> `.engineering/commands.json`; - executable invariant -> tests/tooling when possible. -5. Do not transfer PR numbers, commit diaries, sequence-of-implementation notes or resolved temporary blockers into durable docs. -6. Update `docs/current-state.md` to remove the workstream and expose the next current target/blocker. -7. Delete the completed workstream file by default. -8. Preserve it only when independent audit/regulatory/release/historical value exists; mark it historical and ensure it is not routed as current truth. -9. Search for stale links/references to the removed workstream and update them. -10. Run repository/docs/agent-context validation and relevant project tests. +6. Treat README identity and usage independently. Do not rewrite mission/positioning merely because a feature or command changed. Do update setup/run/use/configuration/examples when the old path would now be incomplete, wrong or misleading. +7. Do not transfer PR numbers, commit diaries, sequence-of-implementation notes or resolved temporary blockers into durable docs. +8. Update `docs/current-state.md` to remove the workstream and expose the next current target/blocker. +9. Delete the completed workstream file by default. +10. Preserve it only when independent audit/regulatory/release/historical value exists; mark it historical and ensure it is not routed as current truth. +11. Search for stale links/references, instructions, examples and configuration claims affected by the completed workstream and update them. +12. Run repository/docs/agent-context validation and relevant project tests. ## Completion questions - Can a future agent understand current behavior without the plan? +- Can a new user/developer follow the README's current setup/run/use path successfully? +- If README usage changed, did we avoid opportunistically rewriting still-valid identity/mission copy? - Is every durable fact in exactly one appropriate canonical owner? +- Are existing feature docs current for the behavior they describe? - Did we avoid copying implementation history into current docs? - Is current state now smaller and truthful? - Is the completed plan gone unless there is a concrete retention reason? -A successful finalization should normally reduce active documentation/context size. +A successful finalization should normally reduce active planning/context size while leaving durable documentation no less truthful than the implementation. From be3cef45556cf046c6464b2107189e540c18b6af Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Sun, 30 Aug 2026 23:21:28 +0200 Subject: [PATCH 009/182] Define documentation impact ownership --- docs/README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/README.md b/docs/README.md index 7d80580a..9610d94a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,6 +2,8 @@ Use documentation by ownership, not chronology. +- README identity sections — what ClosedRoom is, why it exists, its primary audience/outcome and stable positioning. +- README usage sections — current prerequisites, setup, run/start, configuration and public usage/examples. - `architecture.md` — current detailed architecture, ownership and system boundaries. - `features.md` — existing aggregate registry of current product behavior and verification hints. - `current-state.md` — short operational/maturity ledger and current gaps. @@ -11,3 +13,26 @@ Use documentation by ownership, not chronology. - `assets/` — bounded reference/demo assets; generated test evidence should live in CI artifacts instead. Older implementation/refactoring plans in this directory are historical inputs, not automatically current truth. Validate them against code, `architecture.md`, `features.md` and current tests before using them for implementation decisions. + +## Documentation impact contract + +Code and durable documentation ship together. A meaningful change is not complete until every affected canonical owner describes the system as it exists after that change. Do not update every document mechanically: update only affected owners and record plausible-but-unaffected owners as `N/A` during preflight. + +Treat the README as two semantic owners: + +- **Identity** changes only when ClosedRoom's purpose, primary audience/outcome or positioning changes. Do not opportunistically rewrite it for implementation, feature, command or configuration changes. +- **Usage** changes whenever prerequisites, setup, run/start, configuration, public API/UI workflow or copy-paste examples would otherwise become incomplete, incorrect or misleading. + +A change may therefore legitimately report `README_IDENTITY: N/A` and `README_USAGE: UPDATED`. + +Use this routing for other durable impact: feature behavior -> `features.md` or its bounded `features/` owner; architecture/ownership -> `architecture.md`; durable rationale -> ADR; trust/privacy/data lifecycle -> `SECURITY.md` and/or the owning architecture/feature doc; canonical command semantics -> `.engineering/commands.json`; product-experience contracts -> `design/*`; integrated/blocker/next truth -> `current-state.md`. + +## Lifecycle + +Assess documentation impact from observable behavior, not filenames. Search for the existing owner first. Existing feature documentation must be updated in the same change when the behavior it describes changes. Create a new feature document only when durable non-obvious behavior is not sufficiently discoverable from code, public contracts, tests or the existing aggregate registry. + +Active work remains disposable: + +`plan -> implement -> validate -> transfer durable knowledge -> delete plan` + +Do not create documentation merely to record that a PR or task completed. From 6f2b578df18681c3fa0428d372175cbf856f8b8e Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Sun, 30 Aug 2026 23:21:35 +0200 Subject: [PATCH 010/182] Require feature docs to track durable behavior --- docs/features/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/features/README.md b/docs/features/README.md index 8d6f666f..6ae8b8c8 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -3,3 +3,5 @@ `../features.md` remains ClosedRoom's canonical aggregate feature registry during the 0.8 baseline adoption. Do not duplicate the same behavior here. Create a bounded file in this directory only when a feature has enough durable behavior, failure semantics, persistence/configuration or verification detail that splitting it materially improves agent context and ownership. Link the new file once from the aggregate registry and keep one canonical owner for each fact. + +Feature documentation describes current durable behavior, not implementation progress. When a change alters behavior already described by `../features.md` or a bounded feature document, update that owner in the same change. Do not create a new document for a small feature when code, public contracts, tests and the existing registry already make the behavior sufficiently discoverable. From 29d15344c8c52786f3d45ef1624971f30517d6e8 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Sun, 30 Aug 2026 23:21:57 +0200 Subject: [PATCH 011/182] Route documentation freshness through agent workflow --- AGENTS.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index c014bcc4..491044bd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -35,6 +35,8 @@ ClosedRoom is a privacy-first macOS meeting workspace. It records microphone/sys - Prefer deterministic fixtures/mocks over production model downloads for cheap regressions. - Edit `frontend/src/`, not generated `src/local_asr_server/static/assets/` bundles. - Finalized `dist/artifacts/` build directories are immutable; create a new build identity instead of modifying one. +- Code and durable documentation ship together: before publication, assess documentation impact and update every affected canonical owner in the same change. +- README identity and README usage are separate owners. Do not rewrite stable mission/positioning for a usage-only change; do not leave stale setup/run/configuration/public examples because the identity remains valid. ## Ownership and routing @@ -50,6 +52,7 @@ ClosedRoom is a privacy-first macOS meeting workspace. It records microphone/sys | Frontend | `frontend/src/` | `design/*`, API contract, i18n, E2E | | Packaging/artifacts | `scripts/build_artifact.sh`, `build.sh`, `ClosedRoom.spec`, `build_assets/` | finalizer/smoke/E2E | | CI/preflight | selector + `.github/workflows/preflight.yml` | commands/E2E/tests | +| Documentation impact | `docs/README.md` | README identity/usage, feature/architecture/ADR/security/current-state owner | Public API changes require router/schema/service, frontend API consumers and tests. Persisted-data changes require migration/recovery compatibility review. @@ -59,6 +62,8 @@ Use the repo-template-sw 0.8 core skills in `skills/`: `structured-change`, `des ClosedRoom-specific skills remain local specializations; universal 0.8 contracts and this file govern conflicts. +Before publication, `preflight-change` must classify `README_IDENTITY`, `README_USAGE`, feature docs, architecture, ADR, security/data, operations, product experience and current state as `UPDATED` or `N/A`, and `DOCS_CURRENT_WITH_IMPLEMENTATION` must be `PASS`. + ## Project operating commands `.engineering/commands.json` is canonical: @@ -81,8 +86,10 @@ Reuse semantic components/tokens from `frontend/src/components/ui` and `frontend ## Documentation lifecycle +- README identity sections: what ClosedRoom is/why it exists/primary audience and outcome. Change only when those claims materially change. +- README usage sections: prerequisites/setup/run/configuration/public usage/examples. Update in the same change whenever old instructions become incomplete, wrong or misleading. - `docs/architecture.md`: detailed current architecture; intentionally larger local budget. -- `docs/features.md`: aggregate current feature registry; split into `docs/features/` only when useful. +- `docs/features.md`: aggregate current feature registry; split into `docs/features/` only when useful. Existing feature owners change in the same change as the durable behavior they describe. - `docs/current-state.md`: short operational ledger. - `docs/adr/`: accepted durable decisions only. - `docs/workstreams/`: active bounded plans only; delete completed plans after transferring durable truth. @@ -108,4 +115,4 @@ For package/native/runtime evidence use `bash scripts/build_artifact.sh --no-dmg ## Stop conditions -Surface conflicts instead of improvising when a request would create a second owner, silently move data to cloud, weaken auth/privacy, bypass migration review, leave unbounded resources, bypass cleanup/permission/command/E2E/design contracts, weaken tests for green CI, mutate a finalized artifact, or claim evidence that was not executed. +Surface conflicts instead of improvising when a request would create a second owner, silently move data to cloud, weaken auth/privacy, bypass migration review, leave unbounded resources, bypass cleanup/permission/command/E2E/design/documentation-freshness contracts, weaken tests for green CI, mutate a finalized artifact, or claim evidence that was not executed. From 0f757faaa913338625dbb369a0d0c85ba199837c Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Sun, 30 Aug 2026 23:22:07 +0200 Subject: [PATCH 012/182] Expose documentation impact in pull requests --- .github/pull_request_template.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 79121c45..a2807ddc 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -12,7 +12,23 @@ Describe the user/system outcome and the owning boundary changed. - Owning source(s): - Public/persistence/security/runtime/UI contract impact: -- Documentation/design contracts updated or N/A: + +## Documentation impact + +Classify each owner as `UPDATED` or `N/A`; give a short reason when impact was plausible but is `N/A`. + +- README_IDENTITY: +- README_USAGE: +- FEATURE_DOCS: +- ARCHITECTURE: +- ADR: +- SECURITY_DATA: +- OPERATIONS: +- PRODUCT_EXPERIENCE: +- CURRENT_STATE: +- DOCS_CURRENT_WITH_IMPLEMENTATION: PASS / FAIL + +README identity means purpose/audience/outcome/positioning. README usage means prerequisites/setup/run/configuration/public API/UI/examples. A usage-only change must not trigger an opportunistic mission rewrite. ## Validation @@ -29,6 +45,7 @@ If E2E applies, record the journey, `.engineering/e2e.json` environment ID, fide - [ ] Material ambiguity resolved - [ ] Intended target base/head identity checked - [ ] Complete diff reviewed for unrelated/generated/private residue +- [ ] Documentation impact assessed and every affected canonical owner is current - [ ] Required deterministic gates for the selected profile passed or are explicitly routed - [ ] Failure root causes were diagnosed rather than suppressed - [ ] Cleanup/residue expectations are satisfied for executed runtime/E2E/build work From 13d0c7ba029953b722492b3d055235a4e75a7c5b Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Mon, 31 Aug 2026 04:28:49 +0200 Subject: [PATCH 013/182] runtime: enforce phase-scoped model residency Keep at most one registered local LLM/VLM resident per managed phase, return the sidecar to a cold state after local analysis/visual workloads, preserve external-server ownership, and fall back to process stop/restart for explicit/custom model paths. Update architecture and feature documentation with the canonical arbiter/lease/residency ownership, with tests covering success, failure and override propagation. --- docs/architecture.md | 13 ++ docs/features.md | 4 +- src/local_asr_server/runtime/llm_sidecar.py | 209 ++++++++++++++++-- src/local_asr_server/runtime/models.py | 21 ++ .../runtime/service_manager.py | 30 ++- .../services/analysis_service.py | 53 +++-- .../visual_intelligence/service.py | 5 + test/test_llm_sidecar.py | 111 +++++++++- test/test_model_residency.py | 98 ++++++++ test/test_runtime_services.py | 15 ++ test/visual_intelligence_support.py | 7 + 11 files changed, 514 insertions(+), 52 deletions(-) create mode 100644 test/test_model_residency.py diff --git a/docs/architecture.md b/docs/architecture.md index c8d1db90..21016747 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -458,6 +458,19 @@ supera `speaker_diarization_minimum_overlap`. Il valore assegnato ha forma 6. applica un mapping conservativo solo a cluster speaker già esistenti; 7. conserva i JPEG come artefatti della registrazione per consultazione e retry. +Il sidecar LLM/VLM gestito parte con un solo modello residente. Per i modelli +di prodotto registrati (`nemotron-nano-4b`, `nemotron-nano-4b-q8`, +`qwen3-vl-4b`) `RuntimeServiceManager` usa il control plane pubblico di +local-llm-server 0.4 per attivare il modello richiesto dopo aver evacuato +quello della fase precedente. Al termine di visual intelligence o analisi +locale, tutti i runtime registrati vengono scaricati e il sidecar resta sano +in stato zero-resident. Endpoint `external` non vengono mai mutati. Modelli +con path configurato esplicitamente, custom o non registrati mantengono il +boundary più conservativo stop/restart del processo posseduto, così +l'artefatto esplicito non viene perso. Un failure +del control plane durante il cleanup degrada a stop del sidecar posseduto, +evitando residency orfana e senza mascherare il risultato del workload. + Dopo tre errori infrastrutturali consecutivi dal backend visuale, un circuit breaker interrompe le richieste residue, marca il checkpoint come `retryable_failure` e conserva lo staging. Il successivo avvio non riusa dalla diff --git a/docs/features.md b/docs/features.md index e7aa00be..6e05d274 100644 --- a/docs/features.md +++ b/docs/features.md @@ -47,7 +47,7 @@ e cambiata. | Diagnostica meeting e fallback | Rende visibile quando il transcript è valido ma un arricchimento è fallito o ha usato un backend sostitutivo, evitando successi verdi ingannevoli. | Contratto centrale `diagnostics.py`; report condiviso `meeting_diagnostics.py`; outcome ed eventi in `TranscriptionService`/`TranscriptionJobManager`; endpoint autenticato `/v1/meetings/{id}/diagnostics` consumato dal drawer; la pagina risultato Trascrizione mostra immediatamente cause e backend effettivi; CLI `local-asr inspect-meeting`, inoltrata anche dall'eseguibile PyInstaller. `macos_permissions.py` e `/v1/system/accessibility` impediscono l'avvio silenzioso dei global hotkey senza permesso e alimentano il warning Settings. | `stats.diagnostics` e `stats.outcome_status` nel transcript JSON/SQLite, payload degli eventi job e log ruotato `~/Library/Logs/ClosedRoom/closedroom.log`; registra backend richiesto/effettivo per traccia, fallback, causa, errore, contatori e durata. I log includono `recording_id`/`job_id` e redigono token e secret. Il permesso Accessibilità riguarda solo le scorciatoie globali. | `.venv/bin/python -m unittest discover -s test -p 'test_diagnostics.py' -v`; `test_frontend_diagnostics.py`; `test_macos_permissions.py`; `test_bundled_module_dispatch.py`; test diarizzazione/visual/audio intelligence e matrice negativa; `cd frontend && pnpm run build`; `./build.sh --no-dmg` e smoke `Contents/MacOS/ClosedRoom inspect-meeting … --json`. | | Audio intelligence shadow | Arricchisce le trascrizioni di registrazioni con metriche locali leggere su canali, tempo parlato, pause, overlap, speech rate, energia e insight mock provvisori, senza chiamare LLM e senza generare clip audio persistenti; la UI mostra una card dedicata nel dettaglio registrazione e badge sui segmenti trascritti. | Modulo `audio_intelligence`, integrazione in `run_recording_transcription()`, endpoint read-only `/v1/recordings/{id}/intelligence`, client React `ApiClient.recordingIntelligence()` e pannello `AudioIntelligencePanel`; calcolo RMS a finestre con lettura streaming WAV o pipe `ffmpeg`. | `intelligence.json` compatto nella directory registrazione; summary in `stats.audio_intelligence`; segmenti arricchiti con `channel`, `pause_before`, `speech_rate_wpm`, `energy`, `overlap`; `analysis` resta riservato al futuro risultato LLM. | `UV_CACHE_DIR=.cache/uv uv run python -m unittest discover -s test -p 'test_audio_intelligence.py' -v`; `UV_CACHE_DIR=.cache/uv uv run python -m unittest discover -s test -p 'test_recording_api.py' -v`; `cd frontend && npm run build`. | | Diarizzazione speaker post-meeting | Separa gli interlocutori in cluster temporali senza richiedere a priori il numero di partecipanti. Nella fase Configura l'utente sceglie per ogni run `Disattivata`, FluidAudio locale o Speechmatics cloud solo diarizzazione, indipendentemente dal provider ASR e anche per file importati; su una trascrizione salvata può inoltre ricalcolare soltanto gli speaker senza rilanciare l'ASR. Nelle registrazioni a tracce separate il MIC resta un unico speaker noto e soltanto SYSTEM viene processato dal backend scelto. Tutti i cluster rilevati nell'audio restano nominabili anche quando non hanno testo Whisper sovrapposto. | `speaker_diarization.py` possiede FluidAudio, lease runtime, cluster grezzi, copertura transcript e assegnazione temporale; l'helper Swift usa il profilo Community-1 accurato (`stepRatio=0.1`, durata minima embedding `0`, zero-vote re-embedding). `transcription_diarization.py` orchestra diarizzazione iniziale e rerun track-aware, riusa `SpeechmaticsBatchASRProvider`, scarta il testo cloud e sostituisce solo cluster/label; `speaker_labels.py` unisce cluster grezzi e segmenti assegnati. `TranscriptionService` applica la scelta per-run alle registrazioni e `routers/transcriptions.py` ai file upload/path. Il payload usa `diarization_provider`; endpoint rerun `POST /v1/transcriptions/{id}/diarization-jobs`, `TranscriptionJobManager`/`JobStore`, `TranscriptionStore.replace_diarization()`, frontend `ConfigureStep.tsx`/`ResultsStep.tsx`. | La cache pipeline è versionata e include provider, regione e modello di diarizzazione. `speaker-diarization.json` conserva timeline e risultato più recente; transcript JSON/SQLite salvano `provider_speaker`, `speaker_label`, `stats.speaker_diarization`, `clusters_by_track`, `assigned_cluster_count` e `unassigned_clusters_by_track`. Il conteggio speaker deriva dalla timeline del backend, non dai soli segmenti ASR assegnati; ogni mapping espone `transcript_segment_count`. Il rerun rimuove mapping/nominativi precedenti perché gli ID cluster non sono stabili. Speechmatics richiede API key e conferma UI e può generare costi. | `UV_CACHE_DIR=.cache/uv uv run python -m unittest discover -s test -p 'test_transcription_diarization.py' -v`; `test_speaker_diarization.py`; `test_speaker_labels.py`; `test_caching.py`; `test_job_store.py`; `test_recording_api.py`; compilazione release SwiftPM; `cd frontend && pnpm run build`; `./build.sh --no-dmg`; niente chiamate Speechmatics reali nei test automatici. | -| Visual intelligence post-meeting | Raccoglie durante la registrazione frame JPEG temporizzati persistenti e, durante il transcription job post-meeting, usa Qwen3-VL per produrre evidenze strutturate senza riconoscimento facciale. La schermata di configurazione e l'overlay flottante indicano esplicitamente quale finestra o schermo ClosedRoom sta acquisendo, così la sorgente visiva resta verificabile per tutta la registrazione. I frame restano disponibili dopo successi, errori e retry e vengono rimossi soltanto con la cancellazione esplicita della registrazione. La policy stabile `v1` deduplica con dHash e associa cluster `provider_speaker`; la soglia configurabile `visual_frame_similarity_threshold` (default `12`, range `0..64`) tratta come riusabili anche i frame quasi duplicati, riducendo le chiamate Qwen senza contarli come inferenze indipendenti. L'evoluzione task-aware, tracciata in `docs/task-aware-visual-intelligence-plan.md`, separa task (speaker, stato meeting, contenuto condiviso), usa prompt dedicati e aggrega lo stato temporale. Supporta il tracciamento dettagliato tramite run.json, trace.jsonl e routing.jsonl scritti in visual-runs//, con un tab Visual Debug UI in React. La pipeline ottimizza la memoria riducendo le dimensioni dei frame in base al task e salvando preview compresse WEBP per il debugging, serializzando l'esecuzione tramite ModelRuntimeLeaseManager ed eseguendo ASR in un processo separato. | `visual_intelligence/` con contratti, firme, router, prompt, fusion, trace_store.py e pipeline run config; `PostMeetingVisualService` orchestratore. API `POST /v1/recordings/{id}/visual-frames`, `GET /v1/recordings/{id}/visual-frames`, `GET /v2/recordings/{id}/visual-debug`, `GET /v2/recordings/{id}/visual-runs/{gen_id}/previews/{file}`. Frontend React `RecordingPage.tsx`, `useRecorder.ts`, `RecordingOverlayPage.tsx`, `ResultsStep.tsx` e `VisualDebugPanel.tsx`; il nome della sorgente selezionata viene propagato all'overlay tramite il canale di stato della registrazione senza introdurre nuova persistenza. Runtime worker ASR `ASRProcessRunner` in `runtime/asr_worker.py` e CLI `local-asr transcribe`. `RecordingStore` possiede la retention dei frame e `ModelRuntimeLeaseManager` serializza il runtime. | JPEG e manifest restano in `.visual-staging/` nella directory della registrazione; gli artefatti di analisi vengono promossi a `visual-runs//` (run.json, routing.jsonl, trace.jsonl, observations.jsonl, result.json, metrics.json, previews/) con `current_visual_generation.json` nel root. Il cleanup TTL rimuove soltanto checkpoint e generazioni incomplete, non i frame. Settings `visual_frame_similarity_threshold` per la forza del filtro v1 e `visual_debug_previews_enabled` per abilitare le anteprime; l'etichetta mostrata nell'overlay è stato UI effimero e non viene salvata. | `UV_CACHE_DIR=.cache/uv uv run python -m unittest discover -s test -p 'test_visual_intelligence_service.py' -v`, `test_frontend_diagnostics.py`, `test_recording_api.py` e `cd frontend && npm run build`; verifica manuale che overlay compatto ed espanso mostrino la sorgente selezionata. | +| Visual intelligence post-meeting | Raccoglie durante la registrazione frame JPEG temporizzati persistenti e, durante il transcription job post-meeting, usa Qwen3-VL per produrre evidenze strutturate senza riconoscimento facciale. La schermata di configurazione e l'overlay flottante indicano esplicitamente quale finestra o schermo ClosedRoom sta acquisendo, così la sorgente visiva resta verificabile per tutta la registrazione. I frame restano disponibili dopo successi, errori e retry e vengono rimossi soltanto con la cancellazione esplicita della registrazione. La policy stabile `v1` deduplica con dHash e associa cluster `provider_speaker`; la soglia configurabile `visual_frame_similarity_threshold` (default `12`, range `0..64`) tratta come riusabili anche i frame quasi duplicati, riducendo le chiamate Qwen senza contarli come inferenze indipendenti. L'evoluzione task-aware, tracciata in `docs/task-aware-visual-intelligence-plan.md`, separa task (speaker, stato meeting, contenuto condiviso), usa prompt dedicati e aggrega lo stato temporale. Supporta il tracciamento dettagliato tramite run.json, trace.jsonl e routing.jsonl scritti in visual-runs//, con un tab Visual Debug UI in React. La pipeline ottimizza la memoria riducendo le dimensioni dei frame in base al task e salvando preview compresse WEBP per il debugging; i workload pesanti sono serializzati da HeavyWorkloadArbiter, ModelRuntimeLeaseManager resta un phase marker e la residency Qwen viene acquisita solo alla prima inferenza necessaria e rilasciata in finally, riportando il sidecar gestito allo stato cold. ASR continua a essere eseguito in un processo separato. | `visual_intelligence/` con contratti, firme, router, prompt, fusion, trace_store.py e pipeline run config; `PostMeetingVisualService` orchestratore. API `POST /v1/recordings/{id}/visual-frames`, `GET /v1/recordings/{id}/visual-frames`, `GET /v2/recordings/{id}/visual-debug`, `GET /v2/recordings/{id}/visual-runs/{gen_id}/previews/{file}`. Frontend React `RecordingPage.tsx`, `useRecorder.ts`, `RecordingOverlayPage.tsx`, `ResultsStep.tsx` e `VisualDebugPanel.tsx`; il nome della sorgente selezionata viene propagato all'overlay tramite il canale di stato della registrazione senza introdurre nuova persistenza. Runtime worker ASR `ASRProcessRunner` in `runtime/asr_worker.py` e CLI `local-asr transcribe`. `RecordingStore` possiede la retention dei frame; `HeavyWorkloadArbiter` possiede admission e serializzazione dei workload pesanti, mentre `ModelRuntimeLeaseManager` registra soltanto la fase runtime attiva. | JPEG e manifest restano in `.visual-staging/` nella directory della registrazione; gli artefatti di analisi vengono promossi a `visual-runs//` (run.json, routing.jsonl, trace.jsonl, observations.jsonl, result.json, metrics.json, previews/) con `current_visual_generation.json` nel root. Il cleanup TTL rimuove soltanto checkpoint e generazioni incomplete, non i frame. Settings `visual_frame_similarity_threshold` per la forza del filtro v1 e `visual_debug_previews_enabled` per abilitare le anteprime; l'etichetta mostrata nell'overlay è stato UI effimero e non viene salvata. | `UV_CACHE_DIR=.cache/uv uv run python -m unittest discover -s test -p 'test_visual_intelligence_service.py' -v`, `test_frontend_diagnostics.py`, `test_recording_api.py` e `cd frontend && npm run build`; verifica manuale che overlay compatto ed espanso mostrino la sorgente selezionata. | Scelta Qwen per-run: prima di trascrivere una registrazione salvata, la UI permette di abilitare o disabilitare l'analisi immagini per quella singola @@ -157,7 +157,7 @@ fixture condivise. | Meeting workspace | Riorganizza registrazioni, trascrizioni, diarizzazione, analisi e stato job attorno al meeting come unità operativa, senza introdurre una tabella Meeting parallela. Tutti gli ingressi alla trascrizione di una registrazione aprono lo stesso workflow guidato e al completamento tornano al dettaglio meeting canonico, identico a quello aperto da storico, ricerca e progetti. Nel meeting sono visibili stato/backend/conteggi della diarizzazione, nomi speaker modificabili e transcript segmentato per speaker e timestamp; i file importati senza registrazione mantengono invece un risultato transcript standalone. | Endpoint `/v1/meetings`, `/v1/meetings/{recording_id}`, helper `_build_meeting()`, UI React `DashboardPage` ("Oggi"), `MeetingDetailPage`, `RecordingPage`, `ProjectsPage`, `TranscriptionPage` e `ResultsStep`; `SpeakerDiarizationEditor` è il componente condiviso per visibilità diarizzazione e modifica nomi; `utils/transcriptionRoute.ts` possiede costruzione e parsing dei token route. `TranscriptionPage` risolve il risultato completato sul meeting quando esiste `recording_id`; la pagina detail del meeting riorganizza le informazioni principali in un sistema a tab (Trascrizione, Analisi AI, Speaker) che rende la trascrizione immediatamente visibile ed accessibile all'ingresso. | Recording, transcription, `analysis_runs` e `jobs` restano le fonti di verita; lo stato meeting è derivato (`recorded`, `transcribed`, `analyzing`, `ready`) e non persistito separatamente. I nomi speaker sono aggiornati tramite `PATCH /v1/transcriptions/{id}/speakers` e il transcript segmentato usa subito `speaker_name`, mapping salvato o cluster come fallback. L'identità persistita è `Transcription.id`; `saved_id` è accettato soltanto come alias del payload di completamento. | `cd frontend && npm run build`; TestClient su `/v1/meetings`; verifica manuale Oggi -> dettaglio meeting -> Trascrivi -> ritorno allo stesso dettaglio; modificare un nome speaker, salvare e verificare l'etichetta nei segmenti; controllare stato/backend/conteggi, fallback senza cluster e accesso ai tab. | | Progetti | Raggruppa audio, trascrizioni e analisi per contesto di lavoro o meeting in un workspace master-detail con sidebar ottimizzata (descrizione a tooltip, glow ridotto, collassabile su desktop con avatar iniziali e tooltip) e mobile drawer overlay da sinistra, hero bar compatto con pulsante trigger hamburger, grid orizzontale di KPI clickabili, sezione situazione progetto integrata full-width con CTA "Domanda custom" e "Genera" incluse, azioni a tutta larghezza, decisioni e rischi affiancati, timeline collassabile e progressive disclosure. | `project_name` su registrazioni, `_build_projects()`, endpoint `/v1/projects` e `/v1/recordings/{id}/project`, frontend `ProjectsPage`, componenti workspace React (`ProjectSidebar`, pannelli insight, `TaskProcessingLoader`) e storage key frontend per precompilare il progetto quando si avvia un nuovo meeting dal progetto. `_build_projects()` espone anche `analysis_runs` e preferisce l'ultima run completata rispetto al campo legacy `transcription.analysis`; il sistema visuale premium condiviso vive in `frontend/src/index.css` e nei primitive UI React. | Campo `project_name` in metadata registrazione e tabella `recordings`; run analisi in `CatalogStore.analysis_runs`; la situazione progetto MVP è composta lato frontend da `latest_analysis`/`analysis_runs` già disponibili, mostra uno stato di generazione breve e non crea ancora un digest progetto persistente. | `cd frontend && npm run build`; verifica API `/v1/projects`; verifica UI sidebar progetto collassabile (desktop) e mobile drawer con backdrop (mobile), filtro range, CTA nuovo meeting con progetto precompilato, loader situazione progetto, microcopy "cosa aspettarti", digest locale e toggle timeline (Mostra tutti/meno). | | Analisi AI | Trasforma trascrizioni, registrazioni o testo inline in sintesi, punti chiave e azioni tramite job persistenti osservabili e tipizzati per output meeting; il run permette di scegliere provider, modello e setup senza dipendere solo dai default globali. | UI React e API client usano `POST /v1/analysis-jobs`, `POST /v1/analysis-pipelines`, `GET /v1/analysis/templates`, `GET /v1/analysis/pipelines` e polling di `GET /v1/jobs/{job_id}`; `/v1/analysis` resta legacy/debug. `analysis_templates.py` è il registry backend; `AnalysisJobManager`, `AnalysisService`, `LLMService`, provider `mock`, Gemini e locali; `frontend/src/api/config.ts` centralizza cataloghi provider/modelli/preset, `AnalysisSetupModal` apre il setup per le pipeline da meeting e `AnalysisPage` passa gli stessi override. | Provider, `gemini_model`, API key e default locali in `settings.json`; `GET /v1/settings` non restituisce mai la chiave, ma `gemini_api_key_configured`. `AnalysisRequest` e `AnalysisPipelineRequest` accettano override per-run per Gemini e local LLM; `analysis_runs.model`, `analysis_runs.llm_options` e la cache analysis includono provider, modello, preset qualita, temperatura, reasoning, max token, JSON mode, backend/path locali e hash credenziale Gemini senza salvare secret. | `UV_CACHE_DIR=.cache/uv uv run python -m unittest discover -s test -p 'test_analysis_api.py' -v`; `cd frontend && npm run build`; TestClient pipeline mock. | -| Analisi locale tramite local-llm-server | Abilita l'analisi locale offline delle trascrizioni (Nemotron locale) e delle registrazioni audio (Voxtral locale con audio diretto) senza chiamate cloud; in modalita `auto` ClosedRoom avvia il sidecar al bisogno e ricarica automaticamente il runtime quando cambia la configurazione effettiva del modello. L'header offre inoltre un accesso diretto alla Web UI del runtime in una finestra separata. | Modulo `llm.py` (`NemotronLocalProvider`, `VoxtralLocalProvider`), `runtime/models.py` per host/porte default, `runtime/llm_sidecar.py` per processo, porta dinamica, readiness, confronto configurazione e log, `runtime/service_manager.py` per facciata runtime e override per-run, `services/analysis_service.py` per merge impostazioni/request, `ensure_llm_ready` e chiave cache, `CatalogStore.analysis_cache`, `schemas.py` (`AnalysisRequest`, `AnalysisPipelineRequest`, `SettingsRequest`), `routers/system.py` (`POST /v1/analysis`, `GET /v1/runtime/status`, `GET /v1/runtime/services`, `GET /v1/runtime/services/llm`, `POST /start`, `/stop`, `/restart`, `GET /logs`, `POST /v1/settings`), frontend `App.tsx`, `AnalysisSetupModal`, `AnalysisPage`, `SettingsPage` e `apiClient.ts`; il pulsante header legge l'URL effettivo dal runtime e, in modalità gestita, avvia il sidecar se necessario; `run.sh` segue il log sidecar nel terminale di sviluppo. | `llm_provider` sceglie l'adapter, `local_llm_model` il modello caricato; il client locale invia esplicitamente quel modello. Analisi testuali e audio riusano un risultato SQLite quando coincidono contenuto (SHA-256), prompt/domanda/task, provider/modello e impostazioni che possono modificare l'output; per Gemini viene registrato soltanto l'hash della credenziale. `local_llm_mode` (`auto`, `external`, `disabled`), `local_llm_url` come override external/dev, `local_llm_model_path`, `local_llm_backend`, `local_llm_mmproj_path`, `local_llm_ctx_size`, `local_llm_startup_timeout`, `local_llm_llama_server_bin` e `local_llm_model_paths` vivono nei default persistenti ma possono essere risolti per singolo run tramite payload analysis; il sidecar gestito si riavvia se cambia uno di questi valori ed e bindato a `127.0.0.1` con porta interna disponibile. | `PYTHONPATH=src python -m unittest discover -s test -p 'test_caching.py' -v`; `PYTHONPATH=src python -m unittest discover -s test -p 'test_analysis_api.py' -v`; `cd frontend && npm run build`; test runtime esistenti quando il wheel locale `local-llm-server` e coerente. | +| Analisi locale tramite local-llm-server | Abilita l'analisi locale offline delle trascrizioni (Nemotron locale) e delle registrazioni audio (Voxtral locale con audio diretto) senza chiamate cloud. In modalità `auto` ClosedRoom avvia il sidecar al bisogno, mantiene residente al massimo il modello registrato necessario alla fase corrente e, al termine di analisi o visual intelligence, rilascia la residency tornando a uno stato cold/zero-resident. Gli endpoint `external` restano caller-owned e non vengono mutati; path modello espliciti/custom conservano invece l'artefatto esatto tramite il boundary più conservativo stop/restart. L'header offre inoltre accesso diretto alla Web UI del runtime. | Modulo `llm.py` (`NemotronLocalProvider`, `VoxtralLocalProvider`), `runtime/models.py` per host/porte e provenienza del model path, `runtime/llm_sidecar.py` per processo, readiness, control plane admin 0.4, switch/unload e fallback di reclamazione, `runtime/service_manager.py` per facciata runtime, override per-run e release della residency, `services/analysis_service.py` per merge impostazioni/request, `ensure_llm_ready`, cleanup `finally` e chiave cache. `HeavyWorkloadArbiter` possiede admission/serializzazione dei workload pesanti; `ModelRuntimeLeaseManager` è soltanto il phase marker. Il pulsante header legge l'URL effettivo dal runtime e, in modalità gestita, riattiva il modello quando il processo è vivo ma cold. | `llm_provider` sceglie l'adapter e `local_llm_model` il modello richiesto. `local_llm_mode` (`auto`, `external`, `disabled`), `local_llm_url`, `local_llm_model_path`, `local_llm_backend`, `local_llm_mmproj_path`, `local_llm_ctx_size`, `local_llm_startup_timeout`, `local_llm_llama_server_bin` e `local_llm_model_paths` restano nei default persistenti e possono essere risolti per singolo run. Per Nemotron/Qwen registrati senza path esplicito il sidecar usa l'admin API di local-llm-server 0.4 per evacuare il modello precedente e attivare quello richiesto, conservando per Qwen la porta VLM privata scelta da ClosedRoom; path espliciti/custom e failure del control plane ricadono su stop/restart del processo posseduto. Il servizio resta bindato a `127.0.0.1` su porta interna disponibile. Cache e persistenza dell'analisi mantengono gli input/setting già previsti senza salvare secret. | `PYTHONPATH=src python -m unittest discover -s test -p 'test_llm_sidecar.py' -v`; `PYTHONPATH=src python -m unittest discover -s test -p 'test_model_residency.py' -v`; `PYTHONPATH=src python -m unittest discover -s test -p 'test_analysis_api.py' -v`; suite source/integration, build arm64 e smoke della `.app` per cambi lifecycle/runtime. | | Aggiornamento di local-llm-server | Mantiene il runtime locale allineato all'ultimo tag semver stabile pubblicato, usando esclusivamente wheel precompilati e installando in modo riproducibile l'extra `vision` richiesto da Qwen. | `scripts/update_local_llm_server.py` interroga i tag GitHub, riusa il wheel esatto da `../local-llm-server/dist/` oppure lo scarica dagli asset della release con `gh`, preserva gli extra nel requisito `file://`, aggiorna `pyproject.toml` e rigenera `uv.lock`; non contiene fallback a build da sorgente. `LocalLLMSidecar.ensure_ready()` esegue il preflight di `mlx_vlm` per capability image. La dipendenza macOS `mlx==0.31.2` è fissata nella stessa fonte per evitare la regressione di ownership degli stream GPU osservata con `mlx 0.32.0` nel worker PyInstaller. | Wheel versionato nel `dist/` del repository collegato, `pyproject.toml` e `uv.lock`; in caso di errore del lock i due file di dipendenza vengono ripristinati. | `.venv/bin/python -m unittest discover -s test -p 'test_update_local_llm_server.py' -v`; `.venv/bin/python -m unittest discover -s test -p 'test_llm_sidecar.py' -v`; `python3 scripts/update_local_llm_server.py --check`; build bundle e inferenza visuale reale per ogni cambio del pin MLX. | | Runtime visuale nel bundle macOS | Permette alla `.app` PyInstaller di avviare il sidecar Qwen e il server MLX-VLM come processi figli senza richiedere un Python esterno. | `bundled_module_dispatch.py` intercetta esclusivamente `-m local_llm_server` e `-m mlx_vlm.server` prima del lifecycle Cocoa; `ClosedRoom.spec` include moduli e dati dinamici; `build.sh` fissa Python 3.10, versione verificata con MLX, invece di scegliere implicitamente l'interprete più recente. | Runtime e librerie dentro `Contents/Frameworks`; modello riusato dal path utente configurato, inclusa la directory `.lmstudio`; log sidecar in Application Support/Logs. | `test_bundled_module_dispatch.py`; `./build.sh --no-dmg`; esecuzione dei due `--help` dal binario bundle; health e inferenza Qwen positiva dal bundle. | | Impostazioni | Centralizza directory, default trascrizione, provider ASR, provider LLM, workflow meeting e opzioni locali/cloud avanzate senza esporre al frontend i secret già salvati. | Endpoint `/v1/settings`, `/v1/asr/providers`, `settings.py`, `env.py`, `schemas.py`, `asr_provider.py`, Settings UI, directory picker macOS. | `~/Library/Application Support/ClosedRoom/settings.json`, merge con `DEFAULT_SETTINGS`; `env.py` carica `.env` locale in dev senza sovrascrivere variabili già presenti e risolve secret env in modo case-insensitive; `gemini_api_key` e `speechmatics_api_key` sono write-only nella risposta API e gli stati sono esposti solo da `*_api_key_configured`; `asr_provider` default `local`, opzioni Speechmatics e `gemini_model` persistiti; `meeting_auto_analysis` abilita la pipeline asincrona post-trascrizione; scrittura atomica con temp file e `os.replace`. Speechmatics/Gemini sono opt-in cloud e possono inviare audio/testo a terze parti solo quando selezionati. | Testare lettura/salvataggio e regressioni su `RecordingStore.root`; `UV_CACHE_DIR=.cache/uv uv run python -m unittest discover -s test -p 'test_env.py' -v`; `UV_CACHE_DIR=.cache/uv uv run python -m unittest discover -s test -p 'test_analysis_api.py' -v`; `cd frontend && npm run build`. | diff --git a/src/local_asr_server/runtime/llm_sidecar.py b/src/local_asr_server/runtime/llm_sidecar.py index f7f69c3d..74a34884 100644 --- a/src/local_asr_server/runtime/llm_sidecar.py +++ b/src/local_asr_server/runtime/llm_sidecar.py @@ -1,6 +1,7 @@ from __future__ import annotations import importlib.util +import json import os import signal import shutil @@ -11,13 +12,21 @@ from dataclasses import dataclass from pathlib import Path from typing import Any -from urllib.error import URLError -from urllib.request import urlopen +from urllib.error import HTTPError, URLError +from urllib.parse import quote +from urllib.request import Request, urlopen from local_asr_server.paths import get_service_log_file from local_asr_server.runtime.models import LOCAL_SERVICE_HOST +_DYNAMIC_RESIDENCY_MODELS = { + "nemotron-nano-4b", + "nemotron-nano-4b-q8", + "qwen3-vl-4b", +} + + class LocalLLMSidecarError(RuntimeError): """Raised when the managed local LLM service cannot become usable.""" @@ -43,6 +52,7 @@ class LocalLLMProcessConfig: ctx_size: int | None = None startup_timeout: int | None = None llama_server_bin: str = "" + dynamic_residency: bool = True class LocalLLMSidecar: @@ -53,9 +63,11 @@ def __init__(self, log_file: Path | None = None) -> None: self.log_file = log_file or get_service_log_file("llm-server", create_parent=False) self._process: subprocess.Popen[Any] | None = None self._port: int | None = None + self._vision_port: int | None = None self._started_at: float | None = None self._last_error: str | None = None self._process_config: LocalLLMProcessConfig | None = None + self._resident_configs: dict[str, LocalLLMProcessConfig] = {} @property def base_url(self) -> str | None: @@ -96,11 +108,15 @@ def status(self, model: str, mode: str, model_path: str = "") -> dict[str, Any]: loaded_model_id = None loaded_model_path = None loaded_model_backend = None + resident_models: list[str] = [] if health_data: loaded_model = health_data.get("model_key") or health_data.get("model") loaded_model_id = health_data.get("model") loaded_model_path = health_data.get("model_path") loaded_model_backend = health_data.get("backend") + values = health_data.get("loaded_models") + if isinstance(values, list): + resident_models = [str(value) for value in values] return { "name": "llm", @@ -111,6 +127,8 @@ def status(self, model: str, mode: str, model_path: str = "") -> dict[str, Any]: "loaded_model_id": loaded_model_id, "loaded_model_path": loaded_model_path, "loaded_model_backend": loaded_model_backend, + "resident_models": resident_models, + "cold": bool(health_data is not None and not resident_models), "model_path_configured": bool(model_path), "managed": mode == "auto", "url": self.base_url, @@ -135,6 +153,7 @@ def ensure_ready( reasoning: str = "auto", capability: str = "text", timeout: float = 30.0, + dynamic_residency: bool = True, ) -> dict[str, Any]: if model == "custom" and not model_path: raise LocalLLMSidecarError("local_llm_model_missing", "Percorso modello LLM locale non configurato.", 400) @@ -150,9 +169,20 @@ def ensure_ready( "Il backend visuale locale non è installato. Installa local-llm-server con l'extra vision.", 503, ) - config = LocalLLMProcessConfig(model, model_path, backend, mmproj_path, ctx_size, startup_timeout, llama_server_bin) + config = LocalLLMProcessConfig( + model, model_path, backend, mmproj_path, ctx_size, startup_timeout, + llama_server_bin, dynamic_residency, + ) if self._process is None or self._process.poll() is not None: self.start(**config.__dict__) + elif self._supports_dynamic_residency(config): + try: + self._ensure_registered_model(config) + except LocalLLMSidecarError: + # The admin control plane is an optimization boundary. The + # owned process remains the canonical reclamation boundary, + # so recover by restarting with only the requested model. + self.restart(**config.__dict__) elif self._process_config != config: self.restart(**config.__dict__) if not self.wait_until_ready(timeout=timeout): @@ -176,9 +206,19 @@ def start( ctx_size: int | None = None, startup_timeout: int | None = None, llama_server_bin: str = "", + dynamic_residency: bool = True, ) -> dict[str, Any]: - config = LocalLLMProcessConfig(model, model_path, backend, mmproj_path, ctx_size, startup_timeout, llama_server_bin) + config = LocalLLMProcessConfig( + model, model_path, backend, mmproj_path, ctx_size, startup_timeout, + llama_server_bin, dynamic_residency, + ) if self._process is not None and self._process.poll() is None: + if self._supports_dynamic_residency(config): + try: + self._ensure_registered_model(config) + except LocalLLMSidecarError: + return self.restart(**config.__dict__) + return {"base_url": self.base_url, "pid": self._process.pid} if self._process_config != config: return self.restart(**config.__dict__) return {"base_url": self.base_url, "pid": self._process.pid} @@ -206,8 +246,10 @@ def start( start_new_session=True, ) self._started_at = time.time() + self._vision_port = vision_port self._last_error = None self._process_config = config + self._resident_configs = {model: config} except Exception as exc: self._last_error = str(exc) raise LocalLLMSidecarError("local_llm_start_failed", f"Avvio local-llm-server non riuscito: {exc}") from exc @@ -232,8 +274,10 @@ def stop(self, timeout: float = 5.0) -> dict[str, Any]: process.wait(timeout=timeout) self._process = None self._port = None + self._vision_port = None self._started_at = None self._process_config = None + self._resident_configs.clear() return {"stopped": True} def restart(self, **config: Any) -> dict[str, Any]: @@ -315,6 +359,140 @@ def _terminate_stale_vision_workers() -> None: def _vision_runtime_available() -> bool: return importlib.util.find_spec("mlx_vlm") is not None + @staticmethod + def _supports_dynamic_residency(config: LocalLLMProcessConfig) -> bool: + # These are the product-owned registry models whose upstream 0.4 + # entries resolve the same LM Studio/managed artifacts used by + # ClosedRoom. Unknown/custom direct-path models retain the safer + # process restart/stop lifecycle until model_path is an admin API. + return config.dynamic_residency and config.model in _DYNAMIC_RESIDENCY_MODELS + + def _request_json( + self, method: str, path: str, payload: dict[str, Any] | None = None, *, timeout: float = 30.0 + ) -> dict[str, Any]: + if not self.base_url: + raise LocalLLMSidecarError("local_llm_not_running", "Il servizio LLM locale non è avviato.") + data = json.dumps(payload).encode("utf-8") if payload is not None else None + request = Request( + f"{self.base_url}{path}", + data=data, + headers={"Content-Type": "application/json"} if data is not None else {}, + method=method, + ) + try: + with urlopen(request, timeout=timeout) as response: + raw = response.read().decode("utf-8") + return json.loads(raw) if raw else {} + except HTTPError as exc: + try: + detail = exc.read().decode("utf-8", errors="replace")[:1000] + except Exception: + detail = str(exc) + raise LocalLLMSidecarError( + "local_llm_admin_failed", + f"local-llm-server admin {method} {path} failed ({exc.code}): {detail}", + 503, + ) from exc + except (OSError, URLError, ValueError) as exc: + raise LocalLLMSidecarError( + "local_llm_admin_unreachable", + f"local-llm-server admin endpoint non raggiungibile: {exc}", + 503, + ) from exc + + def _resident_model_keys(self) -> list[str]: + payload = self._request_json("GET", "/v1/models", timeout=2.0) + rows = payload.get("data") if isinstance(payload, dict) else None + if not isinstance(rows, list): + return [] + keys: list[str] = [] + for row in rows: + if not isinstance(row, dict): + continue + value = row.get("key") or row.get("id") + if value: + keys.append(str(value)) + return keys + + def _activation_payload( + self, config: LocalLLMProcessConfig, *, include_overrides: bool + ) -> dict[str, Any]: + payload: dict[str, Any] = {"model": config.model} + if not include_overrides: + return payload + for key in ("backend", "mmproj_path", "ctx_size", "startup_timeout", "llama_server_bin"): + value = getattr(config, key) + if value not in (None, ""): + payload[key] = value + if config.model == "qwen3-vl-4b" and self._vision_port is not None: + payload["mlx_vlm_server_port"] = self._vision_port + return payload + + def _ensure_registered_model(self, config: LocalLLMProcessConfig) -> None: + resident = self._resident_model_keys() + unknown = [ + key for key in resident + if (self._resident_configs.get(key) is None + or not self._supports_dynamic_residency(self._resident_configs[key])) + ] + if unknown: + raise LocalLLMSidecarError( + "local_llm_residency_conflict", + "Il sidecar contiene un runtime non gestibile tramite la policy ClosedRoom.", + 503, + ) + for key in resident: + if key == config.model: + continue + self._request_json("DELETE", f"/api/v1/models/{quote(key, safe='')}") + self._resident_configs.pop(key, None) + target_resident = config.model in resident + known_config = self._resident_configs.get(config.model) + include_overrides = (not target_resident) or (known_config is not None and known_config != config) + self._request_json( + "POST", + "/api/v1/models/activate", + self._activation_payload(config, include_overrides=include_overrides), + timeout=float(config.startup_timeout or 300), + ) + self._resident_configs = {config.model: config} + + def release_resident_models(self) -> dict[str, Any]: + """Return the managed sidecar to a cold state after a heavy phase. + + Registered product models use local-llm-server 0.4 zero-resident + semantics. Unknown/custom runtimes fall back to process stop so a + later phase can recreate the exact direct-path configuration. + Cleanup is best-effort and never masks the workload result. + """ + process = self._process + if process is None or process.poll() is not None: + self._resident_configs.clear() + return {"released": True, "cold": True, "resident_models": []} + try: + resident = self._resident_model_keys() + if any( + self._resident_configs.get(key) is None + or not self._supports_dynamic_residency(self._resident_configs[key]) + for key in resident + ): + self.stop() + return {"released": True, "cold": True, "resident_models": [], "fallback": "process_stop"} + released: list[str] = [] + for key in resident: + self._request_json("DELETE", f"/api/v1/models/{quote(key, safe='')}") + released.append(key) + self._resident_configs.clear() + return {"released": True, "cold": True, "resident_models": [], "unloaded_models": released} + except Exception as exc: + import logging + logging.getLogger("uvicorn.error").warning( + "Managed LLM residency cleanup failed; stopping owned sidecar: %s", exc + ) + self._last_error = str(exc) + self.stop() + return {"released": True, "cold": True, "resident_models": [], "fallback": "process_stop"} + def _build_command( self, *, @@ -327,6 +505,7 @@ def _build_command( llama_server_bin: str, port: int, vision_port: int | None = None, + dynamic_residency: bool = True, ) -> list[str]: cmd = [ sys.executable, @@ -337,24 +516,10 @@ def _build_command( cmd.extend(["--host", self.host, "--port", str(port), "--enable-admin-api"]) if vision_port is not None: cmd.extend(["--mlx-vlm-server-port", str(vision_port)]) - from local_asr_server.settings import load_settings - settings = load_settings() - visual_model = settings.get("visual_llm_model") or "qwen3-vl-4b" - - # Load models from local_llm_params.json - from local_asr_server.local_llm_params import load_local_llm_params - params = load_local_llm_params() - config_models = params.get("models") - - if config_models and isinstance(config_models, dict): - models_to_load = list(config_models.keys()) - else: - models_to_load = [] - if model != "custom": - models_to_load.append(model) - models_to_load.append(visual_model) - - cmd.extend(["--models"] + models_to_load) + # Start with exactly the model required by the current phase. + # Additional registered models are activated through the 0.4 admin + # API and the previous resident runtime is evicted first. + cmd.extend(["--model", model]) if model_path: cmd.extend(["--model-path", model_path]) if backend: diff --git a/src/local_asr_server/runtime/models.py b/src/local_asr_server/runtime/models.py index 96805369..23e23788 100644 --- a/src/local_asr_server/runtime/models.py +++ b/src/local_asr_server/runtime/models.py @@ -50,6 +50,27 @@ class AnalysisQualityDefaults: ANALYSIS_QUALITY_DEFAULTS = AnalysisQualityDefaults() +def is_local_llm_model_path_explicit( + settings: dict[str, Any], model: str | None = None, +) -> bool: + """Return whether the selected model path is explicitly user/config supplied. + + Automatic LM Studio discovery is deliberately excluded: registered product + models may be re-resolved by local-llm-server 0.4 after a zero-resident + transition, while explicit paths must preserve their exact artifact through + an owned process restart boundary. + """ + selected_model = model or settings.get("local_llm_model") or "" + model_paths = settings.get("local_llm_model_paths") or {} + if not isinstance(model_paths, dict): + model_paths = {} + if model_paths.get(selected_model) or settings.get("local_llm_model_path"): + return True + if selected_model: + return Path(selected_model).expanduser().exists() + return False + + def resolve_local_llm_model_path(settings: dict[str, Any], model: str | None = None) -> str: """Resolve model-specific paths before the legacy global model path.""" diff --git a/src/local_asr_server/runtime/service_manager.py b/src/local_asr_server/runtime/service_manager.py index 12d177a0..857263c2 100644 --- a/src/local_asr_server/runtime/service_manager.py +++ b/src/local_asr_server/runtime/service_manager.py @@ -3,7 +3,11 @@ from dataclasses import dataclass from typing import Any -from local_asr_server.runtime.models import DEFAULT_LOCAL_LLM_URL, resolve_local_llm_model_path +from local_asr_server.runtime.models import ( + DEFAULT_LOCAL_LLM_URL, + is_local_llm_model_path_explicit, + resolve_local_llm_model_path, +) from local_asr_server.runtime.llm_sidecar import LocalLLMSidecar from local_asr_server.settings import load_settings @@ -53,10 +57,12 @@ def _llm_settings(self, overrides: dict[str, Any] | None = None) -> dict[str, An } model = settings.get("local_llm_model") or "nemotron-nano-4b-q8" model_path = resolve_local_llm_model_path(settings, model) + dynamic_residency = not is_local_llm_model_path_explicit(settings, model) return { "mode": settings.get("local_llm_mode", "auto"), "model": model, "model_path": model_path, + "dynamic_residency": dynamic_residency, "url": settings.get("local_llm_url") or DEFAULT_LOCAL_LLM_URL, "reasoning": settings.get("local_llm_reasoning") or "auto", "backend": settings.get("local_llm_backend") or "", @@ -142,13 +148,31 @@ def ensure_llm_ready( llama_server_bin=llm["llama_server_bin"], reasoning=reasoning or llm["reasoning"], capability=capability, + dynamic_residency=llm["dynamic_residency"], ) + def release_llm_residency( + self, *, overrides: dict[str, Any] | None = None, + ) -> dict[str, Any]: + """Release heavy local LLM/VLM residency after one logical phase. + + Only the process owned by ClosedRoom in ``auto`` mode may be + mutated. External endpoints remain entirely caller-owned. + """ + from local_asr_server.runtime.leases import ModelRuntimeLeaseManager + + settings = self._llm_settings(overrides) + ModelRuntimeLeaseManager.release_lease("vision") + ModelRuntimeLeaseManager.release_lease("llm") + if settings["mode"] != "auto": + return {"released": False, "reason": "not_managed"} + return self.llm_sidecar.release_resident_models() + def start_llm(self) -> dict[str, Any]: llm = self._llm_settings() if llm["mode"] != "auto": return self.llm_status() - return self.llm_sidecar.start(model=llm["model"], model_path=llm["model_path"], backend=llm["backend"], mmproj_path=llm["mmproj_path"], ctx_size=llm["ctx_size"], startup_timeout=llm["startup_timeout"], llama_server_bin=llm["llama_server_bin"]) + return self.llm_sidecar.start(model=llm["model"], model_path=llm["model_path"], backend=llm["backend"], mmproj_path=llm["mmproj_path"], ctx_size=llm["ctx_size"], startup_timeout=llm["startup_timeout"], llama_server_bin=llm["llama_server_bin"], dynamic_residency=llm["dynamic_residency"]) def stop_llm(self) -> dict[str, Any]: return self.llm_sidecar.stop() @@ -157,7 +181,7 @@ def restart_llm(self) -> dict[str, Any]: llm = self._llm_settings() if llm["mode"] != "auto": return self.llm_status() - return self.llm_sidecar.restart(model=llm["model"], model_path=llm["model_path"], backend=llm["backend"], mmproj_path=llm["mmproj_path"], ctx_size=llm["ctx_size"], startup_timeout=llm["startup_timeout"], llama_server_bin=llm["llama_server_bin"]) + return self.llm_sidecar.restart(model=llm["model"], model_path=llm["model_path"], backend=llm["backend"], mmproj_path=llm["mmproj_path"], ctx_size=llm["ctx_size"], startup_timeout=llm["startup_timeout"], llama_server_bin=llm["llama_server_bin"], dynamic_residency=llm["dynamic_residency"]) def llm_logs(self, tail: int = 200) -> dict[str, Any]: return {"service": "llm", "tail": tail, "text": self.llm_sidecar.tail_logs(tail)} diff --git a/src/local_asr_server/services/analysis_service.py b/src/local_asr_server/services/analysis_service.py index 22c9df03..83e1097d 100644 --- a/src/local_asr_server/services/analysis_service.py +++ b/src/local_asr_server/services/analysis_service.py @@ -29,39 +29,44 @@ def __init__(self, services: AppServices) -> None: def analyze(self, body: AnalysisRequest) -> dict[str, Any]: settings = self.settings_with_request_overrides(load_settings(), body) provider_name = settings.get("llm_provider", "mock") + local_phase = provider_name in {"nemotron_local", "voxtral_local"} api_key = body.gemini_api_key or settings.get("gemini_api_key", "") or get_env_var("GEMINI_API_KEY") gemini_model = settings.get("gemini_model") or DEFAULT_GEMINI_MODEL local_llm_url = None local_llm_model = None temperature = self._resolve_temperature(settings) - if provider_name in {"nemotron_local", "voxtral_local"}: - capability = "audio" if provider_name == "voxtral_local" and body.recording_id else "text" - try: - runtime_options = self.services.runtime.ensure_llm_ready( - capability=capability, - reasoning=settings.get("local_llm_reasoning") or "auto", - overrides=settings, - ) - local_llm_url = runtime_options.get("base_url") - local_llm_model = runtime_options.get("model") - except LocalLLMSidecarError as exc: - raise HTTPException(status_code=exc.status, detail={"code": exc.code, "message": str(exc)}) from exc - except RuntimeError as exc: - raise HTTPException(status_code=400, detail=str(exc)) from exc + try: + if local_phase: + capability = "audio" if provider_name == "voxtral_local" and body.recording_id else "text" + try: + runtime_options = self.services.runtime.ensure_llm_ready( + capability=capability, + reasoning=settings.get("local_llm_reasoning") or "auto", + overrides=settings, + ) + local_llm_url = runtime_options.get("base_url") + local_llm_model = runtime_options.get("model") + except LocalLLMSidecarError as exc: + raise HTTPException(status_code=exc.status, detail={"code": exc.code, "message": str(exc)}) from exc + except RuntimeError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc - effective_model = gemini_model if provider_name == "gemini" else local_llm_model - provider = LLMService.get_provider(provider_name, api_key, local_llm_url, local_llm_model, gemini_model) + effective_model = gemini_model if provider_name == "gemini" else local_llm_model + provider = LLMService.get_provider(provider_name, api_key, local_llm_url, local_llm_model, gemini_model) - if provider_name == "voxtral_local" and body.recording_id: - return self._analyze_audio( - body, provider, provider_name=provider_name, - model=local_llm_model, settings=settings, + if provider_name == "voxtral_local" and body.recording_id: + return self._analyze_audio( + body, provider, provider_name=provider_name, + model=local_llm_model, settings=settings, + ) + return self._analyze_text( + body, provider, provider_name=provider_name, model=effective_model, + settings=settings, api_key=api_key, temperature=temperature, ) - return self._analyze_text( - body, provider, provider_name=provider_name, model=effective_model, - settings=settings, api_key=api_key, temperature=temperature, - ) + finally: + if local_phase: + self.services.runtime.release_llm_residency(overrides=settings) @staticmethod def request_setting_overrides(body: Any) -> dict[str, Any]: diff --git a/src/local_asr_server/visual_intelligence/service.py b/src/local_asr_server/visual_intelligence/service.py index 1eefdb3f..bd22c0f3 100644 --- a/src/local_asr_server/visual_intelligence/service.py +++ b/src/local_asr_server/visual_intelligence/service.py @@ -382,6 +382,7 @@ def process( } return payload finally: + services.runtime.release_llm_residency() if not preserve_staging: services.recordings.finish_visual_processing(recording_id) @@ -420,6 +421,7 @@ def _process_v2( completed = False processed_candidates = 0 chat_params = {} + llm_phase_acquired = False try: rejected_candidates = int(routing_summary.get("rejected_task_evaluations") or 0) @@ -547,6 +549,7 @@ def _process_v2( reasoning="off", overrides={"local_llm_model": model}, ) + llm_phase_acquired = True client = self._client(ready["base_url"], model) inf_start = time.perf_counter() qwen_calls += 1 @@ -790,6 +793,8 @@ def _process_v2( } return payload finally: + if llm_phase_acquired: + services.runtime.release_llm_residency() try: trace_path.unlink(missing_ok=True) except Exception: diff --git a/test/test_llm_sidecar.py b/test/test_llm_sidecar.py index 1526c33a..e5b341d0 100644 --- a/test/test_llm_sidecar.py +++ b/test/test_llm_sidecar.py @@ -52,7 +52,7 @@ def test_build_command_keeps_registry_model_when_overriding_model_path(self) -> sys.executable, "-m", "local_asr_server.runtime.local_llm_entrypoint", "serve", "--host", "127.0.0.1", "--port", "45678", "--enable-admin-api", "--mlx-vlm-server-port", "45679", - "--models", "voxtral-mini-3b", "qwen3-vl-4b", "--model-path", "/models/voxtral.gguf", + "--model", "voxtral-mini-3b", "--model-path", "/models/voxtral.gguf", "--backend", "llama_server", "--mmproj-path", "/models/mmproj.gguf", "--ctx-size", "32768", "--startup-timeout", "120", "--llama-server-bin", "/opt/bin/llama-server", @@ -81,6 +81,7 @@ def test_ensure_ready_restarts_when_process_configuration_changes(self) -> None: ctx_size=None, startup_timeout=None, llama_server_bin="", + dynamic_residency=True, ) def test_ensure_ready_reports_missing_vision_extra_before_starting(self) -> None: @@ -98,5 +99,113 @@ def test_ensure_ready_reports_missing_vision_extra_before_starting(self) -> None start.assert_not_called() + def test_registered_model_switch_evicts_previous_before_activation(self) -> None: + sidecar = LocalLLMSidecar() + process = Mock() + process.poll.return_value = None + sidecar._process = process + previous = LocalLLMProcessConfig(model="nemotron-nano-4b-q8") + sidecar._process_config = previous + sidecar._resident_configs = {previous.model: previous} + + with ( + patch.object(sidecar, "_runtime_available", return_value=True), + patch.object(sidecar, "_vision_runtime_available", return_value=True), + patch.object(sidecar, "_resident_model_keys", return_value=[previous.model]), + patch.object(sidecar, "_request_json", return_value={"ok": True}) as request_json, + patch.object(sidecar, "restart") as restart, + patch.object(sidecar, "wait_until_ready", return_value=True), + ): + ready = sidecar.ensure_ready(model="qwen3-vl-4b", capability="image") + + self.assertEqual(ready["model"], "qwen3-vl-4b") + restart.assert_not_called() + self.assertEqual(request_json.call_args_list[0].args[:2], ("DELETE", "/api/v1/models/nemotron-nano-4b-q8")) + self.assertEqual(request_json.call_args_list[1].args[:2], ("POST", "/api/v1/models/activate")) + self.assertEqual(request_json.call_args_list[1].args[2]["model"], "qwen3-vl-4b") + self.assertEqual(list(sidecar._resident_configs), ["qwen3-vl-4b"]) + + def test_start_reactivates_registered_model_when_sidecar_is_cold(self) -> None: + sidecar = LocalLLMSidecar() + process = Mock() + process.poll.return_value = None + process.pid = 123 + sidecar._process = process + config = LocalLLMProcessConfig(model="nemotron-nano-4b-q8") + sidecar._process_config = config + sidecar._resident_configs = {} + with ( + patch.object(sidecar, "_ensure_registered_model") as ensure_model, + patch.object(sidecar, "_runtime_available", return_value=True), + ): + result = sidecar.start(**config.__dict__) + ensure_model.assert_called_once_with(config) + self.assertEqual(result["pid"], 123) + + def test_qwen_activation_reuses_private_vlm_port(self) -> None: + sidecar = LocalLLMSidecar() + sidecar._vision_port = 45679 + payload = sidecar._activation_payload( + LocalLLMProcessConfig(model="qwen3-vl-4b"), include_overrides=True + ) + self.assertEqual(payload["mlx_vlm_server_port"], 45679) + + def test_release_registered_models_keeps_sidecar_alive_and_cold(self) -> None: + sidecar = LocalLLMSidecar() + process = Mock() + process.poll.return_value = None + sidecar._process = process + sidecar._port = 45678 + config = LocalLLMProcessConfig(model="qwen3-vl-4b") + sidecar._resident_configs = {config.model: config} + + with ( + patch.object(sidecar, "_resident_model_keys", return_value=[config.model]), + patch.object(sidecar, "_request_json", return_value={"ok": True}) as request_json, + patch.object(sidecar, "stop") as stop, + ): + result = sidecar.release_resident_models() + + self.assertTrue(result["released"]) + self.assertTrue(result["cold"]) + self.assertEqual(result["unloaded_models"], ["qwen3-vl-4b"]) + request_json.assert_called_once_with("DELETE", "/api/v1/models/qwen3-vl-4b") + stop.assert_not_called() + self.assertEqual(sidecar._resident_configs, {}) + + def test_explicit_path_registered_model_uses_process_stop_reclamation(self) -> None: + sidecar = LocalLLMSidecar() + process = Mock() + process.poll.return_value = None + sidecar._process = process + config = LocalLLMProcessConfig( + model="nemotron-nano-4b-q8", + model_path="/custom/nemotron.gguf", + dynamic_residency=False, + ) + sidecar._resident_configs = {config.model: config} + with ( + patch.object(sidecar, "_resident_model_keys", return_value=[config.model]), + patch.object(sidecar, "stop", return_value={"stopped": True}) as stop, + ): + result = sidecar.release_resident_models() + self.assertEqual(result["fallback"], "process_stop") + stop.assert_called_once_with() + + def test_release_unknown_model_falls_back_to_owned_process_stop(self) -> None: + sidecar = LocalLLMSidecar() + process = Mock() + process.poll.return_value = None + sidecar._process = process + with ( + patch.object(sidecar, "_resident_model_keys", return_value=["custom-runtime"]), + patch.object(sidecar, "stop", return_value={"stopped": True}) as stop, + ): + result = sidecar.release_resident_models() + + self.assertEqual(result["fallback"], "process_stop") + stop.assert_called_once_with() + + if __name__ == "__main__": unittest.main() diff --git a/test/test_model_residency.py b/test/test_model_residency.py new file mode 100644 index 00000000..9b2e72a3 --- /dev/null +++ b/test/test_model_residency.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import unittest +from unittest.mock import Mock, patch + +from fastapi import HTTPException + +from local_asr_server.runtime.service_manager import RuntimeServiceManager +from local_asr_server.schemas import AnalysisRequest +from local_asr_server.services.analysis_service import AnalysisService + + +class ModelResidencyTests(unittest.TestCase): + def test_runtime_release_only_mutates_managed_auto_sidecar(self) -> None: + sidecar = Mock() + sidecar.release_resident_models.return_value = {"released": True, "cold": True} + manager = RuntimeServiceManager(llm_sidecar=sidecar) + with patch("local_asr_server.runtime.service_manager.load_settings", return_value={ + "local_llm_mode": "auto", + "local_llm_model": "nemotron-nano-4b-q8", + "local_llm_model_path": "", + }): + result = manager.release_llm_residency() + self.assertTrue(result["released"]) + sidecar.release_resident_models.assert_called_once_with() + + sidecar.reset_mock() + with patch("local_asr_server.runtime.service_manager.load_settings", return_value={ + "local_llm_mode": "auto", + "local_llm_model": "nemotron-nano-4b-q8", + }): + result = manager.release_llm_residency(overrides={ + "local_llm_mode": "external", + "local_llm_model": "nemotron-nano-4b-q8", + "local_llm_url": "http://127.0.0.1:5555", + }) + self.assertEqual(result["reason"], "not_managed") + sidecar.release_resident_models.assert_not_called() + + def test_local_analysis_releases_residency_after_success(self) -> None: + services = Mock() + services.runtime.ensure_llm_ready.return_value = { + "base_url": "http://127.0.0.1:1235", + "model": "nemotron-nano-4b-q8", + } + provider = Mock() + provider.analyze.return_value = { + "title": "Title", "summary": "Summary", "key_points": [], "action_items": [] + } + services.catalog.get_analysis_cache.return_value = None + settings = { + "llm_provider": "nemotron_local", + "local_llm_mode": "auto", + "local_llm_model": "nemotron-nano-4b-q8", + "local_llm_reasoning": "off", + "local_llm_quality_preset": "balanced", + "local_llm_json_mode": True, + } + with ( + patch("local_asr_server.services.analysis_service.load_settings", return_value=settings), + patch("local_asr_server.services.analysis_service.LLMService.get_provider", return_value=provider), + ): + result = AnalysisService(services).analyze( + AnalysisRequest(text="meeting text", llm_provider="nemotron_local") + ) + self.assertEqual(result["title"], "Title") + services.runtime.release_llm_residency.assert_called_once_with(overrides=settings) + + def test_local_analysis_releases_residency_after_failure(self) -> None: + services = Mock() + services.runtime.ensure_llm_ready.return_value = { + "base_url": "http://127.0.0.1:1235", + "model": "nemotron-nano-4b-q8", + } + provider = Mock() + provider.analyze.side_effect = RuntimeError("inference failed") + services.catalog.get_analysis_cache.return_value = None + settings = { + "llm_provider": "nemotron_local", + "local_llm_mode": "auto", + "local_llm_model": "nemotron-nano-4b-q8", + "local_llm_reasoning": "off", + "local_llm_quality_preset": "balanced", + "local_llm_json_mode": True, + } + with ( + patch("local_asr_server.services.analysis_service.load_settings", return_value=settings), + patch("local_asr_server.services.analysis_service.LLMService.get_provider", return_value=provider), + self.assertRaises(HTTPException), + ): + AnalysisService(services).analyze( + AnalysisRequest(text="meeting text", llm_provider="nemotron_local") + ) + services.runtime.release_llm_residency.assert_called_once_with(overrides=settings) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_runtime_services.py b/test/test_runtime_services.py index cd52797e..ee10ac69 100644 --- a/test/test_runtime_services.py +++ b/test/test_runtime_services.py @@ -21,6 +21,20 @@ def test_model_specific_path_precedes_legacy_global_path(self) -> None: "/models/selected.gguf", ) + def test_model_path_explicit_provenance_distinguishes_config_from_discovery(self) -> None: + from local_asr_server.runtime.models import is_local_llm_model_path_explicit + + self.assertTrue(is_local_llm_model_path_explicit({ + "local_llm_model": "selected", + "local_llm_model_paths": {"selected": "/models/selected.gguf"}, + "local_llm_model_path": "", + })) + self.assertFalse(is_local_llm_model_path_explicit({ + "local_llm_model": "nemotron-nano-4b-q8", + "local_llm_model_paths": {}, + "local_llm_model_path": "", + })) + def test_model_path_falls_back_to_legacy_global_path(self) -> None: settings = { "local_llm_model": "selected", @@ -156,6 +170,7 @@ def test_auto_mode_ensures_managed_sidecar_ready(self) -> None: llama_server_bin="", reasoning="auto", capability="text", + dynamic_residency=True, ) @patch("local_asr_server.runtime.service_manager._query_external_health") diff --git a/test/visual_intelligence_support.py b/test/visual_intelligence_support.py index 8836f964..1c0d0d0c 100644 --- a/test/visual_intelligence_support.py +++ b/test/visual_intelligence_support.py @@ -5,9 +5,16 @@ class RuntimeStub: + def __init__(self): + self.release_calls = 0 + def ensure_llm_ready(self, **kwargs): return {"base_url": "http://127.0.0.1:1235", "model": "qwen3-vl-4b"} + def release_llm_residency(self): + self.release_calls += 1 + return {"released": True, "cold": True} + class TaskAwareClientStub: def __init__(self): From a7fdd7c0b0d2cfa3fbcc0122d49c8faecd665182 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Mon, 31 Aug 2026 07:56:55 +0200 Subject: [PATCH 014/182] refactor(ui): establish UX simplification workstream --- design/ux-contract.json | 36 ++- docs/current-state.md | 4 + docs/workstreams/ux-simplification.md | 127 ++++++++ frontend/src/App.tsx | 97 ++---- .../src/components/ui/AnalysisSetupModal.tsx | 143 +++++---- frontend/src/components/ui/Tooltip.tsx | 21 +- frontend/src/pages/MeetingDetailPage.tsx | 296 ++++++++++-------- 7 files changed, 475 insertions(+), 249 deletions(-) create mode 100644 docs/workstreams/ux-simplification.md diff --git a/design/ux-contract.json b/design/ux-contract.json index 6afb4368..eeb27c39 100644 --- a/design/ux-contract.json +++ b/design/ux-contract.json @@ -1,6 +1,6 @@ { "schema_version": 1, - "contract_version": "0.5.0", + "contract_version": "0.6.0", "applicable": true, "design_source_of_truth": { "type": "code-first", @@ -43,6 +43,40 @@ "bounded_information_density": true, "actionable_error_recovery": true }, + "golden_path": { + "steps": [ + "home", + "new_meeting", + "record", + "stop", + "transcribe", + "analyze", + "review" + ], + "normal_path_rule": "A configured user can complete the golden path without understanding backend, helper, process, port, model-path or routing concepts.", + "surface_hierarchy": { + "home": "understand what happened, what needs attention and what to open or do next", + "new_meeting": "confirm meeting context and start recording", + "record": "understand capture status and safely stop", + "transcribe": "start transcription, understand progress and recover from failure", + "analyze": "choose the trust/runtime boundary in user language and run analysis with strong defaults", + "review": "review transcript, speakers, actions, decisions, risks and derived insights" + } + }, + "complexity_disclosure": { + "tiers": [ + "essential", + "contextual", + "advanced", + "diagnostics" + ], + "essential": "Information and actions required to complete the user's current task.", + "contextual": "Optional task-relevant choices such as project, capture source or enrichment that can change the outcome.", + "advanced": "Expert configuration such as model or quality overrides that normal use does not require.", + "diagnostics": "Internal runtime, helper, process, port, log, routing and model-path details used for troubleshooting rather than normal operation.", + "diagnostics_default_hidden": true, + "recovery_exception": "A technical condition may surface outside diagnostics only when the user must act on it; present the user-facing consequence and recovery action before raw technical detail." + }, "critical_states": [ "loading", "empty", diff --git a/docs/current-state.md b/docs/current-state.md index 72f6395f..50874ba6 100644 --- a/docs/current-state.md +++ b/docs/current-state.md @@ -44,6 +44,10 @@ The gap-closing implementation is on `close-baseline-gaps`. Its deterministic un Historical planning documents still need a separate lifecycle cleanup; they are not treated as current operational truth. +## Active workstream + +- [`docs/workstreams/ux-simplification.md`](workstreams/ux-simplification.md): simplify the primary meeting journey, progressively disclose runtime diagnostics, and harden accessibility/evidence across recording, meeting review, analysis and settings. + ## Next highest-value work 1. Obtain green exact-head remote preflight for the gap-closing PR, including finalized `.app` package smoke on macOS arm64. diff --git a/docs/workstreams/ux-simplification.md b/docs/workstreams/ux-simplification.md new file mode 100644 index 00000000..7eebc2c4 --- /dev/null +++ b/docs/workstreams/ux-simplification.md @@ -0,0 +1,127 @@ +# UX simplification and critical journey hardening + +Status: active +Owner: frontend product experience +Read when: implementing or coordinating the ClosedRoom primary meeting journey and supporting accessibility work + +## Goal + +Make the normal ClosedRoom journey feel simple and task-led from meeting capture through transcript and insights, while preserving expert diagnostics behind progressive disclosure and keeping recovery clear when permissions, capture, transcription or analysis fail. + +## Non-goals + +- Rebrand ClosedRoom or replace the existing semantic design system. +- Remove expert/runtime capabilities that are useful for diagnostics. +- Change recording, transcription or analysis backend contracts unless required to preserve the user-facing journey. +- Treat screenshot polish as evidence of interaction quality. + +## Invariants + +- The canonical journey is `Home -> New Meeting -> Record -> Stop -> Transcribe -> Analyze -> Review`. +- Normal use must not require understanding backend, helper, process, port, model-path or routing concepts. +- Each critical surface has one dominant next action; destructive actions remain visually distinct. +- Complexity is disclosed as `essential -> contextual -> advanced -> diagnostics`. +- Permission, capture, processing and provider failures remain recoverable without losing the meeting. +- Existing semantic UI components/tokens remain the design-system owner. +- Keyboard/focus/assistive semantics and reduced-motion behavior are part of completion, not post-polish cleanup. + +## Work graph + +| ID | Work | Owns/writes | Depends on | Parallel | State | +| --- | --- | --- | --- | --- | --- | +| UX-1 | Codify golden path, hierarchy and disclosure contract | `design/ux-contract.json`, this workstream | — | no | ACTIVE | +| UX-2 | Simplify new-meeting/recording default path and permission recovery | `frontend/src/pages/RecordingPage.tsx`, recording-specific UI helpers/i18n | UX-1 | yes | BLOCKED | +| UX-3 | Make meeting detail the guided transcript-to-insights workspace; hide runtime diagnostics | `frontend/src/pages/MeetingDetailPage.tsx`, meeting-specific UI/i18n | UX-1 | yes | BLOCKED | +| UX-4 | Simplify analysis setup with strong defaults and advanced disclosure | `frontend/src/components/AnalysisSetupModal.tsx`, analysis setup i18n | UX-1 | yes | BLOCKED | +| UX-5 | Separate user preferences from runtime diagnostics; remove runtime action from global header | `frontend/src/pages/SettingsPage.tsx`, `frontend/src/App.tsx`, settings/header i18n | UX-1 | yes | BLOCKED | +| UX-6 | Refine dashboard around attention and next action without redesign | `frontend/src/pages/DashboardPage.tsx`, dashboard-specific UI/i18n | UX-1 | yes | BLOCKED | +| UX-7 | Harden shared accessibility semantics for tooltip/menu/search/tabs and icon actions | `frontend/src/components/ui`, affected semantic consumers | UX-1 | yes | BLOCKED | +| UX-8 | Reduce decorative motion/glow and normalize icon/microcopy treatment | `frontend/src/index.css`, affected presentation-only call sites | UX-2, UX-3, UX-4, UX-5, UX-6, UX-7 | no | BLOCKED | +| UX-9 | Critical-journey automated evidence plus declared macOS REAL_ENVIRONMENT residual checks | tests/scripts/contracts/docs | UX-2, UX-3, UX-4, UX-5, UX-6, UX-7, UX-8 | no | BLOCKED | + +Allowed states: `READY`, `ACTIVE`, `BLOCKED`, `DONE`. + +Parallel work must stay inside the declared write boundaries or use an explicit integration commit. + +## Current executable slice + +`UX-1` + +Acceptance: + +- `design/ux-contract.json` declares the canonical golden path and disclosure tiers. +- The normal-path contract explicitly excludes internal runtime concepts unless they create user value or are needed for recovery. +- Recording, meeting, analysis and settings slices have observable user-outcome acceptance criteria in this workstream. + +Validation: + +- `python3 scripts/verify_product_experience.py` +- `python3 scripts/verify_docs.py` + +## Slice acceptance + +### UX-2 Recording + +- A configured user can start a default `microphone + computer` meeting from the main form without opening technical settings. +- Readiness is summarized as a user-facing ready/blocking state; detailed backend diagnostics are not in the default path. +- Missing microphone/screen permission exposes one clear recovery action and supports retry. +- Source mode, diarization and visual intelligence remain discoverable as contextual/advanced options. + +### UX-3 Meeting workspace + +- If transcription is missing, transcription is the dominant next action. +- If transcript exists and analysis is missing, analysis is the dominant next action. +- Runtime/backend/model/log details are absent from normal processing state and available only through explicit details/diagnostics disclosure. +- Transcript/insights/speakers navigation has correct semantic selection relationships and keyboard focus behavior. + +### UX-4 Analysis setup + +- Default choice is understandable as Local vs Cloud/provider plus a user-facing quality preset. +- Temperature, reasoning, token limits, JSON mode and model paths do not appear until Advanced is opened. +- The current explicit provider/runtime trust choice remains preserved. + +### UX-5 Settings/header + +- Global header contains primary navigation, `New Meeting`, and user-level settings controls; Local LLM runtime UI is not a peer primary action. +- User preference sections remain separate from service lifecycle/log/port/model diagnostics. + +### UX-6 Dashboard + +- Home answers `what happened`, `what needs attention`, and `what should I open/do next` without increasing default density. +- Search and period controls use accessible interaction semantics. + +### UX-7 Accessibility + +- Tooltip content is reachable by keyboard and exposed with assistive semantics. +- Custom menus/search/tabs expose appropriate roles, names, selected/expanded state and focus behavior. +- Icon-only critical controls have accessible names. + +### UX-8 Visual polish + +- Frequent interactions use restrained motion; attention animation is reserved for actual state/progress/urgency. +- Functional icons use Lucide rather than emoji where practical. +- Hard-coded locale-specific functional copy in touched surfaces is removed. + +### UX-9 Evidence + +- Required repository product-experience verification, frontend lint/typecheck and selected E2E/contract checks pass on exact HEAD. +- Validation profile is selected from the actual diff and is not silently downgraded. +- Interactive WKWebView/focus/VoiceOver/TCC evidence remains explicitly `REAL_ENVIRONMENT` where automation cannot prove it. + +## Integration points + +- `design/ux-contract.json` owns journey/disclosure semantics used by every slice. +- Shared component changes in UX-7 land before consumers depend on their semantics. +- UX-8 may tune presentation only after task hierarchy is stable. +- UX-9 validates the integrated exact head rather than reusing evidence from earlier slice commits. + +## Durable documentation destinations + +- `design/ux-contract.json`: canonical journey, hierarchy/disclosure and validation expectations. +- `design/brand-kit.json`: only if semantic visual/motion rules materially change. +- `docs/features/`: only when durable user-visible feature behavior changes beyond the design contract. +- tests/contracts: executable critical-journey and accessibility-adjacent deterministic truth. + +## Completion + +The workstream is complete only when applicable code, interaction states, recovery, accessibility, adaptive behavior, validation/evidence and durable docs agree. Then update `docs/current-state.md` and delete this file by default. diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index e50de95d..25b27e06 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1,5 +1,5 @@ import { useState, useEffect } from 'react'; -import { BarChart3, ChevronDown, ExternalLink, FolderKanban, Languages, Mic, Moon, Palette, PlayCircle, Settings, Sparkles, Sun } from 'lucide-react'; +import { BarChart3, ChevronDown, FolderKanban, Languages, Mic, Moon, Palette, PlayCircle, Settings, Sparkles, Sun } from 'lucide-react'; import { I18nProvider, useTranslation } from './i18n/i18n'; import { ToastProvider, useToast } from './context/ToastContext'; import { ApiClient } from './api/apiClient'; @@ -29,7 +29,6 @@ function MainApp() { const [defaultModel, setDefaultModel] = useState(''); const [theme, setTheme] = useState<'dark' | 'light'>('dark'); const [moreOpen, setMoreOpen] = useState(false); - const [openingLocalLlmUi, setOpeningLocalLlmUi] = useState(false); const [routeDetail, setRouteDetail] = useState(null); const [tourStep, setTourStep] = useState(null); const [tourReturnHash, setTourReturnHash] = useState(''); @@ -67,38 +66,6 @@ function MainApp() { } }; - const openLocalLlmUi = async () => { - const popup = window.open('', 'ClosedRoomLocalLlmUi'); - if (!popup) { - showToast(t('header.localLlmPopupBlocked'), 'warning'); - return; - } - - popup.opener = null; - setOpeningLocalLlmUi(true); - - try { - let service = await ApiClient.getLlmService(); - if (!service.url && service.mode === 'auto') { - service = await ApiClient.startLlmService(); - } - if (!service.url) { - throw new Error(service.error || t('header.localLlmUnavailable')); - } - - popup.location.replace(service.url); - popup.focus(); - } catch (err) { - popup.close(); - const message = err instanceof Error && err.message - ? err.message - : t('header.localLlmUnavailable'); - showToast(message, 'error'); - } finally { - setOpeningLocalLlmUi(false); - } - }; - // Sync hash with activePage useEffect(() => { const handleHashChange = () => { @@ -222,7 +189,7 @@ function MainApp() { return () => clearInterval(interval); }, [isDemoActive]); - // Close more panel on outside click + // Close settings menu on outside click or Escape. useEffect(() => { const handleClick = (e: MouseEvent) => { const target = e.target as HTMLElement; @@ -230,8 +197,15 @@ function MainApp() { setMoreOpen(false); } }; + const handleKeyDown = (e: KeyboardEvent) => { + if (e.key === 'Escape') setMoreOpen(false); + }; document.addEventListener('click', handleClick); - return () => document.removeEventListener('click', handleClick); + document.addEventListener('keydown', handleKeyDown); + return () => { + document.removeEventListener('click', handleClick); + document.removeEventListener('keydown', handleKeyDown); + }; }, []); const renderPage = () => { @@ -298,7 +272,7 @@ function MainApp() { {/* Navigation */} -