diff --git a/.github/workflows_placeholder.md b/.github/workflows_placeholder.md deleted file mode 100644 index 113cdc9..0000000 --- a/.github/workflows_placeholder.md +++ /dev/null @@ -1,3 +0,0 @@ -# CI note - -For the 48-hour local sprint, clean macOS builds are the priority. Add GitHub Actions after Step 2 if it helps the team; do not spend baseline-critical time on CI polish before the simulator compiles and tests locally. diff --git a/.gitignore b/.gitignore index 31325de..17d9ae4 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,18 @@ models/*.pth models/*.ckpt models/checkpoints/ models/runs/ + +# --- Mobile Phone Camera-in-the-Loop --- +# Personal, hardware-specific camera calibration (your phone's FOV, not a +# general default) — see docs/PHONE_CAMERA_METRICS.md "Camera calibration". +configs/ + +# --- FSOC launcher --- +# Local, non-secret overrides (see fsoc.env.example, which IS committed). +fsoc.env + +# --- Local AI-agent session artifacts --- +# Claude Code terminal transcript exports land at repo root as +# YYYY-MM-DD-HHMMSS-.txt (e.g. 2026-09-08-103830-...); local-only, +# never intended to be committed. +/[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]-*.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index ff5eb48..87f4ac9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,6 +15,28 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +# Software-identity provenance for real-session recordings (G2): the commit that was +# actually BUILT, captured at configure time (not re-queried at runtime, which would +# reflect whatever HEAD happens to be when fsoc_live runs, possibly after further +# edits). "unknown" outside a git checkout (e.g. an extracted source tarball) rather +# than failing configure. +find_package(Git QUIET) +set(FSOC_GIT_COMMIT "unknown") +if(GIT_FOUND) + execute_process( + COMMAND ${GIT_EXECUTABLE} rev-parse HEAD + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + OUTPUT_VARIABLE FSOC_GIT_COMMIT_RESULT + RESULT_VARIABLE FSOC_GIT_COMMIT_STATUS + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET + ) + if(FSOC_GIT_COMMIT_STATUS EQUAL 0 AND FSOC_GIT_COMMIT_RESULT) + set(FSOC_GIT_COMMIT "${FSOC_GIT_COMMIT_RESULT}") + endif() +endif() +message(STATUS "FSOC: git commit for recording provenance = ${FSOC_GIT_COMMIT}") + include(cmake/CompilerWarnings.cmake) include(cmake/Sanitizers.cmake) @@ -56,15 +78,55 @@ target_link_libraries(fsoc_control PUBLIC fsoc::core) fsoc_set_project_warnings(fsoc_control) fsoc_enable_sanitizers(fsoc_control) +# --------------------------------------------------------------------------- +# fsoc_camera_calibration — Mobile Phone Camera-in-the-Loop milestone. +# The simplest defensible camera geometry (Phase 8): a declared/measured +# pinhole FOV, stored in a dependency-free key=value file. Deliberately +# OpenCV-FREE and buildable unconditionally (even with FSOC_ENABLE_OPENCV=OFF) +# — it is pure math + file I/O, no camera or image dependency at all. +# --------------------------------------------------------------------------- +add_library(fsoc_camera_calibration src/live_camera_calibration.cpp) +add_library(fsoc::camera_calibration ALIAS fsoc_camera_calibration) +target_include_directories(fsoc_camera_calibration + PUBLIC + $ + $ +) +target_compile_features(fsoc_camera_calibration PUBLIC cxx_std_20) +fsoc_set_project_warnings(fsoc_camera_calibration) +fsoc_enable_sanitizers(fsoc_camera_calibration) + +# --------------------------------------------------------------------------- +# fsoc_virtual_actuator — Mobile Phone Camera-in-the-Loop milestone. +# Honest bookkeeping actuator (Phase 11). Also OpenCV-FREE and buildable +# unconditionally — pure rate-limit/integrate/clamp math, no camera/image +# dependency, no coupling to PanTiltCamera (see fsoc/virtual_actuator.hpp). +# --------------------------------------------------------------------------- +add_library(fsoc_virtual_actuator src/virtual_actuator.cpp) +add_library(fsoc::virtual_actuator ALIAS fsoc_virtual_actuator) +target_include_directories(fsoc_virtual_actuator + PUBLIC + $ + $ +) +target_compile_features(fsoc_virtual_actuator PUBLIC cxx_std_20) +fsoc_set_project_warnings(fsoc_virtual_actuator) +fsoc_enable_sanitizers(fsoc_virtual_actuator) + +add_executable(fsoc_camera_calibrate apps/fsoc_camera_calibrate.cpp) +target_link_libraries(fsoc_camera_calibrate PRIVATE fsoc::camera_calibration) +fsoc_set_project_warnings(fsoc_camera_calibrate) +fsoc_enable_sanitizers(fsoc_camera_calibrate) + # --------------------------------------------------------------------------- # OpenCV discovery (image/perception boundary only — never linked into fsoc_core) # --------------------------------------------------------------------------- set(FSOC_OPENCV_AVAILABLE OFF) if(NOT FSOC_ENABLE_OPENCV STREQUAL "OFF") if(FSOC_ENABLE_OPENCV STREQUAL "ON") - find_package(OpenCV REQUIRED COMPONENTS core imgproc imgcodecs dnn) + find_package(OpenCV REQUIRED COMPONENTS core imgproc imgcodecs dnn videoio) else() - find_package(OpenCV QUIET COMPONENTS core imgproc imgcodecs dnn) + find_package(OpenCV QUIET COMPONENTS core imgproc imgcodecs dnn videoio) endif() if(OpenCV_FOUND) set(FSOC_OPENCV_AVAILABLE ON) @@ -157,6 +219,135 @@ if(FSOC_OPENCV_AVAILABLE) fsoc_set_project_warnings(fsoc_ai_perception) fsoc_enable_sanitizers(fsoc_ai_perception) + # ----------------------------------------------------------------------- + # fsoc_frame_source — Mobile Phone Camera-in-the-Loop milestone. + # Pure I/O boundary (FrameSource interface + cv::VideoCapture adapter). + # Depends ONLY on OpenCV core + videoio. MUST NOT depend on fsoc::core, + # fsoc::perception, fsoc::ai_perception, or fsoc::render: a frame source + # knows nothing about detection, perception mode, or world truth. + # ----------------------------------------------------------------------- + if(TARGET opencv_videoio) + add_library(fsoc_frame_source src/frame_source.cpp src/opencv_camera_frame_source.cpp) + add_library(fsoc::frame_source ALIAS fsoc_frame_source) + target_include_directories(fsoc_frame_source + PUBLIC + $ + $ + ${OpenCV_INCLUDE_DIRS} + ) + target_compile_features(fsoc_frame_source PUBLIC cxx_std_20) + target_link_libraries(fsoc_frame_source PUBLIC opencv_core opencv_videoio) + fsoc_set_project_warnings(fsoc_frame_source) + fsoc_enable_sanitizers(fsoc_frame_source) + set(FSOC_FRAME_SOURCE_AVAILABLE ON) + message(STATUS "FSOC: OpenCV videoio present - phone-camera-in-the-loop targets enabled") + else() + set(FSOC_FRAME_SOURCE_AVAILABLE OFF) + message(STATUS "FSOC: OpenCV videoio absent - phone-camera-in-the-loop targets disabled") + endif() + + if(FSOC_FRAME_SOURCE_AVAILABLE) + # ------------------------------------------------------------------- + # fsoc_atomic_file_io — write-to-temp-then-rename helper shared by + # LiveFramePublisher and RealSessionRecorder. No OpenCV/fsoc::core + # dependency at all; pure filesystem utility. + # ------------------------------------------------------------------- + add_library(fsoc_atomic_file_io src/atomic_file_io.cpp) + add_library(fsoc::atomic_file_io ALIAS fsoc_atomic_file_io) + target_include_directories(fsoc_atomic_file_io + PUBLIC + $ + $ + ) + target_compile_features(fsoc_atomic_file_io PUBLIC cxx_std_20) + fsoc_set_project_warnings(fsoc_atomic_file_io) + fsoc_enable_sanitizers(fsoc_atomic_file_io) + + # ------------------------------------------------------------------- + # fsoc_live_frame_publisher — atomic, frame-identity-safe local file + # transport for fsoc_live's long-running (no finite CSV) output. Pure + # I/O sink: depends only on OpenCV core/imgcodecs, never on + # fsoc::core/perception/control. See docs/LIVE_DATA_AUDIT.md section 2 + # for the gap this closes. + # ------------------------------------------------------------------- + add_library(fsoc_live_frame_publisher src/live_frame_publisher.cpp) + add_library(fsoc::live_frame_publisher ALIAS fsoc_live_frame_publisher) + target_include_directories(fsoc_live_frame_publisher + PUBLIC + $ + $ + ${OpenCV_INCLUDE_DIRS} + ) + target_compile_features(fsoc_live_frame_publisher PUBLIC cxx_std_20) + target_link_libraries(fsoc_live_frame_publisher PUBLIC opencv_core opencv_imgcodecs fsoc::atomic_file_io) + fsoc_set_project_warnings(fsoc_live_frame_publisher) + fsoc_enable_sanitizers(fsoc_live_frame_publisher) + + # ------------------------------------------------------------------- + # fsoc_real_session_recorder — G2 real-data recording sink (raw + # frames + per-frame telemetry + event markers + manifest), never + # pruned unlike the live preview buffer above. Pure I/O sink: no + # fsoc::core/perception/control dependency. + # ------------------------------------------------------------------- + add_library(fsoc_real_session_recorder src/real_session_recorder.cpp) + add_library(fsoc::real_session_recorder ALIAS fsoc_real_session_recorder) + target_include_directories(fsoc_real_session_recorder + PUBLIC + $ + $ + ${OpenCV_INCLUDE_DIRS} + ) + target_compile_features(fsoc_real_session_recorder PUBLIC cxx_std_20) + target_link_libraries(fsoc_real_session_recorder PUBLIC opencv_core opencv_imgcodecs fsoc::atomic_file_io) + fsoc_set_project_warnings(fsoc_real_session_recorder) + fsoc_enable_sanitizers(fsoc_real_session_recorder) + + # ------------------------------------------------------------------- + # fsoc_live_support — real-camera perception/tracking/virtual-actuator + # orchestration (LiveTrackingSession), the real-camera counterpart to + # fsoc_simulation (which similarly backs fsoc_demo via + # fsoc_demo_support). Depends on fsoc::core (camera geometry, target + # tracker), fsoc::control (PID), fsoc::perception (classical + # detector), fsoc::ai_perception (AI detector + Safe Hybrid), and + # fsoc::frame_source (Frame/FrameSourceInfo types only — never + # constructs a FrameSource itself; that stays the CLI app's job). + # ------------------------------------------------------------------- + add_library(fsoc_live_support + src/live_preprocessing.cpp + src/live_tracking_session.cpp + ) + add_library(fsoc::live_support ALIAS fsoc_live_support) + target_include_directories(fsoc_live_support + PUBLIC + $ + $ + ${OpenCV_INCLUDE_DIRS} + ) + target_compile_features(fsoc_live_support PUBLIC cxx_std_20) + target_link_libraries(fsoc_live_support + PUBLIC fsoc::core fsoc::control fsoc::perception fsoc::ai_perception fsoc::frame_source + fsoc::camera_calibration fsoc::virtual_actuator opencv_imgproc) + fsoc_set_project_warnings(fsoc_live_support) + fsoc_enable_sanitizers(fsoc_live_support) + + add_executable(fsoc_camera_probe apps/fsoc_camera_probe.cpp) + target_link_libraries(fsoc_camera_probe PRIVATE fsoc::frame_source) + fsoc_set_project_warnings(fsoc_camera_probe) + fsoc_enable_sanitizers(fsoc_camera_probe) + + add_executable(fsoc_camera_view apps/fsoc_camera_view.cpp) + target_link_libraries(fsoc_camera_view PRIVATE fsoc::frame_source opencv_imgcodecs opencv_imgproc) + fsoc_set_project_warnings(fsoc_camera_view) + fsoc_enable_sanitizers(fsoc_camera_view) + + add_executable(fsoc_live apps/fsoc_live.cpp) + target_link_libraries(fsoc_live PRIVATE fsoc::live_support fsoc::camera_calibration + fsoc::live_frame_publisher fsoc::real_session_recorder opencv_imgcodecs opencv_imgproc) + target_compile_definitions(fsoc_live PRIVATE FSOC_GIT_COMMIT="${FSOC_GIT_COMMIT}") + fsoc_set_project_warnings(fsoc_live) + fsoc_enable_sanitizers(fsoc_live) + endif() + # ----------------------------------------------------------------------- # fsoc_simulation — the deterministic closed-loop integration layer (Step 7). # This is the ONE place the tested modules come together. @@ -401,6 +592,19 @@ if(FSOC_BUILD_TESTS) fsoc_enable_sanitizers(fsoc_target_tracker_tests) add_test(NAME fsoc_target_tracker_tests COMMAND fsoc_target_tracker_tests) + # --- Mobile Phone Camera-in-the-Loop: OpenCV-free pieces, unconditional --- + add_executable(fsoc_virtual_actuator_tests tests/virtual_actuator_tests.cpp) + target_link_libraries(fsoc_virtual_actuator_tests PRIVATE fsoc::virtual_actuator) + fsoc_set_project_warnings(fsoc_virtual_actuator_tests) + fsoc_enable_sanitizers(fsoc_virtual_actuator_tests) + add_test(NAME fsoc_virtual_actuator_tests COMMAND fsoc_virtual_actuator_tests) + + add_executable(fsoc_live_camera_calibration_tests tests/live_camera_calibration_tests.cpp) + target_link_libraries(fsoc_live_camera_calibration_tests PRIVATE fsoc::camera_calibration) + fsoc_set_project_warnings(fsoc_live_camera_calibration_tests) + fsoc_enable_sanitizers(fsoc_live_camera_calibration_tests) + add_test(NAME fsoc_live_camera_calibration_tests COMMAND fsoc_live_camera_calibration_tests) + if(FSOC_OPENCV_AVAILABLE) add_executable(fsoc_step4_tests tests/step4_tests.cpp) target_link_libraries(fsoc_step4_tests PRIVATE fsoc::render) @@ -492,5 +696,38 @@ if(FSOC_BUILD_TESTS) fsoc_set_project_warnings(fsoc_stage4_determinism_tests) fsoc_enable_sanitizers(fsoc_stage4_determinism_tests) add_test(NAME fsoc_stage4_determinism_tests COMMAND fsoc_stage4_determinism_tests) + + # --- Mobile Phone Camera-in-the-Loop: OpenCV/videoio-dependent pieces --- + if(FSOC_FRAME_SOURCE_AVAILABLE) + add_executable(fsoc_frame_source_tests tests/frame_source_tests.cpp) + target_link_libraries(fsoc_frame_source_tests PRIVATE fsoc::frame_source) + fsoc_set_project_warnings(fsoc_frame_source_tests) + fsoc_enable_sanitizers(fsoc_frame_source_tests) + add_test(NAME fsoc_frame_source_tests COMMAND fsoc_frame_source_tests) + + add_executable(fsoc_live_preprocessing_tests tests/live_preprocessing_tests.cpp) + target_link_libraries(fsoc_live_preprocessing_tests PRIVATE fsoc::live_support) + fsoc_set_project_warnings(fsoc_live_preprocessing_tests) + fsoc_enable_sanitizers(fsoc_live_preprocessing_tests) + add_test(NAME fsoc_live_preprocessing_tests COMMAND fsoc_live_preprocessing_tests) + + add_executable(fsoc_live_tracking_session_tests tests/live_tracking_session_tests.cpp) + target_link_libraries(fsoc_live_tracking_session_tests PRIVATE fsoc::live_support) + fsoc_set_project_warnings(fsoc_live_tracking_session_tests) + fsoc_enable_sanitizers(fsoc_live_tracking_session_tests) + add_test(NAME fsoc_live_tracking_session_tests COMMAND fsoc_live_tracking_session_tests) + + add_executable(fsoc_live_frame_publisher_tests tests/live_frame_publisher_tests.cpp) + target_link_libraries(fsoc_live_frame_publisher_tests PRIVATE fsoc::live_frame_publisher) + fsoc_set_project_warnings(fsoc_live_frame_publisher_tests) + fsoc_enable_sanitizers(fsoc_live_frame_publisher_tests) + add_test(NAME fsoc_live_frame_publisher_tests COMMAND fsoc_live_frame_publisher_tests) + + add_executable(fsoc_real_session_recorder_tests tests/real_session_recorder_tests.cpp) + target_link_libraries(fsoc_real_session_recorder_tests PRIVATE fsoc::real_session_recorder) + fsoc_set_project_warnings(fsoc_real_session_recorder_tests) + fsoc_enable_sanitizers(fsoc_real_session_recorder_tests) + add_test(NAME fsoc_real_session_recorder_tests COMMAND fsoc_real_session_recorder_tests) + endif() endif() endif() diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..59d0f74 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,40 @@ +# Contributing + +FSOC is a research/hackathon engineering project (SIH26169). Contributions are +welcome, but the architecture boundaries below are load-bearing — read +`CLAUDE.md` before opening a PR that touches `src/`, `include/`, or `apps/`. + +## Before you start + +- **C++ changes**: know which module you're touching + (`Environment` / `Trajectory` / `PanTiltCamera` / `Detector` / `Estimator` / + `Controller` / `SimulationRunner` / `Telemetry`). Don't let physics, detection, + control, and logging bleed into one function — see `docs/15_INTERFACE_CONTRACTS.md`. +- **Coordinate/unit conventions are frozen** (`docs/04_COORDINATES_AND_MATH.md`). + If a change requires touching a sign or unit, update the math doc and tests in + the same PR. +- **The `v1_baseline` tag is frozen.** Don't modify validated tracking math, + detector behavior, controller tuning, or `presentation_assets/` evidence + unless an actual bug forces it. +- **Frontend changes**: `frontend/DESIGN_SYSTEM.md` documents the token system + (`Orbital Precision`) — reuse existing `components/ui` primitives rather than + introducing new visual patterns. + +## Workflow + +```bash +cmake --preset debug && cmake --build --preset debug && ctest --preset debug +cd frontend && npm run typecheck && npm run lint && npm run build && npx playwright test +``` + +All four must pass before opening a PR. The PR template +(`.github/pull_request_template.md`) asks which module you touched and which +architecture checks apply — fill it in honestly, it's there to catch scope +creep, not to be busywork. + +## Reporting bugs / proposing features + +Open a GitHub issue. For anything touching the frozen baseline or measured +claims in `README.md`, include the exact command/output that shows the +current (and, if applicable, proposed) behavior — this project treats +measured numbers as load-bearing, not decorative. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..1e97cd8 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Kirtan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 4d7b9e8..8c52510 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ OpenCV DNN inference ONNX model Next.js Mission Control + MIT License

@@ -41,7 +42,7 @@ FSOC closes the loop between **what a camera sees and how an optical terminal po The core runs in **C++20**. **TinyBeaconNet** performs neural inference through **OpenCV DNN**, while **Mission Control** makes the resulting trajectories, errors, controller commands, and tracking states inspectable. -**Current scope:** the default branch provides a deterministic software-in-the-loop system. The [phone-camera branch](https://github.com/ThatKJ/FSOC/tree/feat/phone-camera-in-loop) extends it toward real image input with a virtual actuator; its physical validation remains separate from the simulation results below. +**Current scope:** the deterministic software-in-the-loop simulation below is the frozen SIH baseline. This branch also carries the **phone/webcam camera-in-the-loop prototype** (`fsoc_live`, see [Mobile Phone Camera-in-the-Loop](#mobile-phone-camera-in-the-loop-optional-additive-prototype)): a real camera frame through the same perception/tracking/control stack, with an honestly virtual actuator. Its physical validation is tracked separately from the simulation results below. ## Why this exists @@ -239,6 +240,18 @@ The optional [`TargetTracker`](include/fsoc/target_tracker.hpp) adds an alpha-be These are **recorded simulation results**, linked to their committed methods and evidence. They are not physical-camera accuracy measurements or a fresh benchmark of the reader's machine. +| | | +|---|---| +| Step-10 baseline acceptance | **7 / 7 PASS** | +| C++ test suites (`ctest`) | **22 / 22** (100%) | +| Frontend end-to-end tests (Playwright) | **20 / 20** | +| Severe (>50px) closed-loop outliers — Classical | 2,240 | +| Severe closed-loop outliers — Classical + Tracker | 12 (**↓ ~99.5%**) | +| Severe closed-loop outliers — Hybrid | 1,808 | +| Severe closed-loop outliers — Hybrid + Tracker (V2) | 9 (**↓ ~99.5%**) | +| Full-step latency, Hybrid + Tracker, P95 | **~1.2 ms**, vs. the 20 ms / 50 Hz budget | +| Telemetry pipeline | 42-column real CSV export, header-driven | + ### Classical closed-loop baseline The Step-10 comparison runs the same sinusoidal trajectory with and without control: @@ -276,6 +289,73 @@ The Stage-4 ablation evaluates **11 scenarios × 5 seeds × 400 steps = 22,000 s Timing was measured on the documented **Apple M5 development machine**, not embedded or flight hardware. Earlier runs report P95 values of 1.439–1.639 ms under their recorded conditions; timing depends on workload and platform. [Freeze run](docs/SIH_MVP_FREEZE.md) · [Timing methodology](docs/MVP_METRICS.md) +## Mobile Phone Camera-in-the-Loop (optional, additive prototype) + +Alongside the fully-simulated demo above, `fsoc_live` (`apps/fsoc_live.cpp`) makes the +**sensing side** real: a real phone/webcam frame, run through the exact same Classical + +TinyBeaconNet + Safe Hybrid perception, P0-v2 state estimator, and PID controller the +simulation uses — see `docs/PHONE_CAMERA_METRICS.md`. The **actuator side stays honestly +virtual**: `VirtualPanTiltActuator` bookkeeps a commanded angle and drives no physical +hardware. Every telemetry frame and the Mission Control view at `/mission/live` label this +explicitly (`CAMERA SOURCE: REAL_PHONE_CAMERA`, `ACTUATOR: VIRTUAL`). This is a +"real-camera-in-the-loop prototype," never a "physical pan/tilt tracking system" — see +`docs/PHONE_CAMERA_METRICS.md`'s claim boundary. Run it yourself with `./run_fsoc.sh phone` +(or `./run_fsoc.sh golden` for the full narrated walkthrough, +`docs/PHONE_CAMERA_GOLDEN_DEMO.md`) — requires a real camera and, on macOS, granting an OS +permission prompt. + +## Hardware boundary + +No physical beacon, servo, or pan/tilt actuator has been used anywhere in this project, and +the frozen SIH MVP simulation above uses zero physical hardware of any kind. The one +exception is the optional prototype directly above: it reads frames from a real camera, but +still commands no physical actuator. Every measured number in the "Measured results" section +above comes from the deterministic C++ simulation on a desktop-class development machine +(Apple M5), not from the camera prototype. **No claim of embedded, flight, physical +actuation, or real-time-on-target hardware performance is made or implied anywhere in this +repository.** See `docs/MVP_METRICS.md §5`, `docs/SIH_MVP_FREEZE.md §6`, and +`docs/PHONE_CAMERA_METRICS.md`. + +## Working modes + +FSOC runs in more than one context, and each has a different, explicit camera/actuator +boundary — never blurred, never silently upgraded to sound more impressive: + +| Mode | Camera | Actuator | Purpose | +|---|---|---|---| +| **Simulation** (`fsoc_demo`) | Synthetic (rendered) | Simulated | Deterministic development, evaluation, and the frozen SIH baseline. | +| **Mission Control — REPLAY** | Synthetic (recorded) | Simulated | Public/offline viewing of a checked-in run — no C++ build required. | +| **Mission Control — ENGINE** | Synthetic (live) | Simulated | Local viewing of `fsoc_demo` running live, streamed over `/api/simulation/:scenario`. | +| **Phone camera-in-the-loop** (`fsoc_live`) | Real (phone/webcam) | Virtual (bookkept, drives nothing) | Proves the perception → estimation → control stack against a real image, locally. | +| **Physical hardware** | Real | Real (physical pan/tilt) | **Not implemented.** Would replace `FrameSource` / `PanTiltCamera` behind the same interfaces (`docs/09_FUTURE_ARCHITECTURE.md`) — no such hardware exists in this repository today. | + +The public web deployment (see **Deployment** below) only ever serves the first two rows — +it has no access to your camera or a local C++ process, and it never pretends otherwise. + +## Deployment + +The public site is a static/serverless Next.js deployment of `frontend/` — it shows the +project, its architecture, and deterministic replay evidence produced by the real C++ +engine. It is **not** a backend for the phone-camera prototype: your camera and `fsoc_live` +run on your own machine, and Vercel has no way to reach either. `/mission/live` detects the +missing local session and shows a "run this locally" state instead of fabricating one — +see the API route at `frontend/app/api/live-camera/route.ts`. + +Full architecture, project settings, and the local-vs-public split: **`docs/DEPLOYMENT.md`**. + +### One-command launcher + +```bash +./run_fsoc.sh +``` + +One command, one interactive menu — Simulation Demo, Phone Camera Demo, Camera Probe, +Mission Control Only, Run Full Validation, Golden Phone Demo, or Build Everything. It +builds only what's missing, never guesses a port (reads it from +`frontend/package.json`), never touches a process it didn't start, and cleans up on +Ctrl+C. Non-interactive: `./run_fsoc.sh simulation|phone|probe|ui|test|golden|build`. +See `scripts/run_fsoc.sh --help` for every flag (`--rebuild`, `--no-browser`). + ## Validation Run the baseline checks from the repository root: @@ -283,8 +363,14 @@ Run the baseline checks from the repository root: ```bash cmake --preset debug -DFSOC_ENABLE_OPENCV=ON cmake --build --preset debug -ctest --preset debug --output-on-failure -./build/debug/step10_validation_smoke +ctest --preset debug --output-on-failure # 22/22 C++ suites +./build/debug/step10_validation_smoke # 7/7 baseline gates +cmake --preset release && cmake --build --preset release +./build/release/stage4_evaluation --out generated/ai_stage4 # ~10-15 min, frozen protocol +./build/release/stage4_tracker_ablation --out generated/ai_stage4_ablation # ~15 min, clutter mitigation +./build/release/mvp_dynamic_scenarios --out generated/mvp_dynamic_scenarios # velocity/dropout/moving-clutter +./build/release/mvp_latency_budget # full latency budget, seconds +cd frontend && npm run typecheck && npm run lint && npm run build && npx playwright test ``` For the frontend, run the following inside `frontend/` after `npm ci`. The checked-in Playwright configuration uses **installed Google Chrome**, rather than a Playwright-managed Chromium download. @@ -296,6 +382,45 @@ npm run build npx playwright test ``` +## Known limitations + +State these proactively — they're disclosed in `docs/SIH_MVP_FREEZE.md`, not buried: + +1. **A temporally coherent moving distractor defeats the clutter mitigation + completely** — identical outlier counts with or without the tracker + (`docs/MVP_ABLATION.md §6`). The single most important limitation. +2. **The intrinsic single-frame classical false-positive rate (44.9%) is unchanged** — + unfixable without touching the frozen classical detector algorithm. +3. **AI-only reacquisition through `resolve_perception()` is not implemented** — the + prerequisite gate exists (ADR-019); unlocking it is a distinct, deliberately + deferred change to the frozen fusion policy. +4. AI recall is intentionally low (~16-40% depending on scenario) rather than + over-confident — a deliberate precision-over-recall design choice, not a bug. +5. Real coverage cost: Hybrid+Tracker trades ~20 points of coverage for the outlier + reduction above — a disclosed trade, not a free win. +6. Evaluated at n=5 seeds per scenario (matches the frozen protocol) — real, not + large-sample, statistics. + +## Repository structure + +```text +include/fsoc/ Public interfaces +src/ Core implementations +apps/ Executable simulation/demo/benchmark/evaluation programs +tests/ Mathematical/unit validation (CTest) +models/ Committed trained ONNX model + metadata (models/MODEL_CARD.md) +tools/ai/ Offline Python training toolchain (NOT part of the C++ runtime) +tools/beacon_display.html Real, physical test-target page for the phone-camera prototype +frontend/ Next.js Mission Control UI (reads real fsoc_demo telemetry) +cmake/ Build policies +.github/workflows/ CI (build + test, C++ and frontend) +.claude/skills/ Claude Code engineering skills +.claude/agents/ Specialist subagents +docs/ Design docs, measured metrics, ADRs, development history +prompts/ Reusable Vibe Coding prompts +generated/ Git-ignored run artifacts (CSV/PNG/JSON reports) — never committed +``` + The [CI workflow](.github/workflows/ci.yml) separately builds/tests C++ and checks the frontend. Its browser job uses replay fixtures; local tests that require an engine need the native build available. The badge at the top links to current runs; [freeze-time test results](docs/SIH_MVP_FREEZE.md) are historical evidence.

@@ -332,7 +457,7 @@ The evaluation deliberately includes failure cases: | Deterministic C++ coarse-alignment loop | Implemented; baseline acceptance documented | | Neural inference and Hybrid policy | Implemented; model, parity checks, and evaluation committed | | Temporal tracking and bounded recovery | Implemented; ablation and failure cases documented | -| Phone/webcam frame input with virtual actuation | Separate [development branch](https://github.com/ThatKJ/FSOC/tree/feat/phone-camera-in-loop); physical measurements pending in its [test documentation](https://github.com/ThatKJ/FSOC/blob/683f70835c2fd70be5c3bbea03cc3d0fa88db679/docs/PHONE_CAMERA_METRICS.md) | +| Phone/webcam frame input with virtual actuation | Implemented (`fsoc_live`); see [Mobile Phone Camera-in-the-Loop](#mobile-phone-camera-in-the-loop-optional-additive-prototype) and [test documentation](docs/PHONE_CAMERA_METRICS.md) | | Continuous streaming, stronger live-session UX, and evidence export | Further integration work | | Packaged desktop application and physical pan/tilt bench | Planned | | Coarse-to-fine handoff and optical-link validation | Future research | @@ -385,9 +510,13 @@ The evaluation deliberately includes failure cases: ## Contributing -Open an [issue](https://github.com/ThatKJ/FSOC/issues) with the scenario, configuration, commit, and evidence needed to reproduce a problem. Keep changes focused and include relevant tests. +Open an [issue](https://github.com/ThatKJ/FSOC/issues) with the scenario, configuration, commit, and evidence needed to reproduce a problem. Keep changes focused and include relevant tests. See [CONTRIBUTING.md](CONTRIBUTING.md) and [SECURITY.md](SECURITY.md) for the full process and reporting a vulnerability. + +Read [AGENTS.md](AGENTS.md) and [CLAUDE.md](CLAUDE.md) before implementation (human or AI). They define the module boundaries (`Environment` / `Trajectory` / `PanTiltCamera` / `Detector` / `Controller` / ...), coordinate conventions, and the C++20/CMake-only build rules that keep the simulation mathematically traceable. Preserve the frozen baseline, keep truth out of the controller, retain explicit units, and document any algorithm change with its measured effect (`docs/16_AI_CODING_GUARDRAILS.md`). + +## License -Read [AGENTS.md](AGENTS.md) and [CLAUDE.md](CLAUDE.md) before implementation. Preserve the frozen baseline, keep truth out of the controller, retain explicit units, and document any algorithm change with its measured effect. The repository does not currently include a `LICENSE` file. +[MIT](LICENSE).

Built by Team IRODOV
diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..dcc9599 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,19 @@ +# Security Policy + +FSOC is a research/hackathon simulation and evaluation testbed. There is no +production deployment handling user accounts, payments, or sensitive personal +data — the public web deployment is a read-only project site and deterministic +replay viewer (see `docs/DEPLOYMENT.md`), and the phone-camera prototype +(`fsoc_live`) processes camera frames locally on the machine that runs it and +does not transmit them anywhere. + +## Reporting a vulnerability + +If you find a security issue (e.g. a way to make a public API route read +local files it shouldn't, or execute anything server-side), please open a +GitHub issue on `ThatKJ/FSOC` describing it. Since this project has no users +or data at risk beyond its own source and demo evidence, there is no formal +disclosure SLA — but real reports are read and fixed. + +Please do not open an issue containing a working exploit against a third +party; describe the class of problem instead. diff --git a/apps/fsoc_camera_calibrate.cpp b/apps/fsoc_camera_calibrate.cpp new file mode 100644 index 0000000..cac8c22 --- /dev/null +++ b/apps/fsoc_camera_calibrate.cpp @@ -0,0 +1,106 @@ +// fsoc_camera_calibrate — the simplest defensible camera geometry (Phase 8). +// NOT a photogrammetry suite: either declare a manually-known FOV (from the +// phone's spec sheet) or estimate it from one known-size object at a known +// distance (angular-substitution method). Writes a dependency-free +// key=value calibration file that fsoc_live / fsoc_camera_calibrate --check +// can read back. +// +// Usage: +// fsoc_camera_calibrate --manual --width 1920 --height 1080 \ +// --hfov-deg 69 --vfov-deg 42 --out configs/phone_camera.cfg +// +// fsoc_camera_calibrate --from-object --object-width-m 0.05 --distance-m 1.0 \ +// --object-pixel-width-px 120 --image-width-px 1920 --image-height-px 1080 \ +// --out configs/phone_camera.cfg +// +// fsoc_camera_calibrate --check configs/phone_camera.cfg + +#include +#include +#include + +#include "fsoc/live_camera_calibration.hpp" + +namespace { + +void print_usage() { + std::cout + << "Usage:\n" + << " fsoc_camera_calibrate --manual --width W --height H --hfov-deg D --vfov-deg D2 --out PATH\n" + << " fsoc_camera_calibrate --from-object --object-width-m M --distance-m D \\\n" + << " --object-pixel-width-px P --image-width-px W --image-height-px H --out PATH\n" + << " fsoc_camera_calibrate --check PATH\n"; +} + +} // namespace + +int main(int argc, char** argv) { + std::optional mode{}; + int width = 0, height = 0; + double hfov_deg = 0.0, vfov_deg = 0.0; + double object_width_m = 0.0, distance_m = 0.0, object_pixel_width_px = 0.0; + std::string out_path; + std::string check_path; + + for (int i = 1; i < argc; ++i) { + const std::string arg = argv[i]; + auto next = [&]() -> std::string { return (i + 1 < argc) ? argv[++i] : std::string{}; }; + if (arg == "--manual") mode = "manual"; + else if (arg == "--from-object") mode = "from-object"; + else if (arg == "--check") { mode = "check"; check_path = next(); } + else if (arg == "--width" || arg == "--image-width-px") width = std::stoi(next()); + else if (arg == "--height" || arg == "--image-height-px") height = std::stoi(next()); + else if (arg == "--hfov-deg") hfov_deg = std::stod(next()); + else if (arg == "--vfov-deg") vfov_deg = std::stod(next()); + else if (arg == "--object-width-m") object_width_m = std::stod(next()); + else if (arg == "--distance-m") distance_m = std::stod(next()); + else if (arg == "--object-pixel-width-px") object_pixel_width_px = std::stod(next()); + else if (arg == "--out") out_path = next(); + else if (arg == "--help" || arg == "-h") { print_usage(); return 0; } + else { std::cerr << "unrecognized argument '" << arg << "'\n"; print_usage(); return 2; } + } + + if (!mode.has_value()) { + print_usage(); + return 2; + } + + try { + if (*mode == "check") { + const fsoc::LiveCameraCalibrationConfig config = fsoc::load_live_camera_calibration(check_path); + std::cout << "OK: " << check_path << "\n" + << " " << config.width_px << "x" << config.height_px << " hfov=" << config.hfov_deg + << "deg vfov=" << config.vfov_deg << "deg\n"; + return 0; + } + + fsoc::LiveCameraCalibrationConfig config{}; + if (*mode == "manual") { + config.width_px = width; + config.height_px = height; + config.hfov_deg = hfov_deg; + config.vfov_deg = vfov_deg; + } else { + config.width_px = width; + config.height_px = height; + config.hfov_deg = fsoc::estimate_hfov_deg_from_known_object( + object_width_m, distance_m, object_pixel_width_px, width); + config.vfov_deg = fsoc::estimate_vfov_deg_from_hfov(config.hfov_deg, width, height); + std::cout << "Estimated (angular-substitution, no lens-distortion correction):\n" + << " hfov_deg=" << config.hfov_deg << " vfov_deg=" << config.vfov_deg << "\n" + << " This is a first-order approximation -- adequate for coarse alignment,\n" + << " not a substitute for a real checkerboard calibration.\n"; + } + + if (out_path.empty()) { + std::cerr << "--out PATH is required\n"; + return 2; + } + fsoc::save_live_camera_calibration(out_path, config); + std::cout << "Wrote " << out_path << "\n"; + return 0; + } catch (const std::exception& e) { + std::cerr << "fsoc_camera_calibrate: " << e.what() << "\n"; + return 1; + } +} diff --git a/apps/fsoc_camera_probe.cpp b/apps/fsoc_camera_probe.cpp new file mode 100644 index 0000000..a835539 --- /dev/null +++ b/apps/fsoc_camera_probe.cpp @@ -0,0 +1,80 @@ +// fsoc_camera_probe — enumerate plausible camera indices and report what each +// one actually negotiates (resolution / FPS / backend / status). Does NOT +// touch the tracking pipeline; this is pure device discovery (Phase 3). +// +// Run this YOURSELF, interactively, on your own machine — opening a camera +// device may trigger an OS permission prompt (macOS TCC) that only a real +// interactive session can answer. +// +// Usage: +// fsoc_camera_probe [--max-index N] + +#include +#include +#include + +#include "fsoc/opencv_camera_frame_source.hpp" + +namespace { + +struct Args { + int max_index = 4; +}; + +Args parse_args(int argc, char** argv) { + Args args{}; + for (int i = 1; i < argc; ++i) { + const std::string arg = argv[i]; + if (arg == "--max-index" && i + 1 < argc) { + args.max_index = std::stoi(argv[++i]); + } else if (arg == "--help" || arg == "-h") { + std::cout << "Usage: fsoc_camera_probe [--max-index N]\n" + << "Probes camera indices 0..N (default N=4) and reports what each\n" + << "negotiates: resolution, FPS, backend, status.\n"; + std::exit(0); + } + } + return args; +} + +} // namespace + +int main(int argc, char** argv) { + const Args args = parse_args(argc, argv); + + std::cout << "FSOC camera probe -- indices 0.." << args.max_index << "\n"; + std::cout << std::left << std::setw(6) << "INDEX" << std::setw(14) << "RESOLUTION" << std::setw(10) + << "FPS" << std::setw(16) << "BACKEND" << "STATUS\n"; + + int available_count = 0; + for (int index = 0; index <= args.max_index; ++index) { + fsoc::OpenCVCameraFrameSourceConfig config{}; + config.camera_index = index; + fsoc::OpenCVCameraFrameSource source(config); + + if (!source.open()) { + std::cout << std::left << std::setw(6) << index << std::setw(14) << "-" << std::setw(10) + << "-" << std::setw(16) << "-" << "UNAVAILABLE\n"; + continue; + } + + const fsoc::FrameSourceInfo info = source.info(); + ++available_count; + const std::string resolution = std::to_string(info.width_px) + "x" + std::to_string(info.height_px); + const std::string fps = info.fps.has_value() ? std::to_string(static_cast(*info.fps)) : "?"; + std::cout << std::left << std::setw(6) << index << std::setw(14) << resolution << std::setw(10) + << fps << std::setw(16) << info.backend_name << "AVAILABLE\n"; + source.close(); + } + + if (available_count == 0) { + std::cout << "\nNo camera indices responded. If you expected a phone/webcam here:\n" + << " - macOS: check System Settings > Privacy & Security > Camera and grant\n" + << " this terminal/binary permission, then re-run.\n" + << " - a USB/continuity-camera phone link may need the OS to finish pairing\n" + << " before it appears as a camera index.\n" + << " - try --max-index with a larger value.\n"; + return 1; + } + return 0; +} diff --git a/apps/fsoc_camera_view.cpp b/apps/fsoc_camera_view.cpp new file mode 100644 index 0000000..6abb09f --- /dev/null +++ b/apps/fsoc_camera_view.cpp @@ -0,0 +1,175 @@ +// fsoc_camera_view — minimal live-camera viewer. Proves FSOC can receive +// frames from a real camera reliably, BEFORE any tracking logic runs +// (Phase 4). No GUI dependency: this environment cannot assume a display is +// attached, so frames are saved as periodic JPEG snapshots instead of shown +// in a window; per-frame stats go to stdout. +// +// Run this YOURSELF, interactively — opening a camera device may trigger an +// OS permission prompt only a real interactive session can answer. +// +// Usage: +// fsoc_camera_view --camera-index 0 [--seconds 60] [--snapshot-every 2.0] +// [--out-dir generated/live] [--crosshair] +// fsoc_camera_view --camera-url "" [--seconds 60] ... + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "fsoc/opencv_camera_frame_source.hpp" + +namespace { + +struct Args { + std::optional camera_index{}; + std::optional camera_url{}; + double seconds = 60.0; + double snapshot_every_s = 2.0; + std::string out_dir = "generated/live"; + bool crosshair = false; +}; + +void print_usage() { + std::cout << "Usage:\n" + << " fsoc_camera_view --camera-index N [--seconds 60] [--snapshot-every 2.0]\n" + << " [--out-dir generated/live] [--crosshair]\n" + << " fsoc_camera_view --camera-url URL [same options]\n"; +} + +std::optional parse_args(int argc, char** argv) { + Args args{}; + for (int i = 1; i < argc; ++i) { + const std::string arg = argv[i]; + if (arg == "--camera-index" && i + 1 < argc) { + args.camera_index = std::stoi(argv[++i]); + } else if (arg == "--camera-url" && i + 1 < argc) { + args.camera_url = std::string(argv[++i]); + } else if (arg == "--seconds" && i + 1 < argc) { + args.seconds = std::stod(argv[++i]); + } else if (arg == "--snapshot-every" && i + 1 < argc) { + args.snapshot_every_s = std::stod(argv[++i]); + } else if (arg == "--out-dir" && i + 1 < argc) { + args.out_dir = argv[++i]; + } else if (arg == "--crosshair") { + args.crosshair = true; + } else if (arg == "--help" || arg == "-h") { + print_usage(); + std::exit(0); + } else { + std::cerr << "fsoc_camera_view: unrecognized argument '" << arg << "'\n"; + print_usage(); + return std::nullopt; + } + } + if (args.camera_index.has_value() == args.camera_url.has_value()) { + std::cerr << "fsoc_camera_view: specify exactly one of --camera-index or --camera-url\n"; + print_usage(); + return std::nullopt; + } + return args; +} + +} // namespace + +int main(int argc, char** argv) { + const auto parsed = parse_args(argc, argv); + if (!parsed.has_value()) { + return 2; + } + const Args& args = *parsed; + + fsoc::OpenCVCameraFrameSourceConfig config{}; + config.camera_index = args.camera_index; + config.url = args.camera_url; + fsoc::OpenCVCameraFrameSource source(config); + + if (!source.open()) { + std::cerr << "fsoc_camera_view: FAILED to open camera source (" + << (args.camera_index.has_value() ? ("index " + std::to_string(*args.camera_index)) + : ("url " + *args.camera_url)) + << ").\n" + << " This is a clean failure, not a crash. Common causes:\n" + << " - wrong index/URL\n" + << " - camera permission not granted (macOS: System Settings > Privacy & " + "Security > Camera)\n" + << " - another process already holds the device\n" + << "Run fsoc_camera_probe first to see which indices are AVAILABLE.\n"; + return 1; + } + + const fsoc::FrameSourceInfo info = source.info(); + std::cout << "Opened " << fsoc::to_string(info.kind) << " (" << info.backend_name + << "), negotiated " << info.width_px << "x" << info.height_px << "\n"; + std::cout << "Running for " << args.seconds << "s. Snapshots every " << args.snapshot_every_s + << "s to " << args.out_dir << "/\n"; + + std::system(("mkdir -p " + args.out_dir).c_str()); + + const auto start = std::chrono::steady_clock::now(); + double last_snapshot_s = -1.0; + std::size_t frame_count = 0; + std::size_t consecutive_failures = 0; + constexpr std::size_t kMaxConsecutiveFailures = 30; + double last_timestamp_s = 0.0; + + fsoc::Frame frame{}; + while (true) { + const double elapsed_s = + std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + if (elapsed_s >= args.seconds) { + break; + } + + if (!source.read(frame)) { + ++consecutive_failures; + std::cerr << "warning: frame read failed (" << consecutive_failures << " consecutive)\n"; + if (consecutive_failures >= kMaxConsecutiveFailures) { + std::cerr << "fsoc_camera_view: too many consecutive read failures -- camera " + "appears disconnected. Exiting cleanly.\n"; + source.close(); + return 1; + } + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + continue; + } + consecutive_failures = 0; + ++frame_count; + + if (frame.timestamp_s < last_timestamp_s) { + std::cerr << "warning: timestamp went backward (" << frame.timestamp_s << " < " + << last_timestamp_s << ") -- source clock discontinuity\n"; + } + last_timestamp_s = frame.timestamp_s; + + if (frame.timestamp_s - last_snapshot_s >= args.snapshot_every_s) { + cv::Mat out = frame.image.clone(); + if (args.crosshair) { + const cv::Point center(out.cols / 2, out.rows / 2); + const cv::Scalar color = out.channels() == 1 ? cv::Scalar(255) : cv::Scalar(0, 255, 0); + cv::drawMarker(out, center, color, cv::MARKER_CROSS, 40, 2); + } + const std::string path = + args.out_dir + "/frame_" + std::to_string(frame.frame_index) + ".jpg"; + cv::imwrite(path, out); + last_snapshot_s = frame.timestamp_s; + + const double effective_fps = frame.frame_index > 0 ? static_cast(frame.frame_index) / + std::max(frame.timestamp_s, 1e-6) + : 0.0; + std::cout << std::fixed << std::setprecision(2) << "frame " << frame.frame_index << " t=" + << frame.timestamp_s << "s " << frame.image.cols << "x" << frame.image.rows + << " effective_fps=" << effective_fps << " -> " << path << "\n"; + } + } + + source.close(); + std::cout << "Done. Captured " << frame_count << " frames over " << args.seconds << "s.\n"; + return 0; +} diff --git a/apps/fsoc_live.cpp b/apps/fsoc_live.cpp new file mode 100644 index 0000000..0c4d610 --- /dev/null +++ b/apps/fsoc_live.cpp @@ -0,0 +1,604 @@ +// fsoc_live — Mobile Phone Camera-in-the-Loop. +// +// REAL TARGET -> MOBILE PHONE CAMERA -> REAL VIDEO FRAME +// -> Hybrid Perception -> State Estimator -> Prediction -> Controller +// -> VirtualPanTiltActuator -> REAL CONTROL COMMAND TELEMETRY +// +// This is a real-camera-in-the-loop PROTOTYPE, not a physical closed loop: +// the sensing side (camera + perception + estimation) is fully real; the +// actuator side is HONESTLY VIRTUAL (see fsoc/virtual_actuator.hpp) — no +// servo, gimbal, or physical pan/tilt hardware exists or is claimed. See +// docs/PHONE_CAMERA_METRICS.md and README.md's claim-boundary section. +// +// Usage: +// fsoc_live --source camera --camera-index 0 --calibration configs/phone_camera.cfg +// fsoc_live --source camera --camera-index 0 --uncalibrated +// fsoc_live --source camera-url --camera-url "" --calibration configs/phone_camera.cfg +// +// Options: +// --calibration PATH real/measured FOV file (see fsoc_camera_calibrate) -- +// exactly one of --calibration or --uncalibrated is required +// --uncalibrated skip calibration entirely: reports PIXEL measurements +// only. panErrorDeg/tiltErrorDeg/totalErrorDeg are null +// (there is no real FOV to convert pixels->degrees from), +// and control/actuation is force-disabled -- a fabricated +// FOV must never drive a command. Use this to get a real +// preview + pixel-only tracking running before you have a +// calibration file. See docs/PHONE_CAMERA_METRICS.md +// "Camera calibration". +// --mode classical|ai|hybrid (default classical) +// --tracker enable the P0-v2 alpha-beta estimator +// --manual-assist print human-readable PAN/TILT (or, uncalibrated, pixel) +// correction cues +// --no-control observe-only: never compute/apply a command +// --seconds N stop after N seconds (default: run until Ctrl+C / camera ends) +// --live-out DIR where manifest.json / frame_.jpg / telemetry_.json +// are published (default generated/live) +// --record-out DIR where G2 recordings are written (default +// generated/real_sessions) -- see fsoc/real_session_recorder.hpp +// --ai-model PATH required when --mode ai or --mode hybrid +// +// G2 recording: a browser (or anything else) requests start/stop/mark_event by +// writing /command.txt (key=value, one command at a time -- see +// read_pending_command() below); this process polls that file once per loop +// iteration. There is no message queue: two commands written faster than one +// camera frame interval apart will have the earlier one silently superseded. A +// human clicking a UI button will not realistically do this; it is a documented +// limitation, not a bug. +// +// Run this YOURSELF, interactively — opening a camera device may trigger an +// OS permission prompt only a real interactive session can answer. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "fsoc/config.hpp" +#include "fsoc/live_camera_calibration.hpp" +#include "fsoc/live_frame_publisher.hpp" +#include "fsoc/live_tracking_session.hpp" +#include "fsoc/opencv_camera_frame_source.hpp" +#include "fsoc/real_session_recorder.hpp" + +namespace { + +struct Args { + std::optional camera_index{}; + std::optional camera_url{}; + std::string calibration_path{}; + bool uncalibrated = false; + fsoc::PerceptionMode mode{fsoc::PerceptionMode::Classical}; + std::string ai_model_path{}; + bool tracker = false; + bool manual_assist = false; + bool control_enabled = true; + double seconds = -1.0; // -1 = run until stopped + std::string live_out = "generated/live"; + std::string record_out = "generated/real_sessions"; +}; + +// One pending operator command (see the "G2 recording" file comment above). +// key=value, same house convention as fsoc/live_camera_calibration.hpp -- no JSON +// library is linked into the C++ core, and this file is written by hand from a +// tiny Next.js route, so a parser this simple is the honest match for both ends. +struct LiveCommand { + std::string command_id{}; + std::string action{}; // "start" | "stop" | "mark_event" + std::string label{}; // only meaningful for mark_event +}; + +// Returns std::nullopt if the file is missing, unreadable, or missing a required +// key -- callers treat that identically to "no command pending" (never a fatal +// error: a malformed command must not stop tracking). +std::optional read_pending_command(const std::string& path) { + std::ifstream in(path); + if (!in) return std::nullopt; + LiveCommand cmd{}; + std::string line; + while (std::getline(in, line)) { + const auto eq = line.find('='); + if (eq == std::string::npos) continue; + const std::string key = line.substr(0, eq); + const std::string value = line.substr(eq + 1); + if (key == "commandId") cmd.command_id = value; + else if (key == "action") cmd.action = value; + else if (key == "label") cmd.label = value; + } + if (cmd.command_id.empty() || cmd.action.empty()) return std::nullopt; + return cmd; +} + +void print_usage() { + std::cout + << "Usage:\n" + << " fsoc_live --source camera --camera-index N --calibration PATH [options]\n" + << " fsoc_live --source camera --camera-index N --uncalibrated [options]\n" + << " fsoc_live --source camera-url --camera-url URL --calibration PATH [options]\n" + << "Options:\n" + << " --calibration PATH real/measured FOV file (see fsoc_camera_calibrate)\n" + << " --uncalibrated pixel-only mode: no FOV, angular fields are null,\n" + << " control/actuation force-disabled\n" + << " --mode classical|ai|hybrid (default classical)\n" + << " --tracker enable the P0-v2 alpha-beta estimator\n" + << " --manual-assist print human-readable correction cues\n" + << " --no-control observe-only: never compute/apply a command\n" + << " --seconds N stop after N seconds (default: run until stopped)\n" + << " --live-out DIR manifest.json / frame_.jpg / telemetry_.json output dir\n" + << " --record-out DIR G2 recording output dir (default generated/real_sessions)\n" + << " --ai-model PATH required for --mode ai / --mode hybrid\n"; +} + +std::optional parse_args(int argc, char** argv) { + Args args{}; + std::optional source{}; + for (int i = 1; i < argc; ++i) { + const std::string arg = argv[i]; + auto next = [&]() -> std::string { return (i + 1 < argc) ? argv[++i] : std::string{}; }; + if (arg == "--source") source = next(); + else if (arg == "--camera-index") args.camera_index = std::stoi(next()); + else if (arg == "--camera-url") args.camera_url = next(); + else if (arg == "--calibration") args.calibration_path = next(); + else if (arg == "--uncalibrated") args.uncalibrated = true; + else if (arg == "--mode") { + const auto v = next(); + if (v == "classical") args.mode = fsoc::PerceptionMode::Classical; + else if (v == "ai") args.mode = fsoc::PerceptionMode::AI; + else if (v == "hybrid") args.mode = fsoc::PerceptionMode::Hybrid; + else { std::cerr << "--mode must be classical|ai|hybrid\n"; return std::nullopt; } + } + else if (arg == "--ai-model") args.ai_model_path = next(); + else if (arg == "--tracker") args.tracker = true; + else if (arg == "--manual-assist") args.manual_assist = true; + else if (arg == "--no-control") args.control_enabled = false; + else if (arg == "--seconds") args.seconds = std::stod(next()); + else if (arg == "--live-out") args.live_out = next(); + else if (arg == "--record-out") args.record_out = next(); + else if (arg == "--help" || arg == "-h") { print_usage(); std::exit(0); } + else { std::cerr << "unrecognized argument '" << arg << "'\n"; print_usage(); return std::nullopt; } + } + + if (!source.has_value() || (*source != "camera" && *source != "camera-url")) { + std::cerr << "--source must be 'camera' or 'camera-url'\n"; + return std::nullopt; + } + if (*source == "camera" && !args.camera_index.has_value()) { + std::cerr << "--source camera requires --camera-index N\n"; + return std::nullopt; + } + if (*source == "camera-url" && !args.camera_url.has_value()) { + std::cerr << "--source camera-url requires --camera-url URL\n"; + return std::nullopt; + } + if (args.calibration_path.empty() && !args.uncalibrated) { + std::cerr << "exactly one of --calibration PATH or --uncalibrated is required\n" + " (--uncalibrated gives pixel-only measurements with no real FOV --\n" + " see fsoc_camera_calibrate to produce a real calibration file)\n"; + return std::nullopt; + } + if (!args.calibration_path.empty() && args.uncalibrated) { + std::cerr << "--calibration and --uncalibrated are mutually exclusive\n"; + return std::nullopt; + } + if (args.mode != fsoc::PerceptionMode::Classical && args.ai_model_path.empty()) { + std::cerr << "--mode " << (args.mode == fsoc::PerceptionMode::AI ? "ai" : "hybrid") + << " requires --ai-model PATH\n"; + return std::nullopt; + } + return args; +} + +std::string json_escape(const std::string& s) { + std::string out; + for (char c : s) { + if (c == '"' || c == '\\') out += '\\'; + out += c; + } + return out; +} + +std::string opt_json(const std::optional& v) { + if (!v.has_value()) return "null"; + std::ostringstream os; + os << std::setprecision(9) << *v; + return os.str(); +} + +// Hand-rolled, minimal JSON (no JSON library is linked into the C++ core — +// see docs/PHONE_CAMERA_METRICS.md for why). Schema documented in +// docs/PHONE_CAMERA_METRICS.md "Live telemetry JSON schema". +// G2: the live telemetry stream is also how the frontend confirms a recording +// command was actually applied (an accepted HTTP response only means the command +// file was written -- see the file-header comment on command.txt). recording_id is +// "" when inactive. +struct RecordingStatus { + bool active = false; + std::string recording_id{}; + std::size_t recorded_frame_count = 0; + std::size_t error_count = 0; +}; + +std::string to_json(const fsoc::LiveFrameResult& r, fsoc::PerceptionMode mode, bool control_enabled, + const std::string& session_id, bool calibrated, const RecordingStatus& recording) { + std::ostringstream j; + j << std::fixed << std::setprecision(6); + // Deliberately single-line/compact: this string is also appended verbatim as one + // line into RealSessionRecorder's telemetry.jsonl (one JSON object per line), so it + // must never contain an embedded literal newline -- see docs/LIVE_REALDATA_TASK_STATE.md. + j << "{" + << "\"schemaVersion\": 1," + << "\"sessionId\": \"" << json_escape(session_id) << "\"," + << "\"recordingActive\": " << (recording.active ? "true" : "false") << "," + << "\"recordingId\": " << (recording.active ? ("\"" + json_escape(recording.recording_id) + "\"") : "null") + << "," + << "\"recordedFrameCount\": " << recording.recorded_frame_count << "," + << "\"recordingErrorCount\": " << recording.error_count << "," + << "\"frameIndex\": " << r.frame_index << "," + << "\"timestampS\": " << r.timestamp_s << "," + << "\"dtS\": " << r.dt_s << "," + << "\"cameraSource\": \"REAL_PHONE_CAMERA\"," + << "\"actuatorType\": \"VIRTUAL\"," + << "\"sourceKind\": \"" << fsoc::to_string(r.source.kind) << "\"," + << "\"sourceBackend\": \"" << json_escape(r.source.backend_name) << "\"," + << "\"sourceDescription\": \"" << json_escape(r.source.description) << "\"," + << "\"rawWidthPx\": " << r.raw_width_px << "," + << "\"rawHeightPx\": " << r.raw_height_px << "," + << "\"preprocessedWidthPx\": " << r.preprocessed_width_px << "," + << "\"preprocessedHeightPx\": " << r.preprocessed_height_px << "," + << "\"perceptionMode\": \"" << fsoc::to_string(mode) << "\"," + << "\"perceptionSource\": \"" << fsoc::to_string(r.perception.perception_source) << "\"," + << "\"classicalDetected\": " << (r.perception.classical_detected ? "true" : "false") << "," + << "\"aiCandidateDetected\": " << (r.perception.ai_candidate_detected ? "true" : "false") << "," + << "\"aiPresenceProbability\": " << opt_json(r.perception.ai_presence_probability) << "," + << "\"targetDetected\": " << (r.target_detected ? "true" : "false") << "," + << "\"detectedXPx\": " + << (r.detection.has_value() ? std::to_string(r.detection->centroid_px.x_px) : "null") << "," + << "\"detectedYPx\": " + << (r.detection.has_value() ? std::to_string(r.detection->centroid_px.y_px) : "null") << "," + << "\"pixelErrorXPx\": " << (r.tracking_error.has_value() ? std::to_string(r.tracking_error->pixel.x_px) : "null") << "," + << "\"pixelErrorYPx\": " << (r.tracking_error.has_value() ? std::to_string(r.tracking_error->pixel.y_px) : "null") << "," + // Angular fields require a real FOV. In --uncalibrated mode the sensing camera is + // built from a placeholder FOV purely so the (unused) control math has *some* + // angle to compute with -- reporting that placeholder-derived degree value would + // look exactly like a real measurement, so it is always null here regardless of + // whether compute_tracking_error() produced a value. + << "\"panErrorDeg\": " + << ((calibrated && r.tracking_error.has_value()) + ? std::to_string(fsoc::rad_to_deg(r.tracking_error->angular.pan_rad)) + : "null") + << "," + << "\"tiltErrorDeg\": " + << ((calibrated && r.tracking_error.has_value()) + ? std::to_string(fsoc::rad_to_deg(r.tracking_error->angular.tilt_rad)) + : "null") + << "," + << "\"totalErrorDeg\": " + << ((calibrated && r.tracking_error.has_value()) + ? std::to_string(fsoc::rad_to_deg( + std::hypot(r.tracking_error->angular.pan_rad, r.tracking_error->angular.tilt_rad))) + : "null") + << "," + << "\"calibrationStatus\": \"" << (calibrated ? "CALIBRATED" : "UNCALIBRATED") << "\"," + << "\"lockState\": \"" << fsoc::to_string(r.tracked_state.lock_state) << "\"," + << "\"trackerConfidence\": " << r.tracked_state.confidence << "," + << "\"isPrediction\": " << (r.tracked_state.is_prediction ? "true" : "false") << "," + << "\"controlEnabled\": " << (control_enabled ? "true" : "false") << "," + << "\"commandPanRateDegS\": " << fsoc::rad_to_deg(r.command.pan_rate_rad_s) << "," + << "\"commandTiltRateDegS\": " << fsoc::rad_to_deg(r.command.tilt_rate_rad_s) << "," + << "\"virtualPanDeg\": " << fsoc::rad_to_deg(r.actuator_state.pan_rad) << "," + << "\"virtualTiltDeg\": " << fsoc::rad_to_deg(r.actuator_state.tilt_rad) << "," + << "\"virtualPanSaturated\": " << (r.actuator_state.pan_saturated ? "true" : "false") << "," + << "\"virtualTiltSaturated\": " << (r.actuator_state.tilt_saturated ? "true" : "false") + << "}"; + return j.str(); +} + +void print_manual_assist_cue(const fsoc::LiveFrameResult& r, bool calibrated) { + if (!r.tracking_error.has_value()) { + std::cout << " [manual-assist] no target -- hold steady / re-acquire\n"; + return; + } + // Pixel sign convention (frozen, fsoc/tracking_error.hpp): x_px > 0 => beacon RIGHT + // of centre; y_px > 0 => beacon BELOW centre (image +y is down). Same + // "which way to move to reduce this" framing as the calibrated cue below, just in + // pixels instead of degrees -- --uncalibrated has no real FOV to convert with. + if (!calibrated) { + const double x_px = r.tracking_error->pixel.x_px; + const double y_px = r.tracking_error->pixel.y_px; + const char* x_dir = x_px >= 0.0 ? "beacon RIGHT of centre by" : "beacon LEFT of centre by"; + const char* y_dir = y_px >= 0.0 ? "BELOW centre by" : "ABOVE centre by"; + std::cout << std::fixed << std::setprecision(1) << " PIXEL OFFSET (uncalibrated, no degrees) " + << x_dir << " " << std::abs(x_px) << " px " << y_dir << " " << std::abs(y_px) + << " px\n"; + return; + } + const double pan_deg = fsoc::rad_to_deg(r.tracking_error->angular.pan_rad); + const double tilt_deg = fsoc::rad_to_deg(r.tracking_error->angular.tilt_rad); + // Sign convention (frozen, fsoc/tracking_error.hpp): pan_rad > 0 => beacon + // RIGHT => centering requires panning the camera toward +pan (right). + // tilt_rad > 0 => beacon ABOVE => centering requires tilting toward + // +tilt (up). The cue tells a human which way to turn the phone to + // reduce this error, not which way the beacon moved. + const char* pan_dir = pan_deg >= 0.0 ? "PAN RIGHT ->" : "<- PAN LEFT "; + const char* tilt_dir = tilt_deg >= 0.0 ? "TILT UP" : "TILT DOWN"; + std::cout << std::fixed << std::setprecision(1) << " REQUIRED CORRECTION " << pan_dir << " " + << std::abs(pan_deg) << " deg " << tilt_dir << " " << std::abs(tilt_deg) << " deg\n"; +} + +} // namespace + +int main(int argc, char** argv) { + const auto parsed = parse_args(argc, argv); + if (!parsed.has_value()) { + return 2; + } + const Args& args = *parsed; + + fsoc::LiveCameraCalibrationConfig calibration{}; + if (args.uncalibrated) { + // Default-constructed LiveCameraCalibrationConfig carries the header's + // documented PLACEHOLDER hfov/vfov -- never presented to the user as real + // degrees (to_json() nulls every angular field when !calibrated). Control is + // force-disabled below: a placeholder FOV must never drive a command. + calibration = fsoc::LiveCameraCalibrationConfig{}; + std::cout << "fsoc_live: --uncalibrated -- pixel-only mode, no real field of view. " + "Angular fields will be null and control is disabled.\n"; + } else { + try { + calibration = fsoc::load_live_camera_calibration(args.calibration_path); + } catch (const std::exception& e) { + std::cerr << "fsoc_live: failed to load calibration: " << e.what() << "\n" + << " Run fsoc_camera_calibrate first (see docs/PHONE_CAMERA_METRICS.md),\n" + << " or pass --uncalibrated for a pixel-only preview.\n"; + return 1; + } + } + const bool control_enabled = args.control_enabled && !args.uncalibrated; + + fsoc::LiveTrackingSessionConfig session_config{}; + session_config.calibration = calibration; + session_config.perception_mode = args.mode; + session_config.tracker_enabled = args.tracker; + session_config.control_enabled = control_enabled; + if (args.mode != fsoc::PerceptionMode::Classical) { + fsoc::AiBeaconDetectorConfig ai_config{}; + ai_config.model_path = args.ai_model_path; + session_config.ai_detector = ai_config; + } + + std::unique_ptr session; + try { + session = std::make_unique(session_config); + } catch (const std::exception& e) { + std::cerr << "fsoc_live: failed to construct tracking session: " << e.what() << "\n"; + if (args.mode != fsoc::PerceptionMode::Classical) { + std::cerr << " Check --ai-model points at a valid ONNX file (e.g. models/tiny_beacon_net.onnx).\n"; + } + return 1; + } + + fsoc::OpenCVCameraFrameSourceConfig source_config{}; + source_config.camera_index = args.camera_index; + source_config.url = args.camera_url; + fsoc::OpenCVCameraFrameSource source(source_config); + + if (!source.open()) { + std::cerr << "fsoc_live: FAILED to open camera source.\n" + << " Run fsoc_camera_probe first to find an AVAILABLE index, and confirm OS\n" + << " camera permission is granted.\n"; + return 1; + } + const fsoc::FrameSourceInfo source_info = source.info(); + + // A reconnect (re-running fsoc_live) must start a new session identity — never + // reuse the previous run's id, so a client can tell a fresh session from a + // resumed one even if it never observed the disconnect itself. + const std::string session_id = std::to_string( + std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()) + .count()); + + fsoc::LiveFramePublisherConfig publisher_config{}; + publisher_config.output_dir = args.live_out; + std::optional publisher; + try { + publisher.emplace(publisher_config); + } catch (const std::exception& e) { + std::cerr << "fsoc_live: failed to prepare --live-out directory '" << args.live_out + << "': " << e.what() << "\n"; + source.close(); + return 1; + } + + std::cout << "FSOC LIVE -- CAMERA SOURCE = REAL_PHONE_CAMERA ACTUATOR = VIRTUAL\n"; + std::cout << "Source: " << source_info.backend_name << " " << source_info.width_px << "x" + << source_info.height_px << " mode=" << fsoc::to_string(args.mode) + << " tracker=" << (args.tracker ? "on" : "off") + << " control=" << (control_enabled ? "on" : "off") + << " calibration=" << (args.uncalibrated ? "NONE (pixel-only)" : args.calibration_path) << "\n"; + std::cout << "Session: " << session_id << "\n"; + std::cout << "Telemetry: " << args.live_out + << "/manifest.json (names the current frame_.jpg + telemetry_.json pair, " + "polled by Mission Control)\n"; + std::cout << "Recording: write " << args.live_out + << "/command.txt (commandId=..., action=start|stop|mark_event[, label=...]) to control G2 " + "recording; output under " + << args.record_out << "//\n"; + std::cout << "Press Ctrl+C to stop.\n\n"; + + std::string cli_args; + for (int i = 0; i < argc; ++i) { + if (i > 0) cli_args += ' '; + cli_args += argv[i]; + } +#ifdef FSOC_GIT_COMMIT + const std::string software_commit = FSOC_GIT_COMMIT; +#else + const std::string software_commit = "unknown"; +#endif + + std::optional recorder; + std::string last_processed_command_id; + + const auto start = std::chrono::steady_clock::now(); + auto last_frame_time = start; + std::size_t consecutive_failures = 0; + constexpr std::size_t kMaxConsecutiveFailures = 60; + + fsoc::Frame raw_frame{}; + while (true) { + const double elapsed_s = + std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + if (args.seconds > 0.0 && elapsed_s >= args.seconds) { + std::cout << "Reached --seconds limit. Stopping.\n"; + break; + } + + if (auto cmd = read_pending_command(args.live_out + "/command.txt"); + cmd.has_value() && cmd->command_id != last_processed_command_id) { + last_processed_command_id = cmd->command_id; + if (cmd->action == "start") { + if (recorder.has_value()) { + std::cout << "fsoc_live: recording already active (" << recorder->recording_id() + << "), ignoring duplicate start\n"; + } else { + fsoc::RealSessionRecorderConfig rec_config{}; + rec_config.output_root = args.record_out; + rec_config.session_id = session_id; + rec_config.calibration_status = args.uncalibrated ? "UNCALIBRATED" : "CALIBRATED"; + rec_config.calibration_id = args.uncalibrated ? "NONE" : args.calibration_path; + rec_config.perception_mode = fsoc::to_string(args.mode); + rec_config.ai_model_path = args.ai_model_path; + rec_config.software_commit = software_commit; + rec_config.cli_args = cli_args; + rec_config.source_backend = source_info.backend_name; + rec_config.source_description = source_info.description; + rec_config.raw_width_px = source_info.width_px; + rec_config.raw_height_px = source_info.height_px; + rec_config.preprocessed_width_px = session_config.preprocess.target_width_px; + rec_config.preprocessed_height_px = session_config.preprocess.target_height_px; + try { + recorder.emplace(rec_config); + std::cout << "fsoc_live: recording STARTED, id=" << recorder->recording_id() << " -> " + << args.record_out << "/" << recorder->recording_id() << "/\n"; + } catch (const std::exception& e) { + std::cerr << "fsoc_live: failed to start recording: " << e.what() << "\n"; + recorder.reset(); + } + } + } else if (cmd->action == "stop") { + if (recorder.has_value()) { + recorder->stop(); + std::cout << "fsoc_live: recording STOPPED, id=" << recorder->recording_id() + << " frames=" << recorder->recorded_frame_count() + << " errors=" << recorder->error_count() << " events=" << recorder->event_count() + << "\n"; + recorder.reset(); + } else { + std::cout << "fsoc_live: stop requested but no recording is active, ignored\n"; + } + } else if (cmd->action == "mark_event") { + if (recorder.has_value()) { + recorder->mark_event(cmd->label); + std::cout << "fsoc_live: event marked: " << cmd->label << "\n"; + } else { + std::cout << "fsoc_live: mark_event requested but no recording is active, ignored\n"; + } + } else { + std::cerr << "fsoc_live: unknown command action '" << cmd->action << "', ignored\n"; + } + } + + if (!source.read(raw_frame)) { + ++consecutive_failures; + if (consecutive_failures >= kMaxConsecutiveFailures) { + std::cerr << "fsoc_live: too many consecutive frame read failures -- camera appears " + "disconnected. Stopping cleanly (no crash, no fabricated frames).\n"; + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + continue; + } + consecutive_failures = 0; + + const auto now = std::chrono::steady_clock::now(); + const double dt_s = std::chrono::duration(now - last_frame_time).count(); + last_frame_time = now; + if (dt_s <= 0.0) { + continue; // clock hasn't advanced yet on the very first iteration path; skip, don't crash + } + + fsoc::LiveFrameResult result; + try { + result = session->process_frame(raw_frame, source_info, dt_s); + } catch (const std::exception& e) { + std::cerr << "fsoc_live: frame " << raw_frame.frame_index + << " failed to process (" << e.what() << ") -- skipping this frame.\n"; + continue; + } + + RecordingStatus recording_status{}; + if (recorder.has_value()) { + recording_status.active = true; + recording_status.recording_id = recorder->recording_id(); + recording_status.recorded_frame_count = recorder->recorded_frame_count(); + recording_status.error_count = recorder->error_count(); + } + const std::string telemetry_json = + to_json(result, args.mode, control_enabled, session_id, !args.uncalibrated, recording_status); + + try { + publisher->publish(result.frame_index, telemetry_json, raw_frame.image); + } catch (const std::exception& e) { + std::cerr << "fsoc_live: frame " << result.frame_index + << " failed to publish (" << e.what() << ") -- skipping this frame's output.\n"; + continue; + } + + // Recording is a separate sink from the live preview above: it never prunes + // and must not be skipped just because, e.g., the live publish's manifest + // write raced something transient (that already `continue`d above, so this + // line only runs once the frame is confirmed published). + if (recorder.has_value()) { + recorder->record_frame(result.frame_index, result.timestamp_s, telemetry_json, raw_frame.image); + } + + std::cout << std::fixed << std::setprecision(2) << "frame " << result.frame_index << " t=" + << result.timestamp_s << "s lock=" << fsoc::to_string(result.tracked_state.lock_state) + << " detected=" << (result.target_detected ? "yes" : "no"); + if (result.tracking_error.has_value()) { + if (args.uncalibrated) { + std::cout << " error=" << std::hypot(result.tracking_error->pixel.x_px, + result.tracking_error->pixel.y_px) + << "px"; + } else { + std::cout << " error=" + << std::hypot(fsoc::rad_to_deg(result.tracking_error->angular.pan_rad), + fsoc::rad_to_deg(result.tracking_error->angular.tilt_rad)) + << "deg"; + } + } + std::cout << "\n"; + if (args.manual_assist) { + print_manual_assist_cue(result, !args.uncalibrated); + } + } + + if (recorder.has_value()) { + recorder->stop(); + std::cout << "fsoc_live: finalized recording " << recorder->recording_id() + << " on shutdown (frames=" << recorder->recorded_frame_count() + << " errors=" << recorder->error_count() << ")\n"; + } + source.close(); + return 0; +} diff --git a/docs/09_FUTURE_ARCHITECTURE.md b/docs/09_FUTURE_ARCHITECTURE.md index 5053aa0..e718e2b 100644 --- a/docs/09_FUTURE_ARCHITECTURE.md +++ b/docs/09_FUTURE_ARCHITECTURE.md @@ -12,3 +12,15 @@ Upgrade in controlled layers: 8. **Monte Carlo validation:** seeded scenario sweeps and comparative plots. Keep PID as the interpretable reference baseline throughout judging. + +## Real-camera sensing (implemented) and physical actuation (still future) + +The Mobile Phone Camera-in-the-Loop milestone made the **sensing** side of this list +partially real ahead of schedule: `FrameSource` (`fsoc/frame_source.hpp`) is exactly the +swappable-interface boundary this section anticipated for camera input, and +`LiveTrackingSession` proves the existing Classical/AI/Hybrid perception, P0-v2 state +estimator, and PID controller all already worked unchanged on real frames. **Actuator +realism (item 7) is still entirely future** — `VirtualPanTiltActuator` is honest bookkeeping, +not hardware. See `docs/PHONE_CAMERA_METRICS.md` for the full architecture, the hardware- +ready `PanTiltActuator` interface sketch, and the rate-vs-position command mismatch a future +serial/servo adapter will need to solve. diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md new file mode 100644 index 0000000..55c0ef4 --- /dev/null +++ b/docs/DEPLOYMENT.md @@ -0,0 +1,118 @@ +# Deployment + +How FSOC's public web presence is built, deployed, and — most importantly — where its +honest boundary sits relative to the local phone-camera prototype. See `README.md` +"Working modes" for the short version. + +## Architecture + +``` +GitHub ThatKJ/FSOC + │ (push / PR) + ▼ +Vercel builds frontend/ only (Root Directory: frontend) + │ + ▼ +Public site: project pages + Mission Control (REPLAY mode) + checked-in + deterministic telemetry fixtures produced by the real C++ engine +``` + +The public deployment never builds or runs any C++ code. `frontend/` is a self-contained +Next.js app; its REPLAY mode and evidence pages read only fixtures already committed to +the repo (`frontend/lib/telemetry/fixtures/*.json`), generated ahead of time by the real +`fsoc_demo` binary — see `frontend/scripts/generate-fixtures.mjs`. + +## Vercel project + +| Setting | Value | +|---|---| +| Project | `fsoc` (Vercel team: Kirtan's projects) | +| GitHub repo | `ThatKJ/FSOC` | +| Root Directory | `frontend` | +| Framework | Next.js (auto-detected) | +| Production branch | `main` | +| Environment variables | none required — no secrets, no API keys | + +Root Directory is a project-level setting in the Vercel dashboard +(**Project → Settings → General → Root Directory**) — it is not something a `vercel.json` +or the Vercel API-via-MCP can safely change on an already-linked project without risking a +misconfigured push, so it's a one-time manual step rather than something automated here. + +### Preview vs. production + +- Every push to a non-`main` branch (or a PR) gets its own **Preview Deployment** — + a real, shareable URL built from that exact commit. Use this to validate a change + before it ever reaches `main`. +- Pushing/merging to `main` builds **production** at the project's production domain. +- Nothing in this repository auto-merges a feature branch into `main`, and nothing here + should trigger a production deploy without a human deciding `main` is ready. + +## Why the phone-camera prototype cannot run on Vercel + +`fsoc_live` is a long-running native C++ process that opens a real camera device and +writes `generated/live/telemetry.json` after every frame. Vercel's serverless functions +are short-lived, have no access to your Mac's camera hardware or local filesystem, and +cannot host a persistent native process. There is no configuration that changes this — +it's a hardware/process locality problem, not a missing feature. + +`frontend/app/api/live-camera/route.ts` reflects this honestly: it reads +`generated/live/telemetry.json` if present and returns `503 {"error":"no live session"}` +if not — it **never fabricates a reading**, in any environment. On the public deployment +that file will never exist, so `/mission/live` always shows the "Real-Camera Mode runs +locally" state (with the exact command to run it yourself) rather than an error page or +fake data. + +### Running the real-camera mode locally + +```bash +./run_fsoc.sh phone # or: ./run_fsoc.sh golden for the full narrated walkthrough +cd frontend && npm run dev +# open http://localhost:4317/mission/live +``` + +See `docs/PHONE_CAMERA_GOLDEN_DEMO.md` and `docs/PHONE_CAMERA_TEST_PLAN.md` for the full +walkthrough and camera setup. + +## Deployment smoke + +`frontend/tests/e2e/smoke.spec.ts` doubles as a deploy-safe smoke suite: point +it at any deployed URL and it exercises the public routes, navigation, +playback, and API error handling. The one test that needs the local C++ +engine (`engine mode ... reaches the C++ engine`) skips itself automatically +when that engine isn't reachable, so the full suite is safe to run against a +deployment that has no C++ build at all. + +```bash +cd frontend +FSOC_BASE_URL=https:// npx playwright test +``` + +Setting `FSOC_BASE_URL` also disables the config's local `webServer` — no +local build or dev server is started when testing a remote deployment. + +## Troubleshooting + +- **Deployed site shows `framework: null` / doesn't look like the app** — Root + Directory isn't set to `frontend` in the Vercel project settings; fix it there and + redeploy. +- **`/mission/live` shows "Real-Camera Mode" on your own machine** — that's correct + when `fsoc_live` isn't currently running; start it with `./run_fsoc.sh phone`. +- **Build fails on `npm run build`** — reproduce locally first: `cd frontend && npm ci && + npm run typecheck && npm run lint && npm run build`. Vercel runs the same commands. + +## Claude Code + Vercel MCP + +This repository's Vercel project can be inspected and deployed from Claude Code via +Vercel's official MCP server. + +```bash +claude mcp add --transport http vercel https://mcp.vercel.com +``` + +Then, inside Claude Code, run `/mcp` and complete the OAuth authorization in the browser. +Once connected, Claude Code can list projects/deployments and inspect build logs and +runtime errors for this project — never expose or paste OAuth tokens into a session. + +Some project-level settings (notably **Root Directory** on an already-linked project) are +intentionally not exposed as a write operation over MCP and must be changed in the Vercel +dashboard directly. diff --git a/docs/LIVE_DATA_AUDIT.md b/docs/LIVE_DATA_AUDIT.md new file mode 100644 index 0000000..f9a1983 --- /dev/null +++ b/docs/LIVE_DATA_AUDIT.md @@ -0,0 +1,213 @@ +# Live Data Audit (G0) + +Date: 2026-09-12. Scope: trace every live-facing screen/API/provider on `main` to its +actual data source, verify the `feat/phone-camera-in-loop` branch's real content against +the claims in the pasted live-camera/real-data-AI prompt pack, and verify TinyBeaconNet's +actual I/O contract. Read-only — no code changed in this pass. + +## 1. `main` today: every page is a finite-scenario replay, not a live feed + +Every frontend route that looks "live" is actually driven by one API: + +`GET /api/simulation/:scenario?source=engine|replay|auto&mode=classical|ai|hybrid&tracker=0|1` +(`frontend/app/api/simulation/[scenario]/route.ts`) + +- `source=engine` (or `auto` when the engine is available) execs the **existing** + `build/debug/fsoc_demo --csv --quiet [--duration N] [--mode] [--tracker]` + binary to completion, reads the whole CSV it produced, and returns **all frames as one + JSON array** (`frontend/lib/simulation/engine.ts`). This is a bounded, deterministic, + finite run of a named scenario (`static|sinusoidal|loss|open|closed`) — there is no + camera, no continuous frame stream, and no notion of "now." +- `source=replay` (or the `auto` fallback when the engine binary is missing) reads a + **checked-in fixture JSON** under `frontend/lib/telemetry/fixtures/*.json` + (`frontend/lib/simulation/fixtures.ts`) — a frozen array of pre-generated frames. +- The browser side (`frontend/lib/simulation/SimulationProvider.tsx` + + `useScenarioFrames.ts`) fetches this array once per scenario selection and then plays it + back on a local timer/scrubber. Nothing it shows was produced "this second"; it is either + a fresh finite engine run or a frozen fixture, indistinguishable to the viewer. + +This one provider feeds **every** route that consumes scenario frames: Dashboard/root +(`frontend/app/page.tsx`), Mission (`app/mission/page.tsx`), Tracking (`app/tracking/page.tsx`), +Telemetry (`app/telemetry/page.tsx`), Scenarios (`app/scenarios/page.tsx`), Benchmarks +(`app/benchmarks/page.tsx`), Validation (`app/validation/page.tsx`), World +(`app/world/page.tsx`), Architecture (`app/architecture/page.tsx`, static content only). + +| Screen | Source today | Cosmetic or measurement | Live-session replacement | +|---|---|---|---| +| Dashboard / root | finite engine run or fixture via `SimulationProvider` | measurement (pixel/angular error, lock state) | subscribe to the one active `LiveSession` (camera-or-none); no scenario picker in this mode | +| Mission | same | measurement | primary live entry point (G1); real frame + matched detection overlay | +| Tracking | same | measurement | live frame + centroid/tracking-error overlay from the active session | +| Telemetry | same | measurement | active-session series only; reset on new session, not on new browser tab | +| Scenarios | same, `demoScenario` presets | measurement + preset config (cosmetic) | becomes "physical test instructions" (pack §C) — the deterministic scenario picker stays as a **regression/demo tool**, clearly separate from the live route | +| Benchmarks | same, `BenchmarkMetrics` (RMS/P95/max) computed over a finite run | measurement | stays as-is for the **synthetic regression suite**; a live/real-session evaluation view is new and separate (G3/G5), not a replacement of this page | +| Validation | Step-10 acceptance report (`generated/step10/VALIDATION_REPORT.md`) rendered/read | measurement, but explicitly a frozen historical acceptance record | stays as the historical `v1_baseline` record; real-session evaluation gets its own page, not a rewrite of this one | +| World | same provider, 3D-ish scene of world truth (position/FOV cone) | **this is the one page that cannot be honestly kept in the live path**: a single uncalibrated monocular camera has no world-frame pose/range. Per pack §C, remove from live navigation | out of live nav; may stay as a synthetic-scenario visualization only | + +**Nothing here calls a "camera" today.** There is no existing simulation-vs-live toggle to +disable — the gap is that the live path (camera → detector → frontend) does not exist yet on +`main` at all. So "disable the fallback at the service boundary" (pack §C) is really: *build +the live boundary and never let it call `/api/simulation`*, not *patch an existing live route +that currently falls back to simulation*. + +Config numbers that are legitimately cosmetic/engineering limits, not fabricated +measurements (do not flag or remove): `SCENARIOS`/`SIM_RATE_HZ` in +`frontend/lib/baseline/constants.ts` (scenario duration/expected-frame-count config), FOV/ +actuator-rate/tilt-stop constants in the C++ camera config. These describe the **synthetic +regression suite**, which the pack explicitly allows to remain as isolated developer/demo +material. + +## 2. What `feat/phone-camera-in-loop` actually contains (vs. the pasted pack's claims) + +Verified via `git diff main...origin/feat/phone-camera-in-loop --stat` and `git show` (branch +not checked out, no working-tree changes made). The pack's claims are **substantially +accurate** and, if anything, understate how much is already built: + +- `include/fsoc/frame_source.hpp` + `src/frame_source.cpp`: a clean `FrameSource` + interface (`open/read/close/info`) with `FrameSourceKind{Synthetic, OpenCVCamera}`. Pure + I/O boundary — "knows nothing about detection, perception mode, tracking, or control, and + never touches world truth" (header comment, verified against implementation). This is + exactly the seam CLAUDE.md's module-boundary rule requires and it already exists. +- `include/fsoc/opencv_camera_frame_source.hpp/.cpp`: `cv::VideoCapture`-backed + `FrameSource`, by device index or URL. Confirmed real `cv::VideoCapture` usage, not a + synthetic stand-in. +- `include/fsoc/live_camera_calibration.hpp/.cpp`: a config loader for calibration + (intrinsics-adjacent) parameters, with its own tests (`tests/live_camera_calibration_tests.cpp`, + 162 lines). +- `include/fsoc/live_preprocessing.hpp/.cpp`: explicit resize/grayscale step from whatever + the camera reports to the detector's `CV_8UC1` contract — **not** done silently inside the + frame source (matches CLAUDE.md's "no hidden domain math" rule). +- `include/fsoc/live_tracking_session.hpp/.cpp`: `LiveTrackingSession` wires + `FrameSource` output through the **existing, frozen** `BeaconDetector` / + `AiBeaconDetector` / `resolve_perception()` (Safe Hybrid, ADR-018) / `TargetTracker` + (ADR-019) / PID controller pipeline — it does not reimplement detection, perception + fusion, or control. `LiveFrameResult` carries `frame_index`, `dt_s`, `source` info, + perception fields, `tracked_state` (lock state, confidence, `is_prediction`), and a + `command` — i.e. observed-vs-predicted is already a first-class distinct field + (`is_prediction`), and the raw classical/AI candidate flags are preserved alongside the + resolved detection (`classicalDetected`, `aiCandidateDetected` vs. `targetDetected`) — + this already satisfies the pack's "don't let one boolean count predictions as detector + success" requirement. +- `include/fsoc/virtual_actuator.hpp/.cpp`: `VirtualPanTiltActuator` — confirmed **honest + bookkeeping only**. Header comment states explicitly it "does not move anything," must be + labeled `ACTUATOR_TYPE = VIRTUAL`, and deliberately does *not* reuse `PanTiltCamera` to + keep the simulation-vs-virtual-actuator boundary sharp. Matches the pack's claim exactly: + the actuator is real bookkeeping math but zero physical motion. +- `apps/fsoc_live.cpp` (344 lines): the long-running live CLI. Confirmed: + - `--source camera --camera-index N` or `--source camera-url --camera-url URL`. + - `--mode classical|ai|hybrid`, `--tracker`, `--manual-assist` (prints human-readable + PAN/TILT correction cues — this **is** the "manual alignment assist" deliverable tier, + already implemented), `--no-control` (observe-only). + - On read failure: retries up to 60 consecutive failures (~3s at 50ms backoff) before + exiting cleanly with an explicit "camera appears disconnected" message — **no fabricated + frames, no crash.** + - Writes `generated/live/telemetry.json` and `generated/live/frame.jpg`, **overwritten + every processed frame**, non-atomically (plain `ofstream`/`cv::imwrite`, no + write-to-temp-then-rename). The code's own comments in the frontend routes acknowledge + this directly ("fsoc_live overwrites the file every frame, non-atomically... report a + partial write as a clean, retryable miss, never as fabricated telemetry"). + - Telemetry JSON already includes `frameIndex`, `cameraSource: "REAL_PHONE_CAMERA"`, + `actuatorType: "VIRTUAL"`, raw + preprocessed dimensions, perception source/mode, + lock state, `isPrediction`, command rates, and virtual pan/tilt + saturation flags — + i.e. most of the pack's §E "versioned session/frame contract" fields already exist in + an ad-hoc (non-versioned, no schema field, no session id) form. +- `apps/fsoc_camera_probe.cpp`, `apps/fsoc_camera_view.cpp`, `apps/fsoc_camera_calibrate.cpp`: + device discovery/preview/calibration tools, confirmed present. +- `frontend/app/api/live-camera/route.ts` + `.../frame/route.ts`: **exactly** the 500ms + independent-poll pattern the pack describes. Both routes are explicit and honest about + their own limitation in code comments: "this route is a plain snapshot read... an honest + polling read, not a live stream." `route.ts` reports `ageS` and a `stale` flag (>3s) — + so basic staleness detection already exists, just not wired to a UI "disconnected" state + yet, and not frame-ID-matched against the image. +- `frontend/app/mission/live/page.tsx` (255 lines): the live Mission Control page. Polls + both routes independently on a timer; **does not currently verify the fetched image's + frame identity matches the fetched telemetry's `frameIndex`** — this is the concrete gap + the pack flags ("matching the image and measurements by frame identity is an + implementation task"), confirmed real by reading the polling code, not assumed. +- `docs/PHONE_CAMERA_TEST_PLAN.md`, `docs/PHONE_CAMERA_GOLDEN_DEMO.md`, + `docs/PHONE_CAMERA_METRICS.md`: already written on this branch, already candidly + document the polling/latency/actuator-honesty limitations. **Reuse these, do not + duplicate them** — the pack's request for these exact filenames is already satisfied by + this branch's own docs once merged. +- `tools/beacon_display.html`: already exists — the exact "small bright dot on a phone + screen" tool the pack's Setup A calls for. + +**Net finding:** the branch is not a rough prototype; it is close to production-ready for +G1, built with the correct module boundaries, honest labeling, and mostly-graceful failure +handling already in place. It has never been merged into `main` (`main`'s log has no merge +commit for it, and diverged separately through the README/design-system commits). The one +concrete architectural gap for G1 is **frame/telemetry identity + atomicity**: two +independently-overwritten files, polled by two independent unsynchronized HTTP requests, +with no frame ID carried on the image side and no atomic pair swap. Everything else +(camera abstraction, honest actuator, failure handling, manual-assist cues, perception +reuse) is already correct and should be preserved, not rewritten. + +**Recommendation for G1:** merge this branch into a new local integration branch off +`main` (not a rewrite from scratch), then fix identity/atomicity by (a) having +`fsoc_live` write frame-keyed immutable files (`frame_.jpg`, `telemetry_.json`) via +write-to-temp-then-`rename()` (atomic on POSIX) plus a small `manifest.json` naming the +latest complete pair's frame index, and (b) having the frontend fetch the manifest first, +then fetch exactly that frame's image+telemetry by ID, rejecting/retrying on any mismatch. +This satisfies the pack's §E requirements without introducing a WebSocket server, which the +pack explicitly allows ("another transport is acceptable if it proves the same identity, +freshness, and backpressure guarantees"). + +## 3. TinyBeaconNet actual contract (verified against `models/MODEL_CARD.md` and + `tools/ai/common.py`, not assumed from the pasted pack's prose) + +- Native/reference frame: `CV_8UC1`, **640×480**. +- Preprocess: `cv2.resize(..., (320, 240), INTER_AREA)` → float32/255 → NCHW + **`[1, 1, 240, 320]`** — confirms the pack's claim exactly (320×240 net input, 640×480 + reference image). +- Outputs: `presence_logit [N,1]`, `heatmap_logit [N,1,60,80]`, sigmoid applied **outside** + the graph on both, decode via integer argmax + 5×5 soft-argmax, mapped back to original + coordinates via `x_orig = (x_hm + 0.5)·8 − 0.5` — confirms 80×60 heatmap claim exactly. +- Training data: **synthetic only**, `fsoc_ai_datagen` seed 26169, 8400 total frames + (6000/1200/1200 train/val/test), verified integrity (unique hashes, no leakage). Model + card's own header states plainly: "trained + validated on synthetic data... NOT + integrated into the C++ closed loop [was true at Stage 2; Stage 3 has since integrated + it, still synthetic-trained]." +- **Confirmed gap for G3**: `tools/ai/dataset.py` currently reads only the synthetic + `fsoc_ai_datagen` JSONL/manifest format. A real-capture dataset reader, grouped + train/val/test split (by recording session, not frame), and a reviewed-label manifest + format do not exist yet and must be added additively (new dataset reader class/mode, not + a rewrite of the synthetic path — Stage 1–4 synthetic training/eval must keep working + unchanged for regression comparison). + +## 4. Module boundaries a live camera path must respect (already established, reuse as-is) + +- `FrameSource` (`include/fsoc/frame_source.hpp`) — the correct seam for camera vs. + synthetic. Do not add a second competing abstraction. +- `fsoc::BeaconDetection` / classical detector — unchanged, frozen contract. +- `fsoc::AiBeaconDetector`, `PerceptionMode`, `resolve_perception()` (Safe Hybrid, + ADR-018) — unchanged; a live path selects a mode, it does not alter fusion policy. +- `fsoc::TargetTracker` (alpha-beta, ADR-019), `LockState{Searching,Acquiring,Tracking, + Coasting,Lost}`, `is_safe_to_steer()` — unchanged; already distinguishes measured vs. + predicted vs. lost, which the pack requires. +- `fsoc::VirtualPanTiltActuator` — unchanged; stays the actuator until real hardware (G6) + exists. Its telemetry must keep saying `VIRTUAL`/manual-assist, never a claim of physical + motion, consistent with the user's confirmed "no hardware yet" state. +- `fsoc::LiveTrackingSession` — the orchestration seam; the only new work is the + frame/telemetry identity+atomicity fix described above, plus (later) recording hooks + (G2) added as an additional observer, not inline in this class. +- Telemetry: the existing 34/42-column CSV schema is for the **finite scenario** path + (`fsoc_demo`/`SimulationRunner`). The live path's JSON schema is separate (per-frame, + overwritten) and already close to what's needed — G1 work is to version it and add + `sessionId`/`schemaVersion`/calibration/model-hash fields per pack §E, not to unify it + with the CSV schema. + +## 5. Bottom line / next action + +G0 is complete. The single biggest concrete gap between `main` and a real continuous +camera→detector→frontend path is: **the phone-camera branch was never merged, and even once +merged, the frontend/backend frame–telemetry pairing is not identity-safe** (two files, +two independent polls, no shared frame ID enforcement, no atomic pair swap). Everything +else needed for a defensible G1 (camera abstraction, detector reuse, honest actuator, +failure handling, manual-assist) already exists in working, tested C++ form on +`feat/phone-camera-in-loop` and should be integrated, not rebuilt. + +Next action (G1): create a local integration branch off `main`, merge in the relevant +`feat/phone-camera-in-loop` commits/files, resolve conflicts against `main`'s +README/design-system/smoke-test changes, implement the frame-keyed atomic write/read fix, +build, and run the existing branch's own C++ tests (`frame_source_tests`, +`live_camera_calibration_tests`, `live_preprocessing_tests`, `live_tracking_session_tests`, +`virtual_actuator_tests`) plus a new test for the atomic pairing contract. diff --git a/docs/LIVE_REALDATA_TASK_STATE.md b/docs/LIVE_REALDATA_TASK_STATE.md new file mode 100644 index 0000000..db528b3 --- /dev/null +++ b/docs/LIVE_REALDATA_TASK_STATE.md @@ -0,0 +1,355 @@ +# Live Camera + Real-Data AI — Task State + +Tracks the G0–G6 gates from the live-camera/real-data-AI engineering task (2026-09-12). +This supersedes nothing already frozen: `v1_baseline` (Step 10) and the synthetic-data +Stage 1–4 AI perception work stay frozen and reusable. This task adds a **new, additive** +live-camera path and a **real-data** training/eval track alongside them. + +Status values: `BLOCKED` (external dependency) · `IN PROGRESS` · `IMPLEMENTED` (code done, +not yet verified) · `AUTOMATED PASS` (passes an automated test/build) · `PHYSICAL PASS` +(verified against real camera/hardware with evidence) · `FAIL`. + +Operator confirmed setup (2026-09-12): laptop webcam + phone-screen-dot beacon (Setup A). +No motorized pan/tilt hardware yet — G6 stays BLOCKED/out-of-scope until hardware exists. + +## Gate summary + +| Gate | Description | Status | +|---|---|---| +| G0 | Audit and source isolation | AUTOMATED PASS — see `docs/LIVE_DATA_AUDIT.md` | +| G1 | Physical live baseline (real camera → classical tracking → matched frame/telemetry) | AUTOMATED PASS — see below; PHYSICAL PASS pending operator run | +| G2 | Measurement and data collection (recording, annotation, splits) | AUTOMATED PASS (software) — real reviewed recordings pending operator | +| G3 | Real-data AI (train/eval TinyBeaconNet on reviewed real captures) | BLOCKED — no reviewed real recordings exist yet | +| G4 | Native deployment + application integration | BLOCKED (on G3) | +| G5 | Demonstration package (evidence export, manual, repeatable demo) | BLOCKED (on G4) | +| G6 | Physical automatic pointing (motorized hardware) | BLOCKED — no hardware available | + +## G0 — Audit and source isolation + +- [x] Every live-facing screen/API/provider traced to its actual data source — + `docs/LIVE_DATA_AUDIT.md` §1. Every "live-looking" page on `main` is fed by + `/api/simulation/[scenario]` (finite `fsoc_demo` run or checked-in fixture JSON). + No camera path exists on `main` at all yet. +- [x] Phone-camera branch content verified against the pasted prompt pack's claims — + `docs/LIVE_DATA_AUDIT.md` §2. Claims were accurate; branch is close to + production-ready (FrameSource abstraction, honest VirtualPanTiltActuator, + LiveTrackingSession reusing the frozen detector/hybrid/tracker pipeline, + graceful disconnect handling). Never merged into `main`. +- [x] TinyBeaconNet actual I/O contract verified from model metadata — + `docs/LIVE_DATA_AUDIT.md` §3. 640×480 reference → 320×240 net input + `[1,1,240,320]`, heatmap `[1,1,60,80]`, synthetic-only training confirmed. +- [x] Simulation/replay fallback identified — there is no existing live route to + patch; G1 must build the live boundary so it never touches + `/api/simulation`, not retrofit a fallback-disable switch. + +## G1 — Physical live baseline + +**Branch:** `feat/live-camera-integration` (local only — not pushed, `main` untouched). + +- [x] Merged `origin/feat/phone-camera-in-loop` into a new local integration branch off + `main` (commit `ba253a7`). Resolved README.md conflicts by combining main's + rewritten landing-page prose with the branch's Mobile Phone Camera-in-the-Loop / + Hardware boundary / Working modes sections; corrected now-stale claims (LICENSE + file now exists; phone-camera work is merged, not a separate branch). +- [x] Fixed the frame↔telemetry identity/atomicity gap identified in the audit: added + `fsoc::LiveFramePublisher` (`include/fsoc/live_frame_publisher.hpp`, + `src/live_frame_publisher.cpp`) — writes `frame_.jpg` / `telemetry_.json` via + write-to-temp-then-atomic-rename, then flips `manifest.json` (same pattern) LAST, + so any reader that observes a frame index in the manifest is guaranteed both files + for that index already exist, complete. Bounded retention (default 3 pairs) so + `generated/live/` cannot grow unbounded during a long session. + - 3 new unit tests (`tests/live_frame_publisher_tests.cpp`, `fsoc_live_frame_publisher_tests` + in CTest): config validation, single-publish completeness + no leftover `.tmp` files, + and a 25-frame run proving the manifest-named pair always exists on disk and older + pairs are pruned. +- [x] Rewired `apps/fsoc_live.cpp` to use the publisher instead of overwriting + `telemetry.json`/`frame.jpg` in place; added a per-run `sessionId` + (wall-clock-ms token — a reconnect always gets a new one) and `schemaVersion`/ + `sessionId` fields in the telemetry JSON. +- [x] Rewired `frontend/app/api/live-camera/route.ts` to read `manifest.json` then the + exact `telemetryFile` it names, returning `frameIndex` to the client. + Rewired `frontend/app/api/live-camera/frame/route.ts` to require `?frame=` + and serve exactly that file (400 on missing/invalid index, 404 if already pruned — + never a mismatched fallback image). + Rewired `frontend/app/mission/live/page.tsx` to fetch the image by the exact + `frameIndex` the telemetry response named (object-URL fetch, not a cache-busted + ``), and to drop the displayed frame when `sessionId` changes + (reconnect must not inherit the previous session's last image). +- [x] Build: `cmake --preset debug -DFSOC_ENABLE_OPENCV=ON && cmake --build --preset debug` + — clean, no errors/warnings. `ctest --preset debug --output-on-failure` — **23/23 + PASS** (22 pre-existing + `fsoc_live_frame_publisher_tests`). +- [x] Frontend: `npm ci && npm run typecheck && npm run lint && npm run build` — all + clean. `npx playwright test` — **51/51 PASS** (after clearing a stale `next dev` + process from earlier in this session that was holding port 4317 against an old + Next.js version and corrupting `.next/`; not a regression from this work). +- [x] End-to-end wiring smoke-tested with a hand-built fixture (mimicking + `LiveFramePublisher`'s exact output) against a real running server: confirmed + `/api/live-camera` returns the correct `frameIndex`/`ageS`/`stale`, and + `/api/live-camera/frame?frame=` serves the exact matching bytes, 404s on an + unpublished/pruned index, 400s on a missing index — without opening a camera. +- [x] **Uncalibrated (pixel-only) mode added** so a real preview never has to wait on + calibration. `fsoc_live --uncalibrated` (mutually exclusive with `--calibration + PATH`): builds the sensing camera from the header's documented placeholder FOV, + but `to_json()` always nulls `panErrorDeg`/`tiltErrorDeg`/`totalErrorDeg` when + uncalibrated (regardless of what the placeholder-FOV math internally computes), + adds an explicit `calibrationStatus: "UNCALIBRATED"` telemetry field, and + force-disables control/actuation (a fabricated FOV must never drive a command). + `--manual-assist` and the terminal `error=` line switch to real pixel offsets + instead of fabricated degrees in this mode. `/mission/live` shows a + **CALIBRATION** row (`CALIBRATED` vs `UNCALIBRATED (pixel-only)`) and a + **SESSION ID** row (both new). +- [x] **Real per-frame timing surfaced, not just implied.** `/mission/live` now shows a + TIMING panel: PROCESSED FPS (`1/dtS`, the C++ side's own measured wall-clock + interval — explicitly labeled as real per-frame timing, not an exposure + timestamp) and DISPLAYED FPS (measured client-side from actual image updates). + Poll interval tightened 500ms → 200ms (still ~5-6x below a typical webcam's + native 15-30fps — this gap is shown by the two FPS numbers, not hidden). +- [x] **Frontend risk review, focused (not a re-check of already-verified work):** added + `frontend/tests/e2e/live-camera.spec.ts` (6 tests, route-mocked, no camera needed) + covering exactly the previously-untested risks — reconnect (new `sessionId`) drops + the prior session's displayed frame image, a pruned/missing frame (404) doesn't + crash or show a mismatched image, the honest no-session and stale states render + distinctly, and the recording panel reflects telemetry (not local optimistic + state). `/mission/live` was only covered by generic responsive-layout checks + before this. +- [ ] **PHYSICAL PASS — requires the operator.** Opening a real camera device needs an + interactive session to answer the OS permission prompt (macOS TCC) — this cannot + be done from an unattended shell. See "Camera setup checklist" and "Physical G1 + review script" below. + +Known limitation carried forward (not blocking, documented not hidden): fsoc_live only +publishes a new manifest/frame/telemetry entry when a frame is successfully read and +processed. A stalled-but-still-reading camera (never crossing the 60-consecutive-failure +disconnect threshold) is only visible to the frontend indirectly, via the existing +`ageS`/`stale` check on the last published frame — there is no independent +"camera status" heartbeat written on read failure. Acceptable for G1; worth revisiting +if real testing shows this staleness signal is too slow to be trusted. + +## G2 — Measurement and data collection + +**Software: AUTOMATED PASS.** No real reviewed recordings exist yet — that half of G2 +is BLOCKED on the operator (see "Recording assignment" below), and is tracked +separately from the software. + +- [x] **Recording controls.** `/mission/live` has Start/Stop Recording buttons plus + three preset event-mark buttons (Covered/Visible/Scene Change), wired through + `POST /api/live-camera/record` → `generated/live/command.txt` (key=value, + temp-then-rename) → polled once per camera-frame iteration by `fsoc_live`. A 200 + response only means the command file was written; the UI's RECORDING status + always reflects the next polled telemetry (`recordingActive`/`recordingId`/ + `recordedFrameCount`/`recordingErrorCount`), never local optimistic state — same + honesty rule already applied to the camera feed itself. Documented limitation: + `command.txt` is a single slot, not a queue — two commands faster than one camera + frame interval apart will have the earlier one superseded (unrealistic for a + human clicking a button; noted in `apps/fsoc_live.cpp`'s file header). +- [x] **`fsoc::RealSessionRecorder`** (`include/fsoc/real_session_recorder.hpp`, + `src/real_session_recorder.cpp`) — a *second*, separate sink from + `LiveFramePublisher`'s pruned preview buffer. Writes, under + `generated/real_sessions//`: + `manifest.json` (session/recording id, calibration status+id, perception + mode+model path, `softwareCommit` — captured at CMake configure time via `git + rev-parse HEAD`, not re-queried at runtime — full `cliArgs`, raw+preprocessed + dims, source backend/description, running frame/error/event counters, + rewritten atomically after every frame so a crash mid-recording still leaves a + valid partial manifest), `frames/frame_.jpg` (RAW, no overlay — every frame + accepted is kept, **never pruned**, unlike the live preview buffer), + `telemetry.jsonl` (append-only, one line per frame, same content as the live + telemetry), `events.jsonl` (human markers, tagged with the last recorded + frame). Writes are synchronous per frame (no thread, no in-memory queue) — a + disk error increments a counted `error_count()` and is swallowed, never stalls + the tracking loop. + - 4 new unit tests (`tests/real_session_recorder_tests.cpp`, + `fsoc_real_session_recorder_tests`): config validation, a 40-frame recording + surviving in full (nothing pruned) with a correct manifest, event markers tagged + to the right frame, and a simulated disk-write failure counted, not thrown. +- [x] **Annotation tool** at `/mission/annotate` (reachable from `/mission/live` → + "Review Recordings"). Lists local recordings (`GET /api/real-sessions`), loads + one's manifest+telemetry (`GET /api/real-sessions/:id`), serves its raw frames + (`GET /api/real-sessions/:id/frame/:index`), and reads/writes reviewed labels + (`GET`/`POST /api/real-sessions/:id/labels` → `labels.json`, keyed by frame + index so re-labeling a frame updates it rather than appending a duplicate). + Supports: frame scrubbing, presence labeling (present / partial occlusion / + full occlusion / absent / ambiguous), click-to-set beacon center (in **raw** + pixel space — `labelCoordinateSpace: "raw"` recorded explicitly), a detector + **suggestion** marker from the recording's own telemetry (visually distinct, + never auto-saved — only becomes a label if a human accepts/adjusts it and hits + Save), and a reviewed/total progress count. Server-side validation rejects a + self-contradictory label (e.g. `present` with no center, `absent` with a + center) before it can be saved. Raw frame files on disk are never touched — + all overlays are browser-side only. + - 4 new e2e tests (`frontend/tests/e2e/annotate.spec.ts`, route-mocked): empty + state, save-without-center rejected, click-then-save persists a reviewed label, + and navigating frames does not carry over the previous frame's unsaved draft. +- [x] **Real-data dataset loader** (`tools/ai/real_dataset.py`) — additive to + `dataset.py`'s synthetic `BeaconDataset`; the synthetic path is untouched. + `RealBeaconDataset` reads reviewed recordings and returns the *same* + `(input[1,240,320], heatmap[60,80], present, label_xy[2], difficulty)` tuple + shape, so real and synthetic samples can be combined with + `torch.utils.data.ConcatDataset` later without special-casing either. + - Validates: manifest/labels.json/frames exist; `labelCoordinateSpace == "raw"`; + a `present`/`partial_occlusion` label has a center *inside that recording's own* + raw frame bounds; an `absent`/`full_occlusion` label has no center; a labeled + frame's image file actually exists. Raises `RecordingValidationError` on any of + these rather than silently skipping a broken recording. + - Exclusion policy (explicit): `ambiguous` and any frame missing from + `labels.json` (never reviewed) are excluded from every returned sample — + never forced into a positive or negative target. + - Coordinate transform: each raw frame is resized to the frozen `common.ORIG_W × + ORIG_H` (640×480) the same way `LivePreprocessConfig` already resizes a real + frame before detection, and a label's raw-pixel center is scaled by the *same* + per-recording ratio (`_raw_to_orig()` — the one function to touch if crop/mirror + is ever added to the real-camera preprocessing pipeline). +- [x] **Group-based split** (`tools/ai/real_dataset_split.py`) — splits by + **recording** (capture group), never by frame; `assign_splits()` raises + `ValueError` rather than silently leaving a split empty when there are too few + groups (e.g. splitting 1-2 recordings three ways). Deterministic (seeded), + persists one JSON manifest (`save_split_manifest`/`load_split_manifest`) so the + same partition is reused across runs instead of re-derived. CLI: + `python tools/ai/real_dataset_split.py --real-sessions-root generated/real_sessions --out `. +- [x] **Plumbing tests, clearly-labeled fixtures only** + (`tools/ai/real_dataset_tests.py`, 13 checks, run manually — same convention as + `selfcheck.py`, not wired into CTest since it needs `.venv-ai`): missing + manifest/labels/frame-file all raise; wrong `labelCoordinateSpace` raises; + out-of-bounds / missing / contradictory centers raise; `ambiguous` and + never-reviewed frames are excluded; a known raw-space label decodes back out + (via the frozen heatmap encode/decode round trip) close to its expected + resized coordinate; splits have zero group leakage and are deterministic per + seed; too-few-groups raises; split manifest round-trips exactly. Explicitly + documented as fixtures, not real training data. + Run: `.venv-ai/bin/python3 tools/ai/real_dataset_tests.py` (from + `tools/ai/`) — **PASS: all 13 checks**. +- [ ] **Real reviewed recordings.** None exist yet — this is the actual G2/G3 + dependency on you. See "Recording assignment" below. + +## G3 — Real-data AI + +**BLOCKED.** Not started, and must not be claimed complete or attempted with +placeholder data: no reviewed real recordings exist yet, so there is nothing to +train or evaluate on. Unlocked once the first reviewed batch from "Recording +assignment" below exists — the dataset loader and split tooling are already built +and tested against fixtures, ready for that data the moment it exists. + +## Engineering assumptions log + +- Reusing existing frozen interfaces wherever possible: `fsoc::BeaconDetection` contract, + `PerceptionMode`/`AiBeaconDetector`/Safe-Hybrid policy, `fsoc::TargetTracker`, the 34/42 + column telemetry schema. A live camera path is a new `FrameSource` implementation, not a + new detector/controller/telemetry format. +- Actuator stays `NONE`/`MANUAL ASSIST` for the live path until real motor hardware is + provided (G6). No invented pan/tilt angles. +- Python stays confined to `tools/ai/` (offline training/eval), matching existing + precedent (`.venv-ai`, synthetic TinyBeaconNet training already uses this path). + Production capture/detection/control/telemetry stays C++20. +- `v1_baseline` tag and the synthetic-trained Stage 1–4 AI work are not touched or + replaced by this task. +- Recordings/labels/split manifests live under `generated/` (already fully + git-ignored) — device-specific, local by default. Nothing under `generated/` is + committed; `configs/` stays for reusable *templates* (e.g. an example calibration + file), never a guessed real measurement presented as this operator's actual device. + +## Camera setup checklist — laptop webcam + phone-screen-dot beacon + +Verified in this session (commands/flags actually exist and were exercised, short of +opening a real camera). Run from the repo root unless noted. + +1. **Build the camera tools** (if not already built): + `cmake --preset debug -DFSOC_ENABLE_OPENCV=ON && cmake --build --preset debug` + Confirms via `-- FSOC: OpenCV videoio present - phone-camera-in-the-loop targets + enabled` in the configure output. +2. **Probe for your webcam's index**: `./build/debug/fsoc_camera_probe` (probes + indices 0..4 by default; add `--max-index N` for more). Prints a table of + resolution/FPS/backend/status per index — note the first `AVAILABLE` one. +3. **Raw preview — no calibration needed for this step**: + `./build/debug/fsoc_camera_view --camera-index --seconds 15 --out-dir generated/camera_view_test --crosshair` + (a separate `--out-dir` from `generated/live` avoids any confusion with + `fsoc_live`'s own output). Writes periodic JPEG snapshots to that directory — + open one to confirm you're actually seeing your webcam, before calibration is + ever a blocker. +4. **The macOS camera-permission prompt** appears the first time step 2 or 3 opens + the device from your terminal app. If you don't see a prompt and the probe + reports every index `UNAVAILABLE`, permission was likely denied previously — + check System Settings → Privacy & Security → Camera and enable it for the + terminal app you're running this from, then re-run step 2. +5. **Calibration — pixel-only fast path (recommended first)**: skip calibration + entirely with `fsoc_live --uncalibrated` (see step 6) — no file, no measurement, + real pixel-offset tracking immediately, degrees/control explicitly unavailable. + For real angular numbers later: `./build/debug/fsoc_camera_calibrate --manual + --width --height --hfov-deg --vfov-deg --out configs/webcam.cfg` + using the resolution from step 2 and your webcam's actual spec'd or measured + field of view (never copy the simulator's 20°/15°) — or the `--from-object` form + (see `docs/PHONE_CAMERA_METRICS.md` "Camera calibration") if you measure a + known-width object at a known distance instead. `configs/` is currently empty; + treat anything you save there as your own local device file, not something to commit + as if it were a universal value. +6. **Start fsoc_live**: display `tools/beacon_display.html` on your phone (small + bright dot, dark background) where the webcam can see it, then: + `./build/debug/fsoc_live --source camera --camera-index --uncalibrated --manual-assist` + (swap `--uncalibrated` for `--calibration configs/webcam.cfg` once you have one). + Expect continuous `frame ... lock=... detected=...` lines and + `generated/live/manifest.json` + `frame_.jpg` + `telemetry_.json` appearing. +7. **Open Mission Control**: `cd frontend && npm run dev`, then + `http://localhost:4317/mission/live`. Expect LIVE (not STALE), the real feed with + the beacon, `CALIBRATION: UNCALIBRATED (pixel-only)` (or `CALIBRATED` if you used + step 5's file), a SESSION ID, and the pointing-error/TIMING panels updating live. + +## Physical G1 review script — run once camera setup above works + +Return, for each step: what you saw (screenshot if easy), and any terminal +output/error. This is what turns G1 from AUTOMATED PASS to PHYSICAL PASS. + +1. Show an unpredictable hand movement or a handwritten number to the camera — + confirms the feed is current, not a loop/replay (frame index and TIMING numbers + should keep advancing). +2. Show the phone-dot beacon; confirm the pointing-error panel goes non-zero and + `targetDetected`/lock state respond. +3. Move the phone left / right / up / down; confirm the pixel-error/pan-tilt sign + matches the direction (see `docs/PHONE_CAMERA_TEST_PLAN.md` M7 for the exact + convention). +4. Cover the beacon while leaving the camera running: confirm lock state changes + (or `targetDetected: false`) while the page stays LIVE (camera still connected, + just no target) — never a fabricated detection. +5. Uncover it: confirm reacquisition (lock state returns, `isPrediction: false` on + the first fresh detection). +6. Stop `fsoc_live` (Ctrl+C): confirm the page goes STALE within ~3s, then shows the + disconnected/no-session state — no replay, no lingering green "LIVE". +7. Restart `fsoc_live`: confirm the page shows a **new SESSION ID** and does not + keep showing the previous session's last frame under a fresh "LIVE" label. +8. Note the PROCESSED FPS / DISPLAYED FPS panel values you actually observed — this + is the real, measured number, not an assumption. + +Also work through `docs/PHONE_CAMERA_TEST_PLAN.md` M4–M15 if you want the fuller +matrix (M13/M15 are the same disconnect/no-fabrication checks as steps 6-7 above). + +## Recording assignment — the first plumbing clip (G2 → unlocks G3) + +One ~60-second recording, using the Start/Stop Recording controls on +`/mission/live` (camera + beacon set up per the checklist above): + +1. **0:00–0:15 — stationary beacon.** Don't move the phone or camera. +2. **0:15–0:30 — slow horizontal then vertical movement.** Pan the phone/beacon + left-right, then up-down, slowly enough to stay trackable. +3. **0:30–0:45 — cover and uncover.** Use the "Mark: Covered" / "Mark: Visible" + buttons at the moments you actually cover/uncover it. +4. **0:45–1:00 — beacon absent.** Point the camera away from the beacon; include + another bright object (a lamp, a phone flashlight) in frame if you have one + handy — use "Mark: Scene Change" when you do. + +Click **Start Recording** before step 1, work through 1-4, then **Stop Recording**. +The recording lands at `generated/real_sessions//` (the id is shown in +the RECORDING panel and in the terminal). Then: + +5. Open `http://localhost:4317/mission/annotate`, select that recording, and label + a handful of frames across all four segments (present / absent / partial or + full occlusion as appropriate) — you don't need to label every frame for this + first pass, just enough to prove the workflow: click the beacon center, choose a + presence value, hit Save Label, confirm "already reviewed" appears, move to the + next frame. + +Report back: the `recordingId`, roughly how many frames you labeled, and anything +that felt broken or confusing in either page. **This first clip only verifies the +recording/annotation workflow — it is not a sufficient training dataset and not an +independent evaluation set.** Once it works, I'll give you the next batch: several +more short sessions (varied distance/brightness/edges-of-frame per +`docs/LIVE_REALDATA_TASK_STATE.md`'s own future entry once written), with specific +sessions held out for validation and a separate final test group — collected before +any training run, per the split tooling above. diff --git a/docs/PHONE_CAMERA_GOLDEN_DEMO.md b/docs/PHONE_CAMERA_GOLDEN_DEMO.md new file mode 100644 index 0000000..a7a5bda --- /dev/null +++ b/docs/PHONE_CAMERA_GOLDEN_DEMO.md @@ -0,0 +1,132 @@ +# Phone Camera-in-the-Loop — Golden Demo + +The strongest possible demo using zero additional hardware: your Mac's camera (or a phone +used as a webcam/continuity camera), a monitor as the target, no servos, no Arduino. Every +step runs a real command against real hardware — nothing here is scripted or pre-recorded. +See `docs/PHONE_CAMERA_METRICS.md` for the full architecture and claim boundary, and +`docs/PHONE_CAMERA_TEST_PLAN.md` for the manual validation checklist this demo exercises. + +**Run this yourself, interactively.** Opening a camera triggers an OS permission prompt +(macOS: System Settings → Privacy & Security → Camera) that only a real interactive terminal +session can answer — this is why the commands below were not run automatically when this +milestone was built. + +## Prerequisites (once) + +```bash +cmake --preset debug && cmake --build --preset debug && ctest --preset debug --output-on-failure +``` +Expect `100% tests passed out of 22` (17 pre-existing + 5 for this milestone). + +## Setup + +**1. Open the beacon display on a second window/monitor.** +```bash +open tools/beacon_display.html # macOS; or just double-click it +``` +Press `F` for fullscreen. Leave motion on "Static" for now. + +**2. Find your camera index.** +```bash +./build/debug/fsoc_camera_probe +``` +Note the `AVAILABLE` index with the resolution you expect (built-in cameras are usually 0). + +**3. Prove the camera feed is real and stable before any tracking logic runs.** +```bash +./build/debug/fsoc_camera_view --camera-index 0 --seconds 15 --crosshair +``` +Point the camera at the beacon display. Confirm: frames arrive, `t=` timestamps increase, +`generated/live/` (or `--out-dir`) fills with JPEG snapshots you can open and look at. + +**4. Calibrate.** Either declare your camera's known spec-sheet FOV: +```bash +./build/debug/fsoc_camera_calibrate --manual --width 1920 --height 1080 \ + --hfov-deg 69 --vfov-deg 42 --out configs/phone_camera.cfg +``` +...or estimate it from a known object (see `docs/PHONE_CAMERA_METRICS.md` "Camera +calibration" for the method): +```bash +./build/debug/fsoc_camera_calibrate --from-object --object-width-m 0.30 --distance-m 1.0 \ + --object-pixel-width-px 400 --image-width-px 1920 --image-height-px 1080 \ + --out configs/phone_camera.cfg +``` + +## The demo + +**5. Real Classical perception on a real frame.** +```bash +./build/debug/fsoc_live --source camera --camera-index 0 \ + --calibration configs/phone_camera.cfg --mode classical +``` +Point at the beacon. Watch `detected=yes` and a real, changing `error=` in the terminal. +This is the exact same `BeaconDetector` the simulation uses (`docs/PHONE_CAMERA_METRICS.md` +"Architecture"). + +**6. Launch Mission Control and open the live-camera view.** +```bash +cd frontend && npm run build && npm run start & +``` +Open `http://localhost:4317/mission/live`. Confirm it shows `CAMERA SOURCE: +REAL_PHONE_CAMERA` and `ACTUATOR: VIRTUAL` — never a physical-hardware claim. + +**7. Real Hybrid perception + state estimator.** +```bash +./build/debug/fsoc_live --source camera --camera-index 0 \ + --calibration configs/phone_camera.cfg --mode hybrid --tracker \ + --ai-model models/tiny_beacon_net.onnx +``` +Mission Control's PERCEPTION panel now shows `MODE: HYBRID` with a real +`aiPresenceProbability`, and STATE ESTIMATOR shows a real `LOCK STATE`. + +**8. Move the phone/camera.** +Physically pan the camera left/right across the beacon. Watch `panErrorDeg` change sign and +magnitude in real time, and `commandPanRateDegS` track it (`docs/PHONE_CAMERA_METRICS.md`'s +frozen sign convention). + +**9. Try Manual Correction Assist.** +```bash +./build/debug/fsoc_live --source camera --camera-index 0 \ + --calibration configs/phone_camera.cfg --mode hybrid --tracker \ + --ai-model models/tiny_beacon_net.onnx --manual-assist +``` +Follow the printed `REQUIRED CORRECTION` cue by hand; confirm the error shrinks on the next +line. This is manual-in-the-loop, not physical closed-loop control — say so explicitly. + +**10. Hide the beacon briefly.** +Cover the monitor/beacon with your hand for under a second. Watch `lockState` go +`TRACKING -> COASTING` (`isPrediction=true` — the alpha-beta estimator bridging a real, +brief detection gap). + +**11. Reveal it again.** +`lockState` returns to `TRACKING`, `isPrediction=false` — a fresh real measurement. + +**12. Set the beacon in motion.** +On the beacon display, switch Motion to "Sinusoidal", then keep the camera roughly aimed at +it. Watch the pointing error and lock state respond to real, continuous target motion. + +**13. Introduce a second bright light.** +Turn on a lamp, or open a second beacon-display window, near the tracked one. Record what +actually happens — a false lock, a source switch, or correct rejection. Do not expect this +to be solved (`docs/MVP_ABLATION.md` already discloses this as an open limitation for the +synthetic case; this is the real-camera version of the same experiment). This is one of the +most valuable real-camera validations precisely because it might fail informatively. + +**14. Disconnect the camera mid-run.** +Physically unplug a USB camera (or close a continuity-camera connection). Confirm +`fsoc_live` prints a clean "too many consecutive frame read failures" message and exits — +no crash, no fabricated frames. + +**15. Close the loop on claims.** +State explicitly, out loud, to whoever is watching: the sensing side (camera, perception, +state estimation, controller output) is fully real; the actuator side is honestly virtual. +See `docs/PHONE_CAMERA_METRICS.md` "Claim boundary" for the exact acceptable/unacceptable +phrasing. + +## After the demo + +Fill in `docs/PHONE_CAMERA_TEST_PLAN.md`'s manual validation table (M1–M14) with what you +observed, and `docs/PHONE_CAMERA_METRICS.md`'s "Real-camera metrics" table with the numbers +you measured. Optionally capture screenshots into `presentation_assets/phone_camera/` +(filenames suggested in the original milestone brief) — only capture states that actually +occurred. diff --git a/docs/PHONE_CAMERA_METRICS.md b/docs/PHONE_CAMERA_METRICS.md new file mode 100644 index 0000000..89664b8 --- /dev/null +++ b/docs/PHONE_CAMERA_METRICS.md @@ -0,0 +1,247 @@ +# Phone Camera-in-the-Loop — Architecture, Metrics, and Boundaries + +This document covers the **Mobile Phone Camera-in-the-Loop** milestone: extending FSOC from +a synthetic camera + simulated actuator to a **real mobile-phone camera + real perception + +real state estimation + real controller + an honestly-virtual actuator**. It does NOT +replace `docs/MVP_METRICS.md` (the synthetic-simulation metrics) — the two are kept +strictly separate, per this milestone's own rule. + +## What is real vs. what is virtual + +``` +REAL TARGET / BEACON -> MOBILE PHONE CAMERA -> REAL VIDEO FRAME + -> Hybrid Perception -> State Estimator -> Prediction -> Controller + -> VirtualPanTiltActuator -> REAL CONTROL COMMAND TELEMETRY +``` + +| Stage | Real or virtual | Why | +|---|---|---| +| Camera / frames | **REAL** | `OpenCVCameraFrameSource` (`cv::VideoCapture`), a genuine phone/webcam device or network stream | +| Preprocessing | **REAL** | Actual grayscale/resize of the actual captured pixels | +| Classical + AI perception | **REAL** | The exact same `BeaconDetector` / `AiBeaconDetector` / `resolve_perception()` (Safe Hybrid, ADR-018) the simulation uses, run on real frames | +| State estimation (alpha-beta, coasting, reacquisition) | **REAL** | The exact same `TargetTracker` (ADR-019), run on real measurements | +| Controller output (pan/tilt rate command) | **REAL** | The exact same `PIDController`, run on a real image-derived tracking error | +| Actuator | **VIRTUAL** | `VirtualPanTiltActuator` integrates the real command into a bookkeeping angle. It drives **no physical hardware**. See "Claim boundary" below. | + +**Acceptable claims** (from this milestone): "real-camera-in-the-loop prototype", "physical +phone-camera frames processed by the FSOC perception stack", "Hybrid perception validated on +live phone-camera input", "real-camera target-loss / reacquisition demonstrated", +"controller commands generated from real image-derived pointing error". + +**Not acceptable, until physical hardware exists**: "physical pan/tilt tracking", "hardware +closed-loop FSOC terminal", "physical actuator validation", "real FSOC communication", +"laser link established". `fsoc_live`'s own startup banner and every JSON telemetry frame +print `CAMERA SOURCE = REAL_PHONE_CAMERA` / `ACTUATOR = VIRTUAL` explicitly so this +distinction cannot be silently lost in a demo or a screenshot. + +## Architecture + +### FrameSource — the interchangeable frame boundary + +```cpp +class FrameSource { + virtual bool open() = 0; + virtual bool read(Frame& frame) = 0; + virtual void close() = 0; + virtual FrameSourceInfo info() const = 0; +}; +``` + +`fsoc/frame_source.hpp`. One class, `OpenCVCameraFrameSource`, backs BOTH a native camera +index (`--camera-index N`) and a network URL (`--camera-url URL`) — `cv::VideoCapture` +already exposes both through the same `open()`/`read()` surface, so a second +"NetworkCameraFrameSource" class would just duplicate the wrapper. `FrameSource` is a pure +I/O boundary: it depends on nothing from the perception stack, and the perception stack +depends on nothing from it — `LiveTrackingSession::process_frame()` takes a `Frame` and a +`FrameSourceInfo` by value/reference, never a `FrameSource*`. + +### Why the existing perception stack needed almost no changes + +The audit for this milestone found that `BeaconDetector::detect()`, `AiBeaconDetector::detect()`, +`resolve_perception()`, `compute_tracking_error()`, `TargetTracker::update()`, and +`PIDController::update()` were **already** pure functions of pixels/angles with no coupling +to world truth, the renderer, or `SimulationRunner`. Only two real risks existed, both fixed +before any live-camera code was written: + +1. **`AiBeaconDetector`'s fixed `kInputStride`.** Its heatmap-to-pixel decode assumes the + frame handed to it is exactly the network's native resolution (640×480 by default). A raw + phone frame at any other resolution must be resized to exactly that size first — see + `preprocess_live_frame()` (`fsoc/live_preprocessing.hpp`). +2. **Sensing-camera scale.** `PanTiltCamera::fx_px()` = `(width_px/2) / tan(hfov/2)` — it + scales with `width_px` for a fixed field of view. The sensing-reference `PanTiltCamera` + `LiveTrackingSession` builds for `compute_tracking_error()` therefore uses the + **preprocessed frame's** width/height (640×480 by default), not the phone's raw capture + resolution, with the calibration's `hfov`/`vfov` carried over unchanged (field of view is + resolution-independent). Getting this backwards would silently scale every angular error + by the raw/preprocessed size ratio. See `sensing_camera_config()` in + `src/live_tracking_session.cpp`. + +### LiveTrackingSession — the real-camera counterpart to SimulationRunner + +Mirrors `SimulationRunner::step()`'s control-path order exactly: + +``` +preprocess -> classical (+ AI) detect -> resolve_perception + -> [tracker if enabled] -> compute_tracking_error + -> control (or loss/open-loop policy) -> virtual actuator step +``` + +`LiveFrameResult` carries **no** `target_truth`, `observation`, or `target_visible` field — +there is no ground truth for a real camera frame, so none is fabricated or leaked (see +`fsoc/live_tracking_session.hpp`). + +### Camera calibration (`fsoc/live_camera_calibration.hpp`) + +Deliberately **not** a photogrammetry suite. A pinhole model declared by four numbers +(`width_px`, `height_px`, `hfov_deg`, `vfov_deg`), stored in a dependency-free `key=value` +text file — the C++ core links no JSON library, and a four-field calibration doesn't +justify adding one. Two ways to produce a calibration file: + +- **Manual** (`fsoc_camera_calibrate --manual ...`): type in the phone's real spec-sheet FOV. +- **From a known object** (`fsoc_camera_calibrate --from-object ...`): angular-substitution + estimate from one known-size object at a known distance + (`estimate_hfov_deg_from_known_object`), with `vfov` derived from `hfov` assuming square + pixels (`estimate_vfov_deg_from_hfov`). A first-order approximation (no lens-distortion + correction) — adequate for coarse alignment, not a substitute for a real checkerboard + calibration. + +**The synthetic simulation's `CameraConfig` (hfov 20°, vfov 15°) is never assumed to equal a +phone's real FOV.** A calibration file is required before `fsoc_live` will run. + +### Virtual actuator (`fsoc/virtual_actuator.hpp`) + +`VirtualPanTiltActuator` consumes the same `ControlCommand`-shaped angular rate the +simulation's `PanTiltCamera::step()` consumes, integrates it into a bookkeeping angle with +rate saturation and an optional soft tilt-travel limit — but it drives nothing and, just as +importantly, it does **not** feed back into how the next real frame's pixel error is +interpreted (the phone's actual physical orientation is controlled by a human, not by this +class). Deliberately a separate class from `PanTiltCamera` rather than reusing it: reuse +would blur exactly the boundary this milestone exists to keep sharp. Every telemetry frame +labels it `ACTUATOR_TYPE = VIRTUAL`. + +### Manual Correction Assist (`fsoc_live --manual-assist`) + +Turns the real tracking error into a human-readable cue, e.g.: + +``` +REQUIRED CORRECTION PAN RIGHT -> 3.4 deg TILT UP 0.6 deg +``` + +using the same frozen sign convention `PIDController`/`PanTiltCamera` already rely on +(`fsoc/tracking_error.hpp`): a beacon right of centre needs a rightward pan to centre it, a +beacon above centre needs an upward tilt. This is **manual-in-the-loop** — a human physically +moves the phone — never "closed-loop hardware control." + +## Live telemetry JSON schema + +`fsoc_live` overwrites `/telemetry.json` after every processed frame (default +`generated/live/`, gitignored — this is runtime state, not a build artifact) and +`/frame.jpg` with the latest raw frame. Fields (all camelCase): + +| Field | Meaning | +|---|---| +| `frameIndex`, `timestampS`, `dtS` | Frame counter, source-clock time, measured wall-clock interval since the previous frame (a real camera has no fixed timestep) | +| `cameraSource` | Always `"REAL_PHONE_CAMERA"` | +| `actuatorType` | Always `"VIRTUAL"` | +| `sourceKind`, `sourceBackend`, `sourceDescription` | From `FrameSourceInfo` — e.g. `REAL_CAMERA`, `AVFoundation`, `camera index 0` | +| `rawWidthPx`/`rawHeightPx`, `preprocessedWidthPx`/`preprocessedHeightPx` | What the camera actually negotiated vs. what the detector actually measured pixels in | +| `perceptionMode`, `perceptionSource`, `classicalDetected`, `aiCandidateDetected`, `aiPresenceProbability` | `PerceptionDiagnostics`, unchanged from the simulation's own contract | +| `targetDetected`, `detectedXPx`/`detectedYPx` | The control-facing detection, if any | +| `pixelErrorXPx`/`pixelErrorYPx`, `panErrorDeg`/`tiltErrorDeg`/`totalErrorDeg` | `TrackingError`, converted to degrees at the UI boundary (matches the simulation's own degrees-only-at-the-boundary convention) | +| `lockState`, `trackerConfidence`, `isPrediction` | `TrackedState` (P0-v2 estimator), unchanged | +| `controlEnabled`, `commandPanRateDegS`/`commandTiltRateDegS` | The PID's real output | +| `virtualPanDeg`/`virtualTiltDeg`, `virtualPanSaturated`/`virtualTiltSaturated` | `VirtualActuatorState` — bookkeeping only | + +No JSON library is linked into the C++ core (the existing telemetry system is CSV, not +JSON), so this is hand-serialized in `apps/fsoc_live.cpp`. This schema is the contract; the +serialization code must match it exactly. + +## Mission Control transport + +`/api/simulation/:scenario` runs a **finite** scenario to completion and returns every frame +as one JSON array for the frontend to scrub through — that pattern does not fit a real +camera session, which has no natural end. Instead: + +``` +fsoc_live (long-running process, started by YOU) + -> overwrites generated/live/telemetry.json + frame.jpg after every frame + -> GET /api/live-camera (Next.js route, polled every 500ms) + -> GET /api/live-camera/frame (Next.js route, polled every 500ms, cache-busted) + -> frontend/app/mission/live/page.tsx (reuses the existing Panel/KeyValueRow primitives) +``` + +This is an honest **polling snapshot read**, not a push/streaming connection — latency is +bounded by the poll interval (≤500ms) plus whatever `fsoc_live`'s own frame-processing rate +is, not sub-frame real-time. `/mission/live` is reachable by direct link, not one of the 9 +fixed Stitch nav screens (`lib/nav.ts` — see `STITCH_IMPLEMENTATION_MAP.md`), so the existing +navigation rail is unchanged. + +## Real-camera metrics + +**These require a physical phone/camera and OS camera permission on your own machine — they +cannot be measured from this automated session** (see "What could not be measured" below). +Record them here after running `fsoc_camera_view` / `fsoc_live` yourself: + +| Metric | How to measure | Value | +|---|---|---| +| Actual negotiated resolution | `fsoc_camera_probe` output | _(fill in)_ | +| Effective camera FPS | `fsoc_camera_view` stdout (`effective_fps=`) | _(fill in)_ | +| Frame capture + preprocess + perception latency | Add `--seconds`-bounded timing around `process_frame()` if needed, or the wall-clock `dtS` field in `telemetry.json` | _(fill in)_ | +| Centroid jitter on a stationary target | Stddev of `detectedXPx`/`detectedYPx` over N frames of a static beacon | _(fill in)_ | +| Detection availability | Fraction of frames with `targetDetected == true` over a session | _(fill in)_ | +| Short-occlusion recovery | Manual: cover/uncover the beacon, confirm `lockState` goes `TRACKING -> COASTING -> TRACKING` (see `docs/PHONE_CAMERA_TEST_PLAN.md`) | _(fill in)_ | +| False-lock observations under clutter | Manual: introduce a second bright source, record what `perceptionSource`/`lockState` do | _(fill in)_ | + +**No pointing "accuracy" number is reported** — there is no calibrated real-world reference +target position in this milestone, and fabricating one would violate this milestone's core +rule. + +## Hardware-ready actuator interface (future) + +Not implemented now — documented so a future contributor knows what shape to build to. + +``` +PanTiltActuator (interface) +├── SimulatedPanTiltActuator (existing: PanTiltCamera::step(), simulation only) +├── VirtualPanTiltActuator (this milestone: honest bookkeeping, no hardware) +└── SerialPanTiltActuator (FUTURE — not implemented, no fake ACKs, no Arduino claim) +``` + +A future serial/hardware adapter should accept the exact same `ControlCommand`-shaped +`(pan_rate_rad_s, tilt_rate_rad_s)` this milestone's `VirtualPanTiltActuator` and the +simulation's `PanTiltCamera::step()` already accept — no interface change needed upstream of +the actuator. + +**The important mismatch to solve then, not now:** this simulation's controller emits an +**angular rate** command; a cheap hobby servo wants a **position** command. The likely +adapter-side conversion: + +``` +pan_target += pan_rate * dt +tilt_target += tilt_rate * dt +# then clamp to the physical servo's real travel limits +``` + +This conversion belongs entirely inside the future `SerialPanTiltActuator`, not upstream of +it — the controller, tracker, and perception stack should not need to change when real +hardware arrives. + +## Known limitations + +- The AI detector's heatmap decode is calibrated for 640×480 input; real frames are always + resized to that before detection (see "Architecture" above). This is adequate for coarse + alignment but is a real information loss for a phone that captures at much higher + resolution. +- Camera calibration is a single-FOV pinhole approximation with no lens-distortion + correction (Phase 8's own explicit scope boundary). +- The Mission Control live view is a polling read, not a low-latency stream (see "Mission + Control transport"). +- No physical actuation exists. Every "correction" is either bookkeeping + (`VirtualPanTiltActuator`) or a cue for a human to act on (Manual Correction Assist). +- What could not be measured from this automated coding session: opening a real camera + device requires interactive OS permission approval (macOS TCC) and visibly activates the + camera indicator light — an automated agent session should not trigger that unattended. + `fsoc_camera_probe`, `fsoc_camera_view`, and `fsoc_live` are built, unit-tested against + fabricated frames (`tests/live_tracking_session_tests.cpp` and friends), and ready to run, + but the actual live-hardware verification in `docs/PHONE_CAMERA_TEST_PLAN.md`'s "Manual + validation" section is intentionally left for you to run yourself. diff --git a/docs/PHONE_CAMERA_TEST_PLAN.md b/docs/PHONE_CAMERA_TEST_PLAN.md new file mode 100644 index 0000000..0942f2f --- /dev/null +++ b/docs/PHONE_CAMERA_TEST_PLAN.md @@ -0,0 +1,54 @@ +# Phone Camera-in-the-Loop — Test Plan + +Split per this milestone's own instructions: not everything can be automated because the +phone/camera is external hardware. See `docs/PHONE_CAMERA_METRICS.md` for the full +architecture and the honest reason live-hardware capture was not run from this coding +session. + +## Automated (CTest, run in this session — all currently PASS) + +| Test | What it covers | +|---|---| +| `fsoc_virtual_actuator_tests` | Config validation, rate saturation, integration, tilt travel limit, reset, invalid-argument rejection — OpenCV-free | +| `fsoc_live_camera_calibration_tests` | Save/load roundtrip, validation rejection, `CameraConfig` conversion, both FOV-estimation methods — OpenCV-free | +| `fsoc_frame_source_tests` | `FrameSourceKind` labeling, config's exactly-one-of-index-or-url enforcement, an out-of-range camera index failing cleanly (no real device touched) | +| `fsoc_live_preprocessing_tests` | Grayscale passthrough, BGR/BGRA-to-grayscale conversion, custom target size, empty-frame / invalid-size / unsupported-channel-count rejection | +| `fsoc_live_tracking_session_tests` | Pixel-error quadrant signs (fabricated frames, no camera), no-target leaves no ground truth and zero command, `control_enabled=false` never commands, a real command's sign matches the tracking error, tracker wiring coasts through a brief gap and reacquires, `reset()` clears controller/actuator state, invalid-argument rejection, AI/Hybrid mode requires an AI detector config | + +Run: `cmake --build --preset debug && ctest --preset debug --output-on-failure` (these run +alongside every existing Step 1–11 / Stage 3/4 test — all must stay green; see "Baseline +preservation" in the final report for this milestone). + +## Manual validation (run these yourself — fill in PASS/FAIL) + +Each of these requires a real phone/webcam and, on macOS, an interactive terminal session +that can answer the camera-permission prompt — this is why they were not run automatically. +See `docs/PHONE_CAMERA_GOLDEN_DEMO.md` for the exact commands. + +| # | Test | Acceptance criteria | Result | +|---|---|---|---| +| M1 | Camera discovery | `fsoc_camera_probe` lists at least one AVAILABLE index with a sane resolution | ☐ PASS ☐ FAIL | +| M2 | Camera opens and stays stable | `fsoc_camera_view --seconds 60` runs the full 60s with no crash, no repeated reconnection loop, no uncontrolled memory growth (watch `top`/Activity Monitor) | ☐ PASS ☐ FAIL | +| M3 | Timestamps increase correctly | `fsoc_camera_view` stdout shows strictly increasing `t=` values, no backward-time warning | ☐ PASS ☐ FAIL | +| M4 | Calibration produces a sane FOV | `fsoc_camera_calibrate --manual ...` (or `--from-object`) writes a file `fsoc_camera_calibrate --check` reads back correctly | ☐ PASS ☐ FAIL | +| M5 | Real target detected (Classical) | `fsoc_live --mode classical` with `tools/beacon_display.html` on a monitor: `targetDetected=true`, plausible `detectedXPx/YPx` | ☐ PASS ☐ FAIL | +| M6 | Real Hybrid perception | `fsoc_live --mode hybrid --ai-model models/tiny_beacon_net.onnx`: `perceptionMode=HYBRID`, a real `aiPresenceProbability` appears | ☐ PASS ☐ FAIL | +| M7 | Angular error sign convention | Move the beacon right of centre -> `panErrorDeg > 0`; left -> `< 0`; above -> `tiltErrorDeg > 0`; below -> `< 0` (matches `fsoc/tracking_error.hpp`) | ☐ PASS ☐ FAIL | +| M8 | Manual Correction Assist cue direction | With `--manual-assist`, moving the phone in the printed direction visibly reduces the printed error magnitude on the next frame | ☐ PASS ☐ FAIL | +| M9 | Phone motion test | Physically pan the phone left/right across the beacon; `commandPanRateDegS` sign tracks the visible offset direction | ☐ PASS ☐ FAIL | +| M10 | Occlusion / coasting | `--tracker`: briefly cover the beacon; `lockState` goes `TRACKING -> COASTING`, `isPrediction=true`, then `-> TRACKING` on uncovering | ☐ PASS ☐ FAIL | +| M11 | Long occlusion / reacquisition | Cover the beacon beyond `max_coast_frames`; `lockState` reaches `LOST` then `SEARCHING`, then re-acquires (`ACQUIRING -> TRACKING`) once revealed | ☐ PASS ☐ FAIL | +| M12 | Clutter / false-lock investigation | Introduce a second bright source (another screen/lamp) alongside the beacon; record what actually happens (`perceptionSource`, any lock switch) — do NOT expect this to be solved; this is exploratory, per docs/MVP_ABLATION.md's own disclosed limitation | ☐ PASS ☐ FAIL / OBSERVED: _______ | +| M13 | Camera disconnect handled safely | Unplug/close the camera mid-run; `fsoc_live` prints the "too many consecutive frame read failures" message and exits cleanly (no crash) | ☐ PASS ☐ FAIL | +| M14 | Mission Control shows REAL/VIRTUAL labels | With `fsoc_live` running, `/mission/live` shows `CAMERA SOURCE: REAL_PHONE_CAMERA` and `ACTUATOR: VIRTUAL` and the live frame updates | ☐ PASS ☐ FAIL | +| M15 | Mission Control handles no session honestly | With `fsoc_live` NOT running, `/mission/live` shows "No live session" — never fabricated telemetry | ☐ PASS ☐ FAIL (already verified in this session — see final report) | + +## What was verified automatically vs. left for you + +Verified in this session: M15 (the "no live session" honest-failure path), plus every +automated test above and a full manual page-render check with fabricated (clearly +scratch, gitignored, deleted afterward) telemetry — see the final report's "Mission Control +integration" section. + +Left for you (M1–M14): genuine camera hardware access, per `docs/PHONE_CAMERA_METRICS.md`'s +"What could not be measured" note. diff --git a/docs/README.md b/docs/README.md index 9e442b1..8cef6aa 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,6 +11,10 @@ for what it covers. | [`SIH_MVP_FREEZE.md`](SIH_MVP_FREEZE.md) | The frozen SIH MVP state: build/test status, measured metrics, safe vs. unsafe claims for a judge presentation, exact demo commands | | [`MVP_GOLDEN_DEMO.md`](MVP_GOLDEN_DEMO.md) | The 15-step, fully reproducible judge-session walkthrough | | [`RELEASE_NOTES_v2_sih_mvp.md`](RELEASE_NOTES_v2_sih_mvp.md) | Draft GitHub release notes for the `v2_sih_mvp` tag (not yet published) | +| [`PHONE_CAMERA_METRICS.md`](PHONE_CAMERA_METRICS.md) | Mobile Phone Camera-in-the-Loop: architecture, JSON telemetry schema, claim boundary, real-camera metrics | +| [`PHONE_CAMERA_GOLDEN_DEMO.md`](PHONE_CAMERA_GOLDEN_DEMO.md) | The 15-step real-camera demo walkthrough (run yourself — needs a real camera) | +| [`PHONE_CAMERA_TEST_PLAN.md`](PHONE_CAMERA_TEST_PLAN.md) | Automated (CTest) vs. manual (hardware-required) test split, with a PASS/FAIL checklist | +| [`DEPLOYMENT.md`](DEPLOYMENT.md) | Public Vercel deployment architecture, project settings, and why the phone-camera prototype stays local-only | ## Architecture diff --git a/frontend/app/api/live-camera/frame/route.ts b/frontend/app/api/live-camera/frame/route.ts new file mode 100644 index 0000000..14d30e6 --- /dev/null +++ b/frontend/app/api/live-camera/frame/route.ts @@ -0,0 +1,74 @@ +import { NextRequest, NextResponse } from "next/server"; +import { readFile } from "node:fs/promises"; +import { existsSync } from "node:fs"; +import path from "node:path"; + +import { liveDir } from "@/lib/live-camera/paths"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +/** + * SERVER-ONLY. Mobile Phone Camera-in-the-Loop milestone. + * + * Serves one specific frame_.jpg that LiveFramePublisher wrote (see + * include/fsoc/live_frame_publisher.hpp). The caller MUST pass the exact + * frame index it got from GET /api/live-camera's `frameIndex` field -- + * there is no "current frame" concept here on purpose, so an image can + * never be served mismatched against the telemetry that named it. + * + * A 404 here means that pair fell outside LiveFramePublisher's retention + * window between the two requests (the client polled too slowly) -- the + * correct response is to re-poll /api/live-camera for a fresher index, not + * to fall back to any other image. + * + * GET /api/live-camera/frame?frame= + * 200 image/jpeg bytes for exactly that frame index + * 400 missing/invalid ?frame= + * 404 that frame index was never published, or has already been pruned + * 503 no live session / output directory does not exist yet + */ + +export async function GET(req: NextRequest) { + const dir = liveDir(); + if (!existsSync(dir)) { + return NextResponse.json( + { error: "no live session", detail: `${dir} does not exist yet.` }, + { status: 503, headers: { "cache-control": "no-store" } }, + ); + } + + const frameParam = new URL(req.url).searchParams.get("frame"); + const frameIndex = frameParam !== null ? Number(frameParam) : NaN; + if (!Number.isInteger(frameIndex) || frameIndex < 0) { + return NextResponse.json( + { error: "missing or invalid ?frame=" }, + { status: 400, headers: { "cache-control": "no-store" } }, + ); + } + + // Reject path traversal / non-numeric injection outright: the filename is built + // from a validated integer only, never from the raw query string. + const file = path.join(dir, `frame_${frameIndex}.jpg`); + if (!existsSync(file)) { + return NextResponse.json( + { + error: "frame not found", + detail: `frame ${frameIndex} was never published or has already been pruned -- re-fetch /api/live-camera for a current index.`, + }, + { status: 404, headers: { "cache-control": "no-store" } }, + ); + } + try { + const bytes = await readFile(file); + return new NextResponse(bytes, { + status: 200, + headers: { "content-type": "image/jpeg", "cache-control": "no-store" }, + }); + } catch (err) { + return NextResponse.json( + { error: "frame read failed (likely mid-prune, retry)", detail: String(err) }, + { status: 503, headers: { "cache-control": "no-store" } }, + ); + } +} diff --git a/frontend/app/api/live-camera/record/route.ts b/frontend/app/api/live-camera/record/route.ts new file mode 100644 index 0000000..d8b7a3b --- /dev/null +++ b/frontend/app/api/live-camera/record/route.ts @@ -0,0 +1,87 @@ +import { NextRequest, NextResponse } from "next/server"; +import { existsSync } from "node:fs"; +import { mkdir, rename, writeFile } from "node:fs/promises"; +import path from "node:path"; + +import { liveDir } from "@/lib/live-camera/paths"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +/** + * SERVER-ONLY. G2 real-session recording control (docs/LIVE_REALDATA_TASK_STATE.md). + * + * browser (Start/Stop Recording, Mark Event buttons) + * -> POST here -> generated/live/command.txt (key=value, temp-then-rename) + * -> fsoc_live polls that file once per camera-frame iteration and applies it + * -> the NEXT /api/live-camera poll shows the applied recordingActive/ + * recordingId/recordedFrameCount fields + * + * This route's 200 response means "the command file was written," never "fsoc_live + * has applied it" -- there is no synchronous acknowledgement channel. The frontend + * must confirm the actual effect from the next telemetry poll, the same honesty + * rule this project applies to every other command (see fsoc/virtual_actuator.hpp's + * comment on acknowledgement vs. measured effect). + * + * There is no message queue: command.txt is a single slot. Two POSTs faster than + * one camera frame interval apart will have the earlier one silently superseded — + * a documented limitation (see apps/fsoc_live.cpp's file-header comment), not a bug. + * + * POST /api/live-camera/record body: {"action": "start"|"stop"|"mark_event", "label"?: string} + * 200 command file written (commandId returned) + * 400 invalid action / missing label for mark_event + * 503 no live session running (only enforced for "start" -- fsoc_live itself + * also refuses to double-start, so "stop"/"mark_event" with no session is a + * harmless no-op on the C++ side and is not blocked here) + */ + +type Action = "start" | "stop" | "mark_event"; + +function isAction(v: unknown): v is Action { + return v === "start" || v === "stop" || v === "mark_event"; +} + +export async function POST(req: NextRequest) { + let body: unknown; + try { + body = await req.json(); + } catch (err) { + return NextResponse.json({ error: "invalid JSON body", detail: String(err) }, { status: 400 }); + } + + const action = (body as { action?: unknown } | null)?.action; + const label = (body as { label?: unknown } | null)?.label; + if (!isAction(action)) { + return NextResponse.json({ error: "action must be one of: start, stop, mark_event" }, { status: 400 }); + } + if (action === "mark_event" && (typeof label !== "string" || label.trim().length === 0)) { + return NextResponse.json({ error: "mark_event requires a non-empty string label" }, { status: 400 }); + } + + const dir = liveDir(); + if (action === "start" && !existsSync(path.join(dir, "manifest.json"))) { + return NextResponse.json( + { + error: "no live session", + detail: "generated/live/manifest.json does not exist -- start fsoc_live before recording.", + }, + { status: 503, headers: { "cache-control": "no-store" } }, + ); + } + + await mkdir(dir, { recursive: true }); + const commandId = Date.now().toString(); + const lines = [`commandId=${commandId}`, `action=${action}`]; + // command.txt is line-oriented key=value (see apps/fsoc_live.cpp) -- strip any + // newline a caller's label might contain so it can't inject a bogus extra key. + if (action === "mark_event") lines.push(`label=${(label as string).replace(/[\r\n]+/g, " ").trim()}`); + const finalPath = path.join(dir, "command.txt"); + const tmpPath = `${finalPath}.tmp`; + await writeFile(tmpPath, lines.join("\n") + "\n", "utf8"); + await rename(tmpPath, finalPath); + + return NextResponse.json( + { accepted: true, commandId, action }, + { headers: { "cache-control": "no-store" } }, + ); +} diff --git a/frontend/app/api/live-camera/route.ts b/frontend/app/api/live-camera/route.ts new file mode 100644 index 0000000..221dda2 --- /dev/null +++ b/frontend/app/api/live-camera/route.ts @@ -0,0 +1,87 @@ +import { NextResponse } from "next/server"; +import { readFile } from "node:fs/promises"; +import { existsSync } from "node:fs"; +import path from "node:path"; + +import { liveDir } from "@/lib/live-camera/paths"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +/** + * SERVER-ONLY. Mobile Phone Camera-in-the-Loop milestone. + * + * fsoc_live (long-running C++ process, real camera) + * -> generated/live/manifest.json (names the current frame_.jpg / + * telemetry_.json pair, flipped atomically only AFTER both files + * are fully written -- see include/fsoc/live_frame_publisher.hpp and + * docs/LIVE_DATA_AUDIT.md section 2) + * -> (this route, polled) -> browser + * + * fsoc_live is a long-running process (a real camera session has no natural + * "end of run" the way a deterministic scenario does), so it cannot be + * invoked per-request the way /api/simulation/:scenario invokes fsoc_demo. + * This route reads the manifest, then reads exactly the telemetry file it + * names -- LiveFramePublisher's atomicity guarantee means that file is + * always complete by the time the manifest can be observed naming it. The + * frame index returned here is what the client must request from + * /api/live-camera/frame?frame= -- never a bare "current frame.jpg" -- + * so the served image and the served telemetry are guaranteed to describe + * the same camera frame, not two independently-polled snapshots. + * + * GET /api/live-camera + * 200 the current (frameIndex-identified) telemetry, plus staleness info + * 503 no live session is running / no telemetry has been published yet + */ + +interface Manifest { + schemaVersion: number; + frameIndex: number; + frameFile: string; + telemetryFile: string; + publishedAtEpochMs: number; +} + +export async function GET() { + const dir = liveDir(); + const manifestPath = path.join(dir, "manifest.json"); + if (!existsSync(manifestPath)) { + return NextResponse.json( + { + error: "no live session", + detail: + "generated/live/manifest.json does not exist. Start fsoc_live yourself " + + "(see docs/PHONE_CAMERA_GOLDEN_DEMO.md) -- this route never fabricates telemetry.", + }, + { status: 503, headers: { "cache-control": "no-store" } }, + ); + } + + let manifest: Manifest; + try { + manifest = JSON.parse(await readFile(manifestPath, "utf8")); + } catch (err) { + // Extremely unlikely (manifest is written via temp-then-rename), but a read can + // still race a rename mid-flight on some filesystems -- report it as a clean, + // retryable miss, never as fabricated telemetry. + return NextResponse.json( + { error: "manifest read failed (likely mid-write, retry)", detail: String(err) }, + { status: 503, headers: { "cache-control": "no-store" } }, + ); + } + + const telemetryPath = path.join(dir, manifest.telemetryFile); + try { + const frame = JSON.parse(await readFile(telemetryPath, "utf8")); + const ageS = (Date.now() - manifest.publishedAtEpochMs) / 1000; + return NextResponse.json( + { frame, frameIndex: manifest.frameIndex, ageS, stale: ageS > 3 }, + { headers: { "cache-control": "no-store" } }, + ); + } catch (err) { + return NextResponse.json( + { error: "telemetry read failed for the manifest's current frame, retry", detail: String(err) }, + { status: 503, headers: { "cache-control": "no-store" } }, + ); + } +} diff --git a/frontend/app/api/real-sessions/[recordingId]/frame/[index]/route.ts b/frontend/app/api/real-sessions/[recordingId]/frame/[index]/route.ts new file mode 100644 index 0000000..9819ce4 --- /dev/null +++ b/frontend/app/api/real-sessions/[recordingId]/frame/[index]/route.ts @@ -0,0 +1,44 @@ +import { NextResponse } from "next/server"; +import { existsSync } from "node:fs"; +import { readFile } from "node:fs/promises"; +import path from "node:path"; + +import { isValidRecordingId, recordingDir } from "@/lib/real-sessions/paths"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +/** + * SERVER-ONLY. G2 annotation tool: serves one RAW frame from a recording + * (frames/frame_.jpg, exactly as fsoc_live wrote it -- no overlay, no + * crosshair, no dashboard text; see fsoc/real_session_recorder.hpp). + * + * GET /api/real-sessions/:recordingId/frame/:index + * 200 image/jpeg bytes + * 400 invalid recordingId/index + * 404 no such recording or frame + */ + +export async function GET( + _req: Request, + { params }: { params: { recordingId: string; index: string } }, +) { + const { recordingId, index } = params; + if (!isValidRecordingId(recordingId)) { + return NextResponse.json({ error: "invalid recordingId" }, { status: 400 }); + } + const frameIndex = Number(index); + if (!Number.isInteger(frameIndex) || frameIndex < 0) { + return NextResponse.json({ error: "invalid frame index" }, { status: 400 }); + } + + const file = path.join(recordingDir(recordingId), "frames", `frame_${frameIndex}.jpg`); + if (!existsSync(file)) { + return NextResponse.json({ error: "frame not found" }, { status: 404 }); + } + const bytes = await readFile(file); + return new NextResponse(bytes, { + status: 200, + headers: { "content-type": "image/jpeg", "cache-control": "no-store" }, + }); +} diff --git a/frontend/app/api/real-sessions/[recordingId]/labels/route.ts b/frontend/app/api/real-sessions/[recordingId]/labels/route.ts new file mode 100644 index 0000000..efa8b12 --- /dev/null +++ b/frontend/app/api/real-sessions/[recordingId]/labels/route.ts @@ -0,0 +1,159 @@ +import { NextRequest, NextResponse } from "next/server"; +import { existsSync } from "node:fs"; +import { readFile, rename, writeFile } from "node:fs/promises"; +import path from "node:path"; + +import { isValidRecordingId, recordingDir } from "@/lib/real-sessions/paths"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +/** + * SERVER-ONLY. G2 annotation tool: reviewed labels for one recording, stored as a + * single labels.json keyed by frameIndex (not JSONL -- a human re-labeling a frame + * must UPDATE that frame's entry, not append a second, ambiguous one). + * + * `reviewed` is always true for a label written through this route: a human + * explicitly saved it. A detector's own suggestion (from telemetry.jsonl, + * fetched separately) is never written here on its own -- it only becomes a + * label once a person accepts/adjusts it and hits Save, matching the "detector + * suggestions are not independent ground truth until reviewed" requirement. + * + * GET /api/real-sessions/:recordingId/labels + * 200 { schemaVersion, recordingId, sessionId, labelCoordinateSpace, labels } + * (labels: {} if none saved yet -- never fabricated) + * + * POST /api/real-sessions/:recordingId/labels + * body: { frameIndex: number, presence: "present"|"absent"|"partial_occlusion"| + * "full_occlusion"|"ambiguous", centerXPx?: number|null, centerYPx?: number|null } + * 200 the updated labels document + * 400 invalid recordingId/body + * 404 no such recording + */ + +type Presence = "present" | "absent" | "partial_occlusion" | "full_occlusion" | "ambiguous"; +const PRESENCE_VALUES: Presence[] = ["present", "absent", "partial_occlusion", "full_occlusion", "ambiguous"]; + +interface FrameLabel { + presence: Presence; + centerXPx: number | null; + centerYPx: number | null; + reviewed: true; + labeledAtEpochMs: number; +} + +interface LabelsDoc { + schemaVersion: 1; + recordingId: string; + sessionId: string; + labelCoordinateSpace: "raw"; + labels: Record; +} + +function labelsPath(recordingId: string): string { + return path.join(recordingDir(recordingId), "labels.json"); +} + +async function readManifestSessionId(recordingId: string): Promise { + try { + const manifest = JSON.parse(await readFile(path.join(recordingDir(recordingId), "manifest.json"), "utf8")); + return manifest.sessionId ?? ""; + } catch { + return ""; + } +} + +async function readLabelsDoc(recordingId: string): Promise { + const file = labelsPath(recordingId); + if (existsSync(file)) { + try { + return JSON.parse(await readFile(file, "utf8")); + } catch { + // fall through to a fresh doc rather than serving corrupt content + } + } + return { + schemaVersion: 1, + recordingId, + sessionId: await readManifestSessionId(recordingId), + labelCoordinateSpace: "raw", + labels: {}, + }; +} + +export async function GET(_req: NextRequest, { params }: { params: { recordingId: string } }) { + const { recordingId } = params; + if (!isValidRecordingId(recordingId)) { + return NextResponse.json({ error: "invalid recordingId" }, { status: 400 }); + } + if (!existsSync(recordingDir(recordingId))) { + return NextResponse.json({ error: "recording not found" }, { status: 404 }); + } + return NextResponse.json(await readLabelsDoc(recordingId), { headers: { "cache-control": "no-store" } }); +} + +export async function POST(req: NextRequest, { params }: { params: { recordingId: string } }) { + const { recordingId } = params; + if (!isValidRecordingId(recordingId)) { + return NextResponse.json({ error: "invalid recordingId" }, { status: 400 }); + } + if (!existsSync(recordingDir(recordingId))) { + return NextResponse.json({ error: "recording not found" }, { status: 404 }); + } + + let body: unknown; + try { + body = await req.json(); + } catch (err) { + return NextResponse.json({ error: "invalid JSON body", detail: String(err) }, { status: 400 }); + } + + const b = body as { + frameIndex?: unknown; + presence?: unknown; + centerXPx?: unknown; + centerYPx?: unknown; + } | null; + const frameIndex = b?.frameIndex; + const presence = b?.presence; + if (typeof frameIndex !== "number" || !Number.isInteger(frameIndex) || frameIndex < 0) { + return NextResponse.json({ error: "frameIndex must be a non-negative integer" }, { status: 400 }); + } + if (typeof presence !== "string" || !PRESENCE_VALUES.includes(presence as Presence)) { + return NextResponse.json({ error: `presence must be one of: ${PRESENCE_VALUES.join(", ")}` }, { status: 400 }); + } + + const centerXRaw = b?.centerXPx; + const centerYRaw = b?.centerYPx; + const centerProvided = + typeof centerXRaw === "number" && Number.isFinite(centerXRaw) && + typeof centerYRaw === "number" && Number.isFinite(centerYRaw); + + // A visible target needs a center; an absent/fully-occluded one cannot have one -- + // enforced server-side so a client bug can't write a self-contradictory label. + if ((presence === "present" || presence === "partial_occlusion") && !centerProvided) { + return NextResponse.json( + { error: `presence "${presence}" requires numeric centerXPx/centerYPx` }, + { status: 400 }, + ); + } + const forceNull = presence === "absent" || presence === "full_occlusion"; + const centerXPx = forceNull ? null : centerProvided ? (centerXRaw as number) : null; + const centerYPx = forceNull ? null : centerProvided ? (centerYRaw as number) : null; + + const doc = await readLabelsDoc(recordingId); + doc.labels[String(frameIndex)] = { + presence: presence as Presence, + centerXPx, + centerYPx, + reviewed: true, + labeledAtEpochMs: Date.now(), + }; + + const finalPath = labelsPath(recordingId); + const tmpPath = `${finalPath}.tmp`; + await writeFile(tmpPath, JSON.stringify(doc, null, 2), "utf8"); + await rename(tmpPath, finalPath); + + return NextResponse.json(doc, { headers: { "cache-control": "no-store" } }); +} diff --git a/frontend/app/api/real-sessions/[recordingId]/route.ts b/frontend/app/api/real-sessions/[recordingId]/route.ts new file mode 100644 index 0000000..f1bc349 --- /dev/null +++ b/frontend/app/api/real-sessions/[recordingId]/route.ts @@ -0,0 +1,54 @@ +import { NextResponse } from "next/server"; +import { existsSync } from "node:fs"; +import { readFile } from "node:fs/promises"; +import path from "node:path"; + +import { isValidRecordingId, recordingDir } from "@/lib/real-sessions/paths"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +/** + * SERVER-ONLY. G2 annotation tool: one recording's manifest plus its full + * per-frame telemetry (parsed from telemetry.jsonl), so the annotator page can + * build a frame scrubber and show the detector's own (non-ground-truth) + * suggested centroid alongside each frame. + * + * GET /api/real-sessions/:recordingId + * 200 { manifest, telemetry: [...] } -- telemetry sorted by frameIndex + * 400 invalid recordingId + * 404 no such recording + */ + +export async function GET(_req: Request, { params }: { params: { recordingId: string } }) { + const { recordingId } = params; + if (!isValidRecordingId(recordingId)) { + return NextResponse.json({ error: "invalid recordingId" }, { status: 400 }); + } + const dir = recordingDir(recordingId); + const manifestPath = path.join(dir, "manifest.json"); + if (!existsSync(manifestPath)) { + return NextResponse.json({ error: "recording not found" }, { status: 404 }); + } + + const manifest = JSON.parse(await readFile(manifestPath, "utf8")); + + const telemetryPath = path.join(dir, "telemetry.jsonl"); + const telemetry: unknown[] = []; + if (existsSync(telemetryPath)) { + const raw = await readFile(telemetryPath, "utf8"); + for (const line of raw.split("\n")) { + if (!line.trim()) continue; + try { + telemetry.push(JSON.parse(line)); + } catch { + // One malformed line (e.g. a truncated last write from an unclean stop) is + // skipped, not fatal to the rest of the recording's review. + continue; + } + } + } + telemetry.sort((a, b) => (a as { frameIndex: number }).frameIndex - (b as { frameIndex: number }).frameIndex); + + return NextResponse.json({ manifest, telemetry }, { headers: { "cache-control": "no-store" } }); +} diff --git a/frontend/app/api/real-sessions/route.ts b/frontend/app/api/real-sessions/route.ts new file mode 100644 index 0000000..03cefe8 --- /dev/null +++ b/frontend/app/api/real-sessions/route.ts @@ -0,0 +1,57 @@ +import { NextResponse } from "next/server"; +import { existsSync } from "node:fs"; +import { readdir, readFile } from "node:fs/promises"; +import path from "node:path"; + +import { isValidRecordingId, realSessionsDir, recordingDir } from "@/lib/real-sessions/paths"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +/** + * SERVER-ONLY. G2 annotation tool: lists local recordings written by + * fsoc_live's RealSessionRecorder (fsoc/real_session_recorder.hpp) so the + * annotator page can offer a picker. Read-only; never fabricates a recording + * that doesn't exist on disk. + * + * GET /api/real-sessions + * 200 [{ recordingId, sessionId, startedAtEpochMs, endedAtEpochMs, recordedFrameCount, + * errorCount, eventCount, calibrationStatus, perceptionMode }], newest first + */ + +interface Manifest { + recordingId: string; + sessionId: string; + startedAtEpochMs: number; + endedAtEpochMs: number | null; + recordedFrameCount: number; + errorCount: number; + eventCount: number; + calibrationStatus: string; + perceptionMode: string; +} + +export async function GET() { + const dir = realSessionsDir(); + if (!existsSync(dir)) { + return NextResponse.json([], { headers: { "cache-control": "no-store" } }); + } + + const entries = await readdir(dir, { withFileTypes: true }); + const recordings: Manifest[] = []; + for (const entry of entries) { + if (!entry.isDirectory() || !isValidRecordingId(entry.name)) continue; + const manifestPath = path.join(recordingDir(entry.name), "manifest.json"); + if (!existsSync(manifestPath)) continue; + try { + const manifest = JSON.parse(await readFile(manifestPath, "utf8")); + recordings.push(manifest); + } catch { + // A recording mid-write (manifest temp-then-rename raced this read) is skipped + // for this listing, not reported as broken -- it will appear once settled. + continue; + } + } + recordings.sort((a, b) => b.startedAtEpochMs - a.startedAtEpochMs); + return NextResponse.json(recordings, { headers: { "cache-control": "no-store" } }); +} diff --git a/frontend/app/icon.svg b/frontend/app/icon.svg new file mode 100644 index 0000000..c4e82c6 --- /dev/null +++ b/frontend/app/icon.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/frontend/app/layout.tsx b/frontend/app/layout.tsx index e0ec0a6..d38aee1 100644 --- a/frontend/app/layout.tsx +++ b/frontend/app/layout.tsx @@ -12,10 +12,41 @@ const jetbrainsMono = JetBrains_Mono({ display: "swap", }); +const SITE_URL = "https://fsoc-iota.vercel.app"; +const TITLE = "FSOC — AI-Based Virtual Camera Tracking"; +const DESCRIPTION = + "Closed-loop hybrid perception, state estimation and pan/tilt control testbed for coarse alignment of mobile free-space optical communication terminals."; + export const metadata: Metadata = { - title: "FSOC ALIGNMENT — SIH26169", - description: - "Mission-control frontend for the SIH26169 virtual camera tracking engine. Observer / presentation layer over the frozen v1_baseline C++ engine.", + metadataBase: new URL(SITE_URL), + title: { + default: TITLE, + template: "%s — FSOC", + }, + description: DESCRIPTION, + applicationName: "FSOC Mission Control", + keywords: [ + "FSOC", + "free-space optical communication", + "computer vision", + "tracking", + "control systems", + "SIH26169", + ], + authors: [{ name: "Team IRODOV" }], + robots: { index: true, follow: true }, + openGraph: { + type: "website", + url: SITE_URL, + siteName: "FSOC", + title: TITLE, + description: DESCRIPTION, + }, + twitter: { + card: "summary_large_image", + title: TITLE, + description: DESCRIPTION, + }, }; export const viewport: Viewport = { diff --git a/frontend/app/mission/annotate/page.tsx b/frontend/app/mission/annotate/page.tsx new file mode 100644 index 0000000..9d480d8 --- /dev/null +++ b/frontend/app/mission/annotate/page.tsx @@ -0,0 +1,410 @@ +"use client"; + +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; + +import { Screen } from "@/components/shell/AppShell"; +import { Panel, PanelHeader, KeyValueRow } from "@/components/ui/Panel"; +import { fixed } from "@/lib/format"; + +/** + * /mission/annotate — G2 real-session annotation tool. + * + * NOT one of the 9 fixed Stitch nav screens (same convention as /mission/live -- + * see STITCH_IMPLEMENTATION_MAP.md); reachable from the /mission/live page and by + * direct link. Reviews a LOCAL recording written by fsoc_live's + * RealSessionRecorder (generated/real_sessions//) -- there is + * deliberately no upload step; this reads local files a person can inspect + * themselves. + * + * A detector "suggestion" shown here (from the recording's own telemetry.jsonl) + * is never independent ground truth -- it is only ever a starting point a human + * can accept, adjust, or ignore before hitting Save, which is the only action + * that writes a reviewed label. + */ + +type Presence = "present" | "absent" | "partial_occlusion" | "full_occlusion" | "ambiguous"; +const PRESENCE_OPTIONS: { value: Presence; label: string; needsCenter: boolean }[] = [ + { value: "present", label: "Present", needsCenter: true }, + { value: "partial_occlusion", label: "Partial Occlusion", needsCenter: true }, + { value: "full_occlusion", label: "Full Occlusion", needsCenter: false }, + { value: "absent", label: "Absent", needsCenter: false }, + { value: "ambiguous", label: "Ambiguous", needsCenter: false }, +]; + +interface RecordingSummary { + recordingId: string; + sessionId: string; + startedAtEpochMs: number; + endedAtEpochMs: number | null; + recordedFrameCount: number; + errorCount: number; + eventCount: number; + calibrationStatus: string; + perceptionMode: string; +} + +interface Manifest extends RecordingSummary { + rawWidthPx: number; + rawHeightPx: number; + cliArgs: string; + softwareCommit: string; +} + +interface TelemetryFrame { + frameIndex: number; + timestampS: number; + targetDetected: boolean; + detectedXPx: number | null; + detectedYPx: number | null; +} + +interface FrameLabel { + presence: Presence; + centerXPx: number | null; + centerYPx: number | null; + reviewed: true; + labeledAtEpochMs: number; +} + +interface LabelsDoc { + schemaVersion: number; + recordingId: string; + sessionId: string; + labelCoordinateSpace: "raw"; + labels: Record; +} + +export default function AnnotatePage() { + const [recordings, setRecordings] = useState(null); + const [recordingId, setRecordingId] = useState(null); + const [manifest, setManifest] = useState(null); + const [telemetry, setTelemetry] = useState([]); + const [labelsDoc, setLabelsDoc] = useState(null); + // Position WITHIN the telemetry array, not a raw frame number: a recording's + // frameIndex values are the camera's own continuous counter across the whole + // fsoc_live session, so a recording started partway through often begins at a + // large, non-zero frameIndex (e.g. 1000, not 0) -- navigating by array position + // is correct regardless of what the first/last real frameIndex happens to be. + const [cursor, setCursor] = useState(0); + const [pendingPresence, setPendingPresence] = useState(null); + const [pendingCenter, setPendingCenter] = useState<{ x: number; y: number } | null>(null); + const [status, setStatus] = useState(null); + const [loadError, setLoadError] = useState(null); + const imgRef = useRef(null); + + useEffect(() => { + fetch("/api/real-sessions", { cache: "no-store" }) + .then((r) => r.json()) + .then(setRecordings) + .catch((err) => setLoadError(String(err))); + }, []); + + const loadRecording = useCallback((id: string) => { + setRecordingId(id); + setCursor(0); + setLoadError(null); + Promise.all([ + fetch(`/api/real-sessions/${id}`, { cache: "no-store" }).then((r) => r.json()), + fetch(`/api/real-sessions/${id}/labels`, { cache: "no-store" }).then((r) => r.json()), + ]) + .then(([detail, labels]) => { + if (detail.error) throw new Error(detail.error); + setManifest(detail.manifest); + setTelemetry(detail.telemetry); + setLabelsDoc(labels); + }) + .catch((err) => setLoadError(String(err))); + }, []); + + const currentTelemetry = telemetry[cursor] ?? null; + const frameIndex = currentTelemetry?.frameIndex ?? 0; + const existingLabel = labelsDoc?.labels[String(frameIndex)] ?? null; + + // Reset the pending (unsaved) label draft whenever the frame changes -- reload + // from whatever was already reviewed for this frame, if anything. + useEffect(() => { + setPendingPresence(existingLabel?.presence ?? null); + setPendingCenter( + existingLabel?.centerXPx != null && existingLabel?.centerYPx != null + ? { x: existingLabel.centerXPx, y: existingLabel.centerYPx } + : null, + ); + setStatus(null); + // existingLabel is derived from labelsDoc + cursor; depending on it directly + // would re-run this after every save, clobbering the just-saved draft. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [cursor, recordingId]); + + function handleImageClick(e: React.MouseEvent) { + const img = imgRef.current; + if (!img || !manifest) return; + const rect = img.getBoundingClientRect(); + const scaleX = manifest.rawWidthPx / rect.width; + const scaleY = manifest.rawHeightPx / rect.height; + const x = (e.clientX - rect.left) * scaleX; + const y = (e.clientY - rect.top) * scaleY; + setPendingCenter({ x, y }); + } + + function acceptDetectorSuggestion() { + if (currentTelemetry?.detectedXPx != null && currentTelemetry?.detectedYPx != null) { + setPendingCenter({ x: currentTelemetry.detectedXPx, y: currentTelemetry.detectedYPx }); + } + } + + async function save() { + if (!recordingId || !pendingPresence) { + setStatus("choose a presence value first"); + return; + } + const needsCenter = PRESENCE_OPTIONS.find((o) => o.value === pendingPresence)?.needsCenter; + if (needsCenter && !pendingCenter) { + setStatus("click the image to set the beacon center first"); + return; + } + setStatus("saving..."); + try { + const res = await fetch(`/api/real-sessions/${recordingId}/labels`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + frameIndex, + presence: pendingPresence, + centerXPx: pendingCenter?.x ?? null, + centerYPx: pendingCenter?.y ?? null, + }), + }); + const body = await res.json(); + if (!res.ok) { + setStatus(body.error ?? "save failed"); + return; + } + setLabelsDoc(body); + setStatus("saved"); + } catch (err) { + setStatus(String(err)); + } + } + + const reviewedCount = labelsDoc ? Object.keys(labelsDoc.labels).length : 0; + const totalFrames = telemetry.length; + + const markers = useMemo(() => { + if (!imgRef.current || !manifest) return null; + const el = imgRef.current; + // offsetLeft/offsetTop (relative to the `relative` container, the img's + // offsetParent) account for the flex-centering gap when the image doesn't + // fill its container -- getBoundingClientRect()-based scaling alone drops + // that offset and puts the marker near the container's corner instead of on + // the actual point (same bug fixed in /mission/live). + const w = el.offsetWidth; + const h = el.offsetHeight; + const toCss = (x: number, y: number) => ({ + left: `${el.offsetLeft + (x / manifest.rawWidthPx) * w}px`, + top: `${el.offsetTop + (y / manifest.rawHeightPx) * h}px`, + }); + return { toCss }; + }, [manifest, cursor]); // eslint-disable-line react-hooks/exhaustive-deps + + return ( + +

+ + Real-Session Annotation (G2) + + + Local recordings only — reviewed labels are not independent ground truth + until saved by a human + +
+ + {loadError && ( + {loadError} + )} + + {!recordingId && ( + + + Select a recording + + {recordings === null && Loading…} + {recordings != null && recordings.length === 0 && ( + + No recordings found under generated/real_sessions/. Start one from{" "} + /mission/live (Start Recording). + + )} +
+ {recordings?.map((r) => ( + + ))} +
+ + )} + + {recordingId && manifest && ( +
+ + + reviewed {reviewedCount} / {totalFrames} + + } + /> +
+ {/* eslint-disable-next-line @next/next/no-img-element -- local recorded frame, clicked for annotation, not an optimizable static asset */} + {`Recorded + {markers && currentTelemetry?.detectedXPx != null && currentTelemetry?.detectedYPx != null && ( + + )} + {markers && pendingCenter && ( + + )} +
+
+
+ + + setCursor(Math.max(0, Math.min(totalFrames - 1, (Number(e.target.value) || 1) - 1))) + } + className="w-20 border border-outline-variant bg-surface px-margin-sm text-center font-data-mono text-on-surface" + /> + +
+ + t={currentTelemetry ? fixed(currentTelemetry.timestampS, 2) : "—"}s + +
+ + + +
+ )} + + ); +} diff --git a/frontend/app/mission/live/page.tsx b/frontend/app/mission/live/page.tsx new file mode 100644 index 0000000..fcdc5c1 --- /dev/null +++ b/frontend/app/mission/live/page.tsx @@ -0,0 +1,497 @@ +"use client"; + +import { useEffect, useMemo, useRef, useState } from "react"; + +import Link from "next/link"; + +import { Screen } from "@/components/shell/AppShell"; +import { Panel, PanelHeader, KeyValueRow, StatusSquare } from "@/components/ui/Panel"; +import { deg, fixed, px } from "@/lib/format"; + +/** Button-styled link (not a real + + +
+ + + +
+ {lastEventLabel && !recordCommandError && ( + + last marked: {lastEventLabel} + + )} + {recordCommandError && ( + {recordCommandError} + )} + + + + )} + + ); +} diff --git a/frontend/app/mission/page.tsx b/frontend/app/mission/page.tsx index e7d331d..a78e7ab 100644 --- a/frontend/app/mission/page.tsx +++ b/frontend/app/mission/page.tsx @@ -21,9 +21,9 @@ export default function MissionControlPage() { return ( -
+
{/* optical feed */} -
+
{/* Stitch mini FPA overlay */} @@ -76,11 +76,11 @@ export default function MissionControlPage() {
{/* bottom band: pointing error + event log */} -
-
+
+
-
+
Event Log diff --git a/frontend/app/opengraph-image.tsx b/frontend/app/opengraph-image.tsx new file mode 100644 index 0000000..9f3e4dc --- /dev/null +++ b/frontend/app/opengraph-image.tsx @@ -0,0 +1,86 @@ +import { ImageResponse } from "next/og"; + +export const runtime = "edge"; +export const alt = "FSOC — AI-Based Virtual Camera Tracking"; +export const size = { width: 1200, height: 630 }; +export const contentType = "image/png"; + +const BG = "#0e1514"; +const BORDER = "#3c4947"; +const PRIMARY = "#6feee1"; +const TEXT = "#dee4e2"; +const MUTED = "#869491"; +const WARNING = "#ffd2a2"; + +export default async function Image() { + return new ImageResponse( + ( +
+
+
+
+ SIH26169 · TEAM IRODOV +
+
+ github.com/ThatKJ/FSOC +
+
+ +
+
+ FSOC +
+
+ AI-Based Virtual Camera Tracking +
+
+ for Mobile Free-Space Optical Communication +
+
+ +
+ SEE + → + ESTIMATE + → + PREDICT + → + CORRECT +
+
+ ), + { ...size }, + ); +} diff --git a/frontend/app/page.tsx b/frontend/app/page.tsx index 39844c2..6912ad1 100644 --- a/frontend/app/page.tsx +++ b/frontend/app/page.tsx @@ -23,9 +23,9 @@ export default function OverviewPage() { return ( -
+
{/* hero */} -
+
SIH26169 @@ -70,7 +70,7 @@ export default function OverviewPage() {
-
+
@@ -79,11 +79,11 @@ export default function OverviewPage() {
{/* optical viewport */} -
+
- {/* camera chips */} -
+ {/* camera chips -- secondary telemetry, collapsed on mobile to leave room for the label */} +
{[ ["FPS", `${SIM_RATE_HZ}.0`, "text-primary"], ["EXPOSURE", "15 ms", "text-on-surface"], diff --git a/frontend/app/robots.ts b/frontend/app/robots.ts new file mode 100644 index 0000000..c367292 --- /dev/null +++ b/frontend/app/robots.ts @@ -0,0 +1,8 @@ +import type { MetadataRoute } from "next"; + +export default function robots(): MetadataRoute.Robots { + return { + rules: { userAgent: "*", allow: "/" }, + sitemap: "https://fsoc-iota.vercel.app/sitemap.xml", + }; +} diff --git a/frontend/app/sitemap.ts b/frontend/app/sitemap.ts new file mode 100644 index 0000000..0b4eea1 --- /dev/null +++ b/frontend/app/sitemap.ts @@ -0,0 +1,22 @@ +import type { MetadataRoute } from "next"; + +const SITE_URL = "https://fsoc-iota.vercel.app"; + +const ROUTES = [ + "", + "/mission", + "/tracking", + "/world", + "/telemetry", + "/scenarios", + "/benchmarks", + "/validation", + "/architecture", +]; + +export default function sitemap(): MetadataRoute.Sitemap { + return ROUTES.map((route) => ({ + url: `${SITE_URL}${route}`, + lastModified: new Date(), + })); +} diff --git a/frontend/app/telemetry/page.tsx b/frontend/app/telemetry/page.tsx index f8ca9ba..1e76a6d 100644 --- a/frontend/app/telemetry/page.tsx +++ b/frontend/app/telemetry/page.tsx @@ -93,29 +93,30 @@ export default function TelemetryPage() { return ( {/* metrics bar */} -
+ {/* contained horizontal scroll at narrow widths -- never full-page scroll */} +
0 ? "warning" : "default"} /> {/* view switch */} -
+
{VIEWS.map((vw) => { const Icon = vw.icon; const active = vw.id === view; @@ -75,8 +75,9 @@ export default function WorldPage() { ))}
- {/* telemetry panel */} -