diff --git a/.superpowers/brainstorm/42128-1780857831/state/server-stopped b/.superpowers/brainstorm/42128-1780857831/state/server-stopped new file mode 100644 index 000000000..a653e3700 --- /dev/null +++ b/.superpowers/brainstorm/42128-1780857831/state/server-stopped @@ -0,0 +1 @@ +{"reason":"idle timeout","timestamp":1780860058366} diff --git a/README.md b/README.md index 91e69bd95..f34972299 100644 --- a/README.md +++ b/README.md @@ -18,12 +18,12 @@ OpenFlight is an open-source golf launch monitor that uses Doppler radar to meas ### What It Measures -- **Ball Speed**: 30-220 mph range with ±0.5% accuracy (OPS243-A) +- **Ball Speed**: 35-200 mph range with ±0.5% accuracy (OPS243-A) - **Club Speed**: Detected from pre-impact readings (OPS243-A) - **Smash Factor**: Ball speed / club speed ratio - **Launch Angle**: Vertical launch measured by K-LD7 angle radar - **Club Path**: Horizontal aim direction measured by second K-LD7 -- **Spin Rate**: Via rolling buffer I/Q analysis (~50-60% detection rate) +- **Spin Rate**: Via rolling buffer I/Q analysis (the hardest radar measurement — see [Limitations](#limitations)) - **Carry Distance**: Computed from ball speed, launch angle, and spin ### Hardware at a Glance @@ -33,7 +33,7 @@ OpenFlight is an open-source golf launch monitor that uses Doppler radar to meas | OPS243-A Radar | Ball speed, club speed, spin | $249 | | Raspberry Pi 5 | Runs everything | $60 | | 7" Touchscreen | Shows shot data | $46 | -| SparkFun SEN-14262 | Sound trigger for spin detection | $18 | +| SparkFun SEN-14262 | Impact sound trigger for shot capture | $18 | | K-LD7 (×2) + FTDI adapters | Launch angle + club path | $140 | | Power supply + accessories | | $27 | | **Total** | | **~$540** | @@ -52,7 +52,19 @@ Follow the **[Sound Trigger Wiring Guide](docs/sound-trigger-wiring.md)** to con ### 3. Set up the Pi -Follow the **[Raspberry Pi Setup Guide](docs/raspberry-pi-setup.md)** for OS installation, software setup, radar configuration, and auto-start. +Flash Raspberry Pi OS (64-bit), plug in the radars, then run the interactive setup: + +```bash +git clone https://github.com/jewbetcha/openflight.git +cd openflight +./scripts/setup/setup.sh +``` + +The script installs everything and walks you through the one-time hardware +configuration (radar flash setup, K-LD7 device naming, auto-start) with +prompts — no manual config editing needed. It's safe to re-run any time. +See the **[Raspberry Pi Setup Guide](docs/raspberry-pi-setup.md)** for +details and troubleshooting. ### 4. Hit balls @@ -69,6 +81,24 @@ scripts/start-kiosk.sh --mock Then open http://localhost:8080 or use the touchscreen. +### 5. Sync to the cloud (optional) + +OpenFlight can push your sessions to the **FlightWeb** cloud so you can review +shots from any device. It's opt-in, and **raw radar data never leaves your +Pi** — only shot results and session metadata are uploaded (verify with +`openflight-cloud push --dry-run`). + +`setup.sh` offers to enable this and link your Pi. To do it by hand: + +```bash +openflight-cloud link # pair this Pi (enter a short code in your browser) +openflight-cloud status # linked? queued? parked? +``` + +Once linked, sessions sync automatically (on session end and via a ~10-minute +timer that heals wifi outages). See the **[Cloud Sync Guide](docs/cloud-sync.md)** +for details. + ### TV Display Mode OpenFlight also serves a fullscreen-friendly browser display for tablets, TV browsers, or a Chrome tab cast to Chromecast. @@ -86,9 +116,10 @@ This is browser/tab casting only. OpenFlight does not include native Cast SDK su ``` ┌─────────────┐ USB/Serial ┌─────────────┐ Callback ┌─────────────┐ WebSocket ┌─────────────┐ -│ OPS243-A │ ───────────▶ │ Launch │ ──────────▶ │ Flask │ ──────────▶ │ React │ -│ Radar │ Speed data │ Monitor │ on_shot() │ Server │ "shot" │ UI │ -└─────────────┘ └─────────────┘ └─────────────┘ └─────────────┘ +│ OPS243-A │ ───────────▶ │ Rolling │ ──────────▶ │ Flask │ ──────────▶ │ React │ +│ Radar │ I/Q buffer │ Buffer │ on_shot() │ Server │ "shot" │ UI │ +└─────────────┘ │ Monitor │ └─────────────┘ └─────────────┘ + └─────────────┘ ▲ ┌─────────────┐ USB/Serial │ │ K-LD7 (×2) │ ──────────────────── angle data ──────────────┘ @@ -123,32 +154,41 @@ The K-LD7 modules are positioned near the OPS243-A, one mounted vertically (laun ### Radar Settings for Golf -| Setting | Value | Why | -| ----------- | -------- | --------------------------- | -| Sample Rate | 20 kHz | Supports up to ~139 mph | -| Buffer Size | 512 | Faster updates (~10-15 Hz) | -| Min Speed | 10 mph | Filter slow movements | -| Direction | Outbound | Ball moving away from radar | -| Power | Max (0) | Best detection range | +| Setting | Value | Why | +| -------------- | ---------------------- | -------------------------------------------- | +| Mode | Rolling buffer | Raw I/Q capture for spin + precise speeds | +| Sample Rate | 30 ksps | Supports up to ~208 mph ball speed | +| Capture | 4096 I/Q samples | ~136 ms around impact | +| Trigger | Sound (SEN-14262) | ~10 µs hardware latency via HOST_INT | +| Min Ball Speed | 35 mph | Filter club waggle and slow movements | +| DC Mask | ~15 mph exclusion zone | Reject body movement and environmental noise | + +These are applied automatically — the one-time flash configuration is handled +by the setup script. ### Python API ```python -from openflight import LaunchMonitor +from openflight.rolling_buffer import RollingBufferMonitor + +monitor = RollingBufferMonitor() # auto-detects the OPS243-A +monitor.connect() +monitor.start() -with LaunchMonitor() as monitor: - print("Swing when ready...") - shot = monitor.wait_for_shot(timeout=60) +print("Swing when ready...") +shot = monitor.wait_for_shot(timeout=60) +if shot: + print(f"Ball Speed: {shot.ball_speed_mph:.1f} mph") + print(f"Est. Carry: {shot.estimated_carry_yards:.0f} yards") - if shot: - print(f"Ball Speed: {shot.ball_speed_mph:.1f} mph") - print(f"Est. Carry: {shot.estimated_carry_yards:.0f} yards") +monitor.stop() +monitor.disconnect() ``` ## Limitations - **Cosine error**: If ball doesn't travel directly toward/away from radar, measured speed will be slightly lower than actual -- **Spin detection**: ~50-60% reliable — depends on signal quality and trigger timing +- **Spin detection**: The hardest radar measurement, especially indoors — the usable signal window ends when the ball hits the net, and short windows can't resolve low spin (commercial radar units have the same constraint and fall back to estimated spin indoors). Low driver-band readings (≤~3100 RPM) are reported at reduced confidence. When spin isn't measured, carry falls back to club-typical spin values. Improving this is an active focus. - **K-LD7 speed aliasing**: The K-LD7 max speed is 62 mph, so it's used only for angle/distance, not speed ### Ball Markings @@ -183,7 +223,9 @@ openflight/ │ ├── server.py # Flask server, K-LD7 correlation, carry │ ├── session_logger.py # JSONL session logging │ ├── kld7/ # K-LD7 angle radar -│ │ ├── tracker.py # Ring buffer, ball/club detection +│ │ ├── radc.py # FFT, phase interferometry, angle extraction +│ │ ├── tracker.py # Ring buffer, shot correlation +│ │ ├── geometry.py # Launch-angle trajectory fitting │ │ └── types.py # Data types │ └── rolling_buffer/ # Spin rate detection │ ├── monitor.py # Rolling buffer monitor @@ -202,7 +244,7 @@ Contributions welcome! See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines. Areas of interest: -- **Better spin detection**: Increase reliability beyond 50-60% +- **Better spin detection**: A dechirped Doppler-sideband estimator is in development (`scripts/analysis/replay_spin_dechirp.py`) — help validating it against launch-monitor truth data is especially welcome - **K-LD7 signal processing**: Improve ball detection from sparse radar frames - **Mobile app**: Bluetooth connection to phone @@ -217,7 +259,9 @@ uv run pytest tests/ -v - **[Parts List](docs/PARTS.md)** — What to buy - **[Sound Trigger Wiring](docs/sound-trigger-wiring.md)** — How to wire the sound trigger - **[Raspberry Pi Setup](docs/raspberry-pi-setup.md)** — Full setup guide +- **[Cloud Sync](docs/cloud-sync.md)** — Push filtered sessions to FlightWeb - **[Rolling Buffer & Spin Detection](docs/rolling_buffer_spin_detection.md)** — Spin measurement details +- **[Dechirped-Sideband Spin Replay](docs/spin-dechirp-replay.md)** — Next-gen spin estimator test bench - **[K-LD7 Ball Detection Theory](docs/kld7-ball-detection-theory.md)** — How angle detection works - **[K-LD7 Session Review](docs/kld7-session-review.md)** — Offline review workflow for session JSONL files - **[Observability & Log Shipping](docs/observability.md)** — Ship logs to Grafana Cloud diff --git a/docs/cloud-sync-design.md b/docs/cloud-sync-design.md new file mode 100644 index 000000000..b23ad4f47 --- /dev/null +++ b/docs/cloud-sync-design.md @@ -0,0 +1,178 @@ +# OpenFlight Cloud Sync — Client-Side Design (Proposal) + +Status: **draft / not implemented**. Design for the two pieces of the planned +cloud service that live in this (public) repo: the ingest API contract the Pi +speaks, and the uploader that speaks it. The service itself lives in a +separate private repo; this document is the interface between them. + +Design priorities, in order: + +1. **The contract is forever.** Fielded Pis update rarely. The wire format + must tolerate old clients indefinitely and evolve additively. +2. **Never upload raw ADC/I-Q.** Filtered session summaries only — keeps + storage costs sane, uploads fast on bad wifi, and is the + privacy-friendly default. +3. **No user babysitting.** Sessions upload themselves when connectivity + exists; "no wifi at the range" is a non-event, not a manual step. + +--- + +## 1. Ingest API contract (v1) + +All endpoints under `https:///v1/`, TLS only. Authentication is +an opaque per-device bearer token (`of_device_...` prefix) — revocable from +the web app, scoped to one device. + +### Device linking (how a Pi gets its token) + +Simplified RFC 8628 device-code flow — the user never copies a long token +onto the Pi: + +``` +Pi Cloud +── ───── +POST /v1/device-link/start + {device_name, client_version} ─────▶ + ◀───── {link_code: "ABCD-1234", + poll_token, interval_s, expires_s} + + (Pi prints: "Go to cloud.openflight.example/link + and enter code ABCD-1234") + +POST /v1/device-link/poll + {poll_token} (repeat) ─────▶ + ◀───── {status: "pending"} + ◀───── {status: "linked", + device_token, device_id} +``` + +The user enters the code on the website while signed in. Token is stored at +`~/.config/openflight/cloud.json`, mode `0600`, never logged. + +### Session upload + +``` +PUT /v1/sessions/{session_id} + Authorization: Bearer + Content-Type: application/x-ndjson + Content-Encoding: gzip + + +``` + +- **`session_id` is the `session_uuid` embedded in the `session_start` + entry** (a UUID4 written at session creation since format_version 1 / + app 0.2.0 — survives file renames and copies). For older sessions that + predate the field, the uploader falls back to a deterministic UUIDv5 of + `(device_id, session filename)`. Either way the auto-push and the manual + push script can both submit the same session and the server dedupes for + free; PUT semantics make retries safe. +- The client prepends one manifest line to the body: + + ```json + {"type": "upload_manifest", "format_version": 1, "client_version": "0.2.0", + "device_id": "...", "filtered": true, "kept_entry_types": ["session_start", ...]} + ``` + +**Responses:** + +| code | meaning | client behavior | +|---|---|---| +| 201 | accepted (`{session_id, shot_count}`) | mark pushed | +| 200 | duplicate — already stored | mark pushed | +| 401 | token invalid/revoked | stop, flag "needs re-link" | +| 402 | quota/entitlement exceeded | park, retry daily | +| 413 | body too large (cap ~20 MB gzipped) | log error, park file | +| 422 | unparseable (`{reason}`) | log error, park file | +| 429 | rate limited (`Retry-After`) | back off | +| 5xx | server trouble | retry with backoff | + +`GET /v1/health` → 200, used by the uploader to short-circuit when offline. + +### Evolution rules + +- The server must **accept and store unknown entry types** (skip parsing, + don't reject) — old and new clients coexist for years. +- Additive changes only within `/v1`; breaking changes get `/v2` and `/v1` + keeps working. + +--- + +## 2. Client-side filtering (the raw-ADC strip) + +Filtering uses an **allowlist**, not a blocklist — a future heavy entry type +added to the session logger can never leak to the cloud by accident: + +``` +keep: session_start, session_end, shot_detected, trigger_event, session_error +drop: rolling_buffer_capture, iq_blocks, iq_reading, reading_accepted, + and anything not on the keep list +guard: any kept line > 32 KB is dropped and counted (belt and suspenders) +``` + +`shot_detected` carries everything the insights product needs (speeds, spin ++ quality/confidence, angles, carry, K-LD7 diagnostics). `trigger_event` and +`session_error` are small and power reliability insights. Raw I/Q and RADC +stay on the Pi where they belong — they remain available locally for the +offline analysis workflows. + +A typical filtered session is **tens of KB gzipped** vs tens of MB raw. + +--- + +## 3. The uploader (spool-and-retry) + +Lives in the public repo. Three entry points, one mechanism: + +``` +openflight-cloud link # device-link flow (one time) +openflight-cloud push [--dry-run] # filter + upload anything unpushed +openflight-cloud status # linked? queued? parked? last error? +``` + +**Mechanism:** + +- The session directory itself is the queue. A session counts as "pushed" + when a sidecar marker (`.jsonl.pushed`) exists — originals are + never moved or modified, state survives crashes, and no database is + involved. +- A **systemd timer** (every ~10 min) runs `push`; the server process also + fires a non-blocking `push` on session end. The timer makes wifi outages + self-healing; the hook makes the happy path fast. Neither can ever delay + shot processing. +- Per-file attempt counter (in the sidecar); after ~20 failures the file is + parked (`.parked`) and reported by `status` instead of retried forever. +- `--dry-run` prints exactly which entry lines would upload — the privacy + answer to "what are you sending?" + +**Config** (`~/.config/openflight/cloud.json`): + +```json +{"endpoint": "https://cloud.openflight.example", + "device_token": "of_device_...", "device_id": "...", "enabled": true} +``` + +Uploading is **opt-in**: nothing leaves the Pi until the user runs +`openflight-cloud link`. Later, the interactive `setup.sh` can offer linking +as an optional step. + +--- + +## 4. Server-side checklist (private repo, for reference) + +Not designed here, but the contract above implies: the four endpoints; +dedupe on `session_id`; entitlement check at ingest (quota → 402); +blob → object storage; parse kept entries → per-shot rows in Postgres; +device management UI (list/revoke); link-code UI. + +## Open questions + +- Tier gating: does the free tier get full history or last-N sessions? + (Affects only server; contract unchanged.) +- Should `iq_reading` summaries (SNR stats, no raw data) join the allowlist + later for radar-health insights? Cheap to add — allowlist makes it an + explicit decision. +- AGPL hygiene: the uploader (public repo) is AGPL like everything here; + the private service must not import code from this repo unless that code + is dual-licensed or contributor-cleared. Keep the boundary at the wire + contract. diff --git a/docs/cloud-sync.md b/docs/cloud-sync.md new file mode 100644 index 000000000..3c1debece --- /dev/null +++ b/docs/cloud-sync.md @@ -0,0 +1,163 @@ +# Cloud Sync + +OpenFlight can push your session logs to the **FlightWeb** cloud so you can +review shots from any device. It's an opt-in `openflight-cloud` CLI that runs +on the Pi. + +> **Privacy promise — raw radar data never leaves your Pi.** The uploader +> applies an allowlist *before* upload: only shot results and session metadata +> are sent. Raw I/Q captures, rolling-buffer dumps, and per-reading detections +> stay local. The server stores exactly what the Pi sends — so the filter is +> enforced here, on the device. Run `openflight-cloud push --dry-run` any time +> to see precisely what would be uploaded. + +## Quick start + +```bash +openflight-cloud link # one-time: pair this Pi with your account +openflight-cloud status # linked? queued? parked? last error? +openflight-cloud push --dry-run # show exactly which entries would upload +openflight-cloud push # filter + upload anything not yet pushed +openflight-cloud push --retry # also re-attempt parked/failed sessions +``` + +`scripts/setup/setup.sh` offers to enable cloud sync and link the Pi for you +(on a Raspberry Pi). Everything below can also be done by hand. + +> **Upgrading an existing install?** The `openflight-cloud` command is created +> at install time. If you added cloud sync by pulling new code into a venv that +> predates it, reinstall so the console script gets wired up: +> +> ```bash +> uv pip install -e . +> ``` +> +> Until then you can run it as a module: `python -m openflight.cloud.cli link`. + +## Linking a device + +You never copy a long token onto the Pi. Linking uses a short, screen-readable +code (RFC 8628 device flow): + +1. Run `openflight-cloud link`. It prints something like: + + ``` + Go to https://flightweb.fly.dev/link and enter code: ABCD-2345 + + (waiting up to 900s; sign in and enter the code) + ``` + +2. Open that URL in any browser, sign in, and enter the code. +3. The Pi detects the pairing, saves its device token, and enables uploads: + + ``` + Linked! device_id=… . Uploads are now enabled. + ``` + +The code expires after ~15 minutes. If it expires or you mistype, just re-run +`openflight-cloud link`. + +To name the device (defaults to the Pi's hostname): + +```bash +openflight-cloud link --device-name "garage pi" +``` + +## How uploads happen + +Two triggers, one mechanism — both are safe to run at any time and never block +shot processing: + +- **systemd timer** (`openflight-cloud.timer`, ~every 10 min) runs `push`. This + is the safety net that heals wifi outages. +- **On session end**, the server fires a non-blocking `push` for the fast happy + path. If it fails (offline, etc.), the timer picks it up later. + +**The session directory is the queue.** No database is involved — state lives +in sidecar files next to each `session_*.jsonl`, so it survives reboots and +crashes: + +| Sidecar | Meaning | +|---|---| +| `.jsonl.pushed` | Successfully uploaded (won't be sent again). | +| `.jsonl.parked` | Given up on — see `reason`; reported by `status`. | +| `.jsonl.state` | In-flight retry counter / quota cooldown. | + +Originals are never moved or modified. Uploads are idempotent: re-sending a +session that's already stored is safe and dedupes server-side. + +### What gets retried, what gets parked + +| Situation | Behavior | +|---|---| +| Network down / server 5xx | Retried on the next timer tick (exponential per-file backoff); parked after ~20 failures. | +| Rate limited (429) | Backs off for the server-provided interval, then retries. | +| Quota exceeded (402) | Deferred ~24h, then retried automatically. | +| Token rejected (401) | All uploads stop and `status` flags **needs re-link** — run `openflight-cloud link` again. | +| Rejected as malformed/too large (413/422) | Parked (this indicates a client bug and shouldn't happen with correct filtering). | + +## Config + +Stored at `~/.config/openflight/cloud.json`, mode `0600` (the `device_token` is +a bearer credential — keep it secret): + +```json +{ + "endpoint": "https://flightweb.fly.dev", + "device_token": "of_device_…", + "device_id": "…", + "enabled": true +} +``` + +- Written by `link`; read by `push` / `status`. +- Set `"enabled": false` (or delete the file) to turn the uploader into a no-op + without unlinking. +- `endpoint` is configurable in case the production domain moves. + +## systemd units + +Installed by `setup.sh`, or by hand: + +```bash +sudo cp scripts/setup/openflight-cloud.{service,timer} /etc/systemd/system/ +# edit User= and the paths if your install isn't /home/coleman/openflight +sudo systemctl daemon-reload +sudo systemctl enable --now openflight-cloud.timer +``` + +Inspect it: + +```bash +systemctl status openflight-cloud.timer # next run time +journalctl -u openflight-cloud.service # upload logs +``` + +The timer is harmless before you link — uploads stay off until a device token +exists. + +## Troubleshooting + +- **`status` says "not linked"** — run `openflight-cloud link`. +- **"needs re-link" / 401** — the token was revoked or rotated. Re-run + `openflight-cloud link`; the new token replaces the old one in the config. +- **A session is parked** — `status` shows the reason and last error. Parked + sessions are skipped on future runs. Re-attempt all parked/deferred sessions + with `openflight-cloud push --retry`. +- **A session uploaded with 0 shots (or you want to re-send a stored one)** — + `status` flags 0-shot uploads by name. Force a re-upload of a specific + session (even one already marked pushed) with + `openflight-cloud push --retry ` — `` is the + filename or any substring of it. Re-uploads are idempotent (the server + dedupes), so this is always safe. +- **Nothing uploads / "Nothing to upload"** — everything is already pushed. + Confirm `enabled: true` and that the Pi is online (`status` reports + reachability). To re-send a session that was already pushed, use + `push --retry ` as above. +- **"What is it sending?"** — `openflight-cloud push --dry-run` lists every + entry type and count that would be uploaded, and what's dropped. + +## See also + +- [`docs/openflight-cloud-uploader-spec.md`](openflight-cloud-uploader-spec.md) + — the wire contract this client implements (endpoints, status codes, caps). diff --git a/docs/kld7-troubleshooting.md b/docs/kld7-troubleshooting.md index d85c7e9df..19977fb59 100644 --- a/docs/kld7-troubleshooting.md +++ b/docs/kld7-troubleshooting.md @@ -158,9 +158,13 @@ Other possibilities: **Cause:** The K-LD7 uses an FTDI USB-to-serial adapter which shows up as `/dev/ttyUSB*`. If multiple USB-serial devices are connected, auto-detection may pick the wrong one. -**Fix:** Specify the port explicitly or set up udev rules: +**Fix:** Run the device naming wizard so each radar gets a fixed name, or +specify the port explicitly: ```bash -# Explicit port +# Recommended: map the radars to /dev/kld7_vertical / /dev/kld7_horizontal +./scripts/setup/setup_kld7_devices.sh + +# Or: explicit port scripts/start-kiosk.sh --kld7 --kld7-port /dev/ttyUSB0 # Find available ports @@ -174,6 +178,27 @@ If `--kld7` or `--kld7-horizontal` is passed but the radar fails to connect afte ## RADC Streaming Issues +### Measuring real K-LD7 RADC cadence + +Use the guarded timing probe when launch-angle extraction is missing frames or +when one K-LD7 orientation appears slower than the other: + +```bash +uv run python scripts/hardware-test/probe_kld7_timing.py \ + --port /dev/kld7_vertical \ + --duration 10 \ + --frame-mask RADC,DONE \ + --output /tmp/kld7_vertical_timing.jsonl +``` + +At the production `RSPI=3` setting, expect roughly 34 RADC frames per second +with low `done_frame_gaps`. If cadence is much lower or gaps are high, +investigate USB scheduling, serial read duration, or requested packet volume +before changing launch-angle selection logic. + +Undocumented command probing is available only through `--unsafe-probe` and +requires `--output`. Do not use it in production sessions. + ### No RADC frames in buffer **Symptom:** K-LD7 connects but shots show `angle_source: estimated` instead of `radar`. diff --git a/docs/openflight-cloud-uploader-spec.md b/docs/openflight-cloud-uploader-spec.md new file mode 100644 index 000000000..f261ce8e7 --- /dev/null +++ b/docs/openflight-cloud-uploader-spec.md @@ -0,0 +1,272 @@ +# `openflight-cloud` Uploader — Implementation Spec + +**Audience:** an agent building the uploader in the public **openflight** repo +(AGPL-3.0). **Author:** the FlightWeb server team. This describes the wire +contract **as the server actually implements it today** (verified against the +deployed code, not the original draft). It supersedes the speculative parts of +`openflight/docs/cloud-sync-design.md`; where they differ, this wins. + +> Drop this file into the openflight repo (e.g. `docs/openflight-cloud-uploader-spec.md`). +> Nothing here requires importing FlightWeb code — the wire contract *is* the +> boundary between the two repos. + +--- + +## 0. What you're building + +A small CLI, `openflight-cloud`, that lives on the Pi and pushes filtered +session logs to the FlightWeb cloud. Three subcommands, one spool-and-retry +mechanism: + +``` +openflight-cloud link # one-time device pairing +openflight-cloud push [--dry-run] # filter + upload anything unpushed +openflight-cloud status # linked? queued? parked? last error? +``` + +**The single most important rule:** the server stores the device-uploaded blob +**verbatim** — it does *not* re-filter raw radar data out of a device upload. +Therefore **the uploader MUST apply the allowlist filter (§4) before upload.** +The product promise "raw radar data never leaves your Pi" is enforced *here*, +in this client. (The server does filter *manual web uploads* as defense in +depth, but that path is irrelevant to the Pi.) + +--- + +## 1. Endpoint + transport + +- Base URL: configurable; current deployment **`https://flightweb.fly.dev`**. + Store it in config — the production domain may move (openflight vs. flightweb + is undecided). All paths below are under `/v1`. +- TLS only. Bearer auth for session upload; link endpoints are unauthenticated. +- Request/response bodies are JSON except the session-upload body, which is + gzipped NDJSON. +- The server returns `Retry-After` (seconds) on every `429`. + +--- + +## 2. Device linking (RFC 8628-style) + +The user never copies a long token onto the Pi. Flow: + +### 2a. `POST /v1/device-link/start` + +Unauthenticated. Rate limited **per IP: 10 requests / 15 min** (→ `429`). + +Request: +```json +{ "device_name": "garage pi", "client_version": "0.3.0" } +``` +- `device_name`: **required, 1–64 chars** (trimmed). Missing/blank/too-long → `422`. +- `client_version`: optional string. + +Response `200`: +```json +{ "link_code": "ABCD-2345", "poll_token": "", "interval_s": 5, "expires_s": 900 } +``` +- `link_code` format: `^[ABCDEFGHJKLMNPQRSTUVWXYZ]{4}-[2-9]{4}$` (no I/O/0/1 — + built to be read off a screen). Print it for the user: + `Go to /link and enter code ABCD-2345`. +- `poll_token`: high-entropy opaque string; **persist it** for polling. Treat as + a secret (don't log). +- `interval_s` (5): minimum seconds between polls — **honor it**. +- `expires_s` (900): code/poll lifetime (~15 min). + +### 2b. User action (out of band) + +The user opens `/link` in a browser, signs in, and enters the code. + +### 2c. `POST /v1/device-link/poll` + +Rate limited **per poll_token: 30 / 60s** (→ `429`). Poll no faster than +`interval_s`. + +Request: +```json +{ "poll_token": "" } +``` + +Responses (all `200` unless noted): +| Body | Meaning | Client action | +|---|---|---| +| `{"status":"pending"}` | not entered yet | keep polling at `interval_s` | +| `{"status":"expired"}` | code lifetime passed | stop; tell user to re-run `link` | +| `{"status":"linked","device_token":"of_device_…","device_id":""}` | paired | **save token + id, stop polling** | +| `404 {"reason":"unknown_poll_token"}` | unknown OR already consumed | stop; re-run `link` | +| `429` + `Retry-After` | polling too fast | back off | + +**The `linked` response is returned exactly once.** The first successful poll +after the user enters the code issues the token; any subsequent poll with the +same `poll_token` gets `404`. So persist `device_token` + `device_id` +**atomically on first receipt** — if you crash between receiving and saving, +the user must re-link. + +- `device_token` format: `^of_device_[0-9A-Za-z]{32}$`. The server stores only a + SHA-256 hash; this plaintext is shown **once**. + +--- + +## 3. Session upload + +### `PUT /v1/sessions/{session_id}` + +``` +PUT /v1/sessions/1f0e9c2a-7b3d-4e5f-8a9b-0c1d2e3f4a5b +Authorization: Bearer of_device_<…> +Content-Type: application/x-ndjson +Content-Encoding: gzip + + +``` + +Rate limited **per device: 120 / hour** (→ `429`). A full backlog flush stays +well under this; if you somehow hit it, honor `Retry-After`. + +**`{session_id}`** must be a lowercase UUID (server lowercases anyway, but send +lowercase). It is: +- the **`session_uuid`** from the `session_start` entry (UUID4, present since + openflight 0.2.0 / format_version 1) — **preferred**; or +- for older sessions lacking it, a deterministic **UUIDv5 of + `(device_id, session_filename)`** so the same file always maps to the same id + (free dedupe + safe retries). + +If the body contains a `session_start` with a `session_uuid`, it **must equal** +the URL id (case-insensitive) or you get `422 session_uuid_mismatch`. Simplest: +always use the embedded `session_uuid` as the URL id when present. + +**Body:** gzipped NDJSON, filtered per §4, with a manifest first line (§4). +PUT is idempotent — re-uploading the same session is safe and dedupes. + +### Responses → client behavior + +| Code | Body | Meaning | Client action | +|---|---|---|---| +| `201` | `{session_id, shot_count}` | accepted, stored | mark `.pushed` | +| `200` | `{session_id, shot_count}` | duplicate (already stored) | mark `.pushed` | +| `401` | `{reason}` | bad/revoked/missing token (`missing_or_malformed_token` \| `invalid_or_revoked_token`) | stop all uploads; flag "needs re-link" | +| `402` | `{reason:"quota_exceeded"}` | over entitlement | park; retry daily | +| `413` | `{reason}` | too large (`body_too_large` gz>20 MB, or `inflated_too_large` >64 MB) | log + park (should never happen if §4 filtering works) | +| `422` | `{reason}` | unparseable (`invalid_session_id`, `session_uuid_mismatch`, `invalid_gzip`, `no_valid_jsonl`) | log + park; this is a client bug — surface it | +| `429` | `{reason:"rate_limited"}` + `Retry-After` | rate limited | back off `Retry-After` seconds | +| `5xx` | — | server trouble | retry with exponential backoff | + +`200` and `201` are both success — treat identically (mark pushed). Don't +distinguish them for retry purposes. + +### `GET /v1/health` + +`200 {"status":"ok"}`, unauthenticated, no side effects. Use it to +short-circuit `push` when offline (cheap connectivity probe before doing work). + +--- + +## 4. Client-side filtering (the raw-ADC strip) — REQUIRED + +Build the upload body by transforming the session `.jsonl`: + +1. **Prepend one manifest line** as the first line: + ```json + {"type":"upload_manifest","format_version":1,"client_version":"0.3.0", + "device_id":"","filtered":true,"kept_entry_types":[...]} + ``` +2. **Keep only allowlisted entry types** (filter by each line's `type`): + ``` + KEEP: session_start, session_end, shot_detected, trigger_event, session_error + DROP: rolling_buffer_capture, iq_blocks, iq_reading, reading_accepted, + ops_clock_sync, shot_camera, config_change, and ANYTHING not on the keep list + ``` + Use an **allowlist, not a blocklist** — a future heavy entry type the session + logger gains must never leak by default. +3. **Drop any kept line > 32 KB** (and count it) — belt-and-suspenders guard + matching the server's per-line cap. +4. **gzip** the result. Keep it under **20 MB gzipped / 64 MB inflated** (a + filtered session is normally tens of KB, so this is just a safety check — + if a session somehow exceeds it, park the file and report it rather than + uploading raw). + +`--dry-run` should print exactly which entry lines *would* upload (the privacy +answer to "what are you sending?"). The server matches this allowlist exactly, +so anything you keep here is what gets stored. + +> Why this is load-bearing: the server stores the device blob **as received**. +> Whatever you upload is what lives in the cloud. Filtering is not optional. + +--- + +## 5. The uploader mechanism (spool-and-retry) + +- **The session directory is the queue.** A session counts as pushed when a + sidecar marker `".jsonl.pushed"` exists. Never move or modify + originals — state survives crashes, no database needed. +- **Triggers:** a **systemd timer** (~every 10 min) runs `push` (heals wifi + outages); the server process also fires a non-blocking `push` on session end + (fast happy path). Neither may ever block/delay shot processing. +- **Per-file attempt counter** in the sidecar; after ~20 failures, **park** the + file (`".jsonl.parked"`) and report via `status` instead of + retrying forever. +- **Terminal vs. retryable** (see §3 table): `401` stops everything and flags + re-link; `402` parks for daily retry; `413`/`422` park (client bug — these + shouldn't happen with correct filtering); `429`/`5xx` back off and retry. +- Push is **opt-in**: nothing leaves the Pi until the user runs + `openflight-cloud link`. + +--- + +## 6. Config — `~/.config/openflight/cloud.json`, mode `0600` + +```json +{ "endpoint": "https://flightweb.fly.dev", + "device_token": "of_device_…", + "device_id": "…", + "enabled": true } +``` +- `0600`, never logged. `device_token` is a bearer credential. +- `enabled:false` (or file absent) → uploader is a no-op. +- Written by `link` on success; read by `push`/`status`. + +--- + +## 7. Evolution rules (so old Pis keep working forever) + +- The server **accepts and stores unknown entry types** in the blob (it skips + parsing them, doesn't reject) — but you should still filter to the allowlist + so raw data doesn't leak. +- The server **accepts unknown fields** on known entry types. +- `/v1` is append-only and tolerates old clients indefinitely. Breaking changes + would ship as `/v2`; `/v1` keeps working. So a fielded Pi that never updates + keeps uploading forever. +- Send a real `client_version` in `device-link/start` and the manifest — it's + stored per-device and helps the server team support old clients. + +--- + +## 8. Reference values (verified against deployed server) + +| Thing | Value | +|---|---| +| Link code regex | `^[ABCDEFGHJKLMNPQRSTUVWXYZ]{4}-[2-9]{4}$` | +| Device token regex | `^of_device_[0-9A-Za-z]{32}$` | +| Poll interval / code TTL | 5 s / 900 s | +| Rate: link-start | 10 / 15 min per IP | +| Rate: poll | 30 / 60 s per poll_token | +| Rate: upload | 120 / hour per device | +| Caps | 20 MB gzipped, 64 MB inflated, 32 KB per line | +| Allowlist (keep) | session_start, session_end, shot_detected, trigger_event, session_error (+ upload_manifest) | +| Health | `GET /v1/health` → `200 {"status":"ok"}` | + +--- + +## 9. Suggested build order + +1. `link` against `/v1/device-link/{start,poll}` → write config. Verify a real + device row appears (the user will see it on the FlightWeb Devices page). +2. Filtering + manifest (§4) with `--dry-run` first — get the body exactly right + before uploading anything. +3. `push` for a single session → confirm `201`, then re-run → confirm `200` + duplicate. +4. Spool/sidecar/park mechanics + systemd timer. +5. `status` reporting. + +**Exit test (shared with the server team):** a real Pi runs `link`, the user +enters the code, a session uploads, and it appears on FlightWeb. That closes +the Phase 1 loop. diff --git a/docs/raspberry-pi-setup.md b/docs/raspberry-pi-setup.md index b6e87e9c1..c3f90c118 100644 --- a/docs/raspberry-pi-setup.md +++ b/docs/raspberry-pi-setup.md @@ -17,79 +17,110 @@ Make sure you have all the hardware. See the **[Parts List](PARTS.md)** for what **Optional:** - K-LD7 + FTDI adapter (×2) — for launch angle and club path (see [Parts List](PARTS.md)) -## Initial Setup +## Setup ### 1. Install Raspberry Pi OS Use Raspberry Pi Imager to flash **Raspberry Pi OS (64-bit)** to your SD card. -### 2. Clone and Install +### 2. Run the setup script + +Plug in the OPS243-A (and the K-LD7 adapters if you have them), then: ```bash cd ~ git clone https://github.com/jewbetcha/openflight.git cd openflight - -# Run the setup script (handles everything) ./scripts/setup/setup.sh ``` -The setup script will: -- Create a Python virtual environment -- Install all Python dependencies -- Install Node.js dependencies -- Build the UI -- Run tests to verify installation +The script installs everything, then walks you through the one-time hardware +configuration with prompts: + +1. **Dependencies** — Python venv, packages, UI build, test run +2. **OPS243-A radar** — saves rolling buffer mode to the radar's flash + (you'll be asked to unplug/replug the radar once) +3. **K-LD7 radars** (if you have them) — identifies each radar by plugging + them in one at a time, so OpenFlight always knows which is which +4. **Auto-start on boot** — optional systemd service +5. **Desktop shortcut** — optional -Or manually: +Every step can be skipped and the script is **safe to re-run** any time — +it picks up where you left off. + +### 3. Start hitting balls ```bash -curl -LsSf https://astral.sh/uv/install.sh | sh -python -m venv .venv --system-site-packages -source .venv/bin/activate -uv pip install -e ".[ui]" -cd ui && npm install && npm run build && cd .. +./scripts/start-kiosk.sh # Default: rolling buffer + sound trigger +./scripts/start-kiosk.sh --kld7 # With K-LD7 angle radars +./scripts/start-kiosk.sh --mock # Mock mode (no hardware) ``` -## Radar Setup (One-Time) +Then open `http://localhost:8080` or use the touchscreen. + +--- -The OPS243-A needs a one-time configuration to enable rolling buffer mode with hardware sound triggering. This saves settings to flash memory so it boots in the correct mode every time. +## What the Script Configures (Reference) -> **Why?** The OPS243-A has a firmware bug where the HOST_INT pin mode switches unexpectedly when entering rolling buffer mode at runtime. Saving to flash and power cycling bypasses this. Confirmed by OmniPreSense engineering. +You don't need this section unless something went wrong or you prefer to do +things by hand. -### 1. Configure and Save +### OPS243-A Rolling Buffer Mode + +The OPS243-A needs a one-time configuration to enable rolling buffer mode with +hardware sound triggering, saved to flash so it boots correctly every time. + +> **Why?** The OPS243-A has a firmware bug where the HOST_INT pin mode switches +> unexpectedly when entering rolling buffer mode at runtime. Saving to flash and +> power cycling bypasses this. Confirmed by OmniPreSense engineering. + +
+Manual steps ```bash +# 1. Configure and save to flash uv run python scripts/hardware-test/test_rolling_buffer_persist.py --setup + +# 2. Power cycle: unplug the radar's USB cable, wait 3 seconds, plug back in + +# 3. Verify — make a sound near the SEN-14262, you should see I/Q trigger data +uv run python scripts/hardware-test/test_rolling_buffer_persist.py --test ``` -### 2. Power Cycle +
-Unplug the radar's USB cable, wait 3 seconds, plug it back in. +### K-LD7 Device Names -### 3. Verify +USB serial adapters can swap between `/dev/ttyUSB0` and `/dev/ttyUSB1` after a +reboot, so OpenFlight needs fixed names (`/dev/kld7_vertical` and +`/dev/kld7_horizontal`) to tell the two radars apart. The wizard handles this — +you just plug each radar in when asked: ```bash -uv run python scripts/hardware-test/test_rolling_buffer_persist.py --test +./scripts/setup/setup_kld7_devices.sh # run / redo the mapping +./scripts/setup/setup_kld7_devices.sh --show # check the current mapping ``` -Make a sound near the SEN-14262 — you should see trigger data with I/Q samples. +It also installs the FTDI low-latency rule (the K-LD7 RADC stream runs at +3 Mbaud and needs `latency_timer=1ms` instead of the Linux default 16ms). +On startup, the server logs should show both radars at `1ms`: -## K-LD7 Angle Radar Setup - -Each K-LD7 connects via a 3.3V FTDI USB-to-serial adapter and appears as `/dev/ttyUSB*`. +```text +[KLD7:vertical] USB serial latency_timer=1ms ... +[KLD7:horizontal] USB serial latency_timer=1ms ... +``` -### Stable Device Names (udev rules) +
+Manual steps (what the wizard does) -USB serial devices can swap between `/dev/ttyUSB0` and `/dev/ttyUSB1` after a reboot depending on enumeration order. To assign fixed names based on each FTDI adapter's unique serial number: +Find each adapter's serial number: ```bash -# Find the serial numbers for each adapter udevadm info -a /dev/ttyUSB0 | grep '{serial}' | head -1 udevadm info -a /dev/ttyUSB1 | grep '{serial}' | head -1 ``` -Create a udev rule with the serial numbers: +Create a udev rule with the serial numbers (replace `FTXXXXXX`/`FTYYYYYY`): ```bash sudo tee /etc/udev/rules.d/99-kld7.rules << 'EOF' @@ -100,30 +131,57 @@ EOF sudo udevadm control --reload-rules && sudo udevadm trigger ``` -Replace `FTXXXXXX` and `FTYYYYYY` with the actual serial numbers. Now the radars are always at `/dev/kld7_vertical` and `/dev/kld7_horizontal` regardless of plug order. +Then install the latency rule: + +```bash +sudo scripts/setup/setup_kld7_latency.sh +``` + +Use `--dry-run` to preview the rule, or `--all-ftdi` if the `/dev/kld7_*` +names aren't set up yet. + +
-### FTDI Low-Latency Mode +### Auto-Start on Boot -The K-LD7 RADC stream runs at 3 Mbaud, so the FTDI USB adapters should use a -`latency_timer` of `1ms` instead of the Linux default `16ms`. Install the -OpenFlight udev rule once after the stable `/dev/kld7_*` symlinks are working: +The setup script installs and enables a systemd service configured for your +username and install path. + +
+Manual steps and service management ```bash -sudo scripts/setup/setup_kld7_latency.sh +# Install (adjust User= and paths in the file if your username isn't the default) +sudo cp ~/openflight/scripts/setup/openflight.service /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl enable openflight +sudo systemctl start openflight ``` -The script writes `/etc/udev/rules.d/99-openflight-kld7-latency.rules`, applies -the value to currently connected K-LD7 adapters, and reloads udev so it persists -after reboot or replug. Use `--dry-run` to preview the exact rule, or -`--all-ftdi` if the K-LD7 adapters do not have stable symlinks yet. +Management: -On startup, confirm the server logs show both radars at `1ms`: +```bash +sudo systemctl status openflight --no-pager # Check status +journalctl -u openflight -f # View logs +sudo systemctl stop openflight # Stop +sudo systemctl restart openflight # Restart +sudo systemctl disable openflight # Disable auto-start +``` -```text -[KLD7:vertical] USB serial latency_timer=1ms ... -[KLD7:horizontal] USB serial latency_timer=1ms ... +To modify the service: + +```bash +sudo nano /etc/systemd/system/openflight.service +sudo systemctl daemon-reload +sudo systemctl restart openflight ``` +
+ +--- + +## K-LD7 Physical Setup + ### Mounting - **Vertical unit** — measures launch angle. Mount with the antenna plane vertical, aimed at the hitting area. @@ -153,14 +211,9 @@ See [K-LD7 Troubleshooting](kld7-troubleshooting.md) for more details. ### Kiosk Mode (Fullscreen — Recommended) ```bash -# Default: rolling buffer + sound trigger -./scripts/start-kiosk.sh - -# With K-LD7 launch-angle geometry defaults -./scripts/start-kiosk.sh --kld7-geometry - -# Mock mode (no hardware needed) -./scripts/start-kiosk.sh --mock +./scripts/start-kiosk.sh # Default: rolling buffer + sound trigger +./scripts/start-kiosk.sh --kld7-geometry # With K-LD7 launch-angle geometry defaults +./scripts/start-kiosk.sh --mock # Mock mode (no hardware needed) ``` ### Manual Start @@ -178,34 +231,6 @@ Then open `http://localhost:8080`. DISPLAY=:0 ./scripts/start-kiosk.sh ``` -## Auto-Start on Boot - -### Enable the Service - -```bash -sudo cp ~/openflight/scripts/setup/openflight.service /etc/systemd/system/ -sudo systemctl daemon-reload -sudo systemctl enable openflight -sudo systemctl start openflight -``` - -### Service Management - -```bash -sudo systemctl status openflight --no-pager # Check status -journalctl -u openflight -f # View logs -sudo systemctl stop openflight # Stop -sudo systemctl restart openflight # Restart -sudo systemctl disable openflight # Disable auto-start -``` - -To modify the service: -```bash -sudo nano /etc/systemd/system/openflight.service -sudo systemctl daemon-reload -sudo systemctl restart openflight -``` - ## Observability (Grafana Cloud) OpenFlight can ship session logs to Grafana Cloud for long-term analysis. @@ -233,14 +258,17 @@ See the [Sound Trigger Wiring Guide — Troubleshooting](sound-trigger-wiring.md ### K-LD7 Not Connecting ```bash -# Check USB devices -ls /dev/ttyUSB* /dev/kld7_* +# Check the device mapping +./scripts/setup/setup_kld7_devices.sh --show # Test standalone uv run python scripts/hardware-test/test_kld7.py ``` -Look for `[KLD7] Connected on /dev/ttyUSB...` in the server logs. See [K-LD7 Troubleshooting](kld7-troubleshooting.md) for "Wrong length reply" and other connection issues. +If the mapping is missing or points at the wrong radar, re-run the wizard: +`./scripts/setup/setup_kld7_devices.sh`. Look for `[KLD7] Connected on +/dev/ttyUSB...` in the server logs. See [K-LD7 Troubleshooting](kld7-troubleshooting.md) +for "Wrong length reply" and other connection issues. ### Service Won't Start @@ -286,11 +314,20 @@ openflight-server --mock # Mock mode openflight-server --web-port 3000 # Custom port ``` +### Setup + +```bash +./scripts/setup/setup.sh # Full interactive setup (re-run safe) +./scripts/setup/setup.sh --deps-only # Dependencies only +./scripts/setup/setup_kld7_devices.sh # K-LD7 device naming wizard +./scripts/setup/setup_kld7_devices.sh --show # Show current K-LD7 mapping +``` + ### Testing ```bash uv run python scripts/hardware-test/test_rolling_buffer_persist.py --test # Sound trigger uv run python scripts/hardware-test/test_sound_trigger_hardware.py # Direct trigger test uv run python scripts/hardware-test/test_kld7.py # K-LD7 standalone -uv run pytest tests/ -v # Full test suite +uv run pytest tests/ -v # Full test suite ``` diff --git a/docs/spin-dechirp-replay.md b/docs/spin-dechirp-replay.md new file mode 100644 index 000000000..0fc27cd55 --- /dev/null +++ b/docs/spin-dechirp-replay.md @@ -0,0 +1,141 @@ +# Dechirped-Sideband Spin Replay + +Reference for `scripts/analysis/replay_spin_dechirp.py` — the offline test +bench for the next-generation spin estimator. Offline only; it never touches +production behavior. Tune here against truth data first, then port to +`rolling_buffer/processor.py` once the numbers prove out. + +## Why this exists + +A spinning golf ball's rotating asymmetries (seam, logo, dimples) modulate the +radar return, creating **sidebands** flanking the ball's Doppler tone, spaced +at exact multiples of the modulation frequency. Read the spacing → read the +spin. This is how commercial radar units (TrackMan US8845442, FlightScope +US9868044) measure spin. + +The production detector instead FFTs the bandpassed **amplitude envelope**, +which only captures the weak AM component (1-5% modulation depth) and discards +the phase modulation carrying most of the signal energy. + +The trap blocking the sideband approach directly: **ball deceleration**. Drag +at driver speeds chirps the Doppler carrier ~3-4 kHz/s, so over a 60 ms +capture the carrier — and every sideband — sweeps ~200 Hz. Sideband spacings +are 40-270 Hz (2500-8000 RPM), so a plain FFT smears the whole pattern into +one blurred lump. The chirp must be removed first. + +## Pipeline + +1. **Carrier tracking** (`track_carrier`) — short overlapping STFTs (256 + samples ≈ 8.5 ms, 2 ms hop) find the ball Doppler peak near the + OPS243-expected frequency in each frame; a quadratic `f_d(t)` is fitted + through the peaks. SNR collapse marks the end of the usable window (net + impact), which the script reports as `usable_ms`. +2. **Dechirp** (`dechirp`) — multiply the raw I/Q by + `exp(-j·2π·∫f_d(t)dt)` (the conjugate chirp). The decelerating carrier + becomes a stationary tone at 0 Hz; sidebands land at exactly ±m·f_mod, + coherent over the whole window. +3. **Harmonic comb search** (`sideband_spin`) — one large zero-padded FFT, + then a comb over candidate modulation frequencies (33-400 Hz, 0.5 Hz + steps). Each candidate scores `min(upper, lower)` sideband support at + ±1f/±2f/±3f — real spin sidebands are symmetric around the carrier, + noise usually isn't. Each sideband is normalized by its **local** + neighborhood floor, not a global one: the dechirped carrier's skirt + slopes steeply, and with a global floor the lowest candidate frequency + always wins (this was the first-run failure mode). +4. **Harmonic-number disambiguation** — the comb finds the sideband + *spacing*; spin is that spacing (1× logo/asymmetry modulation) or half + of it (the seam's 2-fold symmetry modulates at 2×, the common case per + FlightScope's patent). Sideband support at 1.5× the found spacing — an + odd harmonic — proves the true fundamental is half the spacing. Both + interpretations are emitted (`dechirp_rpm_1x` / `dechirp_rpm_2x`) plus + the pick (`dechirp_rpm`, `dechirp_choice`). +5. **Scoring** — pairs captures with launch-monitor truth and prints + coverage / MAE / median / within-10% for three estimators: production + envelope, dechirped sidebands, and a **harmonic oracle** (the better of + the 1×/2× interpretations per shot — the ceiling a perfect + disambiguation rule could reach). + +## Requirements + +1. **Software** — a normal OpenFlight install (`./scripts/setup/setup.sh + --deps-only`). Runs on any machine, no radar hardware needed. +2. **A session JSONL with shots** — produced automatically by any normal + session. The needed `shot_detected` and `rolling_buffer_capture` (raw + 4096-sample I/Q) entries are logged by default; no special flags. +3. **Spin truth** — hit a session with OpenFlight running alongside a + reference monitor (TrackMan etc.) that exports per-shot spin to CSV, + then pair the two: + + ```bash + uv run python scripts/analysis/compare_trackman.py \ + --openflight session_logs/session_YYYYMMDD_*.jsonl \ + --trackman ~/Downloads/TrackMan_export.csv \ + --output session_logs/comparison_mysession.csv + ``` + + Pairing is by club + chronological order with a ball-speed tolerance, so + the two systems don't need identical shot counts. Hand-rolled CSVs from + other monitors work if they have `shot_number_of`, `match_quality`, + `spin_tm`, and `ball_speed_tm` columns. + +## Running + +```bash +uv run --no-sync python scripts/analysis/replay_spin_dechirp.py \ + --openflight session_logs/session_YYYYMMDD_HHMMSS.jsonl \ + --comparison session_logs/comparison_mysession.csv \ + --output session_logs/spin_dechirp_replay.csv +``` + +Prints the three-row summary and writes a per-shot CSV (both spin +interpretations, comb score, usable window, chirp rate, baseline result). +Under a minute for a ~60-shot session. + +Notes: + +- Only shots with `match_quality == "good"` and a truth spin value are + scored; unmatched warm-up shots are dropped automatically. +- The estimator is research-grade: trust the summary table and + high-comb-score shots, not per-shot `dechirp_rpm` as a final answer. + +## Baseline results (2026-06-11, session_20260511 / comparison_test2, 61 shots) + +| estimator | coverage | result | +|---|---|---| +| production envelope | 12/61 | median error ~3200 rpm | +| dechirp, comb score ≥ 5 | 11/61 | **~1.0% median error** (0.1-1.5% per shot) | +| dechirp + harmonic oracle | 61/61 | 18/61 within 10% | + +Interpretation: the spin information is present in our raw captures and +dechirping recovers it at launch-monitor-grade precision when the comb locks. +The two failure buckets are both visible in the per-shot CSV: + +1. **Harmonic mis-choice** — errors of exactly 2× or ½× where the oracle + column shows the right answer was available. Improving the + disambiguation rule is the highest-value next step. +2. **Low-score noise locks** — shots where no real sideband pattern exists + and the comb picks noise. Needs a comb-score confidence gate before any + value is reported. + +## Next steps (in value order) + +1. Strengthen 1×/2× disambiguation (use odd-harmonic evidence more + aggressively; consider club-band plausibility only as a tiebreaker). +2. Add a comb-score gate and report "not measurable" below it. +3. Validate on a held-out session (different day/rig if possible) before + porting anything to production. More truth-paired sessions from diverse + rigs/nets/balls are the most valuable contribution here. +4. Long term: this estimator is what could lift the production low-band + confidence cap (`SPIN_LOW_BAND_SUSPECT_MAX_RPM` in + `rolling_buffer/processor.py`) — the envelope path cannot distinguish + real ≤3100 RPM spin from red envelope noise in a single 136 ms capture, + but coherent sidebands carry far more evidence. + +## Related + +- `docs/rolling_buffer_spin_detection.md` — production envelope detector +- `scripts/analysis/experiment_spin_windows.py` — earlier window experiments + (shares the JSONL/CSV loaders this script imports) +- TrackMan US8845442 / EP1698380, FlightScope US9868044, Liu 2017 + ("A Micro-Doppler Modulation of Spin Projectile on CW Radar") — the + sideband physics and harmonic-ambiguity math diff --git a/docs/superpowers/plans/2026-06-09-kld7-prf-probing.md b/docs/superpowers/plans/2026-06-09-kld7-prf-probing.md new file mode 100644 index 000000000..2b4da5075 --- /dev/null +++ b/docs/superpowers/plans/2026-06-09-kld7-prf-probing.md @@ -0,0 +1,615 @@ +# K-LD7 Timing and PRF Probe Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build a guarded K-LD7 timing/protocol probe that measures real RADC frame cadence and optionally probes explicitly listed undocumented commands. + +**Architecture:** Add a standalone hardware-test script with a small serial protocol layer, pure summarization helpers, and CLI safety gates. Tests exercise packet construction, split-read handling, frame-gap summaries, and unsafe-mode validation without requiring hardware. + +**Tech Stack:** Python 3.11+, `uv run`, `pytest`, `pyserial`, standard-library `argparse`, `dataclasses`, `json`, `struct`, and `statistics`. + +--- + +## File Structure + +| File | Responsibility | +|------|----------------| +| `scripts/hardware-test/probe_kld7_timing.py` | Standalone K-LD7 protocol probe, CLI, measurement loop, unsafe command probing, JSONL/summary output | +| `tests/test_probe_kld7_timing.py` | Unit tests for protocol helpers, fake serial reads, summarization, CLI safety validation | +| `docs/kld7-troubleshooting.md` | Short operator-facing section explaining when and how to run the timing probe | + +No production tracker code changes are planned. + +## Task 1: Protocol Primitives + +**Files:** +- Create: `scripts/hardware-test/probe_kld7_timing.py` +- Test: `tests/test_probe_kld7_timing.py` + +- [ ] **Step 1: Write failing tests for packet construction and command validation** + +Add tests: + +```python +import importlib.util +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "hardware-test" / "probe_kld7_timing.py" +spec = importlib.util.spec_from_file_location("probe_kld7_timing", SCRIPT) +probe = importlib.util.module_from_spec(spec) +assert spec.loader is not None +spec.loader.exec_module(probe) + + +def test_build_packet_uppercases_command_and_packs_length(): + packet = probe.build_packet("gnfd", (0x21).to_bytes(4, "little")) + + assert packet == b"GNFD\x04\x00\x00\x00!\x00\x00\x00" + + +def test_validate_command_rejects_non_four_byte_command(): + assert probe.validate_probe_command("ABC", "") == "command must be exactly 4 ASCII characters" + + +def test_validate_command_rejects_non_uppercase_command(): + assert probe.validate_probe_command("test", "") == "command must be uppercase ASCII" + + +def test_validate_command_rejects_odd_hex_payload(): + assert probe.validate_probe_command("TST1", "abc") == "hex payload must have an even number of characters" +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: + +```bash +uv run pytest tests/test_probe_kld7_timing.py -v +``` + +Expected: FAIL because `scripts/hardware-test/probe_kld7_timing.py` does not exist. + +- [ ] **Step 3: Implement packet helpers** + +Create `scripts/hardware-test/probe_kld7_timing.py` with: + +```python +#!/usr/bin/env python3 +"""Guarded K-LD7 timing and protocol probe.""" + +from __future__ import annotations + +import argparse +import json +import statistics +import struct +import sys +import time +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import BinaryIO, Iterable, Optional + +try: + import serial + from serial.tools.list_ports import comports +except ImportError: # pragma: no cover - exercised by operator environment + serial = None + comports = None + + +DEFAULT_BAUD = 3_000_000 +DEFAULT_START_BAUD = 115_200 +SUPPORTED_BAUD_RATES = [115_200, 460_800, 921_600, 2_000_000, 3_000_000] +FRAME_CODES = { + "RADC": 0x01, + "RFFT": 0x02, + "PDAT": 0x04, + "TDAT": 0x08, + "DDAT": 0x10, + "DONE": 0x20, +} +DOCUMENTED_WRITE_COMMANDS = { + "RBFR", "RSPI", "RRAI", "THOF", "TRFT", "VISU", "MIRA", "MARA", + "MIAN", "MAAN", "MISP", "MASP", "DEDI", "RATH", "ANTH", "SPTH", + "DIG1", "DIG2", "DIG3", "HOLD", "MIDE", "MIDS", +} +DESTRUCTIVE_COMMANDS = {"RFSE"} + + +def build_packet(command: str, payload: bytes = b"") -> bytes: + cmd = command.upper().encode("ascii") + if len(cmd) != 4: + raise ValueError("command must be exactly 4 ASCII characters") + return struct.pack("<4sI", cmd, len(payload)) + payload + + +def validate_probe_command(command: str, hex_payload: str) -> Optional[str]: + try: + raw = command.encode("ascii") + except UnicodeEncodeError: + return "command must be ASCII" + if len(raw) != 4: + return "command must be exactly 4 ASCII characters" + if command != command.upper(): + return "command must be uppercase ASCII" + if len(hex_payload) % 2: + return "hex payload must have an even number of characters" + try: + bytes.fromhex(hex_payload) + except ValueError: + return "hex payload must be valid hexadecimal" + return None +``` + +- [ ] **Step 4: Run tests to verify Task 1 passes** + +Run: + +```bash +uv run pytest tests/test_probe_kld7_timing.py -v +``` + +Expected: PASS for the four packet-helper tests. + +## Task 2: Packet Reading and Summary Logic + +**Files:** +- Modify: `scripts/hardware-test/probe_kld7_timing.py` +- Modify: `tests/test_probe_kld7_timing.py` + +- [ ] **Step 1: Write failing tests for split reads and `DONE` gap summaries** + +Append tests: + +```python +class FakeSerial: + def __init__(self, chunks): + self.chunks = list(chunks) + self.writes = [] + self.timeout = 0.2 + self.baudrate = 115200 + + def read(self, n): + if not self.chunks: + return b"" + chunk = self.chunks.pop(0) + if len(chunk) > n: + self.chunks.insert(0, chunk[n:]) + return chunk[:n] + return chunk + + def write(self, data): + self.writes.append(data) + return len(data) + + def flush(self): + return None + + def reset_input_buffer(self): + return None + + def close(self): + return None + + +def test_read_packet_handles_split_header_and_payload(): + payload = b"\x00\x01\x02\x03" + header = b"DONE" + len(payload).to_bytes(4, "little") + fake = FakeSerial([header[:3], header[3:8], payload[:1], payload[1:]]) + protocol = probe.KLD7Protocol.__new__(probe.KLD7Protocol) + protocol.port = fake + + packet = protocol.read_packet() + + assert packet.code == "DONE" + assert packet.payload == payload + assert packet.payload_bytes == 4 + + +def test_summarize_measurements_counts_done_gaps(): + packets = [ + probe.PacketRecord(code="DONE", payload_bytes=4, complete_monotonic=1.0, done_frame=10), + probe.PacketRecord(code="RADC", payload_bytes=3072, complete_monotonic=1.1, read_duration_ms=10.0), + probe.PacketRecord(code="DONE", payload_bytes=4, complete_monotonic=2.0, done_frame=12), + probe.PacketRecord(code="RADC", payload_bytes=3072, complete_monotonic=2.1, read_duration_ms=20.0), + ] + + summary = probe.summarize_packets(packets, duration_s=2.0) + + assert summary["radc_frames"] == 2 + assert summary["done_frames"] == 2 + assert summary["done_frame_gaps"] == 1 + assert summary["effective_radc_hz"] == 1.0 + assert summary["read_duration_ms_p95"] == 20.0 +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: + +```bash +uv run pytest tests/test_probe_kld7_timing.py -v +``` + +Expected: FAIL because `KLD7Protocol`, `PacketRecord`, and `summarize_packets` are not implemented. + +- [ ] **Step 3: Implement protocol records, exact reads, and summaries** + +Add to the script: + +```python +@dataclass +class PacketRecord: + code: str + payload_bytes: int + command: Optional[str] = None + response_code: Optional[int] = None + send_monotonic: Optional[float] = None + first_byte_monotonic: Optional[float] = None + header_complete_monotonic: Optional[float] = None + complete_monotonic: Optional[float] = None + read_duration_ms: Optional[float] = None + done_frame: Optional[int] = None + error: Optional[str] = None + payload: bytes = field(default=b"", repr=False) + + +class KLD7Protocol: + def __init__(self, port_path: str, baud: int = DEFAULT_BAUD, timeout: float = 0.2): + if serial is None: + raise RuntimeError("pyserial is required for hardware probing") + self.port_path = port_path + self.baud = baud + self.port = serial.Serial( + port=port_path, + baudrate=DEFAULT_START_BAUD, + parity=serial.PARITY_EVEN, + stopbits=1, + timeout=timeout, + ) + + def _read_exact(self, n: int) -> tuple[bytes, Optional[float]]: + buf = b"" + first_byte_at = None + deadline = time.monotonic() + max(float(getattr(self.port, "timeout", 0.2) or 0.2), 0.2) + while len(buf) < n: + chunk = self.port.read(n - len(buf)) + if chunk: + if first_byte_at is None: + first_byte_at = time.monotonic() + buf += chunk + continue + if time.monotonic() >= deadline: + break + time.sleep(0.001) + return buf, first_byte_at + + def read_packet(self) -> PacketRecord: + started = time.monotonic() + header, first_byte_at = self._read_exact(8) + header_complete = time.monotonic() + if len(header) != 8: + return PacketRecord( + code="", + payload_bytes=0, + first_byte_monotonic=first_byte_at, + header_complete_monotonic=header_complete, + complete_monotonic=header_complete, + error=f"short header read: got {len(header)} of 8 bytes", + ) + raw_code, length = struct.unpack("<4sI", header) + code = raw_code.decode("ascii", errors="replace") + payload = b"" + payload_first = None + if length: + payload, payload_first = self._read_exact(length) + complete = time.monotonic() + error = None + if len(payload) != length: + error = f"short payload read: got {len(payload)} of {length} bytes" + done_frame = None + if code == "DONE" and len(payload) == 4: + done_frame = int.from_bytes(payload, "little", signed=False) + return PacketRecord( + code=code, + payload_bytes=length, + first_byte_monotonic=first_byte_at or payload_first, + header_complete_monotonic=header_complete, + complete_monotonic=complete, + read_duration_ms=(complete - (first_byte_at or started)) * 1000.0, + done_frame=done_frame, + error=error, + payload=payload, + ) + + +def _percentile(values: list[float], percentile: float) -> Optional[float]: + if not values: + return None + values = sorted(values) + index = min(len(values) - 1, max(0, round((percentile / 100.0) * (len(values) - 1)))) + return values[index] + + +def summarize_packets(packets: list[PacketRecord], duration_s: float) -> dict: + radc_packets = [p for p in packets if p.code == "RADC" and not p.error] + done_packets = [p for p in packets if p.code == "DONE" and not p.error] + done_frames = [p.done_frame for p in done_packets if p.done_frame is not None] + gaps = 0 + for previous, current in zip(done_frames, done_frames[1:]): + if current > previous + 1: + gaps += current - previous - 1 + read_durations = [p.read_duration_ms for p in radc_packets if p.read_duration_ms is not None] + errors: dict[str, int] = {} + for packet in packets: + if packet.error: + errors[packet.error] = errors.get(packet.error, 0) + 1 + return { + "duration_s": duration_s, + "radc_frames": len(radc_packets), + "done_frames": len(done_packets), + "effective_radc_hz": round(len(radc_packets) / duration_s, 3) if duration_s > 0 else 0.0, + "effective_done_hz": round(len(done_packets) / duration_s, 3) if duration_s > 0 else 0.0, + "done_frame_gaps": gaps, + "read_duration_ms_mean": statistics.mean(read_durations) if read_durations else None, + "read_duration_ms_p50": statistics.median(read_durations) if read_durations else None, + "read_duration_ms_p95": _percentile(read_durations, 95), + "errors": errors, + } +``` + +- [ ] **Step 4: Run tests to verify Task 2 passes** + +Run: + +```bash +uv run pytest tests/test_probe_kld7_timing.py -v +``` + +Expected: PASS for packet construction, split reads, and summary tests. + +## Task 3: CLI Safety Gates and Measurement Loop + +**Files:** +- Modify: `scripts/hardware-test/probe_kld7_timing.py` +- Modify: `tests/test_probe_kld7_timing.py` + +- [ ] **Step 1: Write failing tests for CLI validation** + +Append tests: + +```python +def test_unsafe_probe_requires_output(): + parser = probe.build_parser() + args = parser.parse_args(["--port", "/dev/null", "--unsafe-probe", "--probe-command", "TEST"]) + + assert probe.validate_args(args) == ["--unsafe-probe requires --output so probe activity is auditable"] + + +def test_rfse_requires_factory_reset_flag(): + parser = probe.build_parser() + args = parser.parse_args([ + "--port", "/dev/null", + "--output", "/tmp/probe.jsonl", + "--unsafe-probe", + "--probe-command", "RFSE", + ]) + + assert probe.validate_args(args) == ["RFSE is refused unless --allow-factory-reset is set"] + + +def test_parse_frame_mask_combines_known_flags(): + assert probe.parse_frame_mask("RADC,DONE") == 0x21 +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: + +```bash +uv run pytest tests/test_probe_kld7_timing.py -v +``` + +Expected: FAIL because parser, arg validation, and frame-mask parsing are missing. + +- [ ] **Step 3: Implement parser, validation, and measurement flow** + +Add: + +```python +def parse_frame_mask(value: str) -> int: + mask = 0 + for raw_name in value.split(","): + name = raw_name.strip().upper() + if not name: + continue + if name not in FRAME_CODES: + raise argparse.ArgumentTypeError(f"unknown frame type {name!r}") + mask |= FRAME_CODES[name] + return mask + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Probe K-LD7 RADC timing and guarded commands.") + parser.add_argument("--port") + parser.add_argument("--baud", type=int, default=DEFAULT_BAUD, choices=SUPPORTED_BAUD_RATES) + parser.add_argument("--duration", type=float, default=10.0) + parser.add_argument("--frame-mask", default="RADC,DONE") + parser.add_argument("--rspi-sweep", action="store_true") + parser.add_argument("--rrai", type=int) + parser.add_argument("--rbfr", type=int) + parser.add_argument("--output", type=Path) + parser.add_argument("--unsafe-probe", action="store_true") + parser.add_argument("--probe-command", action="append", default=[]) + parser.add_argument("--allow-factory-reset", action="store_true") + parser.add_argument("--no-restore-params", action="store_true") + return parser + + +def validate_args(args: argparse.Namespace) -> list[str]: + errors = [] + if args.unsafe_probe and not args.output: + errors.append("--unsafe-probe requires --output so probe activity is auditable") + if args.probe_command and not args.unsafe_probe: + errors.append("--probe-command requires --unsafe-probe") + for command_spec in args.probe_command: + command, _, hex_payload = command_spec.partition(":") + error = validate_probe_command(command, hex_payload) + if error: + errors.append(f"{command_spec}: {error}") + if command.upper() in DESTRUCTIVE_COMMANDS and not args.allow_factory_reset: + errors.append("RFSE is refused unless --allow-factory-reset is set") + return errors +``` + +Extend `KLD7Protocol`: + +```python + def send_command(self, command: str, payload: bytes = b"") -> PacketRecord: + sent = time.monotonic() + self.port.reset_input_buffer() + self.port.write(build_packet(command, payload)) + self.port.flush() + packet = self.read_packet() + packet.command = command.upper() + packet.send_monotonic = sent + if packet.code == "RESP" and packet.payload: + packet.response_code = packet.payload[0] + return packet + + def request_frame(self, frame_mask: int) -> list[PacketRecord]: + records = [self.send_command("GNFD", int(frame_mask).to_bytes(4, "little", signed=True))] + expected = bin(frame_mask).count("1") + for _ in range(expected): + record = self.read_packet() + records.append(record) + if record.code == "DONE": + break + return records + + def close(self) -> None: + try: + self.port.write(build_packet("GBYE")) + self.port.flush() + finally: + self.port.close() +``` + +Add `measure()` and `main()`: + +```python +def measure(protocol: KLD7Protocol, frame_mask: int, duration_s: float) -> list[PacketRecord]: + records = [] + deadline = time.monotonic() + duration_s + while time.monotonic() < deadline: + records.extend(protocol.request_frame(frame_mask)) + return records + + +def write_jsonl(path: Path, records: Iterable[PacketRecord], summary: dict) -> None: + with path.open("w", encoding="utf-8") as handle: + for record in records: + data = asdict(record) + data.pop("payload", None) + handle.write(json.dumps({"type": "packet", **data}, sort_keys=True) + "\n") + handle.write(json.dumps({"type": "summary", **summary}, sort_keys=True) + "\n") + + +def main(argv: Optional[list[str]] = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + errors = validate_args(args) + if errors: + for error in errors: + print(f"error: {error}", file=sys.stderr) + return 2 + frame_mask = parse_frame_mask(args.frame_mask) + if not args.port: + print("error: --port is required for the first implementation", file=sys.stderr) + return 2 + protocol = KLD7Protocol(args.port, baud=args.baud) + records = [] + try: + records = measure(protocol, frame_mask, args.duration) + finally: + protocol.close() + summary = summarize_packets(records, args.duration) + print(json.dumps(summary, indent=2, sort_keys=True)) + if args.output: + write_jsonl(args.output, records, summary) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) +``` + +- [ ] **Step 4: Run focused tests** + +Run: + +```bash +uv run pytest tests/test_probe_kld7_timing.py -v +``` + +Expected: PASS. + +## Task 4: Docs and Verification + +**Files:** +- Modify: `docs/kld7-troubleshooting.md` +- Test: `tests/test_probe_kld7_timing.py` + +- [ ] **Step 1: Add troubleshooting docs** + +Add a section after "RADC Streaming Issues": + +```markdown +### Measuring Real K-LD7 RADC Cadence + +Use the guarded timing probe when launch-angle extraction is missing frames or when one K-LD7 orientation appears slower than the other: + +```bash +uv run python scripts/hardware-test/probe_kld7_timing.py \ + --port /dev/kld7_vertical \ + --duration 10 \ + --frame-mask RADC,DONE \ + --output /tmp/kld7_vertical_timing.jsonl +``` + +At the production `RSPI=3` setting, expect roughly 34 RADC frames per second with low `done_frame_gaps`. If cadence is much lower or gaps are high, investigate USB scheduling, serial read duration, or requested packet volume before changing launch-angle selection logic. + +Undocumented command probing is available only through `--unsafe-probe` and requires `--output`. Do not use it in production sessions. +``` + +- [ ] **Step 2: Run unit tests** + +Run: + +```bash +uv run pytest tests/test_probe_kld7_timing.py -v +``` + +Expected: PASS. + +- [ ] **Step 3: Run lint/format checks for touched Python files** + +Run: + +```bash +uv run ruff check scripts/hardware-test/probe_kld7_timing.py tests/test_probe_kld7_timing.py +uv run ruff format --check scripts/hardware-test/probe_kld7_timing.py tests/test_probe_kld7_timing.py +``` + +Expected: PASS. + +- [ ] **Step 4: Run a no-hardware CLI validation check** + +Run: + +```bash +uv run python scripts/hardware-test/probe_kld7_timing.py --unsafe-probe --probe-command TEST +``` + +Expected: exit code `2` and stderr containing `--unsafe-probe requires --output`. + diff --git a/docs/superpowers/specs/2026-06-09-kld7-prf-probing-design.md b/docs/superpowers/specs/2026-06-09-kld7-prf-probing-design.md new file mode 100644 index 000000000..46992587b --- /dev/null +++ b/docs/superpowers/specs/2026-06-09-kld7-prf-probing-design.md @@ -0,0 +1,219 @@ +# K-LD7 Timing and PRF Probe + +**Date:** 2026-06-09 +**Status:** Approved +**Scope:** Build a guarded hardware exploration path for diagnosing unreliable K-LD7 launch-angle frames, measuring actual RADC frame cadence, and optionally probing explicitly listed undocumented serial commands. + +## Problem + +OpenFlight is not always getting enough reliable K-LD7 frames around impact to extract launch angle. The current production path streams RADC frames at 3 Mbaud and anchors angle extraction to the OPS243 impact timestamp, but a golf ball can cross the useful detection zone in only a few tens of milliseconds. More frames near impact would improve the odds of a clean launch-angle estimate. + +The hypothesis is that a higher PRF and slightly lower pulse width could improve frame capture. The K-LD7 public documentation, however, does not expose PRF or pulse width directly. The module is documented as an FSK Doppler radar. The exposed controls are maximum speed (`RSPI`), maximum range (`RRAI`), base frequency (`RBFR`), threshold/tracking settings, and data-frame request commands. That means the first question is whether the bottleneck is true RF acquisition cadence, serial readout/request strategy, or undocumented firmware capability. + +## Approach + +Create a standalone hardware-test probe that talks to the K-LD7 binary serial protocol directly, but defaults to non-destructive measurement. + +The exploration has two phases: + +1. **Safe measurement:** use documented commands only. Measure actual frame cadence, dropped `DONE` frame numbers, RADC payload read time, and serial errors under current and swept documented settings. +2. **Unsafe probing:** only with an explicit opt-in flag, send explicitly listed candidate undocumented command packets and record response codes plus before/after parameter diffs. + +This is not a broad brute-force scanner. The unsafe phase must only send command names and payloads deliberately supplied on the CLI or in a small allowlisted fixture. The goal is to discover whether a plausible engineering command exists, not to fuzz the firmware. + +## Evidence From Current Code And Docs + +- `src/openflight/kld7/tracker.py` configures `RRAI`, `RSPI`, `RBFR`, `DEDI`, `THOF`, `TRFT`, `MIAN`, `MAAN`, `MIRA`, `MARA`, `MISP`, `MASP`, and `VISU`. +- `src/openflight/kld7/serial_io.py` connects through the existing `kld7.KLD7` package at 3 Mbaud and patches packet reads for robust RADC streaming. +- The installed `kld7` package is a thin wrapper around binary packets: 4-byte command, 4-byte length, optional payload, then `RESP` plus data packets. +- `docs/K-LD7_Datasheet.pdf` lists documented commands: `INIT`, `GNFD`, `GRPS`, `SRPS`, `RFSE`, `GBYE`, and individual parameter setters including `RBFR`, `RSPI`, `RRAI`, `THOF`, `TRFT`, `VISU`, detection bounds, output routing, hold time, and micro-detection settings. +- The datasheet notes RADC is 3072 bytes and recommends the highest baud rate. It also states real-time readout is not possible if requested data readout time exceeds the typical frame duration. + +## Architecture + +**File:** `scripts/hardware-test/probe_kld7_timing.py` + +**Entry point:** `uv run python scripts/hardware-test/probe_kld7_timing.py` + +The script should not depend on the production `KLD7Tracker`. It should use pyserial directly or a small local protocol helper so we can observe raw packet timings without tracker-side buffering or selection logic. + +### Components + +**`KLD7Protocol`** + +Low-level serial protocol wrapper: + +- Opens the port at 115200 even parity. +- Sends `INIT` to negotiate the requested baud rate, usually 3 Mbaud. +- If `INIT` fails, optionally sends `GBYE` at 3 Mbaud and retries, matching the production recovery path for radars left streaming by a crashed process. +- Sends binary packets with `struct.pack("<4sI", cmd, length) + payload`. +- Reads exact packet headers and payloads with the same short-read tolerance used by `src/openflight/kld7/serial_io.py`. +- Exposes `send_command(cmd, payload=b"")`, `read_packet()`, `get_response()`, `read_params()`, `set_param()`, `request_frame(frame_mask)`, and `close()`. + +**`TimingRecorder`** + +Records one row per packet/frame: + +- Host monotonic timestamp at command send. +- First-byte arrival timestamp. +- Header complete timestamp. +- Payload complete timestamp. +- Packet code. +- Payload length. +- Response code, if applicable. +- `DONE` frame number, if requested. +- Short-read or timeout errors. + +**`ProbeResult`** + +Aggregates each run: + +- Effective RADC Hz. +- Effective `DONE` Hz. +- Mean/p50/p95 RADC payload read duration. +- Missing `DONE` frame numbers. +- Number of timeouts, short reads, invalid headers, and sensor-busy responses. +- Parameter snapshot before and after the run. + +## CLI + +```bash +uv run python scripts/hardware-test/probe_kld7_timing.py \ + --port /dev/kld7_vertical \ + --duration 10 \ + --frame-mask RADC,DONE +``` + +### Safe Flags + +- `--port PATH`: serial port or udev alias. Required unless auto-detection finds exactly one K-LD7. +- `--baud 3000000`: target post-`INIT` baud rate. +- `--duration SECONDS`: measurement duration for each configuration. +- `--frame-mask LIST`: comma-separated frame types. Default `RADC,DONE`. +- `--rspi-sweep`: run all documented `RSPI` values. +- `--rrai VALUE`: documented range code or meters. +- `--rbfr VALUE`: documented base frequency code. +- `--output PATH`: write JSONL packet log plus summary JSON. +- `--restore-params`: restore the initial `GRPS` parameter snapshot before exit. Default on. Restore should use individual documented parameter setters, not an opaque `SRPS` write that includes the software-version bytes. + +### Unsafe Flags + +- `--unsafe-probe`: enables undocumented command probing. +- `--probe-command CMD[:HEX_PAYLOAD]`: one explicitly listed 4-byte uppercase command packet to send. Repeatable. Hex payload must have an even number of characters. +- `--allow-factory-reset`: permits `RFSE`. Default refuses `RFSE` even in unsafe mode. +- `--no-restore-params`: leaves changed parameters in place, but only after printing a warning and requiring `--unsafe-probe`. + +Unsafe mode is unavailable unless `--output` is set, so every probe leaves an audit trail. + +## Safe Measurement Flow + +1. Resolve the serial port. +2. Connect at 115200, send `INIT`, switch to target baud. +3. Send `GRPS` and store the initial parameter snapshot. +4. Apply documented settings for the current run. +5. Re-read `GRPS` and record the active settings. +6. Loop until duration expires: + - Send `GNFD` with the requested frame mask. + - Read `RESP`. + - Read expected data packets, including `DONE` when requested. + - Record exact timing and packet sizes. +7. Summarize effective frame cadence and gaps. +8. Restore the initial parameter snapshot unless disabled. +9. Send `GBYE` and close the port. + +## Unsafe Probe Flow + +1. Run the safe connection and initial `GRPS` snapshot. +2. For each explicit `--probe-command`: + - Refuse known destructive commands unless separately allowed. + - Validate the command is exactly four ASCII uppercase bytes. + - Send the command with its exact payload. + - Read and record `RESP`. + - Drain and record any follow-up packets. + - Re-read `GRPS`. + - Emit a before/after parameter diff. +3. If the command returns `OK` or changes parameters, run a short `RADC,DONE` cadence sample. +4. Restore the original parameter snapshot before exit by default. + +## Safety Rules + +- No random command generation. +- No wildcard payload sweeps. +- No persistent production integration until a command is understood and reproducible. +- Refuse `RFSE` by default. +- Always snapshot parameters before writes. +- Restore parameters by default. +- Always send `GBYE` on exit. +- Treat any command that appears to alter RF timing as lab-only until RFbeam confirms regulatory implications. The K-LD7 has modular RF approvals; changing waveform timing or duty cycle outside documented settings could invalidate those assumptions. + +## Data Products + +**JSONL packet log:** one row per command/packet with timing and raw metadata. + +**Summary JSON:** one object per run: + +```json +{ + "port": "/dev/kld7_vertical", + "baud": 3000000, + "frame_mask": ["RADC", "DONE"], + "params_before": {"RSPI": 3, "RRAI": 0, "RBFR": 0}, + "params_active": {"RSPI": 3, "RRAI": 0, "RBFR": 0}, + "duration_s": 10.0, + "radc_frames": 340, + "done_frames": 340, + "effective_radc_hz": 34.0, + "done_frame_gaps": 0, + "read_duration_ms_p95": 12.4, + "errors": {} +} +``` + +## Interpretation + +The first decision point is whether the measured RADC stream is close to the documented `RSPI=3` cadence. + +- If RADC is near 34 Hz with low gaps, the launch-angle issue is probably frame selection, timing alignment, geometry, SNR, or target ambiguity rather than serial acquisition. +- If RADC is well below 34 Hz or has large `DONE` gaps, the next target is readout/request strategy, USB scheduling, or reducing requested packet volume. +- If a documented `RSPI` sweep changes effective cadence as expected, `RSPI=3` remains the highest documented cadence and true higher PRF is not exposed. +- If an undocumented command returns `OK` and changes cadence, it must stay experimental until validated against RF behavior, data quality, and module stability. + +## Testing + +Unit tests should cover protocol and summarization without hardware: + +- Packet builder creates correct command headers and payload lengths. +- Packet reader handles split headers and split payloads. +- `DONE` frame gap detection works across wrap-free monotonically increasing frame numbers. +- Summary statistics are correct for synthetic packet logs. +- Unsafe mode refuses undocumented probes unless `--unsafe-probe` is present. +- `RFSE` is refused unless `--allow-factory-reset` is present. +- Parameter restore runs in `finally` when initial parameters were captured. + +Hardware verification is manual and explicit: + +```bash +uv run python scripts/hardware-test/probe_kld7_timing.py \ + --port /dev/kld7_vertical \ + --duration 10 \ + --frame-mask RADC,DONE \ + --output /tmp/kld7_vertical_timing.jsonl +``` + +## Out Of Scope + +- Production use of undocumented commands. +- Automatic fuzzing of the K-LD7 firmware. +- Firmware extraction or binary reverse engineering. +- RF lab measurement of actual waveform timing. +- Changes to live launch-angle selection. +- UI changes. +- Replacing the `kld7` Python package in production. + +## Files Created Or Modified + +| File | Change | +|------|--------| +| `scripts/hardware-test/probe_kld7_timing.py` | New guarded timing/protocol probe | +| `tests/test_probe_kld7_timing.py` | New protocol, summary, and safety tests | +| `docs/kld7-troubleshooting.md` | Add a short section linking the probe and explaining interpretation | diff --git a/docs/trackman-test-process.md b/docs/trackman-test-process.md index d8a67b872..502856cf9 100644 --- a/docs/trackman-test-process.md +++ b/docs/trackman-test-process.md @@ -40,7 +40,8 @@ For every test pass, preserve enough raw data and diagnostics to answer: spin rail filtering. 3. Confirm OPS243 rolling buffer mode is active and sound trigger re-arms after each shot. -4. Confirm K-LD7 orientation and udev symlinks: +4. Confirm K-LD7 orientation and udev symlinks + (`./scripts/setup/setup_kld7_devices.sh --show`): - horizontal: `/dev/kld7_horizontal` - vertical: `/dev/kld7_vertical` 5. Confirm both K-LD7 FTDI adapters are in low-latency mode. Run this once on diff --git a/pyproject.toml b/pyproject.toml index 97630073d..c4a5906ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ dependencies = [ "flask>=3.1.3", "flask-cors>=6.0.2", "flask-socketio>=5.6.1", + "kld7>=0.2.1", ] [project.optional-dependencies] @@ -52,8 +53,6 @@ camera = [ # "picamera2>=0.3.0; sys_platform == 'linux'", # "inference-sdk>=0.9.0", # Roboflow inference API ] -# K-LD7 radar support -kld7 = ["kld7>=0.2.1"] # Analysis tools for I/Q capture data analysis = [ "numpy>=1.20.0", @@ -63,6 +62,7 @@ analysis = [ [project.scripts] openflight-server = "openflight.server:main" +openflight-cloud = "openflight.cloud.cli:main" [project.urls] Homepage = "https://github.com/jewbetcha/openflight" diff --git a/scripts/analysis/calibrate_kld7_reflector.py b/scripts/analysis/calibrate_kld7_reflector.py index 79461cee5..0d41806e5 100644 --- a/scripts/analysis/calibrate_kld7_reflector.py +++ b/scripts/analysis/calibrate_kld7_reflector.py @@ -234,7 +234,8 @@ def main() -> int: try: from kld7 import FrameCode # type: ignore except ImportError: - print("[calibrate] kld7 package not installed (uv add kld7).", + print("[calibrate] kld7 package not installed. Reinstall the project: " + "uv pip install -e '.[ui]'", file=sys.stderr) return 2 diff --git a/scripts/analysis/capture_kld7_radc.py b/scripts/analysis/capture_kld7_radc.py index 56fe6c9d3..61459bc42 100755 --- a/scripts/analysis/capture_kld7_radc.py +++ b/scripts/analysis/capture_kld7_radc.py @@ -46,7 +46,7 @@ try: from kld7 import KLD7, FrameCode, KLD7Exception except ImportError: - print("kld7 package not installed. Run: pip install kld7") + print("kld7 package not installed. Reinstall the project: uv pip install -e '.[ui]'") sys.exit(1) # Add src to path for OPS243 import. This script lives in scripts/analysis. diff --git a/scripts/analysis/replay_kld7_geometry_candidates.py b/scripts/analysis/replay_kld7_geometry_candidates.py new file mode 100644 index 000000000..32467a6c9 --- /dev/null +++ b/scripts/analysis/replay_kld7_geometry_candidates.py @@ -0,0 +1,630 @@ +#!/usr/bin/env python3 +"""Replay vertical K-LD7 raw ADC candidates through geometry ranking. + +This analysis mode intentionally treats TrackMan as an evaluation label, not +as an input to candidate selection. It asks: if we enumerate physically +plausible RADC candidates near the OPS ball-speed bin and impact time, which +candidate or candidate pair would a geometry-only ranker choose? +""" + +from __future__ import annotations + +import argparse +import csv +import json +import math +import statistics +import sys +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any + +import numpy as np + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent / "src")) + +from replay_kld7_trackman import TrackmanTarget, load_buffers, load_targets # noqa: E402 + +from openflight.kld7.geometry import ( # noqa: E402 + fit_launch_angle_geometric, + fit_launch_angle_single_frame_geometric, +) +from openflight.kld7.radc import ( # noqa: E402 + ball_bin_range_from_speed, + circular_bin_distance, + compute_fft_complex, + expected_ball_bin_from_speed, + parse_radc_payload, + per_bin_angle_deg, + spectrum_from_channel_ffts, + to_complex_iq, +) + + +@dataclass(frozen=True) +class GeometryCandidate: + shot_number: int + club: str + frame_index: int + dt_ms: float + bin_index: int + expected_bin: int + bin_error: int + snr: float + snr_db: float + bearing_deg: float + single_launch_deg: float + single_resid_deg: float + band_rank: int + local_peak: bool + local_prominence: float + trackman_angle_deg: float + single_abs_error_deg: float + + +@dataclass(frozen=True) +class GeometryPick: + shot_number: int + club: str + trackman_angle_deg: float + estimator: str + launch_angle_deg: float | None + abs_error_deg: float | None + score: float | None + frame_count: int + frame_indices: str + dt_ms: str + bins: str + bin_errors: str + snrs: str + fit_rmse_deg: float | None + single_resid_deg: float | None + reason: str + + +@dataclass(frozen=True) +class ScoreWeights: + single_resid: float + pair_rmse: float + bin_error: float + time: float + band_rank: float + local_peak_penalty: float + snr_reward: float + short_span_penalty: float + + +def _to_float(value: Any) -> float | None: + if value in (None, ""): + return None + try: + return float(value) + except (TypeError, ValueError): + return None + + +def _parse_window_ms(raw: str) -> tuple[float, float]: + parts = [float(part.strip()) for part in raw.split(",") if part.strip()] + if len(parts) != 2: + raise argparse.ArgumentTypeError("expected START,END") + if parts[0] > parts[1]: + raise argparse.ArgumentTypeError("window start must be <= end") + return parts[0], parts[1] + + +def _band_bins(bands: list[tuple[int, int]]) -> np.ndarray: + parts = [np.arange(lo, hi, dtype=np.int64) for lo, hi in bands if lo < hi] + if not parts: + return np.array([], dtype=np.int64) + return np.concatenate(parts) + + +def _local_peak(spec: np.ndarray, bin_index: int, radius: int = 2) -> tuple[bool, float]: + lo = max(0, bin_index - radius) + hi = min(len(spec), bin_index + radius + 1) + local_max = float(np.max(spec[lo:hi])) if hi > lo else 0.0 + value = float(spec[bin_index]) + if local_max <= 0.0: + return False, 0.0 + return value >= local_max, value / local_max + + +def enumerate_candidates( + target: TrackmanTarget, + frames: list[dict[str, Any]], + *, + shot_timestamp: float, + near_window_ms: tuple[float, float], + speed_tolerance_mph: float, + max_bin_error: int, + min_snr: float, + angle_offset_deg: float, + mount_deg: float, + distance_ft: float, + fft_size: int, +) -> list[GeometryCandidate]: + expected_bin = expected_ball_bin_from_speed(target.ball_speed_mph, fft_size=fft_size) + bands = ball_bin_range_from_speed( + target.ball_speed_mph, + speed_tolerance_mph, + fft_size=fft_size, + ) + band_bins = _band_bins(bands) + if band_bins.size == 0: + return [] + + candidates: list[GeometryCandidate] = [] + for frame_index, frame in enumerate(frames): + timestamp = _to_float(frame.get("timestamp")) + if timestamp is None: + continue + dt_ms = (timestamp - float(shot_timestamp)) * 1000.0 + if dt_ms < near_window_ms[0] or dt_ms > near_window_ms[1]: + continue + + radc_raw = frame.get("radc") + if radc_raw is None: + continue + try: + channels = parse_radc_payload(radc_raw) if isinstance(radc_raw, bytes) else radc_raw + except (KeyError, TypeError, ValueError): + continue + + f1a_iq = to_complex_iq(channels["f1a_i"], channels["f1a_q"]) + f2a_iq = to_complex_iq(channels["f2a_i"], channels["f2a_q"]) + f1b_iq = ( + to_complex_iq(channels["f1b_i"], channels["f1b_q"]) + if "f1b_i" in channels and "f1b_q" in channels + else None + ) + f1a_fft = compute_fft_complex(f1a_iq, fft_size=fft_size) + f2a_fft = compute_fft_complex(f2a_iq, fft_size=fft_size) + f1b_fft = compute_fft_complex(f1b_iq, fft_size=fft_size) if f1b_iq is not None else None + spec = spectrum_from_channel_ffts(f1a_fft, f2a_fft, f1b_fft, source="f1a") + positive = spec[spec > 0] + if positive.size == 0: + continue + noise_floor = float(np.median(positive)) + if noise_floor <= 0.0: + continue + + snrs = spec[band_bins] / noise_floor + order = np.argsort(-snrs) + band_rank_by_bin = {int(band_bins[idx]): rank + 1 for rank, idx in enumerate(order)} + angles = per_bin_angle_deg(f1a_fft, f2a_fft) + + for bin_index_raw, snr_raw in zip(band_bins, snrs, strict=True): + bin_index = int(bin_index_raw) + snr = float(snr_raw) + if snr < min_snr: + continue + bin_error = circular_bin_distance(bin_index, expected_bin, fft_size=fft_size) + if bin_error > max_bin_error: + continue + + bearing = float(angles[bin_index] + angle_offset_deg) + single = fit_launch_angle_single_frame_geometric( + (dt_ms / 1000.0, bearing, snr * snr), + target.ball_speed_mph, + distance_ft, + mount_deg, + ) + if single is None: + continue + launch, resid = single + is_local_peak, local_prominence = _local_peak(spec, bin_index) + candidates.append( + GeometryCandidate( + shot_number=target.shot_number, + club=target.club, + frame_index=frame_index, + dt_ms=dt_ms, + bin_index=bin_index, + expected_bin=expected_bin, + bin_error=bin_error, + snr=snr, + snr_db=10.0 * math.log10(snr) if snr > 0.0 else 0.0, + bearing_deg=bearing, + single_launch_deg=launch, + single_resid_deg=resid, + band_rank=band_rank_by_bin[bin_index], + local_peak=is_local_peak, + local_prominence=local_prominence, + trackman_angle_deg=target.trackman_angle_deg, + single_abs_error_deg=abs(launch - target.trackman_angle_deg), + ) + ) + return candidates + + +def _candidate_pre_score( + candidate: GeometryCandidate, + *, + target_time_ms: float, + weights: ScoreWeights, +) -> float: + time_penalty = abs(candidate.dt_ms - target_time_ms) / 50.0 + bin_penalty = candidate.bin_error / 25.0 + snr_reward = min(math.log2(max(candidate.snr, 1.0)), 5.0) / 5.0 + rank_penalty = min(candidate.band_rank - 1, 50) / 25.0 + local_penalty = 0.0 if candidate.local_peak else weights.local_peak_penalty + return ( + candidate.single_resid_deg * weights.single_resid + + bin_penalty * weights.bin_error + + time_penalty * weights.time + + rank_penalty * weights.band_rank + + local_penalty + - snr_reward * weights.snr_reward + ) + + +def rank_single( + candidates: list[GeometryCandidate], + *, + target_time_ms: float, + weights: ScoreWeights, + launch_min_deg: float | None, + launch_max_deg: float | None, +) -> GeometryPick | None: + filtered = [ + candidate + for candidate in candidates + if (launch_min_deg is None or candidate.single_launch_deg >= launch_min_deg) + and (launch_max_deg is None or candidate.single_launch_deg <= launch_max_deg) + ] + if not filtered: + return None + best = min( + filtered, + key=lambda cand: _candidate_pre_score(cand, target_time_ms=target_time_ms, weights=weights), + ) + score = _candidate_pre_score(best, target_time_ms=target_time_ms, weights=weights) + return GeometryPick( + shot_number=best.shot_number, + club=best.club, + trackman_angle_deg=best.trackman_angle_deg, + estimator="single", + launch_angle_deg=round(best.single_launch_deg, 3), + abs_error_deg=abs(best.single_launch_deg - best.trackman_angle_deg), + score=score, + frame_count=1, + frame_indices=str([best.frame_index]), + dt_ms=json.dumps([round(best.dt_ms, 1)]), + bins=str([best.bin_index]), + bin_errors=str([best.bin_error]), + snrs=json.dumps([round(best.snr, 3)]), + fit_rmse_deg=None, + single_resid_deg=best.single_resid_deg, + reason="ok", + ) + + +def _pair_score( + first: GeometryCandidate, + second: GeometryCandidate, + *, + fit_rmse: float, + target_time_ms: float, + weights: ScoreWeights, +) -> float: + avg_bin = (first.bin_error + second.bin_error) / 2.0 + avg_rank = (min(first.band_rank, 50) + min(second.band_rank, 50)) / 2.0 + min_snr = min(first.snr, second.snr) + local_penalty = (0.0 if first.local_peak else weights.local_peak_penalty) + ( + 0.0 if second.local_peak else weights.local_peak_penalty + ) + time_penalty = ( + abs(first.dt_ms - target_time_ms) + abs(second.dt_ms - target_time_ms) + ) / 100.0 + span_ms = abs(second.dt_ms - first.dt_ms) + span_penalty = weights.short_span_penalty if span_ms < 8.0 else 0.0 + snr_reward = min(math.log2(max(min_snr, 1.0)), 5.0) / 5.0 + return ( + fit_rmse * weights.pair_rmse + + (avg_bin / 25.0) * weights.bin_error + + (avg_rank / 25.0) * weights.band_rank + + time_penalty * weights.time + + span_penalty + + local_penalty + - snr_reward * weights.snr_reward + ) + + +def rank_pair( + candidates: list[GeometryCandidate], + *, + ball_speed_mph: float, + mount_deg: float, + distance_ft: float, + target_time_ms: float, + max_pairs_per_frame: int, + weights: ScoreWeights, + launch_min_deg: float | None, + launch_max_deg: float | None, +) -> tuple[GeometryPick | None, list[dict[str, Any]]]: + by_frame: dict[int, list[GeometryCandidate]] = {} + for candidate in candidates: + by_frame.setdefault(candidate.frame_index, []).append(candidate) + trimmed: list[GeometryCandidate] = [] + for frame_candidates in by_frame.values(): + trimmed.extend( + sorted( + frame_candidates, + key=lambda cand: _candidate_pre_score( + cand, + target_time_ms=target_time_ms, + weights=weights, + ), + )[:max_pairs_per_frame] + ) + + pair_rows: list[dict[str, Any]] = [] + best_pick: GeometryPick | None = None + best_score = math.inf + ordered = sorted(trimmed, key=lambda cand: (cand.dt_ms, cand.frame_index, cand.bin_index)) + for left_idx, first in enumerate(ordered): + for second in ordered[left_idx + 1 :]: + if second.frame_index == first.frame_index: + continue + if second.dt_ms <= first.dt_ms: + continue + fit = fit_launch_angle_geometric( + [ + (first.dt_ms / 1000.0, first.bearing_deg, first.snr * first.snr), + (second.dt_ms / 1000.0, second.bearing_deg, second.snr * second.snr), + ], + ball_speed_mph, + distance_ft, + mount_deg, + ) + if fit is None: + continue + launch, rmse, _ = fit + if launch_min_deg is not None and launch < launch_min_deg: + continue + if launch_max_deg is not None and launch > launch_max_deg: + continue + score = _pair_score( + first, + second, + fit_rmse=rmse, + target_time_ms=target_time_ms, + weights=weights, + ) + row = { + "shot_number": first.shot_number, + "club": first.club, + "trackman_angle_deg": first.trackman_angle_deg, + "launch_angle_deg": launch, + "abs_error_deg": abs(launch - first.trackman_angle_deg), + "score": score, + "fit_rmse_deg": rmse, + "frame_indices": [first.frame_index, second.frame_index], + "dt_ms": [first.dt_ms, second.dt_ms], + "bins": [first.bin_index, second.bin_index], + "bin_errors": [first.bin_error, second.bin_error], + "snrs": [first.snr, second.snr], + "bearings": [first.bearing_deg, second.bearing_deg], + "local_peaks": [first.local_peak, second.local_peak], + "band_ranks": [first.band_rank, second.band_rank], + } + pair_rows.append(row) + if score < best_score: + best_score = score + best_pick = GeometryPick( + shot_number=first.shot_number, + club=first.club, + trackman_angle_deg=first.trackman_angle_deg, + estimator="pair", + launch_angle_deg=round(launch, 3), + abs_error_deg=abs(launch - first.trackman_angle_deg), + score=score, + frame_count=2, + frame_indices=str(row["frame_indices"]), + dt_ms=json.dumps([round(v, 1) for v in row["dt_ms"]]), + bins=str(row["bins"]), + bin_errors=str(row["bin_errors"]), + snrs=json.dumps([round(v, 3) for v in row["snrs"]]), + fit_rmse_deg=rmse, + single_resid_deg=None, + reason="ok", + ) + return best_pick, sorted(pair_rows, key=lambda row: row["score"]) + + +def _empty_pick(target: TrackmanTarget, reason: str) -> GeometryPick: + return GeometryPick( + shot_number=target.shot_number, + club=target.club, + trackman_angle_deg=target.trackman_angle_deg, + estimator="none", + launch_angle_deg=None, + abs_error_deg=None, + score=None, + frame_count=0, + frame_indices="[]", + dt_ms="[]", + bins="[]", + bin_errors="[]", + snrs="[]", + fit_rmse_deg=None, + single_resid_deg=None, + reason=reason, + ) + + +def _write_csv(path: Path, rows: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fieldnames = sorted({key for row in rows for key in row}) + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + +def _summary(rows: list[GeometryPick]) -> dict[str, Any]: + detected = [row for row in rows if row.abs_error_deg is not None] + errors = [float(row.abs_error_deg) for row in detected] + by_estimator: dict[str, int] = {} + by_reason: dict[str, int] = {} + for row in rows: + by_estimator[row.estimator] = by_estimator.get(row.estimator, 0) + 1 + by_reason[row.reason] = by_reason.get(row.reason, 0) + 1 + return { + "targets": len(rows), + "detected": len(detected), + "detection_rate": len(detected) / len(rows) if rows else 0.0, + "mae": statistics.fmean(errors) if errors else None, + "median_abs_error": statistics.median(errors) if errors else None, + "p90_abs_error": sorted(errors)[round(0.9 * (len(errors) - 1))] if errors else None, + "max_abs_error": max(errors) if errors else None, + "within_1_deg": sum(error <= 1.0 for error in errors), + "within_2_deg": sum(error <= 2.0 for error in errors), + "within_5_deg": sum(error <= 5.0 for error in errors), + "estimator_counts": dict(sorted(by_estimator.items())), + "reason_counts": dict(sorted(by_reason.items())), + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--openflight", required=True, type=Path) + parser.add_argument("--comparison", required=True, type=Path) + parser.add_argument("--summary-output", required=True, type=Path) + parser.add_argument("--rows-output", required=True, type=Path) + parser.add_argument("--candidates-output", required=True, type=Path) + parser.add_argument("--pairs-output", required=True, type=Path) + parser.add_argument("--detail-shot", type=int, default=None) + parser.add_argument("--near-window-ms", type=_parse_window_ms, default=(0.0, 150.0)) + parser.add_argument("--speed-tolerance", type=float, default=10.0) + parser.add_argument("--max-bin-error", type=int, default=25) + parser.add_argument("--min-snr", type=float, default=2.0) + parser.add_argument("--angle-offset", type=float, default=0.0) + parser.add_argument("--mount-deg", type=float, default=10.0) + parser.add_argument("--distance-ft", type=float, default=5.0) + parser.add_argument("--target-time-ms", type=float, default=55.0) + parser.add_argument("--fft-size", type=int, default=2048) + parser.add_argument("--max-pairs-per-frame", type=int, default=8) + parser.add_argument("--single-resid-weight", type=float, default=3.0) + parser.add_argument("--pair-rmse-weight", type=float, default=6.0) + parser.add_argument("--bin-error-weight", type=float, default=1.0) + parser.add_argument("--time-weight", type=float, default=1.0) + parser.add_argument("--band-rank-weight", type=float, default=1.0) + parser.add_argument("--local-peak-penalty", type=float, default=0.75) + parser.add_argument("--snr-reward-weight", type=float, default=1.0) + parser.add_argument("--short-span-penalty", type=float, default=1.0) + parser.add_argument("--launch-min-deg", type=float, default=None) + parser.add_argument("--launch-max-deg", type=float, default=None) + args = parser.parse_args() + weights = ScoreWeights( + single_resid=args.single_resid_weight, + pair_rmse=args.pair_rmse_weight, + bin_error=args.bin_error_weight, + time=args.time_weight, + band_rank=args.band_rank_weight, + local_peak_penalty=args.local_peak_penalty, + snr_reward=args.snr_reward_weight, + short_span_penalty=args.short_span_penalty, + ) + + targets = [ + target + for target in load_targets(args.comparison, axis="vertical") + if target.orientation == "vertical" + ] + buffers = load_buffers(args.openflight) + shot_timestamps: dict[int, float] = {} + with args.openflight.open("r", encoding="utf-8") as handle: + for line in handle: + entry = json.loads(line) + if entry.get("type") != "kld7_buffer" or entry.get("orientation") != "vertical": + continue + shot_number = int(entry.get("shot_number") or -1) + timestamp = _to_float(entry.get("shot_timestamp")) + if shot_number >= 1 and timestamp is not None: + shot_timestamps[shot_number] = timestamp + + rows: list[GeometryPick] = [] + candidate_rows: list[dict[str, Any]] = [] + pair_rows: list[dict[str, Any]] = [] + for target in targets: + buffer = buffers.get((target.shot_number, "vertical")) + if not buffer: + rows.append(_empty_pick(target, "missing_buffer")) + continue + + shot_timestamp = shot_timestamps.get(target.shot_number) + if shot_timestamp is None: + rows.append(_empty_pick(target, "missing_shot_timestamp")) + continue + + candidates = enumerate_candidates( + target, + buffer, + shot_timestamp=shot_timestamp, + near_window_ms=args.near_window_ms, + speed_tolerance_mph=args.speed_tolerance, + max_bin_error=args.max_bin_error, + min_snr=args.min_snr, + angle_offset_deg=args.angle_offset, + mount_deg=args.mount_deg, + distance_ft=args.distance_ft, + fft_size=args.fft_size, + ) + if args.detail_shot is None or target.shot_number == args.detail_shot: + candidate_rows.extend(asdict(candidate) for candidate in candidates) + + single_pick = rank_single( + candidates, + target_time_ms=args.target_time_ms, + weights=weights, + launch_min_deg=args.launch_min_deg, + launch_max_deg=args.launch_max_deg, + ) + pair_pick, pairs = rank_pair( + candidates, + ball_speed_mph=target.ball_speed_mph, + mount_deg=args.mount_deg, + distance_ft=args.distance_ft, + target_time_ms=args.target_time_ms, + max_pairs_per_frame=args.max_pairs_per_frame, + weights=weights, + launch_min_deg=args.launch_min_deg, + launch_max_deg=args.launch_max_deg, + ) + if args.detail_shot is None or target.shot_number == args.detail_shot: + pair_rows.extend(pairs[:200]) + + pick = pair_pick or single_pick + rows.append(pick if pick is not None else _empty_pick(target, "no_geometry_candidate")) + + _write_csv(args.rows_output, [asdict(row) for row in rows]) + _write_csv(args.candidates_output, candidate_rows) + _write_csv(args.pairs_output, pair_rows) + + payload = { + "params": { + "near_window_ms": list(args.near_window_ms), + "speed_tolerance_mph": args.speed_tolerance, + "max_bin_error": args.max_bin_error, + "min_snr": args.min_snr, + "angle_offset_deg": args.angle_offset, + "mount_deg": args.mount_deg, + "distance_ft": args.distance_ft, + "target_time_ms": args.target_time_ms, + "max_pairs_per_frame": args.max_pairs_per_frame, + "launch_min_deg": args.launch_min_deg, + "launch_max_deg": args.launch_max_deg, + "score_weights": asdict(weights), + }, + "summary": _summary(rows), + } + args.summary_output.parent.mkdir(parents=True, exist_ok=True) + args.summary_output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + print(json.dumps(payload, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/analysis/replay_spin_dechirp.py b/scripts/analysis/replay_spin_dechirp.py new file mode 100644 index 000000000..189d82f4b --- /dev/null +++ b/scripts/analysis/replay_spin_dechirp.py @@ -0,0 +1,386 @@ +#!/usr/bin/env python3 +"""Replay raw rolling-buffer captures through a dechirped Doppler-sideband +spin estimator and score it against TrackMan truth alongside the production +envelope detector. + +Offline only — no production behavior changes. Scaffold for the "dechirped +coherent sideband spectrum" approach (TrackMan-style, single window): + +1. Track the ball's Doppler carrier f_d(t) with a short-window STFT. + The ball decelerates ~3-4 kHz/s, smearing the carrier (and every + sideband) ~200 Hz over a 60 ms window — fatal for resolving sideband + spacings of 42-167 Hz (2500-10000 RPM) in a plain FFT. +2. Mix the raw I/Q by the conjugate chirp so the carrier becomes a + stationary tone at 0 Hz and spin sidebands sit at exactly +/- m*f_mod. +3. Search a harmonic comb over candidate modulation frequencies, scoring + symmetric sideband-pair support in the dechirped spectrum. +4. Disambiguate the harmonic number: the seam's 2-fold symmetry + modulates at 2x spin "more often than not" (FlightScope US9868044), + while a logo/asymmetry modulates at 1x. Both interpretations are + emitted; odd-harmonic support at 1.5x the comb fundamental indicates + the true fundamental is half the observed spacing. + +Usage: + uv run --no-sync python scripts/analysis/replay_spin_dechirp.py \ + --openflight session_logs/session_20260511_120001_range.jsonl \ + --comparison session_logs/comparison_test2.csv \ + --output session_logs/spin_dechirp_replay_test2.csv + +Full reference (requirements, pipeline, baseline results, next steps): +docs/spin-dechirp-replay.md +""" + +from __future__ import annotations + +import argparse +import csv +import statistics +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Optional + +import numpy as np + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(PROJECT_ROOT / "src")) +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import compare_trackman as ct # noqa: E402 pylint: disable=wrong-import-position +from experiment_spin_windows import ( # noqa: E402 + _load_session_entries, + _load_trackman_by_shot, + _to_int, +) + +from openflight.rolling_buffer.processor import RollingBufferProcessor # noqa: E402 +from openflight.rolling_buffer.types import IQCapture # noqa: E402 + +SAMPLE_RATE = 30000 +WAVELENGTH_M = 0.01243 +MPS_TO_MPH = 2.23694 + +# Carrier tracking +STFT_WINDOW = 256 # ~8.5 ms — 117 Hz resolution, enough to track the carrier +STFT_STEP = 64 # ~2.1 ms hop +TRACK_TOLERANCE_HZ = 900 # search window around expected ball Doppler +TRACK_MIN_SNR = 3.0 # STFT peak vs frame median to count as "carrier present" + +# Sideband search (after dechirp, carrier at 0 Hz) +SIDEBAND_FFT_SIZE = 1 << 17 # zero-padded for fine comb sampling +MOD_MIN_HZ = 33.0 # 2000 RPM at 1x +MOD_MAX_HZ = 400.0 # 12000 RPM at 2x seam modulation +COMB_STEP_HZ = 0.5 +COMB_HARMONICS = (1.0, 2.0, 3.0) +COMB_HARMONIC_WEIGHTS = (1.0, 0.6, 0.35) +ODD_HARMONIC_RATIO = 0.35 # support at 1.5x fundamental => spacing is really f/2 +MIN_USABLE_MS = 25.0 +SPIN_PLAUSIBLE_RPM = (1500.0, 13000.0) + + +@dataclass +class CarrierTrack: + """Quadratic fit of the ball Doppler carrier over its usable window.""" + + poly: np.ndarray # np.polyfit coeffs of f_d(t) in Hz vs seconds + start_sample: int + end_sample: int + + @property + def usable_ms(self) -> float: + return (self.end_sample - self.start_sample) / SAMPLE_RATE * 1000 + + +def track_carrier( + iq: np.ndarray, + ball_speed_mph: float, + onset_sample: int, +) -> Optional[CarrierTrack]: + """Fit f_d(t) from STFT peaks near the expected ball Doppler. + + Also determines the usable window: the contiguous run of STFT frames + (starting at ball onset) whose carrier peak stays above TRACK_MIN_SNR. + """ + expected_hz = 2 * (ball_speed_mph / MPS_TO_MPH) / WAVELENGTH_M + window = np.hanning(STFT_WINDOW) + freqs = np.fft.fftfreq(STFT_WINDOW, d=1 / SAMPLE_RATE) + band = (np.abs(freqs - expected_hz) <= TRACK_TOLERANCE_HZ) & (freqs > 0) + if not band.any(): + return None + + times, peaks, mags = [], [], [] + end_of_track = onset_sample + for start in range(onset_sample, len(iq) - STFT_WINDOW, STFT_STEP): + seg = iq[start : start + STFT_WINDOW] * window + spec = np.abs(np.fft.fft(seg)) + floor = float(np.median(spec[spec > 0])) or 1.0 + in_band = spec[band] + peak_idx = int(np.argmax(in_band)) + snr = float(in_band[peak_idx]) / floor + if snr < TRACK_MIN_SNR: + # Carrier lost (net impact / out of range): stop at the first + # loss after we have accumulated some track. + if len(times) >= 5: + break + continue + times.append((start + STFT_WINDOW / 2) / SAMPLE_RATE) + peaks.append(float(freqs[band][peak_idx])) + mags.append(float(in_band[peak_idx])) + end_of_track = start + STFT_WINDOW + + if len(times) < 5: + return None + t = np.array(times) + f = np.array(peaks) + w = np.sqrt(np.array(mags)) + order = 2 if len(times) >= 8 else 1 + poly = np.polyfit(t, f, order, w=w) + return CarrierTrack(poly=poly, start_sample=onset_sample, end_sample=end_of_track) + + +def dechirp(iq: np.ndarray, track: CarrierTrack) -> np.ndarray: + """Mix the usable window by the conjugate carrier chirp (carrier -> 0 Hz).""" + segment = iq[track.start_sample : track.end_sample] + t = (track.start_sample + np.arange(len(segment))) / SAMPLE_RATE + phase = 2 * np.pi * np.polyval(np.polyint(track.poly), t) + return segment * np.exp(-1j * phase) + + +@dataclass +class SidebandEstimate: + mod_freq_hz: float # fundamental sideband spacing found by the comb + comb_score: float # harmonic-comb support (vs noise floor) + odd_support: float # support at 1.5x fundamental (harmonic-number evidence) + spin_rpm_1x: float # spin if the modulating feature is 1x (logo) + spin_rpm_2x: float # spin if the modulation is the 2x seam + best_rpm: float # disambiguated pick + harmonic_choice: str # "1x" | "2x" + + +def sideband_spin(baseband: np.ndarray) -> Optional[SidebandEstimate]: + """Find the spin modulation frequency as symmetric sidebands around 0 Hz.""" + windowed = baseband * np.hanning(len(baseband)) + spectrum = np.abs(np.fft.fft(windowed, SIDEBAND_FFT_SIZE)) + + df = SAMPLE_RATE / SIDEBAND_FFT_SIZE + half_cell_bins = max(1, int(round(SAMPLE_RATE / len(baseband) / 2 / df))) + # Local floor window: wide enough to estimate the carrier-skirt level + # at that offset, excluding the sideband cell itself. + floor_bins = 8 * half_cell_bins + + def side_snr(spectrum_half: np.ndarray, k: int) -> float: + cell = spectrum_half[k - half_cell_bins : k + half_cell_bins + 1] + lo = max(1, k - floor_bins) + hi = min(len(spectrum_half) - 1, k + floor_bins) + ring = np.concatenate( + [ + spectrum_half[lo : k - half_cell_bins], + spectrum_half[k + half_cell_bins + 1 : hi], + ] + ) + ring = ring[ring > 0] + if cell.size == 0 or ring.size < 4: + return 0.0 + # The dechirped carrier's skirt slopes steeply: normalize each + # sideband by its own neighborhood, not a global floor, or the + # lowest candidate frequency always wins. + return float(cell.max()) / float(np.median(ring)) + + upper_half = spectrum[: SIDEBAND_FFT_SIZE // 2] + lower_half = spectrum[SIDEBAND_FFT_SIZE // 2 :][::-1] # mirrored negative freqs + + def mag_at(freq_hz: float) -> tuple[float, float]: + # local-floor SNR within half a natural-resolution cell of +/-freq_hz + k = int(round(freq_hz / df)) + if k - floor_bins < 1 or k + floor_bins >= SIDEBAND_FFT_SIZE // 2: + return 0.0, 0.0 + # lower_half[j] = spectrum[N-1-j] = bin at -(j+1)*df, so -k*df is j=k-1 + return side_snr(upper_half, k), side_snr(lower_half, k - 1) + + candidates = np.arange(MOD_MIN_HZ, MOD_MAX_HZ, COMB_STEP_HZ) + scores = np.zeros(len(candidates)) + for i, f0 in enumerate(candidates): + score = 0.0 + for harmonic, weight in zip(COMB_HARMONICS, COMB_HARMONIC_WEIGHTS): + upper, lower = mag_at(f0 * harmonic) + # symmetric-pair support: both sidebands must be present + score += weight * min(upper, lower) + scores[i] = score + + best = int(np.argmax(scores)) + f_mod = float(candidates[best]) + comb_score = float(scores[best]) + if comb_score <= 0: + return None + + # Harmonic-number disambiguation: if there is sideband support at + # 1.5x f_mod (an odd multiple of f_mod/2), the true fundamental is + # f_mod/2 and f_mod was its second harmonic. + upper, lower = mag_at(1.5 * f_mod) + odd_support = min(upper, lower) + + spin_1x = f_mod * 60.0 + spin_2x = f_mod / 2.0 * 60.0 + if odd_support >= ODD_HARMONIC_RATIO * comb_score: + choice, best_rpm = "2x", spin_2x + else: + # Default to 1x when the comb fundamental itself is plausible + # spin; prefer 2x when 1x would be implausibly high. + if spin_1x > SPIN_PLAUSIBLE_RPM[1] and SPIN_PLAUSIBLE_RPM[0] <= spin_2x: + choice, best_rpm = "2x", spin_2x + else: + choice, best_rpm = "1x", spin_1x + + return SidebandEstimate( + mod_freq_hz=f_mod, + comb_score=comb_score, + odd_support=float(odd_support), + spin_rpm_1x=spin_1x, + spin_rpm_2x=spin_2x, + best_rpm=best_rpm, + harmonic_choice=choice, + ) + + +def replay_shot( + processor: RollingBufferProcessor, + capture: IQCapture, +) -> tuple[Optional[Any], Optional[SidebandEstimate], dict[str, Any]]: + """Run production processing and the dechirped estimator on one capture.""" + processed = processor.process_capture(capture) + diag: dict[str, Any] = {} + if not processed: + return None, None, diag + + i_data = np.array(capture.i_samples, dtype=np.float64) + q_data = np.array(capture.q_samples, dtype=np.float64) + iq = (i_data - i_data.mean()) + 1j * (q_data - q_data.mean()) + + onset = max(0, int(processed.ball_timestamp_ms * SAMPLE_RATE / 1000)) + track = track_carrier(iq, processed.ball_speed_mph, onset) + if track is None: + diag["dechirp_skip"] = "no_carrier_track" + return processed, None, diag + + diag["usable_ms"] = round(track.usable_ms, 1) + diag["chirp_hz_per_s"] = round(float(track.poly[-2]) if len(track.poly) >= 2 else 0.0) + if track.usable_ms < MIN_USABLE_MS: + diag["dechirp_skip"] = f"usable_window_{track.usable_ms:.0f}ms" + return processed, None, diag + + estimate = sideband_spin(dechirp(iq, track)) + return processed, estimate, diag + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--openflight", type=Path, required=True, help="Session JSONL") + parser.add_argument("--comparison", type=Path, required=True, help="TrackMan CSV") + parser.add_argument("--output", type=Path, required=True, help="Per-shot output CSV") + args = parser.parse_args() + + shots, captures = _load_session_entries(args.openflight) + trackman_by_shot = _load_trackman_by_shot(args.comparison) + processor = RollingBufferProcessor(sample_rate=SAMPLE_RATE) + + rows: list[dict[str, Any]] = [] + for shot_entry, capture_entry in zip(shots, captures): + shot_data = shot_entry.get("data", shot_entry) + shot_number = _to_int(shot_data.get("shot_number")) + truth = trackman_by_shot.get(shot_number or -1, {}) + if truth.get("match_quality") != "good" or truth.get("spin_tm") is None: + continue + + capture = IQCapture( + sample_time=capture_entry.get("sample_time", 0), + trigger_time=capture_entry.get("trigger_time", 0), + i_samples=capture_entry["i_samples"], + q_samples=capture_entry["q_samples"], + ) + processed, estimate, diag = replay_shot(processor, capture) + if not processed: + continue + + spin_tm = truth["spin_tm"] + baseline = processed.spin + row: dict[str, Any] = { + "shot_number": shot_number, + "club": ct.normalize_club(shot_data.get("club")), + "spin_tm": spin_tm, + "ball_speed_of": round(processed.ball_speed_mph, 1), + "baseline_rpm": baseline.spin_rpm if baseline and baseline.spin_rpm else None, + "baseline_quality": baseline.quality if baseline else None, + "baseline_snr": baseline.snr if baseline else None, + "usable_ms": diag.get("usable_ms"), + "chirp_hz_per_s": diag.get("chirp_hz_per_s"), + "dechirp_skip": diag.get("dechirp_skip"), + } + if estimate: + row.update( + { + "dechirp_rpm": round(estimate.best_rpm), + "dechirp_choice": estimate.harmonic_choice, + "dechirp_rpm_1x": round(estimate.spin_rpm_1x), + "dechirp_rpm_2x": round(estimate.spin_rpm_2x), + "dechirp_score": round(estimate.comb_score, 2), + "dechirp_odd_support": round(estimate.odd_support, 2), + } + ) + rows.append(row) + + if not rows: + print("No paired shots with TrackMan spin truth found.") + return + + fieldnames = sorted({key for row in rows for key in row}) + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + _print_summary(rows) + print(f"\nPer-shot rows written to {args.output}") + + +def _summarize(label: str, pairs: list[tuple[float, float]], total: int) -> None: + if not pairs: + print(f"{label:28s} coverage 0/{total}") + return + errors = [abs(measured - truth) for measured, truth in pairs] + pct = [abs(m - t) / t * 100 for m, t in pairs if t > 0] + within10 = sum(1 for p in pct if p <= 10) + print( + f"{label:28s} coverage {len(pairs)}/{total} " + f"MAE {statistics.mean(errors):6.0f} rpm " + f"median {statistics.median(errors):6.0f} rpm " + f"within10% {within10}/{len(pairs)}" + ) + + +def _print_summary(rows: list[dict[str, Any]]) -> None: + total = len(rows) + baseline_pairs = [ + (row["baseline_rpm"], row["spin_tm"]) for row in rows if row.get("baseline_rpm") + ] + dechirp_pairs = [(row["dechirp_rpm"], row["spin_tm"]) for row in rows if row.get("dechirp_rpm")] + # Oracle: the better of the 1x/2x interpretations per shot — the + # ceiling a perfect harmonic-disambiguation rule could reach. + oracle_pairs = [] + for row in rows: + if row.get("dechirp_rpm_1x"): + truth = row["spin_tm"] + best = min( + (row["dechirp_rpm_1x"], row["dechirp_rpm_2x"]), + key=lambda rpm: abs(rpm - truth), + ) + oracle_pairs.append((best, truth)) + + print(f"\n=== Spin replay summary ({total} TrackMan-paired shots) ===") + _summarize("production envelope", baseline_pairs, total) + _summarize("dechirped sidebands", dechirp_pairs, total) + _summarize("dechirp + harmonic oracle", oracle_pairs, total) + + +if __name__ == "__main__": + main() diff --git a/scripts/analysis/sweep_ballistic_coeffs.py b/scripts/analysis/sweep_ballistic_coeffs.py new file mode 100644 index 000000000..35514a5e1 --- /dev/null +++ b/scripts/analysis/sweep_ballistic_coeffs.py @@ -0,0 +1,566 @@ +"""Tune the four aerodynamic coefficients in +:mod:`openflight.ballistics` against TrackMan-measured carry. + +Parameters swept: + CD_BASE — drag coefficient at zero spin + CD_SPIN_COEFF — slope of Cd(Sp) (linear in spin parameter) + CL_SATURATION — Cl asymptote at high Sp + CL_HALF_SP — Sp at which Cl reaches CL_SATURATION/2 + +Method: scipy.optimize.differential_evolution (global) followed by +Nelder-Mead refinement. Loss is overall RMSE on the TrackMan-inputs +shots (model fed TM's own measurements vs TM's flat carry). + +Air density is set to 1.184 kg/m³ to match TrackMan's "Flat" +normalization (no wind, 0 ft alt, 77 °F). + +Usage (single session):: + + uv run python scripts/analysis/sweep_ballistic_coeffs.py \\ + --trackman session_logs/OpenFlight-Test.Normalized.csv \\ + --output-dir session_logs/sweep_20260506 + +Usage (multiple sessions, with leave-one-session-out CV):: + + uv run python scripts/analysis/sweep_ballistic_coeffs.py \\ + --trackman session_logs/tm_a.csv session_logs/tm_b.csv \\ + --session-label may06 jun02 \\ + --loso \\ + --output-dir session_logs/sweep_multi + +Multi-session shots are concatenated for the main fit. ``--loso`` then +refits on N-1 sessions per fold and scores on the held-out one — if the +held-out RMSE is much worse than the fit RMSE, the optimizer is +absorbing session-specific noise and the coefficients won't generalize. + +The script does not modify ballistics.py — it prints the optimal +coefficients and writes a stats/scatter report so the change can be +reviewed before being committed. +""" + +from __future__ import annotations + +import argparse +import csv +import math +import statistics +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, List, Tuple + +import numpy as np +from scipy.optimize import differential_evolution, minimize + +_REPO_ROOT = Path(__file__).resolve().parents[2] +if str(_REPO_ROOT / "src") not in sys.path: + sys.path.insert(0, str(_REPO_ROOT / "src")) +if str(_REPO_ROOT / "scripts" / "analysis") not in sys.path: + sys.path.insert(0, str(_REPO_ROOT / "scripts" / "analysis")) + +import openflight.ballistics as bl # noqa: E402 +from openflight.ballistics import LaunchConditions, simulate # noqa: E402 + +# Reuse the TM CSV loader from the validation script so the data path is +# identical between the two tools. +from validate_ballistics import ( # noqa: E402 + TM_FLAT_AIR_DENSITY, + TMShot, + _default_session_label, + _normalize_club, + load_trackman, +) + + +# Bounds for the search. Wide enough to span published Cd/Cl ranges for +# dimpled golf balls in the post-drag-crisis regime; narrow enough that +# differential evolution converges in a few minutes. +PARAM_BOUNDS: List[Tuple[float, float]] = [ + (0.16, 0.28), # CD_BASE + (0.00, 0.40), # CD_SPIN_COEFF + (0.18, 0.42), # CL_SATURATION + (0.03, 0.35), # CL_HALF_SP +] +PARAM_NAMES = ["CD_BASE", "CD_SPIN_COEFF", "CL_SATURATION", "CL_HALF_SP"] +DEFAULT_COEFFS = ( + bl.CD_BASE, + bl.CD_SPIN_COEFF, + bl.CL_SATURATION, + bl.CL_HALF_SP, +) + + +@dataclass +class FitResult: + coeffs: Tuple[float, float, float, float] + rmse: float + preds: List[float] + + +def _build_conditions(s: TMShot) -> LaunchConditions: + return LaunchConditions( + ball_speed_mph=s.ball_speed_mph, + launch_angle_v=s.launch_v_deg, + launch_angle_h=s.launch_h_deg if s.launch_h_deg is not None else 0.0, + spin_rpm=s.spin_rpm, + spin_axis_deg=s.spin_axis_deg if s.spin_axis_deg is not None else 0.0, + spin_source="measured", + ) + + +def _filter_shots(shots: List[TMShot]) -> List[TMShot]: + """Keep only shots with the inputs the model needs + a measured carry.""" + out = [] + for s in shots: + if s.ball_speed_mph is None or s.launch_v_deg is None: + continue + if s.spin_rpm is None or s.carry_yards is None: + continue + out.append(s) + return out + + +def simulate_with_coeffs( + shots: List[TMShot], + coeffs: Tuple[float, float, float, float], +) -> List[float]: + """Monkey-patch the ballistics module constants, run simulate() for + every shot, restore the originals on exit. + + Relies on ``ballistics._cd`` and ``ballistics._cl`` resolving the + constants at call time from the module's global namespace. + """ + saved = (bl.CD_BASE, bl.CD_SPIN_COEFF, bl.CL_SATURATION, bl.CL_HALF_SP) + bl.CD_BASE, bl.CD_SPIN_COEFF, bl.CL_SATURATION, bl.CL_HALF_SP = coeffs + try: + results = [] + for s in shots: + traj = simulate(_build_conditions(s), air_density=TM_FLAT_AIR_DENSITY) + results.append(traj.carry_yards) + return results + finally: + bl.CD_BASE, bl.CD_SPIN_COEFF, bl.CL_SATURATION, bl.CL_HALF_SP = saved + + +def make_loss(shots: List[TMShot], measured: np.ndarray): + """Closure that the optimizer can call with a 4-vector.""" + def _loss(x: np.ndarray) -> float: + preds = simulate_with_coeffs(shots, tuple(x)) + return float(np.sqrt(np.mean((np.asarray(preds) - measured) ** 2))) + return _loss + + +def evaluate( + shots: List[TMShot], + measured: np.ndarray, + coeffs: Tuple[float, float, float, float], +) -> FitResult: + preds = simulate_with_coeffs(shots, coeffs) + rmse = float(np.sqrt(np.mean((np.asarray(preds) - measured) ** 2))) + return FitResult(coeffs=coeffs, rmse=rmse, preds=preds) + + +# --------------------------------------------------------------------------- +# Reporting +# --------------------------------------------------------------------------- + +def _stats(deltas: List[float]) -> Dict[str, float]: + if not deltas: + return {"n": 0, "mean": float("nan"), "stdev": float("nan"), + "rmse": float("nan"), "mae": float("nan"), "max_abs": float("nan")} + mean = statistics.fmean(deltas) + stdev = statistics.pstdev(deltas) if len(deltas) > 1 else 0.0 + rmse = math.sqrt(sum(d * d for d in deltas) / len(deltas)) + mae = sum(abs(d) for d in deltas) / len(deltas) + return {"n": len(deltas), "mean": mean, "stdev": stdev, + "rmse": rmse, "mae": mae, "max_abs": max(abs(d) for d in deltas)} + + +def format_table( + label: str, + shots: List[TMShot], + measured: List[float], + preds: List[float], +) -> str: + """Stats block grouped by club + (when multiple) by session.""" + by_club: Dict[str, List[float]] = {} + by_session: Dict[str, List[float]] = {} + for s, m, p in zip(shots, measured, preds): + delta = p - m + by_club.setdefault(_normalize_club(s.club_raw), []).append(delta) + by_session.setdefault(s.session or "default", []).append(delta) + + overall = _stats([p - m for p, m in zip(preds, measured)]) + lines = [f"=== {label} ==="] + lines.append( + f"OVERALL n={overall['n']:3d} bias={overall['mean']:+6.2f} yd " + f"rmse={overall['rmse']:5.2f} yd mae={overall['mae']:5.2f} yd " + f"max|d|={overall['max_abs']:5.2f} yd" + ) + lines.append("By club:") + lines.append(f" {'club':14s} {'n':>3s} {'bias':>8s} {'rmse':>6s} " + f"{'mae':>6s} {'max|d|':>7s}") + for club in sorted(by_club): + s = _stats(by_club[club]) + lines.append( + f" {club:14s} {s['n']:3d} {s['mean']:+7.2f} {s['rmse']:6.2f} " + f"{s['mae']:6.2f} {s['max_abs']:7.2f}" + ) + if len(by_session) > 1: + lines.append("By session:") + lines.append(f" {'session':24s} {'n':>3s} {'bias':>8s} {'rmse':>6s} " + f"{'mae':>6s} {'max|d|':>7s}") + for sess in sorted(by_session): + s = _stats(by_session[sess]) + sess_disp = sess if len(sess) <= 24 else sess[:21] + "..." + lines.append( + f" {sess_disp:24s} {s['n']:3d} {s['mean']:+7.2f} {s['rmse']:6.2f} " + f"{s['mae']:6.2f} {s['max_abs']:7.2f}" + ) + return "\n".join(lines) + + +def write_per_shot_csv( + shots: List[TMShot], + measured: List[float], + preds_default: List[float], + preds_fit: List[float], + out_path: Path, +) -> None: + out_path.parent.mkdir(parents=True, exist_ok=True) + with open(out_path, "w", newline="", encoding="utf-8") as fh: + w = csv.writer(fh) + w.writerow([ + "session", "shot_index", "club", + "ball_speed_mph", "launch_v_deg", "launch_h_deg", + "spin_rpm", "spin_axis_deg", + "measured_carry_yards", + "model_default_yards", "delta_default", + "model_fit_yards", "delta_fit", + ]) + for s, m, pd_, pf in zip(shots, measured, preds_default, preds_fit): + w.writerow([ + s.session or "default", + s.shot_index, _normalize_club(s.club_raw), + f"{s.ball_speed_mph:.2f}", f"{s.launch_v_deg:.2f}", + f"{(s.launch_h_deg or 0.0):.2f}", + f"{s.spin_rpm:.0f}", f"{(s.spin_axis_deg or 0.0):.2f}", + f"{m:.2f}", + f"{pd_:.2f}", f"{pd_ - m:+.2f}", + f"{pf:.2f}", f"{pf - m:+.2f}", + ]) + + +def write_scatter( + shots: List[TMShot], + measured: List[float], + preds_default: List[float], + preds_fit: List[float], + out_path: Path, + fit_coeffs: Tuple[float, float, float, float], +) -> None: + import matplotlib + + matplotlib.use("Agg") + import matplotlib.pyplot as plt + + out_path.parent.mkdir(parents=True, exist_ok=True) + clubs = sorted({_normalize_club(s.club_raw) for s in shots}) + sessions = sorted({(s.session or "default") for s in shots}) + color_for = {c: plt.cm.tab10(i % 10) for i, c in enumerate(clubs)} + marker_cycle = ["o", "s", "^", "D", "v", "P", "X", "*", "<", ">"] + marker_for = {s: marker_cycle[i % len(marker_cycle)] for i, s in enumerate(sessions)} + single_session = len(sessions) == 1 + + fig, axes = plt.subplots(1, 2, figsize=(13, 6.5), sharex=True, sharey=True) + + for ax, preds, label in ( + (axes[0], preds_default, "default"), + (axes[1], preds_fit, "fit"), + ): + for club in clubs: + for sess in sessions: + xs, ys = [], [] + for s, m, p in zip(shots, measured, preds): + if _normalize_club(s.club_raw) != club: + continue + if (s.session or "default") != sess: + continue + xs.append(m) + ys.append(p) + if not xs: + continue + leg_label = ( + f"{club} (n={len(xs)})" if single_session + else f"{club}/{sess} (n={len(xs)})" + ) + ax.scatter(xs, ys, label=leg_label, + color=color_for[club], marker=marker_for[sess], + s=42, alpha=0.85, edgecolor="k", linewidth=0.5) + all_x = list(measured) + all_y = preds + lo = min(min(all_x), min(all_y)) * 0.95 + hi = max(max(all_x), max(all_y)) * 1.05 + ax.plot([lo, hi], [lo, hi], "k--", lw=1, label="1:1") + ax.fill_between([lo, hi], [lo - 5, hi - 5], [lo + 5, hi + 5], + color="grey", alpha=0.15, label="+/-5 yd") + deltas = np.asarray(preds) - np.asarray(measured) + rmse = float(np.sqrt(np.mean(deltas ** 2))) + bias = float(np.mean(deltas)) + ax.set_title(f"{label}: rmse={rmse:.2f} yd bias={bias:+.2f} yd") + ax.set_xlabel("Measured carry (TM Flat, yd)") + ax.set_ylabel("Model carry (yd)") + ax.set_xlim(lo, hi) + ax.set_ylim(lo, hi) + ax.set_aspect("equal") + ax.grid(True, alpha=0.3) + ax.legend(loc="best", fontsize=8) + + fig.suptitle( + f"Default vs fit (fit: CD_BASE={fit_coeffs[0]:.4f}, " + f"CD_SPIN_COEFF={fit_coeffs[1]:.4f}, " + f"CL_SATURATION={fit_coeffs[2]:.4f}, " + f"CL_HALF_SP={fit_coeffs[3]:.4f})", + fontsize=11, + ) + fig.tight_layout() + fig.savefig(out_path, dpi=140) + plt.close(fig) + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def main(argv=None) -> int: + try: + sys.stdout.reconfigure(encoding="utf-8") # type: ignore[attr-defined] + except (AttributeError, ValueError): + pass + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--trackman", required=True, type=Path, nargs="+", + help="One or more TrackMan normalized CSVs. Shots " + "from all files are concatenated for the fit.") + parser.add_argument("--session-label", required=False, type=str, nargs="+", + help="Optional explicit session label per --trackman " + "file (default: filename stem with common " + "prefixes trimmed).") + parser.add_argument("--output-dir", required=True, type=Path) + parser.add_argument("--seed", type=int, default=42, + help="DE seed for reproducibility.") + parser.add_argument("--de-popsize", type=int, default=12, + help="Differential-evolution population size.") + parser.add_argument("--de-maxiter", type=int, default=40, + help="Differential-evolution max iterations.") + parser.add_argument("--de-tol", type=float, default=1e-3, + help="Differential-evolution tolerance.") + parser.add_argument("--loso", action="store_true", + help="Leave-one-session-out cross-validation: for " + "each session, refit on the others and report " + "the held-out RMSE. Only meaningful with " + "multiple --trackman files.") + args = parser.parse_args(argv) + + # Validate inputs. + for p in args.trackman: + if not p.exists(): + print(f"TrackMan CSV not found: {p}", file=sys.stderr) + return 2 + if args.session_label and len(args.session_label) != len(args.trackman): + print( + f"--session-label count ({len(args.session_label)}) must match " + f"--trackman count ({len(args.trackman)})", + file=sys.stderr, + ) + return 2 + + # Load all sessions, tagged with their labels. + labels = args.session_label or [_default_session_label(p) for p in args.trackman] + all_shots: List[TMShot] = [] + for path, label in zip(args.trackman, labels): + shots_in = load_trackman(path, session=label) + usable = _filter_shots(shots_in) + print(f"Loaded {len(shots_in)} TM shots from {path.name} [{label}] " + f"({len(usable)} usable)") + all_shots.extend(usable) + shots = all_shots + if not shots: + print("No usable shots loaded.", file=sys.stderr) + return 2 + measured = np.array([s.carry_yards for s in shots], dtype=float) + print(f"Total usable shots across {len(args.trackman)} session(s): {len(shots)}") + + # Baseline with current ballistics.py constants. + baseline = evaluate(shots, measured, DEFAULT_COEFFS) + print(f"Baseline RMSE: {baseline.rmse:.3f} yd " + f"(coeffs: {DEFAULT_COEFFS})") + + loss_fn = make_loss(shots, measured) + + # --- Global search: differential evolution --- + eval_counter = {"n": 0} + + def _logged_loss(x): + eval_counter["n"] += 1 + return loss_fn(x) + + print(f"\nDifferential evolution: popsize={args.de_popsize}, " + f"maxiter={args.de_maxiter}, seed={args.seed}") + t0 = time.time() + de_result = differential_evolution( + _logged_loss, + bounds=PARAM_BOUNDS, + seed=args.seed, + maxiter=args.de_maxiter, + popsize=args.de_popsize, + tol=args.de_tol, + polish=False, # we do our own Nelder-Mead refinement + updating="deferred", # vectorizable, but more importantly deterministic w/ seed + workers=1, + disp=False, + ) + de_elapsed = time.time() - t0 + print(f"DE converged in {eval_counter['n']} evals " + f"({de_elapsed:.1f}s). Best RMSE: {de_result.fun:.3f} yd") + print(f" -> {dict(zip(PARAM_NAMES, de_result.x))}") + + # --- Local refinement: Nelder-Mead from DE best --- + print("\nNelder-Mead refinement...") + t1 = time.time() + nm = minimize( + loss_fn, + x0=de_result.x, + method="Nelder-Mead", + bounds=PARAM_BOUNDS, + options={"xatol": 1e-5, "fatol": 1e-4, "maxiter": 400}, + ) + nm_elapsed = time.time() - t1 + print(f"NM finished in {nm.nit} iterations ({nm_elapsed:.1f}s). " + f"RMSE: {nm.fun:.3f} yd") + + # Pick whichever is better (NM should be ≤ DE). + if nm.fun <= de_result.fun: + best_x = tuple(float(v) for v in nm.x) + best_rmse = float(nm.fun) + else: + best_x = tuple(float(v) for v in de_result.x) + best_rmse = float(de_result.fun) + fit = evaluate(shots, measured, best_x) + assert abs(fit.rmse - best_rmse) < 1e-6, (fit.rmse, best_rmse) + + # --- Report --- + args.output_dir.mkdir(parents=True, exist_ok=True) + + table_default = format_table( + "Default coefficients", + shots, list(measured), baseline.preds, + ) + table_fit = format_table( + "Fit coefficients", + shots, list(measured), fit.preds, + ) + + fit_lines = [ + "Optimal coefficients (TM-inputs, RMSE objective):", + f" CD_BASE = {best_x[0]:.5f} (default {DEFAULT_COEFFS[0]:.5f})", + f" CD_SPIN_COEFF = {best_x[1]:.5f} (default {DEFAULT_COEFFS[1]:.5f})", + f" CL_SATURATION = {best_x[2]:.5f} (default {DEFAULT_COEFFS[2]:.5f})", + f" CL_HALF_SP = {best_x[3]:.5f} (default {DEFAULT_COEFFS[3]:.5f})", + "", + f"Baseline RMSE: {baseline.rmse:.3f} yd", + f"Fit RMSE: {fit.rmse:.3f} yd", + f"Improvement: {baseline.rmse - fit.rmse:+.3f} yd " + f"({100 * (baseline.rmse - fit.rmse) / baseline.rmse:+.1f}%)", + "", + table_default, + "", + table_fit, + ] + + # --- Leave-one-session-out cross-validation --- + # Only emit when --loso is requested AND there are multiple sessions. + # This is the honest generalization check: refit on N-1 sessions, score + # on the held-out one. If held-out RMSE is much worse than fit RMSE, + # the optimizer is finding session-specific quirks (overfit). + sessions_in_data = sorted({s.session or "default" for s in shots}) + if args.loso and len(sessions_in_data) > 1: + loso_lines = ["", "=== Leave-one-session-out cross-validation ==="] + loso_lines.append( + f"{'held-out':24s} {'n_test':>6s} {'fit_RMSE':>9s} " + f"{'test_RMSE':>10s} {'test_bias':>10s} fit_coeffs" + ) + for held_out in sessions_in_data: + train_shots = [s for s in shots if (s.session or "default") != held_out] + test_shots = [s for s in shots if (s.session or "default") == held_out] + if not train_shots or not test_shots: + continue + train_measured = np.array([s.carry_yards for s in train_shots]) + test_measured = np.array([s.carry_yards for s in test_shots]) + print(f"\n LOSO refit (held out: {held_out}, " + f"n_train={len(train_shots)}, n_test={len(test_shots)})...") + sub_loss = make_loss(train_shots, train_measured) + sub_de = differential_evolution( + sub_loss, + bounds=PARAM_BOUNDS, + seed=args.seed, + maxiter=args.de_maxiter, + popsize=args.de_popsize, + tol=args.de_tol, + polish=False, + updating="deferred", + workers=1, + disp=False, + ) + sub_nm = minimize( + sub_loss, + x0=sub_de.x, + method="Nelder-Mead", + bounds=PARAM_BOUNDS, + options={"xatol": 1e-5, "fatol": 1e-4, "maxiter": 400}, + ) + sub_best = tuple(float(v) for v in ( + sub_nm.x if sub_nm.fun <= sub_de.fun else sub_de.x + )) + sub_fit_rmse = float(min(sub_nm.fun, sub_de.fun)) + # Score on held-out session with the refit coeffs. + test_preds = simulate_with_coeffs(test_shots, sub_best) + test_deltas = np.asarray(test_preds) - test_measured + test_rmse = float(np.sqrt(np.mean(test_deltas ** 2))) + test_bias = float(np.mean(test_deltas)) + ho_disp = held_out if len(held_out) <= 24 else held_out[:21] + "..." + loso_lines.append( + f"{ho_disp:24s} {len(test_shots):>6d} {sub_fit_rmse:>9.3f} " + f"{test_rmse:>10.3f} {test_bias:>+10.3f} " + f"({sub_best[0]:.3f}, {sub_best[1]:.3f}, " + f"{sub_best[2]:.3f}, {sub_best[3]:.3f})" + ) + fit_lines.extend(loso_lines) + elif args.loso: + fit_lines.extend([ + "", + "(LOSO requested but only one session loaded — skipped.)", + ]) + out_stats = args.output_dir / "sweep_stats.txt" + out_stats.write_text("\n".join(fit_lines), encoding="utf-8") + print() + print("\n".join(fit_lines)) + + write_per_shot_csv( + shots, list(measured), baseline.preds, fit.preds, + args.output_dir / "sweep_per_shot.csv", + ) + write_scatter( + shots, list(measured), baseline.preds, fit.preds, + args.output_dir / "sweep_scatter.png", + best_x, + ) + print(f"\nWrote {out_stats}") + print(f"Wrote {args.output_dir / 'sweep_per_shot.csv'}") + print(f"Wrote {args.output_dir / 'sweep_scatter.png'}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/analysis/validate_ballistics.py b/scripts/analysis/validate_ballistics.py new file mode 100644 index 000000000..d10a68180 --- /dev/null +++ b/scripts/analysis/validate_ballistics.py @@ -0,0 +1,686 @@ +"""Validate the ballistic model against paired OpenFlight + TrackMan data. + +Two independent comparisons are produced: + +1. **TM-inputs** — feed TrackMan's measured (ball_speed, launch_v, + launch_h, spin_rpm, spin_axis) into :func:`openflight.ballistics.simulate` + and compare the model's ``carry_yards`` to TrackMan's ``Carry Flat - + Length``. This isolates the physics model from any sensor noise. + +2. **OF-inputs** — feed OpenFlight's measured values for the same shots + into the model and compare to TrackMan's carry. This is the + end-to-end check the user actually sees on screen (sensor + model + error combined). Spin handling matches production + :func:`openflight.ballistics.resolve_launch`: measured spin if + confidence >= SPIN_CONFIDENCE_HIGH, otherwise club-typical fallback. + OpenFlight has no spin-axis measurement, so spin_axis = 0 is assumed. + +TrackMan's "Flat" carry is normalized to no-wind, 0 ft altitude, 77 °F. +We pass ``air_density = 1.184 kg/m³`` (≈ 25 °C sea level) so the model +runs in the same atmosphere TrackMan reports for — otherwise the default +1.225 kg/m³ (15 °C ISA) would add a systematic ~2 yd model bias on +driver-distance shots. + +Usage (single session):: + + uv run python scripts/analysis/validate_ballistics.py \\ + --trackman session_logs/OpenFlight-Test.Normalized.csv \\ + --comparison session_logs/comparison_20260506.csv \\ + --output-dir session_logs/validation_20260506 + +Usage (multiple sessions — concatenates shots, tags each with a session +label, and emits per-session stats alongside the per-club stats):: + + uv run python scripts/analysis/validate_ballistics.py \\ + --trackman session_logs/tm_session_a.csv session_logs/tm_session_b.csv \\ + --comparison session_logs/comp_a.csv session_logs/comp_b.csv \\ + --session-label may06 jun02 \\ + --output-dir session_logs/validation_multi + +``--comparison`` (and ``--session-label``) must match the count of +``--trackman`` when provided. When ``--comparison`` is omitted the +OF-inputs run is skipped. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import math +import statistics +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +# Make the in-repo package importable when running via `uv run python ...` +# from the repo root (uv usually arranges this, but we also support a +# plain `python scripts/analysis/validate_ballistics.py` invocation). +_REPO_ROOT = Path(__file__).resolve().parents[2] +if str(_REPO_ROOT / "src") not in sys.path: + sys.path.insert(0, str(_REPO_ROOT / "src")) + +from openflight.ballistics import ( # noqa: E402 + CLUB_TYPICAL_SPIN_RPM, + LaunchConditions, + simulate, +) +from openflight.launch_monitor import SPIN_CONFIDENCE_HIGH, ClubType # noqa: E402 + +# TrackMan "Flat" normalization: no wind, 0 ft altitude, 77 °F. +# ρ = P / (R_specific · T) with P = 101325 Pa, T = 298.15 K, R = 287.05 J/(kg·K) +# → 1.1839 kg/m³. We round to 1.184 for the input. +TM_FLAT_AIR_DENSITY = 1.184 + +# Club-name → ClubType map. The comparison CSV uses normalized names like +# "driver", "7-iron", "pw"; the raw TM CSV uses "7 Iron", "Driver", "PW". +_CLUB_MAP: Dict[str, ClubType] = { + "driver": ClubType.DRIVER, + "3-wood": ClubType.WOOD_3, + "5-wood": ClubType.WOOD_5, + "7-wood": ClubType.WOOD_7, + "3-hybrid": ClubType.HYBRID_3, + "5-hybrid": ClubType.HYBRID_5, + "7-hybrid": ClubType.HYBRID_7, + "9-hybrid": ClubType.HYBRID_9, + "2-iron": ClubType.IRON_2, + "3-iron": ClubType.IRON_3, + "4-iron": ClubType.IRON_4, + "5-iron": ClubType.IRON_5, + "6-iron": ClubType.IRON_6, + "7-iron": ClubType.IRON_7, + "8-iron": ClubType.IRON_8, + "9-iron": ClubType.IRON_9, + "pw": ClubType.PW, + "gw": ClubType.GW, + "sw": ClubType.SW, + "lw": ClubType.LW, +} + + +def _normalize_club(raw: Optional[str]) -> str: + """Lower-case, collapse "7 Iron" / "7-iron" / "iron 7" → "7-iron" etc.""" + if raw is None: + return "" + s = str(raw).strip().lower().replace("_", "-") + if not s: + return "" + # Strip trailing punctuation. + s = s.rstrip(".") + # "7 iron" → "7-iron", "iron 7" → "7-iron" + parts = s.replace("-", " ").split() + if len(parts) == 2: + a, b = parts + if a.isdigit() and b in ("iron", "i", "wood", "w", "hybrid", "h"): + kind = {"iron": "iron", "i": "iron", "wood": "wood", "w": "wood", + "hybrid": "hybrid", "h": "hybrid"}[b] + return f"{a}-{kind}" + if b.isdigit() and a in ("iron", "wood", "hybrid"): + return f"{b}-{a}" + aliases = {"drv": "driver", "1-wood": "driver", "pitching-wedge": "pw", + "sand-wedge": "sw", "gap-wedge": "gw", "lob-wedge": "lw"} + return aliases.get(s, s) + + +def _club_type(name: str) -> ClubType: + return _CLUB_MAP.get(_normalize_club(name), ClubType.UNKNOWN) + + +def _to_float(v) -> Optional[float]: + if v is None or v == "": + return None + try: + return float(str(v).replace(",", "")) + except (TypeError, ValueError): + return None + + +@dataclass +class TMShot: + """Slim TrackMan record carrying only what the ballistic model needs.""" + + shot_index: int # row index in the CSV (1-based, after header/units rows) + club_raw: str + ball_speed_mph: Optional[float] + launch_v_deg: Optional[float] + launch_h_deg: Optional[float] + spin_rpm: Optional[float] + spin_axis_deg: Optional[float] + carry_yards: Optional[float] + timestamp: str + session: str = "" # session label (typically derived from the source filename) + + +def _default_session_label(path: Path) -> str: + """Cheap session label: filename stem, with any "OpenFlight-Test." + or "comparison_" prefix trimmed so output is readable.""" + stem = path.stem + for prefix in ("OpenFlight-Test.", "comparison_", "session_"): + if stem.startswith(prefix): + stem = stem[len(prefix):] + break + return stem or path.name + + +def load_trackman(path: Path, session: Optional[str] = None) -> List[TMShot]: + """Read a TrackMan "Normalized" CSV. Handles ``sep=,`` preamble and + the units row directly under the header. Each emitted shot is tagged + with ``session`` (defaulting to the file stem) so multi-session + aggregations can keep their origin straight.""" + session_label = session if session is not None else _default_session_label(path) + shots: List[TMShot] = [] + with open(path, "r", encoding="utf-8-sig", newline="") as fh: + # Skip optional ``sep=,`` Excel preamble. + pos = fh.tell() + first = fh.readline().lstrip("").lstrip() + if not first.lower().startswith("sep="): + fh.seek(pos) + reader = csv.DictReader(fh) + if reader.fieldnames is None: + return shots + # Trackman exports sometimes include a units row ("[mph]", "[deg]") + # right under the header. Detect by looking for "[..]" cells. + idx = 0 + for row in reader: + if any( + isinstance(v, str) and v.strip().startswith("[") and v.strip().endswith("]") + for v in row.values() + ): + continue + idx += 1 + shots.append(TMShot( + shot_index=idx, + club_raw=row.get("Club", "") or "", + ball_speed_mph=_to_float(row.get("Ball Speed")), + launch_v_deg=_to_float(row.get("Launch Angle")), + launch_h_deg=_to_float(row.get("Launch Direction")), + spin_rpm=_to_float(row.get("Spin Rate")), + spin_axis_deg=_to_float(row.get("Spin Axis")), + carry_yards=_to_float(row.get("Carry Flat - Length")), + timestamp=row.get("Date", "") or "", + session=session_label, + )) + return shots + + +@dataclass +class ComparisonRow: + """One row from the paired OF/TM comparison CSV.""" + + timestamp_of: str + club_raw: str + match_quality: str + # OpenFlight fields + ball_speed_of: Optional[float] + launch_v_of: Optional[float] + launch_h_of: Optional[float] + spin_of: Optional[float] + carry_of: Optional[float] + # TrackMan fields (for ground truth and fallback spin) + ball_speed_tm: Optional[float] + launch_v_tm: Optional[float] + launch_h_tm: Optional[float] + spin_tm: Optional[float] + spin_axis_tm: Optional[float] # may stay None — comparison CSV usually omits axis + carry_tm: Optional[float] + session: str = "" + + +def load_comparison(path: Path, session: Optional[str] = None) -> List[ComparisonRow]: + session_label = session if session is not None else _default_session_label(path) + rows: List[ComparisonRow] = [] + with open(path, "r", encoding="utf-8-sig", newline="") as fh: + reader = csv.DictReader(fh) + for r in reader: + rows.append(ComparisonRow( + timestamp_of=r.get("timestamp_of", "") or "", + club_raw=r.get("club", "") or "", + match_quality=r.get("match_quality", "") or "", + ball_speed_of=_to_float(r.get("ball_speed_of")), + launch_v_of=_to_float(r.get("launch_v_of")), + launch_h_of=_to_float(r.get("launch_h_of")), + spin_of=_to_float(r.get("spin_of")), + carry_of=_to_float(r.get("carry_of")), + ball_speed_tm=_to_float(r.get("ball_speed_tm")), + launch_v_tm=_to_float(r.get("launch_v_tm")), + launch_h_tm=_to_float(r.get("launch_h_tm")), + spin_tm=_to_float(r.get("spin_tm")), + spin_axis_tm=_to_float(r.get("spin_axis_tm")), + carry_tm=_to_float(r.get("carry_tm")), + session=session_label, + )) + return rows + + +def _spin_axis_lookup(tm_shots: List[TMShot]) -> Dict[Tuple[str, float, float], float]: + """Build (club, ball_speed_rounded, launch_v_rounded) -> spin_axis_deg + so comparison rows (which may lack spin_axis) can borrow TM's axis.""" + out: Dict[Tuple[str, float, float], float] = {} + for s in tm_shots: + if s.spin_axis_deg is None or s.ball_speed_mph is None or s.launch_v_deg is None: + continue + club = _normalize_club(s.club_raw) + key = (club, round(s.ball_speed_mph, 1), round(s.launch_v_deg, 1)) + out.setdefault(key, s.spin_axis_deg) + return out + + +@dataclass +class ValidationRow: + """One row of the per-shot validation CSV.""" + + source: str # "tm" or "of" + shot_label: str + club: str + session: str + ball_speed_mph: float + launch_v_deg: float + launch_h_deg: float + spin_rpm: float + spin_axis_deg: float + spin_source: str # "measured" / "club_typical" / "tm_borrowed" + measured_carry_yards: float + model_carry_yards: float + delta_yards: float # model - measured + + +def _has_required_inputs(*vals) -> bool: + return all(v is not None and not (isinstance(v, float) and math.isnan(v)) for v in vals) + + +def validate_tm_inputs( + tm_shots: List[TMShot], + air_density: float = TM_FLAT_AIR_DENSITY, +) -> List[ValidationRow]: + """Run the model on each TrackMan shot and compare to TM's flat carry.""" + out: List[ValidationRow] = [] + for s in tm_shots: + if not _has_required_inputs( + s.ball_speed_mph, s.launch_v_deg, s.spin_rpm, s.carry_yards + ): + continue + spin_axis = s.spin_axis_deg if s.spin_axis_deg is not None else 0.0 + launch_h = s.launch_h_deg if s.launch_h_deg is not None else 0.0 + conditions = LaunchConditions( + ball_speed_mph=s.ball_speed_mph, + launch_angle_v=s.launch_v_deg, + launch_angle_h=launch_h, + spin_rpm=s.spin_rpm, + spin_axis_deg=spin_axis, + spin_source="measured", + ) + traj = simulate(conditions, air_density=air_density) + session_tag = s.session or "default" + out.append(ValidationRow( + source="tm", + shot_label=f"tm#{s.shot_index}@{session_tag}", + club=_normalize_club(s.club_raw), + session=session_tag, + ball_speed_mph=s.ball_speed_mph, + launch_v_deg=s.launch_v_deg, + launch_h_deg=launch_h, + spin_rpm=s.spin_rpm, + spin_axis_deg=spin_axis, + spin_source="measured", + measured_carry_yards=s.carry_yards, + model_carry_yards=traj.carry_yards, + delta_yards=traj.carry_yards - s.carry_yards, + )) + return out + + +def _resolve_of_spin( + row: ComparisonRow, + club: ClubType, +) -> Tuple[float, str]: + """Mirror production resolve_launch(): use OF spin only when confident, + else club-typical fallback. The comparison CSV doesn't carry spin + confidence, so we treat any present OF spin as measured (this is the + same fallback the production server would log).""" + if row.spin_of is not None and row.spin_of > 0: + return row.spin_of, "measured" + typical = CLUB_TYPICAL_SPIN_RPM.get(club, CLUB_TYPICAL_SPIN_RPM[ClubType.UNKNOWN]) + return typical, "club_typical" + + +def validate_of_inputs( + comp_rows: List[ComparisonRow], + spin_axis_lookup: Dict[Tuple[str, float, float], float], + air_density: float = TM_FLAT_AIR_DENSITY, +) -> List[ValidationRow]: + """Run the model on OpenFlight inputs and compare to TrackMan's + flat carry. Skips rows flagged as ``ball_speed_mismatch`` etc.""" + out: List[ValidationRow] = [] + for i, row in enumerate(comp_rows, start=1): + if row.match_quality != "good": + continue + if not _has_required_inputs( + row.ball_speed_of, row.launch_v_of, row.carry_tm + ): + continue + club_enum = _club_type(row.club_raw) + spin_rpm, spin_source = _resolve_of_spin(row, club_enum) + # OF doesn't measure spin axis. Borrow TM's axis when we can + # match the shot; otherwise assume pure backspin (axis = 0). + spin_axis = 0.0 + if row.ball_speed_tm is not None and row.launch_v_tm is not None: + key = ( + _normalize_club(row.club_raw), + round(row.ball_speed_tm, 1), + round(row.launch_v_tm, 1), + ) + if key in spin_axis_lookup: + spin_axis = spin_axis_lookup[key] + launch_h = row.launch_h_of if row.launch_h_of is not None else 0.0 + conditions = LaunchConditions( + ball_speed_mph=row.ball_speed_of, + launch_angle_v=row.launch_v_of, + launch_angle_h=launch_h, + spin_rpm=spin_rpm, + spin_axis_deg=spin_axis, + spin_source="measured" if spin_source == "measured" else "club_typical", + ) + traj = simulate(conditions, air_density=air_density) + session_tag = row.session or "default" + out.append(ValidationRow( + source="of", + shot_label=f"of#{i}@{session_tag}@{row.timestamp_of[:19]}", + club=_normalize_club(row.club_raw), + session=session_tag, + ball_speed_mph=row.ball_speed_of, + launch_v_deg=row.launch_v_of, + launch_h_deg=launch_h, + spin_rpm=spin_rpm, + spin_axis_deg=spin_axis, + spin_source=spin_source, + measured_carry_yards=row.carry_tm, + model_carry_yards=traj.carry_yards, + delta_yards=traj.carry_yards - row.carry_tm, + )) + return out + + +# --------------------------------------------------------------------------- +# Output +# --------------------------------------------------------------------------- + +def write_per_shot_csv(rows: List[ValidationRow], path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", newline="", encoding="utf-8") as fh: + w = csv.writer(fh) + w.writerow([ + "source", "session", "shot_label", "club", + "ball_speed_mph", "launch_v_deg", "launch_h_deg", + "spin_rpm", "spin_axis_deg", "spin_source", + "measured_carry_yards", "model_carry_yards", + "delta_yards", "abs_delta_yards", + ]) + for r in rows: + w.writerow([ + r.source, r.session, r.shot_label, r.club, + f"{r.ball_speed_mph:.2f}", f"{r.launch_v_deg:.2f}", f"{r.launch_h_deg:.2f}", + f"{r.spin_rpm:.0f}", f"{r.spin_axis_deg:.2f}", r.spin_source, + f"{r.measured_carry_yards:.2f}", f"{r.model_carry_yards:.2f}", + f"{r.delta_yards:.2f}", f"{abs(r.delta_yards):.2f}", + ]) + + +def _stats(deltas: List[float]) -> Dict[str, float]: + if not deltas: + return {"n": 0, "mean": float("nan"), "stdev": float("nan"), + "rmse": float("nan"), "mae": float("nan"), "max_abs": float("nan")} + mean = statistics.fmean(deltas) + stdev = statistics.pstdev(deltas) if len(deltas) > 1 else 0.0 + rmse = math.sqrt(sum(d * d for d in deltas) / len(deltas)) + mae = sum(abs(d) for d in deltas) / len(deltas) + return { + "n": len(deltas), + "mean": mean, + "stdev": stdev, + "rmse": rmse, + "mae": mae, + "max_abs": max(abs(d) for d in deltas), + } + + +def format_stats(label: str, rows: List[ValidationRow]) -> str: + """Return a multi-line stats block: overall, per-club, per-session.""" + lines = [f"=== {label} ==="] + overall = _stats([r.delta_yards for r in rows]) + lines.append( + f"OVERALL n={overall['n']:3d} bias={overall['mean']:+6.2f} yd " + f"rmse={overall['rmse']:5.2f} yd mae={overall['mae']:5.2f} yd " + f"max|d|={overall['max_abs']:5.2f} yd" + ) + + # Per-club table + clubs = sorted({r.club for r in rows}) + lines.append("-" * 80) + lines.append("By club:") + lines.append(f" {'club':10s} {'n':>3s} {'bias':>8s} {'rmse':>6s} " + f"{'mae':>6s} {'max|d|':>7s} {'measured_n':>4s} {'fallback_n':>4s}") + for club in clubs: + club_rows = [r for r in rows if r.club == club] + s = _stats([r.delta_yards for r in club_rows]) + measured_n = sum(1 for r in club_rows if r.spin_source == "measured") + fallback_n = sum(1 for r in club_rows if r.spin_source == "club_typical") + lines.append( + f" {club:10s} {s['n']:3d} {s['mean']:+7.2f} {s['rmse']:6.2f} " + f"{s['mae']:6.2f} {s['max_abs']:7.2f} {measured_n:>10d} {fallback_n:>10d}" + ) + + # Per-session table — only shown when there's more than one session. + sessions = sorted({r.session for r in rows}) + if len(sessions) > 1: + lines.append("By session:") + lines.append(f" {'session':24s} {'n':>3s} {'bias':>8s} {'rmse':>6s} " + f"{'mae':>6s} {'max|d|':>7s}") + for sess in sessions: + sess_rows = [r for r in rows if r.session == sess] + s = _stats([r.delta_yards for r in sess_rows]) + sess_disp = sess if len(sess) <= 24 else sess[:21] + "..." + lines.append( + f" {sess_disp:24s} {s['n']:3d} {s['mean']:+7.2f} {s['rmse']:6.2f} " + f"{s['mae']:6.2f} {s['max_abs']:7.2f}" + ) + lines.append("") + return "\n".join(lines) + + +def write_scatter( + rows: List[ValidationRow], + title: str, + out_path: Path, +) -> None: + """Predicted-vs-measured scatter with 1:1 line and ±5 yd band.""" + try: + import matplotlib + + matplotlib.use("Agg") + import matplotlib.pyplot as plt + except ImportError: + print(f"[warn] matplotlib not available — skipping {out_path.name}", + file=sys.stderr) + return + + if not rows: + return + out_path.parent.mkdir(parents=True, exist_ok=True) + + # Color by club, marker by session (so multi-session runs are legible). + clubs = sorted({r.club for r in rows}) + sessions = sorted({r.session for r in rows}) + color_for = {c: plt.cm.tab10(i % 10) for i, c in enumerate(clubs)} + # Reused matplotlib markers — cycle if there are many sessions. + marker_cycle = ["o", "s", "^", "D", "v", "P", "X", "*", "<", ">"] + marker_for = {s: marker_cycle[i % len(marker_cycle)] for i, s in enumerate(sessions)} + + fig, ax = plt.subplots(figsize=(7, 7)) + for club in clubs: + for sess in sessions: + cr = [r for r in rows if r.club == club and r.session == sess] + if not cr: + continue + label = ( + f"{club} (n={len(cr)})" if len(sessions) == 1 + else f"{club}/{sess} (n={len(cr)})" + ) + ax.scatter( + [r.measured_carry_yards for r in cr], + [r.model_carry_yards for r in cr], + label=label, + color=color_for[club], + marker=marker_for[sess], + s=42, alpha=0.85, edgecolor="k", linewidth=0.5, + ) + all_x = [r.measured_carry_yards for r in rows] + all_y = [r.model_carry_yards for r in rows] + lo = min(min(all_x), min(all_y)) * 0.95 + hi = max(max(all_x), max(all_y)) * 1.05 + ax.plot([lo, hi], [lo, hi], "k--", lw=1, label="1:1") + ax.fill_between([lo, hi], [lo - 5, hi - 5], [lo + 5, hi + 5], + color="grey", alpha=0.15, label="+/-5 yd") + + stats = _stats([r.delta_yards for r in rows]) + ax.set_title( + f"{title}\nn={stats['n']} bias={stats['mean']:+.2f} yd " + f"rmse={stats['rmse']:.2f} yd mae={stats['mae']:.2f} yd" + ) + ax.set_xlabel("Measured carry (TrackMan Flat, yd)") + ax.set_ylabel("Model carry (yd)") + ax.set_xlim(lo, hi) + ax.set_ylim(lo, hi) + ax.set_aspect("equal") + ax.grid(True, alpha=0.3) + ax.legend(loc="best", fontsize=8) + fig.tight_layout() + fig.savefig(out_path, dpi=140) + plt.close(fig) + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def main(argv: Optional[List[str]] = None) -> int: + # Allow non-ASCII (e.g. arrows, Greek letters) in printed stats on + # Windows consoles whose default encoding is cp1252. + try: + sys.stdout.reconfigure(encoding="utf-8") # type: ignore[attr-defined] + except (AttributeError, ValueError): + pass + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--trackman", required=True, type=Path, nargs="+", + help="One or more TrackMan normalized CSVs (each " + "contains Spin Axis column). Multiple files " + "are concatenated; each shot is tagged with a " + "session label derived from the filename " + "(override with --session-label).") + parser.add_argument("--comparison", required=False, type=Path, nargs="+", + help="Paired OF/TM CSV(s) from compare_trackman.py " + "(used for OF-inputs validation). When passing " + "multiple, count must match --trackman and " + "files are paired by position so the spin-axis " + "lookup uses the matching session's TM data.") + parser.add_argument("--session-label", required=False, type=str, nargs="+", + help="Optional explicit labels (one per --trackman). " + "Default = filename stem with prefixes trimmed.") + parser.add_argument("--output-dir", required=True, type=Path, + help="Directory for per-shot CSVs, stats.txt, scatter PNGs.") + parser.add_argument("--air-density", type=float, default=TM_FLAT_AIR_DENSITY, + help=f"Air density (kg/m³) for simulate(). Default " + f"{TM_FLAT_AIR_DENSITY} matches TrackMan 'Flat' (77°F sea level).") + args = parser.parse_args(argv) + + # Resolve session labels. + if args.session_label and len(args.session_label) != len(args.trackman): + print( + f"--session-label count ({len(args.session_label)}) must match " + f"--trackman count ({len(args.trackman)})", + file=sys.stderr, + ) + return 2 + labels = args.session_label or [_default_session_label(p) for p in args.trackman] + + # Validate input files exist. + for p in args.trackman: + if not p.exists(): + print(f"TrackMan CSV not found: {p}", file=sys.stderr) + return 2 + if args.comparison: + if len(args.comparison) != len(args.trackman): + print( + f"--comparison count ({len(args.comparison)}) must match " + f"--trackman count ({len(args.trackman)})", + file=sys.stderr, + ) + return 2 + for p in args.comparison: + if not p.exists(): + print(f"Comparison CSV not found: {p}", file=sys.stderr) + return 2 + + # Load all TM shots and all comparison rows, tagged with session. + all_tm_shots: List[TMShot] = [] + per_session_tm: List[List[TMShot]] = [] # for per-session axis lookup + for path, label in zip(args.trackman, labels): + shots = load_trackman(path, session=label) + print(f"Loaded {len(shots)} TM shots from {path.name} [{label}]") + all_tm_shots.extend(shots) + per_session_tm.append(shots) + + # --- TM inputs --- + tm_rows = validate_tm_inputs(all_tm_shots, air_density=args.air_density) + write_per_shot_csv(tm_rows, args.output_dir / "validation_tm_inputs.csv") + write_scatter( + tm_rows, + f"TM-inputs validation (rho={args.air_density:.3f} kg/m³)", + args.output_dir / "scatter_tm_inputs.png", + ) + stats_text = [format_stats( + f"TM-inputs (model fed TrackMan measurements, rho={args.air_density:.3f})", + tm_rows, + )] + + # --- OF inputs --- + of_rows: List[ValidationRow] = [] + if args.comparison: + all_comp_rows: List[ComparisonRow] = [] + for comp_path, label, tm_subset in zip(args.comparison, labels, per_session_tm): + comp = load_comparison(comp_path, session=label) + print(f"Loaded {len(comp)} paired rows from {comp_path.name} [{label}]") + # Build axis lookup from this session's TM data — keeps the + # mapping unambiguous when different sessions share ball speeds. + axis_lookup = _spin_axis_lookup(tm_subset) + of_rows.extend(validate_of_inputs( + comp, axis_lookup, air_density=args.air_density, + )) + all_comp_rows.extend(comp) + write_per_shot_csv(of_rows, args.output_dir / "validation_of_inputs.csv") + write_scatter( + of_rows, + f"OF-inputs end-to-end (rho={args.air_density:.3f} kg/m³)", + args.output_dir / "scatter_of_inputs.png", + ) + stats_text.append(format_stats( + f"OF-inputs (OpenFlight measurements → model → TM carry, rho={args.air_density:.3f})", + of_rows, + )) + + out_stats = args.output_dir / "stats.txt" + out_stats.parent.mkdir(parents=True, exist_ok=True) + out_stats.write_text("\n".join(stats_text), encoding="utf-8") + + # Echo to stdout so the test runner immediately sees the result. + print() + print("\n".join(stats_text)) + print(f"\nWrote {out_stats}") + print(f"Wrote {args.output_dir / 'validation_tm_inputs.csv'}") + if of_rows: + print(f"Wrote {args.output_dir / 'validation_of_inputs.csv'}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/hardware-test/probe_kld7_timing.py b/scripts/hardware-test/probe_kld7_timing.py new file mode 100644 index 000000000..b8cd05ca0 --- /dev/null +++ b/scripts/hardware-test/probe_kld7_timing.py @@ -0,0 +1,543 @@ +#!/usr/bin/env python3 +"""Guarded K-LD7 timing and protocol probe.""" + +from __future__ import annotations + +import argparse +import json +import statistics +import struct +import sys +import time +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Iterable, Optional + +try: + import serial + from serial.tools.list_ports import comports +except ImportError: # pragma: no cover - operator environment issue + serial = None + comports = None + + +DEFAULT_BAUD = 3_000_000 +DEFAULT_START_BAUD = 115_200 +SUPPORTED_BAUD_RATES = [115_200, 460_800, 921_600, 2_000_000, 3_000_000] +MAX_PACKET_PAYLOAD_BYTES = 8192 +FRAME_CODES = { + "RADC": 0x01, + "RFFT": 0x02, + "PDAT": 0x04, + "TDAT": 0x08, + "DDAT": 0x10, + "DONE": 0x20, +} +DOCUMENTED_WRITE_COMMANDS = { + "RBFR", + "RSPI", + "RRAI", + "THOF", + "TRFT", + "VISU", + "MIRA", + "MARA", + "MIAN", + "MAAN", + "MISP", + "MASP", + "DEDI", + "RATH", + "ANTH", + "SPTH", + "DIG1", + "DIG2", + "DIG3", + "HOLD", + "MIDE", + "MIDS", +} +DESTRUCTIVE_COMMANDS = {"RFSE"} +PARAM_STRUCT_FORMAT = "<19s8B2b4Bb4BH2B" +PARAM_FIELDS = [ + "software_version", + "RBFR", + "RSPI", + "RRAI", + "THOF", + "TRFT", + "VISU", + "MIRA", + "MARA", + "MIAN", + "MAAN", + "MISP", + "MASP", + "DEDI", + "RATH", + "ANTH", + "SPTH", + "DIG1", + "DIG2", + "DIG3", + "HOLD", + "MIDE", + "MIDS", +] + + +@dataclass +class PacketRecord: + """One packet or packet-level error observed while probing.""" + + code: str + payload_bytes: int + command: Optional[str] = None + response_code: Optional[int] = None + send_monotonic: Optional[float] = None + first_byte_monotonic: Optional[float] = None + header_complete_monotonic: Optional[float] = None + complete_monotonic: Optional[float] = None + read_duration_ms: Optional[float] = None + done_frame: Optional[int] = None + error: Optional[str] = None + payload: bytes = field(default=b"", repr=False) + + +def build_packet(command: str, payload: bytes = b"") -> bytes: + """Build a K-LD7 binary command packet.""" + cmd = command.upper().encode("ascii") + if len(cmd) != 4: + raise ValueError("command must be exactly 4 ASCII characters") + return struct.pack("<4sI", cmd, len(payload)) + payload + + +def validate_probe_command(command: str, hex_payload: str) -> Optional[str]: + """Validate one explicit unsafe probe command specification.""" + try: + raw = command.encode("ascii") + except UnicodeEncodeError: + return "command must be ASCII" + if len(raw) != 4: + return "command must be exactly 4 ASCII characters" + if command != command.upper(): + return "command must be uppercase ASCII" + if len(hex_payload) % 2: + return "hex payload must have an even number of characters" + try: + bytes.fromhex(hex_payload) + except ValueError: + return "hex payload must be valid hexadecimal" + return None + + +def parse_frame_mask(value: str) -> int: + """Parse a comma-separated frame mask such as RADC,DONE.""" + mask = 0 + for raw_name in value.split(","): + name = raw_name.strip().upper() + if not name: + continue + if name not in FRAME_CODES: + raise argparse.ArgumentTypeError(f"unknown frame type {name!r}") + mask |= FRAME_CODES[name] + if mask == 0: + raise argparse.ArgumentTypeError("at least one frame type is required") + return mask + + +def _response_name(code: Optional[int]) -> Optional[str]: + names = { + 0: "OK", + 1: "UnknownCommand", + 2: "InvalidParameter", + 3: "InvalidRPSTVersion", + 4: "UARTError", + 5: "SensorBusy", + } + return names.get(code) + + +class KLD7Protocol: + """Small direct serial protocol wrapper for K-LD7 packets.""" + + def __init__(self, port_path: str, baud: int = DEFAULT_BAUD, timeout: float = 0.2): + if serial is None: + raise RuntimeError("pyserial is required for hardware probing") + self.port_path = port_path + self.baud = baud + self.port = serial.Serial( + port=port_path, + baudrate=DEFAULT_START_BAUD, + parity=serial.PARITY_EVEN, + stopbits=serial.STOPBITS_ONE, + timeout=timeout, + ) + self._init_device() + + def _init_device(self) -> None: + """Negotiate the requested baud rate, recovering once from a stuck 3M session.""" + try: + self._send_init() + except Exception: + self._send_gbye_at_3mbaud() + self._send_init() + + def _send_init(self) -> None: + if self.baud not in SUPPORTED_BAUD_RATES: + raise ValueError(f"unsupported baud rate: {self.baud}") + response = self.send_command( + "INIT", + SUPPORTED_BAUD_RATES.index(self.baud).to_bytes(4, "little", signed=True), + ) + if response.response_code != 0: + name = _response_name(response.response_code) or response.response_code + raise RuntimeError(f"INIT failed: {name}") + if self.baud != DEFAULT_START_BAUD: + self.port.baudrate = self.baud + + def _send_gbye_at_3mbaud(self) -> None: + try: + self.port.baudrate = DEFAULT_BAUD + self.port.reset_input_buffer() + self.port.write(build_packet("GBYE")) + self.port.flush() + time.sleep(0.3) + finally: + self.port.baudrate = DEFAULT_START_BAUD + + def _read_exact(self, n: int) -> tuple[bytes, Optional[float]]: + buf = b"" + first_byte_at = None + timeout = max(float(getattr(self.port, "timeout", 0.2) or 0.2), 0.2) + deadline = time.monotonic() + timeout + while len(buf) < n: + chunk = self.port.read(n - len(buf)) + if chunk: + if first_byte_at is None: + first_byte_at = time.monotonic() + buf += chunk + deadline = time.monotonic() + timeout + continue + if time.monotonic() >= deadline: + break + time.sleep(0.001) + return buf, first_byte_at + + def read_packet(self) -> PacketRecord: + started = time.monotonic() + header, first_byte_at = self._read_exact(8) + header_complete = time.monotonic() + if len(header) != 8: + return PacketRecord( + code="", + payload_bytes=0, + first_byte_monotonic=first_byte_at, + header_complete_monotonic=header_complete, + complete_monotonic=header_complete, + read_duration_ms=(header_complete - started) * 1000.0, + error=f"short header read: got {len(header)} of 8 bytes", + ) + raw_code, length = struct.unpack("<4sI", header) + code = raw_code.decode("ascii", errors="replace") + if length > MAX_PACKET_PAYLOAD_BYTES: + complete = time.monotonic() + return PacketRecord( + code=code, + payload_bytes=length, + first_byte_monotonic=first_byte_at, + header_complete_monotonic=header_complete, + complete_monotonic=complete, + read_duration_ms=(complete - (first_byte_at or started)) * 1000.0, + error=f"invalid payload length: {length} bytes", + ) + payload = b"" + payload_first = None + if length: + payload, payload_first = self._read_exact(length) + complete = time.monotonic() + error = None + if len(payload) != length: + error = f"short payload read: got {len(payload)} of {length} bytes" + done_frame = None + if code == "DONE" and len(payload) == 4: + done_frame = int.from_bytes(payload, "little", signed=False) + return PacketRecord( + code=code, + payload_bytes=length, + first_byte_monotonic=first_byte_at or payload_first, + header_complete_monotonic=header_complete, + complete_monotonic=complete, + read_duration_ms=(complete - (first_byte_at or started)) * 1000.0, + done_frame=done_frame, + error=error, + payload=payload, + ) + + def send_command(self, command: str, payload: bytes = b"") -> PacketRecord: + sent = time.monotonic() + self.port.reset_input_buffer() + self.port.write(build_packet(command, payload)) + self.port.flush() + packet = self.read_packet() + packet.command = command.upper() + packet.send_monotonic = sent + if packet.code == "RESP" and packet.payload: + packet.response_code = packet.payload[0] + return packet + + def request_frame(self, frame_mask: int) -> list[PacketRecord]: + records = [self.send_command("GNFD", int(frame_mask).to_bytes(4, "little", signed=True))] + expected = bin(frame_mask).count("1") + for _ in range(expected): + record = self.read_packet() + records.append(record) + if record.code == "DONE": + break + return records + + def read_params(self) -> tuple[dict, list[PacketRecord]]: + response = self.send_command("GRPS") + records = [response] + if response.response_code != 0: + return {}, records + packet = self.read_packet() + records.append(packet) + if packet.code != "RPST" or packet.error: + return {}, records + values = struct.unpack(PARAM_STRUCT_FORMAT, packet.payload) + params = {} + for name, value in zip(PARAM_FIELDS, values): + if name == "software_version": + params[name] = value.split(b"\x00", 1)[0].decode("ascii", errors="ignore") + else: + params[name] = int(value) + return params, records + + def set_param(self, command: str, value: int) -> PacketRecord: + if command not in DOCUMENTED_WRITE_COMMANDS: + raise ValueError(f"not a documented parameter command: {command}") + return self.send_command(command, int(value).to_bytes(4, "little", signed=True)) + + def restore_params(self, params: dict) -> list[PacketRecord]: + records = [] + for command in PARAM_FIELDS[1:]: + if command in params: + records.append(self.set_param(command, int(params[command]))) + return records + + def close(self) -> None: + try: + self.port.write(build_packet("GBYE")) + self.port.flush() + finally: + self.port.close() + + +def _percentile(values: list[float], percentile: float) -> Optional[float]: + if not values: + return None + values = sorted(values) + index = min(len(values) - 1, max(0, round((percentile / 100.0) * (len(values) - 1)))) + return values[index] + + +def summarize_packets(packets: list[PacketRecord], duration_s: float) -> dict: + """Summarize packet-level timing and frame continuity.""" + radc_packets = [p for p in packets if p.code == "RADC" and not p.error] + done_packets = [p for p in packets if p.code == "DONE" and not p.error] + done_frames = [p.done_frame for p in done_packets if p.done_frame is not None] + gaps = 0 + for previous, current in zip(done_frames, done_frames[1:]): + if current > previous + 1: + gaps += current - previous - 1 + read_durations = [p.read_duration_ms for p in radc_packets if p.read_duration_ms is not None] + errors: dict[str, int] = {} + for packet in packets: + if packet.error: + errors[packet.error] = errors.get(packet.error, 0) + 1 + return { + "duration_s": duration_s, + "radc_frames": len(radc_packets), + "done_frames": len(done_packets), + "effective_radc_hz": round(len(radc_packets) / duration_s, 3) if duration_s > 0 else 0.0, + "effective_done_hz": round(len(done_packets) / duration_s, 3) if duration_s > 0 else 0.0, + "done_frame_gaps": gaps, + "read_duration_ms_mean": statistics.mean(read_durations) if read_durations else None, + "read_duration_ms_p50": statistics.median(read_durations) if read_durations else None, + "read_duration_ms_p95": _percentile(read_durations, 95), + "errors": errors, + } + + +def parse_probe_command(command_spec: str) -> tuple[str, bytes]: + command, _, hex_payload = command_spec.partition(":") + return command, bytes.fromhex(hex_payload) + + +def find_kld7_port() -> Optional[str]: + """Return a K-LD7-like serial port only when exactly one candidate is present.""" + if comports is None: + return None + matches = [] + for port in comports(): + desc = (port.description or "").lower() + mfg = (port.manufacturer or "").lower() + if any(kw in desc for kw in ["ftdi", "cp210", "usb-serial", "uart"]) or any( + kw in mfg for kw in ["ftdi", "silicon labs"] + ): + matches.append(port.device) + return matches[0] if len(matches) == 1 else None + + +def measure(protocol: KLD7Protocol, frame_mask: int, duration_s: float) -> list[PacketRecord]: + """Request frames until the duration expires.""" + records = [] + deadline = time.monotonic() + duration_s + while time.monotonic() < deadline: + records.extend(protocol.request_frame(frame_mask)) + return records + + +def apply_documented_settings( + protocol: KLD7Protocol, args: argparse.Namespace +) -> list[PacketRecord]: + records = [] + if args.rrai is not None: + records.append(protocol.set_param("RRAI", args.rrai)) + if args.rbfr is not None: + records.append(protocol.set_param("RBFR", args.rbfr)) + return records + + +def run_unsafe_probes(protocol: KLD7Protocol, args: argparse.Namespace) -> list[PacketRecord]: + records = [] + for command_spec in args.probe_command: + command, payload = parse_probe_command(command_spec) + records.append(protocol.send_command(command, payload)) + _params, param_records = protocol.read_params() + records.extend(param_records) + return records + + +def write_jsonl(path: Path, records: Iterable[PacketRecord], summary: dict) -> None: + with path.open("w", encoding="utf-8") as handle: + for record in records: + data = asdict(record) + data.pop("payload", None) + handle.write(json.dumps({"type": "packet", **data}, sort_keys=True) + "\n") + handle.write(json.dumps({"type": "summary", **summary}, sort_keys=True) + "\n") + + +def _run_one_configuration( + args: argparse.Namespace, rspi: Optional[int] = None +) -> tuple[list, dict]: + port = args.port or find_kld7_port() + if not port: + raise RuntimeError( + "--port is required unless exactly one K-LD7-like serial port is present" + ) + frame_mask = parse_frame_mask(args.frame_mask) + protocol = KLD7Protocol(port, baud=args.baud) + records: list[PacketRecord] = [] + params_before: dict = {} + try: + params_before, param_records = protocol.read_params() + records.extend(param_records) + records.extend(apply_documented_settings(protocol, args)) + if rspi is not None: + records.append(protocol.set_param("RSPI", rspi)) + params_active, param_records = protocol.read_params() + records.extend(param_records) + if args.unsafe_probe: + records.extend(run_unsafe_probes(protocol, args)) + measurement_records = measure(protocol, frame_mask, args.duration) + records.extend(measurement_records) + summary = summarize_packets(measurement_records, args.duration) + summary.update( + { + "port": port, + "baud": args.baud, + "frame_mask": [name for name, value in FRAME_CODES.items() if frame_mask & value], + "params_before": params_before, + "params_active": params_active, + } + ) + if rspi is not None: + summary["rspi_sweep_value"] = rspi + return records, summary + finally: + if params_before and not args.no_restore_params: + records.extend(protocol.restore_params(params_before)) + protocol.close() + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Probe K-LD7 RADC timing and guarded commands.") + parser.add_argument("--port") + parser.add_argument("--baud", type=int, default=DEFAULT_BAUD, choices=SUPPORTED_BAUD_RATES) + parser.add_argument("--duration", type=float, default=10.0) + parser.add_argument("--frame-mask", default="RADC,DONE") + parser.add_argument("--rspi-sweep", action="store_true") + parser.add_argument("--rrai", type=int) + parser.add_argument("--rbfr", type=int) + parser.add_argument("--output", type=Path) + parser.add_argument("--unsafe-probe", action="store_true") + parser.add_argument("--probe-command", action="append", default=[]) + parser.add_argument("--allow-factory-reset", action="store_true") + parser.add_argument("--no-restore-params", action="store_true") + return parser + + +def validate_args(args: argparse.Namespace) -> list[str]: + errors = [] + if args.duration <= 0: + errors.append("--duration must be positive") + if args.unsafe_probe and not args.output: + errors.append("--unsafe-probe requires --output so probe activity is auditable") + if args.probe_command and not args.unsafe_probe: + errors.append("--probe-command requires --unsafe-probe") + if args.no_restore_params and not args.unsafe_probe: + errors.append("--no-restore-params requires --unsafe-probe") + try: + parse_frame_mask(args.frame_mask) + except argparse.ArgumentTypeError as exc: + errors.append(str(exc)) + for command_spec in args.probe_command: + command, _, hex_payload = command_spec.partition(":") + error = validate_probe_command(command, hex_payload) + if error: + errors.append(f"{command_spec}: {error}") + if command.upper() in DESTRUCTIVE_COMMANDS and not args.allow_factory_reset: + errors.append("RFSE is refused unless --allow-factory-reset is set") + return errors + + +def main(argv: Optional[list[str]] = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + errors = validate_args(args) + if errors: + for error in errors: + print(f"error: {error}", file=sys.stderr) + return 2 + all_records = [] + summaries = [] + try: + rspi_values: Iterable[Optional[int]] = range(4) if args.rspi_sweep else [None] + for rspi in rspi_values: + records, summary = _run_one_configuration(args, rspi) + all_records.extend(records) + summaries.append(summary) + except Exception as exc: # pylint: disable=broad-except + print(f"error: {exc}", file=sys.stderr) + return 1 + output = summaries[0] if len(summaries) == 1 else {"runs": summaries} + print(json.dumps(output, indent=2, sort_keys=True)) + if args.output: + write_jsonl(args.output, all_records, output) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/hardware-test/test_kld7.py b/scripts/hardware-test/test_kld7.py index 34970e9c7..aaf92f0a0 100755 --- a/scripts/hardware-test/test_kld7.py +++ b/scripts/hardware-test/test_kld7.py @@ -23,7 +23,7 @@ (tracked target) since the tracking filter may not lock on in time. Prerequisites: - pip install kld7 pyserial + uv pip install -e '.[ui]' # kld7 ships as a base dependency Usage: # Basic capture (auto-detect port, Ctrl+C to stop) @@ -56,7 +56,7 @@ try: from kld7 import KLD7, FrameCode, KLD7Exception except ImportError: - print("Error: kld7 package not installed. Run: pip install kld7") + print("Error: kld7 package not installed. Reinstall the project: uv pip install -e '.[ui]'") sys.exit(1) from serial.tools.list_ports import comports diff --git a/scripts/setup/openflight-cloud.service b/scripts/setup/openflight-cloud.service new file mode 100644 index 000000000..987f75a2f --- /dev/null +++ b/scripts/setup/openflight-cloud.service @@ -0,0 +1,12 @@ +[Unit] +Description=OpenFlight cloud uploader (push filtered sessions to FlightWeb) +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User=coleman +WorkingDirectory=/home/coleman/openflight +# Uses the venv's console script; falls back to module form if not on PATH. +ExecStart=/home/coleman/openflight/.venv/bin/openflight-cloud push +# Opt-in: this no-ops until the user runs `openflight-cloud link`. diff --git a/scripts/setup/openflight-cloud.timer b/scripts/setup/openflight-cloud.timer new file mode 100644 index 000000000..6d27d415c --- /dev/null +++ b/scripts/setup/openflight-cloud.timer @@ -0,0 +1,13 @@ +[Unit] +Description=Run the OpenFlight cloud uploader every ~10 minutes (heals wifi outages) + +[Timer] +# Fire shortly after boot, then every 10 minutes. RandomizedDelaySec spreads +# load so a fleet of Pis doesn't hit the server in lockstep. +OnBootSec=2min +OnUnitActiveSec=10min +RandomizedDelaySec=60 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/scripts/setup/setup.sh b/scripts/setup/setup.sh index bdac4b619..88821355d 100755 --- a/scripts/setup/setup.sh +++ b/scripts/setup/setup.sh @@ -1,8 +1,20 @@ #!/bin/bash # # OpenFlight Setup Script -# Installs all Python and Node.js dependencies for first-time setup # +# Installs all dependencies, then (on a Raspberry Pi) walks through the +# one-time hardware configuration interactively: +# - OPS243-A rolling buffer flash config +# - K-LD7 device naming + FTDI low-latency rules +# - Auto-start on boot (systemd service) +# - Desktop shortcut +# +# Usage: +# ./scripts/setup/setup.sh # full interactive setup +# ./scripts/setup/setup.sh --deps-only # install dependencies, skip hardware +# ./scripts/setup/setup.sh --non-interactive # no prompts (deps only) +# +# Safe to re-run at any time. set -e @@ -16,20 +28,46 @@ YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # No Color -log() { - echo -e "${GREEN}[OpenFlight]${NC} $1" -} +log() { echo -e "${GREEN}[OpenFlight]${NC} $1"; } +warn() { echo -e "${YELLOW}[OpenFlight]${NC} $1"; } +error() { echo -e "${RED}[OpenFlight]${NC} $1"; } +info() { echo -e "${BLUE}[OpenFlight]${NC} $1"; } -warn() { - echo -e "${YELLOW}[OpenFlight]${NC} $1" -} +INTERACTIVE=true +DEPS_ONLY=false -error() { - echo -e "${RED}[OpenFlight]${NC} $1" -} +while [[ $# -gt 0 ]]; do + case "$1" in + --non-interactive) + INTERACTIVE=false + DEPS_ONLY=true + shift + ;; + --deps-only) + DEPS_ONLY=true + shift + ;; + --help|-h) + awk 'NR>1 && !/^#/{exit} NR>1{sub(/^# ?/,""); print}' "$0" + exit 0 + ;; + *) + error "Unknown option: $1 (try --help)" + exit 1 + ;; + esac +done -info() { - echo -e "${BLUE}[OpenFlight]${NC} $1" +# Ask a yes/no question. Returns 0 for yes. Usage: confirm "Question?" [Y|N] +confirm() { + local question="$1" + local default="${2:-N}" + local prompt suffix answer + if [ "$default" == "Y" ]; then suffix="[Y/n]"; else suffix="[y/N]"; fi + prompt="$(echo -e "${BLUE}[OpenFlight]${NC} ${question} ${suffix} ")" + read -r -p "$prompt" answer + answer="${answer:-$default}" + [[ "$answer" =~ ^[Yy]$ ]] } cd "$PROJECT_DIR" @@ -58,12 +96,16 @@ else warn "Unknown platform: $OSTYPE" fi +# ────────────────────────────────────────────────────────────────────── +# Phase 1: Dependencies +# ────────────────────────────────────────────────────────────────────── + # Check for Python 3.9+ log "Checking Python version..." if command -v python3 &> /dev/null; then PYTHON_VERSION=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') - PYTHON_MAJOR=$(echo $PYTHON_VERSION | cut -d. -f1) - PYTHON_MINOR=$(echo $PYTHON_VERSION | cut -d. -f2) + PYTHON_MAJOR=$(echo "$PYTHON_VERSION" | cut -d. -f1) + PYTHON_MINOR=$(echo "$PYTHON_VERSION" | cut -d. -f2) if [ "$PYTHON_MAJOR" -ge 3 ] && [ "$PYTHON_MINOR" -ge 9 ]; then log "Python $PYTHON_VERSION found ✓" @@ -102,15 +144,12 @@ if ! command -v uv &> /dev/null; then log "Installing uv (fast Python package manager)..." curl -LsSf https://astral.sh/uv/install.sh | sh # Source the new path - export PATH="$HOME/.cargo/bin:$PATH" + export PATH="$HOME/.cargo/bin:$HOME/.local/bin:$PATH" fi # Create virtual environment -log "Creating Python virtual environment..." -if [ "$PLATFORM" == "pi" ]; then - python3 -m venv .venv - log "Created venv" -else +if [ ! -d .venv ]; then + log "Creating Python virtual environment..." python3 -m venv .venv log "Created venv" fi @@ -123,14 +162,12 @@ log "Activated virtual environment" log "Installing Python dependencies..." if command -v uv &> /dev/null; then uv pip install -e ".[ui,analysis]" - - # Camera dependencies are disabled for the radar-only production path. - # If camera support returns, re-enable the optional camera extra in - # pyproject.toml and restore installation here. else pip install -e ".[ui,analysis]" - # Camera dependencies are disabled for the radar-only production path. fi +# Camera dependencies are disabled for the radar-only production path. +# If camera support returns, re-enable the optional camera extra in +# pyproject.toml and restore installation here. log "Python dependencies installed ✓" # Install dev dependencies @@ -154,9 +191,7 @@ log "UI built ✓" cd .. # Make scripts executable -log "Making scripts executable..." -chmod +x scripts/*.sh -chmod +x scripts/setup/*.sh +chmod +x scripts/*.sh scripts/setup/*.sh # Run tests to verify installation log "Running tests to verify installation..." @@ -166,38 +201,140 @@ else warn "Some tests failed - installation may be incomplete" fi +# ────────────────────────────────────────────────────────────────────── +# Phase 2: Hardware configuration (Raspberry Pi, interactive) +# ────────────────────────────────────────────────────────────────────── + +if [ "$PLATFORM" == "pi" ] && [ "$DEPS_ONLY" == "false" ] && [ "$INTERACTIVE" == "true" ]; then + echo "" + echo -e "${GREEN}=== Hardware Setup ===${NC}" + echo "" + info "Dependencies are installed. The next steps configure your hardware." + info "You can skip any step and re-run this script later." + + # --- OPS243-A rolling buffer flash config --- + echo "" + if confirm "Configure the OPS243-A radar now? (it must be plugged in)" "Y"; then + log "Saving rolling buffer mode to the radar's flash memory..." + if python scripts/hardware-test/test_rolling_buffer_persist.py --setup; then + echo "" + info "Now power cycle the radar: unplug its USB cable, wait 3 seconds," + info "and plug it back in. (This works around a firmware bug — one time only.)" + read -r -p "Press Enter after plugging it back in... " _ + sleep 2 + log "Verifying (make a sharp sound near the sound detector when asked)..." + if python scripts/hardware-test/test_rolling_buffer_persist.py --test; then + log "OPS243-A configured ✓" + else + warn "Verification failed. See docs/raspberry-pi-setup.md → Radar Setup." + fi + else + warn "Radar configuration failed — is the OPS243-A plugged in?" + warn "You can re-run this script, or see docs/raspberry-pi-setup.md." + fi + else + info "Skipped. Run later with:" + info " uv run python scripts/hardware-test/test_rolling_buffer_persist.py --setup" + fi + + # --- K-LD7 device naming + latency --- + echo "" + if confirm "Do you have K-LD7 angle radars to set up?" "N"; then + "$SCRIPT_DIR/setup_kld7_devices.sh" + else + info "Skipped. Run later with: ./scripts/setup/setup_kld7_devices.sh" + fi + + # --- Auto-start service --- + echo "" + if confirm "Start OpenFlight automatically on boot?" "N"; then + log "Installing systemd service for user '$USER'..." + sed -e "s|^User=.*|User=$USER|" \ + -e "s|/home/coleman/openflight|$PROJECT_DIR|g" \ + "$SCRIPT_DIR/openflight.service" | sudo tee /etc/systemd/system/openflight.service > /dev/null + sudo systemctl daemon-reload + sudo systemctl enable openflight + log "Service installed and enabled ✓ (starts on next boot)" + info "Manage it with: sudo systemctl {start|stop|status} openflight" + else + info "Skipped. See docs/raspberry-pi-setup.md → Auto-Start on Boot." + fi + + # --- Desktop shortcut --- + echo "" + if confirm "Add an OpenFlight shortcut to the desktop?" "N"; then + mkdir -p "$HOME/Desktop" + sed -e "s|/home/coleman/openflight|$PROJECT_DIR|g" \ + "$SCRIPT_DIR/OpenFlight.desktop" > "$HOME/Desktop/OpenFlight.desktop" + chmod +x "$HOME/Desktop/OpenFlight.desktop" + log "Desktop shortcut added ✓" + fi + + # --- Cloud sync (opt-in) --- + echo "" + info "Cloud sync pushes *filtered* session logs (shots, not raw radar) to" + info "FlightWeb so you can review them from anywhere. It is opt-in: nothing" + info "leaves this Pi until you link it to your account." + echo "" + if confirm "Enable cloud sync for this Pi?" "N"; then + log "Installing the cloud uploader timer (runs every ~10 min)..." + for unit in openflight-cloud.service openflight-cloud.timer; do + sed -e "s|^User=.*|User=$USER|" \ + -e "s|/home/coleman/openflight|$PROJECT_DIR|g" \ + "$SCRIPT_DIR/$unit" | sudo tee "/etc/systemd/system/$unit" > /dev/null + done + sudo systemctl daemon-reload + sudo systemctl enable --now openflight-cloud.timer + log "Cloud uploader timer installed and enabled ✓" + + echo "" + if confirm "Link this Pi to your FlightWeb account now?" "Y"; then + log "Starting device linking..." + echo "" + # `openflight-cloud` is on PATH via the activated venv. The command + # prints a short code; enter it in your browser when prompted. + if openflight-cloud link; then + log "Pi linked ✓ — sessions will sync automatically from now on." + else + warn "Linking did not complete. Re-run any time with:" + warn " openflight-cloud link" + fi + else + info "Skipped. Link later (the timer keeps running, but uploads stay" + info "off until you do) with:" + info " openflight-cloud link" + fi + info "Check sync state any time with: openflight-cloud status" + else + info "Skipped. Enable later by re-running this script, or see" + info " docs/cloud-sync.md" + fi +elif [ "$PLATFORM" == "pi" ]; then + info "Skipping hardware setup ($([ "$INTERACTIVE" == "false" ] && echo "non-interactive" || echo "--deps-only"))." + info "Run ./scripts/setup/setup.sh again without flags to configure hardware." +fi + +# ────────────────────────────────────────────────────────────────────── +# Done +# ────────────────────────────────────────────────────────────────────── + echo "" echo -e "${GREEN}╔═══════════════════════════════════════════╗${NC}" echo -e "${GREEN}║ Setup Complete! 🎉 ║${NC}" echo -e "${GREEN}╚═══════════════════════════════════════════╝${NC}" echo "" -log "To activate the virtual environment:" -echo " source .venv/bin/activate" +log "Start OpenFlight:" +echo " ./scripts/start-kiosk.sh # Default: rolling buffer + sound trigger" +echo " ./scripts/start-kiosk.sh --kld7 # With K-LD7 angle radars" +echo " ./scripts/start-kiosk.sh --mock # Mock mode (no hardware)" echo "" -if [ "$PLATFORM" == "pi" ]; then - log "IMPORTANT: Configure the radar for rolling buffer mode (one-time):" - echo " uv run python scripts/hardware-test/test_rolling_buffer_persist.py --setup" - echo " # Then power cycle the radar (unplug USB, wait 3s, replug)" - echo " uv run python scripts/hardware-test/test_rolling_buffer_persist.py --test" - echo "" -fi -log "To start the server:" -echo " ./scripts/start-kiosk.sh # Default: rolling buffer + sound trigger" -echo " ./scripts/start-kiosk.sh --mock # Mock mode (no radar)" +log "Then open http://localhost:8080 (or use the touchscreen)." echo "" -if [ "$PLATFORM" == "pi" ]; then - log "To set up auto-start on boot:" - echo " sudo cp scripts/openflight.service /etc/systemd/system/" - echo " sudo systemctl daemon-reload" - echo " sudo systemctl enable openflight" - echo "" - log "To set up log shipping to Grafana Cloud:" - echo " sudo scripts/setup/setup_alloy.sh" - echo "" - log "To add desktop shortcut:" - echo " cp scripts/OpenFlight.desktop ~/Desktop/" - echo " chmod +x ~/Desktop/OpenFlight.desktop" - echo "" -fi -log "For more info, see docs/raspberry-pi-setup.md" +log "Cloud sync (optional):" +echo " openflight-cloud link # pair this Pi with FlightWeb" +echo " openflight-cloud status # linked? queued? parked?" +echo " openflight-cloud push --dry-run # see exactly what would upload" +echo "" +log "For details and troubleshooting, see docs/raspberry-pi-setup.md" +log "For cloud sync details, see docs/cloud-sync.md" echo "" diff --git a/scripts/setup/setup_kld7_devices.sh b/scripts/setup/setup_kld7_devices.sh new file mode 100755 index 000000000..c9f609ee1 --- /dev/null +++ b/scripts/setup/setup_kld7_devices.sh @@ -0,0 +1,221 @@ +#!/bin/bash +# +# Interactive K-LD7 USB adapter naming wizard. +# +# Identifies each K-LD7's FTDI adapter by plug-in order — no serial numbers +# to look up, no udev rules to edit by hand. Writes a udev rule so the +# radars always appear at /dev/kld7_vertical and /dev/kld7_horizontal no +# matter which USB port they use or what order they enumerate in, then +# installs the FTDI low-latency rule. +# +# Usage: +# scripts/setup/setup_kld7_devices.sh # interactive wizard +# scripts/setup/setup_kld7_devices.sh --show # show current mapping +# +# Re-run any time to redo the mapping (e.g. after replacing an adapter). + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RULE_FILE="/etc/udev/rules.d/99-kld7.rules" + +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +BLUE='\033[0;34m' +NC='\033[0m' + +log() { echo -e "${GREEN}[K-LD7 Setup]${NC} $1"; } +warn() { echo -e "${YELLOW}[K-LD7 Setup]${NC} $1"; } +err() { echo -e "${RED}[K-LD7 Setup]${NC} $1"; } +ask() { echo -e "${BLUE}[K-LD7 Setup]${NC} $1"; } + +show_mapping() { + echo "" + if [ -e /dev/kld7_vertical ] || [ -e /dev/kld7_horizontal ]; then + log "Current K-LD7 device names:" + for name in kld7_vertical kld7_horizontal; do + if [ -e "/dev/$name" ]; then + echo " /dev/$name -> $(readlink -f "/dev/$name")" + else + echo " /dev/$name -> (radar not connected or not mapped)" + fi + done + else + warn "No /dev/kld7_* device names found." + fi + if [ -f "$RULE_FILE" ]; then + echo "" + log "Installed rule ($RULE_FILE):" + sed 's/^/ /' "$RULE_FILE" + fi +} + +if [ "${1:-}" == "--show" ]; then + show_mapping + exit 0 +fi + +if ! command -v udevadm &> /dev/null; then + err "udevadm not found — this wizard only runs on Linux (Raspberry Pi)." + exit 1 +fi + +list_ttyusb() { + ls /dev/ttyUSB* 2>/dev/null || true +} + +# Wait for exactly one new /dev/ttyUSB* device to appear vs a snapshot. +# Prints the new device path. +wait_for_new_device() { + local before="$1" + local waited=0 + while [ "$waited" -lt 90 ]; do + sleep 1 + waited=$((waited + 1)) + local now new + now="$(list_ttyusb)" + new="$(comm -13 <(echo "$before" | sort) <(echo "$now" | sort) | head -1)" + if [ -n "$new" ]; then + # Give udev a moment to finish setting the device up + sleep 2 + echo "$new" + return 0 + fi + done + return 1 +} + +device_serial() { + udevadm info -q property -n "$1" 2>/dev/null | sed -n 's/^ID_SERIAL_SHORT=//p' +} + +device_usb_path() { + udevadm info -q property -n "$1" 2>/dev/null | sed -n 's/^ID_PATH=//p' +} + +identify_adapter() { + # $1 = orientation label. Asks the user to plug that radar in and + # prints "serial|usb_path" for the adapter that appears. + local label="$1" + local before + before="$(list_ttyusb)" + echo "" >&2 + ask "Plug in the ${label} K-LD7's USB cable now (waiting up to 90s)..." >&2 + local dev + if ! dev="$(wait_for_new_device "$before")"; then + err "No new USB serial device appeared. Check the cable and re-run the wizard." >&2 + exit 1 + fi + log "Found ${label} adapter at $dev" >&2 + echo "$(device_serial "$dev")|$(device_usb_path "$dev")" +} + +echo "" +echo -e "${GREEN}=== K-LD7 Device Naming Wizard ===${NC}" +echo "" +echo "This identifies which USB adapter belongs to which radar so OpenFlight" +echo "always knows which K-LD7 is which. You'll unplug both adapters, then" +echo "plug them back in one at a time when asked." +echo "" + +if [ -f "$RULE_FILE" ]; then + show_mapping + echo "" + read -r -p "A mapping already exists. Redo it? [y/N] " redo + if [[ ! "$redo" =~ ^[Yy]$ ]]; then + log "Keeping the existing mapping." + exit 0 + fi +fi + +read -r -p "How many K-LD7 radars do you have? [2/1] " count +count="${count:-2}" +if [[ "$count" != "1" && "$count" != "2" ]]; then + err "Please answer 1 or 2." + exit 1 +fi + +echo "" +ask "Unplug BOTH K-LD7 USB cables from the Pi (leave the OPS243 connected)." +read -r -p "Press Enter when both are unplugged... " _ + +VERT_INFO="$(identify_adapter "VERTICAL (launch angle — mounted upright)")" +VERT_SERIAL="${VERT_INFO%%|*}" +VERT_PATH="${VERT_INFO##*|}" + +HORIZ_SERIAL="" +HORIZ_PATH="" +if [ "$count" == "2" ]; then + HORIZ_INFO="$(identify_adapter "HORIZONTAL (club path — mounted flat)")" + HORIZ_SERIAL="${HORIZ_INFO%%|*}" + HORIZ_PATH="${HORIZ_INFO##*|}" +fi + +# Build the rule. Prefer the adapter's unique serial number; fall back to +# the physical USB port path for cheap clones with no serial (in that case +# the cables must stay in the same USB ports). +RULE_LINES=() +USED_PATH_FALLBACK=false + +rule_for() { + local serial="$1" path="$2" name="$3" + if [ -n "$serial" ]; then + echo "SUBSYSTEM==\"tty\", ATTRS{serial}==\"$serial\", SYMLINK+=\"$name\"" + else + USED_PATH_FALLBACK=true + echo "SUBSYSTEM==\"tty\", ENV{ID_PATH}==\"$path\", SYMLINK+=\"$name\"" + fi +} + +if [ "$count" == "2" ] && [ -n "$VERT_SERIAL" ] && [ "$VERT_SERIAL" == "$HORIZ_SERIAL" ]; then + warn "Both adapters report the same serial number — falling back to USB port position." + VERT_SERIAL="" + HORIZ_SERIAL="" +fi + +RULE_LINES+=("$(rule_for "$VERT_SERIAL" "$VERT_PATH" "kld7_vertical")") +if [ "$count" == "2" ]; then + RULE_LINES+=("$(rule_for "$HORIZ_SERIAL" "$HORIZ_PATH" "kld7_horizontal")") +fi + +echo "" +log "Writing $RULE_FILE (you may be asked for your password):" +printf ' %s\n' "${RULE_LINES[@]}" +printf '%s\n' "${RULE_LINES[@]}" | sudo tee "$RULE_FILE" > /dev/null +sudo udevadm control --reload-rules +sudo udevadm trigger +sleep 2 + +echo "" +ok=true +if [ ! -e /dev/kld7_vertical ]; then + ok=false +fi +if [ "$count" == "2" ] && [ ! -e /dev/kld7_horizontal ]; then + ok=false +fi + +if [ "$ok" == "true" ]; then + log "Device names verified ✓" + show_mapping +else + err "Device names did not appear. Re-run the wizard, or see the manual" + err "steps in docs/raspberry-pi-setup.md (K-LD7 Angle Radar Setup)." + exit 1 +fi + +if [ "$USED_PATH_FALLBACK" == "true" ]; then + echo "" + warn "Your adapters have no unique serial number, so the mapping is tied" + warn "to the physical USB ports. Keep each cable in the port it's in now." +fi + +# FTDI low-latency rule (the 3 Mbaud RADC stream needs latency_timer=1ms) +echo "" +log "Installing the FTDI low-latency rule..." +sudo "$SCRIPT_DIR/setup_kld7_latency.sh" + +echo "" +log "K-LD7 setup complete. The radars are now at:" +echo " /dev/kld7_vertical$([ "$count" == "2" ] && echo " and /dev/kld7_horizontal")" diff --git a/scripts/start-kiosk.sh b/scripts/start-kiosk.sh index 57f2a1025..b9d35b326 100755 --- a/scripts/start-kiosk.sh +++ b/scripts/start-kiosk.sh @@ -42,6 +42,9 @@ EXPERIMENTAL_KLD7_VERTICAL_IMPACT_ENERGY="" EXPERIMENTAL_KLD7_HORIZONTAL_IMPACT_ENERGY="" EXPERIMENTAL_KLD7_HORIZONTAL_RETRY_IMPACT_ENERGY="" EXPERIMENTAL_KLD7_HORIZONTAL_ANGLE_LIMIT="" +BALLISTICS=false +CALCULATED_SPIN=false +BALL_SPEED_COSINE=false # Buffer split presets (pre/post trigger segments out of 32 total) # At 20ksps: each segment = 6.4ms, total buffer = 204.8ms @@ -198,6 +201,18 @@ while [[ $# -gt 0 ]]; do EXPERIMENTAL_KLD7_HORIZONTAL_ANGLE_LIMIT="$2" shift 2 ;; + --ballistics) + BALLISTICS=true + shift + ;; + --calculated-spin) + CALCULATED_SPIN=true + shift + ;; + --ball-speed-cosine-correction) + BALL_SPEED_COSINE=true + shift + ;; --port|-p) PORT="$2" shift 2 @@ -321,6 +336,18 @@ if [ "$NO_CAMERA" = true ]; then SERVER_CMD="$SERVER_CMD --no-camera" fi +if [ "$BALLISTICS" = true ]; then + SERVER_CMD="$SERVER_CMD --ballistics" +fi + +if [ "$CALCULATED_SPIN" = true ]; then + SERVER_CMD="$SERVER_CMD --calculated-spin" +fi + +if [ "$BALL_SPEED_COSINE" = true ]; then + SERVER_CMD="$SERVER_CMD --ball-speed-cosine-correction" +fi + if [ -n "$TRIGGER" ]; then SERVER_CMD="$SERVER_CMD --trigger $TRIGGER" fi @@ -414,14 +441,13 @@ if [ "$DRY_RUN" = true ]; then exit 0 fi -# Check if venv exists -if [ ! -d ".venv" ]; then - error "Virtual environment not found. Run: uv venv && uv pip install -e '.[ui,kld7]'" +# Ensure the environment is in sync (uv recreates/repairs .venv as needed, +# so a moved project dir self-heals instead of failing with "command not found") +if ! command -v uv >/dev/null 2>&1; then + error "uv not found. Install it: https://docs.astral.sh/uv/" exit 1 fi - -# Activate venv -source .venv/bin/activate +uv sync --quiet configure_kld7_latency @@ -483,7 +509,13 @@ else log "Camera enabled (Hough + ByteTrack)" fi -$SERVER_CMD & +if [ "$BALLISTICS" = true ]; then + log "Ballistic carry model enabled (simulator + drag/Magnus)" +else + log "Ballistic carry model disabled (using legacy table)" +fi + +uv run $SERVER_CMD & SERVER_PID=$! # Wait for server to be ready diff --git a/session_logs/comparison_20260506_plots/compare_ball_speed.png b/session_logs/comparison_20260506_plots/compare_ball_speed.png deleted file mode 100644 index 48a59853d..000000000 Binary files a/session_logs/comparison_20260506_plots/compare_ball_speed.png and /dev/null differ diff --git a/session_logs/comparison_20260506_plots/compare_carry.png b/session_logs/comparison_20260506_plots/compare_carry.png deleted file mode 100644 index 90f4770c3..000000000 Binary files a/session_logs/comparison_20260506_plots/compare_carry.png and /dev/null differ diff --git a/session_logs/comparison_20260506_plots/compare_club_speed.png b/session_logs/comparison_20260506_plots/compare_club_speed.png deleted file mode 100644 index 25a2d040c..000000000 Binary files a/session_logs/comparison_20260506_plots/compare_club_speed.png and /dev/null differ diff --git a/session_logs/comparison_20260506_plots/compare_launch_h.png b/session_logs/comparison_20260506_plots/compare_launch_h.png deleted file mode 100644 index 221401191..000000000 Binary files a/session_logs/comparison_20260506_plots/compare_launch_h.png and /dev/null differ diff --git a/session_logs/comparison_20260506_plots/compare_launch_v.png b/session_logs/comparison_20260506_plots/compare_launch_v.png deleted file mode 100644 index 8a7e04e48..000000000 Binary files a/session_logs/comparison_20260506_plots/compare_launch_v.png and /dev/null differ diff --git a/session_logs/comparison_20260506_plots/compare_overview.png b/session_logs/comparison_20260506_plots/compare_overview.png deleted file mode 100644 index f03f8665b..000000000 Binary files a/session_logs/comparison_20260506_plots/compare_overview.png and /dev/null differ diff --git a/session_logs/comparison_20260506_plots/compare_spin.png b/session_logs/comparison_20260506_plots/compare_spin.png deleted file mode 100644 index f078288fa..000000000 Binary files a/session_logs/comparison_20260506_plots/compare_spin.png and /dev/null differ diff --git a/session_logs/comparison_20260506_plots/compare_takeaways.png b/session_logs/comparison_20260506_plots/compare_takeaways.png deleted file mode 100644 index 8bdbf1d5d..000000000 Binary files a/session_logs/comparison_20260506_plots/compare_takeaways.png and /dev/null differ diff --git a/src/openflight/ballistics.py b/src/openflight/ballistics.py new file mode 100644 index 000000000..85a53ed2c --- /dev/null +++ b/src/openflight/ballistics.py @@ -0,0 +1,358 @@ +""" +Physics-based golf ball flight simulator. + +Integrates the drag + Magnus ODE using RK4 to produce a deterministic +trajectory from launch conditions. When measured spin is low-confidence +or missing, `resolve_launch` substitutes a club-typical spin value so the +output remains committable (no probabilistic range). + +Coordinate system (world frame): + x — downrange, target direction + y — lateral, +right + z — height, up + +Aerodynamic model: Cd and Cl are functions of the spin parameter +Sp = r*omega / v. Fits are consistent with Bearman & Harvey (1976) and +Kensrud & Smith (2018) measurements for dimpled golf balls in the +post-drag-crisis regime (Re ~ 5e4 to 2e5). Spin decay follows +Kiratidis & Leinweber (2018) at ~4%/s. +""" + +import math +from dataclasses import dataclass +from typing import Literal, Optional + +from .launch_monitor import SPIN_CONFIDENCE_HIGH, ClubType, Shot + +MPH_TO_MPS = 0.44704 +MPS_TO_MPH = 1.0 / MPH_TO_MPS +M_TO_YD = 1.09361 + +# USGA maximum-conforming ball (45.93 g, 42.7 mm diameter). +# Using the max — not an average — so carry estimates are upper-bounded by +# the rules rather than by a guess at the specific ball in play. +BALL_MASS_KG = 0.04593 +BALL_RADIUS_M = 0.02135 +BALL_AREA_M2 = math.pi * BALL_RADIUS_M ** 2 +AIR_DENSITY_STD = 1.225 # kg/m³ at sea level, 15 °C ISA + +# Cd = CD_BASE + CD_SPIN_COEFF * Sp +# Linear rise with spin parameter Sp = r·ω/v. +# Cl = CL_SATURATION * Sp / (CL_HALF_SP + Sp) +# Hill-type saturating form: Cl → CL_SATURATION as Sp → ∞, +# reaches CL_SATURATION/2 at Sp = CL_HALF_SP. +# These are simple parametric forms consistent with Bearman & Harvey (1976) +# and Kensrud & Smith (2018) for dimpled balls past the drag crisis +# (Re ~ 5e4–2e5), which covers the full range of realistic golf shots. +CD_BASE = 0.205 +CD_SPIN_COEFF = 0.18 +CL_SATURATION = 0.32 +CL_HALF_SP = 0.15 + +# Exponential spin decay: ω(t) = ω₀·exp(-rate·t). +# ~4%/s per Kiratidis & Leinweber (2018); small but matters over ~6 s flights. +SPIN_DECAY_RATE = 0.04 + +GRAVITY = 9.81 +# 500 Hz integration. RK4 error is O(dt⁵) so this is effectively exact for +# the timescales involved; larger dt starts to visibly shorten long drives. +DT_SECONDS = 0.002 +# Safety cap — real shots terminate in 5–9 s; anything longer implies the +# solver went unstable or physical inputs are pathological. +MAX_FLIGHT_SECONDS = 15.0 +# Trajectory sample cadence for the returned point list. Integration still +# runs at DT_SECONDS; this only controls how many points the caller sees, +# keeping payload size reasonable for UI/log consumers. +SAMPLE_INTERVAL_S = 0.05 + +# Club-typical spin (RPM) from TrackMan PGA Tour averages. +# Used as fallback when measured spin is missing or low-confidence. +CLUB_TYPICAL_SPIN_RPM: dict[ClubType, float] = { + ClubType.DRIVER: 2700, + ClubType.WOOD_3: 3500, + ClubType.WOOD_5: 4200, + ClubType.WOOD_7: 4800, + ClubType.HYBRID_3: 4400, + ClubType.HYBRID_5: 4900, + ClubType.HYBRID_7: 5300, + ClubType.HYBRID_9: 5800, + ClubType.IRON_2: 4000, + ClubType.IRON_3: 4500, + ClubType.IRON_4: 5000, + ClubType.IRON_5: 5400, + ClubType.IRON_6: 6000, + ClubType.IRON_7: 6500, + ClubType.IRON_8: 7500, + ClubType.IRON_9: 8500, + ClubType.PW: 9000, + ClubType.GW: 9500, + ClubType.SW: 10000, + ClubType.LW: 10500, + ClubType.UNKNOWN: 5000, +} + + +@dataclass +class LaunchConditions: + """Deterministic launch parameters for the physics model or simulator export.""" + + ball_speed_mph: float + launch_angle_v: float + launch_angle_h: float + spin_rpm: float + spin_axis_deg: float + spin_source: Literal["measured", "club_typical"] + + +@dataclass +class TrajectoryPoint: + t: float + x: float + y: float + z: float + speed_mph: float + spin_rpm: float + + +@dataclass +class Trajectory: + points: list[TrajectoryPoint] + carry_yards: float + apex_yards: float + lateral_yards: float + flight_time_s: float + landing_speed_mph: float + landing_angle_deg: float + + @property + def total_yards(self) -> float: + """Carry plus a simple rollout estimate (flatter landings roll farther).""" + # Rough heuristic, not a physical model: a flat landing (~20°) adds + # ~28 yd, a steep wedge landing (~60°) adds ~15 yd. Surface firmness, + # wetness, and slope are ignored — fine for a display hint, not for + # course-play distance. + rollout = max(0.0, 30.0 * math.cos(math.radians(self.landing_angle_deg))) + return self.carry_yards + rollout + + +def resolve_launch(shot: Shot) -> Optional[LaunchConditions]: + """ + Produce committed launch conditions from a shot. + + Returns None if the vertical launch angle is unavailable (no physics + simulation possible without it). Spin is taken from the measurement only + when confidence >= SPIN_CONFIDENCE_HIGH; otherwise a club-typical value + is substituted and `spin_source` is set to "club_typical". + """ + if shot.launch_angle_vertical is None: + return None + + use_measured = ( + shot.spin_rpm is not None + and shot.spin_confidence is not None + and shot.spin_confidence >= SPIN_CONFIDENCE_HIGH + ) + if use_measured: + spin_rpm = float(shot.spin_rpm) + source: Literal["measured", "club_typical"] = "measured" + else: + spin_rpm = CLUB_TYPICAL_SPIN_RPM.get( + shot.club, CLUB_TYPICAL_SPIN_RPM[ClubType.UNKNOWN] + ) + source = "club_typical" + + return LaunchConditions( + ball_speed_mph=shot.ball_speed_mph, + launch_angle_v=shot.launch_angle_vertical, + launch_angle_h=shot.launch_angle_horizontal or 0.0, + spin_rpm=spin_rpm, + spin_axis_deg=shot.spin_axis_deg or 0.0, + spin_source=source, + ) + + +def _cd(sp: float) -> float: + return CD_BASE + CD_SPIN_COEFF * sp + + +def _cl(sp: float) -> float: + return CL_SATURATION * sp / (CL_HALF_SP + sp) if sp > 0 else 0.0 + + +def _derivatives( + state: tuple, + omega: float, + axis: tuple, + air_density: float, +) -> tuple: + """d/dt of (x, y, z, vx, vy, vz) under gravity + drag + Magnus.""" + _, _, _, vx, vy, vz = state + v = math.sqrt(vx * vx + vy * vy + vz * vz) + # At v ≈ 0 both drag and Magnus vanish (F ∝ v²) and division below + # would be unsafe. Should only happen at the instantaneous apex of a + # pathological straight-up launch. + if v < 1e-6: + return (vx, vy, vz, 0.0, 0.0, -GRAVITY) + + # Spin parameter Sp drives both Cd and Cl. Note ω is tracked separately + # (not in the state tuple) because its decay is exponential, not an ODE + # we want to couple into RK4. + sp = BALL_RADIUS_M * omega / v + cd = _cd(sp) + cl = _cl(sp) + + # q: dynamic-pressure × reference-area / mass = acceleration per unit C. + # Pre-dividing by mass here lets us return accelerations directly. + q = 0.5 * air_density * v * v * BALL_AREA_M2 / BALL_MASS_KG + drag_a = cd * q + lift_a = cl * q + + # Drag opposes velocity: -v̂ · |F_drag|. + ax = -drag_a * vx / v + ay = -drag_a * vy / v + az = -drag_a * vz / v + + # Magnus direction is (ω̂ × v̂). We compute axis × velocity and normalize + # by the cross-product magnitude rather than |ω|·|v|·sin(θ), which avoids + # an extra sin() and gracefully degrades when axis ∥ v (spin does no work). + ox, oy, oz = axis + cx = oy * vz - oz * vy + cy = oz * vx - ox * vz + cz = ox * vy - oy * vx + c_mag = math.sqrt(cx * cx + cy * cy + cz * cz) + if c_mag > 1e-6: + ax += lift_a * cx / c_mag + ay += lift_a * cy / c_mag + az += lift_a * cz / c_mag + + az -= GRAVITY + + return (vx, vy, vz, ax, ay, az) + + +def _rk4_step( + state: tuple, + omega: float, + axis: tuple, + air_density: float, + dt: float, +) -> tuple: + k1 = _derivatives(state, omega, axis, air_density) + s2 = tuple(state[i] + 0.5 * dt * k1[i] for i in range(6)) + k2 = _derivatives(s2, omega, axis, air_density) + s3 = tuple(state[i] + 0.5 * dt * k2[i] for i in range(6)) + k3 = _derivatives(s3, omega, axis, air_density) + s4 = tuple(state[i] + dt * k3[i] for i in range(6)) + k4 = _derivatives(s4, omega, axis, air_density) + return tuple( + state[i] + (dt / 6.0) * (k1[i] + 2 * k2[i] + 2 * k3[i] + k4[i]) + for i in range(6) + ) + + +def simulate( + conditions: LaunchConditions, + air_density: float = AIR_DENSITY_STD, + dt: float = DT_SECONDS, +) -> Trajectory: + """ + Integrate flight from launch to first ground contact (z = 0). + """ + v0 = conditions.ball_speed_mph * MPH_TO_MPS + la_v = math.radians(conditions.launch_angle_v) + la_h = math.radians(conditions.launch_angle_h) + + # Initial velocity in world frame (x downrange, y right, z up). + vx = v0 * math.cos(la_v) * math.cos(la_h) + vy = v0 * math.cos(la_v) * math.sin(la_h) + vz = v0 * math.sin(la_v) + + # Spin axis convention: + # spin_axis_deg = 0 → pure backspin, axis = -y + # (ω × v with v ≈ +x gives Magnus = +z, i.e. lift up — correct) + # spin_axis_deg > 0 → top of axis tilts toward +y (right); + # Magnus gains a +y component → fade/slice. + # spin_axis_deg < 0 → draw/hook. + # World-frame (not velocity-frame) definition, matching launch-monitor + # and simulator conventions where spin axis is specified at launch. + axis_rad = math.radians(conditions.spin_axis_deg) + axis = (0.0, -math.cos(axis_rad), math.sin(axis_rad)) + + # rpm → rad/s. Decays exponentially each step; see loop below. + omega = conditions.spin_rpm * 2 * math.pi / 60.0 + + points: list[TrajectoryPoint] = [ + TrajectoryPoint(0.0, 0.0, 0.0, 0.0, conditions.ball_speed_mph, conditions.spin_rpm) + ] + + state = (0.0, 0.0, 0.0, vx, vy, vz) + t = 0.0 + max_z = 0.0 + last_sample_t = 0.0 + + while t < MAX_FLIGHT_SECONDS: + new_state = _rk4_step(state, omega, axis, air_density, dt) + t += dt + omega *= math.exp(-SPIN_DECAY_RATE * dt) + + _, _, z, _, _, _ = new_state + if z > max_z: + max_z = z + + # Ground contact. Linear interpolation between prev and new state + # is sufficient because dt (2 ms) is tiny relative to the timescale + # of the descent — the state is effectively linear across one step + # even though the full trajectory is not. + if z <= 0.0 and t > dt: + prev_z = state[2] + denom = prev_z - z + frac = prev_z / denom if abs(denom) > 1e-9 else 0.0 + t_hit = (t - dt) + frac * dt + final = tuple(state[i] + frac * (new_state[i] - state[i]) for i in range(6)) + fx, fy, fz, fvx, fvy, fvz = final + v_final = math.sqrt(fvx * fvx + fvy * fvy + fvz * fvz) + landing_angle = math.degrees( + math.atan2(-fvz, math.sqrt(fvx * fvx + fvy * fvy)) + ) + points.append(TrajectoryPoint( + t_hit, + fx * M_TO_YD, fy * M_TO_YD, max(fz, 0.0) * M_TO_YD, + v_final * MPS_TO_MPH, + omega * 60 / (2 * math.pi), + )) + return Trajectory( + points=points, + carry_yards=fx * M_TO_YD, + apex_yards=max_z * M_TO_YD, + lateral_yards=fy * M_TO_YD, + flight_time_s=t_hit, + landing_speed_mph=v_final * MPS_TO_MPH, + landing_angle_deg=landing_angle, + ) + + state = new_state + + if t - last_sample_t >= SAMPLE_INTERVAL_S: + sx_, sy_, sz_, svx, svy, svz = state + v = math.sqrt(svx * svx + svy * svy + svz * svz) + points.append(TrajectoryPoint( + t, + sx_ * M_TO_YD, sy_ * M_TO_YD, sz_ * M_TO_YD, + v * MPS_TO_MPH, + omega * 60 / (2 * math.pi), + )) + last_sample_t = t + + # Flight did not terminate — return current state as best-effort + fx, fy, fz, fvx, fvy, fvz = state + v_final = math.sqrt(fvx * fvx + fvy * fvy + fvz * fvz) + landing_angle = math.degrees(math.atan2(-fvz, math.sqrt(fvx * fvx + fvy * fvy))) + return Trajectory( + points=points, + carry_yards=fx * M_TO_YD, + apex_yards=max_z * M_TO_YD, + lateral_yards=fy * M_TO_YD, + flight_time_s=t, + landing_speed_mph=v_final * MPS_TO_MPH, + landing_angle_deg=landing_angle, + ) diff --git a/src/openflight/cloud/__init__.py b/src/openflight/cloud/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/openflight/cloud/cli.py b/src/openflight/cloud/cli.py new file mode 100644 index 000000000..d098048b3 --- /dev/null +++ b/src/openflight/cloud/cli.py @@ -0,0 +1,112 @@ +"""``openflight-cloud`` command-line entry point. + +openflight-cloud link # one-time device pairing +openflight-cloud push [--dry-run] # filter + upload anything unpushed +openflight-cloud status # linked? queued? parked? last error? +""" + +import argparse +from pathlib import Path +from typing import List, Optional + +from ..session_logger import SessionLogger +from . import commands +from .client import CloudClient +from .config import CONFIG_PATH, CloudConfig, load_config + +DEFAULT_LOG_DIR = SessionLogger.DEFAULT_LOG_DIR + + +def _build_parser() -> argparse.ArgumentParser: + # Shared options usable either before or after the subcommand. + common = argparse.ArgumentParser(add_help=False) + common.add_argument( + "--config", + type=Path, + default=CONFIG_PATH, + help=f"Path to cloud config (default: {CONFIG_PATH}).", + ) + common.add_argument( + "--log-dir", + type=Path, + default=DEFAULT_LOG_DIR, + help=f"Session log directory (default: {DEFAULT_LOG_DIR}).", + ) + + parser = argparse.ArgumentParser( + prog="openflight-cloud", + description="Push filtered OpenFlight session logs to the FlightWeb cloud.", + parents=[common], + ) + sub = parser.add_subparsers(dest="command") + + link = sub.add_parser( + "link", parents=[common], help="Pair this device with a FlightWeb account." + ) + link.add_argument("--device-name", default=None, help="Device label (default: hostname).") + + push = sub.add_parser("push", parents=[common], help="Upload any unpushed sessions.") + push.add_argument( + "--dry-run", + action="store_true", + help="Show exactly which entries would upload; send nothing.", + ) + push.add_argument( + "--retry", + nargs="?", + const="__ALL__", + default=None, + metavar="SESSION", + help=( + "Re-upload sessions that were parked or failed. Give a session " + "filename (or substring) to force re-upload a specific one even if " + "it was already sent." + ), + ) + + sub.add_parser("status", parents=[common], help="Show link state, queue, and parked sessions.") + return parser + + +def _client(config: CloudConfig) -> CloudClient: + """Build a CloudClient from persisted config.""" + return CloudClient(config.endpoint, token=config.device_token or None) + + +def main(argv: Optional[List[str]] = None) -> int: + parser = _build_parser() + args = parser.parse_args(argv) + + if not args.command: + parser.print_help() + return 1 + + config = load_config(args.config) or CloudConfig() + + if args.command == "link": + ok = commands.cmd_link(config, args.config, _client(config), device_name=args.device_name) + return 0 if ok else 1 + + if args.command == "push": + retry = args.retry is not None + session = None if args.retry == "__ALL__" else args.retry + summary = commands.cmd_push( + config, + args.log_dir, + _client(config), + dry_run=args.dry_run, + retry=retry, + session=session, + ) + return 1 if summary.get("needs_relink") else 0 + + if args.command == "status": + commands.cmd_status(config, args.log_dir, _client(config)) + return 0 + + parser.print_help() + return 1 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/src/openflight/cloud/client.py b/src/openflight/cloud/client.py new file mode 100644 index 000000000..9be3aef39 --- /dev/null +++ b/src/openflight/cloud/client.py @@ -0,0 +1,247 @@ +"""HTTP client for the FlightWeb cloud wire contract. + +Uses the standard library (``urllib``) — no third-party HTTP dependency on the +Pi. The low-level transport is injectable (``request_fn``) so tests run without +the network. + +The contract lives entirely in this module; nothing here imports FlightWeb +code. See docs/openflight-cloud-uploader-spec.md. +""" + +import json +import urllib.error +import urllib.request +from collections import namedtuple +from dataclasses import dataclass +from typing import Callable, Dict, Optional + +API_PREFIX = "/v1" +DEFAULT_TIMEOUT = 30 + +HttpResponse = namedtuple("HttpResponse", ["status", "headers", "body"]) + + +class CloudNetworkError(Exception): + """Raised when the request never reached the server (DNS, TLS, offline).""" + + +class RateLimited(Exception): + """Raised on a 429 for link endpoints; carries Retry-After seconds.""" + + def __init__(self, retry_after: Optional[int]): + super().__init__(f"rate limited, retry after {retry_after}s") + self.retry_after = retry_after + + +class LinkError(Exception): + """Raised when a link request is rejected (e.g. 422 invalid device name).""" + + def __init__(self, message: str, status: int, reason: Optional[str] = None): + super().__init__(message) + self.status = status + self.reason = reason + + +@dataclass +class LinkStart: + """Response from ``device-link/start``: code to show + token to poll.""" + + link_code: str + poll_token: str + interval_s: int + expires_s: int + + +@dataclass +class LinkPoll: + """Response from ``device-link/poll``: pending/expired/linked/unknown.""" + + status: str + device_token: Optional[str] = None + device_id: Optional[str] = None + + +@dataclass +class UploadResult: + """Outcome of an upload, with the action the spool layer should take.""" + + status_code: int + # one of: success, relink, quota, park, rate_limited, retry + action: str + reason: Optional[str] = None + retry_after: Optional[int] = None + session_id: Optional[str] = None + shot_count: Optional[int] = None + + +def _header(headers: Dict[str, str], name: str) -> Optional[str]: + """Case-insensitive header lookup.""" + lowered = {k.lower(): v for k, v in (headers or {}).items()} + return lowered.get(name.lower()) + + +def _retry_after(headers: Dict[str, str]) -> Optional[int]: + value = _header(headers, "Retry-After") + if value is None: + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def urllib_request( + method: str, + url: str, + data: Optional[bytes] = None, + headers: Optional[Dict[str, str]] = None, + timeout: int = DEFAULT_TIMEOUT, +) -> HttpResponse: + """Default transport. Returns HttpResponse for any HTTP status (including + 4xx/5xx); raises CloudNetworkError only when the server was unreachable.""" + req = urllib.request.Request(url, data=data, headers=headers or {}, method=method) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + return HttpResponse( + status=resp.status, + headers=dict(resp.headers.items()), + body=resp.read(), + ) + except urllib.error.HTTPError as exc: + # HTTP error responses are valid contract responses, not failures. + return HttpResponse( + status=exc.code, + headers=dict(exc.headers.items()) if exc.headers else {}, + body=exc.read(), + ) + except urllib.error.URLError as exc: + raise CloudNetworkError(str(exc.reason)) from exc + except (TimeoutError, OSError) as exc: + raise CloudNetworkError(str(exc)) from exc + + +class CloudClient: + """Thin client over the ``/v1`` wire contract.""" + + def __init__( + self, + endpoint: str, + token: Optional[str] = None, + request_fn: Callable[..., HttpResponse] = urllib_request, + timeout: int = DEFAULT_TIMEOUT, + ): + self.endpoint = endpoint.rstrip("/") + self.token = token + self._request = request_fn + self.timeout = timeout + + def _url(self, path: str) -> str: + return f"{self.endpoint}{API_PREFIX}{path}" + + def _json_request( + self, method: str, path: str, payload: Optional[dict] = None, headers: Optional[dict] = None + ) -> HttpResponse: + body = json.dumps(payload).encode("utf-8") if payload is not None else None + hdrs = {"Accept": "application/json"} + if body is not None: + hdrs["Content-Type"] = "application/json" + if headers: + hdrs.update(headers) + return self._request(method, self._url(path), data=body, headers=hdrs, timeout=self.timeout) + + @staticmethod + def _body_json(resp: HttpResponse) -> dict: + if not resp.body: + return {} + try: + parsed = json.loads(resp.body.decode("utf-8")) + return parsed if isinstance(parsed, dict) else {} + except (json.JSONDecodeError, ValueError): + return {} + + def health(self) -> bool: + """Cheap connectivity probe; True iff the server reports healthy.""" + try: + resp = self._request( + "GET", self._url("/health"), data=None, headers=None, timeout=self.timeout + ) + except CloudNetworkError: + return False + return resp.status == 200 and self._body_json(resp).get("status") == "ok" + + def device_link_start(self, device_name: str, client_version: str) -> LinkStart: + """Begin pairing; returns a link code to show the user and a poll token.""" + resp = self._json_request( + "POST", + "/device-link/start", + {"device_name": device_name, "client_version": client_version}, + ) + if resp.status == 429: + raise RateLimited(_retry_after(resp.headers)) + body = self._body_json(resp) + if resp.status != 200: + raise LinkError(f"link start failed ({resp.status})", resp.status, body.get("reason")) + return LinkStart( + link_code=body["link_code"], + poll_token=body["poll_token"], + interval_s=int(body.get("interval_s", 5)), + expires_s=int(body.get("expires_s", 900)), + ) + + def device_link_poll(self, poll_token: str) -> LinkPoll: + """Poll for pairing completion; on ``linked`` returns the device token.""" + resp = self._json_request("POST", "/device-link/poll", {"poll_token": poll_token}) + if resp.status == 429: + raise RateLimited(_retry_after(resp.headers)) + if resp.status == 404: + # Unknown OR already-consumed poll token. + return LinkPoll(status="unknown") + body = self._body_json(resp) + if resp.status != 200: + raise LinkError(f"link poll failed ({resp.status})", resp.status, body.get("reason")) + return LinkPoll( + status=body.get("status", "pending"), + device_token=body.get("device_token"), + device_id=body.get("device_id"), + ) + + def upload_session(self, session_id: str, gzipped_body: bytes) -> UploadResult: + """PUT a filtered, gzipped session. Maps status -> spool action.""" + headers = { + "Content-Type": "application/x-ndjson", + "Content-Encoding": "gzip", + } + if self.token: + headers["Authorization"] = f"Bearer {self.token}" + resp = self._request( + "PUT", + self._url(f"/sessions/{session_id}"), + data=gzipped_body, + headers=headers, + timeout=self.timeout, + ) + body = self._body_json(resp) + reason = body.get("reason") + + if resp.status in (200, 201): + return UploadResult( + status_code=resp.status, + action="success", + session_id=body.get("session_id"), + shot_count=body.get("shot_count"), + ) + if resp.status == 401: + return UploadResult(resp.status, action="relink", reason=reason) + if resp.status == 402: + return UploadResult(resp.status, action="quota", reason=reason) + if resp.status in (413, 422): + return UploadResult(resp.status, action="park", reason=reason) + if resp.status == 429: + return UploadResult( + resp.status, + action="rate_limited", + reason=reason, + retry_after=_retry_after(resp.headers), + ) + # 5xx and anything unexpected: back off and retry. + return UploadResult(resp.status, action="retry", reason=reason) diff --git a/src/openflight/cloud/commands.py b/src/openflight/cloud/commands.py new file mode 100644 index 000000000..aacd3e6da --- /dev/null +++ b/src/openflight/cloud/commands.py @@ -0,0 +1,296 @@ +"""Command orchestration for the openflight-cloud CLI. + +Each function takes its dependencies explicitly (config, client, output sink, +sleep) so the logic is testable without the network or real timers. +""" + +import socket +import time +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional + +from . import filtering, spool +from .client import CloudNetworkError, RateLimited +from .config import CloudConfig, save_config + +OutFn = Callable[[str], None] + + +def cmd_link( + config: CloudConfig, + config_path: Path, + client, + device_name: Optional[str] = None, + sleep: Callable[[float], None] = time.sleep, + now_fn: Callable[[], float] = time.monotonic, + out: OutFn = print, +) -> bool: + """One-time device pairing (RFC 8628-style). Returns True on success.""" + device_name = (device_name or socket.gethostname() or "openflight pi").strip()[:64] + try: + start = client.device_link_start(device_name, filtering.CLIENT_VERSION) + except RateLimited as exc: + out(f"Rate limited starting link; try again in {exc.retry_after}s.") + return False + + link_url = f"{config.endpoint.rstrip('/')}/link" + out("") + out(f" Go to {link_url} and enter code: {start.link_code}") + out("") + out(f" (waiting up to {start.expires_s}s; sign in and enter the code)") + + deadline = now_fn() + start.expires_s + interval = max(1, start.interval_s) + while now_fn() < deadline: + sleep(interval) + try: + poll = client.device_link_poll(start.poll_token) + except RateLimited as exc: + sleep(exc.retry_after or interval) + continue + + if poll.status == "pending": + continue + if poll.status == "linked": + # Persist token + id atomically on first receipt — the linked + # response is returned exactly once. + config.device_token = poll.device_token + config.device_id = poll.device_id + config.enabled = True + save_config(config, config_path) + out(f"Linked! device_id={poll.device_id}. Uploads are now enabled.") + return True + # expired or unknown (consumed/invalid token) + out(f"Link {poll.status}. Re-run `openflight-cloud link`.") + return False + + out("Link timed out. Re-run `openflight-cloud link`.") + return False + + +def _describe_dry_run(filename: str, session_id: str, result: filtering.FilterResult, out: OutFn): + out(f"\n{filename} -> session {session_id}") + if not result.kept_lines: + out(" (nothing to upload — no allowlisted entries)") + for entry_type in sorted(result.kept_type_counts): + out(f" keep {result.kept_type_counts[entry_type]:>5} x {entry_type}") + if result.dropped_oversize: + out(f" drop {result.dropped_oversize:>5} oversize line(s) (>32 KB)") + + +def _apply_retry(log_dir: Path, session: Optional[str], out: OutFn) -> int: + """Clear markers so failed/parked (or named) sessions upload again. + + Bulk retry (no ``session``) un-parks and clears cooldowns but leaves + already-pushed sessions alone. A named ``session`` (filename substring) + force-clears even ``.pushed`` so a session the server already stored — e.g. + one that landed with 0 shots — can be re-sent. Returns how many matched. + """ + targeted = bool(session) + matched = 0 + for path in spool.session_files(log_dir): + if targeted: + if session not in path.name: + continue + elif not (spool.is_parked(path) or spool.in_cooldown(path)): + continue + cleared = spool.clear_markers(path, include_pushed=targeted) + matched += 1 + if cleared: + out(f"{path.name}: reset {', '.join(cleared)} for retry.") + if matched == 0: + if targeted: + out(f"No session matching '{session}'. Run `openflight-cloud status` to list them.") + else: + out("No parked or deferred sessions to retry.") + return matched + + +def cmd_push( + config: CloudConfig, + log_dir: Path, + client, + dry_run: bool = False, + retry: bool = False, + session: Optional[str] = None, + out: OutFn = print, +) -> Dict[str, Any]: + """Filter and upload anything unpushed. Returns a summary dict. + + With ``retry``, first reset markers so previously parked/failed sessions + (or a specific ``session`` by filename substring) are eligible again. + """ + summary: Dict[str, Any] = { + "uploaded": 0, + "parked": 0, + "deferred": 0, + "failed": 0, + "offline": False, + "needs_relink": False, + "dry_run": dry_run, + } + + if not dry_run and not config.is_active(): + out("Uploader inactive (not linked or disabled). Run `openflight-cloud link`.") + summary["skipped"] = "inactive" + return summary + + if retry: + _apply_retry(log_dir, session, out) + + # Cheap connectivity probe so we don't churn while offline. + if not dry_run and not client.health(): + out("Cloud unreachable; will retry later.") + summary["offline"] = True + return summary + + pending = spool.pending_sessions(log_dir) + if not pending: + out("Nothing to upload.") + return summary + + for path in pending: + if not dry_run and spool.in_cooldown(path): + summary["deferred"] += 1 + continue + + # Stream the file: a raw-ADC session can be hundreds of MB, but we only + # keep the (tiny) shot lines. Loading it whole would risk OOM on a Pi. + result = filtering.filter_session_file(path, config.device_id) + session_id = result.session_id + + if dry_run: + _describe_dry_run(path.name, session_id, result, out) + continue + + try: + body = filtering.build_upload_body(result) + except filtering.BodyTooLargeError as exc: + spool.mark_parked( + path, + reason="body_too_large", + attempts=spool.read_attempts(path), + last_error=str(exc), + ) + summary["parked"] += 1 + out(f"{path.name}: too large after filtering — parked ({exc}).") + continue + + try: + upload = client.upload_session(session_id, body) + except CloudNetworkError as exc: + out(f"{path.name}: network error ({exc}); will retry later.") + summary["offline"] = True + break + + action = upload.action + if action == "success": + spool.mark_pushed(path, session_id, upload.shot_count) + summary["uploaded"] += 1 + out(f"{path.name}: uploaded ({upload.status_code}).") + elif action == "relink": + summary["needs_relink"] = True + out("Device token rejected. Stopping uploads — re-run `openflight-cloud link`.") + break + elif action == "quota": + spool.record_cooldown(path, "quota_exceeded", spool.QUOTA_COOLDOWN_S) + summary["deferred"] += 1 + out(f"{path.name}: quota exceeded — deferring ~24h.") + elif action == "park": + spool.mark_parked( + path, + reason=upload.reason or "client_error", + attempts=spool.read_attempts(path), + last_error=str(upload.status_code), + ) + summary["parked"] += 1 + out(f"{path.name}: rejected ({upload.status_code} {upload.reason}) — parked.") + elif action == "rate_limited": + out(f"Rate limited; backing off {upload.retry_after}s. Will retry later.") + summary["rate_limited"] = upload.retry_after + break + else: # retry (5xx / unexpected) + attempts = spool.record_failure( + path, f"{upload.status_code} {upload.reason or ''}".strip() + ) + summary["failed"] += 1 + if spool.is_parked(path): + summary["parked"] += 1 + out(f"{path.name}: failed {attempts}x — parked.") + else: + out(f"{path.name}: server error ({upload.status_code}); attempt {attempts}.") + + return summary + + +def cmd_status( + config: CloudConfig, + log_dir: Path, + client=None, + out: OutFn = print, +) -> Dict[str, Any]: + """Report link state, queue counts, and parked sessions.""" + out(f"Endpoint: {config.endpoint}") + if config.is_linked(): + out(f"Linked: yes (device_id={config.device_id})") + out(f"Enabled: {'yes' if config.enabled else 'no (uploads paused)'}") + else: + out("Linked: no — this device is not linked. Run `openflight-cloud link` to pair it.") + + online = None + if client is not None and config.is_active(): + online = client.health() + out(f"Reachable: {'yes' if online else 'no (cloud unreachable)'}") + + counts = spool.summarize(log_dir) + out( + f"Sessions: {counts['total']} total | {counts['pushed']} pushed | " + f"{counts['pending']} pending | {counts['parked']} parked" + ) + + parked = _parked_details(log_dir) + if parked: + out("Parked:") + for name, info in parked: + out(f" {name}: {info.get('reason')} (last_error={info.get('last_error')})") + + zero_shot = _zero_shot_uploads(log_dir) + if zero_shot: + out("Uploaded with 0 shots (re-upload with `openflight-cloud push --retry `):") + for name in zero_shot: + out(f" {name}") + + return { + "counts": counts, + "parked": [name for name, _ in parked], + "zero_shot": zero_shot, + "linked": config.is_linked(), + "online": online, + } + + +def _read_marker(path: Path, suffix: str) -> dict: + import json + + marker = path.with_name(path.name + suffix) + try: + return json.loads(marker.read_text()) + except (json.JSONDecodeError, ValueError, OSError): + return {} + + +def _parked_details(log_dir: Path) -> List: + return [ + (path.name, _read_marker(path, spool.PARKED_SUFFIX)) + for path in spool.session_files(log_dir) + if spool.is_parked(path) + ] + + +def _zero_shot_uploads(log_dir: Path) -> List[str]: + """Pushed sessions the server stored with no shots — likely worth re-sending.""" + names = [] + for path in spool.session_files(log_dir): + if spool.is_pushed(path) and not _read_marker(path, spool.PUSHED_SUFFIX).get("shot_count"): + names.append(path.name) + return names diff --git a/src/openflight/cloud/config.py b/src/openflight/cloud/config.py new file mode 100644 index 000000000..0708f518c --- /dev/null +++ b/src/openflight/cloud/config.py @@ -0,0 +1,65 @@ +"""Persistent config for the openflight-cloud uploader. + +Stored at ``~/.config/openflight/cloud.json`` with mode ``0600``. The +``device_token`` is a bearer credential and must never be logged. The file is +written by ``link`` on success and read by ``push``/``status``. When the file +is absent (or ``enabled`` is false) the uploader is a no-op. +""" + +import json +import os +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Optional + +# Current deployment. The production domain is undecided (openflight vs. +# flightweb), so it lives in config and may move. +DEFAULT_ENDPOINT = "https://flightweb.fly.dev" + +CONFIG_PATH = Path.home() / ".config" / "openflight" / "cloud.json" + + +@dataclass +class CloudConfig: + """Uploader configuration persisted to disk.""" + + endpoint: str = DEFAULT_ENDPOINT + device_token: str = "" + device_id: str = "" + enabled: bool = True + + def is_linked(self) -> bool: + """True when a device token and id are both present.""" + return bool(self.device_token and self.device_id) + + def is_active(self) -> bool: + """True when the uploader should actually push (linked and enabled).""" + return self.enabled and self.is_linked() + + +def load_config(path: Path = CONFIG_PATH) -> Optional[CloudConfig]: + """Load config from ``path``; return None if the file is absent.""" + path = Path(path) + if not path.exists(): + return None + data = json.loads(path.read_text()) + return CloudConfig( + endpoint=data.get("endpoint", DEFAULT_ENDPOINT), + device_token=data.get("device_token", ""), + device_id=data.get("device_id", ""), + enabled=data.get("enabled", True), + ) + + +def save_config(config: CloudConfig, path: Path = CONFIG_PATH) -> None: + """Write config to ``path`` with mode 0600, creating parent dirs.""" + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + # Create with restrictive permissions from the start so the bearer token is + # never briefly world-readable. + fd = os.open(str(path), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as handle: + json.dump(asdict(config), handle, indent=2) + handle.write("\n") + # Re-assert mode in case the file already existed with looser permissions. + os.chmod(path, 0o600) diff --git a/src/openflight/cloud/filtering.py b/src/openflight/cloud/filtering.py new file mode 100644 index 000000000..a3a67ca83 --- /dev/null +++ b/src/openflight/cloud/filtering.py @@ -0,0 +1,184 @@ +"""Client-side filtering — the raw-ADC strip. + +This is the load-bearing privacy boundary. The FlightWeb server stores a +device upload **verbatim**; it does not re-filter raw radar data out of a +device upload. So the product promise "raw radar data never leaves your Pi" +is enforced *here*, by applying an allowlist before upload. + +Use an allowlist, not a blocklist — any future heavy entry type the session +logger gains must never leak by default. +""" + +import gzip +import json +import uuid +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional + +from .. import __version__ + +CLIENT_VERSION = __version__ + +# Allowlisted entry types — only these are uploaded. ``error`` and +# ``session_error`` are both kept: the session logger currently emits ``error`` +# (see session_logger.py), while the server spec names ``session_error``; +# keeping both is privacy-safe (error entries carry only error strings/context, +# never raw ADC) and future-proofs a rename. +KEEP_ENTRY_TYPES = frozenset( + { + "session_start", + "session_end", + "shot_detected", + "trigger_event", + "session_error", + "error", + } +) + +MANIFEST_TYPE = "upload_manifest" +MANIFEST_FORMAT_VERSION = 1 + +# Per-line cap mirroring the server's per-line guard. Belt-and-suspenders. +MAX_LINE_BYTES = 32 * 1024 +# Body caps mirroring the server. A filtered session is normally tens of KB, +# so these are safety checks, not normal operating limits. +MAX_GZIP_BYTES = 20 * 1024 * 1024 +MAX_INFLATED_BYTES = 64 * 1024 * 1024 + +# Fixed namespace for deterministic UUIDv5 of (device_id, session_filename), +# used for older sessions that predate the embedded session_uuid. A stable +# namespace makes the same file always map to the same id (dedupe + safe retry). +SESSION_NAMESPACE = uuid.UUID("8d8ac610-566d-4ef0-9c22-186b2a5ed793") + + +class BodyTooLargeError(Exception): + """Raised when a filtered body exceeds the gzip/inflated caps.""" + + +@dataclass +class FilterResult: + """Outcome of filtering one session file.""" + + manifest: Dict[str, Any] + kept_lines: List[str] + dropped_oversize: int = 0 + kept_type_counts: Dict[str, int] = field(default_factory=dict) + session_id: str = "" + + +def _iter_entries(lines: Iterable[str]): + """Yield (raw_line, parsed_dict) for parseable JSON lines, skipping junk.""" + for raw in lines: + stripped = raw.strip() + if not stripped: + continue + try: + parsed = json.loads(stripped) + except (json.JSONDecodeError, ValueError): + continue + if isinstance(parsed, dict): + yield stripped, parsed + + +def _filter_entries( + lines: Iterable[str], device_id: str, filename: str, client_version: str +) -> FilterResult: + """Single-pass core: consume ``lines`` once, applying the allowlist. + + Works on any line iterable — including an open file object, which streams + line by line so a huge raw-ADC session is never held in memory at once. + Captures the ``session_uuid`` from ``session_start`` during the same pass + so we don't need a second read to resolve the upload id. + """ + kept_lines: List[str] = [] + kept_type_counts: Dict[str, int] = {} + dropped_oversize = 0 + embedded_uuid = "" + + for raw, entry in _iter_entries(lines): + entry_type = entry.get("type") + if entry_type == "session_start" and not embedded_uuid: + session_uuid = entry.get("session_uuid") + if session_uuid: + embedded_uuid = str(session_uuid).lower() + if entry_type not in KEEP_ENTRY_TYPES: + continue + if len(raw.encode("utf-8")) > MAX_LINE_BYTES: + dropped_oversize += 1 + continue + kept_lines.append(raw) + kept_type_counts[entry_type] = kept_type_counts.get(entry_type, 0) + 1 + + session_id = embedded_uuid or str(uuid.uuid5(SESSION_NAMESPACE, f"{device_id}:{filename}")) + manifest = { + "type": MANIFEST_TYPE, + "format_version": MANIFEST_FORMAT_VERSION, + "client_version": client_version, + "device_id": device_id, + "filtered": True, + "kept_entry_types": sorted(kept_type_counts), + } + return FilterResult( + manifest=manifest, + kept_lines=kept_lines, + dropped_oversize=dropped_oversize, + kept_type_counts=kept_type_counts, + session_id=session_id, + ) + + +def filter_session_lines( + lines: Iterable[str], + device_id: str, + client_version: str = CLIENT_VERSION, + filename: str = "", +) -> FilterResult: + """Filter raw session lines to the allowlist and build the manifest. + + Drops non-allowlisted types, drops any kept line over ``MAX_LINE_BYTES`` + (counting them), and skips blank/unparseable lines. ``filename`` is only + used for the UUIDv5 session-id fallback when no ``session_uuid`` is present. + """ + return _filter_entries(lines, device_id, filename, client_version) + + +def filter_session_file(path, device_id: str, client_version: str = CLIENT_VERSION) -> FilterResult: + """Stream-filter a session file by path without loading it into memory. + + The raw ADC (rolling_buffer_capture / kld7_buffer / iq_blocks) is dropped as + each line is read, so peak memory stays near a single line regardless of how + large the raw-ADC file is. This is the production path for ``push``. + """ + path = Path(path) + with path.open(encoding="utf-8", errors="replace") as handle: + return _filter_entries(handle, device_id, path.name, client_version) + + +def build_upload_body( + result: FilterResult, + max_gzip_bytes: Optional[int] = None, + max_inflated_bytes: Optional[int] = None, +) -> bytes: + """Build the gzipped NDJSON upload body (manifest first), enforcing caps. + + Raises BodyTooLargeError if the body would exceed either cap — the caller + should park the session and report it rather than upload raw. + """ + # Resolve at call time so tests (and config) can adjust the module caps. + max_gzip_bytes = MAX_GZIP_BYTES if max_gzip_bytes is None else max_gzip_bytes + max_inflated_bytes = MAX_INFLATED_BYTES if max_inflated_bytes is None else max_inflated_bytes + out_lines = [json.dumps(result.manifest)] + out_lines.extend(result.kept_lines) + ndjson = ("\n".join(out_lines) + "\n").encode("utf-8") + + if len(ndjson) > max_inflated_bytes: + raise BodyTooLargeError( + f"inflated body {len(ndjson)} bytes exceeds cap {max_inflated_bytes}" + ) + + # mtime=0 keeps the gzip output deterministic (stable retries/dedupe). + body = gzip.compress(ndjson, mtime=0) + if len(body) > max_gzip_bytes: + raise BodyTooLargeError(f"gzip body {len(body)} bytes exceeds cap {max_gzip_bytes}") + return body diff --git a/src/openflight/cloud/spool.py b/src/openflight/cloud/spool.py new file mode 100644 index 000000000..e6a82f62a --- /dev/null +++ b/src/openflight/cloud/spool.py @@ -0,0 +1,184 @@ +"""Spool-and-retry mechanics — the session directory *is* the queue. + +State lives in sidecar files next to each ``session_*.jsonl`` so it survives +crashes with no database: + +- ``.jsonl.pushed`` — present once accepted by the server (terminal). +- ``.jsonl.parked`` — present once given up on (terminal). +- ``.jsonl.state`` — JSON attempt counter + last error for in-flight + retries; removed on success. + +Originals are never moved or modified. +""" + +import json +import time +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, List, Optional + +PUSHED_SUFFIX = ".pushed" +PARKED_SUFFIX = ".parked" +STATE_SUFFIX = ".state" + +SESSION_GLOB = "session_*.jsonl" + +# After this many failures, park the file and report via ``status`` instead of +# retrying forever. +MAX_ATTEMPTS = 20 + +# How long to defer a session after a quota (402) rejection — retry daily +# rather than every timer tick. +QUOTA_COOLDOWN_S = 24 * 60 * 60 + + +def _sidecar(path: Path, suffix: str) -> Path: + # Append (not replace) so "session_x.jsonl" -> "session_x.jsonl.pushed". + return path.with_name(path.name + suffix) + + +def _now() -> str: + return datetime.now().isoformat() + + +def session_files(log_dir: Path) -> List[Path]: + """All session JSONL files in ``log_dir`` (sorted); empty if dir missing.""" + log_dir = Path(log_dir) + if not log_dir.is_dir(): + return [] + return sorted(log_dir.glob(SESSION_GLOB)) + + +def is_pushed(path: Path) -> bool: + """True if a ``.pushed`` marker exists for this session.""" + return _sidecar(path, PUSHED_SUFFIX).exists() + + +def is_parked(path: Path) -> bool: + """True if a ``.parked`` marker exists for this session.""" + return _sidecar(path, PARKED_SUFFIX).exists() + + +def pending_sessions(log_dir: Path) -> List[Path]: + """Session files that are neither pushed nor parked.""" + return [p for p in session_files(log_dir) if not is_pushed(p) and not is_parked(p)] + + +def read_attempts(path: Path) -> int: + """Number of recorded failed attempts for this session (0 if none).""" + state_path = _sidecar(path, STATE_SUFFIX) + if not state_path.exists(): + return 0 + try: + return int(json.loads(state_path.read_text()).get("attempts", 0)) + except (json.JSONDecodeError, ValueError): + return 0 + + +def _write_json(path: Path, data: Dict[str, Any]) -> None: + path.write_text(json.dumps(data, indent=2) + "\n") + + +def record_failure(path: Path, error: str) -> int: + """Increment the attempt counter, store the error, and park if maxed out. + + Returns the new attempt count. + """ + attempts = read_attempts(path) + 1 + _write_json( + _sidecar(path, STATE_SUFFIX), + {"attempts": attempts, "last_error": error, "last_attempt_at": _now()}, + ) + if attempts >= MAX_ATTEMPTS: + mark_parked(path, reason="max_attempts", attempts=attempts, last_error=error) + return attempts + + +def record_cooldown(path: Path, reason: str, seconds: float, now: Optional[float] = None) -> None: + """Defer retries for this session until ``seconds`` from now (e.g. quota). + + Does not increment the failure counter or park — quota is not a client bug. + """ + now = time.time() if now is None else now + state_path = _sidecar(path, STATE_SUFFIX) + state: Dict[str, Any] = {} + if state_path.exists(): + try: + state = json.loads(state_path.read_text()) + except (json.JSONDecodeError, ValueError): + state = {} + state.update( + {"cooldown_until": now + seconds, "cooldown_reason": reason, "last_attempt_at": _now()} + ) + _write_json(state_path, state) + + +def in_cooldown(path: Path, now: Optional[float] = None) -> bool: + """True if this session is deferred (cooldown not yet elapsed).""" + now = time.time() if now is None else now + state_path = _sidecar(path, STATE_SUFFIX) + if not state_path.exists(): + return False + try: + until = json.loads(state_path.read_text()).get("cooldown_until") + except (json.JSONDecodeError, ValueError): + return False + return until is not None and now < until + + +def mark_pushed(path: Path, session_id: str, shot_count: Optional[int]) -> None: + """Mark a session as successfully uploaded and clear retry state.""" + _write_json( + _sidecar(path, PUSHED_SUFFIX), + {"session_id": session_id, "shot_count": shot_count, "pushed_at": _now()}, + ) + state_path = _sidecar(path, STATE_SUFFIX) + if state_path.exists(): + state_path.unlink() + + +def mark_parked(path: Path, reason: str, attempts: int, last_error: Optional[str]) -> None: + """Mark a session as parked (given up on); reported via ``status``.""" + _write_json( + _sidecar(path, PARKED_SUFFIX), + { + "reason": reason, + "attempts": attempts, + "last_error": last_error, + "parked_at": _now(), + }, + ) + + +def clear_markers(path: Path, include_pushed: bool = False) -> List[str]: + """Remove terminal/retry markers so a session becomes pending again. + + Clears ``.parked`` and ``.state`` (un-park + reset attempts/cooldown). With + ``include_pushed`` also clears ``.pushed`` to force re-upload of a session + the server already stored (idempotent server-side). Returns the suffixes + actually removed. + """ + suffixes = [PARKED_SUFFIX, STATE_SUFFIX] + if include_pushed: + suffixes.append(PUSHED_SUFFIX) + cleared: List[str] = [] + for suffix in suffixes: + marker = _sidecar(path, suffix) + if marker.exists(): + marker.unlink() + cleared.append(suffix) + return cleared + + +def summarize(log_dir: Path) -> Dict[str, int]: + """Count pushed / parked / pending sessions for ``status``.""" + files = session_files(log_dir) + pushed = sum(1 for p in files if is_pushed(p)) + parked = sum(1 for p in files if is_parked(p)) + pending = sum(1 for p in files if not is_pushed(p) and not is_parked(p)) + return { + "total": len(files), + "pushed": pushed, + "parked": parked, + "pending": pending, + } diff --git a/src/openflight/cloud/trigger.py b/src/openflight/cloud/trigger.py new file mode 100644 index 000000000..18b13e828 --- /dev/null +++ b/src/openflight/cloud/trigger.py @@ -0,0 +1,50 @@ +"""Fire-and-forget push trigger for the server's session-end happy path. + +Spawns ``openflight-cloud push`` as a detached subprocess so an upload never +blocks or delays shot processing. The systemd timer remains the safety net that +heals wifi outages; this is just the fast path. +""" + +import subprocess +import sys +from pathlib import Path +from typing import Callable + +from .config import CONFIG_PATH, CloudConfig + + +def fire_push_async( + config: CloudConfig, + log_dir: Path, + config_path: Path = CONFIG_PATH, + popen_fn: Callable[..., object] = subprocess.Popen, +) -> bool: + """Spawn a detached ``push`` if the uploader is active. Never raises. + + Returns True if a push was spawned, False otherwise (inactive or spawn + failed). The caller is on the shot/session path, so all errors are + swallowed. + """ + if not config.is_active(): + return False + + cmd = [ + sys.executable, + "-m", + "openflight.cloud.cli", + "--config", + str(config_path), + "--log-dir", + str(log_dir), + "push", + ] + try: + popen_fn( + cmd, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + return True + except (OSError, ValueError): + return False diff --git a/src/openflight/kld7/radc.py b/src/openflight/kld7/radc.py index 9bdb8190b..a626907ed 100644 --- a/src/openflight/kld7/radc.py +++ b/src/openflight/kld7/radc.py @@ -9,6 +9,7 @@ import logging import math from dataclasses import asdict, dataclass +from functools import lru_cache import numpy as np @@ -48,10 +49,29 @@ DEFAULT_BALL_ALIASED_MIN_KMH = -39.0 DEFAULT_BALL_ALIASED_MAX_KMH = -7.0 +# DC alias blind zone: ball speeds whose Doppler alias lands within this +# margin of 0 km/h sit in the DC clutter region (and partially under the +# DC mask), where angle extraction is unreliable. At the ±100 km/h wrap, +# 124 mph aliases exactly onto DC; ±11 km/h covers the empirically bad +# ~118-131 mph band (sub-frame STFT findings, 2026-06-09: 26% two-ray +# pass rate vs ~70% for clear speeds). Affects 4i/5i/3h ball speeds. +DC_ALIAS_BLIND_ZONE_KMH = 11.0 +# Confidence ceiling for shots measured inside the blind zone — same +# ceiling as legacy-naive-suspect results, since both are clutter-prone. +DC_BLIND_ZONE_CONFIDENCE_MAX = 0.35 + +# Sub-frame STFT instrumentation: 64-sample windows at 75% overlap give +# 13 sub-frames per 256-sample RADC frame (~7.2 ms each, ~1.8 ms apart) — +# short enough to resolve ground-bounce fringe oscillation within a frame. +SUBFRAME_WINDOW_SAMPLES = 64 +SUBFRAME_STEP_SAMPLES = 16 + # K-LD7 antenna parameters (24 GHz) WAVELENGTH_M = 3e8 / 24.125e9 # ~12.43 mm ANTENNA_SPACING_M = 8.0e-3 # ~0.64λ, calibrated against PDAT reference data +MPH_TO_KMH = 1.609 + # Vertical rule-stack candidate selection (impact-relative, OPS-anchored). # Primary frames are far enough after impact to usually be ball-only. The # early context window can support a primary anchor when it is OPS-bin aligned, @@ -92,17 +112,35 @@ def parse_radc_payload(payload: bytes) -> dict[str, np.ndarray]: } +def _channels_from_frame(radc_raw): + """Return the channel dict for a frame's RADC payload. + + Frames carry either the raw 3072-byte payload (live path) or an + already-parsed channel dict (offline replay). + """ + return parse_radc_payload(radc_raw) if isinstance(radc_raw, bytes) else radc_raw + + def to_complex_iq(i_channel: np.ndarray, q_channel: np.ndarray) -> np.ndarray: """Convert uint16 I/Q arrays to complex float, removing DC offset. Uses per-channel mean removal instead of a fixed midpoint, since the K-LD7 ADC bias varies across channels and units. """ - i_float = i_channel.astype(np.float64) - np.mean(i_channel.astype(np.float64)) - q_float = q_channel.astype(np.float64) - np.mean(q_channel.astype(np.float64)) + i_float = i_channel.astype(np.float64) + i_float -= i_float.mean() + q_float = q_channel.astype(np.float64) + q_float -= q_float.mean() return i_float + 1j * q_float +@lru_cache(maxsize=4) +def _hann_window(n: int) -> np.ndarray: + # Cached per length: this runs twice per RADC frame at ~34 Hz, and the + # window never changes. Callers must not mutate the returned array. + return np.hanning(n) + + def compute_spectrum( iq: np.ndarray, fft_size: int = 2048, dc_mask_bins: int = DC_MASK_BINS ) -> np.ndarray: @@ -116,11 +154,7 @@ def compute_spectrum( Returns: Magnitude spectrum (linear scale), length = fft_size """ - windowed = iq * np.hanning(len(iq)) - padded = np.zeros(fft_size, dtype=np.complex128) - padded[: len(windowed)] = windowed - fft_result = np.fft.fft(padded) - magnitude = np.abs(fft_result) + magnitude = np.abs(np.fft.fft(iq * _hann_window(len(iq)), n=fft_size)) # Mask DC leakage at both ends of the spectrum if dc_mask_bins > 0: magnitude[:dc_mask_bins] = 0.0 @@ -132,10 +166,7 @@ def compute_fft_complex( iq: np.ndarray, fft_size: int = 2048, dc_mask_bins: int = DC_MASK_BINS ) -> np.ndarray: """Compute complex FFT output (not magnitude) for phase-based processing.""" - windowed = iq * np.hanning(len(iq)) - padded = np.zeros(fft_size, dtype=np.complex128) - padded[: len(windowed)] = windowed - result = np.fft.fft(padded) + result = np.fft.fft(iq * _hann_window(len(iq)), n=fft_size) if dc_mask_bins > 0: result[:dc_mask_bins] = 0.0 result[-dc_mask_bins:] = 0.0 @@ -345,7 +376,7 @@ def aliased_velocity_from_ball_speed_mph( max_speed_kmh: float = 100.0, ) -> float: """Map true ball speed to the K-LD7 aliased Doppler velocity in km/h.""" - ball_speed_kmh = ball_speed_mph * 1.609 + ball_speed_kmh = ball_speed_mph * MPH_TO_KMH unambiguous_range = max_speed_kmh * 2.0 aliased_kmh = ball_speed_kmh % unambiguous_range if aliased_kmh > max_speed_kmh: @@ -363,6 +394,21 @@ def expected_ball_bin_from_speed( return _velocity_to_bin(aliased_kmh, fft_size, max_speed_kmh) +def is_dc_alias_blind_zone( + ball_speed_mph: float, + margin_kmh: float = DC_ALIAS_BLIND_ZONE_KMH, + max_speed_kmh: float = 100.0, +) -> bool: + """Return True when a ball speed's Doppler alias lands near DC. + + These shots' spectra sit in the DC clutter region where the ball + return is masked or blended with clutter — angle output should be + treated as low-confidence regardless of its apparent SNR. + """ + aliased_kmh = aliased_velocity_from_ball_speed_mph(ball_speed_mph, max_speed_kmh) + return abs(aliased_kmh) <= margin_kmh + + def circular_bin_distance(a: int, b: int, fft_size: int = 2048) -> int: """Shortest distance between FFT bins, accounting for circular wrap.""" distance = abs(int(a) - int(b)) @@ -411,7 +457,7 @@ def ball_bin_range_from_speed( tolerance_mph: Search window around the expected velocity (±) """ aliased_kmh = aliased_velocity_from_ball_speed_mph(ball_speed_mph, max_speed_kmh) - tol_kmh = tolerance_mph * 1.609 + tol_kmh = tolerance_mph * MPH_TO_KMH lo_vel = aliased_kmh - tol_kmh hi_vel = aliased_kmh + tol_kmh @@ -604,9 +650,9 @@ def _find_peak_near_expected_bin( if sub_hi <= sub_lo: continue indices = np.arange(sub_lo, sub_hi, dtype=int) - mask = np.array( - [circular_bin_distance(idx, expected_bin, fft_size) <= tolerance for idx in indices] - ) + # Vectorized circular_bin_distance over the whole sub-band + direct = np.abs(indices - expected_bin) + mask = np.minimum(direct, fft_size - direct) <= tolerance if not mask.any(): continue near_indices = indices[mask] @@ -705,6 +751,90 @@ def _phase_coherence_for_peak( return float(np.clip(coherence, 0.0, 1.0)) +def subframe_fringe_metrics( + f1a_iq: np.ndarray, + f2a_iq: np.ndarray, + peak_bin: int, + fft_size: int = 2048, + search_bins: int = CENTROID_SEARCH_BINS, +) -> dict | None: + """Sub-frame STFT fringe metrics for one RADC frame. + + Slices the frame into short overlapping windows and tracks the + interferometric elevation and F1A/F2A magnitude balance at the peak + across them. Indoor ground-bounce multipath sweeps several fringe + cycles within one ~29 ms frame, which shows up as coherent + oscillation in both metrics — flat metrics indicate a clean + single-ray return. Diagnostic only: logged per shot so the fringe + hypothesis can be checked against this rig's sessions offline. + + Returns None when the frame is too short or no sub-frame has usable + signal at the peak. + """ + n = len(f1a_iq) + if n < SUBFRAME_WINDOW_SAMPLES: + return None + lo = max(0, peak_bin - search_bins) + hi = min(fft_size, peak_bin + search_bins + 1) + if hi <= lo: + return None + + balances: list[float] = [] + elevations: list[float] = [] + for start in range(0, n - SUBFRAME_WINDOW_SAMPLES + 1, SUBFRAME_STEP_SAMPLES): + window = slice(start, start + SUBFRAME_WINDOW_SAMPLES) + f1 = compute_fft_complex(f1a_iq[window], fft_size=fft_size) + f2 = compute_fft_complex(f2a_iq[window], fft_size=fft_size) + # The sub-frame peak wanders within the frame peak's neighborhood + # as the fringe modulates — track it rather than reading a fixed bin. + local = np.abs(f1[lo:hi]) + np.abs(f2[lo:hi]) + sub_bin = lo + int(np.argmax(local)) + mag1 = float(np.abs(f1[sub_bin])) + mag2 = float(np.abs(f2[sub_bin])) + if mag1 <= 0 or mag2 <= 0: + continue + balances.append(mag2 / mag1) + phase_diff = float(np.angle(f1[sub_bin] * np.conj(f2[sub_bin]))) + sin_theta = phase_diff * WAVELENGTH_M / (2 * np.pi * ANTENNA_SPACING_M) + elevations.append(float(np.degrees(np.arcsin(np.clip(sin_theta, -1.0, 1.0))))) + + if not balances: + return None + elev = np.array(elevations) + bal = np.array(balances) + return { + "subframe_count": len(balances), + "elev_p2p_deg": round(float(elev.max() - elev.min()), 2), + "elev_std_deg": round(float(elev.std()), 2), + "balance_min": round(float(bal.min()), 3), + "balance_max": round(float(bal.max()), 3), + } + + +def _fringe_metrics_for_selected_frames( + frames: list[dict], + frame_indices, + peak_bins, + fft_size: int, +) -> list[dict]: + """Compute fringe metrics for the frames a shot result selected.""" + out: list[dict] = [] + for fi, peak_bin in zip(frame_indices, peak_bins): + radc_raw = frames[int(fi)].get("radc") + if radc_raw is None: + continue + try: + channels = _channels_from_frame(radc_raw) + f1a_iq = to_complex_iq(channels["f1a_i"], channels["f1a_q"]) + f2a_iq = to_complex_iq(channels["f2a_i"], channels["f2a_q"]) + except (KeyError, TypeError, ValueError): + continue + metrics = subframe_fringe_metrics(f1a_iq, f2a_iq, int(peak_bin), fft_size=fft_size) + if metrics is not None: + out.append({"frame_index": int(fi), **metrics}) + return out + + @dataclass(frozen=True) class _VerticalFrameCandidate: frame_index: int @@ -713,8 +843,6 @@ class _VerticalFrameCandidate: snr_linear: float angle_deg: float speed_mph: float - raw_angle_deg: float - geom_bearing_deg: float t_after_impact_s: float | None phase_coherence: float | None peak_width_bins: int @@ -1001,7 +1129,7 @@ def empty(reason: str, has_radc: bool, warnings: tuple[str, ...]) -> RADCFrameDi return empty("missing_radc", has_radc=False, warnings=("missing_radc",)) try: - channels = parse_radc_payload(radc_raw) if isinstance(radc_raw, bytes) else radc_raw + channels = _channels_from_frame(radc_raw) except ValueError: return empty("invalid_payload_size", has_radc=True, warnings=("invalid_payload",)) if not isinstance(channels, dict): @@ -1100,7 +1228,7 @@ def empty(reason: str, has_radc: bool, warnings: tuple[str, ...]) -> RADCFrameDi else None ) peak_velocity_kmh = bin_to_velocity_kmh(peak_bin, fft_size, max_speed_kmh) - peak_ball_speed_mph = (2.0 * max_speed_kmh + peak_velocity_kmh) / 1.609 + peak_ball_speed_mph = (2.0 * max_speed_kmh + peak_velocity_kmh) / MPH_TO_KMH speed_error_mph = ( peak_ball_speed_mph - ops243_ball_speed_mph if ops243_ball_speed_mph is not None else None ) @@ -1247,7 +1375,7 @@ def find_impact_frames( energies.append(0.0) continue try: - channels = parse_radc_payload(radc) if isinstance(radc, bytes) else radc + channels = _channels_from_frame(radc) iq = to_complex_iq(channels["f1a_i"], channels["f1a_q"]) except (KeyError, TypeError, ValueError): energies.append(0.0) @@ -1279,7 +1407,6 @@ def extract_launch_angle( frames: list[dict], fft_size: int = 2048, max_speed_kmh: float = 100.0, - cfar_threshold: float = 2.5, impact_energy_threshold: float = 3.0, angle_offset_deg: float = 0.0, ops243_ball_speed_mph: float | None = None, @@ -1303,7 +1430,8 @@ def extract_launch_angle( Pipeline: 1. Find impact frames (high-velocity energy spikes) 2. Group consecutive impacts into shot events - 3. For each shot, run band-limited CFAR in the ball velocity range + 3. For each shot, find the strongest peak in the ball velocity band + (median-SNR gated, OPS-anchored when ball speed is available) 4. Per-bin interferometric angle estimation on ball detections. The per-frame angle is the magnitude²-weighted centroid of the per-bin angles inside the spectral peak (bins whose magnitude @@ -1405,6 +1533,18 @@ def extract_launch_angle( ball_bands = default_ball_bin_ranges(fft_size, max_speed_kmh) ops_expected_bin = None + dc_blind_zone = ops243_ball_speed_mph is not None and is_dc_alias_blind_zone( + ops243_ball_speed_mph, max_speed_kmh=max_speed_kmh + ) + if dc_blind_zone: + logger.warning( + "[RADC] Ball speed %.1f mph aliases within ±%.0f km/h of DC — " + "angle extraction is degraded in this band (confidence capped at %.2f)", + ops243_ball_speed_mph, + DC_ALIAS_BLIND_ZONE_KMH, + DC_BLIND_ZONE_CONFIDENCE_MAX, + ) + min_velocity_bin = 150 # skip low-velocity body/clutter impact_indices = find_impact_frames( frames, @@ -1414,9 +1554,7 @@ def extract_launch_angle( ball_bands=ball_bands, ) if not impact_indices: - import logging - - logging.getLogger("openflight.kld7.radc").info( + logger.info( "[KLD7-RADC] No impact frames found (energy_threshold=%.1f, ball_bands=%s, %d frames)", impact_energy_threshold, ball_bands, @@ -1451,25 +1589,20 @@ def extract_launch_angle( peak_snrs = [] peak_speeds_mph = [] peak_bins: list[int] = [] - peak_times: list[float | None] = [] # flight time per frame (s), for geometry peak_coherences: list[float | None] = [] peak_widths: list[int] = [] peak_ops_anchor_weak: list[bool] = [] peak_frame_indices: list[int] = [] + # Time after impact per frame (s) — doubles as the flight time + # used by the geometry estimator. per_frame_t_after_impact: list[float | None] = [] for fi in sorted(frame_set): radc_raw = frames[fi].get("radc") if radc_raw is None: continue - frame_ts = frames[fi].get("timestamp") - flight_time_s = ( - float(frame_ts) - float(time_ref) - if frame_ts is not None and time_ref is not None - else None - ) try: - channels = parse_radc_payload(radc_raw) if isinstance(radc_raw, bytes) else radc_raw + channels = _channels_from_frame(radc_raw) except (KeyError, TypeError, ValueError): continue @@ -1539,47 +1672,23 @@ def extract_launch_angle( # Magnitude²-weighted centroid of the per-bin angles across # the spectral peak, rather than the raw angle at a single - # bin. Search a small neighborhood (`centroid_search_bins`) - # around the peak and include bins whose magnitude is at - # least `centroid_floor_frac` of the peak. For a range- - # spread target this integrates the angle estimate across - # all the energy in the peak; restricting to a neighborhood - # prevents random noise bins elsewhere in the band (which - # have similar magnitudes when there is no real ball signal) - # from contributing. This is the wideband monopulse - # formulation (Zhang et al., Sensors 2016). - if centroid_floor_frac < 1.0: - # Clip the centroid neighborhood to the sub-band that - # contains the peak. This prevents the neighborhood - # from spilling across the wrap into an unrelated - # spectral region when the ball band wraps around DC. - # Fall back to FFT bounds if peak_bin sits outside any - # listed sub-band (defensive — shouldn't happen). - sub_for_peak = next( - (sub for sub in ball_bands if sub[0] <= peak_bin < sub[1]), - (0, fft_size), - ) - sub_lo, sub_hi = sub_for_peak - lo_n = max(sub_lo, peak_bin - CENTROID_SEARCH_BINS) - hi_n = min(sub_hi, peak_bin + CENTROID_SEARCH_BINS + 1) - neigh = spec[lo_n:hi_n] - neigh_mask = neigh >= peak_val * centroid_floor_frac - if neigh_mask.any(): - neigh_indices = np.flatnonzero(neigh_mask) + lo_n - neigh_w = neigh[neigh_mask] ** 2 - neigh_w_sum = float(neigh_w.sum()) - if neigh_w_sum > 0: - centroid_angle = float( - np.sum(angles[neigh_indices] * neigh_w) / neigh_w_sum - ) - else: - centroid_angle = float(angles[peak_bin]) - else: - centroid_angle = float(angles[peak_bin]) - else: - # Disabled (frac=1.0) — fall back to the legacy - # single-peak-bin angle for exact backward compatibility. - centroid_angle = float(angles[peak_bin]) + # bin. For a range-spread target this integrates the angle + # estimate across all the energy in the peak; restricting to + # a neighborhood prevents random noise bins elsewhere in the + # band (which have similar magnitudes when there is no real + # ball signal) from contributing. This is the wideband + # monopulse formulation (Zhang et al., Sensors 2016). + # `peak_band` is the sub-band the peak was found in, so the + # neighborhood cannot spill across the DC wrap into an + # unrelated spectral region. + centroid_angle, peak_width = _centroid_angle_for_peak( + angles, + spec, + peak_bin, + peak_val, + peak_band, + centroid_floor_frac, + ) phase_coherence = _phase_coherence_for_peak( f1a_fft, f2a_fft, @@ -1589,19 +1698,9 @@ def extract_launch_angle( peak_band, coherence_bins=4, ) - peak_width = int( - _centroid_angle_for_peak( - angles, - spec, - peak_bin, - peak_val, - peak_band, - centroid_floor_frac, - )[1] - ) vel = bin_to_velocity_kmh(peak_bin, fft_size, max_speed_kmh) - frame_speed_mph = (200.0 + vel) / 1.609 + frame_speed_mph = (2.0 * max_speed_kmh + vel) / MPH_TO_KMH frame_ts = _optional_float(frames[fi].get("timestamp")) t_after_impact: float | None = None if frame_ts is not None and time_ref is not None: @@ -1611,7 +1710,6 @@ def extract_launch_angle( peak_snrs.append(snr) peak_bins.append(peak_bin) peak_speeds_mph.append(frame_speed_mph) - peak_times.append(flight_time_s) peak_coherences.append(phase_coherence) peak_widths.append(peak_width) peak_ops_anchor_weak.append(ops_anchor_weak) @@ -1624,7 +1722,6 @@ def extract_launch_angle( angs = np.array(peak_angles) snrs = np.array(peak_snrs) bins_arr = np.array(peak_bins, dtype=int) - times_arr = np.array([np.nan if t is None else t for t in peak_times], dtype=float) weak_ops_anchor_arr = np.array(peak_ops_anchor_weak, dtype=bool) frame_indices_arr = np.array(peak_frame_indices, dtype=int) t_after_arr = np.array( @@ -1640,8 +1737,6 @@ def extract_launch_angle( ], dtype=float, ) - raw_angs = angs.copy() - geom_arr = np.zeros_like(angs, dtype=float) used_rule_stack = False if orientation == "vertical" and time_ref is not None: candidates: list[_VerticalFrameCandidate] = [] @@ -1659,8 +1754,6 @@ def extract_launch_angle( snr_linear=float(snrs[i]), angle_deg=float(angle), speed_mph=float(peak_speeds_mph[i]), - raw_angle_deg=float(raw_angs[i]), - geom_bearing_deg=float(geom_arr[i]), t_after_impact_s=per_frame_t_after_impact[i], phase_coherence=peak_coherences[i], peak_width_bins=int(peak_widths[i]), @@ -1677,7 +1770,6 @@ def extract_launch_angle( clean_angs = angs[keep_mask] clean_snrs = snrs[keep_mask] clean_bins = bins_arr[keep_mask] - clean_times = times_arr[keep_mask] clean_weak_ops_anchor = weak_ops_anchor_arr[keep_mask] clean_frame_indices = frame_indices_arr[keep_mask] clean_t_after = t_after_arr[keep_mask] @@ -1710,7 +1802,6 @@ def extract_launch_angle( angs = angs[strong_mask] snrs = snrs[strong_mask] bins_arr = bins_arr[strong_mask] - times_arr = times_arr[strong_mask] weak_ops_anchor_arr = weak_ops_anchor_arr[strong_mask] frame_indices_arr = frame_indices_arr[strong_mask] t_after_arr = t_after_arr[strong_mask] @@ -1725,7 +1816,6 @@ def extract_launch_angle( clean_angs = angs clean_snrs = snrs clean_bins = bins_arr - clean_times = times_arr clean_weak_ops_anchor = weak_ops_anchor_arr clean_frame_indices = frame_indices_arr clean_t_after = t_after_arr @@ -1753,7 +1843,6 @@ def extract_launch_angle( clean_angs = angs[clean_mask] clean_snrs = snrs[clean_mask] clean_bins = bins_arr[clean_mask] - clean_times = times_arr[clean_mask] clean_weak_ops_anchor = weak_ops_anchor_arr[clean_mask] clean_frame_indices = frame_indices_arr[clean_mask] clean_t_after = t_after_arr[clean_mask] @@ -1819,27 +1908,27 @@ def extract_launch_angle( and distance_ft is not None ): per_frame_geom = [ - (float(clean_times[i]), float(clean_angs[i] + angle_offset_deg), float(w[i])) + (float(clean_t_after[i]), float(clean_angs[i] + angle_offset_deg), float(w[i])) for i in range(len(clean_angs)) - if not math.isnan(clean_times[i]) + if not math.isnan(clean_t_after[i]) ] strong_per_frame_geom = [ - (float(clean_times[i]), float(clean_angs[i] + angle_offset_deg), float(w[i])) + (float(clean_t_after[i]), float(clean_angs[i] + angle_offset_deg), float(w[i])) for i in range(len(clean_angs)) - if not math.isnan(clean_times[i]) and not bool(clean_weak_ops_anchor[i]) + if not math.isnan(clean_t_after[i]) and not bool(clean_weak_ops_anchor[i]) ] strong_single_frame_indices = [ i for i in range(len(clean_angs)) - if not math.isnan(clean_times[i]) + if not math.isnan(clean_t_after[i]) and not bool(clean_weak_ops_anchor[i]) - and 0.0 < float(clean_times[i]) <= GEOM_FLIGHT_T_MAX_S + and 0.0 < float(clean_t_after[i]) <= GEOM_FLIGHT_T_MAX_S ] single_frame_fallback_idx = ( min( strong_single_frame_indices, key=lambda i: ( - float(clean_times[i]), + float(clean_t_after[i]), ( math.inf if math.isnan(float(clean_bin_errors[i])) @@ -1887,7 +1976,7 @@ def extract_launch_angle( single_geom = ( fit_launch_angle_single_frame_geometric( ( - float(clean_times[single_frame_fallback_idx]), + float(clean_t_after[single_frame_fallback_idx]), float(clean_angs[single_frame_fallback_idx] + angle_offset_deg), float(w[single_frame_fallback_idx]), ), @@ -1905,7 +1994,6 @@ def extract_launch_angle( clean_angs = clean_angs[selected_single] clean_snrs = clean_snrs[selected_single] clean_bins = clean_bins[selected_single] - clean_times = clean_times[selected_single] clean_weak_ops_anchor = clean_weak_ops_anchor[selected_single] clean_frame_indices = clean_frame_indices[selected_single] clean_t_after = clean_t_after[selected_single] @@ -1919,7 +2007,7 @@ def extract_launch_angle( corrected_angle, geom_single_frame_resid, int(clean_frame_indices[0]), - float(clean_times[0]) * 1000.0, + float(clean_t_after[0]) * 1000.0, float(clean_angs[0] + angle_offset_deg), ) else: @@ -1934,7 +2022,7 @@ def extract_launch_angle( logger.info( "[RADC] Geometry estimator: <2 in-flight frames " "(flight times ms=%s); falling back to naive bearing average", - [None if math.isnan(t) else round(float(t) * 1000.0, 1) for t in clean_times], + [None if math.isnan(t) else round(float(t) * 1000.0, 1) for t in clean_t_after], ) # Hard physical bounds — reject obvious outliers before they @@ -2025,6 +2113,9 @@ def extract_launch_angle( elif selection_path == "legacy_naive_suspect": confidence = min(confidence, VERTICAL_LEGACY_NAIVE_CONFIDENCE_MAX) + if dc_blind_zone: + confidence = min(confidence, DC_BLIND_ZONE_CONFIDENCE_MAX) + selected_t_ms = [ None if math.isnan(float(t)) else round(float(t) * 1000.0, 1) for t in clean_t_after ] @@ -2032,6 +2123,9 @@ def extract_launch_angle( None if math.isnan(float(bin_error)) else int(round(float(bin_error))) for bin_error in clean_bin_errors ] + fringe_metrics = _fringe_metrics_for_selected_frames( + frames, clean_frame_indices, clean_bins, fft_size + ) results.append( { @@ -2054,6 +2148,8 @@ def extract_launch_angle( "selected_t_ms": selected_t_ms, "selected_bin_errors": selected_bin_errors, "weak_adjacent_frame_used": bool(np.any(clean_weak_ops_anchor)), + "dc_blind_zone": dc_blind_zone, + "fringe_metrics": fringe_metrics, "ball_speed_mph": round(avg_speed_mph, 1), "confidence": confidence, "detection_count": len(peak_angles), diff --git a/src/openflight/kld7/tracker.py b/src/openflight/kld7/tracker.py index bec8a72dd..2bd6af263 100644 --- a/src/openflight/kld7/tracker.py +++ b/src/openflight/kld7/tracker.py @@ -25,6 +25,8 @@ "geom_fit_rmse_deg", "geom_single_frame_resid_deg", "weak_adjacent_frame_used", + "dc_blind_zone", + "fringe_metrics", "raw_angle_deg", "angle_offset_deg", "spectrum_source", @@ -196,7 +198,10 @@ def _init_ring_buffer(self): def connect(self) -> bool: """Connect to K-LD7 and configure for golf.""" if find_spec("kld7") is None: - logger.error("[KLD7] kld7 package not installed. Run: pip install kld7") + logger.error( + "[KLD7] kld7 package not installed. Reinstall the project: " + "uv pip install -e '.[ui]'" + ) return False port = self.port or _find_port() @@ -454,27 +459,19 @@ def _stream_loop(self): "[KLD7] Stream generator exited (frames=%d, %s)", frame_count, self.orientation ) - except KLD7Exception as e: - errors += 1 - logger.warning( - "[KLD7] Stream error %d/%d (%s): %s", errors, max_errors, self.orientation, e - ) - if errors < max_errors: - self._drain_after_stream_error() - continue - reconnects += 1 - if reconnects > max_reconnects: - break - if self._reconnect_after_stream_errors(errors): - errors = 0 - continue - break - except Exception as e: - if _is_recoverable_stream_error(e): + # KLD7Exception is always treated as recoverable; other + # exceptions only when they match known transient patterns. + if isinstance(e, KLD7Exception) or _is_recoverable_stream_error(e): errors += 1 + label = ( + "Stream error" + if isinstance(e, KLD7Exception) + else "Recoverable stream error" + ) logger.warning( - "[KLD7] Recoverable stream error %d/%d (%s): %s", + "[KLD7] %s %d/%d (%s): %s", + label, errors, max_errors, self.orientation, @@ -557,8 +554,8 @@ def _radc_frames_for_extraction( if shot_timestamp is None: return frames, frames_available, 0 - window_before_s = max(float(getattr(self, "buffer_seconds", 0.0) or 0.0), 0.0) - window_after_s = max(float(getattr(self, "shot_window_after_s", 0.0) or 0.0), 0.0) + window_before_s = max(float(self.buffer_seconds or 0.0), 0.0) + window_after_s = max(float(self.shot_window_after_s or 0.0), 0.0) start = shot_timestamp - window_before_s end = shot_timestamp + window_after_s diff --git a/src/openflight/launch_monitor.py b/src/openflight/launch_monitor.py index 1654aef27..4630ae08b 100644 --- a/src/openflight/launch_monitor.py +++ b/src/openflight/launch_monitor.py @@ -247,6 +247,9 @@ class Shot: impact_timestamp: Optional[float] = None impact_timestamp_kld7: Optional[float] = None club_speed_mph: Optional[float] = None + # Raw OPS radial ball speed, kept when the cosine correction rewrites + # ball_speed_mph (radar bin anchoring must keep using the radial value) + ball_speed_raw_mph: Optional[float] = None peak_magnitude: Optional[float] = None readings: List[SpeedReading] = field(default_factory=list) club: ClubType = ClubType.DRIVER @@ -259,6 +262,10 @@ class Shot: launch_angle_horizontal_source: Optional[str] = None spin_rpm: Optional[float] = None spin_confidence: Optional[float] = None + # Raw radar-measured spin, kept when --calculated-spin rewrites + # spin_rpm with the kinematic estimate (for offline scoring) + spin_rpm_measured: Optional[float] = None + spin_source: Optional[str] = None # "measured", "calculated", or None spin_result_quality: Optional[str] = None spin_snr: Optional[float] = None spin_modulation_depth: Optional[float] = None diff --git a/src/openflight/ops243.py b/src/openflight/ops243.py index d1b7d6a73..ae5a26839 100644 --- a/src/openflight/ops243.py +++ b/src/openflight/ops243.py @@ -272,6 +272,7 @@ def read_clock_sync( per_read_timeout: float = 0.2, max_sync_duration_s: float = 1.25, sample_interval_s: float = 0.01, + store: bool = True, ) -> dict: """Map the OPS internal clock to host epoch via repeated ``C?`` reads. @@ -296,9 +297,10 @@ def read_clock_sync( Sound-triggered captures use this mapping to convert the radar's internal ``trigger_time`` to host epoch only when - ``usable_for_trigger_timestamps`` is true. Returns a summary dict (also - stored on ``self.last_clock_sync``); never raises on a missing/garbled - reply. + ``usable_for_trigger_timestamps`` is true. Returns a summary dict. By + default it is also stored on ``self.last_clock_sync``; pass + ``store=False`` for diagnostics that should not affect the live timing + path. Never raises on a missing/garbled reply. """ if not self.serial or not self.serial.is_open: raise ConnectionError("Not connected to radar") @@ -409,7 +411,8 @@ def find_integer_rollover() -> Optional[tuple[dict, dict]]: "rollover_uncertainty_ms": rollover_uncertainty_ms, "reads": reads, } - self.last_clock_sync = summary + if store: + self.last_clock_sync = summary if usable_for_trigger_timestamps: logger.info( "[OPS] Clock sync: method=%s offset=%.3fs best_read_latency=%.1fms " @@ -1048,7 +1051,7 @@ def trigger_capture(self, timeout: float = 10.0) -> str: return full_response - def wait_for_hardware_trigger(self, timeout: float = 30.0) -> str: + def wait_for_hardware_trigger(self, timeout: float = 30.0, dump_grace: float = 8.0) -> str: """ Wait for hardware trigger to fire and read the buffer dump. @@ -1057,7 +1060,11 @@ def wait_for_hardware_trigger(self, timeout: float = 30.0) -> str: (HOST_INT). Used with SoundTrigger (SparkFun SEN-14262). Args: - timeout: Maximum time to wait for trigger data + timeout: Maximum time to wait for the trigger to fire + dump_grace: Extra time allowed for the dump to finish once the + first byte has arrived. The ~46KB rolling-buffer dump takes + ~4-5s; a trigger firing near the end of the timeout window + must not have its dump cut off by the original deadline. Returns: Raw response string containing JSON lines, or empty string on timeout @@ -1070,11 +1077,12 @@ def wait_for_hardware_trigger(self, timeout: float = 30.0) -> str: response_lines = [] start_time = time.time() + deadline = start_time + timeout last_data_time = None bytes_received = 0 self.last_hardware_trigger_first_byte_timestamp = None - while (time.time() - start_time) < timeout: + while time.time() < deadline: if self.serial.in_waiting: first_byte_timestamp = time.time() if last_data_time is None else None chunk = self.serial.read(self.serial.in_waiting) @@ -1083,6 +1091,9 @@ def wait_for_hardware_trigger(self, timeout: float = 30.0) -> str: if first_byte_timestamp is not None: last_data_time = first_byte_timestamp self.last_hardware_trigger_first_byte_timestamp = last_data_time + # The trigger fired — the dump is now in flight. Extend + # the deadline so a late trigger gets its full dump. + deadline = max(deadline, last_data_time + dump_grace) logger.debug( "[OPS] Hardware trigger: first byte after %.1fs", last_data_time - start_time, diff --git a/src/openflight/rolling_buffer/processor.py b/src/openflight/rolling_buffer/processor.py index 9240904a2..75de10463 100644 --- a/src/openflight/rolling_buffer/processor.py +++ b/src/openflight/rolling_buffer/processor.py @@ -89,17 +89,37 @@ class RollingBufferProcessor: # Rail-rejection guards. The envelope FFT has two pathological # regions where the peak picker hunts for noise rather than a real # seam tone: - # - The lowest few bins of the valid seam range are dominated by - # residual DC leakage from the envelope subtraction (the Hann - # main lobe is ~2 bins wide but the leakage shoulder extends - # several bins further). On real captures this produces a - # pile-up at ~2637-3076 RPM. Zero the lowest N bins so they - # cannot win the argmax. + # - Slow envelope drift (range falloff, residual DC) leaks into + # the lowest bins of the seam range. The polynomial detrend + # removes the drift itself; a 1-bin guard absorbs what little + # leakage remains. (This guard was previously 5 bins, which + # zeroed 33-51.3 Hz = 1980-3080 RPM — typical driver backspin — + # and produced a rail-artifact pile-up at ~3296 RPM instead.) # - The highest 1-2 bins are the bandpass shoulder of the # prefilter. Even a moderate noise spike there reads as # ~12000 RPM. Reject the pick when the peak lands there # and SNR isn't strong enough to override. - SPIN_DC_LEAKAGE_BINS = 5 # Zero this many low bins of valid range + SPIN_DC_LEAKAGE_BINS = 1 # Zero this many low bins of valid range + # Picks at or below this RPM are reported but capped to low quality: + # red envelope noise produces sustained narrowband mimics in this + # band that a single ~136 ms capture cannot distinguish from a real + # seam tone (see the low-band cap in detect_spin). + SPIN_LOW_BAND_SUSPECT_MAX_RPM = 3100.0 + # Polynomial order for envelope detrending before the spin FFT. + # Order 3 follows the smooth range-falloff decay across the window + # but cannot track a >=33 Hz seam tone (>=2 cycles in any window + # long enough to pass SPIN_MIN_SAMPLES), so real spin is preserved. + SPIN_DETREND_POLY_ORDER = 3 + # Ball-signal-end detection: the analysis window must stop when the + # ball signal dies (net impact indoors), not at the end of the + # capture — the amplitude cliff plus dead air corrupts the envelope + # FFT. Signal is "lost" when the smoothed envelope stays below + # SPIN_SIGNAL_LOSS_THRESHOLD x (early-window median) for + # SPIN_SIGNAL_LOSS_HOLD_SAMPLES. + SPIN_SIGNAL_LOSS_SMOOTH_SAMPLES = 90 # ~3 ms moving average + SPIN_SIGNAL_LOSS_REF_SAMPLES = 450 # first ~15 ms sets reference level + SPIN_SIGNAL_LOSS_THRESHOLD = 0.15 # fraction of reference level + SPIN_SIGNAL_LOSS_HOLD_SAMPLES = 150 # ~5 ms sustained loss SPIN_UPPER_RAIL_BINS = 2 # Top N bins of valid range = "upper rail" SPIN_PRIOR_MIN_RELATIVE_MAG = 0.40 # Candidate must be this strong to displace argmax SPIN_PRIOR_MAX_RELATIVE_ERROR = 0.55 # Candidate must be within this fraction of expected @@ -416,6 +436,116 @@ def process_overlapping(self, capture: IQCapture) -> SpeedTimeline: """ return self._process_capture(capture, self.STEP_SIZE_OVERLAP) + def _local_noise_floor( + self, valid_mag: np.ndarray, peak_idx: int, window_samples: int + ) -> float: + """Noise level around the peak, excluding the peak's main lobe. + + Takes the *max* of the two sides' medians: a genuine narrow tone + has both sides at the true noise floor, while a peak riding the + top of a red-noise slope has at least one side nearly as high as + the peak itself. A pooled median would let the rolled-off side + mask the slope. + + The exclusion zone scales with the Hann main-lobe width in + zero-padded bins (2 x FFT_SIZE / window_samples per side), so a + short analysis window's wider tone skirt is not counted as noise. + """ + # A real Hann tone reaches its first null 2 x FFT/N padded bins + # from the peak; beyond that only -31 dB sidelobes remain. The + # neighborhood is the immediate shoulder just past the null — a + # red-noise bump's tell-tale is that this shoulder stays near + # peak level, and a wider window would dilute it with far bins + # that have already rolled off. + exclude = int(np.ceil(2 * self.SPIN_ENVELOPE_FFT_SIZE / max(window_samples, 1))) + half_width = 3 * exclude + lo = max(0, peak_idx - half_width) + hi = min(len(valid_mag), peak_idx + half_width + 1) + side_floors = [] + for side in ( + valid_mag[lo : max(lo, peak_idx - exclude)], + valid_mag[min(hi, peak_idx + exclude + 1) : hi], + ): + live = side[side > 0] + # A side needs a few live bins to give a meaningful median + # (the leakage guard zeroes bins near the low band edge). + if live.size >= 3: + side_floors.append(float(np.median(live))) + return max(side_floors) if side_floors else 0.0 + + def _spin_peak_is_persistent( + self, ball_envelope: np.ndarray, peak_freq_hz: float + ) -> bool: + """Whether the picked seam tone is present in both halves of the window. + + A real seam tone persists for the whole ball flight; an envelope-noise + fluctuation concentrates its energy in part of the window. Demoting + non-persistent picks is the single-window analogue of tracking + sideband traces across successive spectra. + """ + half = len(ball_envelope) // 2 + if half < self.SPIN_MIN_SAMPLES // 2: + return True # window too short to split meaningfully + for segment in (ball_envelope[:half], ball_envelope[half:]): + seg = segment - np.mean(segment) + windowed = seg * np.hanning(len(seg)) + magnitude = np.abs(np.fft.fft(windowed, self.SPIN_ENVELOPE_FFT_SIZE)) + freqs = np.fft.fftfreq(self.SPIN_ENVELOPE_FFT_SIZE, d=1 / self.SAMPLE_RATE) + half_fft = self.SPIN_ENVELOPE_FFT_SIZE // 2 + magnitude, freqs = magnitude[1:half_fft], freqs[1:half_fft] + valid = (freqs >= self.SPIN_MIN_SEAM_HZ) & (freqs <= self.SPIN_MAX_SEAM_HZ) + valid_mag, valid_freqs = magnitude[valid], freqs[valid] + if not np.any(valid_mag > 0): + return False + floor = float(np.median(valid_mag[valid_mag > 0])) + # Tolerance: the half-window's natural resolution (±2 bins) + tol_hz = 2.0 * self.SAMPLE_RATE / len(seg) + near = np.abs(valid_freqs - peak_freq_hz) <= tol_hz + if not near.any() or floor <= 0: + return False + near_max = float(valid_mag[near].max()) + # The pick must be above the floor AND (near-)dominant in + # this half. A broad noise bump keeps energy near the pick + # but its dominant frequency wanders across the bump; a real + # tone stays pinned at the same frequency in every half. + if near_max < 2.5 * floor or near_max < 0.7 * float(valid_mag.max()): + return False + return True + + def _ball_signal_end_sample(self, envelope: np.ndarray, start_sample: int) -> int: + """Return the absolute sample index where the ball signal ends. + + Detects sustained loss of the bandpassed ball tone (net impact + indoors) by comparing the smoothed envelope against the level at + the start of the ball window. Returns len(envelope) when no loss + is found, so outdoor shots keep the full capture. + """ + tail = envelope[start_sample:] + if len(tail) < self.SPIN_SIGNAL_LOSS_REF_SAMPLES: + return len(envelope) + + kernel = np.ones(self.SPIN_SIGNAL_LOSS_SMOOTH_SAMPLES) + kernel /= self.SPIN_SIGNAL_LOSS_SMOOTH_SAMPLES + smoothed = np.convolve(tail, kernel, mode="same") + reference = float(np.median(smoothed[: self.SPIN_SIGNAL_LOSS_REF_SAMPLES])) + if reference <= 0: + return len(envelope) + + below = smoothed < reference * self.SPIN_SIGNAL_LOSS_THRESHOLD + hold = self.SPIN_SIGNAL_LOSS_HOLD_SAMPLES + if len(below) < hold: + return len(envelope) + sustained = np.convolve(below.astype(float), np.ones(hold), mode="valid") >= hold + if not sustained.any(): + return len(envelope) + loss_offset = int(np.argmax(sustained)) + logger.info( + "[PROCESSOR] Ball signal lost at %.1fms (window trimmed from %.1fms)", + (start_sample + loss_offset) / self.SAMPLE_RATE * 1000, + len(envelope) / self.SAMPLE_RATE * 1000, + ) + return start_sample + loss_offset + def detect_spin( self, capture: IQCapture, @@ -468,10 +598,11 @@ def detect_spin( # Amplitude envelope envelope = np.abs(filtered) - # Trim to ball-present window (from ball onset to end of capture) + # Trim to ball-present window: from ball onset to where the ball + # signal dies (net impact indoors), falling back to capture end. start_sample = max(0, int(ball_timestamp_ms * self.SAMPLE_RATE / 1000)) spin_window_start_sample = start_sample - spin_window_end_sample = len(envelope) + spin_window_end_sample = self._ball_signal_end_sample(envelope, start_sample) ball_envelope = envelope[spin_window_start_sample:spin_window_end_sample] # Trim filter transients from both ends. sosfiltfilt's internal @@ -512,12 +643,17 @@ def detect_spin( weak_modulation = modulation_depth < 0.01 # Remove DC and apply Hann window - ball_envelope -= envelope_mean + ball_envelope = ball_envelope - envelope_mean if envelope_std < 1e-6: return SpinResult.no_spin_detected( "Envelope variation too low", modulation_depth=modulation_depth, ) + # Detrend slow envelope drift (range falloff) so it cannot leak + # into the low end of the seam band and shadow real driver spin. + x = np.arange(len(ball_envelope), dtype=np.float64) + trend = np.polyval(np.polyfit(x, ball_envelope, self.SPIN_DETREND_POLY_ORDER), x) + ball_envelope = ball_envelope - trend windowed = ball_envelope * np.hanning(len(ball_envelope)) # --- Primary: FFT on envelope --- @@ -563,8 +699,17 @@ def detect_spin( at_lower_rail = peak_idx < leakage + self.SPIN_UPPER_RAIL_BINS at_upper_rail = peak_idx >= n_valid - self.SPIN_UPPER_RAIL_BINS - # SNR: peak vs median noise floor in valid range + # SNR: peak vs the stricter of two noise floors. The global + # median catches white noise; the local median catches red + # envelope noise, which forms broad low-frequency bumps whose + # peaks tower over the global median while sitting barely above + # their own neighborhood. A real seam tone is a narrow line + # (Hann main lobe) above its local floor; a noise-bump peak is + # not. Without the local floor, toneless red noise reads as + # high-confidence ~2000-3300 RPM spin. noise_floor = np.median(valid_mag[valid_mag > 0]) if np.any(valid_mag > 0) else 1.0 + local_floor = self._local_noise_floor(valid_mag, peak_idx, len(windowed)) + noise_floor = max(noise_floor, local_floor) fft_snr = peak_mag / noise_floor if noise_floor > 0 else 0 spin_candidates = self._build_spin_candidates( valid_mag, @@ -627,23 +772,27 @@ def detect_spin( candidates=spin_candidates, ) - # Lower-rail picks survive the leakage zeroing only if energy - # leaks just past the guard. Treat them as suspect: require - # modulation depth to clearly exceed the weak-modulation - # threshold (i.e., a real seam tone rather than envelope wander). + # Lower-rail picks sit where residual envelope drift leaks past + # the detrend. Treat them as suspect: require modulation depth + # to clearly exceed the weak-modulation threshold AND medium + # SNR (a real low-spin driver tone is strong; noise picks in + # this zone hover just above the report floor). if at_lower_rail and ( - modulation_depth is None or modulation_depth < 0.012 + modulation_depth is None + or modulation_depth < 0.012 + or fft_snr < self.SPIN_SNR_MEDIUM ): logger.warning( "[PROCESSOR] Spin rejected: lower-rail peak at %.0f RPM " - "(mod %.4f, envelope-DC leakage)", + "(mod %.4f, SNR %.1f, envelope-drift leakage suspected)", spin_rpm, modulation_depth if modulation_depth is not None else float("nan"), + fft_snr, ) return SpinResult.no_spin_detected( f"Lower-rail peak at {spin_rpm:.0f} RPM " - f"(mod {modulation_depth or 0:.4f}, " - f"envelope-DC leakage suspected)", + f"(mod {modulation_depth or 0:.4f}, SNR {fft_snr:.1f}, " + f"envelope-drift leakage suspected)", snr=fft_snr, modulation_depth=modulation_depth, peak_freq_hz=peak_freq, @@ -815,6 +964,18 @@ def detect_spin( quality = "low" confidence = 0.3 + # A seam tone must persist across the whole window — a pick whose + # energy lives in only half the window is an envelope-noise + # fluctuation, however sharp its spectral peak looks. + if not self._spin_peak_is_persistent(ball_envelope, peak_freq): + logger.info( + "[PROCESSOR] Spin demoted: %.0f RPM peak not persistent " + "across both window halves", + spin_rpm, + ) + confidence = min(confidence, 0.3) + quality = "low" + # Weak modulation caps confidence — the envelope FFT peak may be # noise rather than real seam modulation. if weak_modulation: @@ -822,11 +983,17 @@ def detect_spin( if quality == "high": quality = "medium" - # Low-edge picks can be real low-spin driver candidates, but - # real Trackman comparison sessions also show 3300-3500 RPM - # rail artifacts on irons/wedges. Keep the candidate visible for - # analysis, but never treat it as a reliable spin measurement. - if at_lower_rail: + # Low-band picks (<= ~3100 RPM / ~52 Hz) can be real driver spin, + # and the detrended FFT now reports their correct value — but at + # a ~136 ms window this band has an irreducible noise-mimic + # problem: lowpassed envelope noise produces sustained narrowband + # components here that are physically indistinguishable from a + # seam tone within a single capture (and real TrackMan sessions + # showed exactly such artifacts on irons/wedges). Report the + # value, but never as reliable. Lifting this cap requires an + # estimator with more signal energy (dechirped Doppler sidebands) + # or a longer observation window. + if at_lower_rail or spin_rpm <= self.SPIN_LOW_BAND_SUSPECT_MAX_RPM: confidence = min(confidence, 0.5) if quality in ("high", "medium"): quality = "low" diff --git a/src/openflight/rolling_buffer/trigger.py b/src/openflight/rolling_buffer/trigger.py index 980359127..eb5f8c568 100644 --- a/src/openflight/rolling_buffer/trigger.py +++ b/src/openflight/rolling_buffer/trigger.py @@ -780,6 +780,11 @@ class SoundTrigger(TriggerStrategy): instead, which uses Pi GPIO (lower threshold) + software S! trigger. """ + CLOCK_SYNC_SAMPLES = 36 + CLOCK_SYNC_MAX_ROLLOVER_UNCERTAINTY_MS = 40.0 + CLOCK_SYNC_MAX_TIMEOUT_READ_MS = 50.0 + CLOCK_SYNC_MAX_FALLBACK_AGE_S = 60.0 + def __init__( self, pre_trigger_segments: int = 12, @@ -796,6 +801,207 @@ def __init__( """ super().__init__(pre_trigger_segments=pre_trigger_segments) + @staticmethod + def _clock_sync_last_read_host_time(clock_sync: dict) -> Optional[float]: + """Return the host time of the last C? read in a clock-sync summary.""" + reads = clock_sync.get("reads") or [] + if not reads or not isinstance(reads[-1], dict): + return None + return reads[-1].get("host_after") or reads[-1].get("host_mid") + + @classmethod + def _clock_sync_age_s(cls, clock_sync: dict) -> Optional[float]: + """Return age in seconds for a clock-sync summary.""" + last_host_time = cls._clock_sync_last_read_host_time(clock_sync) + if last_host_time is None: + return None + try: + return time.time() - float(last_host_time) + except (TypeError, ValueError): + return None + + @classmethod + def _clock_sync_quality(cls, clock_sync: object) -> tuple[bool, str]: + """Return whether a clock sync is trustworthy enough for shot timing.""" + if not isinstance(clock_sync, dict): + return False, "missing" + + if not clock_sync.get("usable_for_trigger_timestamps"): + return False, f"unusable_method:{clock_sync.get('clock_sync_method', 'unknown')}" + + if clock_sync.get("best_offset_s") is None: + return False, "missing_best_offset" + + reads = clock_sync.get("reads") or [] + slow_invalid_reads = [ + read + for read in reads + if isinstance(read, dict) + and read.get("radar_clock_s") is None + and (read.get("read_latency_ms") or 0.0) >= cls.CLOCK_SYNC_MAX_TIMEOUT_READ_MS + ] + if slow_invalid_reads: + return False, f"timeout_reads:{len(slow_invalid_reads)}" + + method = clock_sync.get("clock_sync_method") + if method == "integer_rollover": + uncertainty = clock_sync.get("rollover_uncertainty_ms") + if uncertainty is None: + return False, "missing_rollover_uncertainty" + try: + if float(uncertainty) > cls.CLOCK_SYNC_MAX_ROLLOVER_UNCERTAINTY_MS: + return False, f"rollover_uncertainty:{float(uncertainty):.1f}ms" + except (TypeError, ValueError): + return False, "invalid_rollover_uncertainty" + return True, "valid_integer_rollover" + + if method == "fractional_clock": + return True, "valid_fractional_clock" + + return False, f"unsupported_method:{method or 'unknown'}" + + @classmethod + def _clock_sync_summary_for_log(cls, clock_sync: object) -> Optional[dict]: + """Return a compact JSONL-safe summary of a clock-sync candidate.""" + if not isinstance(clock_sync, dict): + return None + valid, reason = cls._clock_sync_quality(clock_sync) + return { + "valid": valid, + "reason": reason, + "source": clock_sync.get("source"), + "samples": clock_sync.get("samples"), + "valid_samples": clock_sync.get("valid_samples"), + "clock_sync_method": clock_sync.get("clock_sync_method"), + "best_offset_s": clock_sync.get("best_offset_s"), + "raw_best_offset_s": clock_sync.get("raw_best_offset_s"), + "best_read_latency_ms": clock_sync.get("best_read_latency_ms"), + "offset_spread_ms": clock_sync.get("offset_spread_ms"), + "rollover_uncertainty_ms": clock_sync.get("rollover_uncertainty_ms"), + "age_s": ( + round(cls._clock_sync_age_s(clock_sync), 3) + if cls._clock_sync_age_s(clock_sync) is not None + else None + ), + } + + def _select_clock_sync_for_capture( + self, + radar: "OPS243Radar", + capture: IQCapture, + ) -> Optional[dict]: + """Choose and apply the best OPS clock sync for this capture.""" + previous_sync = getattr(radar, "last_clock_sync", None) + previous_valid, previous_reason = self._clock_sync_quality(previous_sync) + previous_age_s = ( + self._clock_sync_age_s(previous_sync) if isinstance(previous_sync, dict) else None + ) + + fresh_sync = None + fresh_error = None + if hasattr(radar, "read_clock_sync"): + try: + fresh_sync = radar.read_clock_sync( + samples=self.CLOCK_SYNC_SAMPLES, + store=False, + ) + if isinstance(fresh_sync, dict): + fresh_sync["source"] = "per_shot" + except Exception as exc: # pylint: disable=broad-except + fresh_error = str(exc) + logger.warning("[TRIGGER] Per-shot OPS clock sync failed: %s", exc, exc_info=True) + + fresh_valid, fresh_reason = self._clock_sync_quality(fresh_sync) + + selected_sync = None + selected_source = "first_byte" + selected_reason = "no_valid_clock_sync" + + if fresh_valid and isinstance(fresh_sync, dict): + selected_sync = fresh_sync + selected_source = "fresh" + selected_reason = fresh_reason + radar.last_clock_sync = fresh_sync + elif ( + previous_valid + and isinstance(previous_sync, dict) + and previous_age_s is not None + and previous_age_s <= self.CLOCK_SYNC_MAX_FALLBACK_AGE_S + ): + selected_sync = previous_sync + selected_source = "previous" + selected_reason = ( + f"fresh_rejected:{fresh_reason};previous_age:{previous_age_s:.1f}s" + ) + elif previous_valid and previous_age_s is not None: + selected_reason = ( + f"fresh_rejected:{fresh_reason};previous_too_old:{previous_age_s:.1f}s" + ) + elif previous_reason != "missing": + selected_reason = f"fresh_rejected:{fresh_reason};previous_rejected:{previous_reason}" + elif fresh_error: + selected_reason = f"fresh_error:{fresh_error}" + else: + selected_reason = f"fresh_rejected:{fresh_reason}" + + selected_offset_s = None + selected_age_s = None + if isinstance(selected_sync, dict): + selected_offset_s = selected_sync.get("best_offset_s") + selected_age_s = self._clock_sync_age_s(selected_sync) + try: + capture.apply_trigger_timestamp_from_clock_sync(float(selected_offset_s)) + except (TypeError, ValueError): + logger.warning( + "[TRIGGER] Ignoring invalid selected OPS clock-sync offset: %r", + selected_offset_s, + ) + selected_sync = None + selected_source = "first_byte" + selected_reason = "selected_offset_invalid" + selected_offset_s = None + + previous_offset_s = ( + previous_sync.get("best_offset_s") if isinstance(previous_sync, dict) else None + ) + fresh_offset_s = fresh_sync.get("best_offset_s") if isinstance(fresh_sync, dict) else None + + selection_log = { + "selection": selected_source, + "selection_reason": selected_reason, + "selected_offset_s": selected_offset_s, + "selected_age_s": round(selected_age_s, 3) if selected_age_s is not None else None, + "fresh": self._clock_sync_summary_for_log(fresh_sync), + "previous": self._clock_sync_summary_for_log(previous_sync), + "fresh_error": fresh_error, + "fresh_delta_from_previous_ms": ( + round((fresh_offset_s - previous_offset_s) * 1000.0, 3) + if fresh_offset_s is not None and previous_offset_s is not None + else None + ), + } + + if selected_sync is not None: + logger.info( + "[TRIGGER] OPS clock sync selected: %s offset=%.6fs age=%sms " + "(fresh=%s, previous=%s, delta=%sms)", + selected_source, + selected_offset_s, + "n/a" if selected_age_s is None else f"{selected_age_s:.1f}", + fresh_reason, + previous_reason, + "n/a" + if selection_log["fresh_delta_from_previous_ms"] is None + else f"{selection_log['fresh_delta_from_previous_ms']:.1f}", + ) + else: + logger.info( + "[TRIGGER] OPS clock sync fallback to first-byte timing: %s", + selected_reason, + ) + + return selection_log + def wait_for_trigger( self, radar: "OPS243Radar", @@ -849,44 +1055,6 @@ def wait_for_trigger( if first_byte_timestamp is not None and capture.first_byte_timestamp is None: capture.first_byte_timestamp = float(first_byte_timestamp) - clock_sync = getattr(radar, "last_clock_sync", None) - clock_sync_usable = ( - bool(clock_sync.get("usable_for_trigger_timestamps")) - if isinstance(clock_sync, dict) - else False - ) - clock_offset_s = ( - clock_sync.get("best_offset_s") - if isinstance(clock_sync, dict) and clock_sync_usable - else None - ) - if clock_offset_s is not None: - try: - capture.apply_trigger_timestamp_from_clock_sync(float(clock_offset_s)) - except (TypeError, ValueError): - logger.warning( - "[TRIGGER] Ignoring invalid OPS clock-sync offset: %r", - clock_offset_s, - ) - elif isinstance(clock_sync, dict) and clock_sync.get("best_offset_s") is not None: - logger.info( - "[TRIGGER] Ignoring unusable OPS clock sync for trigger timestamp (method=%s)", - clock_sync.get("clock_sync_method", "unknown"), - ) - - if capture.first_byte_timestamp is not None and capture.trigger_timestamp is None: - capture.apply_trigger_timestamp_from_first_byte() - - if capture.trigger_timestamp is not None and capture.first_byte_timestamp is not None: - logger.info( - "[TRIGGER] Sound trigger wall time %.3f " - "(source=%s, first byte %.3f, post-trigger %.1fms)", - capture.trigger_timestamp, - capture.trigger_timestamp_source or "unknown", - capture.first_byte_timestamp, - capture.post_trigger_duration_ms, - ) - # Quick validation: does the capture contain any real swing data? # At a driving range, a nearby player's impact sound can trip the # trigger even though nothing was moving in front of our radar. @@ -909,6 +1077,21 @@ def wait_for_trigger( ) return None + self._select_clock_sync_for_capture(radar, capture) + + if capture.first_byte_timestamp is not None and capture.trigger_timestamp is None: + capture.apply_trigger_timestamp_from_first_byte() + + if capture.trigger_timestamp is not None and capture.first_byte_timestamp is not None: + logger.info( + "[TRIGGER] Sound trigger wall time %.3f " + "(source=%s, first byte %.3f, post-trigger %.1fms)", + capture.trigger_timestamp, + capture.trigger_timestamp_source or "unknown", + capture.first_byte_timestamp, + capture.post_trigger_duration_ms, + ) + logger.info( "[TRIGGER] Sound trigger accepted — peak %.1f mph, %d outbound readings", summary["valid_peak_outbound_mph"], diff --git a/src/openflight/server.py b/src/openflight/server.py index aa6beec3e..a532ae98d 100644 --- a/src/openflight/server.py +++ b/src/openflight/server.py @@ -20,10 +20,13 @@ from flask_cors import CORS from flask_socketio import SocketIO -from .launch_monitor import ClubType, Shot +from .ballistics import resolve_launch, simulate +from .launch_monitor import SPIN_CONFIDENCE_HIGH, ClubType, Shot from .ops243 import Direction, SpeedReading, set_show_raw_readings from .rolling_buffer.monitor import estimate_carry_with_spin, get_optimal_spin_for_ball_speed from .session_logger import get_session_logger, init_session_logger, log_session_error +from .speed_correction import correct_ball_speed +from .spin_estimate import calculated_spin_rpm # Configure logging logger = logging.getLogger(__name__) @@ -68,6 +71,12 @@ experimental_kld7_radc_tuning: bool = False experimental_kld7_raw_radc_logging: bool = False +# Ballistic model toggle. When True, shot carry comes from the physics +# simulator whenever a vertical launch angle is available. When False +# (default), all carry computations go through the legacy table estimator. +# The simulator is opt-in until coefficients are validated against TM. +ballistics_enabled: bool = False + _DEFAULT_KLD7_RADC_TUNING = { "radc_speed_tolerance_mph": 10.0, "radc_centroid_floor_frac": 0.5, @@ -754,10 +763,19 @@ def _session_start_config() -> dict: return config +ball_speed_correction_enabled = False +ball_speed_correction_distance_ft = 5.5 +ball_speed_correction_ball_above_radar_ft = -4.0 / 12.0 +calculated_spin_enabled = False + + def shot_to_dict(shot: Shot) -> dict: """Convert Shot to JSON-serializable dict.""" return { "ball_speed_mph": round(shot.ball_speed_mph, 1), + "ball_speed_raw_mph": ( + round(shot.ball_speed_raw_mph, 1) if shot.ball_speed_raw_mph else None + ), "club_speed_mph": round(shot.club_speed_mph, 1) if shot.club_speed_mph else None, "smash_factor": round(shot.smash_factor, 2) if shot.smash_factor else None, "estimated_carry_yards": round(shot.estimated_carry_yards), @@ -782,6 +800,10 @@ def shot_to_dict(shot: Shot) -> dict: "spin_axis_deg": shot.spin_axis_deg, # Spin data from rolling buffer mode "spin_rpm": round(shot.spin_rpm) if shot.spin_rpm else None, + "spin_rpm_measured": ( + round(shot.spin_rpm_measured) if shot.spin_rpm_measured else None + ), + "spin_source": shot.spin_source, "spin_confidence": round(shot.spin_confidence, 2) if shot.spin_confidence else None, "spin_quality": shot.spin_quality, "spin_snr": round(shot.spin_snr, 2) if shot.spin_snr is not None else None, @@ -1478,6 +1500,38 @@ def handle_shutdown(): threading.Thread(target=_shutdown_process_after_delay, daemon=True).start() +def _apply_calculated_spin(shot: Shot) -> bool: + """Replace radar-measured spin with the kinematic estimate. + + The 24 GHz OPS return carries no usable spin line (see + spin_estimate.py), so when the vertical launch angle was actually + measured (radar/camera, not the club-table estimate), spin_rpm is + rewritten with 170*v*sin(LA)^1.2. The displaced measured value is + kept in spin_rpm_measured for offline scoring. Returns True when + the shot was rewritten. + """ + if shot.launch_angle_vertical is None: + return False + if shot.launch_angle_vertical_source not in ("radar", "camera"): + return False + spin_calc = calculated_spin_rpm(shot.ball_speed_mph, shot.launch_angle_vertical) + if spin_calc is None: + return False + shot.spin_rpm_measured = shot.spin_rpm + shot.spin_rpm = spin_calc + shot.spin_confidence = SPIN_CONFIDENCE_HIGH + shot.spin_source = "calculated" + shot.spin_rejection_reason = None + logger.info( + "[SERVER] Calculated spin: %.0f rpm (v=%.1f mph, LA=%.1f deg, measured was %s)", + spin_calc, + shot.ball_speed_mph, + shot.launch_angle_vertical, + "%.0f rpm" % shot.spin_rpm_measured if shot.spin_rpm_measured else "none", + ) + return True + + def on_shot_detected(shot: Shot): """Callback when a shot is detected - emit to all clients.""" global ball_detected, ball_detection_confidence # pylint: disable=global-statement @@ -1770,32 +1824,72 @@ def on_shot_detected(shot: Shot): # rejected or missing axes fall back to conservative estimates. _ensure_user_facing_launch_angles(shot) - # Compute spin-adjusted carry using measured spin (if reliable) or club average - _MIN_RELIABLE_SPIN_CONF = 0.6 - if shot.carry_spin_adjusted is None and shot.mode != "mock": - has_reliable_spin = ( - shot.spin_rpm - and shot.spin_rpm > 0 - and shot.spin_confidence is not None - and shot.spin_confidence >= _MIN_RELIABLE_SPIN_CONF - ) - spin_for_carry = ( - shot.spin_rpm - if has_reliable_spin - else get_optimal_spin_for_ball_speed(shot.ball_speed_mph, shot.club) - ) - shot.carry_spin_adjusted = estimate_carry_with_spin( - shot.ball_speed_mph, - spin_for_carry, - shot.club, - club_speed_mph=shot.club_speed_mph, + # Ball-speed cosine correction: the OPS reads the radial component of + # a ball departing at the launch angle. Applied AFTER the K-LD7 (which + # must anchor to the radial speed) and BEFORE carry/ballistics. + if ball_speed_correction_enabled and shot.launch_angle_vertical is not None: + raw_speed = shot.ball_speed_mph + shot.ball_speed_raw_mph = raw_speed + shot.ball_speed_mph = correct_ball_speed( + raw_speed, + shot.launch_angle_vertical, + ball_speed_correction_distance_ft, + ball_speed_correction_ball_above_radar_ft, ) logger.info( - "[SERVER] Spin-adjusted carry: %.0f yds (spin: %.0f rpm%s)", - shot.carry_spin_adjusted, - spin_for_carry, - "" if shot.spin_rpm and shot.spin_rpm > 0 else " avg", + "[SERVER] Ball speed cosine correction: %.1f -> %.1f mph (LA %.1f)", + raw_speed, + shot.ball_speed_mph, + shot.launch_angle_vertical, ) + + # Calculated spin runs AFTER the cosine correction (the model is + # calibrated on true ball speed) and BEFORE carry/ballistics. + if calculated_spin_enabled: + _apply_calculated_spin(shot) + + # Compute carry. Prefer the physics simulator (drag + Magnus, RK4) when + # ballistics is enabled and a vertical launch angle is available; fall + # back to the table estimator otherwise (either ballistics disabled or + # angle missing → resolve_launch returns None). + _MIN_RELIABLE_SPIN_CONF = 0.6 + if shot.carry_spin_adjusted is None and shot.mode != "mock": + conditions = resolve_launch(shot) if ballistics_enabled else None + if conditions is not None: + trajectory = simulate(conditions) + shot.carry_spin_adjusted = trajectory.carry_yards + logger.info( + "[SERVER] Ballistic carry: %.0f yds (spin: %.0f rpm, source: %s)", + shot.carry_spin_adjusted, + conditions.spin_rpm, + conditions.spin_source, + ) + else: + has_reliable_spin = ( + shot.spin_rpm + and shot.spin_rpm > 0 + and shot.spin_confidence is not None + and shot.spin_confidence >= _MIN_RELIABLE_SPIN_CONF + ) + spin_for_carry = ( + shot.spin_rpm + if has_reliable_spin + else get_optimal_spin_for_ball_speed(shot.ball_speed_mph, shot.club) + ) + shot.carry_spin_adjusted = estimate_carry_with_spin( + shot.ball_speed_mph, + spin_for_carry, + shot.club, + club_speed_mph=shot.club_speed_mph, + ) + reason = "ballistics disabled" if not ballistics_enabled else "no launch angle" + logger.info( + "[SERVER] Table carry (%s): %.0f yds (spin: %.0f rpm%s)", + reason, + shot.carry_spin_adjusted, + spin_for_carry, + "" if shot.spin_rpm and shot.spin_rpm > 0 else " avg", + ) if shot.spin_rejection_reason: logger.info( "[SERVER] Spin unavailable: %s (snr=%s, candidate=%s rpm)", @@ -2014,6 +2108,26 @@ def on_trigger_diagnostic(data: dict): monitor.start(shot_callback=on_shot_detected, live_callback=on_live_reading) +def _fire_cloud_push(session_logger): + """Best-effort, non-blocking cloud push on session end. + + Fully guarded: the uploader is opt-in and must never delay or break the + shot/session path. The systemd timer is the safety net if this no-ops. + """ + try: + from .cloud.config import load_config + from .cloud.trigger import fire_push_async + + config = load_config() + if config is None or not config.is_active(): + return + log_dir = getattr(session_logger, "log_dir", None) + if log_dir is not None: + fire_push_async(config, log_dir=log_dir) + except Exception: # pylint: disable=broad-exception-caught + pass + + def stop_monitor(): """Stop the launch monitor.""" global monitor # pylint: disable=global-statement @@ -2022,6 +2136,7 @@ def stop_monitor(): session_logger = get_session_logger() if session_logger: session_logger.end_session() + _fire_cloud_push(session_logger) if monitor: monitor.stop() @@ -2299,6 +2414,16 @@ def main(): "--log-dir", help="Directory for session logs (default: ~/openflight_sessions)" ) parser.add_argument("--no-logging", action="store_true", help="Disable session logging") + parser.add_argument( + "--ballistics", + action="store_true", + help=( + "Enable the physics-based carry simulator (drag + Magnus, RK4). " + "When set, shots with a vertical launch angle use the simulator " + "for carry; otherwise they fall back to the legacy table estimator. " + "Default: disabled (all shots use the table)." + ), + ) parser.add_argument( "--trigger", choices=["polling", "threshold", "speed", "sound"], @@ -2338,6 +2463,26 @@ def main(): default=0.0, help="K-LD7 vertical angle offset in degrees (default: 0.0)", ) + parser.add_argument( + "--ball-speed-cosine-correction", + action="store_true", + help=( + "Correct OPS radial ball speed to true ball speed using the launch " + "angle and radar geometry (validated vs TrackMan; see " + "src/openflight/speed_correction.py)" + ), + ) + parser.add_argument( + "--calculated-spin", + action="store_true", + help=( + "Replace radar-measured spin with the kinematic estimate " + "(170*v*sin(LA)^1.2) when the launch angle was measured. The 24 GHz " + "OPS return carries no usable spin line (see " + "src/openflight/spin_estimate.py); the measured value is kept in " + "spin_rpm_measured for offline scoring" + ), + ) parser.add_argument( "--kld7-vertical-estimator", choices=("geometry", "naive"), @@ -2362,6 +2507,16 @@ def main(): "(weak lever; default: 5.5)" ), ) + parser.add_argument( + "--kld7-radar-height-inches", + dest="kld7_radar_height_inches", + type=float, + default=4.0, + help=( + "K-LD7 radar height above the ball in inches, used by the ball-speed " + "cosine correction geometry (default: 4.0)" + ), + ) parser.add_argument( "--kld7-horizontal", action="store_true", @@ -2455,8 +2610,18 @@ def main(): global experimental_kld7_radc_tuning global experimental_kld7_raw_radc_logging global active_kld7_radc_tuning + global ballistics_enabled experimental_kld7_raw_radc_logging = args.experimental_kld7_raw_radc_logging experimental_kld7_radc_tuning = args.experimental_kld7_radc_tuning + global ball_speed_correction_enabled + global ball_speed_correction_distance_ft + global ball_speed_correction_ball_above_radar_ft + ball_speed_correction_enabled = args.ball_speed_cosine_correction + ball_speed_correction_distance_ft = args.kld7_ball_distance + ball_speed_correction_ball_above_radar_ft = -args.kld7_radar_height_inches / 12.0 + global calculated_spin_enabled + calculated_spin_enabled = args.calculated_spin + ballistics_enabled = args.ballistics kld7_radc_tuning_kwargs = _kld7_radc_tuning_kwargs(args) active_kld7_radc_tuning = dict(kld7_radc_tuning_kwargs) @@ -2486,6 +2651,11 @@ def main(): init_session_logger(enabled=False) print("Session logging DISABLED") + if ballistics_enabled: + print("Ballistic carry model: ENABLED (simulator + drag/Magnus)") + else: + print("Ballistic carry model: DISABLED (table fallback for all shots)") + # Configure radar logging if requested if args.radar_log: logging.basicConfig( diff --git a/src/openflight/session_logger.py b/src/openflight/session_logger.py index 585642064..f8f0ca855 100644 --- a/src/openflight/session_logger.py +++ b/src/openflight/session_logger.py @@ -7,14 +7,21 @@ import json import logging +import uuid from dataclasses import asdict, dataclass from datetime import datetime from pathlib import Path from typing import Any, Dict, List, Optional +from . import __version__ from .kld7.radc import RADC_PAYLOAD_BYTES from .ops243 import SpeedReading +# Version of the session JSONL format itself. Bump on breaking changes to +# entry structure; additive changes (new fields, new entry types) do not +# require a bump. Consumed by offline analysis and (eventually) cloud sync. +SESSION_FORMAT_VERSION = 1 + @dataclass class SessionMetadata: @@ -29,6 +36,13 @@ class SessionMetadata: config: Dict[str, Any] mode: str # "rolling-buffer" or "mock" trigger_type: Optional[str] # For rolling-buffer mode: "polling", "threshold", etc. + # Globally unique session identity for cloud sync dedupe. The + # timestamp-based session_id stays for filenames and display; this + # UUID travels inside the data so renamed/copied session files keep + # their identity (see docs/cloud-sync-design.md). + session_uuid: str = "" + format_version: int = 1 + app_version: str = "" class SessionLogger: @@ -146,6 +160,9 @@ def start_session( config=config or {}, mode=mode, trigger_type=trigger_type, + session_uuid=str(uuid.uuid4()), + format_version=SESSION_FORMAT_VERSION, + app_version=__version__, ) self._write_entry("session_start", asdict(metadata)) diff --git a/src/openflight/speed_correction.py b/src/openflight/speed_correction.py new file mode 100644 index 000000000..73f399c10 --- /dev/null +++ b/src/openflight/speed_correction.py @@ -0,0 +1,76 @@ +"""Ball-speed cosine correction. + +The OPS243 measures RADIAL speed — the component of the ball's velocity +along the radar's line of sight. The ball departs upward at the launch +angle while the radar sits low behind the tee, so the radial reading is +compressed by cos(angle between velocity and LOS). This is the dominant +cause of the long-observed ~2-2.5 mph OPS-below-TrackMan ball speed gap. + +Model (zero free parameters): the OPS mode-based extraction reads near +the MAXIMUM of the radial-speed profile — radial speed rises early in +flight as the LOS aligns with the velocity vector, then falls with drag, +so the profile has a peak. The correction divides the measured speed by +that predicted peak fraction. + +Validated offline against TrackMan ball speeds (2026-06): +- 2026-06-08 bay, 128 shots: bias -2.15 -> +0.33 mph, |err| 2.83 -> 1.48 +- 2026-05-30 holdout, 26 shots: bias -2.26 -> +0.30, |err| 2.35 -> 1.31 +- Coleman cross-rig outdoor, 62 shots: bias -2.09 -> +0.65, median 0.58 +- Production configuration (OUR launch angles — 100 two-ray + 28 + club-fallback — instead of TrackMan's): bias +0.32, |err| 1.52, + median 0.67. The LA dependence couples speed accuracy to launch-angle + accuracy, but LA errors are zero-mean post-calibration, so the + coupling adds ~0.1 mph of scatter and no bias. + +Known caveat: high-launch wedges on one outdoor rig overcorrected (~+3); +under observation. Club speed does NOT need this correction — the club +head's delivery is nearly parallel to the LOS (error ~0.1-0.2 mph). +""" + +from __future__ import annotations + +import math + +MPH_TO_FTS = 1.4666667 +DRAG_MPH_PER_MS = 0.027 # iron-speed drag deceleration of the ball + + +def radial_speed_factor( + launch_angle_deg: float, + ball_speed_mph: float, + ball_distance_ft: float, + ball_above_radar_ft: float, + window_ms: float = 70.0, +) -> float: + """Predicted (OPS radial reading) / (true ball speed), in (0, 1]. + + Maximum of the radial-speed profile over the capture window: + radial(t) = v(t) * cos(launch - elevation_of_ball_from_radar(t)). + """ + if ball_speed_mph <= 0: + return 1.0 + la = math.radians(launch_angle_deg) + v_fts = ball_speed_mph * MPH_TO_FTS + best = 0.0 + t_ms = 0.0 + while t_ms <= window_ms: + v_frac = max(1.0 - DRAG_MPH_PER_MS * t_ms / ball_speed_mph, 0.0) + t = t_ms / 1000.0 + x = ball_distance_ft + v_fts * math.cos(la) * t + y = ball_above_radar_ft + v_fts * math.sin(la) * t + best = max(best, v_frac * math.cos(la - math.atan2(y, x))) + t_ms += 2.0 + return min(max(best, 0.5), 1.0) + + +def correct_ball_speed( + measured_mph: float, + launch_angle_deg: float, + ball_distance_ft: float, + ball_above_radar_ft: float, +) -> float: + """True ball speed from the OPS radial measurement.""" + factor = radial_speed_factor( + launch_angle_deg, measured_mph, ball_distance_ft, ball_above_radar_ft + ) + return measured_mph / factor diff --git a/src/openflight/spin_estimate.py b/src/openflight/spin_estimate.py new file mode 100644 index 000000000..bdb4819b1 --- /dev/null +++ b/src/openflight/spin_estimate.py @@ -0,0 +1,71 @@ +"""Calculated spin from measured ball speed and launch angle. + +The OPS243 return carries no usable spin line: golf-ball dimples +(~0.3 mm) are Rayleigh-smooth at 24 GHz (λ = 12.4 mm) and the specular +point does not rotate with the ball, so rotation barely modulates the +echo within our ~65 ms ball dwell. Offline analysis of the 2026-06-08 +TrackMan session (131 paired shots, PW-3h) found no spectral line at +the TrackMan spin frequency on any club after dechirp and carrier +removal — and the production envelope estimator's output had ~zero +within-club correlation with TrackMan spin (r ≈ +0.19). + +What does work is impact kinematics. Spin loft tracks launch angle and +the friction impulse gives spin ∝ v·sin(spin loft), yielding a single +global formula with no club input: + + spin_rpm = 170 · ball_speed_mph · sin(LA)^1.2 + +Validation against the 2026-06-08 TrackMan truth set: +- TrackMan inputs (physics ceiling): 10.5% median error across the + bag — better than an oracle per-club median table (12.0%) +- Leave-one-club-out blind: 11.3% median, 97/131 within 25%; the + fitted (coefficient, exponent) were stable across all folds +- Simulated with production launch-angle noise (2.5° MAE two-ray): + ~21% median expected live + +Caveats: calibrated on one player/session (range balls). Attack-angle +style shifts spin loft at a fixed launch angle, so cross-player error +is likely a few points worse. Accuracy degrades at low launch angles +(cot(LA) error amplification) — the same clubs where measured launch +angle is weakest. +""" + +import math +from typing import Optional + +# Fitted on the 2026-06-08 TrackMan session (131 shots, PW-3h); +# leave-one-club-out stable at (170, 1.2) across all folds. +SPIN_COEFF_RPM_PER_MPH = 170.0 +SPIN_LA_EXPONENT = 1.2 + +# Outside these bounds the kinematic model is extrapolating into +# regimes it was never calibrated on (top-spinned thins, pop-ups). +MIN_LAUNCH_ANGLE_DEG = 2.0 +MAX_LAUNCH_ANGLE_DEG = 60.0 + +# Physical ceiling — beyond fresh-groove lob wedge territory. +MAX_SPIN_RPM = 13000.0 + + +def calculated_spin_rpm( + ball_speed_mph: float, + launch_angle_deg: float, +) -> Optional[float]: + """Kinematic spin estimate from ball speed and vertical launch angle. + + Use the true (cosine-corrected) ball speed when available — the + model was calibrated against TrackMan ball speed. + + Returns None when inputs are missing or outside the calibrated + range; callers should fall back to club-typical spin. + """ + if ball_speed_mph is None or launch_angle_deg is None: + return None + if ball_speed_mph <= 0: + return None + if not MIN_LAUNCH_ANGLE_DEG <= launch_angle_deg <= MAX_LAUNCH_ANGLE_DEG: + return None + + sin_la = math.sin(math.radians(launch_angle_deg)) + spin = SPIN_COEFF_RPM_PER_MPH * ball_speed_mph * sin_la**SPIN_LA_EXPONENT + return float(min(spin, MAX_SPIN_RPM)) diff --git a/tests/test_ballistics.py b/tests/test_ballistics.py new file mode 100644 index 000000000..4b4ff085c --- /dev/null +++ b/tests/test_ballistics.py @@ -0,0 +1,170 @@ +"""Tests for the ballistics flight simulator and launch resolution.""" + +from datetime import datetime + +import pytest + +from openflight.ballistics import ( + CLUB_TYPICAL_SPIN_RPM, + LaunchConditions, + resolve_launch, + simulate, +) +from openflight.launch_monitor import ClubType, Shot + + +def _shot(**kwargs) -> Shot: + defaults = dict( + ball_speed_mph=160.0, + timestamp=datetime.now(), + club=ClubType.DRIVER, + launch_angle_vertical=12.0, + ) + defaults.update(kwargs) + return Shot(**defaults) + + +class TestResolveLaunch: + def test_returns_none_without_vertical_launch_angle(self): + shot = _shot(launch_angle_vertical=None) + assert resolve_launch(shot) is None + + def test_uses_measured_spin_when_high_confidence(self): + shot = _shot(spin_rpm=2500, spin_confidence=0.85) + cond = resolve_launch(shot) + assert cond is not None + assert cond.spin_rpm == 2500 + assert cond.spin_source == "measured" + + def test_uses_club_typical_when_low_confidence(self): + shot = _shot(spin_rpm=1500, spin_confidence=0.3, club=ClubType.DRIVER) + cond = resolve_launch(shot) + assert cond is not None + assert cond.spin_rpm == CLUB_TYPICAL_SPIN_RPM[ClubType.DRIVER] + assert cond.spin_source == "club_typical" + + def test_uses_club_typical_when_spin_missing(self): + shot = _shot(spin_rpm=None, club=ClubType.IRON_7) + cond = resolve_launch(shot) + assert cond is not None + assert cond.spin_rpm == CLUB_TYPICAL_SPIN_RPM[ClubType.IRON_7] + assert cond.spin_source == "club_typical" + + def test_medium_confidence_still_falls_back(self): + # Medium confidence (~0.5) is below the high threshold — use typical. + shot = _shot(spin_rpm=3000, spin_confidence=0.5) + cond = resolve_launch(shot) + assert cond is not None + assert cond.spin_source == "club_typical" + + def test_defaults_horizontal_angle_to_zero(self): + shot = _shot(launch_angle_horizontal=None) + cond = resolve_launch(shot) + assert cond.launch_angle_h == 0.0 + + def test_defaults_spin_axis_to_zero(self): + shot = _shot(spin_axis_deg=None) + cond = resolve_launch(shot) + assert cond.spin_axis_deg == 0.0 + + +def _driver(spin_rpm=2700, launch=11.0, ball_speed=165.0, axis=0.0, la_h=0.0): + return LaunchConditions( + ball_speed_mph=ball_speed, + launch_angle_v=launch, + launch_angle_h=la_h, + spin_rpm=spin_rpm, + spin_axis_deg=axis, + spin_source="measured", + ) + + +class TestSimulate: + def test_driver_carry_in_expected_range(self): + # 165 mph ball speed / 11° / 2700 RPM is close to PGA Tour averages. + # TrackMan data: ~270–285 yards carry. + traj = simulate(_driver()) + assert 250 <= traj.carry_yards <= 300, ( + f"Driver carry {traj.carry_yards:.1f} yd outside plausible range" + ) + + def test_iron_carry_in_expected_range(self): + # 7-iron: 120 mph ball speed, 17° launch, 6500 RPM → ~160-180 yd + cond = LaunchConditions( + ball_speed_mph=120.0, + launch_angle_v=17.0, + launch_angle_h=0.0, + spin_rpm=6500, + spin_axis_deg=0.0, + spin_source="measured", + ) + traj = simulate(cond) + assert 140 <= traj.carry_yards <= 200, ( + f"7-iron carry {traj.carry_yards:.1f} yd outside plausible range" + ) + + def test_higher_launch_produces_higher_apex(self): + low = simulate(_driver(launch=8.0)) + high = simulate(_driver(launch=15.0)) + assert high.apex_yards > low.apex_yards + + def test_more_spin_produces_higher_apex(self): + low_spin = simulate(_driver(spin_rpm=1800)) + high_spin = simulate(_driver(spin_rpm=3500)) + assert high_spin.apex_yards > low_spin.apex_yards + + def test_fade_lands_right_of_target(self): + traj = simulate(_driver(axis=10.0)) # +axis = fade + assert traj.lateral_yards > 3.0 + + def test_draw_lands_left_of_target(self): + traj = simulate(_driver(axis=-10.0)) # -axis = draw + assert traj.lateral_yards < -3.0 + + def test_straight_shot_stays_near_center(self): + traj = simulate(_driver(axis=0.0, la_h=0.0)) + assert abs(traj.lateral_yards) < 1.0 + + def test_horizontal_launch_offsets_landing(self): + # +la_h should push ball right + traj = simulate(_driver(la_h=2.0)) + assert traj.lateral_yards > 1.0 + + def test_trajectory_ends_at_ground(self): + traj = simulate(_driver()) + assert traj.points[-1].z <= 0.01 + assert traj.points[-1].t == pytest.approx(traj.flight_time_s, rel=0.01) + + def test_spin_decays_over_flight(self): + traj = simulate(_driver(spin_rpm=3000)) + final_spin = traj.points[-1].spin_rpm + # 4%/s for ~6s flight → ~80% of initial + assert 2300 < final_spin < 2900 + + def test_flight_time_reasonable(self): + traj = simulate(_driver()) + # Drivers typically spend 5-8 seconds in the air. + assert 4.0 < traj.flight_time_s < 9.0 + + def test_landing_angle_is_positive_descent(self): + traj = simulate(_driver()) + # Ball descends on landing — angle below horizontal is positive. + assert 20.0 < traj.landing_angle_deg < 60.0 + + def test_zero_launch_angle_does_not_crash(self): + # Extreme input should still produce a terminated trajectory. + cond = LaunchConditions( + ball_speed_mph=100.0, + launch_angle_v=0.5, + launch_angle_h=0.0, + spin_rpm=3000, + spin_axis_deg=0.0, + spin_source="measured", + ) + traj = simulate(cond) + assert traj.carry_yards > 0 + assert traj.flight_time_s < 5.0 + + def test_total_distance_includes_rollout(self): + traj = simulate(_driver()) + assert traj.total_yards > traj.carry_yards diff --git a/tests/test_cloud_cli.py b/tests/test_cloud_cli.py new file mode 100644 index 000000000..1786edfd1 --- /dev/null +++ b/tests/test_cloud_cli.py @@ -0,0 +1,103 @@ +"""Tests for the openflight-cloud CLI argument wiring.""" + +import pytest + +from openflight.cloud import cli + + +class TestArgParsing: + def test_requires_subcommand(self, capsys): + rc = cli.main([]) + assert rc != 0 + + def test_unknown_subcommand_errors(self): + with pytest.raises(SystemExit): + cli.main(["frobnicate"]) + + +class TestDispatch: + def test_status_dispatches(self, monkeypatch, tmp_path): + called = {} + + def fake_status(config, log_dir, client=None, out=print): + called["log_dir"] = log_dir + out("status-ran") + return {} + + monkeypatch.setattr(cli.commands, "cmd_status", fake_status) + rc = cli.main(["status", "--log-dir", str(tmp_path), "--config", str(tmp_path / "c.json")]) + assert rc == 0 + assert called["log_dir"] == tmp_path + + def test_push_passes_dry_run_flag(self, monkeypatch, tmp_path): + captured = {} + + def fake_push(config, log_dir, client, dry_run=False, retry=False, session=None, out=print): + captured["dry_run"] = dry_run + return {"needs_relink": False} + + monkeypatch.setattr(cli.commands, "cmd_push", fake_push) + cli.main( + ["push", "--dry-run", "--log-dir", str(tmp_path), "--config", str(tmp_path / "c.json")] + ) + assert captured["dry_run"] is True + + def test_push_retry_all(self, monkeypatch, tmp_path): + captured = {} + + def fake_push(config, log_dir, client, dry_run=False, retry=False, session=None, out=print): + captured.update(retry=retry, session=session) + return {"needs_relink": False} + + monkeypatch.setattr(cli.commands, "cmd_push", fake_push) + cli.main( + ["push", "--retry", "--log-dir", str(tmp_path), "--config", str(tmp_path / "c.json")] + ) + assert captured == {"retry": True, "session": None} + + def test_push_retry_named_session(self, monkeypatch, tmp_path): + captured = {} + + def fake_push(config, log_dir, client, dry_run=False, retry=False, session=None, out=print): + captured.update(retry=retry, session=session) + return {"needs_relink": False} + + monkeypatch.setattr(cli.commands, "cmd_push", fake_push) + cli.main( + [ + "push", + "--retry", + "session_20260527", + "--log-dir", + str(tmp_path), + "--config", + str(tmp_path / "c.json"), + ] + ) + assert captured == {"retry": True, "session": "session_20260527"} + + def test_push_no_retry_by_default(self, monkeypatch, tmp_path): + captured = {} + + def fake_push(config, log_dir, client, dry_run=False, retry=False, session=None, out=print): + captured.update(retry=retry, session=session) + return {"needs_relink": False} + + monkeypatch.setattr(cli.commands, "cmd_push", fake_push) + cli.main(["push", "--log-dir", str(tmp_path), "--config", str(tmp_path / "c.json")]) + assert captured == {"retry": False, "session": None} + + def test_push_returns_nonzero_when_relink_needed(self, monkeypatch, tmp_path): + monkeypatch.setattr(cli.commands, "cmd_push", lambda *a, **k: {"needs_relink": True}) + rc = cli.main(["push", "--log-dir", str(tmp_path), "--config", str(tmp_path / "c.json")]) + assert rc != 0 + + def test_link_dispatches(self, monkeypatch, tmp_path): + monkeypatch.setattr(cli.commands, "cmd_link", lambda *a, **k: True) + rc = cli.main(["link", "--config", str(tmp_path / "c.json")]) + assert rc == 0 + + def test_link_returns_nonzero_on_failure(self, monkeypatch, tmp_path): + monkeypatch.setattr(cli.commands, "cmd_link", lambda *a, **k: False) + rc = cli.main(["link", "--config", str(tmp_path / "c.json")]) + assert rc != 0 diff --git a/tests/test_cloud_client.py b/tests/test_cloud_client.py new file mode 100644 index 000000000..03ece5dde --- /dev/null +++ b/tests/test_cloud_client.py @@ -0,0 +1,174 @@ +"""Tests for the openflight-cloud HTTP client (wire contract).""" + +import json + +import pytest + +from openflight.cloud import client as cl + + +class FakeTransport: + """Records requests and returns queued responses.""" + + def __init__(self, responses): + self._responses = list(responses) + self.calls = [] + + def __call__(self, method, url, data=None, headers=None, timeout=30): + self.calls.append({"method": method, "url": url, "data": data, "headers": headers or {}}) + return self._responses.pop(0) + + +def _resp(status, body=None, headers=None): + raw = json.dumps(body).encode() if body is not None else b"" + return cl.HttpResponse(status=status, headers=headers or {}, body=raw) + + +class TestHealth: + def test_ok_returns_true(self): + client = cl.CloudClient( + "https://e.test", request_fn=FakeTransport([_resp(200, {"status": "ok"})]) + ) + assert client.health() is True + + def test_non_ok_returns_false(self): + client = cl.CloudClient("https://e.test", request_fn=FakeTransport([_resp(503)])) + assert client.health() is False + + def test_network_error_returns_false(self): + def boom(*a, **k): + raise cl.CloudNetworkError("offline") + + client = cl.CloudClient("https://e.test", request_fn=boom) + assert client.health() is False + + def test_uses_v1_health_path(self): + t = FakeTransport([_resp(200, {"status": "ok"})]) + cl.CloudClient("https://e.test/", request_fn=t).health() + assert t.calls[0]["url"] == "https://e.test/v1/health" + assert t.calls[0]["method"] == "GET" + + +class TestDeviceLinkStart: + def test_parses_link_start_response(self): + body = { + "link_code": "ABCD-2345", + "poll_token": "secret-token", + "interval_s": 5, + "expires_s": 900, + } + t = FakeTransport([_resp(200, body)]) + client = cl.CloudClient("https://e.test", request_fn=t) + result = client.device_link_start("garage pi", "0.2.0") + assert result.link_code == "ABCD-2345" + assert result.poll_token == "secret-token" + assert result.interval_s == 5 + assert result.expires_s == 900 + + def test_sends_device_name_and_version(self): + t = FakeTransport( + [_resp(200, {"link_code": "A", "poll_token": "p", "interval_s": 5, "expires_s": 900})] + ) + cl.CloudClient("https://e.test", request_fn=t).device_link_start("garage pi", "9.9.9") + sent = json.loads(t.calls[0]["data"].decode()) + assert sent == {"device_name": "garage pi", "client_version": "9.9.9"} + assert t.calls[0]["url"] == "https://e.test/v1/device-link/start" + + def test_422_raises_link_error(self): + t = FakeTransport([_resp(422, {"reason": "invalid_device_name"})]) + with pytest.raises(cl.LinkError): + cl.CloudClient("https://e.test", request_fn=t).device_link_start("", "0.2.0") + + def test_429_raises_rate_limited_with_retry_after(self): + t = FakeTransport([_resp(429, {"reason": "rate_limited"}, headers={"Retry-After": "42"})]) + with pytest.raises(cl.RateLimited) as exc: + cl.CloudClient("https://e.test", request_fn=t).device_link_start("pi", "0.2.0") + assert exc.value.retry_after == 42 + + +class TestDeviceLinkPoll: + def test_pending(self): + t = FakeTransport([_resp(200, {"status": "pending"})]) + result = cl.CloudClient("https://e.test", request_fn=t).device_link_poll("tok") + assert result.status == "pending" + + def test_linked_returns_token_and_id(self): + body = {"status": "linked", "device_token": "of_device_x", "device_id": "uuid-1"} + t = FakeTransport([_resp(200, body)]) + result = cl.CloudClient("https://e.test", request_fn=t).device_link_poll("tok") + assert result.status == "linked" + assert result.device_token == "of_device_x" + assert result.device_id == "uuid-1" + + def test_404_maps_to_unknown(self): + t = FakeTransport([_resp(404, {"reason": "unknown_poll_token"})]) + result = cl.CloudClient("https://e.test", request_fn=t).device_link_poll("tok") + assert result.status == "unknown" + + def test_429_raises_rate_limited(self): + t = FakeTransport([_resp(429, {}, headers={"Retry-After": "5"})]) + with pytest.raises(cl.RateLimited): + cl.CloudClient("https://e.test", request_fn=t).device_link_poll("tok") + + +class TestUploadSession: + def _client(self, responses): + return cl.CloudClient( + "https://e.test", token="of_device_tok", request_fn=FakeTransport(responses) + ) + + def test_201_is_success(self): + r = self._client([_resp(201, {"session_id": "s1", "shot_count": 7})]).upload_session( + "s1", b"gz" + ) + assert r.action == "success" + assert r.shot_count == 7 + + def test_200_is_success(self): + r = self._client([_resp(200, {"session_id": "s1", "shot_count": 7})]).upload_session( + "s1", b"gz" + ) + assert r.action == "success" + + def test_401_needs_relink(self): + r = self._client([_resp(401, {"reason": "invalid_or_revoked_token"})]).upload_session( + "s1", b"gz" + ) + assert r.action == "relink" + + def test_402_quota(self): + r = self._client([_resp(402, {"reason": "quota_exceeded"})]).upload_session("s1", b"gz") + assert r.action == "quota" + + def test_413_parks(self): + r = self._client([_resp(413, {"reason": "body_too_large"})]).upload_session("s1", b"gz") + assert r.action == "park" + + def test_422_parks(self): + r = self._client([_resp(422, {"reason": "invalid_gzip"})]).upload_session("s1", b"gz") + assert r.action == "park" + assert r.reason == "invalid_gzip" + + def test_429_retry_with_retry_after(self): + r = self._client( + [_resp(429, {"reason": "rate_limited"}, headers={"Retry-After": "30"})] + ).upload_session("s1", b"gz") + assert r.action == "rate_limited" + assert r.retry_after == 30 + + def test_5xx_retry(self): + r = self._client([_resp(503)]).upload_session("s1", b"gz") + assert r.action == "retry" + + def test_sends_bearer_auth_and_gzip_headers(self): + t = FakeTransport([_resp(201, {"session_id": "s1", "shot_count": 1})]) + cl.CloudClient("https://e.test", token="of_device_tok", request_fn=t).upload_session( + "1f0e9c2a-7b3d-4e5f-8a9b-0c1d2e3f4a5b", b"gzbytes" + ) + call = t.calls[0] + assert call["method"] == "PUT" + assert call["url"] == "https://e.test/v1/sessions/1f0e9c2a-7b3d-4e5f-8a9b-0c1d2e3f4a5b" + assert call["headers"]["Authorization"] == "Bearer of_device_tok" + assert call["headers"]["Content-Type"] == "application/x-ndjson" + assert call["headers"]["Content-Encoding"] == "gzip" + assert call["data"] == b"gzbytes" diff --git a/tests/test_cloud_commands.py b/tests/test_cloud_commands.py new file mode 100644 index 000000000..e4beb4356 --- /dev/null +++ b/tests/test_cloud_commands.py @@ -0,0 +1,298 @@ +"""Tests for openflight-cloud command orchestration (link/push/status).""" + +import json + +import pytest + +from openflight.cloud import commands, spool +from openflight.cloud.client import LinkPoll, LinkStart, UploadResult +from openflight.cloud.config import CloudConfig + + +class FakeClient: + def __init__(self, *, healthy=True, link_start=None, polls=None, uploads=None): + self._healthy = healthy + self._link_start = link_start + self._polls = list(polls or []) + self._uploads = list(uploads or []) + self.uploaded = [] + + def health(self): + return self._healthy + + def device_link_start(self, device_name, client_version): + return self._link_start + + def device_link_poll(self, poll_token): + return self._polls.pop(0) + + def upload_session(self, session_id, body): + self.uploaded.append(session_id) + return self._uploads.pop(0) + + +def _write_session(tmp_path, name, *entries): + path = tmp_path / name + path.write_text("\n".join(json.dumps(e) for e in entries) + "\n") + return path + + +def _linked_config(): + return CloudConfig( + endpoint="https://e.test", + device_token="of_device_tok", + device_id="dev-1", + enabled=True, + ) + + +class TestPush: + def test_offline_short_circuits(self, tmp_path): + _write_session(tmp_path, "session_a.jsonl", {"type": "session_start"}) + client = FakeClient(healthy=False) + out = [] + result = commands.cmd_push(_linked_config(), tmp_path, client, out=out.append) + assert result["offline"] is True + assert client.uploaded == [] + + def test_inactive_config_is_noop(self, tmp_path): + _write_session(tmp_path, "session_a.jsonl", {"type": "session_start"}) + config = CloudConfig(enabled=False) + client = FakeClient() + result = commands.cmd_push(config, tmp_path, client, out=lambda _m: None) + assert result["skipped"] == "inactive" + assert client.uploaded == [] + + def test_success_marks_pushed(self, tmp_path): + path = _write_session( + tmp_path, + "session_a.jsonl", + {"type": "session_start", "session_uuid": "1f0e9c2a-7b3d-4e5f-8a9b-0c1d2e3f4a5b"}, + {"type": "shot_detected", "ball_speed_mph": 90}, + ) + client = FakeClient( + uploads=[UploadResult(201, action="success", session_id="x", shot_count=1)] + ) + result = commands.cmd_push(_linked_config(), tmp_path, client, out=lambda _m: None) + assert client.uploaded == ["1f0e9c2a-7b3d-4e5f-8a9b-0c1d2e3f4a5b"] + assert spool.is_pushed(path) + assert result["uploaded"] == 1 + + def test_dry_run_does_not_upload_or_mark(self, tmp_path): + path = _write_session( + tmp_path, + "session_a.jsonl", + {"type": "session_start", "session_uuid": "u"}, + {"type": "shot_detected", "ball_speed_mph": 90}, + {"type": "rolling_buffer_capture", "i_samples": [1, 2, 3]}, + ) + client = FakeClient() + out = [] + commands.cmd_push(_linked_config(), tmp_path, client, dry_run=True, out=out.append) + assert client.uploaded == [] + assert not spool.is_pushed(path) + printed = "\n".join(out) + # The privacy answer: shows kept types, hides dropped raw data. + assert "shot_detected" in printed + assert "session_start" in printed + assert "rolling_buffer_capture" not in printed + + def test_relink_stops_and_flags(self, tmp_path): + _write_session(tmp_path, "session_a.jsonl", {"type": "session_start", "session_uuid": "a"}) + _write_session(tmp_path, "session_b.jsonl", {"type": "session_start", "session_uuid": "b"}) + client = FakeClient( + uploads=[UploadResult(401, action="relink", reason="invalid_or_revoked_token")] + ) + result = commands.cmd_push(_linked_config(), tmp_path, client, out=lambda _m: None) + assert result["needs_relink"] is True + # Stops after the first 401 — does not attempt the second session. + assert len(client.uploaded) == 1 + + def test_park_on_422(self, tmp_path): + path = _write_session( + tmp_path, "session_a.jsonl", {"type": "session_start", "session_uuid": "a"} + ) + client = FakeClient(uploads=[UploadResult(422, action="park", reason="invalid_gzip")]) + commands.cmd_push(_linked_config(), tmp_path, client, out=lambda _m: None) + assert spool.is_parked(path) + + def test_quota_sets_cooldown_not_park(self, tmp_path): + path = _write_session( + tmp_path, "session_a.jsonl", {"type": "session_start", "session_uuid": "a"} + ) + client = FakeClient(uploads=[UploadResult(402, action="quota", reason="quota_exceeded")]) + commands.cmd_push(_linked_config(), tmp_path, client, out=lambda _m: None) + assert not spool.is_parked(path) + assert spool.in_cooldown(path) + + def test_5xx_records_failure_and_leaves_pending(self, tmp_path): + path = _write_session( + tmp_path, "session_a.jsonl", {"type": "session_start", "session_uuid": "a"} + ) + client = FakeClient(uploads=[UploadResult(503, action="retry")]) + commands.cmd_push(_linked_config(), tmp_path, client, out=lambda _m: None) + assert spool.read_attempts(path) == 1 + assert not spool.is_pushed(path) + assert not spool.is_parked(path) + + def test_oversize_body_parks(self, tmp_path, monkeypatch): + path = _write_session( + tmp_path, "session_a.jsonl", {"type": "shot_detected", "ball_speed_mph": 90} + ) + from openflight.cloud import filtering + + monkeypatch.setattr(filtering, "MAX_GZIP_BYTES", 1) + client = FakeClient(uploads=[UploadResult(201, action="success")]) + commands.cmd_push(_linked_config(), tmp_path, client, out=lambda _m: None) + assert client.uploaded == [] + assert spool.is_parked(path) + + def test_skips_sessions_in_cooldown(self, tmp_path): + path = _write_session( + tmp_path, "session_a.jsonl", {"type": "session_start", "session_uuid": "a"} + ) + spool.record_cooldown(path, "quota_exceeded", seconds=spool.QUOTA_COOLDOWN_S) + client = FakeClient(uploads=[UploadResult(201, action="success")]) + result = commands.cmd_push(_linked_config(), tmp_path, client, out=lambda _m: None) + assert client.uploaded == [] + assert result["deferred"] == 1 + + +class TestPushRetry: + def test_retry_all_reuploads_parked_session(self, tmp_path): + path = _write_session( + tmp_path, "session_a.jsonl", {"type": "session_start", "session_uuid": "a"} + ) + spool.mark_parked(path, reason="max_attempts", attempts=20, last_error="503") + client = FakeClient(uploads=[UploadResult(201, action="success", shot_count=1)]) + result = commands.cmd_push( + _linked_config(), tmp_path, client, retry=True, out=lambda _m: None + ) + assert client.uploaded == ["a"] + assert spool.is_pushed(path) + assert result["uploaded"] == 1 + + def test_retry_all_reuploads_cooled_down_session(self, tmp_path): + path = _write_session( + tmp_path, "session_a.jsonl", {"type": "session_start", "session_uuid": "a"} + ) + spool.record_cooldown(path, "quota_exceeded", seconds=spool.QUOTA_COOLDOWN_S) + client = FakeClient(uploads=[UploadResult(201, action="success", shot_count=1)]) + commands.cmd_push(_linked_config(), tmp_path, client, retry=True, out=lambda _m: None) + assert client.uploaded == ["a"] + + def test_retry_all_leaves_pushed_sessions_alone(self, tmp_path): + path = _write_session( + tmp_path, "session_a.jsonl", {"type": "session_start", "session_uuid": "a"} + ) + spool.mark_pushed(path, "a", 1) + client = FakeClient(uploads=[]) + commands.cmd_push(_linked_config(), tmp_path, client, retry=True, out=lambda _m: None) + assert client.uploaded == [] + assert spool.is_pushed(path) + + def test_retry_named_session_force_reuploads_pushed(self, tmp_path): + path = _write_session( + tmp_path, "session_20260527_x.jsonl", {"type": "session_start", "session_uuid": "a"} + ) + spool.mark_pushed(path, "a", 0) # previously "uploaded" with 0 shots + client = FakeClient(uploads=[UploadResult(201, action="success", shot_count=5)]) + commands.cmd_push( + _linked_config(), tmp_path, client, retry=True, session="20260527", out=lambda _m: None + ) + assert client.uploaded == ["a"] + assert spool.is_pushed(path) + + def test_retry_named_no_match_reports_and_uploads_nothing(self, tmp_path): + path = _write_session( + tmp_path, "session_a.jsonl", {"type": "session_start", "session_uuid": "a"} + ) + spool.mark_pushed(path, "a", 1) + client = FakeClient(uploads=[]) + out = [] + commands.cmd_push( + _linked_config(), tmp_path, client, retry=True, session="nope", out=out.append + ) + assert client.uploaded == [] + assert "no" in "\n".join(out).lower() + + +class TestLink: + def test_links_and_saves_config(self, tmp_path): + config_path = tmp_path / "cloud.json" + client = FakeClient( + link_start=LinkStart("ABCD-2345", "poll-tok", 5, 900), + polls=[ + LinkPoll("pending"), + LinkPoll("linked", device_token="of_device_new", device_id="dev-99"), + ], + ) + out = [] + ok = commands.cmd_link( + CloudConfig(endpoint="https://e.test"), + config_path, + client, + device_name="garage pi", + sleep=lambda _s: None, + out=out.append, + ) + assert ok is True + saved = json.loads(config_path.read_text()) + assert saved["device_token"] == "of_device_new" + assert saved["device_id"] == "dev-99" + assert saved["enabled"] is True + assert "ABCD-2345" in "\n".join(out) + + def test_expired_returns_false(self, tmp_path): + client = FakeClient( + link_start=LinkStart("ABCD-2345", "poll-tok", 5, 900), + polls=[LinkPoll("expired")], + ) + ok = commands.cmd_link( + CloudConfig(endpoint="https://e.test"), + tmp_path / "cloud.json", + client, + device_name="pi", + sleep=lambda _s: None, + out=lambda _m: None, + ) + assert ok is False + assert not (tmp_path / "cloud.json").exists() + + +class TestStatus: + def test_reports_unlinked(self, tmp_path): + out = [] + commands.cmd_status(CloudConfig(), tmp_path, out=out.append) + assert "not linked" in "\n".join(out).lower() + + def test_reports_reachability_when_client_given(self, tmp_path): + out = [] + client = FakeClient(healthy=False) + result = commands.cmd_status(_linked_config(), tmp_path, client=client, out=out.append) + assert result["online"] is False + assert "unreachable" in "\n".join(out).lower() + + def test_flags_zero_shot_uploads_with_retry_hint(self, tmp_path): + a = _write_session(tmp_path, "session_zero.jsonl", {"type": "session_start"}) + b = _write_session(tmp_path, "session_ok.jsonl", {"type": "session_start"}) + spool.mark_pushed(a, "id-a", 0) + spool.mark_pushed(b, "id-b", 12) + out = [] + result = commands.cmd_status(_linked_config(), tmp_path, out=out.append) + text = "\n".join(out) + assert "session_zero.jsonl" in text + assert "--retry" in text + assert "session_ok.jsonl" not in text # healthy uploads aren't nagged + assert result["zero_shot"] == ["session_zero.jsonl"] + + def test_reports_counts_and_parked(self, tmp_path): + a = _write_session(tmp_path, "session_a.jsonl", {"type": "session_start"}) + b = _write_session(tmp_path, "session_b.jsonl", {"type": "session_start"}) + spool.mark_pushed(a, "id-a", 2) + spool.mark_parked(b, reason="invalid_gzip", attempts=3, last_error="422") + out = [] + commands.cmd_status(_linked_config(), tmp_path, out=out.append) + text = "\n".join(out) + assert "dev-1" in text + assert "invalid_gzip" in text diff --git a/tests/test_cloud_config.py b/tests/test_cloud_config.py new file mode 100644 index 000000000..7f637f138 --- /dev/null +++ b/tests/test_cloud_config.py @@ -0,0 +1,84 @@ +"""Tests for the openflight-cloud config module.""" + +import json +import stat + +import pytest + +from openflight.cloud import config as cfg + + +class TestLoadConfig: + def test_returns_none_when_file_absent(self, tmp_path): + assert cfg.load_config(tmp_path / "cloud.json") is None + + def test_loads_all_fields(self, tmp_path): + path = tmp_path / "cloud.json" + path.write_text( + json.dumps( + { + "endpoint": "https://example.test", + "device_token": "of_device_" + "a" * 32, + "device_id": "abc-123", + "enabled": True, + } + ) + ) + loaded = cfg.load_config(path) + assert loaded.endpoint == "https://example.test" + assert loaded.device_token == "of_device_" + "a" * 32 + assert loaded.device_id == "abc-123" + assert loaded.enabled is True + + def test_defaults_endpoint_and_enabled_when_missing(self, tmp_path): + path = tmp_path / "cloud.json" + path.write_text(json.dumps({"device_token": "t", "device_id": "i"})) + loaded = cfg.load_config(path) + assert loaded.endpoint == cfg.DEFAULT_ENDPOINT + assert loaded.enabled is True + + +class TestSaveConfig: + def test_writes_file_with_0600_permissions(self, tmp_path): + path = tmp_path / "nested" / "cloud.json" + config = cfg.CloudConfig(device_token="tok", device_id="id") + cfg.save_config(config, path) + + assert path.exists() + mode = stat.S_IMODE(path.stat().st_mode) + assert mode == 0o600 + + def test_round_trips_through_load(self, tmp_path): + path = tmp_path / "cloud.json" + config = cfg.CloudConfig( + endpoint="https://e.test", + device_token="of_device_" + "b" * 32, + device_id="dev-9", + enabled=False, + ) + cfg.save_config(config, path) + loaded = cfg.load_config(path) + assert loaded == config + + +class TestIsLinked: + def test_true_when_token_and_id_present(self): + config = cfg.CloudConfig(device_token="t", device_id="i") + assert config.is_linked() + + def test_false_when_token_missing(self): + assert not cfg.CloudConfig(device_id="i").is_linked() + + def test_false_when_id_missing(self): + assert not cfg.CloudConfig(device_token="t").is_linked() + + +class TestIsActive: + def test_active_requires_enabled_and_linked(self): + assert cfg.CloudConfig(device_token="t", device_id="i", enabled=True).is_active() + + def test_inactive_when_disabled(self): + assert not cfg.CloudConfig(device_token="t", device_id="i", enabled=False).is_active() + + def test_inactive_when_not_linked(self): + assert not cfg.CloudConfig(enabled=True).is_active() diff --git a/tests/test_cloud_filtering.py b/tests/test_cloud_filtering.py new file mode 100644 index 000000000..40c722e80 --- /dev/null +++ b/tests/test_cloud_filtering.py @@ -0,0 +1,191 @@ +"""Tests for the openflight-cloud client-side filtering (raw-ADC strip).""" + +import gzip +import json +import uuid + +import pytest + +from openflight.cloud import filtering as flt + + +def _line(entry_type, **fields): + return json.dumps({"ts": "2026-06-14T00:00:00", "type": entry_type, **fields}) + + +class TestFilterSessionLines: + def test_keeps_only_allowlisted_types(self): + lines = [ + _line("session_start", session_uuid="u"), + _line("rolling_buffer_capture", i_samples=[1, 2, 3]), + _line("shot_detected", ball_speed_mph=100), + _line("iq_blocks", blocks=[]), + _line("trigger_event", accepted=True), + _line("session_end"), + ] + result = flt.filter_session_lines(lines, device_id="dev-1") + kept_types = [json.loads(line)["type"] for line in result.kept_lines] + assert kept_types == [ + "session_start", + "shot_detected", + "trigger_event", + "session_end", + ] + + def test_keeps_both_error_and_session_error(self): + lines = [ + _line("error", error="boom"), + _line("session_error", error="boom2"), + ] + result = flt.filter_session_lines(lines, device_id="dev-1") + kept_types = [json.loads(line)["type"] for line in result.kept_lines] + assert kept_types == ["error", "session_error"] + + def test_drops_unknown_future_type_by_default(self): + lines = [_line("some_future_heavy_type", data="x" * 10)] + result = flt.filter_session_lines(lines, device_id="dev-1") + assert result.kept_lines == [] + + def test_drops_kept_line_over_32kb_and_counts_it(self): + big = _line("shot_detected", note="x" * (33 * 1024)) + small = _line("shot_detected", ball_speed_mph=90) + result = flt.filter_session_lines([big, small], device_id="dev-1") + assert result.dropped_oversize == 1 + assert len(result.kept_lines) == 1 + + def test_ignores_blank_and_unparseable_lines(self): + lines = ["", " ", "not json", _line("shot_detected", ball_speed_mph=90)] + result = flt.filter_session_lines(lines, device_id="dev-1") + assert len(result.kept_lines) == 1 + + def test_manifest_has_expected_shape(self): + lines = [ + _line("session_start", session_uuid="u"), + _line("shot_detected", ball_speed_mph=90), + ] + result = flt.filter_session_lines(lines, device_id="dev-7", client_version="9.9.9") + m = result.manifest + assert m["type"] == "upload_manifest" + assert m["format_version"] == 1 + assert m["client_version"] == "9.9.9" + assert m["device_id"] == "dev-7" + assert m["filtered"] is True + assert set(m["kept_entry_types"]) == {"session_start", "shot_detected"} + + def test_kept_entry_types_are_sorted_and_unique(self): + lines = [ + _line("shot_detected"), + _line("shot_detected"), + _line("session_start"), + ] + result = flt.filter_session_lines(lines, device_id="d") + assert result.manifest["kept_entry_types"] == ["session_start", "shot_detected"] + + +class TestFilterSessionFile: + """Streaming filter that reads a file path without materializing it.""" + + def _write(self, path, *entries): + path.write_text("\n".join(json.dumps(e) for e in entries) + "\n") + + def test_keeps_shots_and_strips_raw_adc(self, tmp_path): + path = tmp_path / "session_x.jsonl" + self._write( + path, + {"type": "session_start", "session_uuid": "1F0E9C2A-7B3D-4E5F-8A9B-0C1D2E3F4A5B"}, + {"type": "rolling_buffer_capture", "i_samples": [1, 2, 3], "q_samples": [4, 5]}, + {"type": "shot_detected", "ball_speed_mph": 142}, + {"type": "kld7_buffer", "frames": [{"radc_b64": "AAAA"}]}, + {"type": "shot_detected", "ball_speed_mph": 99}, + {"type": "session_end"}, + ) + result = flt.filter_session_file(path, device_id="dev-1") + kept_types = [json.loads(line)["type"] for line in result.kept_lines] + assert kept_types == ["session_start", "shot_detected", "shot_detected", "session_end"] + assert result.kept_type_counts["shot_detected"] == 2 + + def test_resolves_session_id_from_embedded_uuid(self, tmp_path): + path = tmp_path / "session_x.jsonl" + self._write( + path, + {"type": "session_start", "session_uuid": "1F0E9C2A-7B3D-4E5F-8A9B-0C1D2E3F4A5B"}, + {"type": "shot_detected", "ball_speed_mph": 90}, + ) + result = flt.filter_session_file(path, device_id="dev-1") + assert result.session_id == "1f0e9c2a-7b3d-4e5f-8a9b-0c1d2e3f4a5b" + + def test_session_id_uuid5_fallback_uses_filename(self, tmp_path): + path = tmp_path / "session_x.jsonl" + self._write(path, {"type": "session_start"}, {"type": "shot_detected"}) + result = flt.filter_session_file(path, device_id="dev-1") + expected = str(uuid.uuid5(flt.SESSION_NAMESPACE, "dev-1:session_x.jsonl")) + assert result.session_id == expected + + def test_does_not_load_whole_file_into_memory(self, tmp_path): + """Regression: filtering a large raw-ADC file must use bounded memory. + + The old path read the entire file via read_text().splitlines(), peaking + at ~2x the file size — enough to OOM-kill the push on a Pi, dropping the + whole upload ("0 shots uploaded"). Streaming keeps peak ~one line. + """ + import tracemalloc + + path = tmp_path / "session_big.jsonl" + # ~20 MB of raw ADC (100 lines x ~200 KB), plus a handful of shots. + big = {"type": "rolling_buffer_capture", "i_samples": list(range(25000))} + with path.open("w") as f: + f.write(json.dumps({"type": "session_start", "session_uuid": "u"}) + "\n") + for i in range(100): + f.write(json.dumps(big) + "\n") + if i % 25 == 0: + f.write(json.dumps({"type": "shot_detected", "ball_speed_mph": 90}) + "\n") + f.write(json.dumps({"type": "session_end"}) + "\n") + + file_size = path.stat().st_size + assert file_size > 15 * 1024 * 1024 # sanity: the file really is large + + tracemalloc.start() + result = flt.filter_session_file(path, device_id="d") + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + + assert result.kept_type_counts["shot_detected"] == 4 + # Streaming peaks at roughly one line; the file is >15 MB. A generous + # 8 MB ceiling cleanly fails the old whole-file-in-memory approach. + assert peak < 8 * 1024 * 1024, f"peak {peak / 1024 / 1024:.1f} MB — file not streamed" + + +class TestBuildUploadBody: + def test_body_is_gzip_with_manifest_first(self): + lines = [ + _line("session_start", session_uuid="u"), + _line("shot_detected", ball_speed_mph=90), + ] + result = flt.filter_session_lines(lines, device_id="dev-1") + body = flt.build_upload_body(result) + + ndjson = gzip.decompress(body).decode("utf-8") + out_lines = ndjson.strip().split("\n") + first = json.loads(out_lines[0]) + assert first["type"] == "upload_manifest" + assert json.loads(out_lines[1])["type"] == "session_start" + assert json.loads(out_lines[2])["type"] == "shot_detected" + + def test_raises_when_gzip_exceeds_cap(self): + result = flt.FilterResult( + manifest={"type": "upload_manifest"}, + kept_lines=[_line("shot_detected", ball_speed_mph=90)], + dropped_oversize=0, + ) + # 1-byte caps make any real body too large. + with pytest.raises(flt.BodyTooLargeError): + flt.build_upload_body(result, max_gzip_bytes=1, max_inflated_bytes=1_000_000) + + def test_raises_when_inflated_exceeds_cap(self): + result = flt.FilterResult( + manifest={"type": "upload_manifest"}, + kept_lines=[_line("shot_detected", ball_speed_mph=90)], + dropped_oversize=0, + ) + with pytest.raises(flt.BodyTooLargeError): + flt.build_upload_body(result, max_gzip_bytes=1_000_000, max_inflated_bytes=1) diff --git a/tests/test_cloud_spool.py b/tests/test_cloud_spool.py new file mode 100644 index 000000000..ba3497498 --- /dev/null +++ b/tests/test_cloud_spool.py @@ -0,0 +1,159 @@ +"""Tests for the openflight-cloud spool-and-retry sidecar mechanics.""" + +import json + +import pytest + +from openflight.cloud import spool + + +def _session(tmp_path, name="session_20260614_120000_range.jsonl"): + path = tmp_path / name + path.write_text('{"type":"session_start"}\n') + return path + + +class TestDiscovery: + def test_session_files_finds_only_session_jsonl(self, tmp_path): + _session(tmp_path, "session_a.jsonl") + _session(tmp_path, "session_b.jsonl") + (tmp_path / "radar_raw_x.log").write_text("noise") + (tmp_path / "other.jsonl").write_text("{}") + found = {p.name for p in spool.session_files(tmp_path)} + assert found == {"session_a.jsonl", "session_b.jsonl"} + + def test_session_files_empty_when_dir_missing(self, tmp_path): + assert spool.session_files(tmp_path / "nope") == [] + + def test_pending_excludes_pushed_and_parked(self, tmp_path): + a = _session(tmp_path, "session_a.jsonl") + b = _session(tmp_path, "session_b.jsonl") + c = _session(tmp_path, "session_c.jsonl") + spool.mark_pushed(b, session_id="id-b", shot_count=3) + spool.mark_parked(c, reason="quota_exceeded", attempts=20, last_error="402") + pending = {p.name for p in spool.pending_sessions(tmp_path)} + assert pending == {"session_a.jsonl"} + assert a # referenced + + +class TestPushedMarker: + def test_mark_pushed_creates_sidecar(self, tmp_path): + path = _session(tmp_path) + spool.mark_pushed(path, session_id="abc", shot_count=5) + marker = tmp_path / (path.name + ".pushed") + assert marker.exists() + data = json.loads(marker.read_text()) + assert data["session_id"] == "abc" + assert data["shot_count"] == 5 + assert spool.is_pushed(path) + + def test_mark_pushed_clears_attempt_state(self, tmp_path): + path = _session(tmp_path) + spool.record_failure(path, "5xx") + spool.mark_pushed(path, session_id="abc", shot_count=1) + assert spool.read_attempts(path) == 0 + assert not (tmp_path / (path.name + ".state")).exists() + + +class TestParkedMarker: + def test_mark_parked_creates_sidecar(self, tmp_path): + path = _session(tmp_path) + spool.mark_parked(path, reason="invalid_session_id", attempts=1, last_error="422") + marker = tmp_path / (path.name + ".parked") + assert marker.exists() + data = json.loads(marker.read_text()) + assert data["reason"] == "invalid_session_id" + assert data["attempts"] == 1 + assert data["last_error"] == "422" + assert spool.is_parked(path) + + +class TestAttemptCounter: + def test_attempts_start_at_zero(self, tmp_path): + path = _session(tmp_path) + assert spool.read_attempts(path) == 0 + + def test_record_failure_increments_and_returns_count(self, tmp_path): + path = _session(tmp_path) + assert spool.record_failure(path, "5xx") == 1 + assert spool.record_failure(path, "5xx") == 2 + assert spool.read_attempts(path) == 2 + + def test_record_failure_stores_last_error(self, tmp_path): + path = _session(tmp_path) + spool.record_failure(path, "boom") + state = json.loads((tmp_path / (path.name + ".state")).read_text()) + assert state["last_error"] == "boom" + + def test_record_failure_parks_after_max_attempts(self, tmp_path): + path = _session(tmp_path) + for _ in range(spool.MAX_ATTEMPTS - 1): + spool.record_failure(path, "5xx") + assert not spool.is_parked(path) + spool.record_failure(path, "5xx") + assert spool.is_parked(path) + assert spool.read_attempts(path) == spool.MAX_ATTEMPTS + + +class TestCooldown: + def test_no_cooldown_by_default(self, tmp_path): + path = _session(tmp_path) + assert spool.in_cooldown(path, now=1000.0) is False + + def test_record_cooldown_blocks_until_elapsed(self, tmp_path): + path = _session(tmp_path) + spool.record_cooldown(path, "quota_exceeded", seconds=100, now=1000.0) + assert spool.in_cooldown(path, now=1050.0) is True + assert spool.in_cooldown(path, now=1101.0) is False + + def test_cooldown_does_not_park(self, tmp_path): + path = _session(tmp_path) + spool.record_cooldown(path, "quota_exceeded", seconds=100, now=1000.0) + assert not spool.is_parked(path) + + +class TestClearMarkers: + def test_clears_parked_and_state_keeps_pushed_by_default(self, tmp_path): + path = _session(tmp_path) + spool.mark_pushed(path, "id", 1) + spool.mark_parked(path, reason="r", attempts=2, last_error="e") + spool.record_failure(path, "e") # writes .state + cleared = spool.clear_markers(path) + assert spool.PARKED_SUFFIX in cleared + assert spool.STATE_SUFFIX in cleared + assert spool.PUSHED_SUFFIX not in cleared + assert spool.is_pushed(path) + assert not spool.is_parked(path) + + def test_include_pushed_clears_everything(self, tmp_path): + path = _session(tmp_path) + spool.mark_pushed(path, "id", 1) + cleared = spool.clear_markers(path, include_pushed=True) + assert spool.PUSHED_SUFFIX in cleared + assert not spool.is_pushed(path) + + def test_returns_empty_when_no_markers(self, tmp_path): + path = _session(tmp_path) + assert spool.clear_markers(path) == [] + + def test_after_clear_session_is_pending_again(self, tmp_path): + a = _session(tmp_path, "session_a.jsonl") + spool.mark_parked(a, reason="r", attempts=20, last_error="e") + assert a not in spool.pending_sessions(tmp_path) + spool.clear_markers(a) + assert a in spool.pending_sessions(tmp_path) + + +class TestStatusSummary: + def test_summarizes_counts(self, tmp_path): + a = _session(tmp_path, "session_a.jsonl") + b = _session(tmp_path, "session_b.jsonl") + c = _session(tmp_path, "session_c.jsonl") + spool.mark_pushed(a, session_id="a", shot_count=1) + spool.mark_parked(b, reason="r", attempts=20, last_error="402") + assert c + summary = spool.summarize(tmp_path) + assert summary["pushed"] == 1 + assert summary["parked"] == 1 + assert summary["pending"] == 1 + assert summary["total"] == 3 diff --git a/tests/test_cloud_trigger.py b/tests/test_cloud_trigger.py new file mode 100644 index 000000000..f0c6db413 --- /dev/null +++ b/tests/test_cloud_trigger.py @@ -0,0 +1,49 @@ +"""Tests for the non-blocking session-end push trigger.""" + +import pytest + +from openflight.cloud import trigger +from openflight.cloud.config import CloudConfig + + +class TestFirePushAsync: + def test_fires_when_active(self, tmp_path): + calls = [] + config = CloudConfig(device_token="t", device_id="i", enabled=True) + ok = trigger.fire_push_async( + config, log_dir=tmp_path, popen_fn=lambda cmd, **k: calls.append(cmd) + ) + assert ok is True + assert len(calls) == 1 + assert "push" in calls[0] + + def test_skips_when_inactive(self, tmp_path): + calls = [] + config = CloudConfig(enabled=False) + ok = trigger.fire_push_async( + config, log_dir=tmp_path, popen_fn=lambda cmd, **k: calls.append(cmd) + ) + assert ok is False + assert calls == [] + + def test_swallows_spawn_errors(self, tmp_path): + def boom(cmd, **k): + raise OSError("no exec") + + config = CloudConfig(device_token="t", device_id="i", enabled=True) + # Must never raise into the caller (server shot path). + assert trigger.fire_push_async(config, log_dir=tmp_path, popen_fn=boom) is False + + def test_passes_config_and_log_dir(self, tmp_path): + calls = [] + config = CloudConfig(device_token="t", device_id="i", enabled=True) + cfg_path = tmp_path / "cloud.json" + trigger.fire_push_async( + config, + log_dir=tmp_path, + config_path=cfg_path, + popen_fn=lambda cmd, **k: calls.append(cmd), + ) + cmd = calls[0] + assert str(cfg_path) in cmd + assert str(tmp_path) in cmd diff --git a/tests/test_kld7.py b/tests/test_kld7.py index 865787567..aba7f0c14 100644 --- a/tests/test_kld7.py +++ b/tests/test_kld7.py @@ -1049,8 +1049,6 @@ def test_vertical_rules_can_pair_strong_anchor_with_weak_rising_neighbor(self): snr_linear=10.0, angle_deg=7.0, speed_mph=117.0, - raw_angle_deg=7.0, - geom_bearing_deg=0.0, t_after_impact_s=0.038, phase_coherence=0.99, peak_width_bins=5, @@ -1062,8 +1060,6 @@ def test_vertical_rules_can_pair_strong_anchor_with_weak_rising_neighbor(self): snr_linear=VERTICAL_RULE_WEAK_ADJACENT_SNR_FLOOR + 0.1, angle_deg=11.0, speed_mph=117.0, - raw_angle_deg=11.0, - geom_bearing_deg=0.0, t_after_impact_s=0.071, phase_coherence=0.99, peak_width_bins=5, @@ -1088,8 +1084,6 @@ def test_vertical_rules_can_pair_primary_anchor_with_early_ops_matched_neighbor( snr_linear=6.0, angle_deg=2.0, speed_mph=117.0, - raw_angle_deg=2.0, - geom_bearing_deg=0.0, t_after_impact_s=0.012, phase_coherence=0.99, peak_width_bins=5, @@ -1101,8 +1095,6 @@ def test_vertical_rules_can_pair_primary_anchor_with_early_ops_matched_neighbor( snr_linear=10.0, angle_deg=7.0, speed_mph=117.0, - raw_angle_deg=7.0, - geom_bearing_deg=0.0, t_after_impact_s=0.038, phase_coherence=0.99, peak_width_bins=5, @@ -1126,8 +1118,6 @@ def test_vertical_rules_prefer_primary_pair_over_early_context_pair(self): snr_linear=6.0, angle_deg=2.0, speed_mph=117.0, - raw_angle_deg=2.0, - geom_bearing_deg=0.0, t_after_impact_s=0.012, phase_coherence=0.99, peak_width_bins=5, @@ -1139,8 +1129,6 @@ def test_vertical_rules_prefer_primary_pair_over_early_context_pair(self): snr_linear=10.0, angle_deg=7.0, speed_mph=117.0, - raw_angle_deg=7.0, - geom_bearing_deg=0.0, t_after_impact_s=0.038, phase_coherence=0.99, peak_width_bins=5, @@ -1152,8 +1140,6 @@ def test_vertical_rules_prefer_primary_pair_over_early_context_pair(self): snr_linear=8.0, angle_deg=11.0, speed_mph=117.0, - raw_angle_deg=11.0, - geom_bearing_deg=0.0, t_after_impact_s=0.071, phase_coherence=0.99, peak_width_bins=5, @@ -1177,8 +1163,6 @@ def test_vertical_rules_do_not_anchor_on_early_frame_alone(self): snr_linear=12.0, angle_deg=2.0, speed_mph=117.0, - raw_angle_deg=2.0, - geom_bearing_deg=0.0, t_after_impact_s=0.012, phase_coherence=0.99, peak_width_bins=5, @@ -1202,8 +1186,6 @@ def test_vertical_rules_require_strong_anchor_for_weak_adjacent_frames(self): snr_linear=VERTICAL_RULE_WEAK_ADJACENT_SNR_FLOOR + 0.1, angle_deg=5.0 + idx, speed_mph=117.0, - raw_angle_deg=5.0 + idx, - geom_bearing_deg=0.0, t_after_impact_s=0.040 + idx * 0.030, phase_coherence=0.99, peak_width_bins=5, @@ -1229,8 +1211,6 @@ def test_vertical_rules_reject_weak_neighbor_when_bearing_is_not_rising(self): snr_linear=10.0, angle_deg=7.0, speed_mph=117.0, - raw_angle_deg=7.0, - geom_bearing_deg=0.0, t_after_impact_s=0.038, phase_coherence=0.99, peak_width_bins=5, @@ -1242,8 +1222,6 @@ def test_vertical_rules_reject_weak_neighbor_when_bearing_is_not_rising(self): snr_linear=VERTICAL_RULE_WEAK_ADJACENT_SNR_FLOOR + 0.1, angle_deg=3.0, speed_mph=117.0, - raw_angle_deg=3.0, - geom_bearing_deg=0.0, t_after_impact_s=0.071, phase_coherence=0.99, peak_width_bins=5, @@ -1702,3 +1680,172 @@ def fake_extract_launch_angle(frames, **kwargs): "distance_ft": 5.5, } ] + + +def _make_iq_pair_with_fringe( + velocity_kmh, + angle_deg, + wobble_deg=0.0, + wobble_cycles=2, + am_depth=0.0, + amplitude=1000.0, + n=256, +): + """Synthesize complex F1A/F2A I/Q for one frame. + + The F2A steering phase tracks ``angle_deg`` plus an optional sinusoidal + wobble (simulating a ground-bounce fringe sweeping the blended angle), + and F2A amplitude is optionally modulated by ``am_depth`` at the same + rate (simulating the channel-balance oscillation of a fringe). + """ + from openflight.kld7.radc import ANTENNA_SPACING_M, WAVELENGTH_M + + max_speed_kmh = 100.0 + if velocity_kmh >= 0: + norm_freq = velocity_kmh / (2 * max_speed_kmh) + else: + norm_freq = 1.0 + velocity_kmh / (2 * max_speed_kmh) + + t = np.arange(n) + carrier = 2 * np.pi * norm_freq * t + fringe = 2 * np.pi * wobble_cycles * t / n + inst_angle_rad = np.radians(angle_deg + wobble_deg * np.sin(fringe)) + steering = 2 * np.pi * ANTENNA_SPACING_M * np.sin(inst_angle_rad) / WAVELENGTH_M + + f1a = amplitude * np.exp(1j * carrier) + f2a = amplitude * (1.0 + am_depth * np.sin(fringe)) * np.exp(1j * (carrier + steering)) + return f1a, f2a + + +class TestDCAliasBlindZone: + """Ball speeds that alias onto DC are buried in the DC clutter mask + (K-LD7 ±100 km/h wrap: ~118-131 mph lands within ~11 km/h of bin 0).""" + + def test_blind_zone_band(self): + from openflight.kld7.radc import is_dc_alias_blind_zone + + # 124 mph = 199.5 km/h aliases to -0.5 km/h — dead center of DC + assert is_dc_alias_blind_zone(124.0) + # Edges of the empirical 118-131 mph band + assert is_dc_alias_blind_zone(119.0) # aliases to -8.5 km/h + assert is_dc_alias_blind_zone(130.0) # aliases to +9.2 km/h + # Clear speeds on either side of the band + assert not is_dc_alias_blind_zone(110.0) # -23.0 km/h + assert not is_dc_alias_blind_zone(140.0) # +25.3 km/h + # Typical iron / minimum pipeline speeds are nowhere near DC + assert not is_dc_alias_blind_zone(72.0) + assert not is_dc_alias_blind_zone(35.0) + + def _frames_with_tone(self, aliased_kmh, amplitude=6000): + synth = TestRADCAngleExtraction() + impact_ts = time.time() + quiet = synth._make_quiet_radc_payload() + frames = [{"timestamp": impact_ts - (6 - i) * 0.056, "radc": quiet} for i in range(6)] + tone = synth._make_radc_payload_with_tone(aliased_kmh, angle_deg=5.0, amplitude=amplitude) + frames.append({"timestamp": impact_ts + 0.056, "radc": tone}) + return frames + + def test_extract_flags_blind_zone_and_caps_confidence(self): + from openflight.kld7.radc import extract_launch_angle + + # 119 mph aliases to -8.5 km/h: detectable (outside the 8-bin DC + # mask) but inside the blind-zone margin where clutter dominates. + ball_speed_mph = 119.0 + aliased_kmh = (ball_speed_mph * 1.609) % 200.0 - 200.0 + + results = extract_launch_angle( + self._frames_with_tone(aliased_kmh), + ops243_ball_speed_mph=ball_speed_mph, + ) + + assert results + assert results[0]["dc_blind_zone"] is True + assert results[0]["confidence"] <= 0.35 + + def test_extract_clear_speed_not_flagged(self): + from openflight.kld7.radc import extract_launch_angle + + ball_speed_mph = 72.0 + aliased_kmh = (ball_speed_mph * 1.609) % 200.0 - 200.0 + + results = extract_launch_angle( + self._frames_with_tone(aliased_kmh), + ops243_ball_speed_mph=ball_speed_mph, + ) + + assert results + assert results[0]["dc_blind_zone"] is False + + +class TestSubframeFringeMetrics: + """Per-frame sub-frame STFT metrics exposing ground-bounce fringes: + elevation ripple and F1A/F2A balance excursion across 64-sample windows.""" + + def _peak_bin(self, velocity_kmh, fft_size=2048): + from openflight.kld7.radc import _velocity_to_bin + + return _velocity_to_bin(velocity_kmh, fft_size) + + def test_constant_angle_has_low_ripple(self): + from openflight.kld7.radc import subframe_fringe_metrics + + f1a, f2a = _make_iq_pair_with_fringe(-84.0, angle_deg=8.0) + metrics = subframe_fringe_metrics(f1a, f2a, self._peak_bin(-84.0)) + + assert metrics is not None + assert metrics["subframe_count"] == 13 + assert metrics["elev_p2p_deg"] < 1.0 + assert metrics["balance_max"] - metrics["balance_min"] < 0.1 + + def test_fringe_wobble_detected(self): + from openflight.kld7.radc import subframe_fringe_metrics + + f1a, f2a = _make_iq_pair_with_fringe( + -84.0, angle_deg=8.0, wobble_deg=6.0, wobble_cycles=2, am_depth=0.4 + ) + metrics = subframe_fringe_metrics(f1a, f2a, self._peak_bin(-84.0)) + + assert metrics is not None + assert metrics["elev_p2p_deg"] > 4.0 + assert metrics["balance_max"] - metrics["balance_min"] > 0.3 + + def test_zero_signal_returns_none(self): + from openflight.kld7.radc import subframe_fringe_metrics + + zeros = np.zeros(256, dtype=np.complex128) + assert subframe_fringe_metrics(zeros, zeros, 1186) is None + + def test_extract_attaches_fringe_metrics_for_selected_frames(self): + from openflight.kld7.radc import extract_launch_angle + + synth = TestRADCAngleExtraction() + impact_ts = time.time() + quiet = synth._make_quiet_radc_payload() + frames = [{"timestamp": impact_ts - (6 - i) * 0.056, "radc": quiet} for i in range(6)] + tone = synth._make_radc_payload_with_tone(-84.0, angle_deg=5.0, amplitude=6000) + frames.append({"timestamp": impact_ts + 0.056, "radc": tone}) + + results = extract_launch_angle(frames, ops243_ball_speed_mph=72.0) + + assert results + fringe = results[0]["fringe_metrics"] + assert isinstance(fringe, list) and fringe + entry = fringe[0] + assert entry["frame_index"] in results[0]["selected_frame_indices"] + for key in ("subframe_count", "elev_p2p_deg", "balance_min", "balance_max"): + assert key in entry + + +class TestRADCDiagnosticsPassthrough: + def test_selection_diagnostics_include_blind_zone_and_fringe(self): + from openflight.kld7.tracker import _radc_selection_diagnostics + + best = { + "estimator": "naive", + "dc_blind_zone": True, + "fringe_metrics": [{"frame_index": 6, "elev_p2p_deg": 7.2}], + } + diagnostics = _radc_selection_diagnostics(best, relaxed_retry=False) + + assert diagnostics["dc_blind_zone"] is True + assert diagnostics["fringe_metrics"] == [{"frame_index": 6, "elev_p2p_deg": 7.2}] diff --git a/tests/test_ops243.py b/tests/test_ops243.py index f7bec8895..fb9d076a8 100644 --- a/tests/test_ops243.py +++ b/tests/test_ops243.py @@ -1,5 +1,7 @@ """Tests for OPS243 radar driver.""" +import time + import pytest from openflight.ops243 import Direction, OPS243Radar, SpeedReading @@ -286,6 +288,16 @@ def test_offset_spread_reported_for_multiple_reads(self): assert summary["offset_spread_ms"] is not None assert summary["offset_spread_ms"] >= 0.0 + def test_store_false_does_not_replace_last_clock_sync(self): + existing = {"best_offset_s": 42.0} + radar = self._radar(_FakeClockSerial(clock_value="137.429")) + radar.last_clock_sync = existing + + summary = radar.read_clock_sync(samples=2, per_read_timeout=0.05, store=False) + + assert summary["best_offset_s"] is not None + assert radar.last_clock_sync is existing + def test_no_response_is_handled(self): radar = self._radar(_FakeClockSerial(respond=False)) summary = radar.read_clock_sync(samples=2, per_read_timeout=0.01) @@ -330,3 +342,88 @@ def test_raises_when_not_connected(self): radar.serial = None with pytest.raises(ConnectionError): radar.read_clock_sync(samples=1) + + +class _ScheduledSerial: + """Serial stand-in that releases bytes on a wall-clock schedule. + + Models the rolling-buffer dump: nothing arrives until the hardware + trigger fires, then ~46KB streams over several seconds. Schedule is + a list of (seconds_after_reset, bytes) pairs. + """ + + def __init__(self, schedule): + self.is_open = True + self._schedule = sorted(schedule, key=lambda item: item[0]) + self._t0 = time.time() + self._consumed = 0 + + def reset_input_buffer(self): + self._t0 = time.time() + self._consumed = 0 + + def _released(self): + elapsed = time.time() - self._t0 + return b"".join(data for t, data in self._schedule if t <= elapsed) + + @property + def in_waiting(self): + return len(self._released()) - self._consumed + + def read(self, n): + released = self._released() + chunk = released[self._consumed : self._consumed + n] + self._consumed += len(chunk) + return chunk + + +class TestWaitForHardwareTrigger: + """Tests for the hardware-trigger read loop (sound trigger path).""" + + # A complete dump ends with a closed Q array — the loop's completeness + # check requires '"Q"' followed by ']}'. + _DUMP = [ + b'{"sample_time":946.077}\r\n{"trigger_time":946.145}\r\n', + b'{"I":[2168,2187,2155,2154]}\r\n', + b'{"Q":[2048,2050,2047,2049]}', + ] + + def _radar(self, serial_obj): + radar = OPS243Radar.__new__(OPS243Radar) + radar.serial = serial_obj + radar.last_hardware_trigger_first_byte_timestamp = None + return radar + + def test_trigger_near_timeout_still_reads_full_dump(self): + """A trigger firing just before the timeout must not truncate the dump. + + Regression test: the wait loop bounded TOTAL elapsed time, so a + trigger at 26s of a 30s window had its ~4.5s dump cut off at 30.0s + ("Incomplete capture (missing: Q)"). Scaled down: trigger at 0.3s + of a 0.4s window, dump completes at 0.7s. + """ + schedule = [ + (0.30, self._DUMP[0]), + (0.50, self._DUMP[1]), + (0.70, self._DUMP[2]), + ] + radar = self._radar(_ScheduledSerial(schedule)) + response = radar.wait_for_hardware_trigger(timeout=0.4) + assert response == b"".join(self._DUMP).decode("ascii") + + def test_no_trigger_returns_empty_after_timeout(self): + """With no data at all, the wait still times out promptly.""" + radar = self._radar(_ScheduledSerial([])) + start = time.time() + response = radar.wait_for_hardware_trigger(timeout=0.3) + assert response == "" + assert time.time() - start < 1.5 + + def test_complete_dump_returns_before_grace_expires(self): + """A dump that finishes early returns immediately on completeness.""" + schedule = [(0.05, b"".join(self._DUMP))] + radar = self._radar(_ScheduledSerial(schedule)) + start = time.time() + response = radar.wait_for_hardware_trigger(timeout=5.0) + assert response == b"".join(self._DUMP).decode("ascii") + assert time.time() - start < 1.0 diff --git a/tests/test_probe_kld7_timing.py b/tests/test_probe_kld7_timing.py new file mode 100644 index 000000000..1128232af --- /dev/null +++ b/tests/test_probe_kld7_timing.py @@ -0,0 +1,189 @@ +"""Tests for the guarded K-LD7 timing probe.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "hardware-test" / "probe_kld7_timing.py" +spec = importlib.util.spec_from_file_location("probe_kld7_timing", SCRIPT) +probe = importlib.util.module_from_spec(spec) +assert spec.loader is not None +sys.modules[spec.name] = probe +spec.loader.exec_module(probe) + + +class FakeSerial: + """Small serial fake that can return deliberately split chunks.""" + + def __init__(self, chunks): + self.chunks = list(chunks) + self.writes = [] + self.timeout = 0.2 + self.baudrate = 115200 + self.closed = False + + def read(self, n): + if not self.chunks: + return b"" + chunk = self.chunks.pop(0) + if len(chunk) > n: + self.chunks.insert(0, chunk[n:]) + return chunk[:n] + return chunk + + def write(self, data): + self.writes.append(data) + return len(data) + + def flush(self): + return None + + def reset_input_buffer(self): + return None + + def close(self): + self.closed = True + + +def test_build_packet_uppercases_command_and_packs_length(): + packet = probe.build_packet("gnfd", (0x21).to_bytes(4, "little")) + + assert packet == b"GNFD\x04\x00\x00\x00!\x00\x00\x00" + + +def test_validate_command_rejects_non_four_byte_command(): + assert probe.validate_probe_command("ABC", "") == "command must be exactly 4 ASCII characters" + + +def test_validate_command_rejects_non_uppercase_command(): + assert probe.validate_probe_command("test", "") == "command must be uppercase ASCII" + + +def test_validate_command_rejects_odd_hex_payload(): + assert ( + probe.validate_probe_command("TST1", "abc") + == "hex payload must have an even number of characters" + ) + + +def test_read_packet_handles_split_header_and_payload(): + payload = b"\x00\x01\x02\x03" + header = b"DONE" + len(payload).to_bytes(4, "little") + fake = FakeSerial([header[:3], header[3:8], payload[:1], payload[1:]]) + protocol = probe.KLD7Protocol.__new__(probe.KLD7Protocol) + protocol.port = fake + + packet = protocol.read_packet() + + assert packet.code == "DONE" + assert packet.payload == payload + assert packet.payload_bytes == 4 + + +def test_summarize_measurements_counts_done_gaps(): + packets = [ + probe.PacketRecord(code="DONE", payload_bytes=4, complete_monotonic=1.0, done_frame=10), + probe.PacketRecord( + code="RADC", payload_bytes=3072, complete_monotonic=1.1, read_duration_ms=10.0 + ), + probe.PacketRecord(code="DONE", payload_bytes=4, complete_monotonic=2.0, done_frame=12), + probe.PacketRecord( + code="RADC", payload_bytes=3072, complete_monotonic=2.1, read_duration_ms=20.0 + ), + ] + + summary = probe.summarize_packets(packets, duration_s=2.0) + + assert summary["radc_frames"] == 2 + assert summary["done_frames"] == 2 + assert summary["done_frame_gaps"] == 1 + assert summary["effective_radc_hz"] == 1.0 + assert summary["read_duration_ms_p95"] == 20.0 + + +def test_unsafe_probe_requires_output(): + parser = probe.build_parser() + args = parser.parse_args(["--port", "/dev/null", "--unsafe-probe", "--probe-command", "TEST"]) + + assert probe.validate_args(args) == [ + "--unsafe-probe requires --output so probe activity is auditable" + ] + + +def test_rfse_requires_factory_reset_flag(): + parser = probe.build_parser() + args = parser.parse_args( + [ + "--port", + "/dev/null", + "--output", + "/tmp/probe.jsonl", + "--unsafe-probe", + "--probe-command", + "RFSE", + ] + ) + + assert probe.validate_args(args) == ["RFSE is refused unless --allow-factory-reset is set"] + + +def test_parse_frame_mask_combines_known_flags(): + assert probe.parse_frame_mask("RADC,DONE") == 0x21 + + +def test_probe_command_requires_unsafe_probe(): + parser = probe.build_parser() + args = parser.parse_args(["--port", "/dev/null", "--probe-command", "TEST"]) + + assert probe.validate_args(args) == ["--probe-command requires --unsafe-probe"] + + +def test_rfse_allowed_when_factory_reset_flag_is_present(): + parser = probe.build_parser() + args = parser.parse_args( + [ + "--port", + "/dev/null", + "--output", + "/tmp/probe.jsonl", + "--unsafe-probe", + "--allow-factory-reset", + "--probe-command", + "RFSE", + ] + ) + + assert probe.validate_args(args) == [] + + +def test_send_command_records_response_code_and_written_packet(): + payload = b"\x00" + header = b"RESP" + len(payload).to_bytes(4, "little") + fake = FakeSerial([header, payload]) + protocol = probe.KLD7Protocol.__new__(probe.KLD7Protocol) + protocol.port = fake + + response = protocol.send_command("TEST", b"\x01\x02") + + assert fake.writes == [b"TEST\x02\x00\x00\x00\x01\x02"] + assert response.command == "TEST" + assert response.response_code == 0 + + +def test_parse_probe_command_decodes_hex_payload(): + assert probe.parse_probe_command("TEST:0102ff") == ("TEST", b"\x01\x02\xff") + + +def test_write_jsonl_strips_payload_bytes(tmp_path): + output = tmp_path / "probe.jsonl" + records = [probe.PacketRecord(code="RADC", payload_bytes=3072, payload=b"\x00\x01")] + summary = {"radc_frames": 1} + + probe.write_jsonl(output, records, summary) + + lines = output.read_text(encoding="utf-8").splitlines() + assert '"payload"' not in lines[0] + assert '"type": "packet"' in lines[0] + assert '"type": "summary"' in lines[1] diff --git a/tests/test_project_metadata.py b/tests/test_project_metadata.py index ab0e4ccae..c5f36110c 100644 --- a/tests/test_project_metadata.py +++ b/tests/test_project_metadata.py @@ -1,5 +1,6 @@ """Tests for packaging metadata that affects setup/install behavior.""" +import re from pathlib import Path try: @@ -12,6 +13,25 @@ def _pyproject() -> dict: return tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8")) +def _requirement_name(dependency: str) -> str: + """Extract the distribution name from a PEP 508 dependency string.""" + return re.split(r"[<>=!~ \[;]", dependency, maxsplit=1)[0].strip() + + +def test_kld7_is_installed_by_default(): + """K-LD7 driver must be a base dependency so every install path includes it. + + The package is a tiny pure-Python wheel whose only requirement (pyserial) is + already a base dependency, and the --kld7 runtime flag gates actual hardware + use. Shipping it by default avoids the recurring "kld7 package not installed" + failure on clean installs, since setup.sh, `uv sync`, and CI do not pull + optional extras. + """ + dependencies = _pyproject()["project"]["dependencies"] + + assert any(_requirement_name(dep) == "kld7" for dep in dependencies) + + def test_camera_dependencies_are_not_installed_by_default(): """Camera tracking packages should not be part of the base install.""" dependencies = _pyproject()["project"]["dependencies"] diff --git a/tests/test_rolling_buffer.py b/tests/test_rolling_buffer.py index 1c438c994..417d761ab 100644 --- a/tests/test_rolling_buffer.py +++ b/tests/test_rolling_buffer.py @@ -1,6 +1,7 @@ """Tests for rolling_buffer module.""" import math +import time from datetime import datetime from unittest.mock import MagicMock @@ -592,9 +593,96 @@ def test_sound_trigger_prefers_ops_clock_sync_for_trigger_timestamp(self): radar = MagicMock() radar.wait_for_hardware_trigger.return_value = '{"sample_time": 0.0}' radar.last_hardware_trigger_first_byte_timestamp = 12345.678 - radar.last_clock_sync = { + radar.last_clock_sync = None + radar.read_clock_sync.return_value = { + "source": "per_shot", + "samples": 3, + "valid_samples": 3, + "best_offset_s": 12000.0, + "clock_sync_method": "integer_rollover", + "usable_for_trigger_timestamps": True, + "rollover_uncertainty_ms": 10.0, + "reads": [ + { + "host_after": time.time(), + "host_mid": time.time(), + "radar_clock_s": 100.0, + "read_latency_ms": 1.0, + } + ], + } + + capture = IQCapture( + sample_time=100.000, + trigger_time=100.068, + i_samples=[2048] * 4096, + q_samples=[2048] * 4096, + ) + processor = MagicMock() + processor.parse_capture.return_value = capture + processor.process_standard.return_value = SpeedTimeline( + readings=[ + SpeedReading( + speed_mph=100.0, + magnitude=1000.0, + timestamp_ms=68.0, + direction="outbound", + ) + ], + sample_rate_hz=937.5, + ) + + trigger = SoundTrigger(pre_trigger_segments=12) + result = trigger.wait_for_trigger(radar, processor, timeout=1.0) + + assert result is capture + assert result.trigger_timestamp == pytest.approx(12100.068) + assert result.trigger_timestamp_source == "ops_clock_sync" + assert radar.last_clock_sync is radar.read_clock_sync.return_value + radar.read_clock_sync.assert_called_once_with(samples=36, store=False) + + def test_sound_trigger_uses_recent_previous_sync_when_fresh_sync_is_bad(self): + """A bad per-shot C? read should not discard a recent valid sync.""" + from openflight.rolling_buffer.trigger import SoundTrigger + + now = time.time() + radar = MagicMock() + radar.wait_for_hardware_trigger.return_value = '{"sample_time": 0.0}' + radar.last_hardware_trigger_first_byte_timestamp = 12345.678 + previous_sync = { + "source": "startup", + "samples": 3, + "valid_samples": 3, "best_offset_s": 12000.0, + "clock_sync_method": "integer_rollover", + "usable_for_trigger_timestamps": True, + "rollover_uncertainty_ms": 10.0, + "reads": [ + { + "host_after": now, + "host_mid": now, + "radar_clock_s": 100.0, + "read_latency_ms": 1.0, + } + ], + } + radar.last_clock_sync = previous_sync + radar.read_clock_sync.return_value = { + "source": "per_shot", + "samples": 4, + "valid_samples": 2, + "best_offset_s": 12100.0, + "clock_sync_method": "integer_rollover", "usable_for_trigger_timestamps": True, + "rollover_uncertainty_ms": 900.0, + "reads": [ + { + "host_after": now, + "host_mid": now, + "radar_clock_s": None, + "read_latency_ms": 200.0, + } + ], } capture = IQCapture( @@ -623,6 +711,7 @@ def test_sound_trigger_prefers_ops_clock_sync_for_trigger_timestamp(self): assert result is capture assert result.trigger_timestamp == pytest.approx(12100.068) assert result.trigger_timestamp_source == "ops_clock_sync" + assert radar.last_clock_sync is previous_sync def test_sound_trigger_ignores_unusable_ops_clock_sync(self): """Whole-second-only clock sync should not override first-byte timing.""" @@ -636,6 +725,14 @@ def test_sound_trigger_ignores_unusable_ops_clock_sync(self): "usable_for_trigger_timestamps": False, "clock_sync_method": "integer_unusable_no_rollover", } + radar.read_clock_sync.return_value = { + "samples": 1, + "valid_samples": 0, + "best_offset_s": None, + "usable_for_trigger_timestamps": False, + "clock_sync_method": "no_valid_reads", + "reads": [], + } capture = IQCapture( sample_time=100.000, @@ -2259,14 +2356,17 @@ def test_lower_rail_peak_rejected_with_toneless_envelope(self): ) if result.spin_rpm > 0: accepted.append((seed, result)) - # No accepted detection may sit at the bottom 5 bins of the seam - # search range (≈ 33-50 Hz, ≈ 2000-3000 RPM) — that whole region - # is dominated by envelope-DC leakage on real data. + # With the envelope detrend, 2000-3100 RPM is measurable for real + # tones — but toneless noise must never produce a *reliable* + # result there. Rail-zone picks additionally require medium SNR, + # which envelope noise does not reach. for seed, r in accepted: - assert r.spin_rpm > 3100, ( - f"seed={seed}: lower rail not rejected, got " - f"{r.spin_rpm} RPM (quality={r.quality}, snr={r.snr})" - ) + if r.spin_rpm <= 3100: + assert not r.is_reliable and r.confidence <= 0.5, ( + f"seed={seed}: noise accepted as reliable low spin, got " + f"{r.spin_rpm} RPM (quality={r.quality}, snr={r.snr}, " + f"confidence={r.confidence})" + ) def test_upper_rail_peak_rejected_with_toneless_envelope(self): """A toneless envelope must not be reported as ~12000 RPM @@ -2379,11 +2479,16 @@ def test_real_seam_modulation_still_passes(self): assert result.at_upper_rail is False, "Interior peak should not flag at_upper_rail" def test_lower_rail_candidate_confidence_is_capped(self): - """Lower-rail candidates may be visible but must not be reliable.""" + """Lower-rail candidates may be visible but must not be reliable. + + The rail zone is the lowest 3 live bins of the seam range + (~33-44 Hz, <=~2640 RPM) now that the detrend replaced the wide + leakage guard; 2300 RPM sits inside it. + """ processor = RollingBufferProcessor() i, q = self._amplitude_modulated_iq( base_speed_mph=160.0, - mod_freq_hz=3000.0 / 60.0, + mod_freq_hz=2300.0 / 60.0, modulation_depth=0.03, ) capture = IQCapture( @@ -2400,6 +2505,7 @@ def test_lower_rail_candidate_confidence_is_capped(self): ) assert result.spin_rpm > 0 + assert abs(result.spin_rpm - 2300) < 250 assert result.at_lower_rail is True assert result.confidence <= 0.5 assert result.is_reliable is False @@ -2518,7 +2624,8 @@ def test_phase_confirmation_accepts_matching_witness(self): assert witness is not None assert witness["confirmed"] is True - assert witness["method"] == "phase_residual" + # Either phase witness may win the ranking; both are valid confirmations. + assert witness["method"] in ("phase_residual", "instant_frequency") assert witness["rpm"] == pytest.approx(7031.25) assert witness["snr"] >= processor.SPIN_PHASE_SNR_MIN assert witness["agreement_pct"] <= processor.SPIN_PHASE_AGREEMENT_PCT @@ -2613,3 +2720,108 @@ def test_disconnect_stops_capture_thread(self): assert monitor._running is False, ( "disconnect() must call stop() so the capture thread shuts down" ) + + +class TestSpinWindowTrimming: + """The spin window must end where ball signal ends (net impact), not at + the end of the capture — the amplitude cliff plus dead air corrupts the + envelope FFT.""" + + def _make_capture_with_signal_loss( + self, + base_speed_mph=120, + spin_rpm=6000, + signal_end_ms=55.0, + modulation_depth=0.03, + sample_rate=30000, + num_samples=4096, + ): + """Ball tone with seam modulation that dies mid-capture (net impact).""" + rng = np.random.default_rng(7) + wavelength = 0.01243 + doppler_hz = 2 * (base_speed_mph / 2.23694) / wavelength + seam_hz = spin_rpm / 60.0 + + t = np.arange(num_samples) / sample_rate + phase = 2 * np.pi * doppler_hz * t + amplitude = 200 * (1.0 + modulation_depth * np.sin(2 * np.pi * seam_hz * t)) + i_signal = amplitude * np.cos(phase) + q_signal = amplitude * np.sin(phase) + + end_sample = min(int(signal_end_ms * sample_rate / 1000), num_samples) + i_signal[end_sample:] = rng.normal(0, 2, num_samples - end_sample) + q_signal[end_sample:] = rng.normal(0, 2, num_samples - end_sample) + + return IQCapture( + sample_time=0.0, + trigger_time=0.068, + i_samples=(i_signal + 2048).astype(int).clip(0, 4095).tolist(), + q_samples=(q_signal + 2048).astype(int).clip(0, 4095).tolist(), + ) + + def test_spin_correct_when_ball_signal_dies_mid_capture(self): + capture = self._make_capture_with_signal_loss(spin_rpm=6000, signal_end_ms=55.0) + processor = RollingBufferProcessor() + result = processor.detect_spin(capture, ball_speed_mph=120, ball_timestamp_ms=5.0) + assert result.spin_rpm > 0, f"Should detect spin, got: {result.rejection_reason}" + assert abs(result.spin_rpm - 6000) < 300, ( + f"Expected ~6000 RPM, got {result.spin_rpm} (quality={result.quality})" + ) + + def test_full_length_signal_is_not_truncated(self): + """A ball tone spanning the whole capture must keep its full window + (seam_cycles reflects the full ~126 ms analysis window).""" + capture = self._make_capture_with_signal_loss(spin_rpm=6000, signal_end_ms=10000.0) + processor = RollingBufferProcessor() + result = processor.detect_spin(capture, ball_speed_mph=120, ball_timestamp_ms=5.0) + assert result.spin_rpm > 0 + assert abs(result.spin_rpm - 6000) < 300 + # 6000 RPM = 100 Hz; >= 11 cycles requires >= ~110 ms of window + assert result.seam_cycles >= 11, ( + f"Window appears wrongly truncated: {result.seam_cycles} cycles" + ) + + +class TestSpinDriverDeadZone: + """Typical driver backspin (2000-3000 RPM) must be measurable: the old + DC-leakage guard zeroed 33-51.3 Hz (1980-3080 RPM) of the envelope FFT.""" + + def _tone_capture(self, spin_rpm, modulation_depth=0.03, decay=None): + sample_rate, num_samples = 30000, 4096 + wavelength = 0.01243 + doppler_hz = 2 * (160 / 2.23694) / wavelength + t = np.arange(num_samples) / sample_rate + phase = 2 * np.pi * doppler_hz * t + amplitude = np.full(num_samples, 200.0) + if spin_rpm: + amplitude *= 1.0 + modulation_depth * np.sin(2 * np.pi * (spin_rpm / 60.0) * t) + if decay is not None: + amplitude *= np.linspace(1.0, decay, num_samples) + return IQCapture( + sample_time=0.0, + trigger_time=0.068, + i_samples=(amplitude * np.cos(phase) + 2048).astype(int).clip(0, 4095).tolist(), + q_samples=(amplitude * np.sin(phase) + 2048).astype(int).clip(0, 4095).tolist(), + ) + + def test_low_driver_spin_measurable(self): + """2400 RPM (40 Hz) sits inside the old zeroed band.""" + processor = RollingBufferProcessor() + result = processor.detect_spin( + self._tone_capture(2400), ball_speed_mph=160, ball_timestamp_ms=5.0 + ) + assert result.spin_rpm > 0, f"Should detect spin, got: {result.rejection_reason}" + assert abs(result.spin_rpm - 2400) < 250, f"Expected ~2400 RPM, got {result.spin_rpm}" + + def test_range_falloff_decay_does_not_fake_driver_spin(self): + """A smoothly decaying envelope with no seam modulation (ball flying + away, no spin signal) must not produce a confident driver-band spin.""" + processor = RollingBufferProcessor() + result = processor.detect_spin( + self._tone_capture(spin_rpm=0, decay=0.4), + ball_speed_mph=160, + ball_timestamp_ms=5.0, + ) + assert result.spin_rpm == 0 or result.quality == "low", ( + f"Decay ramp faked spin: {result.spin_rpm} RPM quality={result.quality}" + ) diff --git a/tests/test_server.py b/tests/test_server.py index db36a7597..3e94acb3e 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -1978,3 +1978,159 @@ def reset(self): assert shot.angle_source == "radar" assert shot.launch_angle_vertical == pytest.approx(18.7) assert shot.launch_angle_horizontal == pytest.approx(0.0) + + +class TestCarryComputation: + """Tests for the ballistic carry path in on_shot_detected.""" + + def _patch_environment(self, monkeypatch): + monkeypatch.setattr(server_module, "kld7_vertical", None) + monkeypatch.setattr(server_module, "kld7_horizontal", None) + monkeypatch.setattr(server_module, "camera_tracker", None) + monkeypatch.setattr(server_module, "camera_enabled", False) + monkeypatch.setattr(server_module, "monitor", None) + monkeypatch.setattr(server_module, "debug_mode", False) + monkeypatch.setattr(server_module, "get_session_logger", lambda: None) + monkeypatch.setattr(server_module.socketio, "emit", lambda *args, **kwargs: None) + + def test_carry_uses_ballistic_simulator_when_launch_angle_present(self, monkeypatch): + """A shot with a vertical launch angle should get carry from the physics sim.""" + self._patch_environment(monkeypatch) + monkeypatch.setattr(server_module, "ballistics_enabled", True) + + captured = {} + + from openflight import ballistics as ballistics_module + real_simulate = ballistics_module.simulate + + def spying_simulate(conditions, *args, **kwargs): + captured["conditions"] = conditions + return real_simulate(conditions, *args, **kwargs) + + monkeypatch.setattr(server_module, "simulate", spying_simulate) + + shot = Shot( + ball_speed_mph=165.0, + club_speed_mph=112.0, + timestamp=datetime.now(), + club=ClubType.DRIVER, + launch_angle_vertical=11.0, + launch_angle_confidence=0.8, + spin_rpm=2700, + spin_confidence=0.85, + angle_source="radar", + ) + + on_shot_detected(shot) + + assert "conditions" in captured, "simulate() should have been called" + assert captured["conditions"].spin_source == "measured" + assert shot.carry_spin_adjusted is not None + assert 250 < shot.carry_spin_adjusted < 300 + + def test_carry_falls_back_to_table_when_resolve_returns_none(self, monkeypatch): + """When resolve_launch returns None, the table path should compute carry.""" + self._patch_environment(monkeypatch) + + monkeypatch.setattr(server_module, "resolve_launch", lambda shot: None) + + def fail_simulate(*args, **kwargs): + raise AssertionError("simulate() must not be called when resolve_launch is None") + + monkeypatch.setattr(server_module, "simulate", fail_simulate) + + shot = Shot( + ball_speed_mph=150.0, + club_speed_mph=105.0, + timestamp=datetime.now(), + club=ClubType.DRIVER, + launch_angle_vertical=12.0, + spin_rpm=2700, + spin_confidence=0.85, + angle_source="radar", + ) + + on_shot_detected(shot) + + assert shot.carry_spin_adjusted is not None + assert shot.carry_spin_adjusted > 0 + + def test_carry_skips_ballistic_when_ballistics_disabled(self, monkeypatch): + """When ballistics_enabled is False, the simulator must not run even + if a valid launch angle is present — carry falls through to the + table estimator. This is the default; `--ballistics` opts in.""" + self._patch_environment(monkeypatch) + monkeypatch.setattr(server_module, "ballistics_enabled", False) + + def fail_resolve(*args, **kwargs): + raise AssertionError("resolve_launch must not run when ballistics disabled") + + def fail_simulate(*args, **kwargs): + raise AssertionError("simulate() must not run when ballistics disabled") + + monkeypatch.setattr(server_module, "resolve_launch", fail_resolve) + monkeypatch.setattr(server_module, "simulate", fail_simulate) + + shot = Shot( + ball_speed_mph=165.0, + club_speed_mph=112.0, + timestamp=datetime.now(), + club=ClubType.DRIVER, + launch_angle_vertical=11.0, + launch_angle_confidence=0.8, + spin_rpm=2700, + spin_confidence=0.85, + angle_source="radar", + ) + + on_shot_detected(shot) + + assert shot.carry_spin_adjusted is not None + assert shot.carry_spin_adjusted > 0 + + +class TestApplyCalculatedSpin: + """Tests for the --calculated-spin shot rewrite.""" + + def _shot(self, la=18.0, la_source="radar", ball_speed=115.0, spin=6800.0): + return Shot( + ball_speed_mph=ball_speed, + timestamp=datetime.now(), + club=ClubType.IRON_7, + launch_angle_vertical=la, + launch_angle_vertical_source=la_source, + spin_rpm=spin, + spin_confidence=0.3, + spin_rejection_reason="SNR too low", + ) + + def test_rewrites_spin_when_launch_angle_measured(self): + shot = self._shot() + assert server_module._apply_calculated_spin(shot) is True + # 170 * 115 * sin(18deg)^1.2 ~= 4800 rpm + assert 4500 < shot.spin_rpm < 5100 + assert shot.spin_rpm_measured == 6800.0 + assert shot.spin_source == "calculated" + assert shot.spin_confidence == pytest.approx(0.7) + assert shot.spin_rejection_reason is None + + def test_untouched_when_launch_angle_estimated(self): + shot = self._shot(la_source="estimated") + assert server_module._apply_calculated_spin(shot) is False + assert shot.spin_rpm == 6800.0 + assert shot.spin_source is None + + def test_untouched_when_no_launch_angle(self): + shot = self._shot(la=None) + assert server_module._apply_calculated_spin(shot) is False + assert shot.spin_rpm == 6800.0 + + def test_untouched_when_launch_angle_outside_model_range(self): + shot = self._shot(la=1.0) + assert server_module._apply_calculated_spin(shot) is False + assert shot.spin_rpm == 6800.0 + + def test_camera_launch_angle_accepted(self): + shot = self._shot(la_source="camera") + assert server_module._apply_calculated_spin(shot) is True + assert shot.spin_source == "calculated" diff --git a/tests/test_session_logger.py b/tests/test_session_logger.py index 8ad87b88f..1ec574daf 100644 --- a/tests/test_session_logger.py +++ b/tests/test_session_logger.py @@ -500,3 +500,36 @@ def test_clock_sync_disabled_skips_write(self, tmp_path): logger = SessionLogger(log_dir=tmp_path, enabled=False) logger.log_clock_sync(device="ops243", port="x", summary=self._summary()) assert logger.session_path is None + + +class TestSessionIdentity: + """session_start must carry a globally unique ID and format version so + cloud sync can dedupe sessions by content, not filename.""" + + def _start_entry(self, tmp_path): + logger = SessionLogger(log_dir=tmp_path, enabled=True) + logger.start_session(mode="rolling-buffer", trigger_type="sound") + logger.end_session() + session_file = next(tmp_path.glob("session_*.jsonl")) + with session_file.open() as handle: + first = json.loads(handle.readline()) + return first + + def test_session_start_has_uuid_and_format_version(self, tmp_path): + import uuid + + import openflight + + entry = self._start_entry(tmp_path) + assert entry["type"] == "session_start" + # Valid UUID4, distinct from the timestamp-based session_id + parsed = uuid.UUID(entry["session_uuid"]) + assert parsed.version == 4 + assert entry["session_uuid"] != entry["session_id"] + assert entry["format_version"] == 1 + assert entry["app_version"] == openflight.__version__ + + def test_session_uuid_is_unique_per_session(self, tmp_path): + first = self._start_entry(tmp_path / "a") + second = self._start_entry(tmp_path / "b") + assert first["session_uuid"] != second["session_uuid"] diff --git a/tests/test_speed_correction.py b/tests/test_speed_correction.py new file mode 100644 index 000000000..e339b5262 --- /dev/null +++ b/tests/test_speed_correction.py @@ -0,0 +1,55 @@ +"""Tests for the ball-speed cosine correction.""" + +import pytest + +from openflight.speed_correction import correct_ball_speed, radial_speed_factor + +D_FT = 5.0 +H_FT = -4.0 / 12.0 + + +class TestRadialSpeedFactor: + def test_factor_below_one_for_lofted_launch(self): + # A ball departing upward always reads slow on a low radar + f = radial_speed_factor(19.0, 110.0, D_FT, H_FT) + assert 0.95 < f < 1.0 + + def test_typical_iron_compression_matches_observed_bias(self): + # The validated datasets showed ~2.1-2.6 mph of compression at + # iron speeds (~2-2.5% of ball speed) + f = radial_speed_factor(19.0, 108.0, D_FT, H_FT) + compression_mph = 108.0 * (1.0 - f) + assert 1.5 < compression_mph < 3.5 + + def test_higher_launch_compresses_more(self): + f_wedge = radial_speed_factor(30.0, 90.0, D_FT, H_FT) + f_iron = radial_speed_factor(17.0, 110.0, D_FT, H_FT) + f_driver = radial_speed_factor(11.0, 150.0, D_FT, H_FT) + assert f_wedge < f_iron < f_driver + + def test_farther_tee_compresses_more(self): + # LOS flattens with distance while the velocity stays pitched up + assert radial_speed_factor(18.0, 110.0, 6.5, H_FT) < radial_speed_factor( + 18.0, 110.0, 5.0, H_FT + ) + + def test_zero_launch_is_nearly_uncorrected(self): + f = radial_speed_factor(0.0, 110.0, D_FT, H_FT) + assert f > 0.995 + + def test_degenerate_inputs_clamp(self): + assert radial_speed_factor(19.0, 0.0, D_FT, H_FT) == 1.0 + assert 0.5 <= radial_speed_factor(44.0, 60.0, D_FT, H_FT) <= 1.0 + + +class TestCorrectBallSpeed: + def test_correction_raises_speed(self): + corrected = correct_ball_speed(108.0, 19.0, D_FT, H_FT) + assert corrected > 108.0 + assert corrected == pytest.approx(110.3, abs=0.8) + + def test_roundtrip_consistency(self): + # Correcting then re-deriving the radial reading lands back + true_speed = correct_ball_speed(108.0, 19.0, D_FT, H_FT) + radial = true_speed * radial_speed_factor(19.0, true_speed, D_FT, H_FT) + assert radial == pytest.approx(108.0, abs=0.15) diff --git a/tests/test_spin_estimate.py b/tests/test_spin_estimate.py new file mode 100644 index 000000000..d46898cac --- /dev/null +++ b/tests/test_spin_estimate.py @@ -0,0 +1,50 @@ +"""Tests for the kinematic calculated-spin model.""" + +import math + +import pytest + +from openflight.spin_estimate import ( + MAX_SPIN_RPM, + SPIN_COEFF_RPM_PER_MPH, + SPIN_LA_EXPONENT, + calculated_spin_rpm, +) + + +class TestCalculatedSpin: + def test_seven_iron_regime(self): + """115 mph at 17.8 deg (TrackMan 7i medians) lands in 7i spin range.""" + spin = calculated_spin_rpm(115.0, 17.8) + assert spin == pytest.approx( + SPIN_COEFF_RPM_PER_MPH * 115.0 * math.sin(math.radians(17.8)) ** SPIN_LA_EXPONENT + ) + assert 3500 <= spin <= 5500 + + def test_wedge_spins_more_than_long_iron(self): + """At realistic speed/LA pairs, wedges out-spin long irons.""" + wedge = calculated_spin_rpm(92.0, 28.0) + long_iron = calculated_spin_rpm(130.0, 12.0) + assert wedge > long_iron + + def test_monotonic_in_launch_angle(self): + spins = [calculated_spin_rpm(110.0, la) for la in (10, 15, 20, 25, 30)] + assert spins == sorted(spins) + + def test_monotonic_in_ball_speed(self): + spins = [calculated_spin_rpm(v, 20.0) for v in (80, 100, 120, 140)] + assert spins == sorted(spins) + + def test_capped_at_physical_maximum(self): + assert calculated_spin_rpm(200.0, 55.0) == MAX_SPIN_RPM + + def test_none_outside_calibrated_launch_range(self): + assert calculated_spin_rpm(110.0, 1.0) is None + assert calculated_spin_rpm(110.0, -5.0) is None + assert calculated_spin_rpm(110.0, 65.0) is None + + def test_none_for_missing_or_invalid_inputs(self): + assert calculated_spin_rpm(None, 20.0) is None + assert calculated_spin_rpm(110.0, None) is None + assert calculated_spin_rpm(0.0, 20.0) is None + assert calculated_spin_rpm(-10.0, 20.0) is None diff --git a/ui/README.md b/ui/README.md index c987b941e..f6e89f241 100644 --- a/ui/README.md +++ b/ui/README.md @@ -1,73 +1,93 @@ -# React + TypeScript + Vite +# OpenFlight UI -This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules. +The OpenFlight dashboard: a React + TypeScript + Vite app that connects to the +backend over `socket.io` and renders live shot data, session stats, camera ball +detection, and a screen-mounted display mode. -Currently, two official plugins are available: +This README covers frontend development. For the hardware, the radar pipeline, +and how the whole system fits together, see the [root README](../README.md). -- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react) uses [Babel](https://babeljs.io/) (or [oxc](https://oxc.rs) when used in [rolldown-vite](https://vite.dev/guide/rolldown)) for Fast Refresh -- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh +## Quick start -## React Compiler +You need Node 20+ and a running OpenFlight backend. To start a backend without +hardware, run `scripts/start-kiosk.sh --mock` from the repo root (see the +[root README](../README.md#getting-started)). -The React Compiler is not enabled on this template because of its impact on dev & build performances. To add it, see [this documentation](https://react.dev/learn/react-compiler/installation). +```bash +npm install +npm run dev +``` + +The dev server runs on port `5173`. When served there, the UI assumes the +backend is at `http://localhost:8080`. Point it elsewhere with +`VITE_SOCKET_URL`: -## Expanding the ESLint configuration +```bash +VITE_SOCKET_URL="http://localhost:8081" npm run dev +``` -If you are developing a production application, we recommend updating the configuration to enable type-aware lint rules: +## Scripts -```js -export default defineConfig([ - globalIgnores(['dist']), - { - files: ['**/*.{ts,tsx}'], - extends: [ - // Other configs... +| Script | Description | +| ---------------------- | -------------------------------------------- | +| `npm run dev` | Dev server with hot reload | +| `npm run build` | Type-check and build the production bundle | +| `npm run preview` | Serve the production build locally | +| `npm run lint` | ESLint | +| `npm run test` | Vitest unit tests | +| `npm run format` | Format `src/` with Prettier | +| `npm run format:check` | Check formatting without writing | - // Remove tseslint.configs.recommended and replace with this - tseslint.configs.recommendedTypeChecked, - // Alternatively, use this for stricter rules - tseslint.configs.strictTypeChecked, - // Optionally, add this for stylistic rules - tseslint.configs.stylisticTypeChecked, +## How the UI connects - // Other configs... - ], - languageOptions: { - parserOptions: { - project: ['./tsconfig.node.json', './tsconfig.app.json'], - tsconfigRootDir: import.meta.dirname, - }, - // other options... - }, - }, -]); -``` +The app is entirely client-side. Everything flows through one socket connection. + +- **`utils/serverOrigin.ts`** resolves the backend origin: `VITE_SOCKET_URL` if + set, otherwise `http://localhost:8080` when running on the Vite dev port + (`5173`), otherwise the page's own origin (the production case, where the + backend serves the built UI). +- **`hooks/useSocket.ts`** owns the connection. It receives events like `shot`, + `session_state`, `camera_status`, `ball_detection`, and `trigger_status`, and + sends commands like `set_club`, `clear_session`, `simulate_shot`, and + `toggle_camera`. It's the source of truth for the event contract — read it + before assuming what the backend emits. +- **State** lives in `state/` (shot history and unit preferences) via React + context providers. +- **Shutdown** posts to `/api/shutdown` to stop the connected backend. + +**Display mode** lives at `/display`: a compact, fullscreen-friendly dashboard +for mounted screens and TVs. The [root README](../README.md#tv-display-mode) +covers casting it. + +**Launch Daddy** is a hidden mode toggled by a tap area in the header. When on, +new shots can fire an animated overlay. -You can also install [eslint-plugin-react-x](https://github.com/Rel1cx/eslint-react/tree/main/packages/plugins/eslint-plugin-react-x) and [eslint-plugin-react-dom](https://github.com/Rel1cx/eslint-react/tree/main/packages/plugins/eslint-plugin-react-dom) for React-specific lint rules: - -```js -// eslint.config.js -import reactX from 'eslint-plugin-react-x'; -import reactDom from 'eslint-plugin-react-dom'; - -export default defineConfig([ - globalIgnores(['dist']), - { - files: ['**/*.{ts,tsx}'], - extends: [ - // Other configs... - // Enable lint rules for React - reactX.configs['recommended-typescript'], - // Enable lint rules for React DOM - reactDom.configs.recommended, - ], - languageOptions: { - parserOptions: { - project: ['./tsconfig.node.json', './tsconfig.app.json'], - tsconfigRootDir: import.meta.dirname, - }, - // other options... - }, - }, -]); +## Project layout + +A few files do most of the work. This is illustrative, not exhaustive — +components carry co-located `.css` and `.test.tsx` files. + +```text +src/ + App.tsx # navigation, view selection, display routing + main.tsx # entry point + hooks/useSocket.ts # socket connection, events, backend commands + utils/serverOrigin.ts # backend origin resolution + state/ # shot history + unit preference context + components/ # CameraFeed, ShotDisplay, StatsView, DebugPanel, … + LaunchDaddy/ # the hidden overlay mode ``` + +## Troubleshooting + +**Socket won't connect.** Confirm the backend is running and reachable from the +browser. If it isn't on the default port, set `VITE_SOCKET_URL`. Connection logs +come from `hooks/useSocket.ts`. + +**Build fails.** `npm run build` surfaces TypeScript and bundling errors; `npm +run lint` catches the rest. + +--- + +Contributing guidelines (setup, code quality, PRs) live in +[CONTRIBUTING.md](../CONTRIBUTING.md). diff --git a/ui/package-lock.json b/ui/package-lock.json index 44d2c2073..9023f87d4 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -14,14 +14,14 @@ }, "devDependencies": { "@eslint/js": "^9.39.1", - "@types/node": "^25.6.0", + "@types/node": "^25.9.3", "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", "@vitejs/plugin-react": "^5.1.1", "eslint": "^9.39.1", - "eslint-plugin-react-hooks": "^7.0.1", + "eslint-plugin-react-hooks": "^7.1.1", "eslint-plugin-react-refresh": "^0.4.24", - "globals": "^17.5.0", + "globals": "^17.6.0", "prettier": "^3.8.3", "typescript": "~6.0.3", "typescript-eslint": "^8.46.4", @@ -1418,13 +1418,13 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "25.6.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-25.6.0.tgz", - "integrity": "sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ==", + "version": "25.9.3", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.9.3.tgz", + "integrity": "sha512-603BddQMv3pUcr4U2dhujk83N2tTDVr/34wII2B6bJy6g+8WD6yUb11jszNs0gdi4PesVWl7ABt8nYMVpnLUcg==", "dev": true, "license": "MIT", "dependencies": { - "undici-types": "~7.19.0" + "undici-types": ">=7.24.0 <7.24.7" } }, "node_modules/@types/react": { @@ -2328,9 +2328,9 @@ } }, "node_modules/eslint-plugin-react-hooks": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-7.0.1.tgz", - "integrity": "sha512-O0d0m04evaNzEPoSW+59Mezf8Qt0InfgGIBJnpC0h3NH/WjUAR7BIKUfysC6todmtiZ/A0oUVS8Gce0WhBrHsA==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-7.1.1.tgz", + "integrity": "sha512-f2I7Gw6JbvCexzIInuSbZpfdQ44D7iqdWX01FKLvrPgqxoE7oMj8clOfto8U6vYiz4yd5oKu39rRSVOe1zRu0g==", "dev": true, "license": "MIT", "dependencies": { @@ -2344,7 +2344,7 @@ "node": ">=18" }, "peerDependencies": { - "eslint": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0-0 || ^9.0.0" + "eslint": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0-0 || ^9.0.0 || ^10.0.0" } }, "node_modules/eslint-plugin-react-refresh": { @@ -2600,9 +2600,9 @@ } }, "node_modules/globals": { - "version": "17.5.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-17.5.0.tgz", - "integrity": "sha512-qoV+HK2yFl/366t2/Cb3+xxPUo5BuMynomoDmiaZBIdbs+0pYbjfZU+twLhGKp4uCZ/+NbtpVepH5bGCxRyy2g==", + "version": "17.6.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-17.6.0.tgz", + "integrity": "sha512-sepffkT8stwnIYbsMBpoCHJuJM5l98FUF2AnE07hfvE0m/qp3R586hw4jF4uadbhvg1ooIdzuu7CsfD2jzCaNA==", "dev": true, "license": "MIT", "engines": { @@ -3411,9 +3411,9 @@ } }, "node_modules/undici-types": { - "version": "7.19.2", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.19.2.tgz", - "integrity": "sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg==", + "version": "7.24.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", "dev": true, "license": "MIT" }, diff --git a/ui/package.json b/ui/package.json index 54bbc7c45..e722638e9 100644 --- a/ui/package.json +++ b/ui/package.json @@ -19,14 +19,14 @@ }, "devDependencies": { "@eslint/js": "^9.39.1", - "@types/node": "^25.6.0", + "@types/node": "^25.9.3", "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", "@vitejs/plugin-react": "^5.1.1", "eslint": "^9.39.1", - "eslint-plugin-react-hooks": "^7.0.1", + "eslint-plugin-react-hooks": "^7.1.1", "eslint-plugin-react-refresh": "^0.4.24", - "globals": "^17.5.0", + "globals": "^17.6.0", "prettier": "^3.8.3", "typescript": "~6.0.3", "typescript-eslint": "^8.46.4", diff --git a/ui/src/components/DisplayMode.test.tsx b/ui/src/components/DisplayMode.test.tsx index cab8891ec..3edbe1522 100644 --- a/ui/src/components/DisplayMode.test.tsx +++ b/ui/src/components/DisplayMode.test.tsx @@ -32,6 +32,7 @@ const shot: Shot = { spin_rpm: 2450, spin_confidence: 0.8, spin_quality: 'high', + spin_source: 'calculated', carry_spin_adjusted: 261, }; diff --git a/ui/src/components/ShotDisplay.tsx b/ui/src/components/ShotDisplay.tsx index d43f889f9..b08e4ed2d 100644 --- a/ui/src/components/ShotDisplay.tsx +++ b/ui/src/components/ShotDisplay.tsx @@ -234,6 +234,13 @@ export function ShotDisplay({ shot, animate = false }: ShotDisplayProps) { value={hasSpin ? formatSpinRpm(shot.spin_rpm!) : '—'} unit={hasSpin ? 'rpm' : undefined} label="Spin Rate" + subtext={ + hasSpin && shot.spin_source + ? shot.spin_source === 'calculated' + ? 'estimated' + : 'radar' + : undefined + } variant="spin" confidence={hasSpin ? shot.spin_quality : null} /> diff --git a/ui/src/types/shot.ts b/ui/src/types/shot.ts index adf575712..bb0deec61 100644 --- a/ui/src/types/shot.ts +++ b/ui/src/types/shot.ts @@ -19,6 +19,7 @@ export interface Shot { spin_rpm: number | null; spin_confidence: number | null; spin_quality: 'high' | 'medium' | 'low' | null; + spin_source: 'measured' | 'calculated' | null; carry_spin_adjusted: number | null; }