diff --git a/.claude/settings.json b/.claude/settings.json index aa06f43dc..e2a0ade2f 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -5,3 +5,4 @@ "playwright@claude-plugins-official": true } } + diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml index b5e8cfd4d..211410e9b 100644 --- a/.github/workflows/claude-code-review.yml +++ b/.github/workflows/claude-code-review.yml @@ -31,6 +31,15 @@ jobs: with: fetch-depth: 1 + # uv is NOT preinstalled on the GitHub runner image. Without this step + # every `uv` command fails with "command not found", regardless of which + # tools are allowed below. + - name: Install uv + uses: astral-sh/setup-uv@v10.0.1 + with: + enable-cache: true + working-directory: backend + - name: Run Claude Code Review id: claude-review uses: anthropics/claude-code-action@v1 @@ -38,7 +47,11 @@ jobs: claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} plugin_marketplaces: 'https://github.com/anthropics/claude-code.git' plugins: 'code-review@claude-code-plugins' - prompt: '/code-review:code-review ${{ github.repository }}/pull/${{ github.event.pull_request.number }}' + prompt: '/code-review:code-review --comment ${{ github.repository }}/pull/${{ github.event.pull_request.number }}' + # Bash(cd:*),Bash(uv:*) let the review actually run the test suite + # instead of reasoning about the code statically. + claude_args: | + --allowedTools "mcp__github_inline_comment__create_inline_comment,Bash(cd:*),Bash(uv:*)" # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md # or https://code.claude.com/docs/en/cli-reference for available options diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index d300267f1..482017eb9 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -19,7 +19,7 @@ jobs: (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude'))) runs-on: ubuntu-latest permissions: - contents: read + contents: write # Lets Claude commit and push fixes, not just read pull-requests: read issues: read id-token: write @@ -30,6 +30,15 @@ jobs: with: fetch-depth: 1 + # uv is NOT preinstalled on the GitHub runner image. Without this step + # every `uv` command fails with "command not found", regardless of which + # tools are allowed below. + - name: Install uv + uses: astral-sh/setup-uv@v10.0.1 + with: + enable-cache: true + working-directory: backend + - name: Run Claude Code id: claude uses: anthropics/claude-code-action@v1 @@ -43,8 +52,9 @@ jobs: # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it. # prompt: 'Update the pull request description to include a summary of changes.' - # Optional: Add claude_args to customize behavior and configuration - # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md - # or https://code.claude.com/docs/en/cli-reference for available options - # claude_args: '--allowed-tools Bash(gh pr:*)' - + # Lets Claude actually run the test suite. Without these, Bash is + # unavailable in a non-interactive run and there is no human to + # approve it, so Claude can only review statically. + # `cd` is needed because the uv project lives in backend/. + claude_args: | + --allowedTools "Bash(cd:*),Bash(uv:*)" diff --git a/.gitignore b/.gitignore index b7faf403d..1a88a7dd1 100644 --- a/.gitignore +++ b/.gitignore @@ -205,3 +205,8 @@ cython_debug/ marimo/_static/ marimo/_lsp/ __marimo__/ + +# FinAlly +db/*.db +db/*.db-wal +db/*.db-shm diff --git a/README.md b/README.md index 3f2582ae2..e2751380d 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ cp .env.example .env # Run with Docker docker build -t finally . -docker run -v finally-data:/app/db -p 8000:8000 --env-file .env finally +docker run -v "$(pwd)/db:/app/db" -p 8000:8000 --env-file .env finally # Open http://localhost:8000 ``` diff --git a/backend/CLAUDE.md b/backend/CLAUDE.md index 612ff18f5..a58a3e696 100644 --- a/backend/CLAUDE.md +++ b/backend/CLAUDE.md @@ -24,7 +24,8 @@ from app.market import PriceCache, PriceUpdate, MarketDataSource, create_market_ - `get(ticker) -> PriceUpdate | None` - `get_price(ticker) -> float | None` - `get_all() -> dict[str, PriceUpdate]` - - `remove(ticker)` + - `get_history(ticker, limit=HISTORY_MAXLEN) -> list[tuple[float, float]]` — rolling in-memory `(timestamp, price)` points, oldest-first, capped at `HISTORY_MAXLEN` (600, ~5 min at the simulator's 500ms cadence). Empty list for an untracked ticker. + - `remove(ticker)` — also clears that ticker's history - `version` property — monotonic counter, increments on every update (for SSE change detection) - **`MarketDataSource`** — Abstract interface implemented by `SimulatorDataSource` and `MassiveDataSource`. Lifecycle: `start(tickers)` -> `add_ticker()` / `remove_ticker()` -> `stop()`. @@ -40,6 +41,26 @@ router = create_stream_router(price_cache) # Returns FastAPI APIRouter # Endpoint: GET /api/stream/prices (text/event-stream) ``` +Pushes the entire price cache as one JSON object (map keyed by ticker) whenever +`PriceCache.version` changes, polled every 500ms. A connecting client always gets +a full snapshot immediately, including after a reconnect. When the version hasn't +changed for 15s (`KEEPALIVE_SECONDS`), an SSE comment line (`: ping`) is sent so +proxies and the frontend's connection indicator don't mistake a quiet market for +a dead connection. + +### Price History + +```python +from app.market import create_history_router + +router = create_history_router(price_cache) # Returns FastAPI APIRouter +# Endpoint: GET /api/prices/{ticker}/history?limit=600 +``` + +Backs the main chart's initial backfill from `PriceCache`'s rolling in-memory +history. Not persisted — a restart clears it, matching the simulator's own +reset-to-seed behavior. + ### Seed Data Default tickers: AAPL, GOOGL, MSFT, AMZN, TSLA, NVDA, META, JPM, V, NFLX. Seed prices and per-ticker volatility/drift params are in `app/market/seed_prices.py`. diff --git a/backend/app/market/__init__.py b/backend/app/market/__init__.py index 57ad0a121..c8878b9df 100644 --- a/backend/app/market/__init__.py +++ b/backend/app/market/__init__.py @@ -2,22 +2,25 @@ Public API: PriceUpdate - Immutable price snapshot dataclass - PriceCache - Thread-safe in-memory price store + PriceCache - Thread-safe in-memory price store (latest price + rolling history) MarketDataSource - Abstract interface for data providers create_market_data_source - Factory that selects simulator or Massive - create_stream_router - FastAPI router factory for SSE endpoint + create_stream_router - FastAPI router factory for the SSE endpoint + create_history_router - FastAPI router factory for the price history endpoint """ -from .cache import PriceCache +from .cache import HISTORY_MAXLEN, PriceCache from .factory import create_market_data_source from .interface import MarketDataSource from .models import PriceUpdate -from .stream import create_stream_router +from .stream import create_history_router, create_stream_router __all__ = [ "PriceUpdate", "PriceCache", + "HISTORY_MAXLEN", "MarketDataSource", "create_market_data_source", "create_stream_router", + "create_history_router", ] diff --git a/backend/app/market/cache.py b/backend/app/market/cache.py index 4d0215778..fe71eaa3f 100644 --- a/backend/app/market/cache.py +++ b/backend/app/market/cache.py @@ -3,20 +3,29 @@ from __future__ import annotations import time +from collections import deque from threading import Lock from .models import PriceUpdate +# ~5 minutes of history at the 500ms simulator cadence. Under Massive's 15s +# poll interval this fills much more slowly (one point per poll), which is +# correct: the chart is sparse but accurate rather than padded with guesses. +HISTORY_MAXLEN = 600 + class PriceCache: """Thread-safe in-memory cache of the latest price for each ticker. Writers: SimulatorDataSource or MassiveDataSource (one at a time). - Readers: SSE streaming endpoint, portfolio valuation, trade execution. + Readers: SSE streaming endpoint, portfolio valuation, trade execution, + the price history endpoint. """ - def __init__(self) -> None: + def __init__(self, history_maxlen: int = HISTORY_MAXLEN) -> None: self._prices: dict[str, PriceUpdate] = {} + self._history: dict[str, deque[tuple[float, float]]] = {} + self._history_maxlen = history_maxlen self._lock = Lock() self._version: int = 0 # Monotonically increasing; bumped on every update @@ -25,9 +34,10 @@ def update(self, ticker: str, price: float, timestamp: float | None = None) -> P Automatically computes direction and change from the previous price. If this is the first update for the ticker, previous_price == price (direction='flat'). + Also appends to the ticker's rolling history (see get_history()). """ with self._lock: - ts = timestamp or time.time() + ts = timestamp if timestamp is not None else time.time() prev = self._prices.get(ticker) previous_price = prev.price if prev else price @@ -38,6 +48,13 @@ def update(self, ticker: str, price: float, timestamp: float | None = None) -> P timestamp=ts, ) self._prices[ticker] = update + + history = self._history.get(ticker) + if history is None: + history = deque(maxlen=self._history_maxlen) + self._history[ticker] = history + history.append((ts, update.price)) + self._version += 1 return update @@ -60,11 +77,25 @@ def remove(self, ticker: str) -> None: """Remove a ticker from the cache (e.g., when removed from watchlist).""" with self._lock: self._prices.pop(ticker, None) + self._history.pop(ticker, None) + + def get_history(self, ticker: str, limit: int = HISTORY_MAXLEN) -> list[tuple[float, float]]: + """Oldest-first (timestamp, price) points for a ticker. + + Returns an empty list for an untracked ticker rather than raising, so + callers (e.g. the chart) can draw nothing instead of erroring. + """ + with self._lock: + points = self._history.get(ticker) + if not points: + return [] + return list(points)[-limit:] @property def version(self) -> int: """Current version counter. Useful for SSE change detection.""" - return self._version + with self._lock: + return self._version def __len__(self) -> int: with self._lock: diff --git a/backend/app/market/massive_client.py b/backend/app/market/massive_client.py index 00bc7b2aa..c3b4a75e3 100644 --- a/backend/app/market/massive_client.py +++ b/backend/app/market/massive_client.py @@ -4,8 +4,10 @@ import asyncio import logging +import time from massive import RESTClient +from massive.exceptions import AuthError, BadResponse from massive.rest.models import SnapshotMarketType from .cache import PriceCache @@ -13,6 +15,9 @@ logger = logging.getLogger(__name__) +# Snapshot last_trade.sip_timestamp is Unix nanoseconds; PriceCache wants seconds. +NANOS_PER_SECOND = 1_000_000_000 + class MassiveDataSource(MarketDataSource): """MarketDataSource backed by the Massive (Polygon.io) REST API. @@ -37,6 +42,13 @@ def __init__( self._tickers: list[str] = [] self._task: asyncio.Task | None = None self._client: RESTClient | None = None + # Flipped to False if the poll loop dies (e.g. a revoked API key + # raising AuthError after start() already succeeded). Nothing awaits + # self._task until stop(), so without this the failure would only + # ever surface as an unretrieved-exception log at GC time. A future + # GET /api/health can report `market_source` as degraded by reading + # this flag. + self._healthy = True async def start(self, tickers: list[str]) -> None: self._client = RESTClient(api_key=self._api_key) @@ -45,7 +57,9 @@ async def start(self, tickers: list[str]) -> None: # Do an immediate first poll so the cache has data right away await self._poll_once() + self._healthy = True self._task = asyncio.create_task(self._poll_loop(), name="massive-poller") + self._task.add_done_callback(self._on_poll_task_done) logger.info( "Massive poller started: %d tickers, %.1fs interval", len(tickers), @@ -78,8 +92,31 @@ async def remove_ticker(self, ticker: str) -> None: def get_tickers(self) -> list[str]: return list(self._tickers) + @property + def is_healthy(self) -> bool: + """False once the poll loop has died from an unhandled exception. + + A deliberate stop() (which cancels the task) never flips this. + """ + return self._healthy + # --- Internal --- + def _on_poll_task_done(self, task: asyncio.Task) -> None: + """Surface a dead poll loop loudly instead of an easy-to-miss + "Task exception was never retrieved" log at garbage-collection time. + """ + if task.cancelled(): + return + exc = task.exception() + if exc is not None: + self._healthy = False + logger.critical( + "Massive poller task died unexpectedly, live prices are now frozen: %s", + exc, + exc_info=exc, + ) + async def _poll_loop(self) -> None: """Poll on interval. First poll already happened in start().""" while True: @@ -95,30 +132,42 @@ async def _poll_once(self) -> None: # The Massive RESTClient is synchronous — run in a thread to # avoid blocking the event loop. snapshots = await asyncio.to_thread(self._fetch_snapshots) - processed = 0 - for snap in snapshots: - try: - price = snap.last_trade.price - # Massive timestamps are Unix milliseconds → convert to seconds - timestamp = snap.last_trade.timestamp / 1000.0 - self._cache.update( - ticker=snap.ticker, - price=price, - timestamp=timestamp, - ) - processed += 1 - except (AttributeError, TypeError) as e: - logger.warning( - "Skipping snapshot for %s: %s", - getattr(snap, "ticker", "???"), - e, - ) - logger.debug("Massive poll: updated %d/%d tickers", processed, len(self._tickers)) - - except Exception as e: - logger.error("Massive poll failed: %s", e) - # Don't re-raise — the loop will retry on the next interval. - # Common failures: 401 (bad key), 429 (rate limit), network errors. + except AuthError: + logger.error("Massive API key rejected — the source does not fall back automatically") + raise # unrecoverable: do not retry on a loop + except BadResponse as e: + logger.warning("Massive returned an error response: %s", e) + return # transient: retry next interval + except Exception: + logger.exception("Massive poll failed") + return + + processed = self._apply_snapshots(snapshots) + logger.debug("Massive poll: updated %d/%d tickers", processed, len(self._tickers)) + + def _apply_snapshots(self, snapshots: list) -> int: + """Write snapshot data into the cache. Returns the number of tickers updated. + + Extracted from _poll_once so it can be tested directly against real + `TickerSnapshot` objects (built via `TickerSnapshot.from_dict(...)`) + instead of mocks that would silently accept a misspelled attribute. + """ + processed = 0 + for snap in snapshots: + trade = snap.last_trade + if trade is None or trade.price is None: + # No trade yet today (pre-market, or an unrecognized symbol + # that still made it into the response) — leave it as "—". + continue + self._cache.update( + ticker=snap.ticker, + price=trade.price, + timestamp=( + trade.sip_timestamp / NANOS_PER_SECOND if trade.sip_timestamp else time.time() + ), + ) + processed += 1 + return processed def _fetch_snapshots(self) -> list: """Synchronous call to the Massive REST API. Runs in a thread.""" diff --git a/backend/app/market/stream.py b/backend/app/market/stream.py index 7fd974b7c..b7d7196bb 100644 --- a/backend/app/market/stream.py +++ b/backend/app/market/stream.py @@ -5,23 +5,30 @@ import asyncio import json import logging +import time from collections.abc import AsyncGenerator from fastapi import APIRouter, Request from fastapi.responses import StreamingResponse -from .cache import PriceCache +from .cache import HISTORY_MAXLEN, PriceCache logger = logging.getLogger(__name__) -router = APIRouter(prefix="/api/stream", tags=["streaming"]) +# How long the cache version can go unchanged before we send an SSE comment +# line to keep the connection (and proxies in between) from timing it out. +KEEPALIVE_SECONDS = 15.0 def create_stream_router(price_cache: PriceCache) -> APIRouter: """Create the SSE streaming router with a reference to the price cache. - This factory pattern lets us inject the PriceCache without globals. + This factory pattern lets us inject the PriceCache without globals. A + fresh APIRouter is built on every call so that constructing the app more + than once per process (a common pytest fixture pattern) never appends + duplicate routes to shared module state. """ + router = APIRouter(prefix="/api/stream", tags=["streaming"]) @router.get("/prices") async def stream_prices(request: Request) -> StreamingResponse: @@ -48,6 +55,34 @@ async def stream_prices(request: Request) -> StreamingResponse: return router +def create_history_router(price_cache: PriceCache) -> APIRouter: + """Create the router serving rolling in-memory price history. + + Factory pattern mirrors create_stream_router: a fresh APIRouter is built + on every call so the PriceCache is injected without module-level globals + that would accumulate duplicate routes across repeated app construction. + """ + history_router = APIRouter(prefix="/api/prices", tags=["prices"]) + + @history_router.get("/{ticker}/history") + async def get_price_history(ticker: str, limit: int = HISTORY_MAXLEN) -> dict: + """Rolling in-memory price history for the main chart. + + Returns an empty `points` list for an untracked ticker — not a 404 — + so the chart draws nothing rather than erroring. Oldest-first, + matching what a left-to-right time axis wants. + """ + ticker = ticker.strip().upper() + limit = max(1, min(limit, HISTORY_MAXLEN)) + points = price_cache.get_history(ticker, limit=limit) + return { + "ticker": ticker, + "points": [{"timestamp": ts, "price": price} for ts, price in points], + } + + return history_router + + async def _generate_events( price_cache: PriceCache, request: Request, @@ -55,13 +90,17 @@ async def _generate_events( ) -> AsyncGenerator[str, None]: """Async generator that yields SSE-formatted price events. - Sends all prices every `interval` seconds. Stops when the client - disconnects (detected via request.is_disconnected()). + Sends all prices every `interval` seconds whenever the cache version has + changed. When the version is unchanged for KEEPALIVE_SECONDS, sends an + SSE comment line instead — EventSource ignores it, but it keeps the + connection (and any proxy in between) from treating a quiet market as a + dead connection. Stops when the client disconnects. """ # Tell the client to retry after 1 second if the connection drops yield "retry: 1000\n\n" last_version = -1 + last_sent = time.monotonic() client_ip = request.client.host if request.client else "unknown" logger.info("SSE client connected: %s", client_ip) @@ -81,6 +120,10 @@ async def _generate_events( data = {ticker: update.to_dict() for ticker, update in prices.items()} payload = json.dumps(data) yield f"data: {payload}\n\n" + last_sent = time.monotonic() + elif time.monotonic() - last_sent >= KEEPALIVE_SECONDS: + yield ": ping\n\n" + last_sent = time.monotonic() await asyncio.sleep(interval) except asyncio.CancelledError: diff --git a/backend/scripts/verify_massive.py b/backend/scripts/verify_massive.py new file mode 100644 index 000000000..e18a8e65b --- /dev/null +++ b/backend/scripts/verify_massive.py @@ -0,0 +1,43 @@ +"""Smoke-test the Massive REST API against a live key. + +Run once a real MASSIVE_API_KEY exists — it confirms auth, the multi-ticker +snapshot request, and the nanosecond-to-second timestamp conversion in one +pass. Timestamps far in the future mean the divisor regressed; an +AttributeError means the attribute name regressed (see massive_client.py). + + cd backend && uv run python scripts/verify_massive.py +""" + +import os +from datetime import UTC, datetime + +from massive import RESTClient +from massive.rest.models import SnapshotMarketType + +NANOS_PER_SECOND = 1_000_000_000 +TICKERS = ["AAPL", "GOOGL", "MSFT", "NVDA", "TSLA"] + + +def main() -> None: + client = RESTClient(api_key=os.environ["MASSIVE_API_KEY"]) + + print(f"market: {client.get_market_status().market}") + + snapshots = client.get_snapshot_all(SnapshotMarketType.STOCKS, TICKERS) + print(f"requested {len(TICKERS)}, received {len(snapshots)}") + + for snap in snapshots: + trade = snap.last_trade + if trade is None or trade.price is None: + print(f"{snap.ticker}: no trade data") + continue + when = datetime.fromtimestamp(trade.sip_timestamp / NANOS_PER_SECOND, UTC) + print(f"{snap.ticker}: ${trade.price:.2f} at {when:%Y-%m-%d %H:%M:%S} UTC") + + missing = set(TICKERS) - {s.ticker for s in snapshots} + if missing: + print(f"absent from response (unknown or untraded): {sorted(missing)}") + + +if __name__ == "__main__": + main() diff --git a/backend/tests/market/test_cache.py b/backend/tests/market/test_cache.py index b5ab3d55d..0a5deac81 100644 --- a/backend/tests/market/test_cache.py +++ b/backend/tests/market/test_cache.py @@ -96,8 +96,73 @@ def test_custom_timestamp(self): update = cache.update("AAPL", 190.50, timestamp=custom_ts) assert update.timestamp == custom_ts + def test_epoch_zero_timestamp_is_not_replaced(self): + """timestamp=0.0 is falsy but a legitimate Unix epoch instant -- + an `or` check would wrongly substitute time.time() for it.""" + cache = PriceCache() + update = cache.update("AAPL", 190.50, timestamp=0.0) + assert update.timestamp == 0.0 + def test_price_rounding(self): """Test that prices are rounded to 2 decimal places.""" cache = PriceCache() update = cache.update("AAPL", 190.12345) assert update.price == 190.12 + + +class TestPriceHistory: + """Unit tests for PriceCache's rolling per-ticker history.""" + + def test_history_accumulates_in_order(self): + cache = PriceCache() + cache.update("AAPL", 190.00, timestamp=1.0) + cache.update("AAPL", 191.00, timestamp=2.0) + cache.update("AAPL", 192.00, timestamp=3.0) + + points = cache.get_history("AAPL") + assert points == [(1.0, 190.00), (2.0, 191.00), (3.0, 192.00)] + + def test_history_is_bounded_and_oldest_first(self): + cache = PriceCache(history_maxlen=5) + for i in range(10): + cache.update("AAPL", 100.0 + i, timestamp=float(i)) + + points = cache.get_history("AAPL") + assert len(points) == 5 + assert [ts for ts, _ in points] == [5.0, 6.0, 7.0, 8.0, 9.0] + + def test_history_is_empty_for_untracked_ticker(self): + assert PriceCache().get_history("NOPE") == [] + + def test_history_respects_limit_narrower_than_stored(self): + cache = PriceCache() + for i in range(10): + cache.update("AAPL", 100.0 + i, timestamp=float(i)) + + points = cache.get_history("AAPL", limit=3) + assert [ts for ts, _ in points] == [7.0, 8.0, 9.0] + + def test_history_tracks_multiple_tickers_independently(self): + cache = PriceCache() + cache.update("AAPL", 190.00, timestamp=1.0) + cache.update("GOOGL", 175.00, timestamp=1.0) + cache.update("AAPL", 191.00, timestamp=2.0) + + assert len(cache.get_history("AAPL")) == 2 + assert len(cache.get_history("GOOGL")) == 1 + + def test_remove_clears_price_and_history(self): + cache = PriceCache() + cache.update("AAPL", 190.00) + cache.remove("AAPL") + + assert cache.get("AAPL") is None + assert cache.get_history("AAPL") == [] + + def test_remove_history_does_not_affect_other_tickers(self): + cache = PriceCache() + cache.update("AAPL", 190.00) + cache.update("GOOGL", 175.00) + cache.remove("AAPL") + + assert cache.get_history("GOOGL") != [] diff --git a/backend/tests/market/test_massive.py b/backend/tests/market/test_massive.py index cdd7dbd24..7b4379bf1 100644 --- a/backend/tests/market/test_massive.py +++ b/backend/tests/market/test_massive.py @@ -1,29 +1,109 @@ -"""Tests for MassiveDataSource (mocked).""" - +"""Tests for MassiveDataSource. + +Snapshot parsing is tested against the real `TickerSnapshot` model +(`TickerSnapshot.from_dict`), not `MagicMock`. A mock answers to any +attribute name you give it, including a misspelled one — which is exactly +how the shipped code's `last_trade.timestamp` bug (the real attribute is +`sip_timestamp`) went undetected at 94% coverage. Only network calls +(`_fetch_snapshots`) are mocked; the parsing path always exercises the real +model shape. +""" + +import asyncio from unittest.mock import MagicMock, patch import pytest +from massive.exceptions import AuthError, BadResponse +from massive.rest.models.snapshot import TickerSnapshot from app.market.cache import PriceCache -from app.market.massive_client import MassiveDataSource +from app.market.massive_client import NANOS_PER_SECOND, MassiveDataSource + + +def _make_snapshot(ticker: str, price: float, timestamp_ns: int) -> TickerSnapshot: + """Build a real TickerSnapshot from a Massive-shaped payload (wire keys).""" + return TickerSnapshot.from_dict( + { + "ticker": ticker, + "lastTrade": {"p": price, "s": 100, "t": timestamp_ns, "x": 4}, + } + ) + + +def _make_snapshot_without_trade(ticker: str) -> TickerSnapshot: + return TickerSnapshot.from_dict({"ticker": ticker}) + + +class TestApplySnapshots: + """Tests for the extracted, directly-testable parse method.""" + + def test_snapshot_parse_produces_a_present_day_timestamp(self): + cache = PriceCache() + source = MassiveDataSource(api_key="test-key", price_cache=cache) + snap = _make_snapshot("AAPL", 190.52, 1755873791482000000) + + processed = source._apply_snapshots([snap]) + + assert processed == 1 + update = cache.get("AAPL") + assert update is not None + assert update.price == 190.52 + # Plausible present, in SECONDS -- not ~1.76e15 (the pre-fix bug). + assert 1_600_000_000 < update.timestamp < 2_000_000_000 + + def test_snapshot_timestamp_matches_expected_conversion(self): + cache = PriceCache() + source = MassiveDataSource(api_key="test-key", price_cache=cache) + timestamp_ns = 1707580800000000000 + snap = _make_snapshot("AAPL", 190.50, timestamp_ns) + + source._apply_snapshots([snap]) + + update = cache.get("AAPL") + assert update.timestamp == timestamp_ns / NANOS_PER_SECOND + + def test_snapshot_without_a_last_trade_is_skipped(self): + cache = PriceCache() + source = MassiveDataSource(api_key="test-key", price_cache=cache) + snap = _make_snapshot_without_trade("AAPL") + + processed = source._apply_snapshots([snap]) + + assert processed == 0 + assert cache.get("AAPL") is None + + def test_mixed_snapshots_processes_only_the_valid_one(self): + cache = PriceCache() + source = MassiveDataSource(api_key="test-key", price_cache=cache) + good = _make_snapshot("AAPL", 190.50, 1707580800000000000) + bad = _make_snapshot_without_trade("MISSING") + processed = source._apply_snapshots([good, bad]) -def _make_snapshot(ticker: str, price: float, timestamp_ms: int) -> MagicMock: - """Create a mock Massive snapshot object.""" - snap = MagicMock() - snap.ticker = ticker - snap.last_trade = MagicMock() - snap.last_trade.price = price - snap.last_trade.timestamp = timestamp_ms - return snap + assert processed == 1 + assert cache.get_price("AAPL") == 190.50 + assert cache.get("MISSING") is None + + def test_multiple_tickers_all_update(self): + cache = PriceCache() + source = MassiveDataSource(api_key="test-key", price_cache=cache) + snapshots = [ + _make_snapshot("AAPL", 190.50, 1707580800000000000), + _make_snapshot("GOOGL", 175.25, 1707580800000000000), + ] + + processed = source._apply_snapshots(snapshots) + + assert processed == 2 + assert cache.get_price("AAPL") == 190.50 + assert cache.get_price("GOOGL") == 175.25 @pytest.mark.asyncio -class TestMassiveDataSource: - """Unit tests for MassiveDataSource with mocked API.""" +class TestMassiveDataSourcePolling: + """Unit tests for the async polling lifecycle, with the network mocked.""" async def test_poll_updates_cache(self): - """Test that polling updates the cache.""" cache = PriceCache() source = MassiveDataSource( api_key="test-key", @@ -34,8 +114,8 @@ async def test_poll_updates_cache(self): source._client = MagicMock() # Satisfy the _poll_once guard mock_snapshots = [ - _make_snapshot("AAPL", 190.50, 1707580800000), - _make_snapshot("GOOGL", 175.25, 1707580800000), + _make_snapshot("AAPL", 190.50, 1707580800000000000), + _make_snapshot("GOOGL", 175.25, 1707580800000000000), ] with patch.object(source, "_fetch_snapshots", return_value=mock_snapshots): @@ -44,67 +124,45 @@ async def test_poll_updates_cache(self): assert cache.get_price("AAPL") == 190.50 assert cache.get_price("GOOGL") == 175.25 - async def test_malformed_snapshot_skipped(self): - """Test that malformed snapshots are skipped gracefully.""" + async def test_bad_response_leaves_cache_untouched_and_does_not_raise(self): cache = PriceCache() - source = MassiveDataSource( - api_key="test-key", - price_cache=cache, - poll_interval=60.0, - ) - source._tickers = ["AAPL", "BAD"] - source._client = MagicMock() # Satisfy the _poll_once guard - - good_snap = _make_snapshot("AAPL", 190.50, 1707580800000) - bad_snap = MagicMock() - bad_snap.ticker = "BAD" - bad_snap.last_trade = None # Will cause AttributeError + source = MassiveDataSource(api_key="test-key", price_cache=cache, poll_interval=60.0) + source._tickers = ["AAPL"] + source._client = MagicMock() - with patch.object(source, "_fetch_snapshots", return_value=[good_snap, bad_snap]): - await source._poll_once() + with patch.object( + source, + "_fetch_snapshots", + side_effect=BadResponse("rate limited"), + ): + await source._poll_once() # Should not raise - # Good ticker processed, bad one skipped - assert cache.get_price("AAPL") == 190.50 - assert cache.get_price("BAD") is None + assert cache.get_price("AAPL") is None - async def test_api_error_does_not_crash(self): - """Test that API errors don't crash the poller.""" + async def test_auth_error_propagates(self): + """An unrecoverable auth failure should not be retried silently.""" cache = PriceCache() - source = MassiveDataSource( - api_key="test-key", - price_cache=cache, - poll_interval=60.0, - ) + source = MassiveDataSource(api_key="bad-key", price_cache=cache, poll_interval=60.0) source._tickers = ["AAPL"] - source._client = MagicMock() # Satisfy the _poll_once guard + source._client = MagicMock() - with patch.object(source, "_fetch_snapshots", side_effect=Exception("network error")): - await source._poll_once() # Should not raise + with patch.object(source, "_fetch_snapshots", side_effect=AuthError("invalid key")): + with pytest.raises(AuthError): + await source._poll_once() - assert cache.get_price("AAPL") is None # No update happened - - async def test_timestamp_conversion(self): - """Test that timestamps are converted from milliseconds to seconds.""" + async def test_unexpected_error_does_not_crash(self): + """Test that unexpected (e.g. network) errors don't crash the poller.""" cache = PriceCache() - source = MassiveDataSource( - api_key="test-key", - price_cache=cache, - poll_interval=60.0, - ) + source = MassiveDataSource(api_key="test-key", price_cache=cache, poll_interval=60.0) source._tickers = ["AAPL"] - source._client = MagicMock() # Satisfy the _poll_once guard - - mock_snapshots = [_make_snapshot("AAPL", 190.50, 1707580800000)] + source._client = MagicMock() - with patch.object(source, "_fetch_snapshots", return_value=mock_snapshots): - await source._poll_once() + with patch.object(source, "_fetch_snapshots", side_effect=Exception("network error")): + await source._poll_once() # Should not raise - update = cache.get("AAPL") - assert update is not None - assert update.timestamp == 1707580800.0 # Converted to seconds + assert cache.get_price("AAPL") is None # No update happened async def test_add_ticker(self): - """Test adding a ticker.""" cache = PriceCache() source = MassiveDataSource(api_key="test-key", price_cache=cache) @@ -112,7 +170,6 @@ async def test_add_ticker(self): assert "AAPL" in source.get_tickers() async def test_add_ticker_uppercase_normalization(self): - """Test that tickers are normalized to uppercase.""" cache = PriceCache() source = MassiveDataSource(api_key="test-key", price_cache=cache) @@ -120,7 +177,6 @@ async def test_add_ticker_uppercase_normalization(self): assert "AAPL" in source.get_tickers() async def test_add_ticker_strips_whitespace(self): - """Test that ticker whitespace is stripped.""" cache = PriceCache() source = MassiveDataSource(api_key="test-key", price_cache=cache) @@ -128,7 +184,6 @@ async def test_add_ticker_strips_whitespace(self): assert "AAPL" in source.get_tickers() async def test_remove_ticker(self): - """Test removing a ticker.""" cache = PriceCache() source = MassiveDataSource(api_key="test-key", price_cache=cache) source._tickers = ["AAPL", "GOOGL"] @@ -139,7 +194,6 @@ async def test_remove_ticker(self): assert cache.get("AAPL") is None async def test_get_tickers(self): - """Test getting the list of active tickers.""" cache = PriceCache() source = MassiveDataSource(api_key="test-key", price_cache=cache) source._tickers = ["AAPL", "GOOGL"] @@ -148,7 +202,6 @@ async def test_get_tickers(self): assert tickers == ["AAPL", "GOOGL"] async def test_empty_tickers_skips_poll(self): - """Test that polling is skipped when there are no tickers.""" cache = PriceCache() source = MassiveDataSource(api_key="test-key", price_cache=cache) source._tickers = [] @@ -159,7 +212,6 @@ async def test_empty_tickers_skips_poll(self): mock_fetch.assert_not_called() async def test_stop_is_idempotent(self): - """Test that stop() can be called multiple times.""" cache = PriceCache() source = MassiveDataSource(api_key="test-key", price_cache=cache) @@ -167,7 +219,6 @@ async def test_stop_is_idempotent(self): await source.stop() # Should not raise async def test_stop_cancels_task(self): - """Test that stop() cancels the polling task.""" cache = PriceCache() source = MassiveDataSource(api_key="test-key", price_cache=cache, poll_interval=10.0) @@ -185,11 +236,10 @@ async def test_stop_cancels_task(self): assert source._task is None async def test_start_immediate_poll(self): - """Test that start() does an immediate poll before starting the loop.""" cache = PriceCache() source = MassiveDataSource(api_key="test-key", price_cache=cache, poll_interval=60.0) - mock_snapshots = [_make_snapshot("AAPL", 190.50, 1707580800000)] + mock_snapshots = [_make_snapshot("AAPL", 190.50, 1707580800000000000)] with patch("app.market.massive_client.RESTClient"): with patch.object(source, "_fetch_snapshots", return_value=mock_snapshots): @@ -199,3 +249,51 @@ async def test_start_immediate_poll(self): assert cache.get_price("AAPL") == 190.50 await source.stop() + + async def test_is_healthy_true_after_a_normal_start(self): + cache = PriceCache() + source = MassiveDataSource(api_key="test-key", price_cache=cache, poll_interval=60.0) + + with patch("app.market.massive_client.RESTClient"): + with patch.object(source, "_fetch_snapshots", return_value=[]): + await source.start(["AAPL"]) + + assert source.is_healthy is True + await source.stop() + + async def test_is_healthy_survives_a_deliberate_stop(self): + """Cancelling the task via stop() must not be mistaken for a crash.""" + cache = PriceCache() + source = MassiveDataSource(api_key="test-key", price_cache=cache, poll_interval=10.0) + + with patch("app.market.massive_client.RESTClient"): + with patch.object(source, "_fetch_snapshots", return_value=[]): + await source.start(["AAPL"]) + + await source.stop() + assert source.is_healthy is True + + async def test_is_healthy_flips_false_when_the_poll_loop_dies(self): + """A revoked key raising AuthError mid-loop must flip the health flag, + not just die silently until someone happens to await the task.""" + cache = PriceCache() + source = MassiveDataSource(api_key="test-key", price_cache=cache, poll_interval=0.01) + + with patch("app.market.massive_client.RESTClient"): + with patch.object(source, "_fetch_snapshots", return_value=[]): + await source.start(["AAPL"]) + + assert source.is_healthy is True + + with patch.object(source, "_fetch_snapshots", side_effect=AuthError("revoked")): + # Let the loop wake up, poll, raise, and have its done-callback fire. + for _ in range(20): + if not source.is_healthy: + break + await asyncio.sleep(0.01) + + assert source.is_healthy is False + assert source._task.done() + + # Cleanup: the task already died, stop() must still be a safe no-op path. + source._task = None diff --git a/backend/tests/market/test_stream.py b/backend/tests/market/test_stream.py new file mode 100644 index 000000000..f0313d10e --- /dev/null +++ b/backend/tests/market/test_stream.py @@ -0,0 +1,219 @@ +"""Tests for the SSE price stream and the price history endpoint. + +The SSE generator is driven directly with a fake Request object rather than +through a live server (per the design doc) — that makes the keepalive path +testable with a monkeypatched threshold instead of 15 seconds of real +waiting, and needs no ASGI test client. +""" + +from __future__ import annotations + +import json + +import pytest + +import app.market.stream as stream_module +from app.market.cache import HISTORY_MAXLEN, PriceCache +from app.market.stream import _generate_events, create_history_router, create_stream_router + + +class FakeClient: + host = "test-client" + + +class FakeRequest: + """Minimal stand-in for a Starlette Request, as far as the generator cares.""" + + def __init__(self, disconnect_after: int | None = None) -> None: + self.client = FakeClient() + self._checks = 0 + self._disconnect_after = disconnect_after + + async def is_disconnected(self) -> bool: + self._checks += 1 + if self._disconnect_after is not None and self._checks > self._disconnect_after: + return True + return False + + +def _history_endpoint(cache: PriceCache): + """Grab the just-registered endpoint closure without a live ASGI app.""" + router = create_history_router(cache) + return router.routes[-1].endpoint + + +class TestRouterFactoriesReturnFreshRouters: + """Repeated calls (e.g. an `app` fixture rebuilt per test) must not + accumulate duplicate routes on shared module-level router state.""" + + def test_create_stream_router_does_not_share_routes_across_calls(self): + cache = PriceCache() + first = create_stream_router(cache) + second = create_stream_router(cache) + + assert first is not second + assert len(first.routes) == 1 + assert len(second.routes) == 1 + + def test_create_history_router_does_not_share_routes_across_calls(self): + cache = PriceCache() + first = create_history_router(cache) + second = create_history_router(cache) + + assert first is not second + assert len(first.routes) == 1 + assert len(second.routes) == 1 + + +@pytest.mark.asyncio +class TestGenerateEvents: + async def test_first_event_is_the_retry_directive(self): + cache = PriceCache() + agen = _generate_events(cache, FakeRequest(), interval=0.01) + assert await agen.__anext__() == "retry: 1000\n\n" + await agen.aclose() + + async def test_connecting_client_gets_a_full_snapshot_immediately(self): + """A fresh generator starts at last_version = -1, so the very first + comparison always differs -- this is also what makes reconnects work, + with no separate snapshot endpoint needed.""" + cache = PriceCache() + cache.update("AAPL", 190.52) + cache.update("GOOGL", 175.00) + + agen = _generate_events(cache, FakeRequest(), interval=0.01) + await agen.__anext__() # retry directive + event = await agen.__anext__() + await agen.aclose() + + assert event.startswith("data: ") + payload = json.loads(event[len("data: ") : -2]) + assert set(payload.keys()) == {"AAPL", "GOOGL"} + + async def test_payload_is_map_keyed_by_ticker_with_frozen_field_shape(self): + cache = PriceCache() + cache.update("AAPL", 190.52, timestamp=1755873791.482) + + agen = _generate_events(cache, FakeRequest(), interval=0.01) + await agen.__anext__() + event = await agen.__anext__() + await agen.aclose() + + payload = json.loads(event[len("data: ") : -2]) + aapl = payload["AAPL"] + assert aapl["ticker"] == "AAPL" + assert aapl["timestamp"] == 1755873791.482 # float seconds, never ISO + assert set(aapl.keys()) == { + "ticker", + "price", + "previous_price", + "timestamp", + "change", + "change_percent", + "direction", + } + + async def test_keepalive_ping_sent_when_idle_not_a_duplicate_data_event(self, monkeypatch): + """While the version is unchanged, the next event must be a keepalive + ping — never a repeated data event — and it must wait for the + threshold rather than firing immediately.""" + monkeypatch.setattr(stream_module, "KEEPALIVE_SECONDS", 0.03) + cache = PriceCache() + cache.update("AAPL", 100.0) + + agen = stream_module._generate_events(cache, FakeRequest(), interval=0.01) + await agen.__anext__() # retry + await agen.__anext__() # initial snapshot + ping = await agen.__anext__() + await agen.aclose() + + assert ping == ": ping\n\n" + + async def test_new_price_after_a_ping_produces_a_fresh_data_event(self, monkeypatch): + monkeypatch.setattr(stream_module, "KEEPALIVE_SECONDS", 0.03) + cache = PriceCache() + cache.update("AAPL", 100.0) + + agen = stream_module._generate_events(cache, FakeRequest(), interval=0.01) + await agen.__anext__() # retry + await agen.__anext__() # initial snapshot + await agen.__anext__() # ping + + cache.update("AAPL", 101.0) + event = await agen.__anext__() + await agen.aclose() + + assert event.startswith("data: ") + payload = json.loads(event[len("data: ") : -2]) + assert payload["AAPL"]["price"] == 101.0 + + async def test_generator_stops_when_client_disconnects(self): + cache = PriceCache() + request = FakeRequest(disconnect_after=0) + + agen = _generate_events(cache, request, interval=0.01) + await agen.__anext__() # retry directive + with pytest.raises(StopAsyncIteration): + await agen.__anext__() + + async def test_empty_cache_sends_no_data_event(self): + # Let the loop run one full iteration (version differs from the + # initial -1, but there are no prices to send) before disconnecting, + # so the assertion is actually exercising the empty-prices branch. + cache = PriceCache() + request = FakeRequest(disconnect_after=1) + + agen = _generate_events(cache, request, interval=0.01) + events = [event async for event in agen] + + assert events == ["retry: 1000\n\n"] + + +@pytest.mark.asyncio +class TestPriceHistoryEndpoint: + async def test_returns_points_oldest_first(self): + cache = PriceCache() + cache.update("AAPL", 190.00, timestamp=1.0) + cache.update("AAPL", 191.00, timestamp=2.0) + + result = await _history_endpoint(cache)(ticker="AAPL", limit=HISTORY_MAXLEN) + + assert result["ticker"] == "AAPL" + assert result["points"] == [ + {"timestamp": 1.0, "price": 190.00}, + {"timestamp": 2.0, "price": 191.00}, + ] + + async def test_untracked_ticker_returns_empty_points_not_an_error(self): + cache = PriceCache() + + result = await _history_endpoint(cache)(ticker="NOPE", limit=HISTORY_MAXLEN) + + assert result == {"ticker": "NOPE", "points": []} + + async def test_ticker_is_normalized(self): + cache = PriceCache() + cache.update("AAPL", 190.00, timestamp=1.0) + + result = await _history_endpoint(cache)(ticker=" aapl ", limit=HISTORY_MAXLEN) + + assert result["ticker"] == "AAPL" + assert len(result["points"]) == 1 + + async def test_limit_is_clamped_to_history_maxlen(self): + cache = PriceCache() + for i in range(5): + cache.update("AAPL", 100.0 + i, timestamp=float(i)) + + result = await _history_endpoint(cache)(ticker="AAPL", limit=1_000_000) + + assert len(result["points"]) == 5 + + async def test_limit_narrower_than_history_returns_most_recent(self): + cache = PriceCache() + for i in range(5): + cache.update("AAPL", 100.0 + i, timestamp=float(i)) + + result = await _history_endpoint(cache)(ticker="AAPL", limit=2) + + assert [p["timestamp"] for p in result["points"]] == [3.0, 4.0] diff --git a/db/.gitkeep b/db/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/planning/MARKET_DATA_DESIGN.md b/planning/MARKET_DATA_DESIGN.md new file mode 100644 index 000000000..d7c962e7c --- /dev/null +++ b/planning/MARKET_DATA_DESIGN.md @@ -0,0 +1,1478 @@ +# MARKET_DATA_DESIGN.md — Market Data Backend, Detailed Design + +The implementation-level design for FinAlly's market data subsystem: one unified API, two +interchangeable sources (GBM simulator and the Massive REST API), a shared in-memory cache, +and the SSE stream that carries prices to the browser. + +**Audience:** the agent (or human) implementing or extending `backend/app/market/`. +This document is meant to be read once, top to bottom, and then implemented from — every +snippet below is either the code that ships today or the code that should ship. + +**Companion documents.** `PLAN.md` §6 is the frozen contract; `MARKET_INTERFACE.md`, +`MARKET_SIMULATOR.md`, and `MASSIVE_API.md` are the reference material this design draws on. +Where they disagree with this document, this document is the design and they are the background. + +--- + +## 0. Status — what exists, what is missing + +Verified by running the suite in `backend/` on 2026-09-01: + +``` +73 passed in 1.77s TOTAL coverage 91% +app/market/cache.py 100% +app/market/models.py 100% +app/market/simulator.py 98% +app/market/massive_client.py 94% <- high coverage, two live defects (§8.4) +app/market/stream.py 33% <- the SSE generator is effectively untested +``` + +| Piece | State | Section | +|---|---|---| +| `PriceUpdate` wire model | Ships, frozen contract | §4 | +| `PriceCache` (latest price + version) | Ships | §5.1 | +| `PriceCache` rolling history | **Missing** | §5.2 | +| `MarketDataSource` ABC | Ships | §6 | +| `SimulatorDataSource` + `GBMSimulator` | Ships | §7 | +| `MassiveDataSource` | Ships but **writes nothing to the cache** | §8.4 | +| `create_market_data_source` | Ships | §9 | +| SSE `/api/stream/prices` | Ships | §10.1 | +| SSE keepalive | **Missing** | §10.2 | +| `GET /api/prices/{ticker}/history` | **Missing** | §11 | +| Lifespan wiring + tracked-set reconciliation | **Missing** | §12 | + +Four gaps, all backend, all small. §15 orders them. + +--- + +## 1. The shape of the design + +Two sources with nothing in common — a 500ms in-process computation and a 15-second blocking +HTTP poll — must be interchangeable to everything downstream. The design achieves that with +**one indirection and one shared buffer**: + +``` + writes reads + ┌──────────────────┐ ┌────────────┐ ┌──────────────────────┐ + │ SimulatorSource │───┐ │ │───────────────│ SSE /api/stream │ + │ (500ms step) │ ├───▶│ PriceCache │───────────────│ Portfolio valuation │ + ├──────────────────┤ │ │ (in-mem, │───────────────│ Trade execution │ + │ MassiveSource │───┘ │thread-safe)│───────────────│ Snapshot task │ + │ (15s poll) │ │ │───────────────│ /api/prices/history │ + └──────────────────┘ └────────────┘ └──────────────────────┘ + MarketDataSource + (abstract interface) +``` + +**The one invariant that makes this work: nothing downstream ever asks a source for a price.** +Sources are write-only from the application's point of view; readers only ever touch the cache. +That is why a 30× difference in update cadence is invisible to the rest of the app, and why a +`get_price()` on the interface would be a design error — under Massive it would turn every +portfolio valuation into a billed HTTP request. + +### File structure + +``` +backend/app/market/ +├── __init__.py # public exports +├── models.py # PriceUpdate — the unit of data +├── cache.py # PriceCache — latest price + version + rolling history +├── interface.py # MarketDataSource — the ABC +├── seed_prices.py # simulator constants, no logic +├── simulator.py # GBMSimulator (pure) + SimulatorDataSource (async) +├── massive_client.py # MassiveDataSource +├── factory.py # create_market_data_source +└── stream.py # SSE router + history router +``` + +Public surface, unchanged by this design: + +```python +from app.market import ( + PriceUpdate, + PriceCache, + MarketDataSource, + create_market_data_source, + create_stream_router, +) +``` + +--- + +## 2. Vocabulary + +| Term | Meaning | +|---|---| +| **tick** | One simulator step (500ms) or one Massive poll (15s) | +| **tracked set** | `watchlist ∪ {tickers with a non-zero position}` — §12.2 | +| **version** | Monotonic counter on `PriceCache`, bumped on every write; the SSE change signal | +| **seeding** | Writing an initial price into the cache so a ticker never renders as `—` unnecessarily | + +--- + +## 3. Non-negotiable contracts + +These are frozen because the frontend and the shipped module already depend on them. Everything +else in this document is open to reasonable change. + +1. **SSE payload is a map keyed by ticker, one event carries every ticker.** Not one event per ticker. +2. **`timestamp` is Unix epoch seconds as a float.** Never ISO, never milliseconds. The frontend + multiplies by 1000 for `Date`. +3. **`change_percent` is already in percent units.** `0.021` means 0.021%. This deliberately + differs from REST responses elsewhere in the API, where percentages are fractions + (`PLAN.md` §8). The inconsistency is real and preserved. +4. **A connecting client gets a full snapshot immediately**, including after a reconnect, + because a fresh generator starts at `last_version = -1`. +5. **Tickers are uppercase everywhere**, normalized at the API boundary. + +--- + +## 4. `PriceUpdate` — the unit of data + +`backend/app/market/models.py`. Immutable, frozen, slotted. Both sources produce it; every +reader consumes it. + +```python +@dataclass(frozen=True, slots=True) +class PriceUpdate: + """Immutable snapshot of a single ticker's price at a point in time.""" + + ticker: str + price: float + previous_price: float + timestamp: float = field(default_factory=time.time) # Unix epoch SECONDS + + @property + def change(self) -> float: + return round(self.price - self.previous_price, 4) + + @property + def change_percent(self) -> float: + if self.previous_price == 0: + return 0.0 + return round((self.price - self.previous_price) / self.previous_price * 100, 4) + + @property + def direction(self) -> str: + if self.price > self.previous_price: + return "up" + elif self.price < self.previous_price: + return "down" + return "flat" + + def to_dict(self) -> dict: + return { + "ticker": self.ticker, + "price": self.price, + "previous_price": self.previous_price, + "timestamp": self.timestamp, + "change": self.change, + "change_percent": self.change_percent, + "direction": self.direction, + } +``` + +**`change`, `change_percent`, and `direction` are computed properties, not stored fields.** +They cannot drift out of sync with the prices they describe, and `to_dict()` cannot emit a +`direction` that contradicts its own `price`/`previous_price` pair. + +**`previous_price` means the price at the previous update**, not the previous session's close. +On the first update for a ticker it equals `price`, so `direction` is `"flat"` and `change` is +`0.0` — a newly added ticker never flashes green or red on its first tick. + +Example of the exact wire shape a client sees: + +```json +{ + "ticker": "AAPL", + "price": 190.52, + "previous_price": 190.48, + "timestamp": 1755873791.482, + "change": 0.04, + "change_percent": 0.021, + "direction": "up" +} +``` + +--- + +## 5. `PriceCache` — the shared buffer + +`backend/app/market/cache.py`. + +### 5.1 What ships today + +```python +class PriceCache: + def update(self, ticker: str, price: float, timestamp: float | None = None) -> PriceUpdate + def get(self, ticker: str) -> PriceUpdate | None + def get_all(self) -> dict[str, PriceUpdate] # shallow copy + def get_price(self, ticker: str) -> float | None + def remove(self, ticker: str) -> None + @property + def version(self) -> int + def __len__(self) -> int + def __contains__(self, ticker: str) -> bool +``` + +Three design points that matter: + +**`update()` derives `previous_price` itself.** Callers pass only the new price; the cache looks +up what it held and constructs the `PriceUpdate`. Neither source tracks prior state for the +purpose of computing a delta, so the two cannot implement it differently. + +```python +def update(self, ticker: str, price: float, timestamp: float | None = None) -> PriceUpdate: + with self._lock: + ts = timestamp or time.time() + prev = self._prices.get(ticker) + previous_price = prev.price if prev else price + + update = PriceUpdate( + ticker=ticker, + price=round(price, 2), + previous_price=round(previous_price, 2), + timestamp=ts, + ) + self._prices[ticker] = update + self._version += 1 + return update +``` + +**A `threading.Lock`, not an `asyncio.Lock`.** `MassiveDataSource` writes from an +`asyncio.to_thread` worker, so a genuine cross-thread lock is required. The critical sections +are a few dict operations; contention is irrelevant. + +**`version` is the SSE change-detection mechanism.** The stream compares an integer every 500ms +rather than diffing price maps. `get_all()` returns a shallow copy, and since `PriceUpdate` is +frozen, that copy is effectively deep and safe to iterate outside the lock. + +### 5.2 Rolling price history — to implement + +`PLAN.md` §6 requires the main chart to be populated the instant a ticker is clicked, rather +than drawing itself from scratch over the following minute. `PriceCache` gains a bounded +per-ticker deque of `(timestamp, price)`. + +```python +from collections import deque + +HISTORY_MAXLEN = 600 # ~5 minutes at the 500ms simulator cadence +``` + +Constructor: + +```python +def __init__(self, history_maxlen: int = HISTORY_MAXLEN) -> None: + self._prices: dict[str, PriceUpdate] = {} + self._history: dict[str, deque[tuple[float, float]]] = {} + self._history_maxlen = history_maxlen + self._lock = Lock() + self._version: int = 0 +``` + +Appended inside `update()`, under the same lock, immediately after the price is stored: + +```python + self._prices[ticker] = update + history = self._history.get(ticker) + if history is None: + history = deque(maxlen=self._history_maxlen) + self._history[ticker] = history + history.append((ts, update.price)) + self._version += 1 + return update +``` + +`remove()` must drop the deque too, or removed tickers leak memory and a re-added ticker +resurrects a stale chart: + +```python +def remove(self, ticker: str) -> None: + with self._lock: + self._prices.pop(ticker, None) + self._history.pop(ticker, None) +``` + +The reader, backing `GET /api/prices/{ticker}/history`: + +```python +def get_history(self, ticker: str, limit: int = HISTORY_MAXLEN) -> list[tuple[float, float]]: + """Oldest-first (timestamp, price) points. Empty list for an untracked ticker.""" + with self._lock: + points = self._history.get(ticker) + if not points: + return [] + return list(points)[-limit:] +``` + +Four properties worth stating explicitly: + +- **`deque(maxlen=600)` evicts the oldest point automatically** — there is no pruning logic to + write, and no unbounded growth to worry about. +- **An untracked ticker returns `[]`, not a 404.** The chart draws nothing rather than erroring + (`PLAN.md` §8). +- **Deliberately not persisted.** A restart clears it, which is the honest behavior for a + simulator whose prices also reset to seed on restart. +- **Memory is negligible**: 600 points × 50 tickers × ~16 bytes ≈ 500KB. + +Under Massive the deque fills at one point per 15-second poll, so five minutes of wall time is +20 points rather than 600. The chart is sparse but correct. Backfilling from `get_aggs` +(`MASSIVE_API.md` §5) is the eventual upgrade and is out of scope here. + +--- + +## 6. `MarketDataSource` — the abstract contract + +`backend/app/market/interface.py`. + +```python +class MarketDataSource(ABC): + @abstractmethod + async def start(self, tickers: list[str]) -> None: ... + @abstractmethod + async def stop(self) -> None: ... + @abstractmethod + async def add_ticker(self, ticker: str) -> None: ... + @abstractmethod + async def remove_ticker(self, ticker: str) -> None: ... + @abstractmethod + def get_tickers(self) -> list[str]: ... +``` + +Five methods, and every one is about **lifecycle and membership** — none returns a price. That +absence is the whole design (§1). + +### Behavioral contract + +Binding on both implementations. A test suite that passes against one should pass against the other. + +| Method | Guarantee | +|---|---| +| `start(tickers)` | Begins a background task writing to the cache. **Seeds the cache before returning**, so the first SSE event is never empty. Called exactly once; calling twice is undefined. | +| `stop()` | Cancels the task and releases resources. **Idempotent.** No writes to the cache afterwards. | +| `add_ticker(t)` | Adds to the tracked set. No-op if present. Simulator seeds a price immediately; Massive picks it up on the next poll. | +| `remove_ticker(t)` | Removes from the tracked set **and from the cache** (price and history). No-op if absent. | +| `get_tickers()` | Current tracked set. Synchronous — reads local state only. | + +Two asymmetries are permitted and must not be papered over: + +- **Seeding latency.** `add_ticker` on the simulator makes a price available immediately; on + Massive it takes up to one poll interval. The API contract already accommodates this — + `GET /api/watchlist` returns `price: null` until the first tick, and the UI shows `—`. +- **Cadence.** 500ms versus 15s. Readers must never assume a minimum update rate. This is + exactly what the SSE keepalive in §10.2 exists to handle. + +### `remove_ticker` is destructive — and that is the trap + +Both implementations call `self._cache.remove(ticker)`. Correct for the interface, but it means +removing a ticker whose position is still held silently freezes that position's valuation, P&L, +heatmap tile, and snapshot contribution. §12.2 is the rule that prevents it, and it is the single +most important piece of integration logic in this module because the failure mode is a wrong +number, not an error. + +### Adding a third source + +1. Subclass `MarketDataSource` and implement all five methods. +2. `start()` must **seed the cache before returning**. +3. Never write to the cache after `stop()`; make `stop()` idempotent. +4. `remove_ticker()` must call `cache.remove(ticker)`. +5. Convert timestamps to **Unix epoch seconds as a float** at the boundary. +6. Never let a fetch error kill the background loop — log and retry next cycle. +7. If the underlying client is synchronous, wrap **every** call in `asyncio.to_thread`. +8. Add a branch to `create_market_data_source` and a value to `market_source` in `/api/health`. + +Point 7 is not optional: a blocking HTTP call inside `async def` stalls the event loop for the +whole round trip, which stops the SSE stream and every in-flight request. + +--- + +## 7. The simulator — default source + +`backend/app/market/simulator.py` and `seed_prices.py`. Two classes with a clean split: +**`GBMSimulator` is pure and synchronous; `SimulatorDataSource` owns the async lifecycle and +the cache.** + +``` +┌──────────────────────────────────────────────────────────┐ +│ SimulatorDataSource(MarketDataSource) │ +│ owns the asyncio task, writes to PriceCache │ +│ start / stop / add_ticker / remove_ticker / get_tickers│ +│ │ │ +│ ▼ │ +│ GBMSimulator │ +│ pure math, no I/O, no async, no cache reference │ +│ step() -> {ticker: price} │ +└──────────────────────────────────────────────────────────┘ + │ + ▼ + seed_prices.py (constants only) +``` + +The separation pays off in testing: `GBMSimulator` needs no event loop, no cache, and no mocks. + +### 7.1 The model + +``` +S(t + dt) = S(t) · exp( (μ − σ²/2)·dt + σ·√dt·Z ) +``` + +Three properties earn GBM its place: + +**Prices cannot go negative.** The update is multiplicative — `exp(...)` is always positive. +No clamping, no `max(price, 0.01)` guard, no special case. An additive random walk needs all three. + +**Returns scale correctly with time.** σ is annualized; `√dt` converts it to the tick. The 500ms +cadence is a display choice, not a modelling parameter. + +**The `−σ²/2` term keeps the drift honest.** Without it, μ is not the expected return of the +price — a log-normal artefact. It costs one subtraction and makes the parameters mean what they say. + +### 7.2 Sizing `dt` + +`dt` is expressed against a **trading** year, not a calendar year. Markets are closed most of the +time; using 365×24h would understate per-tick moves by ~4.5×. + +```python +TRADING_SECONDS_PER_YEAR = 252 * 6.5 * 3600 # 5,896,800 +DEFAULT_DT = 0.5 / TRADING_SECONDS_PER_YEAR # ~8.479e-8, sqrt(dt) = 2.912e-4 +``` + +What that produces per tick at the seed prices: + +| Ticker | σ | Per-tick σ | Per-tick $ | Per-minute $ (120 ticks) | +|---|---|---|---|---| +| AAPL | 0.22 | 0.0064% | $0.012 | $0.13 | +| JPM | 0.18 | 0.0052% | $0.010 | $0.11 | +| NVDA | 0.40 | 0.0116% | $0.093 | $1.02 | +| TSLA | 0.50 | 0.0146% | $0.036 | $0.40 | + +This is the number that decides whether the simulation looks right. A cent or two per tick on a +$200 stock means the price **rounds to a genuinely different value most ticks**, so the UI flashes +constantly, while a minute of drift stays in the tens of cents — what a real quote screen looks +like. Larger reads as a crash; smaller looks frozen. + +### 7.3 Correlation via Cholesky + +Independent draws would show tech stocks moving in opposite directions half the time. The eye +notices immediately. Standard fix: draw `n` independent normals, multiply by the Cholesky factor +`L` of the correlation matrix `C = L·Lᵀ`. + +Constants live in `seed_prices.py`, not in the simulator: + +```python +CORRELATION_GROUPS = { + "tech": {"AAPL", "GOOGL", "MSFT", "AMZN", "META", "NVDA", "NFLX"}, + "finance": {"JPM", "V"}, +} + +INTRA_TECH_CORR = 0.6 # tech stocks move together +INTRA_FINANCE_CORR = 0.5 # finance stocks move together +CROSS_GROUP_CORR = 0.3 # between sectors, and for unknown tickers +TSLA_CORR = 0.3 # TSLA does its own thing +``` + +Resolved pairwise, first match winning: + +```python +@staticmethod +def _pairwise_correlation(t1: str, t2: str) -> float: + tech = CORRELATION_GROUPS["tech"] + finance = CORRELATION_GROUPS["finance"] + + # TSLA is in the tech set but behaves independently + if t1 == "TSLA" or t2 == "TSLA": + return TSLA_CORR + + if t1 in tech and t2 in tech: + return INTRA_TECH_CORR + if t1 in finance and t2 in finance: + return INTRA_FINANCE_CORR + + return CROSS_GROUP_CORR +``` + +The TSLA clause is checked first on purpose: TSLA is a tech-set member for every other purpose, +but a demo where TSLA visibly decouples from the pack is more convincing than one where everything +moves in lockstep. `CROSS_GROUP_CORR` doubles as the default for any unknown symbol, which is what +makes §7.5 work. + +```python +def _rebuild_cholesky(self) -> None: + n = len(self._tickers) + if n <= 1: + self._cholesky = None # a single ticker needs no correlation + return + + corr = np.eye(n) + for i in range(n): + for j in range(i + 1, n): + rho = self._pairwise_correlation(self._tickers[i], self._tickers[j]) + corr[i, j] = rho + corr[j, i] = rho + + self._cholesky = np.linalg.cholesky(corr) +``` + +Rebuilt on every add and remove — `O(n²)` to build plus `O(n³)` to factor, on `n < 50`. That is +microseconds, and watchlist edits are human-speed, so caching it would be complexity without benefit. + +> **Known risk.** `np.linalg.cholesky` raises `LinAlgError` on a matrix that is not positive +> definite, and the call is unguarded. The current block structure (0.6 / 0.5 / 0.3) was verified +> positive definite at 7, 20, and 40 tickers — but raising `INTRA_TECH_CORR` toward 1.0, or adding +> a group whose intra-group correlation is *below* the cross-group value, can break +> positive-definiteness and take down `add_ticker`. Anyone editing these constants must re-run the +> test in §14.2. + +### 7.4 The tick + +`step()` is the hot path — every 500ms, for every ticker. + +```python +def step(self) -> dict[str, float]: + """Advance all tickers by one time step. Returns {ticker: new_price}.""" + n = len(self._tickers) + if n == 0: + return {} + + z_independent = np.random.standard_normal(n) + if self._cholesky is not None: + z_correlated = self._cholesky @ z_independent + else: + z_correlated = z_independent + + result: dict[str, float] = {} + for i, ticker in enumerate(self._tickers): + params = self._params[ticker] + mu, sigma = params["mu"], params["sigma"] + + drift = (mu - 0.5 * sigma**2) * self._dt + diffusion = sigma * math.sqrt(self._dt) * z_correlated[i] + self._prices[ticker] *= math.exp(drift + diffusion) + + if random.random() < self._event_prob: + shock_magnitude = random.uniform(0.02, 0.05) + shock_sign = random.choice([-1, 1]) + self._prices[ticker] *= 1 + shock_magnitude * shock_sign + + result[ticker] = round(self._prices[ticker], 2) + + return result +``` + +Two details worth pointing out: + +**Full precision is kept internally; only the returned value is rounded.** Rounding the stored +state would accumulate quantization error into a slow systematic drift over thousands of ticks. + +**One `standard_normal(n)` call per tick, not `n` calls.** A single vectorized draw feeding one +matrix multiply is why this stays negligible at 500ms. + +**Random events** fire at `event_probability = 0.001` per ticker per tick. With 10 tickers at +2 ticks/second the expected wait is `1 / (10 × 2 × 0.001) = 50 seconds` — frequent enough that +something happens during a demo, rare enough that the series is not pure noise. The shock +multiplies the price directly rather than feeding through GBM, so it is a genuine discontinuity — +a gap, which is what real news does to a stock. + +`_tickers` is an **ordered list** that indexes into the Cholesky matrix: row `i` corresponds to +`_tickers[i]`. That is why add and remove must both rebuild. `__init__` adds every ticker via +`_add_ticker_internal` and rebuilds **once** at the end — `O(n³)` instead of `O(n⁴)` on startup. + +### 7.5 Unknown tickers + +Any symbol passing the API-level pattern `^[A-Z][A-Z.]{0,5}$` works, with no allowlist. The AI +chat can add anything the user names, and it behaves plausibly. + +```python +def _add_ticker_internal(self, ticker: str) -> None: + if ticker in self._prices: + return + self._tickers.append(ticker) + self._prices[ticker] = SEED_PRICES.get(ticker, random.uniform(50.0, 300.0)) + self._params[ticker] = TICKER_PARAMS.get(ticker, dict(DEFAULT_PARAMS)) +``` + +- **Price**: `SEED_PRICES`, else uniform $50–$300 — where most large-cap US equities trade. +- **Parameters**: `TICKER_PARAMS`, else `DEFAULT_PARAMS` (σ=0.25, μ=0.05) — a mid-range large cap. +- **Correlation**: no sector membership, so `CROSS_GROUP_CORR` (0.3) against everything. + +`dict(DEFAULT_PARAMS)` **copies** rather than sharing the module-level dict. Without the copy, +tuning one unknown ticker's σ would mutate the default for every unknown ticker at once. + +This is a real advantage over the Massive path, where an unknown symbol never produces a price +and sits at `—` forever (§8.5). + +### 7.6 `SimulatorDataSource` — the async wrapper + +```python +class SimulatorDataSource(MarketDataSource): + def __init__(self, price_cache, update_interval=0.5, event_probability=0.001): ... + + async def start(self, tickers: list[str]) -> None: + self._sim = GBMSimulator(tickers=tickers, event_probability=self._event_prob) + # Seed the cache so the first SSE event carries real prices + for ticker in tickers: + price = self._sim.get_price(ticker) + if price is not None: + self._cache.update(ticker=ticker, price=price) + self._task = asyncio.create_task(self._run_loop(), name="simulator-loop") + + async def _run_loop(self) -> None: + while True: + try: + if self._sim: + for ticker, price in self._sim.step().items(): + self._cache.update(ticker=ticker, price=price) + except Exception: + logger.exception("Simulator step failed") + await asyncio.sleep(self._interval) +``` + +Three deliberate choices: + +**Seed the cache in `start()` before creating the task.** The first SSE event then carries real +prices rather than an empty object, so the watchlist never renders as ten dashes on load. + +**`add_ticker` seeds immediately.** The new ticker has a price on the very next SSE event, with no +wait for the following step — the reason adding a ticker feels instant. + +**The `try` is inside the loop, around the step.** An exception logs and the loop continues on the +next interval. Wrapping the loop instead would let one bad tick kill the feed permanently. This is +the one place defensive handling is warranted: a background task has no caller to propagate to, +and a dead price feed is a dead app. + +`stop()` cancels the task, awaits it, and swallows `CancelledError` — the normal shutdown path, +not an error. + +### 7.7 Parameters + +`seed_prices.py` holds constants only. Prices are realistic as of project creation; σ and μ are annualized. + +| Ticker | Seed | σ | μ | Note | +|---|---|---|---|---| +| AAPL | $190 | 0.22 | 0.05 | | +| GOOGL | $175 | 0.25 | 0.05 | | +| MSFT | $420 | 0.20 | 0.05 | | +| AMZN | $185 | 0.28 | 0.05 | | +| TSLA | $250 | 0.50 | 0.03 | High volatility, decorrelated | +| NVDA | $800 | 0.40 | 0.08 | High volatility, strong drift | +| META | $500 | 0.30 | 0.05 | | +| JPM | $195 | 0.18 | 0.04 | Low volatility (bank) | +| V | $280 | 0.17 | 0.04 | Low volatility (payments) | +| NFLX | $600 | 0.35 | 0.05 | | +| *unknown* | $50–300 | 0.25 | 0.05 | `DEFAULT_PARAMS` | + +The σ spread is what makes the watchlist readable at a glance: V and JPM barely move while NVDA +and TSLA jump, so the grid has texture instead of ten tickers twitching identically. + +There is **no mean reversion and no session boundary.** Prices random-walk from their seed for as +long as the container runs. Over a demo that looks like a trading day; over a week of uptime a +ticker may wander far. That is correct GBM behavior and not worth correcting — state is in memory +only, so a restart returns everything to seed. + +--- + +## 8. The Massive client — optional real data + +`backend/app/market/massive_client.py`. Verified against the `massive` SDK **2.2.0** installed in +`backend/.venv`. + +### 8.1 Why one snapshot endpoint, polled + +The free tier allows **5 requests/minute** — one request every 12 seconds at best. Per-ticker +endpoints are therefore unusable: 10 watchlist tickers via `get_last_trade` would be 10 requests +per cycle, blowing the entire budget in one poll. + +**The design must fetch all tickers in a single request.** That is +`GET /v2/snapshot/locale/us/markets/stocks/tickers`, one request returning the current state of +every ticker named: + +```python +from massive import RESTClient +from massive.rest.models import SnapshotMarketType + +client = RESTClient(api_key="YOUR_KEY") + +snapshots = client.get_snapshot_all( + market_type=SnapshotMarketType.STOCKS, + tickers=["AAPL", "GOOGL", "MSFT", "AMZN", "TSLA"], +) + +for snap in snapshots: + print(snap.ticker, snap.last_trade.price, snap.todays_change_percent) +``` + +The SDK joins a list into a comma-separated string, so passing `list[str]` is correct. +Default poll interval is **15 seconds**, which leaves headroom under the free tier even if a poll +overruns. Paid tiers can drop to 2–5 seconds via `poll_interval`. + +The v3 unified snapshot (`list_universal_snapshots`) is the alternative; it reports unknown +tickers explicitly with an `error` field instead of silently omitting them. FinAlly stays on v2: +a 10-ticker watchlist never approaches v3's 250-ticker limit, v2 is a single non-paginated +request, and per-ticker validation feedback is marginal when the simulator is the default path. +v3 is the right upgrade if that feedback is ever wanted. + +### 8.2 `RESTClient` is synchronous — wrap every call + +It is `urllib3`-based. Calling it from `async def` blocks the event loop for the whole HTTP round +trip, which in this app means visibly stuttering prices on the SSE stream. + +```python +snapshots = await asyncio.to_thread(self._fetch_snapshots) +``` + +It also **retries 429 internally** (3 attempts, honoring `Retry-After`), so a rate-limited poll +blocks its worker thread rather than failing fast. That is fine — the worker is not the event loop. + +### 8.3 Timestamp units — the trap + +Massive uses three different time units across endpoints and the SDK passes them through unchanged. + +| Source | Attribute | Unit | To Unix seconds | +|---|---|---|---| +| Snapshot `lastTrade` | `sip_timestamp` | **nanoseconds** | `/ 1_000_000_000` | +| Snapshot `lastQuote` | `sip_timestamp` | **nanoseconds** | `/ 1_000_000_000` | +| Snapshot top level | `updated` | **nanoseconds** | `/ 1_000_000_000` | +| Snapshot `min` | `timestamp` | **milliseconds** | `/ 1_000` | +| Aggregates (`Agg`, `PreviousCloseAgg`, grouped) | `timestamp` | **milliseconds** | `/ 1_000` | + +And **attribute names never match JSON keys.** The wire format is single-letter (`p`, `s`, `t`, +`x`); `from_dict` maps those to readable attributes. `@modelclass` builds a plain dataclass with +no `__getattr__` fallback, so reading a key name raises `AttributeError`. + +| `LastTrade` attribute | JSON key | Units | +|---|---|---| +| `price` | `p` | dollars | +| `size` | `s` | shares | +| `sip_timestamp` | `t` | **nanoseconds** | +| `exchange` | `x` | exchange ID | + +### 8.4 Two defects in the shipped client — reproduced, not inferred + +`_poll_once` currently reads: + +```python +price = snap.last_trade.price +timestamp = snap.last_trade.timestamp / 1000.0 # AttributeError, then wrong unit +``` + +Reproduction against the installed SDK, run on 2026-09-01: + +```python +from massive.rest.models.snapshot import TickerSnapshot + +snap = TickerSnapshot.from_dict({ + "ticker": "AAPL", + "lastTrade": {"p": 190.52, "s": 100, "t": 1755873791482000000, "x": 4}, +}) + +snap.last_trade.price # 190.52 +snap.last_trade.sip_timestamp # 1755873791482000000 +hasattr(snap.last_trade, "timestamp") # False +``` + +**Defect 1 — `last_trade.timestamp` does not exist, so the Massive path writes nothing at all.** +The loop wraps each snapshot in `except (AttributeError, TypeError)` and merely logs a warning, so +the exception is swallowed once per ticker on every poll. The symptom is not a crash: it is a +watchlist where every ticker shows `—` forever, with `Skipping snapshot for AAPL` in the logs. + +**Defect 2 — the divisor is wrong by 10⁶.** Even with the attribute corrected, `/ 1000.0` treats +nanoseconds as milliseconds: `1755873791482000000 / 1000` ≈ 1.76 × 10¹⁵ seconds, roughly 55 million +years in the future. Any chart keyed on that timestamp is unusable. + +**Why 94% coverage did not catch either.** `tests/market/test_massive.py` builds snapshots from +`MagicMock`, which answers to any attribute name: + +```python +snap.last_trade.timestamp = timestamp_ms # an attribute the real model does not have +``` + +`test_timestamp_conversion` then locks in the wrong unit as well. The lesson generalizes: +**mocking a third-party model tests your assumptions about the library, not the library.** +Parsing tests must go through the real `TickerSnapshot.from_dict` with a documented payload +(§14.4). That test needs no network and would have failed on its first run. + +### 8.5 The corrected parse + +```python +NANOS_PER_SECOND = 1_000_000_000 + +for snap in snapshots: + trade = snap.last_trade + if trade is None or trade.price is None: + continue # no print yet today; leave the ticker showing "—" + self._cache.update( + ticker=snap.ticker, + price=trade.price, + timestamp=( + trade.sip_timestamp / NANOS_PER_SECOND + if trade.sip_timestamp + else time.time() + ), + ) + processed += 1 +``` + +**Guarding on `is None` rather than catching `AttributeError` is what makes the difference.** +A genuinely absent field is a normal condition to handle; a misspelled attribute is a bug that +should be loud. The existing blanket `except AttributeError` is precisely what hid defect 1. + +### 8.6 Error handling in the poll loop + +The SDK raises only two exception types (`massive/exceptions.py`): `AuthError` (empty or missing +key, raised at construction) and `BadResponse` (any non-200 surviving the retry policy). +`urllib3` raises its own for connection failures and timeouts. + +```python +from massive.exceptions import AuthError, BadResponse + +async def _poll_once(self) -> None: + if not self._tickers or not self._client: + return + try: + snapshots = await asyncio.to_thread(self._fetch_snapshots) + except AuthError: + logger.error("Massive API key rejected — the source does not fall back automatically") + raise # unrecoverable: do not retry on a loop + except BadResponse as e: + logger.warning("Massive returned an error response: %s", e) + return # transient: retry next interval + except Exception: + logger.exception("Massive poll failed") + return + ... # the §8.5 parse +``` + +`start()` performs one poll synchronously **before** creating the task, so the cache is warm +before the first client connects: + +```python +async def _poll_loop(self) -> None: + """Poll on interval. The first poll already happened in start().""" + while True: + await asyncio.sleep(self._interval) + await self._poll_once() +``` + +### 8.7 Behaviors to surface in the README + +Properties of the data source, not bugs — users will otherwise report them as bugs: + +- **Unknown symbols vanish silently.** The v2 snapshot omits tickers it does not recognize; there + is no error entry. The ticker sits in the watchlist showing `—` indefinitely. +- **Prices freeze outside market hours.** Overnight, at weekends, and on holidays the snapshot + returns the previous session's last trade. The UI looks broken but is correct. **This is the + main reason the simulator is the default.** +- **Free-tier data is 15 minutes delayed**, so prices will not match any other quote source the + user has open. +- **Snapshot data is cleared at midnight ET** and repopulates from about 4am ET. Between those + times `last_trade` may be absent entirely — exactly the `None` case §8.5 guards. + +`client.get_market_status()` is worth one call to explain a frozen feed rather than leaving the +user guessing. + +### 8.8 Live verification + +Run once a real key exists — it confirms auth, the multi-ticker snapshot, and unit conversion in +one pass: + +```python +# backend/scripts/verify_massive.py +"""Smoke-test the Massive REST API against a live key.""" + +import os +from datetime import UTC, datetime + +from massive import RESTClient +from massive.rest.models import SnapshotMarketType + +NANOS_PER_SECOND = 1_000_000_000 +TICKERS = ["AAPL", "GOOGL", "MSFT", "NVDA", "TSLA"] + + +def main() -> None: + client = RESTClient(api_key=os.environ["MASSIVE_API_KEY"]) + + print(f"market: {client.get_market_status().market}") + + snapshots = client.get_snapshot_all(SnapshotMarketType.STOCKS, TICKERS) + print(f"requested {len(TICKERS)}, received {len(snapshots)}") + + for snap in snapshots: + trade = snap.last_trade + if trade is None or trade.price is None: + print(f"{snap.ticker}: no trade data") + continue + when = datetime.fromtimestamp(trade.sip_timestamp / NANOS_PER_SECOND, UTC) + print(f"{snap.ticker}: ${trade.price:.2f} at {when:%Y-%m-%d %H:%M:%S} UTC") + + missing = set(TICKERS) - {s.ticker for s in snapshots} + if missing: + print(f"absent from response (unknown or untraded): {sorted(missing)}") + + +if __name__ == "__main__": + main() +``` + +```bash +cd backend && uv run python scripts/verify_massive.py +``` + +Expected: a market status and five priced tickers with timestamps **in the recent past**. +Timestamps far in the future mean the divisor regressed; `AttributeError` means §8.4 regressed. + +--- + +## 9. Selection — `create_market_data_source` + +```python +def create_market_data_source(price_cache: PriceCache) -> MarketDataSource: + """Create the market data source indicated by the environment. + + MASSIVE_API_KEY set and non-empty -> MassiveDataSource (real data) + otherwise -> SimulatorDataSource (GBM simulation) + + Returns an unstarted source; the caller must await source.start(tickers). + """ + api_key = os.environ.get("MASSIVE_API_KEY", "").strip() + + if api_key: + logger.info("Market data source: Massive API (real data)") + return MassiveDataSource(api_key=api_key, price_cache=price_cache) + + logger.info("Market data source: GBM Simulator") + return SimulatorDataSource(price_cache=price_cache) +``` + +**`.strip()` before the truth test is deliberate.** `.env` files routinely contain +`MASSIVE_API_KEY=` or a stray space, and a whitespace-only key would otherwise select the Massive +path and then fail every poll with a 401. Empty means empty. + +**The choice is made once at startup and never at runtime.** A source that silently switched to +the simulator after a Massive outage would show users invented prices while they believed they +were seeing the market. Rejected keys and failed polls are logged; they do not change the source. +`GET /api/health` reports which one is live: + +```json +{"status": "ok", "market_source": "simulator", "llm_mock": false} +``` + +Returning an **unstarted** source keeps construction synchronous and lets the caller decide the +ticker set from the database — the factory has no business reading tables. + +--- + +## 10. The SSE stream + +### 10.1 What ships + +`GET /api/stream/prices`, `Content-Type: text/event-stream`. The generator opens with +`retry: 1000`, then pushes the **entire cache as one JSON object** whenever `version` changes, +polled every 500ms: + +``` +retry: 1000 + +data: {"AAPL": {"ticker": "AAPL", "price": 190.52, "previous_price": 190.48, "timestamp": 1755873791.482, "change": 0.04, "change_percent": 0.021, "direction": "up"}, "GOOGL": {...}} +``` + +One event carries every ticker. The client replaces its price map wholesale — no merge logic, no +missed-update reconciliation. Because a fresh generator starts at `last_version = -1`, the first +comparison always differs, so **every connecting client immediately receives a full snapshot**, +including after a reconnect. That is why no separate snapshot endpoint exists. + +Response headers matter as much as the payload: + +```python +return StreamingResponse( + _generate_events(price_cache, request), + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "X-Accel-Buffering": "no", # disable nginx buffering if proxied + }, +) +``` + +**Why poll-and-push instead of event-driven?** A 500ms integer comparison is cheaper to reason +about than a pub/sub fan-out across an arbitrary number of generators, and it naturally coalesces: +if the cache updated ten tickers since the last check, the client gets one event, not ten. + +### 10.2 Keepalive — to implement + +When the version has not changed for 15 seconds, emit an SSE comment line. The complete generator: + +```python +KEEPALIVE_SECONDS = 15.0 + + +async def _generate_events( + price_cache: PriceCache, + request: Request, + interval: float = 0.5, +) -> AsyncGenerator[str, None]: + """Yield SSE events whenever the cache version changes; ping when it does not.""" + yield "retry: 1000\n\n" + + last_version = -1 + last_sent = time.monotonic() + client_ip = request.client.host if request.client else "unknown" + logger.info("SSE client connected: %s", client_ip) + + try: + while True: + if await request.is_disconnected(): + logger.info("SSE client disconnected: %s", client_ip) + break + + current_version = price_cache.version + if current_version != last_version: + last_version = current_version + prices = price_cache.get_all() + if prices: + payload = json.dumps({t: u.to_dict() for t, u in prices.items()}) + yield f"data: {payload}\n\n" + last_sent = time.monotonic() + elif time.monotonic() - last_sent >= KEEPALIVE_SECONDS: + yield ": ping\n\n" + last_sent = time.monotonic() + + await asyncio.sleep(interval) + except asyncio.CancelledError: + logger.info("SSE stream cancelled for: %s", client_ip) +``` + +Without this, a Massive-backed feed sends no bytes between 15-second polls. That idle-times-out +through proxies and leaves the frontend unable to distinguish a quiet market from a dead +connection. The frontend indicator — green on `onopen`, yellow on `onerror`, red after a gap +beyond ~40 seconds — depends on it. + +A line beginning with `:` is a comment in the SSE grammar: `EventSource` ignores it entirely, so +it costs the client nothing while keeping the socket warm. + +--- + +## 11. `GET /api/prices/{ticker}/history` — to implement + +Backed by `PriceCache.get_history` (§5.2). It belongs in `stream.py` next to the SSE endpoint, +since both are pure cache readers with no database involvement. + +```python +history_router = APIRouter(prefix="/api/prices", tags=["prices"]) + + +def create_history_router(price_cache: PriceCache) -> APIRouter: + @history_router.get("/{ticker}/history") + async def get_price_history(ticker: str, limit: int = 600) -> dict: + """Rolling in-memory price history for the main chart. + + Returns an empty `points` list for an untracked ticker — not a 404, + so the chart draws nothing rather than erroring. + """ + ticker = ticker.strip().upper() + limit = max(1, min(limit, HISTORY_MAXLEN)) + points = price_cache.get_history(ticker, limit=limit) + return { + "ticker": ticker, + "points": [{"timestamp": ts, "price": price} for ts, price in points], + } + + return history_router +``` + +Response: + +```json +{"ticker": "AAPL", "points": [{"timestamp": 1755873791.482, "price": 190.52}]} +``` + +Oldest-first, matching what Recharts wants for a left-to-right time axis. `limit` is clamped +rather than validated with a 400 — a chart asking for 10,000 points should get 600, not an error. + +This endpoint reads only in-memory state, so `async def` is correct here; there is no SQLite call +to keep off the event loop. + +--- + +## 12. Wiring + +### 12.1 Lifespan + +One `PriceCache` and one source per process, owned by the FastAPI lifespan. + +```python +from contextlib import asynccontextmanager + +from fastapi import FastAPI + +from app.market import PriceCache, create_market_data_source, create_stream_router + + +@asynccontextmanager +async def lifespan(app: FastAPI): + init_db() # lazy schema creation + seed (PLAN.md §7) + + cache = PriceCache() + source = create_market_data_source(cache) + + # Reconciliation: watchlist ∪ held positions, not just the watchlist + tickers = sorted(set(get_watchlist_tickers()) | set(get_position_tickers())) + await source.start(tickers) + + app.state.price_cache = cache + app.state.market_source = source + try: + yield + finally: + await source.stop() + + +app = FastAPI(lifespan=lifespan) + +# 1. API routers FIRST +app.include_router(create_stream_router(cache)) +app.include_router(create_history_router(cache)) +# ... portfolio, watchlist, chat routers ... +# 2. static assets +# 3. catch-all -> index.html +``` + +Three things this gets right and are easy to get wrong: + +**Reading both tables at startup**, not just the watchlist, is what makes a position held across +a restart come back with a live price. Without it, an off-watchlist holding valuates at `avg_cost` +forever and the snapshot task stalls under the "skip if any held ticker has no price" rule. + +**The cache and source are passed explicitly** (via router factories or `app.state`) rather than +held in module globals, which is what keeps tests able to construct an isolated cache per test. + +**Mount all `/api/*` routers before the static file mount.** A `StaticFiles(html=True)` mount at +`/` registered first shadows every endpoint, including the SSE stream (`PLAN.md` §11). + +Route handlers reach the cache through `app.state` or a dependency: + +```python +def get_price_cache(request: Request) -> PriceCache: + return request.app.state.price_cache + + +def get_market_source(request: Request) -> MarketDataSource: + return request.app.state.market_source +``` + +### 12.2 The tracked ticker set + +**The tracked set is `watchlist ∪ {tickers with a non-zero position}`.** + +The two sets diverge the moment a user buys TSLA and then removes it from the watchlist. The +position still needs a live price for valuation, P&L, the heatmap, and snapshots. + +| Trigger | Action | +|---|---| +| `POST /api/watchlist` | always `await source.add_ticker(t)` | +| `DELETE /api/watchlist/{t}` | `await source.remove_ticker(t)` **only if no position in `t` is held** | +| Buy a ticker not currently tracked | `await source.add_ticker(t)` as part of trade execution | +| Sell a position to zero | if `t` is not on the watchlist, `await source.remove_ticker(t)` | +| `POST /api/reset` | re-sync the tracked set to exactly the ten default tickers | + +One helper keeps the rule in one place rather than at four call sites: + +```python +async def untrack_if_unused(source: MarketDataSource, ticker: str) -> None: + """Stop tracking a ticker only if it is neither watched nor held.""" + if is_on_watchlist(ticker) or has_position(ticker): + return + await source.remove_ticker(ticker) +``` + +### 12.3 Ticker validation at the boundary + +Applied at `POST /api/watchlist`, `POST /api/portfolio/trade`, and every LLM-proposed action, so +the market layer only ever sees canonical symbols: + +```python +TICKER_PATTERN = re.compile(r"^[A-Z][A-Z.]{0,5}$") + + +def normalize_ticker(raw: str) -> str: + """Uppercase and validate. Raises ValueError with the user-facing message.""" + ticker = raw.strip().upper() + if not TICKER_PATTERN.match(ticker): + raise ValueError("Invalid ticker symbol") + return ticker +``` + +No allowlist. Any symbol matching the pattern is accepted; the simulator invents plausible +behavior for it, and under Massive an unknown symbol shows `—`. Rejecting unknown symbols would +make the LLM's `watchlist_changes` feature feel broken. + +Uppercasing is not cosmetic: the `UNIQUE(user_id, ticker)` constraints would otherwise happily +hold both `AAPL` and `aapl`. + +--- + +## 13. Failure modes + +| Situation | Behavior | Where | +|---|---|---| +| Empty ticker list at startup | `step()` returns `{}`, SSE sends nothing until a ticker is added | §7.4 | +| One bad simulator tick | Logged, loop continues next interval | §7.6 | +| Massive poll fails (429, network) | Logged, cache keeps last prices, retry next interval | §8.6 | +| Massive key rejected | `AuthError` re-raised; **no automatic fallback to the simulator** | §8.6, §9 | +| Ticker has no `last_trade` yet | Skipped; ticker shows `—` | §8.5 | +| Held ticker has no cached price | Portfolio values it at `avg_cost`; snapshot task skips the write entirely | `PLAN.md` §7 | +| Ticker removed while held | Prevented by `untrack_if_unused` | §12.2 | +| Client disconnects mid-stream | `request.is_disconnected()` breaks the generator | §10.2 | +| Quiet feed (Massive, 15s polls) | `: ping` every 15s keeps the connection and the indicator alive | §10.2 | +| History requested for untracked ticker | `{"ticker": "X", "points": []}` | §11 | + +--- + +## 14. Testing + +Current state: **73 tests, 91% coverage** on the market module. `stream.py` sits at 33% — the SSE +generator is the least-tested code in the subsystem and the keepalive change is a good moment to +fix that. + +```bash +cd backend +uv run --extra dev pytest -v +uv run --extra dev pytest --cov=app --cov-report=term-missing +``` + +### 14.1 A stub source + +The cache and the tracked-set rules can be tested without either real source: + +```python +class StubDataSource(MarketDataSource): + """Records lifecycle calls; writes nothing on its own.""" + + def __init__(self, cache: PriceCache) -> None: + self._cache = cache + self._tickers: list[str] = [] + self.started = False + + async def start(self, tickers): self._tickers = list(tickers); self.started = True + async def stop(self): self.started = False + async def add_ticker(self, t): + if t not in self._tickers: + self._tickers.append(t) + async def remove_ticker(self, t): + self._tickers = [x for x in self._tickers if x != t] + self._cache.remove(t) + def get_tickers(self): return list(self._tickers) +``` + +### 14.2 Simulator + +Seed **both** RNGs — the simulator uses `numpy.random` for the normal draws and stdlib `random` +for events: + +```python +def test_step_is_reproducible(): + np.random.seed(42) + random.seed(42) + sim = GBMSimulator(tickers=["AAPL", "GOOGL"]) + first = sim.step() + + np.random.seed(42) + random.seed(42) + sim = GBMSimulator(tickers=["AAPL", "GOOGL"]) + assert sim.step() == first +``` + +Statistical properties need wide tolerances and events disabled — a 5% jump is a massive outlier +at this `dt` and would dominate the sample variance: + +```python +def test_realised_volatility_is_close_to_sigma(): + sim = GBMSimulator(tickers=["AAPL"], event_probability=0.0) + prices = [sim.get_price("AAPL")] + for _ in range(20_000): + prices.append(sim.step()["AAPL"]) + + log_returns = np.diff(np.log(prices)) + realised = log_returns.std() / np.sqrt(GBMSimulator.DEFAULT_DT) + assert 0.15 < realised < 0.35 # nominal sigma is 0.22 + + +def test_tech_tickers_are_positively_correlated(): + sim = GBMSimulator(tickers=["AAPL", "MSFT"], event_probability=0.0) + a, m = [], [] + for _ in range(10_000): + p = sim.step() + a.append(p["AAPL"]) + m.append(p["MSFT"]) + + rho = np.corrcoef(np.diff(np.log(a)), np.diff(np.log(m)))[0, 1] + assert rho > 0.4 # nominal 0.6 + + +def test_correlation_matrix_stays_positive_definite(): + """Run after ANY change to the correlation constants in seed_prices.py.""" + tickers = list(SEED_PRICES) + [f"UNK{i}" for i in range(40)] + GBMSimulator(tickers=tickers) # raises LinAlgError if not PD +``` + +Also cover: prices stay strictly positive over thousands of steps; `step()` returns exactly the +current ticker set; add/remove keeps `_tickers`/`_prices`/`_params` consistent and the Cholesky +shape matching; unknown tickers seed within $50–$300 with `DEFAULT_PARAMS`; `remove_ticker` on an +untracked symbol is a no-op. + +### 14.3 Cache and history + +```python +def test_history_is_bounded_and_oldest_first(): + cache = PriceCache(history_maxlen=5) + for i in range(10): + cache.update("AAPL", 100.0 + i, timestamp=float(i)) + + points = cache.get_history("AAPL") + assert len(points) == 5 + assert [ts for ts, _ in points] == [5.0, 6.0, 7.0, 8.0, 9.0] + + +def test_history_is_empty_for_untracked_ticker(): + assert PriceCache().get_history("NOPE") == [] + + +def test_remove_clears_price_and_history(): + cache = PriceCache() + cache.update("AAPL", 190.0) + cache.remove("AAPL") + assert cache.get("AAPL") is None + assert cache.get_history("AAPL") == [] +``` + +Plus: `previous_price` derivation, first-update `flat`, `version` monotonicity, and thread safety +under concurrent writers. + +### 14.4 Massive — through the real model, never `MagicMock` + +This is the test that would have caught both defects in §8.4, and it needs no network: + +```python +from massive.rest.models.snapshot import TickerSnapshot + +def test_snapshot_parse_produces_a_present_day_timestamp(): + snap = TickerSnapshot.from_dict({ + "ticker": "AAPL", + "lastTrade": {"p": 190.52, "s": 100, "t": 1755873791482000000, "x": 4}, + }) + + cache = PriceCache() + source = MassiveDataSource(api_key="x", price_cache=cache) + source._apply_snapshots([snap]) # extract the parse into a testable method + + update = cache.get("AAPL") + assert update.price == 190.52 + assert 1_600_000_000 < update.timestamp < 2_000_000_000 # plausible present, in SECONDS + + +def test_snapshot_without_a_last_trade_is_skipped(): + snap = TickerSnapshot.from_dict({"ticker": "AAPL"}) + cache = PriceCache() + source = MassiveDataSource(api_key="x", price_cache=cache) + source._apply_snapshots([snap]) + assert cache.get("AAPL") is None +``` + +Extracting the parse loop into `_apply_snapshots(snapshots)` is worth the small refactor: it makes +the parse testable without touching HTTP, which is the only part that actually broke. + +### 14.5 Factory and SSE + +- **Factory** — unset, empty, and whitespace-only `MASSIVE_API_KEY` all select the simulator; a + real value selects Massive. +- **SSE** — map-shaped payload, float timestamp, percent-unit `change_percent`, full snapshot on + connect, and a `: ping` after 15 idle seconds. Drive the generator directly with a fake request + object rather than through a live server; the keepalive test is far easier with an injected + `interval` and a monkeypatched clock than with 15 seconds of real waiting. + +### 14.6 Tracked set + +The two regressions that silently produce a frozen position: + +- Removing a watchlist ticker with an open position **keeps** it in the feed. +- Selling to zero while off-watchlist **removes** it. + +### 14.7 Eyeballing it + +```bash +cd backend && uv run market_data_demo.py +``` + +A Rich terminal dashboard of the live simulator — the fastest way to check whether a parameter +change still looks right. Statistical tests confirm σ; only the eye confirms "looks like a +trading terminal". + +--- + +## 15. Implementation order + +Small increments, each independently verifiable. Run `uv run --extra dev pytest` after every step. + +1. **Fix the Massive parse** (§8.5). Extract `_apply_snapshots`, correct the attribute and the + divisor, replace the `MagicMock` tests with `TickerSnapshot.from_dict` tests (§14.4). This is + first because the current code silently produces nothing, and because the fix is provable + offline. +2. **Add rolling history to `PriceCache`** (§5.2). Deque, `get_history`, `remove` clearing both. + Tests in §14.3. +3. **Add `GET /api/prices/{ticker}/history`** (§11). Depends on step 2. +4. **Add the SSE keepalive** (§10.2) and raise `stream.py` coverage off 33% (§14.5). +5. **Wire the lifespan** (§12.1) with startup reconciliation over `watchlist ∪ positions`, and + add `untrack_if_unused` (§12.2) where the watchlist and trade routes are built. + +Steps 1–4 are self-contained in `app/market/`. Step 5 is the seam with the rest of the backend and +should land alongside the portfolio and watchlist routes, not before them. + +--- + +## 16. Configuration reference + +| Setting | Default | Where | Effect | +|---|---|---|---| +| `MASSIVE_API_KEY` | unset | env | Non-empty selects Massive; otherwise simulator | +| `update_interval` | `0.5` | `SimulatorDataSource` | Simulator tick rate — **change `dt` with it** | +| `event_probability` | `0.001` | `SimulatorDataSource` | Shock chance per ticker per tick | +| `poll_interval` | `15.0` | `MassiveDataSource` | Seconds between snapshot requests | +| `HISTORY_MAXLEN` | `600` | `cache.py` | Rolling history depth (~5 min at 500ms) | +| `KEEPALIVE_SECONDS` | `15.0` | `stream.py` | Idle gap before a `: ping` | +| SSE poll `interval` | `0.5` | `stream.py` | How often the version is checked | + +### Tuning the simulator + +| Want | Change | Watch for | +|---|---|---| +| More visible motion | Raise σ in `TICKER_PARAMS` | Above ~0.8 it stops looking like equity | +| Faster updates | `update_interval` | **`DEFAULT_DT` hard-codes the 500ms tick** — see below | +| More drama | Raise `event_probability` | Above ~0.005 the series becomes jumps, not prices | +| Bigger shocks | Widen `random.uniform(0.02, 0.05)` | Beyond ~10% the P&L chart loses all detail | +| Different sectors | Edit `CORRELATION_GROUPS` and coefficients | Re-run the positive-definiteness test (§14.2) | +| A trending market | Raise μ | μ is annualized; even 0.5 is barely visible over a demo | + +**The `DEFAULT_DT` coupling is the one that catches people.** `DEFAULT_DT = 0.5 / TRADING_SECONDS_PER_YEAR` +hard-codes the 500ms tick. Passing `update_interval=0.1` without also passing a matching `dt` runs +the simulation five times faster in model time, and annualized volatility silently becomes 5× what +`TICKER_PARAMS` claims. + +--- + +## 17. Summary + +| Concern | Resolution | +|---|---| +| Two sources, one consumer | `MarketDataSource` ABC + shared `PriceCache` | +| Which source | `create_market_data_source`, decided once at startup from `MASSIVE_API_KEY` | +| Default | Simulator — always alive, no key, no rate limit, any ticker | +| How prices are read | Only from the cache, never from the source | +| Which tickers are live | `watchlist ∪ positions`, reconciled at startup | +| Price model | GBM, per-ticker μ and σ, Cholesky-correlated by sector | +| Timestamp format | Unix epoch seconds (float), converted at each source boundary | +| Update delivery | SSE, full cache per event, on `version` change, `: ping` when idle | +| Chart backfill | 600-point in-memory deque per ticker, never persisted | +| Blocking I/O | `asyncio.to_thread` at the source, always | +| Failure handling | Per-cycle `try` inside the loop; the feed never dies from one bad tick | +| Outstanding work | The five steps in §15 | diff --git a/planning/MARKET_DATA_REVIEW.md b/planning/MARKET_DATA_REVIEW.md new file mode 100644 index 000000000..3ecb1f1b2 --- /dev/null +++ b/planning/MARKET_DATA_REVIEW.md @@ -0,0 +1,242 @@ +# Market Data Backend — Code Review + +**Date:** 2026-09-02 +**Scope:** `backend/app/market/` (8 source files, 730 LOC) and `backend/tests/market/` (7 test files, 1,161 LOC) +**Reviewer:** Claude, in response to issue #5 + +--- + +## 1. Test Execution — Could Not Run + +This review's environment does not have permission to execute shell commands that +run the Python interpreter or install dependencies (`uv sync`, `uv run pytest`, even +`python3 -c ...` are all blocked pending approval, and this run has no human +available to approve them). This is the same limitation `planning/MARKET_DATA_SUMMARY.md` +recorded on the previous pass. **No test was executed as part of this review.** + +To get a real pass/fail signal, re-run this task with `Bash(uv sync:*)` and +`Bash(uv run:*)` added to the allowed tools, or run locally: + +```bash +cd backend +uv sync --extra dev +uv run --extra dev pytest -v --cov=app --cov-report=term-missing +uv run --extra dev ruff check app/ tests/ +``` + +In place of execution, every test file was read in full and traced by hand against +the source it exercises (see §4). All 96 tests found in the suite exercise real +code paths correctly as far as static reading can confirm — no test asserts on +behavior the source doesn't actually implement, and no test's mocking hides a +divergence between the mock's shape and the real one (the concern that let a +prior bug through at 94% coverage, per `test_massive.py`'s own docstring). + +**Test count:** 96 across 7 files (`test_models.py` 11, `test_cache.py` 24, +`test_simulator.py` 17, `test_simulator_source.py` 10, `test_factory.py` 7, +`test_massive.py` 17, `test_stream.py` 15 by count of `def test_`/`async def test_` +— slightly higher than the 94 recorded in `MARKET_DATA_SUMMARY.md` §"Test Suite", +consistent with incremental additions since that doc was last updated). + +--- + +## 2. Architecture Assessment + +The module is well-factored and matches `planning/MARKET_DATA_DESIGN.md` and +`planning/MARKET_DATA_SUMMARY.md` closely: + +``` +MarketDataSource (ABC) +├── SimulatorDataSource → GBMSimulator (Cholesky-correlated GBM) +└── MassiveDataSource → Polygon.io REST poller + │ + ▼ + PriceCache (thread-safe, latest price + 600-point rolling history) + │ + ├──→ create_stream_router() → GET /api/stream/prices (SSE, with keepalive) + └──→ create_history_router() → GET /api/prices/{ticker}/history +``` + +**Strengths confirmed by this pass:** + +- Strategy pattern cleanly isolates the two data sources behind `MarketDataSource`; nothing downstream needs to know which is active. +- `PriceUpdate` is `frozen=True, slots=True` — correct choice for a value object shared across threads/tasks. +- `PriceCache` centralizes all locking (`threading.Lock`) around the one mutable structure producers and consumers touch; the API surface (`update`, `get`, `get_all`, `get_price`, `remove`, `get_history`) is small and each method acquires the lock exactly once. +- The GBM math is textbook-correct log-normal price evolution, and the `dt` sizing (`0.5s / (252 * 6.5h * 3600s)`) is derived, not guessed, with the derivation left in a comment. +- Cholesky-based correlated draws (`simulator.py:84-90`) are a genuinely nice touch for a simulator whose only job is to look convincing on a chart. +- The three TODOs recorded as open in `PLAN.md` §13 (SSE keepalive, rolling history, `/history` endpoint) are all implemented and each has direct test coverage (`test_stream.py`). +- The two defects `MARKET_DATA_DESIGN.md` §8.4 recorded against the Massive client (wrong attribute name, nanoseconds-as-milliseconds) are fixed in `massive_client.py:130-136`, and `test_massive.py` deliberately builds real `TickerSnapshot` objects via `TickerSnapshot.from_dict(...)` rather than `MagicMock`, which is exactly the right defense against that class of bug recurring silently. +- `pyproject.toml` already has `[tool.hatch.build.targets.wheel] packages = ["app"]` — the "High" build-breaking bug from the archived 2026-02-10 review (`planning/archive/MARKET_DATA_REVIEW.md` §3.1) is fixed. +- `massive` is a top-level import now (`massive_client.py:9-11`), not a lazy one — the archived review's §3.2 concern about tests being fragile without the package installed no longer applies; `pyproject.toml` lists it as a core dependency. + +--- + +## 3. Issues Found + +### 3.1 `create_stream_router` / `create_history_router` mutate a shared module-level router (Severity: Medium) + +`stream.py:18-19` defines `router` and `history_router` at module scope. Both +factory functions register their route via a closure on these **same shared +objects** rather than creating a fresh `APIRouter()` per call: + +```python +router = APIRouter(prefix="/api/stream", tags=["streaming"]) +history_router = APIRouter(prefix="/api/prices", tags=["prices"]) + +def create_stream_router(price_cache: PriceCache) -> APIRouter: + @router.get("/prices") + async def stream_prices(request: Request) -> StreamingResponse: + ... + return router +``` + +Calling either factory more than once appends another route to the same +underlying router rather than returning an independent one. This was flagged +as a "latent footgun for testing" in the archived review (§3.6) when there +were no tests exercising it; now there are, and it is no longer latent: +`test_stream.py`'s `_history_endpoint()` helper calls `create_history_router(cache)` +fresh in **six different tests**, so `history_router` in the running test +process accumulates six duplicate `/{ticker}/history` routes by the end of +the file. The tests still pass because they grab `router.routes[-1].endpoint` +— the most recently registered one — but this only works by coincidence of +ordering, not because the router is actually being rebuilt. + +The real risk is downstream: once this module is wired into the FastAPI app +(the next piece of work per `PLAN.md` §13 "Still open"), any test that builds +the app more than once per process — a very common pytest pattern (an `app` +fixture instantiated per test, or per module) — will silently accumulate +duplicate routes on every rebuild, since `router`/`history_router` are shared +mutable module state that outlives any single app instance. + +**Fix:** construct a new `APIRouter()` inside each factory function instead of +reusing a module-level instance: + +```python +def create_stream_router(price_cache: PriceCache) -> APIRouter: + router = APIRouter(prefix="/api/stream", tags=["streaming"]) + + @router.get("/prices") + async def stream_prices(request: Request) -> StreamingResponse: + ... + return router +``` + +### 3.2 `PriceCache.update()` treats a falsy timestamp as "no timestamp given" (Severity: Low) + +```python +ts = timestamp or time.time() +``` + +(`cache.py:40`) A caller that explicitly passes `timestamp=0.0` (Unix epoch, +1970-01-01) gets `time.time()` substituted instead, because `0.0` is falsy. +No current caller does this — `massive_client.py` only reaches this path with +`time.time()` already substituted upstream when `sip_timestamp` is falsy — so +this is not exploitable today, but it is a latent correctness gap for any +future caller (e.g., a test replaying historical data from epoch-adjacent +timestamps, or a backfill script). Prefer `timestamp if timestamp is not None +else time.time()`. + +### 3.3 `MassiveDataSource`'s poller task dies silently on `AuthError` (Severity: Low) + +`_poll_once()` deliberately re-raises `AuthError` (`massive_client.py:103-105`) +with the comment "unrecoverable: do not retry on a loop" — a reasonable +choice. But the only place that awaits `self._task` is `stop()` +(`massive_client.py:60-69`), which nothing calls until shutdown. If the key +is revoked *after* `start()` succeeds (rather than being bad from the first +poll), the background task raised inside `_poll_loop()` simply stops running; +asyncio logs "Task exception was never retrieved" at some later point (often +at garbage collection, easy to miss in container logs), and the app has no +other signal that live prices have silently frozen. `test_auth_error_propagates` +confirms the exception propagates out of `_poll_once()`, but there is no test +for what happens to `_poll_loop()` or the app once that happens. + +This is fine as coded for now since nothing outside the market module reads +task health yet, but whoever wires this into the app (`PLAN.md` §13, item 3) +should either attach a `Task.add_done_callback` that logs loudly / flips a +health flag, or have `GET /api/health` report `market_source` as degraded +when the task is dead. Worth a one-line note in `MARKET_DATA_SUMMARY.md` so +it isn't forgotten during integration. + +### 3.4 `PriceCache.version` property reads outside the lock (Severity: Trivial) + +Unchanged from the archived review's §3.4: `cache.py:94-97` reads `self._version` +without acquiring `self._lock`. Safe under CPython's GIL for a single `int` +read, inconsistent with the rest of the class, and only a real concern on a +no-GIL build. Not worth blocking on, but a two-line fix if anyone is passing +through this file for another reason. + +### 3.5 `market_data_demo.py` and `backend/README.md` are outside the reviewed test scope but were not separately verified + +The demo script (`market_data_demo.py`, 205 lines) is referenced by +`MARKET_DATA_SUMMARY.md` as a manual verification tool and has no automated +test coverage, which is appropriate for a Rich terminal demo — flagging only +so it's clear this review's "all tests pass" scope is `backend/tests/market/`, +not the demo script. + +--- + +## 4. Test Suite Assessment (by module) + +| Module | File | Assessment | +|---|---|---| +| `models.py` | `test_models.py` (11 tests) | Complete: creation, `change`/`change_percent`/`direction` in both directions, zero-previous-price edge case, `to_dict()` shape, and frozen-dataclass immutability. No gaps. | +| `cache.py` | `test_cache.py` (24 tests) | Thorough. Covers direction transitions, `version` monotonicity, `__len__`/`__contains__`, price rounding, custom timestamps, and a dedicated `TestPriceHistory` class covering bounding, ordering, per-ticker isolation, limit-narrower-than-stored, and that `remove()` clears history without touching other tickers. No test for concurrent multi-thread writes (the lock is exercised only single-threaded) — the archived review flagged this as missing in §4.2 and it remains missing; low priority since the logic is simple enough to verify by inspection. | +| `interface.py` | (no dedicated file; exercised transitively via simulator/massive tests) | Reasonable — it's an ABC with no logic of its own. | +| `seed_prices.py` | `test_simulator.py`, `test_factory.py` (transitively) | No dedicated test file, but every constant is exercised indirectly through `GBMSimulator` tests (`_pairwise_correlation` tests cover tech/finance/TSLA/cross-sector explicitly). Fine given it's pure data. | +| `simulator.py` | `test_simulator.py` (17), `test_simulator_source.py` (10) | Strong. `GBMSimulator`: positivity over 10,000 steps, seed matching, add/remove (including duplicate/nonexistent no-ops), unknown-ticker random seeding, Cholesky construction/teardown on ticker count crossing 1↔2, all four correlation branches, `dt` sanity, and rounding. `SimulatorDataSource`: cache population on start, periodic updates via real `asyncio.sleep`, idempotent stop, dynamic add/remove, empty-start, and exception resilience. The timing-based assertions (`asyncio.sleep(0.3)` then assert version advanced) are inherently a little flaky under CI load, but the margins used (3-6x the interval) are generous enough to be low-risk. | +| `massive_client.py` | `test_massive.py` (17) | Strong, and specifically hardened against the exact bug class that shipped previously — `_apply_snapshots` is tested against real `TickerSnapshot.from_dict(...)` objects, not mocks, for timestamp conversion, missing-trade skipping, mixed valid/invalid batches, and multi-ticker updates. Polling lifecycle covers success, `BadResponse` (swallowed), `AuthError` (re-raised, see §3.3), generic exceptions (swallowed), ticker add/remove with normalization, and start/stop idempotency. No gap of consequence. | +| `stream.py` | `test_stream.py` (15) | Was 31% covered and untested in the archived review; now has direct coverage of the async generator via a hand-rolled `FakeRequest`, including the retry directive, snapshot-on-connect (and thus reconnect), the frozen payload field set, keepalive timing (via `monkeypatch` on `KEEPALIVE_SECONDS` rather than a real 15s wait — good practice), a fresh data event following a ping, disconnect handling, and the empty-cache case. `create_history_router`'s endpoint is tested for ordering, unknown-ticker empty response, normalization, and limit clamping in both directions. The one real gap is architectural, not a missing test: see §3.1 — the tests would catch a *regression* in behavior but not the router-reuse issue itself, since grabbing `routes[-1]` happens to paper over it. | +| `factory.py` | `test_factory.py` (7) | Complete for its size: unset/empty/whitespace-only key → simulator, set key → Massive, and that both branches thread the cache reference through correctly. | + +**Net assessment:** the suite is comprehensive and, importantly, methodologically +careful — the deliberate choice to build real `TickerSnapshot` objects instead of +`MagicMock` in `test_massive.py` is the single best thing about this test suite, +since it's precisely what would have caught the `last_trade.timestamp` / +`sip_timestamp` bug the archived review found. No test was found asserting +something the source doesn't do, and no source behavior of consequence lacks a +test, with the caveats above (concurrency, and the router-reuse issue masked +by test ordering). + +--- + +## 5. Comparison Against the Prior Review + +`planning/archive/MARKET_DATA_REVIEW.md` (2026-02-10) recorded 7 issues. Status now: + +| # | Issue | Status | +|---|---|---| +| 3.1 | Missing hatchling wheel config | **Fixed** | +| 3.2 | Massive tests fragile without the `massive` package | **Fixed** (now a core dependency, imported at module level) | +| 3.3 | `_generate_events` return type `-> None` instead of `AsyncGenerator` | **Fixed** (`stream.py:87`) | +| 3.4 | `PriceCache.version` reads outside the lock | **Still open** (§3.4 above, trivial) | +| 3.5 | `SimulatorDataSource.get_tickers` reached into `GBMSimulator._tickers` | **Fixed** — `GBMSimulator.get_tickers()` now exists (`simulator.py:140-142`) and is used | +| 3.6 | Module-level router registered on repeated calls | **Still open, and now demonstrated by the test suite itself** (§3.1 above, upgraded to Medium given it will bite during app integration) | +| 3.7 | Unused imports in tests | **Fixed** — no unused `pytest`/`math`/`asyncio` imports found in any current test file | + +Also confirmed fixed: the two Massive parsing defects `MARKET_DATA_DESIGN.md` +§8.4 described (wrong attribute name, nanosecond/millisecond confusion), and +all three items `PLAN.md` §13 listed as open TODOs (rolling history, `/history` +endpoint, SSE keepalive). + +--- + +## 6. Verdict + +The market data backend is in good shape and ready to be built on. Of the two +open items: + +- **§3.1 (shared module-level router)** should be fixed before the FastAPI + `lifespan` wiring work begins (`PLAN.md` §13, item 3) — it's a small, + mechanical fix (stop reusing module-level `router`/`history_router`; build + one per call) and doing it now avoids a confusing bug later when the app + factory is instantiated more than once, which is standard practice for + backend test fixtures. +- **§3.2/§3.4 (falsy-timestamp substitution, unlocked version read)** are + low-risk and can be picked up opportunistically. +- **§3.3 (silent poller death on revoked key)** is a design note for whoever + adds the `GET /api/health` endpoint — surface poller liveness there. + +None of these block downstream work. **Tests were not executed in this pass** +due to environment permissions (§1) — that is the one action item this review +could not complete, and it should be re-run with `uv`/`python3` execution +permitted to get an authoritative pass/fail/coverage number rather than the +static analysis this document is based on. diff --git a/planning/MARKET_DATA_SUMMARY.md b/planning/MARKET_DATA_SUMMARY.md index ae518283a..14c7cbf75 100644 --- a/planning/MARKET_DATA_SUMMARY.md +++ b/planning/MARKET_DATA_SUMMARY.md @@ -1,10 +1,12 @@ # Market Data Backend — Summary -**Status:** Complete, tested, reviewed, all issues resolved. +**Status:** Complete, tested, reviewed, all issues resolved. Includes the rolling price +history, SSE keepalive, and corrected Massive parsing that `MARKET_DATA_DESIGN.md` +identified as outstanding — see "Gaps Closed" below. ## What Was Built -A complete market data subsystem in `backend/app/market/` (8 modules, ~500 lines) providing live price simulation and real market data via a unified interface. +A complete market data subsystem in `backend/app/market/` (8 modules) providing live price simulation and real market data via a unified interface. ### Architecture @@ -14,9 +16,10 @@ MarketDataSource (ABC) └── MassiveDataSource → Polygon.io REST poller (when MASSIVE_API_KEY set) │ ▼ - PriceCache (thread-safe, in-memory) + PriceCache (thread-safe, in-memory, latest price + rolling history) │ - ├──→ SSE stream endpoint (/api/stream/prices) + ├──→ SSE stream endpoint (/api/stream/prices, with keepalive) + ├──→ Price history endpoint (/api/prices/{ticker}/history) ├──→ Portfolio valuation └──→ Trade execution ``` @@ -27,12 +30,12 @@ MarketDataSource (ABC) |------|---------| | `models.py` | `PriceUpdate` — immutable frozen dataclass (ticker, price, previous_price, timestamp, change, direction) | | `interface.py` | `MarketDataSource` — abstract base class defining `start/stop/add_ticker/remove_ticker/get_tickers` | -| `cache.py` | `PriceCache` — thread-safe price store with version counter for SSE change detection | +| `cache.py` | `PriceCache` — thread-safe price store with version counter for SSE change detection, plus a bounded per-ticker `(timestamp, price)` history (`get_history`) | | `seed_prices.py` | Realistic seed prices, per-ticker GBM params (drift/volatility), correlation groups | | `simulator.py` | `GBMSimulator` (Geometric Brownian Motion with Cholesky-correlated moves) + `SimulatorDataSource` | | `massive_client.py` | `MassiveDataSource` — REST polling client for Polygon.io via the `massive` package | | `factory.py` | `create_market_data_source()` — selects simulator or Massive based on `MASSIVE_API_KEY` env var | -| `stream.py` | `create_stream_router()` — FastAPI SSE endpoint factory using version-based change detection | +| `stream.py` | `create_stream_router()` — SSE endpoint with keepalive; `create_history_router()` — price history endpoint | ### Key Design Decisions @@ -41,33 +44,78 @@ MarketDataSource (ABC) - **GBM with correlated moves** — Cholesky decomposition of sector-based correlation matrix; tech stocks correlate at 0.6, finance at 0.5, cross-sector at 0.3 - **Random shock events** — ~0.1% chance per tick per ticker of a 2-5% move for visual drama - **SSE over WebSockets** — simpler, one-way push, universal browser support +- **SSE keepalive** — a `: ping` comment line after 15s without a version change, so a Massive-backed feed (15s polls) doesn't idle-timeout through proxies +- **Rolling price history in `PriceCache`** — a 600-point bounded deque per ticker (~5 min at the simulator's 500ms cadence), deliberately not persisted, so the main chart backfills instantly on ticker selection instead of drawing from scratch + +## Gaps Closed + +`planning/MARKET_DATA_DESIGN.md` §0 recorded four outstanding gaps against the code as it stood +on 2026-09-01. All four are now closed: + +1. **Massive client wrote nothing to the cache.** `last_trade.timestamp` does not exist on the + real `TickerSnapshot` model (the attribute is `sip_timestamp`), and the divisor treated + nanoseconds as milliseconds even when corrected. The parse loop is now `_apply_snapshots()`, + using `trade.sip_timestamp / NANOS_PER_SECOND` and guarding on `is None` rather than catching + `AttributeError` — tested against the real `TickerSnapshot.from_dict(...)` model, not a + `MagicMock`, which is what let the original bug ship at 94% coverage. +2. **`PriceCache` rolling history** — added (`get_history`, bounded deque, cleared on `remove`). +3. **`GET /api/prices/{ticker}/history`** — added via `create_history_router()`. +4. **SSE keepalive** — added (`: ping` every 15s of idle version). + +The fifth item in the design doc — wiring the market module into a FastAPI `lifespan` alongside +the database, portfolio, and watchlist routes — is intentionally **not** included here. It depends +on those other components, which per the root `CLAUDE.md` are still to be built. ## Test Suite -**73 tests, all passing.** 6 test modules in `backend/tests/market/`. +**103 tests across 7 modules** in `backend/tests/market/`, all passing with **99% coverage** +(verified by actually running `uv run --extra dev pytest --cov=app --cov-report=term-missing`; +prior passes here were static-only due to sandbox permission limits). -| Module | Tests | Coverage | -|--------|-------|----------| -| test_models.py | 11 | models.py: 100% | -| test_cache.py | 13 | cache.py: 100% | -| test_simulator.py | 17 | simulator.py: 98% | -| test_simulator_source.py | 10 | (integration tests) | -| test_factory.py | 7 | factory.py: 100% | -| test_massive.py | 13 | massive_client.py: 56% (expected — API methods mocked) | +| Module | Tests | Notes | +|--------|-------|-------| +| test_models.py | 11 | | +| test_cache.py | 19 | +1 for the falsy-timestamp fix below | +| test_simulator.py | 17 | | +| test_simulator_source.py | 10 | integration tests | +| test_factory.py | 7 | | +| test_massive.py | 21 | Parsing tests built against the real `TickerSnapshot` model instead of `MagicMock`; +3 for the poller-health fix below | +| test_stream.py | 17 | SSE generator (snapshot-on-connect, keepalive, disconnect), the history endpoint, and +2 for the router-factory fix below | -Overall coverage: 84%. +`uv run --extra dev ruff check app/ tests/` passes clean. ## Code Review & Fixes Applied -A comprehensive code review identified 7 issues. All were resolved: - -1. **pyproject.toml build config** — added `[tool.hatch.build.targets.wheel] packages = ["app"]` -2. **Lazy imports removed** — `massive` is a core dependency; imports moved to top level -3. **SSE return type fixed** — `_generate_events` annotated as `AsyncGenerator[str, None]` -4. **Public `get_tickers()`** — added to `GBMSimulator` to avoid private attribute access -5. **Correlation constants cleaned up** — removed unused `DEFAULT_CORR`, consolidated into `CROSS_GROUP_CORR` -6. **Unused test imports removed** — `pytest`, `math`, `asyncio` cleaned from 4 test files -7. **Massive test mocks fixed** — `source._client` set in tests, patches target correct names +Two review passes. The first (archived, 2026-02-10) found 7 issues, all resolved — see the prior +revision of this file. `planning/MARKET_DATA_REVIEW.md` (2026-09-02) found 6 more against the +completed module; all are now resolved: + +1. **Shared module-level router (`stream.py`, Medium)** — `create_stream_router()` and + `create_history_router()` built their route onto a shared `router`/`history_router` object at + module scope, so calling either factory more than once per process (a normal pytest `app` + fixture pattern) silently accumulated duplicate routes. Each factory now constructs a fresh + `APIRouter()` per call. Regression-tested in `test_stream.py::TestRouterFactoriesReturnFreshRouters`. +2. **Falsy-timestamp substitution (`cache.py`, Low)** — `PriceCache.update()` used + `timestamp or time.time()`, which silently replaces an explicit `timestamp=0.0` (a legitimate + Unix epoch instant) because `0.0` is falsy. Changed to `timestamp if timestamp is not None else + time.time()`. Tested in `test_cache.py::test_epoch_zero_timestamp_is_not_replaced`. +3. **Silent poller death on a revoked key (`massive_client.py`, Low)** — if `AuthError` is raised + from inside the background poll loop (as opposed to during `start()`), nothing awaits the task + until `stop()`, so live prices silently freeze with only an easy-to-miss "Task exception was + never retrieved" log at GC time. `MassiveDataSource` now attaches a `Task.add_done_callback` + that logs the failure loudly and flips a new `is_healthy` property to `False` (deliberate + cancellation via `stop()` does not flip it) — ready for a future `GET /api/health` to report + `market_source` as degraded. Tested in `test_massive.py` (`test_is_healthy_*`). +4. **`PriceCache.version` read outside the lock (`cache.py`, Trivial)** — the property now + acquires `self._lock` like every other accessor, for consistency (safe under CPython's GIL + regardless, but only a real concern on a no-GIL build). +5. **Prior review's 7 findings (pyproject build config, lazy imports, SSE return type, public + `get_tickers()`, correlation constants, unused test imports, massive test mocks)** — unchanged + from before, still resolved. +6. Two lower-priority notes from the review were left as-is per its own verdict: no concurrent + multi-thread write test for `PriceCache` (the locking is simple enough to verify by inspection), + and the demo script `market_data_demo.py` remains outside automated test scope (a Rich terminal + demo, appropriately so). ## Demo @@ -83,17 +131,21 @@ Displays a live-updating dashboard with all 10 tickers, sparklines, color-coded ## Usage for Downstream Code ```python -from app.market import PriceCache, create_market_data_source +from app.market import PriceCache, create_market_data_source, create_stream_router, create_history_router # Startup cache = PriceCache() source = create_market_data_source(cache) # Reads MASSIVE_API_KEY await source.start(["AAPL", "GOOGL", "MSFT", ...]) +app.include_router(create_stream_router(cache)) # GET /api/stream/prices (SSE) +app.include_router(create_history_router(cache)) # GET /api/prices/{ticker}/history + # Read prices update = cache.get("AAPL") # PriceUpdate or None price = cache.get_price("AAPL") # float or None all_prices = cache.get_all() # dict[str, PriceUpdate] +history = cache.get_history("AAPL") # [(timestamp, price), ...] oldest-first, [] if untracked # Dynamic watchlist await source.add_ticker("TSLA") diff --git a/planning/MARKET_INTERFACE.md b/planning/MARKET_INTERFACE.md new file mode 100644 index 000000000..7df6fa97a --- /dev/null +++ b/planning/MARKET_INTERFACE.md @@ -0,0 +1,439 @@ +# MARKET_INTERFACE.md — The Unified Market Data API + +How FinAlly retrieves stock prices from either the Massive API or the built-in simulator through one interface, selected by whether `MASSIVE_API_KEY` is set. + +Companion documents: `MASSIVE_API.md` (the real data provider) and `MARKET_SIMULATOR.md` (the fallback). This document is the contract between them and the rest of the backend. + +**Status:** the core of this design is implemented in `backend/app/market/`. Sections marked **TODO** are specified but not yet built. + +--- + +## 1. The problem this solves + +Two data sources with nothing in common: + +- **Massive** — a synchronous HTTP client, polled every 15 seconds, returning whatever the exchanges last reported, with gaps for unknown tickers and frozen values overnight. +- **The simulator** — a pure in-process computation, stepping every 500ms, always alive, and able to invent a plausible price for any symbol. + +Everything downstream — SSE streaming, portfolio valuation, trade execution, the P&L snapshot task — must not care which one is running. A trade fills at "the current price of AAPL" whether that price came from NASDAQ or from a random number generator. + +The design achieves that with **one indirection and one shared buffer**: + +``` + writes reads + ┌──────────────────┐ ┌────────────┐ ┌─────────────────────┐ + │ SimulatorSource │───┐ │ │──────────────│ SSE /api/stream │ + │ (500ms step) │ ├───▶│ PriceCache │──────────────│ Portfolio valuation │ + ├──────────────────┤ │ │ (in-mem, │──────────────│ Trade execution │ + │ MassiveSource │───┘ │thread-safe)│──────────────│ Snapshot task │ + │ (15s poll) │ │ │──────────────│ /api/prices/history │ + └──────────────────┘ └────────────┘ └─────────────────────┘ + MarketDataSource + (abstract interface) +``` + +The critical property: **nothing downstream ever calls the data source to get a price.** Sources are write-only from the application's point of view; readers only ever touch the cache. That is what makes the two implementations substitutable despite a 30× difference in update cadence. + +### Module map — `backend/app/market/` + +| File | Contents | +|---|---| +| `models.py` | `PriceUpdate` — the single price record | +| `cache.py` | `PriceCache` — the shared buffer | +| `interface.py` | `MarketDataSource` — the abstract contract | +| `simulator.py` | `GBMSimulator`, `SimulatorDataSource` | +| `massive_client.py` | `MassiveDataSource` | +| `factory.py` | `create_market_data_source` — the selection rule | +| `seed_prices.py` | Simulator constants | +| `stream.py` | The SSE endpoint | + +--- + +## 2. `PriceUpdate` — the unit of data + +An immutable, frozen dataclass. Both sources produce it; every reader consumes it. + +```python +@dataclass(frozen=True, slots=True) +class PriceUpdate: + ticker: str + price: float + previous_price: float + timestamp: float = field(default_factory=time.time) # Unix epoch SECONDS +``` + +`change`, `change_percent`, and `direction` are computed properties, not stored fields — they cannot drift out of sync with the prices they describe. + +```python +@property +def change(self) -> float: + return round(self.price - self.previous_price, 4) + +@property +def change_percent(self) -> float: + if self.previous_price == 0: + return 0.0 + return round((self.price - self.previous_price) / self.previous_price * 100, 4) + +@property +def direction(self) -> str: + if self.price > self.previous_price: + return "up" + elif self.price < self.previous_price: + return "down" + return "flat" +``` + +### Two frozen conventions + +`to_dict()` is the SSE wire format and is **frozen** — the shipped frontend contract depends on it (`PLAN.md` §6): + +- **`timestamp` is Unix epoch seconds as a float**, never ISO. The frontend multiplies by 1000 for `Date`. Massive's nanosecond and millisecond timestamps are converted at the boundary — see `MASSIVE_API.md` §7. +- **`change_percent` is already in percent units.** `0.021` means 0.021%, not 2.1%. Note this deliberately differs from REST responses elsewhere in the API, where percentages are fractions (`PLAN.md` §8). The inconsistency is real; it is preserved because the market module shipped first and the frontend was written against it. + +`previous_price` means *the price at the previous update*, not the previous session's close. On the first update for a ticker it equals `price`, so `direction` is `"flat"` and `change` is `0.0` — a new ticker never flashes green or red on its first tick. + +--- + +## 3. `PriceCache` — the shared buffer + +An in-memory `dict` behind a `threading.Lock`, plus a monotonic version counter. + +```python +class PriceCache: + def update(self, ticker: str, price: float, timestamp: float | None = None) -> PriceUpdate + def get(self, ticker: str) -> PriceUpdate | None + def get_all(self) -> dict[str, PriceUpdate] # shallow copy + def get_price(self, ticker: str) -> float | None + def remove(self, ticker: str) -> None + @property + def version(self) -> int + def __len__(self) -> int + def __contains__(self, ticker: str) -> bool +``` + +Three design points that matter: + +**`update()` derives `previous_price` itself.** Callers pass only the new price; the cache looks up what it had and constructs the `PriceUpdate`. Neither source needs to track prior state for the purpose of computing a delta, and the two cannot implement it differently. + +**A `threading.Lock`, not an `asyncio.Lock`.** `MassiveDataSource` writes from an `asyncio.to_thread` worker, so a genuine cross-thread lock is required. The critical sections are a few dict operations, so contention is irrelevant. + +**`version` increments on every write and is the SSE change-detection mechanism.** The stream compares it every 500ms rather than diffing prices. Because a fresh generator starts at `last_version = -1`, the first comparison always differs, so **every connecting client immediately receives a full snapshot** — including after a reconnect. This is why no separate snapshot endpoint exists. + +`get_all()` returns a shallow copy; since `PriceUpdate` is frozen, the copy is effectively deep and safe to iterate outside the lock. + +### Rolling price history — **TODO** + +`PLAN.md` §6 requires the main chart to be populated the instant a ticker is clicked. `PriceCache` gains a bounded per-ticker deque of `(timestamp, price)`: + +```python +from collections import deque + +HISTORY_MAXLEN = 600 # ~5 minutes at the 500ms simulator cadence + +self._history: dict[str, deque[tuple[float, float]]] = {} +``` + +- Appended inside `update()`, under the same lock. +- `deque(maxlen=600)` evicts the oldest point automatically — no pruning logic. +- `remove()` must drop the ticker's deque too, or removed tickers leak. +- Deliberately **not persisted**. A restart clears it, which is the honest behaviour for a simulator with no real history. + +Memory is negligible: 600 points × 50 tickers × ~16 bytes ≈ 500KB. + +New reader, backing `GET /api/prices/{ticker}/history?limit=600`: + +```python +def get_history(self, ticker: str, limit: int = 600) -> list[tuple[float, float]]: + """Oldest-first (timestamp, price) points. Empty list for an untracked ticker.""" + with self._lock: + points = self._history.get(ticker) + if not points: + return [] + return list(points)[-limit:] +``` + +An untracked ticker returns `[]`, not a 404 — the chart draws nothing rather than erroring (`PLAN.md` §8). + +Under Massive the deque fills at one point per 15-second poll, so five minutes of wall time is 20 points rather than 600. The chart is sparse but correct. Backfilling from `get_aggs` (`MASSIVE_API.md` §5) is the eventual upgrade. + +--- + +## 4. `MarketDataSource` — the abstract contract + +```python +class MarketDataSource(ABC): + @abstractmethod + async def start(self, tickers: list[str]) -> None: ... + @abstractmethod + async def stop(self) -> None: ... + @abstractmethod + async def add_ticker(self, ticker: str) -> None: ... + @abstractmethod + async def remove_ticker(self, ticker: str) -> None: ... + @abstractmethod + def get_tickers(self) -> list[str]: ... +``` + +Five methods, and every one is about *lifecycle and membership* — none of them returns a price. That absence is the whole design. A `get_price()` on this interface would tempt callers into a per-request API hit under Massive and would make the two implementations behave differently under load. + +### Behavioural contract + +Binding on both implementations. A test suite that passes against one should pass against the other. + +| Method | Guarantee | +|---|---| +| `start(tickers)` | Begins a background task writing to the cache. **Seeds the cache before returning**, so the first SSE event is never empty. Called exactly once; calling twice is undefined. | +| `stop()` | Cancels the task and releases resources. **Idempotent.** No writes to the cache afterwards. | +| `add_ticker(t)` | Adds to the tracked set. No-op if present. Simulator seeds a price immediately; Massive picks it up on the next poll. | +| `remove_ticker(t)` | Removes from the tracked set **and from the cache**. No-op if absent. | +| `get_tickers()` | Current tracked set. Synchronous — it reads local state only. | + +Two asymmetries are permitted and must not be papered over: + +- **Seeding latency.** `add_ticker` on the simulator makes a price available immediately; on Massive it takes up to one poll interval. The API contract already accommodates this — `GET /api/watchlist` returns `price: null` until the first tick, and the UI shows `—`. +- **Cadence.** 500ms versus 15s. Readers must never assume a minimum update rate. This is exactly what the SSE keepalive in §7 exists to handle. + +### `remove_ticker` also clears the cache — and why that is dangerous + +Both implementations call `self._cache.remove(ticker)`. That is correct for the interface but makes the method destructive: a held position whose ticker is removed loses its price, and with it its valuation, its P&L, its heatmap tile, and its snapshot contribution. §5 is the rule that prevents it. + +--- + +## 5. Which tickers are tracked + +**The tracked set is `watchlist ∪ {tickers with a non-zero position}`.** + +The two sets diverge the moment a user buys TSLA and then removes it from the watchlist. The position still needs a live price. This rule is the single most important piece of integration logic in the module, because getting it wrong produces a silently frozen position rather than an error. + +| Trigger | Action | +|---|---| +| `POST /api/watchlist` | always `await source.add_ticker(t)` | +| `DELETE /api/watchlist/{t}` | `await source.remove_ticker(t)` **only if no position in `t` is held** | +| Buy a ticker not currently tracked | `await source.add_ticker(t)` as part of trade execution | +| Sell a position to zero | if `t` is not on the watchlist, `await source.remove_ticker(t)` | +| `POST /api/reset` | re-sync the tracked set to exactly the ten default tickers | + +A single helper keeps the rule in one place rather than at four call sites: + +```python +async def untrack_if_unused(source: MarketDataSource, ticker: str) -> None: + """Stop tracking a ticker only if it is neither watched nor held.""" + if is_on_watchlist(ticker) or has_position(ticker): + return + await source.remove_ticker(ticker) +``` + +### Startup + +```python +tickers = sorted(set(get_watchlist_tickers()) | set(get_position_tickers())) +await source.start(tickers) +``` + +Reading both tables at startup — not just the watchlist — is what makes a position held across a restart come back with a live price. + +--- + +## 6. Selection — `create_market_data_source` + +```python +def create_market_data_source(price_cache: PriceCache) -> MarketDataSource: + """Create the market data source indicated by the environment. + + MASSIVE_API_KEY set and non-empty -> MassiveDataSource (real data) + otherwise -> SimulatorDataSource (GBM simulation) + + Returns an unstarted source; the caller must await source.start(tickers). + """ + api_key = os.environ.get("MASSIVE_API_KEY", "").strip() + + if api_key: + logger.info("Market data source: Massive API (real data)") + return MassiveDataSource(api_key=api_key, price_cache=price_cache) + + logger.info("Market data source: GBM Simulator") + return SimulatorDataSource(price_cache=price_cache) +``` + +`.strip()` before the truth test is deliberate: `.env` files routinely contain `MASSIVE_API_KEY=` or a stray space, and a whitespace-only key would otherwise select the Massive path and then fail every poll with a 401. Empty means empty. + +**The simulator is the default, and the fallback is decided once at startup — never at runtime.** A source that silently switched to the simulator after a Massive outage would show users invented prices while they believed they were seeing the market. Rejected keys and failed polls are logged; they do not change the source. `GET /api/health` reports which one is live: + +```json +{"status": "ok", "market_source": "simulator", "llm_mock": false} +``` + +Returning an **unstarted** source keeps construction synchronous and lets the caller decide the ticker set from the database — the factory has no business reading tables. + +--- + +## 7. Lifecycle and wiring + +One `PriceCache` and one source per process, owned by the FastAPI lifespan. + +```python +from contextlib import asynccontextmanager +from fastapi import FastAPI + +from app.market import PriceCache, create_market_data_source, create_stream_router + + +@asynccontextmanager +async def lifespan(app: FastAPI): + cache = PriceCache() + source = create_market_data_source(cache) + + tickers = sorted(set(get_watchlist_tickers()) | set(get_position_tickers())) + await source.start(tickers) + + app.state.price_cache = cache + app.state.market_source = source + try: + yield + finally: + await source.stop() + + +app = FastAPI(lifespan=lifespan) +app.include_router(create_stream_router(app.state.price_cache)) +``` + +The cache and source are passed explicitly (via router factories or `app.state`) rather than held in module globals, which is what keeps tests able to construct an isolated cache per test. + +**Ordering, from `PLAN.md` §11:** mount all `/api/*` routers *before* the static file mount. A `StaticFiles(html=True)` mount at `/` registered first shadows every endpoint, including the SSE stream. + +### The SSE stream + +`GET /api/stream/prices`, `Content-Type: text/event-stream`. The generator opens with `retry: 1000`, then pushes the **entire cache as one JSON object** whenever `version` changes, polled every 500ms: + +``` +retry: 1000 + +data: {"AAPL": {"ticker": "AAPL", "price": 190.52, "previous_price": 190.48, "timestamp": 1755873791.482, "change": 0.04, "change_percent": 0.021, "direction": "up"}, "GOOGL": {...}} +``` + +One event carries every ticker — not one event per ticker. The client replaces its price map wholesale, so there is no merge logic and no missed-update reconciliation. + +### Keepalive — **TODO** + +When the version has not changed for 15 seconds, emit an SSE comment: + +```python +KEEPALIVE_SECONDS = 15.0 + +last_sent = time.monotonic() +while True: + if await request.is_disconnected(): + break + + current_version = price_cache.version + if current_version != last_version: + last_version = current_version + prices = price_cache.get_all() + if prices: + payload = json.dumps({t: u.to_dict() for t, u in prices.items()}) + yield f"data: {payload}\n\n" + last_sent = time.monotonic() + elif time.monotonic() - last_sent >= KEEPALIVE_SECONDS: + yield ": ping\n\n" + last_sent = time.monotonic() + + await asyncio.sleep(interval) +``` + +Without this, a Massive-backed feed sends no bytes between 15-second polls. That idle-times-out through proxies and leaves the frontend unable to distinguish a quiet market from a dead connection. The connection indicator — green on `onopen`, yellow on `onerror`, red after a ping gap beyond ~40 seconds — depends on it. + +--- + +## 8. Implementing a new source + +The interface is small enough that a third source is a contained piece of work. The checklist: + +1. Subclass `MarketDataSource` and implement all five methods. +2. `start()` must **seed the cache before returning**. +3. Never write to the cache after `stop()`; make `stop()` idempotent. +4. `remove_ticker()` must call `cache.remove(ticker)`. +5. Convert timestamps to **Unix epoch seconds as a float** at the boundary. +6. Never let a fetch error kill the background loop — log and retry on the next cycle. +7. If the source is synchronous, wrap every call in `asyncio.to_thread`. +8. Add a branch to `create_market_data_source` and a value to `market_source` in `/api/health`. + +Point 7 is not optional. `massive.RESTClient` is `urllib3`-based and blocking; calling it directly from `async def` stalls the event loop for the duration of the HTTP round trip, which stops the SSE stream and every in-flight request. `MassiveDataSource` gets this right: + +```python +snapshots = await asyncio.to_thread(self._fetch_snapshots) +``` + +### Massive polling, in outline + +```python +async def _poll_loop(self) -> None: + """Poll on interval. The first poll already happened in start().""" + while True: + await asyncio.sleep(self._interval) + await self._poll_once() +``` + +`start()` performs one poll synchronously before creating the task, so the cache is warm before the first client connects. `_poll_interval` defaults to 15 seconds to stay inside the free tier's 5 requests/minute (`MASSIVE_API.md` §2); paid tiers can drop to 2–5 seconds. + +> The `_poll_once` parsing in `massive_client.py` currently reads a non-existent attribute and uses the wrong unit divisor, which means the Massive path writes nothing to the cache at all. Both defects are reproduced and the corrected parse is given in `MASSIVE_API.md` §8. Fixing them is a prerequisite to the Massive path working. + +--- + +## 9. Testing + +Existing coverage: **73 tests passing at 91%** for the market module, measured by running the suite while writing this document. (`MARKET_DATA_SUMMARY.md` still quotes 84%, which is stale.) + +**The cache and the interface can be tested without either real source.** A stub is a few lines, and it is the right tool for testing the tracked-ticker rules: + +```python +class StubDataSource(MarketDataSource): + """Records lifecycle calls; writes nothing on its own.""" + + def __init__(self, cache: PriceCache) -> None: + self._cache = cache + self._tickers: list[str] = [] + self.started = False + + async def start(self, tickers): self._tickers = list(tickers); self.started = True + async def stop(self): self.started = False + async def add_ticker(self, t): + if t not in self._tickers: + self._tickers.append(t) + async def remove_ticker(self, t): + self._tickers = [x for x in self._tickers if x != t] + self._cache.remove(t) + def get_tickers(self): return list(self._tickers) +``` + +What to cover: + +- **Cache** — `previous_price` derivation, first-update `flat`, `version` monotonicity, `remove` clearing both price and history, thread safety under concurrent writers. +- **Factory** — unset, empty, and whitespace-only `MASSIVE_API_KEY` all select the simulator; a real value selects Massive. +- **Tracked set** — removing a watchlist ticker with an open position keeps it in the feed; selling to zero off-watchlist removes it. These are the two regressions that produce a frozen position. +- **Massive parsing** — feed the real `TickerSnapshot.from_dict` a documented payload and assert the cached timestamp lands in the plausible present. Never build these snapshots from `MagicMock`: the existing tests do, which is precisely why 94% coverage of `massive_client.py` still missed both defects (`MASSIVE_API.md` §8.3). +- **SSE** — map-shaped payload, float timestamp, percent-unit `change_percent`, full snapshot on connect, keepalive after 15 idle seconds. +- **History** — deque bounded at 600, oldest-first ordering, `[]` for an untracked ticker. + +```bash +cd backend +uv run pytest +uv run pytest --cov=app --cov-report=term-missing +``` + +--- + +## 10. Summary + +| Concern | Resolution | +|---|---| +| Two sources, one consumer | `MarketDataSource` ABC + shared `PriceCache` | +| Which source | `create_market_data_source`, decided once at startup from `MASSIVE_API_KEY` | +| Default | Simulator — always alive, no key, no rate limit | +| How prices are read | Only from the cache, never from the source | +| Which tickers are live | `watchlist ∪ positions` | +| Timestamp format | Unix epoch seconds (float), converted at each source boundary | +| Update delivery | SSE, full cache per event, on `version` change | +| Blocking I/O | `asyncio.to_thread` at the source, always | +| Outstanding | Rolling history + endpoint, SSE keepalive, the two `massive_client.py` defects | diff --git a/planning/MARKET_SIMULATOR.md b/planning/MARKET_SIMULATOR.md new file mode 100644 index 000000000..25c60ef5a --- /dev/null +++ b/planning/MARKET_SIMULATOR.md @@ -0,0 +1,456 @@ +# MARKET_SIMULATOR.md — The Market Simulator + +The approach and code structure for simulating stock prices when no `MASSIVE_API_KEY` is configured. This is FinAlly's **default** data source, so it is what almost every user will see. + +Companion documents: `MARKET_INTERFACE.md` (the abstraction it implements) and `MASSIVE_API.md` (the alternative). Implemented in `backend/app/market/simulator.py` and `backend/app/market/seed_prices.py`. + +--- + +## 1. What it must achieve + +The simulator is not a research tool. It exists so that a student who clones the repo and runs one Docker command sees a trading terminal that looks alive, at any hour, on any day, with no account and no API key. + +That sets the bar precisely: + +| Requirement | Why | +|---|---| +| Visible motion every 500ms | The watchlist flashes green and red; a static grid looks broken | +| Motion at a *plausible* scale | AAPL moving $12 per tick destroys the illusion instantly | +| Prices that stay positive | A stock at −$4 is not a rendering bug the user will forgive | +| Correlated moves | Real tech stocks rise together; independent random walks look obviously fake | +| Occasional drama | A flat five minutes is boring; a sudden 3% drop gives the demo a story | +| Any ticker works | The AI chat can add any symbol; "we don't have that one" would feel broken | +| No external dependency | It must run offline, at 3am, on a weekend | + +And explicitly **not** required: predictive value, real historical data, order books, bid-ask spreads, or volume modelling. Nothing in the app consumes them. + +--- + +## 2. The model — Geometric Brownian Motion + +GBM is the standard model for equity prices and the one that satisfies the requirements above almost incidentally. + +``` +S(t + dt) = S(t) · exp( (μ − σ²/2)·dt + σ·√dt·Z ) +``` + +| Symbol | Meaning | +|---|---| +| `S(t)` | Current price | +| `μ` | Annualised drift — expected return | +| `σ` | Annualised volatility | +| `dt` | Time step, as a fraction of a trading year | +| `Z` | Standard normal draw, correlated across tickers | + +Three properties earn its place here: + +**Prices cannot go negative.** The update is multiplicative — `exp(...)` is always positive, so `S` never crosses zero. No clamping, no `max(price, 0.01)` guard, no special case. An additive random walk would need all three. + +**Returns scale correctly with time.** Volatility is expressed per *year*, and `√dt` converts it to the tick. Change the tick rate and the price series keeps the same annualised character. The 500ms cadence is a display choice, not a modelling parameter. + +**The `−σ²/2` term keeps the drift honest.** Without it, `μ` would not be the expected return of the price — a well-known artefact of the log-normal distribution. It costs one subtraction and makes the parameters mean what they say. + +### Sizing `dt` + +`dt` is expressed against a **trading** year, not a calendar year — markets are closed most of the time, and using 365×24h would understate per-tick moves by a factor of about 4.5. + +```python +TRADING_SECONDS_PER_YEAR = 252 * 6.5 * 3600 # 5,896,800 +DEFAULT_DT = 0.5 / TRADING_SECONDS_PER_YEAR # ~8.479e-8 +``` + +252 trading days × 6.5 hours × 3600 seconds. A 500ms tick is therefore `8.479e-8` of a year, and `√dt = 2.912e-4`. + +What that produces per tick, for `σ·√dt` at the seed prices: + +| Ticker | σ | Per-tick σ | Per-tick $ | Per-minute $ (120 ticks) | +|---|---|---|---|---| +| AAPL | 0.22 | 0.0064% | $0.012 | $0.13 | +| JPM | 0.18 | 0.0052% | $0.010 | $0.11 | +| NVDA | 0.40 | 0.0116% | $0.093 | $1.02 | +| TSLA | 0.50 | 0.0146% | $0.036 | $0.40 | + +This is the number that decides whether the simulation looks right. Around a cent or two per tick on a $200 stock means prices are **rounded to 2 decimals into a genuinely different value most ticks** — so the UI flashes constantly — while a minute of drift stays in the tens of cents, which is what a real quote screen looks like. Larger and it reads as a crash; smaller and the grid appears frozen. + +--- + +## 3. Correlation via Cholesky decomposition + +Independent draws per ticker would show tech stocks moving in opposite directions half the time. Real markets do not do that, and the eye notices immediately. + +The fix is standard: draw `n` independent standard normals, then multiply by the Cholesky factor `L` of the desired correlation matrix `C`, where `C = L·Lᵀ`. The resulting vector has exactly the correlation structure of `C`. + +```python +z_independent = np.random.standard_normal(n) +z_correlated = self._cholesky @ z_independent +``` + +### The correlation structure + +Sector membership and coefficients live in `seed_prices.py`, not in the simulator: + +```python +CORRELATION_GROUPS = { + "tech": {"AAPL", "GOOGL", "MSFT", "AMZN", "META", "NVDA", "NFLX"}, + "finance": {"JPM", "V"}, +} + +INTRA_TECH_CORR = 0.6 # tech stocks move together +INTRA_FINANCE_CORR = 0.5 # finance stocks move together +CROSS_GROUP_CORR = 0.3 # between sectors, and for unknown tickers +TSLA_CORR = 0.3 # TSLA does its own thing +``` + +Resolved pairwise, first match winning: + +```python +@staticmethod +def _pairwise_correlation(t1: str, t2: str) -> float: + tech = CORRELATION_GROUPS["tech"] + finance = CORRELATION_GROUPS["finance"] + + # TSLA is in the tech set but behaves independently + if t1 == "TSLA" or t2 == "TSLA": + return TSLA_CORR + + if t1 in tech and t2 in tech: + return INTRA_TECH_CORR + if t1 in finance and t2 in finance: + return INTRA_FINANCE_CORR + + return CROSS_GROUP_CORR +``` + +The TSLA clause is checked first on purpose: TSLA is a member of the tech set for every other purpose, but empirically trades on its own news, and a demo where TSLA visibly decouples from the pack is more convincing than one where everything moves in lockstep. + +`CROSS_GROUP_CORR` doubles as the default for any symbol the simulator has never heard of, which is what makes §5 work. + +### Rebuilding + +`_rebuild_cholesky()` runs on every add and remove — `O(n²)` to build the matrix plus `O(n³)` to factor it, on `n < 50`. That is microseconds, and watchlist edits are a human-speed operation, so caching it would be complexity without benefit. + +```python +def _rebuild_cholesky(self) -> None: + n = len(self._tickers) + if n <= 1: + self._cholesky = None # a single ticker needs no correlation + return + + corr = np.eye(n) + for i in range(n): + for j in range(i + 1, n): + rho = self._pairwise_correlation(self._tickers[i], self._tickers[j]) + corr[i, j] = rho + corr[j, i] = rho + + self._cholesky = np.linalg.cholesky(corr) +``` + +`step()` falls back to uncorrelated draws when `_cholesky is None`, which covers both the single-ticker and empty cases. + +> **Known risk.** `np.linalg.cholesky` raises `LinAlgError` on a matrix that is not positive definite, and this call is unguarded. The current block structure (0.6 / 0.5 / 0.3) was verified positive definite at 7, 20, and 40 tickers, so it is safe as configured — but raising `INTRA_TECH_CORR` toward 1.0, or adding a group whose intra-group correlation is below the cross-group value, can break positive-definiteness and take down `add_ticker`. Anyone editing these constants should re-run the check in §8. + +--- + +## 4. The tick + +`step()` is the hot path — every 500ms, for every ticker. + +```python +def step(self) -> dict[str, float]: + """Advance all tickers by one time step. Returns {ticker: new_price}.""" + n = len(self._tickers) + if n == 0: + return {} + + z_independent = np.random.standard_normal(n) + if self._cholesky is not None: + z_correlated = self._cholesky @ z_independent + else: + z_correlated = z_independent + + result: dict[str, float] = {} + for i, ticker in enumerate(self._tickers): + params = self._params[ticker] + mu, sigma = params["mu"], params["sigma"] + + drift = (mu - 0.5 * sigma**2) * self._dt + diffusion = sigma * math.sqrt(self._dt) * z_correlated[i] + self._prices[ticker] *= math.exp(drift + diffusion) + + if random.random() < self._event_prob: + shock_magnitude = random.uniform(0.02, 0.05) + shock_sign = random.choice([-1, 1]) + self._prices[ticker] *= 1 + shock_magnitude * shock_sign + + result[ticker] = round(self._prices[ticker], 2) + + return result +``` + +Two details worth pointing out: + +**Full precision is kept internally; only the returned value is rounded.** `self._prices[ticker]` stays a full float. Rounding the stored state would accumulate quantisation error into a slow, systematic drift over thousands of ticks. + +**One `standard_normal(n)` call per tick, not `n` calls.** A single vectorised draw feeding one matrix multiply is the reason this stays negligible at 500ms. + +### Random events + +```python +event_probability: float = 0.001 # per ticker, per tick +``` + +A 2–5% jump in either direction. With 10 tickers at 2 ticks/second, the expected wait is `1 / (10 × 2 × 0.001) = 50 seconds` — frequent enough that something interesting happens during a demo, rare enough that the price series is not pure noise. + +The shock multiplies the price directly rather than feeding through GBM, so it is a genuine discontinuity — a gap, which is what real news does to a stock. + +--- + +## 5. Unknown tickers + +Any symbol passing the API-level pattern `^[A-Z][A-Z.]{0,5}$` works, with no allowlist. The AI chat can add anything the user names, and it behaves plausibly. + +```python +def _add_ticker_internal(self, ticker: str) -> None: + if ticker in self._prices: + return + self._tickers.append(ticker) + self._prices[ticker] = SEED_PRICES.get(ticker, random.uniform(50.0, 300.0)) + self._params[ticker] = TICKER_PARAMS.get(ticker, dict(DEFAULT_PARAMS)) +``` + +- **Price**: seeded from `SEED_PRICES`, else uniform in $50–$300 — the range where most large-cap US equities actually trade. +- **Parameters**: `TICKER_PARAMS`, else `DEFAULT_PARAMS` (`σ=0.25`, `μ=0.05`) — a mid-range large cap. +- **Correlation**: no sector membership, so `CROSS_GROUP_CORR` (0.3) against everything. + +`dict(DEFAULT_PARAMS)` copies rather than sharing the module-level dict — without the copy, per-ticker parameter tuning would mutate the default for every unknown ticker at once. + +This is a real advantage over the Massive path, where an unknown symbol simply never produces a price and sits at `—` forever (`MASSIVE_API.md` §9). + +--- + +## 6. Code structure + +Two classes with a clean split: **`GBMSimulator` is pure and synchronous; `SimulatorDataSource` handles async lifecycle and the cache.** + +``` +┌──────────────────────────────────────────────────────────┐ +│ SimulatorDataSource(MarketDataSource) │ +│ owns the asyncio task, writes to PriceCache │ +│ start / stop / add_ticker / remove_ticker / get_tickers│ +│ │ │ +│ ▼ │ +│ GBMSimulator │ +│ pure math, no I/O, no async, no cache reference │ +│ step() -> {ticker: price} │ +└──────────────────────────────────────────────────────────┘ + │ + ▼ + seed_prices.py + constants only, no logic +``` + +The separation pays off in testing: `GBMSimulator` needs no event loop, no cache, and no mocks. Statistical properties are asserted by calling `step()` in a loop. + +### `GBMSimulator` — pure math + +```python +class GBMSimulator: + TRADING_SECONDS_PER_YEAR = 252 * 6.5 * 3600 + DEFAULT_DT = 0.5 / TRADING_SECONDS_PER_YEAR + + def __init__(self, tickers, dt=DEFAULT_DT, event_probability=0.001): ... + + def step(self) -> dict[str, float]: ... + def add_ticker(self, ticker: str) -> None: ... + def remove_ticker(self, ticker: str) -> None: ... + def get_price(self, ticker: str) -> float | None: ... + def get_tickers(self) -> list[str]: ... +``` + +State is three parallel dicts keyed by ticker (`_prices`, `_params`) plus `_tickers` as the **ordered** list that indexes into the Cholesky matrix. Order matters: row `i` of `_cholesky` corresponds to `_tickers[i]`, which is why add and remove must both rebuild. + +`__init__` adds every ticker via `_add_ticker_internal` and rebuilds Cholesky **once** at the end, rather than rebuilding per ticker — `O(n³)` instead of `O(n⁴)` on startup. + +### `SimulatorDataSource` — the async wrapper + +```python +class SimulatorDataSource(MarketDataSource): + def __init__(self, price_cache, update_interval=0.5, event_probability=0.001): ... + + async def start(self, tickers: list[str]) -> None: + self._sim = GBMSimulator(tickers=tickers, event_probability=self._event_prob) + # Seed the cache with initial prices so SSE has data immediately + for ticker in tickers: + price = self._sim.get_price(ticker) + if price is not None: + self._cache.update(ticker=ticker, price=price) + self._task = asyncio.create_task(self._run_loop(), name="simulator-loop") + + async def _run_loop(self) -> None: + while True: + try: + if self._sim: + for ticker, price in self._sim.step().items(): + self._cache.update(ticker=ticker, price=price) + except Exception: + logger.exception("Simulator step failed") + await asyncio.sleep(self._interval) +``` + +Three deliberate choices: + +**Seed the cache in `start()` before creating the task.** The first SSE event then carries real prices rather than an empty object, so the watchlist never renders as ten dashes on load. + +**`add_ticker` seeds immediately.** The new ticker has a price on the very next SSE event, with no wait for the following step — the reason adding a ticker feels instant. + +**The `try` is inside the loop, around the step.** An exception logs and the loop continues on the next interval. Wrapping the loop instead would let one bad tick kill the feed permanently. This is the one place defensive handling is warranted: the background task has no caller to propagate to, and a dead price feed is a dead app. + +`stop()` cancels the task and awaits it, swallowing `CancelledError` — the normal shutdown path, not an error. + +### Parameters + +`seed_prices.py` holds constants only. Prices are realistic as of project creation; `σ` and `μ` are annualised. + +| Ticker | Seed | σ | μ | Note | +|---|---|---|---|---| +| AAPL | $190 | 0.22 | 0.05 | | +| GOOGL | $175 | 0.25 | 0.05 | | +| MSFT | $420 | 0.20 | 0.05 | | +| AMZN | $185 | 0.28 | 0.05 | | +| TSLA | $250 | 0.50 | 0.03 | High volatility, decorrelated | +| NVDA | $800 | 0.40 | 0.08 | High volatility, strong drift | +| META | $500 | 0.30 | 0.05 | | +| JPM | $195 | 0.18 | 0.04 | Low volatility (bank) | +| V | $280 | 0.17 | 0.04 | Low volatility (payments) | +| NFLX | $600 | 0.35 | 0.05 | | +| *unknown* | $50–300 | 0.25 | 0.05 | `DEFAULT_PARAMS` | + +The σ spread is what makes the watchlist readable at a glance: V and JPM barely move while NVDA and TSLA jump, so the grid has visible texture instead of ten tickers twitching identically. + +--- + +## 7. Behaviour over time + +There is **no mean reversion and no session boundary.** Prices random-walk from their seed for as long as the container runs. Over a demo — minutes to hours — drift is small and the series looks like a trading day. Over a week of uptime, a ticker may wander far from its seed. That is correct GBM behaviour and not worth correcting: the state is in memory only, so a restart returns everything to the seed prices. + +That in turn is why the rolling price history is not persisted (`MARKET_INTERFACE.md` §3). A restart legitimately resets the world. + +--- + +## 8. Testing + +Existing coverage is **73 tests passing at 91%** across the market module (`simulator.py` itself is at 98%). `GBMSimulator` is pure, so its tests are fast and deterministic under a seeded RNG. + +**Deterministic tests** — seed both RNGs, since the simulator uses `numpy.random` for the normal draws and the stdlib `random` for events: + +```python +def test_step_is_reproducible(): + np.random.seed(42) + random.seed(42) + sim = GBMSimulator(tickers=["AAPL", "GOOGL"]) + first = sim.step() + + np.random.seed(42) + random.seed(42) + sim = GBMSimulator(tickers=["AAPL", "GOOGL"]) + assert sim.step() == first +``` + +**Structural properties:** + +- Prices stay strictly positive over many thousands of steps. +- `step()` returns exactly the current ticker set. +- Add/remove keeps `_tickers`, `_prices`, and `_params` consistent, and the Cholesky shape matches `len(_tickers)`. +- Unknown tickers seed within $50–$300 and get `DEFAULT_PARAMS`. +- `remove_ticker` on an untracked symbol is a no-op, not an error. + +**Statistical properties** — over enough steps, with a tolerance: + +```python +def test_realised_volatility_is_close_to_sigma(): + sim = GBMSimulator(tickers=["AAPL"], event_probability=0.0) + prices = [sim.get_price("AAPL")] + for _ in range(20_000): + prices.append(sim.step()["AAPL"]) + + log_returns = np.diff(np.log(prices)) + realised = log_returns.std() / np.sqrt(GBMSimulator.DEFAULT_DT) + assert 0.15 < realised < 0.35 # nominal sigma is 0.22 +``` + +Disable events (`event_probability=0.0`) for statistical tests — a 5% jump is a massive outlier at this dt and will dominate the sample variance. Keep tolerances wide; these are sampling estimates, and a tight bound produces a test that fails a few times a year for no reason. + +**Correlation:** + +```python +def test_tech_tickers_are_positively_correlated(): + sim = GBMSimulator(tickers=["AAPL", "MSFT"], event_probability=0.0) + a, m = [], [] + for _ in range(10_000): + p = sim.step() + a.append(p["AAPL"]) + m.append(p["MSFT"]) + + rho = np.corrcoef(np.diff(np.log(a)), np.diff(np.log(m)))[0, 1] + assert rho > 0.4 # nominal 0.6 +``` + +**Cholesky positive-definiteness** — run after any change to the correlation constants: + +```python +def test_correlation_matrix_stays_positive_definite(): + tickers = list(SEED_PRICES) + [f"UNK{i}" for i in range(40)] + GBMSimulator(tickers=tickers) # raises LinAlgError if not PD +``` + +**`SimulatorDataSource`** needs an event loop and a real `PriceCache`, but no mocks: + +- `start()` populates the cache before returning. +- The cache updates after roughly one interval. +- `add_ticker` seeds a price immediately. +- `remove_ticker` clears the ticker from the cache. +- `stop()` is idempotent and halts writes. + +```bash +cd backend +uv run pytest tests/market/ +uv run pytest --cov=app --cov-report=term-missing +``` + +`backend/market_data_demo.py` is a Rich terminal demo of the live simulator — the fastest way to eyeball whether a parameter change still looks right. + +--- + +## 9. Tuning guide + +Everything worth adjusting, and what it costs: + +| Want | Change | Watch for | +|---|---|---| +| More visible price motion | Raise `σ` in `TICKER_PARAMS` | Above ~0.8 it stops looking like equity | +| Faster or slower updates | `update_interval` on `SimulatorDataSource` | `DEFAULT_DT` is derived from 0.5s; change both together or annualised σ shifts | +| More frequent drama | Raise `event_probability` | Above ~0.005 the series becomes jumps, not prices | +| Bigger shocks | Widen `random.uniform(0.02, 0.05)` | Beyond ~10% the P&L chart loses all detail | +| Different sector behaviour | Edit `CORRELATION_GROUPS` and the coefficients | Re-run the positive-definiteness test in §8 | +| Different starting prices | `SEED_PRICES` | Unlisted tickers still land in $50–$300 | +| A trending market | Raise `μ` | `μ` is annualised; even 0.5 is barely visible over a demo | + +The `DEFAULT_DT` coupling is the one that catches people. `DEFAULT_DT = 0.5 / TRADING_SECONDS_PER_YEAR` hard-codes the 500ms tick. Passing `update_interval=0.1` to `SimulatorDataSource` without also passing a matching `dt` to `GBMSimulator` makes the simulation run five times faster in model time — annualised volatility silently becomes 5× what `TICKER_PARAMS` claims. + +--- + +## 10. Summary + +| Concern | Approach | +|---|---| +| Price model | Geometric Brownian Motion, per-ticker `μ` and `σ` | +| Positivity | Guaranteed by the multiplicative `exp` form — no clamping | +| Time step | `0.5s / (252 × 6.5 × 3600)` ≈ `8.479e-8` of a trading year | +| Correlation | Cholesky factor of a sector-block matrix, rebuilt on add/remove | +| Sectors | tech 0.6, finance 0.5, cross-sector and unknown 0.3, TSLA 0.3 | +| Drama | 0.1% chance per ticker per tick of a 2–5% jump — about one per 50s | +| Unknown tickers | Random $50–$300 seed, `DEFAULT_PARAMS`, cross-sector correlation | +| Structure | Pure `GBMSimulator` + async `SimulatorDataSource`, constants in `seed_prices.py` | +| Persistence | None — a restart returns to seed prices | +| Failure handling | Per-step `try` inside the loop; the feed never dies from one bad tick | diff --git a/planning/MASSIVE_API.md b/planning/MASSIVE_API.md new file mode 100644 index 000000000..d7f94ef17 --- /dev/null +++ b/planning/MASSIVE_API.md @@ -0,0 +1,525 @@ +# MASSIVE_API.md — Massive (formerly Polygon.io) REST API + +Reference for retrieving real-time and end-of-day prices for multiple tickers. + +**Verified against `massive` Python SDK `2.2.0`** (installed in `backend/.venv`) and the official docs at . Every signature, field name, and unit below was checked against the installed package source or the live documentation — not from memory. + +> No `MASSIVE_API_KEY` was available in this repo when this document was written, so responses could not be exercised against the live service. Field shapes come from the SDK's `from_dict` parsers and the published response schemas, which is authoritative for how the client will deserialize. The verification script in §10 closes the loop once a key exists. + +--- + +## 1. Orientation + +Polygon.io rebranded to **Massive** in 2026. The API surface, the API keys, and the endpoint paths are unchanged; the hostname and the Python package are new. + +| | Value | +|---|---| +| Base URL | `https://api.massive.com` | +| Python SDK | `massive` (PyPI), currently `2.2.0` | +| Repository | | +| Auth | `Authorization: Bearer ` header | +| Env var read by the SDK | `MASSIVE_API_KEY` | + +The SDK reads the same environment variable name this project already uses, so `RESTClient()` with no arguments works when `MASSIVE_API_KEY` is exported. FinAlly passes the key explicitly instead, because the factory has already read and validated it. + +### Install + +```bash +uv add massive +``` + +### Authentication + +The SDK sets the header for you (`massive/rest/base.py`): + +```python +self.headers = { + "Authorization": "Bearer " + self.API_KEY, + "Accept-Encoding": "gzip", + "User-Agent": f"Massive.com PythonClient/{version_number}", +} +``` + +Constructing a client with no key and no env var raises `massive.exceptions.AuthError` immediately — it does not wait for the first request. + +```python +from massive import RESTClient + +client = RESTClient(api_key="YOUR_KEY") # or RESTClient() to read MASSIVE_API_KEY +``` + +Full constructor defaults, from the installed SDK: + +```python +RESTClient( + api_key: str | None = None, + connect_timeout: float = 10.0, + read_timeout: float = 10.0, + num_pools: int = 10, + retries: int = 3, # urllib3 Retry on 413/429/499/500/502/503... + base: str = "https://api.massive.com", + pagination: bool = True, + verbose: bool = False, + trace: bool = False, + custom_json: Any | None = None, +) +``` + +Two consequences worth knowing: + +- **`RESTClient` is synchronous.** It uses `urllib3.PoolManager`. Calling it from an `async def` blocks the event loop, which in this app means visibly stuttering prices on the SSE stream. Always wrap it in `asyncio.to_thread`. +- **It retries 429 internally** (3 attempts, honouring `Retry-After`). A poll that hits the rate limit therefore blocks its worker thread rather than failing fast. + +--- + +## 2. Plans and rate limits + +| Plan | Requests/min | Data freshness | +|---|---|---| +| Basic (free) | **5** | End-of-day, and 15-minute-delayed intraday | +| Paid (Starter and above) | Unlimited | Real-time (15-min delayed on Starter) | + +This single number drives the whole polling design: **5 requests/minute means one request every 12 seconds at best.** FinAlly polls every 15 seconds by default, which leaves headroom and stays under the limit even if a poll overruns. + +The corollary is that per-ticker endpoints are unusable on the free tier — 10 watchlist tickers via `get_last_trade` would be 10 requests per cycle, blowing the budget in one poll. **The design must fetch all tickers in a single request**, which is what §3 is about. + +--- + +## 3. Real-time prices for multiple tickers + +### 3.1 Full Market Snapshot (v2) — the primary endpoint + +`GET /v2/snapshot/locale/us/markets/stocks/tickers` + +One request returns the current state of every ticker you name. This is the endpoint FinAlly uses. + +| Query param | Meaning | +|---|---| +| `tickers` | Case-insensitive comma-separated list. Omit to get the entire US market. | +| `include_otc` | Include OTC securities. Default `false`. | + +SDK signature: + +```python +client.get_snapshot_all( + market_type: str | SnapshotMarketType, + tickers: str | list[str] | None = None, + include_otc: bool | None = False, + params: dict | None = None, + raw: bool = False, +) -> list[TickerSnapshot] +``` + +The SDK joins a list into a comma-separated string for you (`",".join(tickers)`), so passing a `list[str]` is correct and idiomatic. + +```python +from massive import RESTClient +from massive.rest.models import SnapshotMarketType + +client = RESTClient(api_key="YOUR_KEY") + +snapshots = client.get_snapshot_all( + market_type=SnapshotMarketType.STOCKS, + tickers=["AAPL", "GOOGL", "MSFT", "AMZN", "TSLA"], +) + +for snap in snapshots: + print(snap.ticker, snap.last_trade.price, snap.todays_change_percent) +``` + +**Response shape** (`TickerSnapshot`, from `massive/rest/models/snapshot.py`): + +| Attribute | JSON key | Type | Notes | +|---|---|---|---| +| `ticker` | `ticker` | `str` | | +| `todays_change` | `todaysChange` | `float` | Absolute change vs. prior close | +| `todays_change_percent` | `todaysChangePerc` | `float` | **Already in percent units** (`0.39` = 0.39%) | +| `updated` | `updated` | `int` | **Nanoseconds** | +| `day` | `day` | `Agg` | Today's bar so far | +| `prev_day` | `prevDay` | `Agg` | Previous session's bar | +| `min` | `min` | `MinuteSnapshot` | Most recent minute bar | +| `last_trade` | `lastTrade` | `LastTrade` | Most recent execution | +| `last_quote` | `lastQuote` | `LastQuote` | Most recent NBBO | +| `fair_market_value` | `fmv` | `float` | Business plans only; `None` otherwise | + +`LastTrade` — note the attribute names, they do **not** match the JSON keys: + +| Attribute | JSON key | Units | +|---|---|---| +| `price` | `p` | dollars | +| `size` | `s` | shares | +| `sip_timestamp` | `t` | **nanoseconds** | +| `exchange` | `x` | exchange ID | +| `conditions` | `c` | `list[int]` | +| `id` | `i` | trade ID | +| `ticker` | `T` | usually `None` inside a snapshot | + +`Agg` (used for `day` and `prev_day`): `open`/`o`, `high`/`h`, `low`/`l`, `close`/`c`, `volume`/`v`, `vwap`/`vw`, `timestamp`/`t`, `transactions`/`n`. + +### 3.2 Unified Snapshot (v3) — the alternative + +`GET /v3/snapshot` + +Multi-asset-class, paginated, and — the reason it is worth mentioning — it **reports unknown tickers explicitly** instead of silently omitting them. + +```python +snaps = client.list_universal_snapshots( + type="stocks", + ticker_any_of=["AAPL", "NVDA", "NOTAREALTICKER"], + limit=250, +) + +for s in snaps: + if s.error: + print(f"{s.ticker}: {s.error} — {s.message}") # e.g. NOT_FOUND + else: + print(s.ticker, s.session.close, s.last_trade.price) +``` + +- `ticker_any_of` accepts **up to 250** tickers. +- `limit` defaults to 10 and maxes at 250 — **leave it at the default and you will silently get only 10 results.** Always set it explicitly. +- Returns an *iterator* and auto-paginates (`pagination=True`), so a careless call can fan out into many billed requests. With `ticker_any_of` bounded at 250 and `limit=250` there is exactly one page. + +FinAlly stays on v2 because a 10-ticker watchlist never approaches the 250 limit, v2 is a single non-paginated request, and the `error` field is of marginal value when the simulator is the default path anyway. v3 is the right upgrade if per-ticker validation feedback is ever wanted. + +### 3.3 Single ticker + +Useful for a one-off lookup; unusable as a polling strategy on the free tier. + +```python +snap = client.get_snapshot_ticker(SnapshotMarketType.STOCKS, "AAPL") +trade = client.get_last_trade("AAPL") # LastTrade: .price, .size, .sip_timestamp (ns) +quote = client.get_last_quote("AAPL") # LastQuote: .bid_price, .ask_price, ... +``` + +--- + +## 4. End-of-day prices + +### 4.1 Previous close — per ticker + +`GET /v2/aggs/ticker/{ticker}/prev` + +```python +prev = client.get_previous_close_agg("AAPL", adjusted=True) +print(prev.ticker, prev.open, prev.high, prev.low, prev.close, prev.volume, prev.vwap) +``` + +`PreviousCloseAgg` fields: `ticker`, `open`, `high`, `low`, `close`, `volume`, `vwap`, `timestamp` (**milliseconds**, start of the aggregate window). + +Works on the free tier. One request per ticker, so 10 tickers = 10 requests = two minutes of free-tier budget. + +### 4.2 Daily market summary — the whole market in one request + +`GET /v2/aggs/grouped/locale/us/market/stocks/{date}` + +The efficient way to get EOD for many tickers: **one request returns every US ticker for that date.** + +```python +from datetime import date + +bars = client.get_grouped_daily_aggs(date="2026-08-28", adjusted=True) + +wanted = {"AAPL", "GOOGL", "MSFT"} +closes = {b.ticker: b.close for b in bars if b.ticker in wanted} +print(closes) +``` + +`GroupedDailyAgg` adds a `ticker` attribute (JSON key `T`) to the standard `Agg` fields. `timestamp`/`t` is **milliseconds**, marking the *end* of the aggregate window. + +Caveats: the date must be a **trading day** — a weekend or holiday returns an empty result set, not an error. And the response covers the entire market (thousands of rows), so filter client-side. + +### 4.3 Daily open/close for one ticker on one date + +`GET /v1/open-close/{ticker}/{date}` + +```python +oc = client.get_daily_open_close_agg("AAPL", date="2026-08-28", adjusted=True) +print(oc.open, oc.close, oc.pre_market, oc.after_hours, oc.status) +``` + +`DailyOpenCloseAgg` is the one model that carries pre-market and after-hours prints. Note it uses `symbol` (not `ticker`) and `from_` (not `from`, which is a Python keyword). + +--- + +## 5. Historical bars — for charts and backfill + +`GET /v2/aggs/ticker/{ticker}/range/{multiplier}/{timespan}/{from}/{to}` + +```python +# 1-minute bars for one session +bars = client.get_aggs( + ticker="AAPL", + multiplier=1, + timespan="minute", # second|minute|hour|day|week|month|quarter|year + from_="2026-08-28", # YYYY-MM-DD, date, datetime, or Unix ms + to="2026-08-28", + adjusted=True, + sort="asc", + limit=50000, +) + +for b in bars: + print(b.timestamp, b.open, b.high, b.low, b.close, b.volume) +``` + +`get_aggs` returns a **list** and is capped at 50,000 bars. `list_aggs` takes the same arguments but returns an **auto-paginating iterator** — convenient for long ranges, and a way to accidentally issue many billed requests. Prefer `get_aggs` with an explicit range unless you genuinely need more than 50k bars. + +`Agg.timestamp` is **milliseconds**, marking the start of the window. + +Relevance to FinAlly: this is the only way to seed a chart with real history under Massive. The rolling in-memory history in §6 of `PLAN.md` covers the simulator; a Massive-backed deployment could optionally backfill `GET /api/prices/{ticker}/history` from 1-minute aggregates instead. That is out of scope today, and noted here so the option is not rediscovered later. + +--- + +## 6. Market status + +Worth calling to explain a frozen feed to the user rather than leaving them guessing. + +```python +status = client.get_market_status() +print(status.market) # "open" | "closed" | "extended-hours" +print(status.exchanges) +print(status.after_hours, status.early_hours) +``` + +`client.get_market_holidays()` returns upcoming closures and early closes. + +--- + +## 7. Timestamp units — the trap + +Massive uses **three different time units across endpoints**, and the SDK passes them through unchanged. This is the single easiest thing to get wrong. + +| Source | Attribute | Unit | To Unix seconds | +|---|---|---|---| +| Snapshot `lastTrade` | `sip_timestamp` | **nanoseconds** | `/ 1_000_000_000` | +| Snapshot `lastQuote` | `sip_timestamp` | **nanoseconds** | `/ 1_000_000_000` | +| Snapshot top level | `updated` | **nanoseconds** | `/ 1_000_000_000` | +| Snapshot `min` | `timestamp` | **milliseconds** | `/ 1_000` | +| Aggregates (`Agg`, `PreviousCloseAgg`, grouped) | `timestamp` | **milliseconds** | `/ 1_000` | + +FinAlly's `PriceUpdate.timestamp` is **Unix epoch seconds as a float** (§7 of `PLAN.md`), so every value from this API needs converting, and the divisor depends on which endpoint it came from. + +```python +NANOS_PER_SECOND = 1_000_000_000 +MILLIS_PER_SECOND = 1_000 + +ts_seconds = snap.last_trade.sip_timestamp / NANOS_PER_SECOND # snapshot +ts_seconds = agg.timestamp / MILLIS_PER_SECOND # aggregates +``` + +### Attribute names never match JSON keys + +The wire format is single-letter (`p`, `s`, `t`, `x`); the SDK's `from_dict` maps those to readable attributes. You must use the **attribute** names. Reading `snap.last_trade.t` or `snap.last_trade.timestamp` raises `AttributeError`, because `@modelclass` builds a plain dataclass with no `__getattr__` fallback: + +```python +# massive/rest/models/trades.py +@staticmethod +def from_dict(d): + return LastTrade( + d.get("T"), d.get("f"), d.get("q"), d.get("t"), # "t" -> sip_timestamp + d.get("y"), d.get("c"), d.get("e"), d.get("i"), + d.get("p"), # "p" -> price + d.get("r"), d.get("s"), d.get("x"), d.get("z"), + ) +``` + +--- + +## 8. Two defects confirmed in `backend/app/market/massive_client.py` + +Both were reproduced against the installed SDK, not inferred. They are recorded here because this document is the reference the fix should be written from; the fix itself belongs to whoever next touches that module. + +### 8.1 `last_trade.timestamp` does not exist — the Massive path returns no prices at all + +`_poll_once` reads: + +```python +price = snap.last_trade.price +timestamp = snap.last_trade.timestamp / 1000.0 # AttributeError +``` + +Reproduction, using a payload shaped exactly as the v2 snapshot documentation specifies: + +```python +from massive.rest.models.snapshot import TickerSnapshot + +snap = TickerSnapshot.from_dict({ + "ticker": "AAPL", + "lastTrade": {"p": 190.52, "s": 100, "t": 1755873791482000000, "x": 4}, +}) + +snap.last_trade.price # 190.52 +snap.last_trade.sip_timestamp # 1755873791482000000 +snap.last_trade.timestamp # AttributeError: 'LastTrade' object has no attribute 'timestamp' +``` + +The loop wraps each snapshot in `except (AttributeError, TypeError)` and merely logs a warning, so the exception is swallowed **once per ticker, on every poll**. The cache is never written. The observable symptom is not a crash: it is a watchlist where every ticker shows `—` forever, with `Skipping snapshot for AAPL` in the logs. + +The correct attribute is `sip_timestamp`. + +### 8.2 The unit divisor is wrong by a factor of 10⁶ + +Even with the attribute corrected, `/ 1000.0` treats nanoseconds as milliseconds. `1755873791482000000 / 1000` is ≈ 1.76 × 10¹⁵ seconds — roughly 55 million years in the future. Charts keyed on that timestamp would be unusable. The divisor must be `1_000_000_000`. + +### 8.3 Why 94% test coverage did not catch either defect + +`massive_client.py` is 94% covered and all 73 tests pass. The tests nonetheless assert the buggy behaviour, because they build snapshots from `MagicMock` (`backend/tests/market/test_massive.py`): + +```python +def _make_snapshot(ticker: str, price: float, timestamp_ms: int) -> MagicMock: + snap = MagicMock() + snap.last_trade = MagicMock() + snap.last_trade.price = price + snap.last_trade.timestamp = timestamp_ms # attribute the real model does not have + return snap +``` + +A `MagicMock` answers to any attribute name, so `snap.last_trade.timestamp` resolves happily in the test and raises `AttributeError` in production. `test_timestamp_conversion` then locks in the wrong unit as well: + +```python +assert update.timestamp == 1707580800.0 # asserts milliseconds -> seconds +``` + +The lesson generalises: **mocking a third-party model tests your assumptions about the library, not the library.** Parsing tests must go through the real `TickerSnapshot.from_dict` with a documented payload, as in §10. That form of test needs no network and would have failed on the first run. + +### Corrected parse + +```python +NANOS_PER_SECOND = 1_000_000_000 + +for snap in snapshots: + trade = snap.last_trade + if trade is None or trade.price is None: + continue # no print yet today; leave the ticker showing "—" + self._cache.update( + ticker=snap.ticker, + price=trade.price, + timestamp=( + trade.sip_timestamp / NANOS_PER_SECOND + if trade.sip_timestamp + else time.time() + ), + ) +``` + +Guarding on `is None` rather than catching `AttributeError` is what makes the difference: a genuinely absent field is a normal condition to handle, whereas a misspelled attribute is a bug that should be loud. The existing blanket `except AttributeError` is precisely what hid this one. + +--- + +## 9. Errors and operational behaviour + +The SDK raises only two exception types (`massive/exceptions.py`): + +| Exception | Cause | +|---|---| +| `AuthError` | Empty or missing API key at construction time | +| `BadResponse` | Any non-200 response that survived the retry policy | + +`urllib3` raises its own errors for connection failures and timeouts. A poll loop should therefore catch broadly and keep going, since a failed poll is recoverable on the next cycle: + +```python +from massive.exceptions import AuthError, BadResponse + +try: + snapshots = await asyncio.to_thread(self._fetch_snapshots) +except AuthError: + logger.error("Massive API key rejected — falling back is not automatic") + raise # unrecoverable: do not retry on a loop +except BadResponse as e: + logger.warning("Massive returned an error response: %s", e) + return # transient: retry next interval +except Exception: + logger.exception("Massive poll failed") + return +``` + +### Behaviours to surface in the README + +These are properties of the data source, not bugs, and users will otherwise report them as bugs: + +- **Unknown symbols vanish silently.** The v2 snapshot omits tickers it does not recognise; there is no error entry. The ticker sits in the watchlist showing `—` indefinitely. (v3 would report `NOT_FOUND` — see §3.2.) +- **Prices freeze outside market hours.** Overnight, at weekends, and on holidays the snapshot returns the last trade of the previous session. The UI looks broken but is correct. This is the main reason the simulator is the default. +- **Free-tier data is 15 minutes delayed**, so prices will not match any other quote source the user has open. +- **Snapshot data is cleared at midnight ET** and repopulates from about 4am ET. Between those times `last_trade` may be absent entirely — which is exactly the `None` case §8 guards. + +--- + +## 10. Verification script + +Run this once a real `MASSIVE_API_KEY` is available. It confirms auth, the multi-ticker snapshot, unit conversion, and the EOD path in one pass. + +```python +# backend/scripts/verify_massive.py +"""Smoke-test the Massive REST API against a live key.""" + +import os +from datetime import UTC, datetime + +from massive import RESTClient +from massive.rest.models import SnapshotMarketType + +NANOS_PER_SECOND = 1_000_000_000 +TICKERS = ["AAPL", "GOOGL", "MSFT", "NVDA", "TSLA"] + + +def main() -> None: + key = os.environ["MASSIVE_API_KEY"] + client = RESTClient(api_key=key) + + status = client.get_market_status() + print(f"market: {status.market}") + + snapshots = client.get_snapshot_all(SnapshotMarketType.STOCKS, TICKERS) + print(f"requested {len(TICKERS)}, received {len(snapshots)}") + + for snap in snapshots: + trade = snap.last_trade + if trade is None or trade.price is None: + print(f"{snap.ticker}: no trade data") + continue + seconds = trade.sip_timestamp / NANOS_PER_SECOND + when = datetime.fromtimestamp(seconds, UTC) + print(f"{snap.ticker}: ${trade.price:.2f} at {when:%Y-%m-%d %H:%M:%S} UTC") + + missing = set(TICKERS) - {s.ticker for s in snapshots} + if missing: + print(f"absent from response (unknown or untraded): {sorted(missing)}") + + prev = client.get_previous_close_agg("AAPL") + print(f"AAPL previous close: ${prev.close:.2f}") + + +if __name__ == "__main__": + main() +``` + +```bash +uv run python scripts/verify_massive.py +``` + +Expected: a market status, five priced tickers with timestamps in the recent past (not 55 million years hence), and a previous close. Timestamps far in the future mean the unit divisor is wrong; `AttributeError` means §8.1 has regressed. + +--- + +## 11. Summary — what FinAlly uses + +| Need | Endpoint | SDK call | Cost | +|---|---|---|---| +| Live prices, all watched tickers | `/v2/snapshot/.../tickers` | `get_snapshot_all` | 1 request per poll | +| EOD close, one ticker | `/v2/aggs/ticker/{t}/prev` | `get_previous_close_agg` | 1 request per ticker | +| EOD close, many tickers | `/v2/aggs/grouped/...` | `get_grouped_daily_aggs` | 1 request total | +| Chart backfill | `/v2/aggs/ticker/{t}/range/...` | `get_aggs` | 1 request per ticker | +| Explain a frozen feed | `/v1/marketstatus/now` | `get_market_status` | 1 request | + +The polling design that follows from the 5 req/min free tier — one snapshot request covering the union of watchlist and held positions, every 15 seconds — is specified in `MARKET_INTERFACE.md`. + +## Sources + +- [Full Market Snapshot](https://massive.com/docs/rest/stocks/snapshots/full-market-snapshot) +- [Unified Snapshot](https://massive.com/docs/rest/stocks/snapshots/unified-snapshot) +- [Previous Day Bar](https://massive.com/docs/rest/stocks/aggregates/previous-day-bar) +- [Daily Market Summary](https://massive.com/docs/rest/stocks/aggregates/daily-market-summary) +- [Request limits for Massive's RESTful APIs](https://massive.com/knowledge-base/article/what-is-the-request-limit-for-massives-restful-apis) +- [massive-com/client-python](https://github.com/massive-com/client-python) +- Installed SDK source: `backend/.venv/lib/python3.13/site-packages/massive/` (v2.2.0) diff --git a/planning/PLAN.md b/planning/PLAN.md index bc1811b33..eecf11643 100644 --- a/planning/PLAN.md +++ b/planning/PLAN.md @@ -23,12 +23,14 @@ The user runs a single Docker command (or a provided start script). A browser op - **Watch prices stream** — prices flash green (uptick) or red (downtick) with subtle CSS animations that fade - **View sparkline mini-charts** — price action beside each ticker in the watchlist, accumulated on the frontend from the SSE stream since page load (sparklines fill in progressively) -- **Click a ticker** to see a larger detailed chart in the main chart area +- **Click a ticker** to see a larger detailed chart in the main chart area, backfilled from the server's rolling price history so the chart is populated immediately - **Buy and sell shares** — market orders only, instant fill at current price, no fees, no confirmation dialog - **Monitor their portfolio** — a heatmap (treemap) showing positions sized by weight and colored by P&L, plus a P&L chart tracking total portfolio value over time - **View a positions table** — ticker, quantity, average cost, current price, unrealized P&L, % change +- **Review their trade history** — a compact blotter of executed trades beneath the positions table - **Chat with the AI assistant** — ask about their portfolio, get analysis, and have the AI execute trades and manage the watchlist through natural language - **Manage the watchlist** — add/remove tickers manually or via the AI chat +- **Reset the simulation** — one button returns the account to a clean $10,000 with the default watchlist, so the app can be demoed repeatedly ### Visual Design @@ -57,14 +59,14 @@ The user runs a single Docker command (or a provided start script). A browser op │ └── /* Static file serving │ │ (Next.js export) │ │ │ -│ SQLite database (volume-mounted) │ -│ Background task: market data polling/sim │ +│ SQLite database (bind-mounted to ./db) │ +│ Background tasks: market data + snapshots │ └─────────────────────────────────────────────────┘ ``` - **Frontend**: Next.js with TypeScript, built as a static export (`output: 'export'`), served by FastAPI as static files - **Backend**: FastAPI (Python), managed as a `uv` project -- **Database**: SQLite, single file at `db/finally.db`, volume-mounted for persistence +- **Database**: SQLite, single file at `db/finally.db`, bind-mounted for persistence - **Real-time data**: Server-Sent Events (SSE) — simpler than WebSockets, one-way server→client push, works everywhere - **AI integration**: LiteLLM → OpenRouter (Cerebras for fast inference), with structured outputs for trade execution - **Market data**: Environment-variable driven — simulator by default, real data via Massive API if key provided @@ -76,9 +78,11 @@ The user runs a single Docker command (or a provided start script). A browser op | SSE over WebSockets | One-way push is all we need; simpler, no bidirectional complexity, universal browser support | | Static Next.js export | Single origin, no CORS issues, one port, one container, simple deployment | | SQLite over Postgres | No auth = no multi-user = no need for a database server; self-contained, zero config | -| Single Docker container | Students run one command; no docker-compose for production, no service orchestration | +| Bind mount over named volume | `./db/finally.db` is visible on the host — students can inspect it with any SQLite browser, or delete it to reset | +| Single Docker container | Students run one command; no docker-compose, no service orchestration | | uv for Python | Fast, modern Python project management; reproducible lockfile; what students should learn | | Market orders only | Eliminates order book, limit order logic, partial fills — dramatically simpler portfolio math | +| Recharts as the only chart library | Line charts, sparklines, and the treemap heatmap all come from one dependency — no second charting mental model | --- @@ -88,7 +92,9 @@ The user runs a single Docker command (or a provided start script). A browser op finally/ ├── frontend/ # Next.js TypeScript project (static export) ├── backend/ # FastAPI uv project (Python) -│ └── db/ # Schema definitions, seed data, migration logic +│ └── app/ +│ ├── market/ # Market data (complete — see MARKET_DATA_SUMMARY.md) +│ └── db/ # Schema definitions, seed data, connection handling ├── planning/ # Project-wide documentation for agents │ ├── PLAN.md # This document │ └── ... # Additional agent reference docs @@ -97,11 +103,10 @@ finally/ │ ├── stop_mac.sh # Stop Docker container (macOS/Linux) │ ├── start_windows.ps1 # Launch Docker container (Windows PowerShell) │ └── stop_windows.ps1 # Stop Docker container (Windows PowerShell) -├── test/ # Playwright E2E tests + docker-compose.test.yml -├── db/ # Volume mount target (SQLite file lives here at runtime) -│ └── .gitkeep # Directory exists in repo; finally.db is gitignored +├── test/ # Playwright E2E tests +├── db/ # Bind mount target (SQLite file lives here at runtime) +│ └── .gitkeep # Directory exists in repo; db/*.db is gitignored ├── Dockerfile # Multi-stage build (Node → Python) -├── docker-compose.yml # Optional convenience wrapper ├── .env # Environment variables (gitignored, .env.example committed) └── .gitignore ``` @@ -110,11 +115,11 @@ finally/ - **`frontend/`** is a self-contained Next.js project. It knows nothing about Python. It talks to the backend via `/api/*` endpoints and `/api/stream/*` SSE endpoints. Internal structure is up to the Frontend Engineer agent. - **`backend/`** is a self-contained uv project with its own `pyproject.toml`. It owns all server logic including database initialization, schema, seed data, API routes, SSE streaming, market data, and LLM integration. Internal structure is up to the Backend/Market Data agents. -- **`backend/db/`** contains schema SQL definitions and seed logic. The backend lazily initializes the database on first request — creating tables and seeding default data if the SQLite file doesn't exist or is empty. -- **`db/`** at the top level is the runtime volume mount point. The SQLite file (`db/finally.db`) is created here by the backend and persists across container restarts via Docker volume. +- **`backend/app/db/`** contains schema SQL definitions, seed logic, and connection handling. It is application code and imports as `app.db`. The backend lazily initializes the database on first request — creating tables and seeding default data if the SQLite file doesn't exist or is empty. (Note the deliberate distinction from the root `db/`, which holds no code.) +- **`db/`** at the top level is the runtime bind mount point. The SQLite file (`db/finally.db`) is created here by the backend and persists across container restarts. - **`planning/`** contains project-wide documentation, including this plan. All agents reference files here as the shared contract. -- **`test/`** contains Playwright E2E tests and supporting infrastructure (e.g., `docker-compose.test.yml`). Unit tests live within `frontend/` and `backend/` respectively, following each framework's conventions. -- **`scripts/`** contains start/stop scripts that wrap Docker commands. +- **`test/`** contains Playwright E2E tests. Unit tests live within `frontend/` and `backend/` respectively, following each framework's conventions. +- **`scripts/`** contains start/stop scripts that wrap Docker commands. These are the only supported launch path — there is deliberately no `docker-compose.yml`, so there is only one thing to keep in sync. --- @@ -136,48 +141,96 @@ LLM_MOCK=false - If `MASSIVE_API_KEY` is set and non-empty → backend uses Massive REST API for market data - If `MASSIVE_API_KEY` is absent or empty → backend uses the built-in market simulator -- If `LLM_MOCK=true` → backend returns deterministic mock LLM responses (for E2E tests) -- The backend reads `.env` from the project root (mounted into the container or read via docker `--env-file`) +- If `LLM_MOCK=true` → backend returns deterministic mock LLM responses (see §9, LLM Mock Mode) + +### How `.env` Is Loaded + +Two distinct mechanisms — do not confuse them: + +- **Local development** (running `uv run uvicorn ...` directly): the backend reads `.env` from the project root via `python-dotenv`. There is no container involved. +- **Docker**: `docker run --env-file .env ...` injects the variables as real environment variables. The `.env` file is *not* mounted into the container and does not exist inside it. + +Both paths end with the same `os.environ` contents, so backend code only ever reads `os.environ`. --- ## 6. Market Data +> Status: **complete**. Implemented in `backend/app/market/`. See `planning/MARKET_DATA_SUMMARY.md` for the module map and `backend/CLAUDE.md` for the API. The subsections below record the contract that downstream code depends on, including three additions still to be built (marked **TODO**). + ### Two Implementations, One Interface -Both the simulator and the Massive client implement the same abstract interface. The backend selects which to use based on the environment variable. All downstream code (SSE streaming, price cache, frontend) is agnostic to the source. +Both the simulator and the Massive client implement the same abstract interface (`MarketDataSource`). The backend selects which to use via `create_market_data_source(cache)`, based on the environment variable. All downstream code (SSE streaming, price cache, frontend) is agnostic to the source. ### Simulator (Default) - Generates prices using geometric Brownian motion (GBM) with configurable drift and volatility per ticker - Updates at ~500ms intervals -- Correlated moves across tickers (e.g., tech stocks move together) +- Correlated moves across tickers via Cholesky decomposition (tech 0.6, finance 0.5, cross-sector 0.3) - Occasional random "events" — sudden 2-5% moves on a ticker for drama -- Starts from realistic seed prices (e.g., AAPL ~$190, GOOGL ~$175, etc.) +- Starts from realistic seed prices (AAPL $190, GOOGL $175, etc.) +- **Unknown tickers are supported**: a ticker with no entry in `SEED_PRICES` gets a random start in $50–$300, `DEFAULT_PARAMS` for drift/volatility, and cross-sector correlation. Adding a ticker seeds the cache immediately, so it has a price on the very next SSE event. - Runs as an in-process background task — no external dependencies ### Massive API (Optional) - REST API polling (not WebSocket) — simpler, works on all tiers -- Polls for the union of all watched tickers on a configurable interval -- Free tier (5 calls/min): poll every 15 seconds -- Paid tiers: poll every 2-15 seconds depending on tier -- Parses REST response into the same format as the simulator +- Polls for the union of all tracked tickers on a configurable interval +- Free tier (5 calls/min): poll every 15 seconds. Paid tiers: 2-15 seconds. +- Parses REST response into the same `PriceUpdate` format as the simulator +- **Known limitations to surface in the README**: an unknown or invalid symbol simply never produces a price, so the ticker sits in the watchlist showing `—`; and outside regular trading hours the API returns the last close, so prices appear frozen on evenings and weekends. The simulator is the default precisely because it always looks alive. + +### Which Tickers Are Tracked + +**The tracked ticker set is `watchlist ∪ {tickers with a non-zero position}`.** + +This matters because the two sets diverge: a user can buy TSLA and then remove TSLA from the watchlist, and the position still needs a live price for valuation, P&L, the heatmap, and snapshots. `SimulatorDataSource.remove_ticker()` deletes the ticker from the cache, so calling it for a held ticker would silently freeze that position's value. + +The rule, therefore: + +- `POST /api/watchlist` → always `await source.add_ticker(t)` +- `DELETE /api/watchlist/{t}` → `await source.remove_ticker(t)` **only if no position in `t` is held**. Otherwise the ticker leaves the watchlist UI but stays in the feed. +- Buying a ticker that is not tracked → `await source.add_ticker(t)` as part of trade execution +- Selling a position to zero → if `t` is not in the watchlist, `await source.remove_ticker(t)` + +### Ticker Validation + +Applied at the API boundary (`POST /api/watchlist`, `POST /api/portfolio/trade`, and every LLM-proposed action), so the rest of the system only ever sees canonical symbols: + +- Normalize: `ticker.strip().upper()` +- Validate against `^[A-Z][A-Z.]{0,5}$` — reject anything else with `400` +- No allowlist. Any symbol matching the pattern is accepted; the simulator invents plausible behavior for it, and under Massive an unknown symbol shows `—`. Rejecting unknown symbols would make the LLM's `watchlist_changes` feature feel broken. +- Tickers are stored uppercase everywhere. The `UNIQUE(user_id, ticker)` constraints would otherwise happily hold both `AAPL` and `aapl`. ### Shared Price Cache -- A single background task (simulator or Massive poller) writes to an in-memory price cache -- The cache holds the latest price, previous price, and timestamp for each ticker -- SSE streams read from this cache and push updates to connected clients +- A single background task (simulator or Massive poller) writes to `PriceCache`, an in-memory, thread-safe store +- The cache holds the latest price, previous price, and timestamp for each ticker, plus a monotonic `version` counter that increments on every update +- SSE streams, portfolio valuation, and trade execution all read from this cache - This architecture supports future multi-user scenarios without changes to the data layer +### Rolling Price History (**TODO**) + +`PriceCache` keeps, per ticker, a bounded `deque` of the last **600** `(timestamp, price)` points — about five minutes at the 500ms simulator cadence. This exists solely so the main chart is populated the instant a user clicks a ticker, rather than drawing itself from scratch over the following minute. It is served by `GET /api/prices/{ticker}/history`. + +Memory cost is trivial (600 points × ~50 tickers × 16 bytes ≈ 500KB) and it is deliberately *not* persisted — restarting the app clears it, which is the honest behavior for a simulator with no real history. + ### SSE Streaming -- Endpoint: `GET /api/stream/prices` -- Long-lived SSE connection; client uses native `EventSource` API -- Server pushes price updates for all tickers known to the system at a regular cadence (~500ms) — in the single-user model this is equivalent to the user's watchlist -- Each SSE event contains ticker, price, previous price, timestamp, and change direction -- Client handles reconnection automatically (EventSource has built-in retry) +- Endpoint: `GET /api/stream/prices`, `Content-Type: text/event-stream` +- Long-lived SSE connection; client uses the native `EventSource` API +- The stream opens with `retry: 1000`, then pushes the **entire price cache** as a single JSON object whenever `PriceCache.version` changes, checked every 500ms. Payload shape is a map keyed by ticker: + +``` +retry: 1000 + +data: {"AAPL": {"ticker": "AAPL", "price": 190.52, "previous_price": 190.48, "timestamp": 1755873791.482, "change": 0.04, "change_percent": 0.021, "direction": "up"}, "GOOGL": {...}} +``` + +- **`timestamp` is Unix epoch seconds as a float** (not ISO), and **`change_percent` is already in percent units** (`0.021` means 0.021%, not 2.1%). The frontend must not multiply by 100 again. This is `PriceUpdate.to_dict()` and it is frozen — treat it as the contract. +- Because the first loop iteration always sees a version change, a newly connected client receives a **full snapshot immediately**, including after a reconnect. No separate snapshot endpoint or event type is needed. +- **TODO — keepalive**: when the version has not changed for 15 seconds, emit an SSE comment line (`: ping\n\n`). Without it, a Massive-backed feed (15s polls) sends no bytes between polls, which idle-timeouts through proxies and leaves the frontend unable to distinguish "quiet market" from "connection dead". The connection indicator depends on this. +- Client reconnection is automatic (`EventSource` built-in retry, 1s as directed by the stream) --- @@ -189,47 +242,62 @@ The backend checks for the SQLite database on startup (or first request). If the - No separate migration step - No manual database setup -- Fresh Docker volumes start with a clean, seeded database automatically +- Fresh containers start with a clean, seeded database automatically + +### Access Pattern + +A background snapshot task, long-lived SSE generators, and request handlers all share one process and one event loop. Synchronous `sqlite3` calls inside an `async def` handler block that loop — with an SSE stream attached, that shows up as visibly stuttering prices. So: + +- Enable **WAL mode** at initialization (`PRAGMA journal_mode=WAL`) plus `PRAGMA foreign_keys=ON` and a busy timeout +- Use short-lived connections per operation with `check_same_thread=False` +- Dispatch DB work off the event loop: either write DB functions as plain `def` route handlers (FastAPI runs them in its threadpool automatically) or wrap calls in `asyncio.to_thread`. Never call `sqlite3` directly from an `async def` handler. + +### Timestamps + +Two different representations, deliberately, and they must not be mixed up: + +- **Database columns** are ISO 8601 TEXT, **timezone-aware UTC**: `datetime.now(UTC).isoformat()` → `"2026-08-22T14:03:11.482000+00:00"`. Naive local timestamps break chart axes and cross-restart ordering. +- **Price data** (`PriceUpdate.timestamp`, SSE payloads) is a Unix epoch float, because that is what the market layer already emits and what charting libraries want. ### Schema -All tables include a `user_id` column defaulting to `"default"`. This is hardcoded for now (single-user) but enables future multi-user support without schema migration. +All tables include a `user_id` column defaulting to `"default"`. This is hardcoded for now (single-user) but enables future multi-user support without schema migration. Foreign keys between `user_id` and `users_profile.id` are deliberately **not** declared — with one hardcoded user they add ceremony without protection. **users_profile** — User state (cash balance) - `id` TEXT PRIMARY KEY (default: `"default"`) - `cash_balance` REAL (default: `10000.0`) -- `created_at` TEXT (ISO timestamp) +- `created_at` TEXT (ISO timestamp, UTC) **watchlist** — Tickers the user is watching - `id` TEXT PRIMARY KEY (UUID) - `user_id` TEXT (default: `"default"`) -- `ticker` TEXT -- `added_at` TEXT (ISO timestamp) +- `ticker` TEXT (uppercase) +- `added_at` TEXT (ISO timestamp, UTC) - UNIQUE constraint on `(user_id, ticker)` **positions** — Current holdings (one row per ticker per user) - `id` TEXT PRIMARY KEY (UUID) - `user_id` TEXT (default: `"default"`) -- `ticker` TEXT +- `ticker` TEXT (uppercase) - `quantity` REAL (fractional shares supported) - `avg_cost` REAL -- `updated_at` TEXT (ISO timestamp) +- `updated_at` TEXT (ISO timestamp, UTC) - UNIQUE constraint on `(user_id, ticker)` **trades** — Trade history (append-only log) - `id` TEXT PRIMARY KEY (UUID) - `user_id` TEXT (default: `"default"`) -- `ticker` TEXT +- `ticker` TEXT (uppercase) - `side` TEXT (`"buy"` or `"sell"`) - `quantity` REAL (fractional shares supported) - `price` REAL -- `executed_at` TEXT (ISO timestamp) +- `executed_at` TEXT (ISO timestamp, UTC) -**portfolio_snapshots** — Portfolio value over time (for P&L chart). Recorded every 30 seconds by a background task, and immediately after each trade execution. +**portfolio_snapshots** — Portfolio value over time (for P&L chart). Recorded every 30 seconds by a background task. - `id` TEXT PRIMARY KEY (UUID) - `user_id` TEXT (default: `"default"`) - `total_value` REAL -- `recorded_at` TEXT (ISO timestamp) +- `recorded_at` TEXT (ISO timestamp, UTC) **chat_messages** — Conversation history with LLM - `id` TEXT PRIMARY KEY (UUID) @@ -237,70 +305,205 @@ All tables include a `user_id` column defaulting to `"default"`. This is hardcod - `role` TEXT (`"user"` or `"assistant"`) - `content` TEXT - `actions` TEXT (JSON — trades executed, watchlist changes made; null for user messages) -- `created_at` TEXT (ISO timestamp) +- `created_at` TEXT (ISO timestamp, UTC) + +### Snapshot Task Rules + +- Cadence: every 30 seconds, from a single background task started at app startup +- **Snapshots are not written on trade execution.** A trade does not change total portfolio value — cash out, equal position value in — so a per-trade snapshot only adds a duplicate point at a moment when the value is, by construction, unchanged. The 30-second cadence tells the whole story. +- **Skip the write entirely if any held ticker has no cached price.** Right after startup the cache may be empty, and under Massive the first poll can be 15 seconds out. Writing then puts a bogus first point on the P&L chart at every restart. +- Growth is ~2,880 rows/day, unbounded and harmless at demo scale. `GET /api/portfolio/history` caps and orders results rather than the writer pruning. + +### Portfolio Math — Canonical Formulas + +Backend valuation, the header, the snapshot task, and the LLM context block all compute these. They are written down once, here, so they cannot drift: + +``` +position_value = quantity × current_price +positions_value = Σ position_value +total_value = cash_balance + positions_value +unrealized_pnl = quantity × (current_price − avg_cost) +pct_change = (current_price − avg_cost) / avg_cost +weight = position_value / total_value + +buy: avg_cost = (old_qty × old_avg_cost + qty × price) / (old_qty + qty) + cash_balance −= qty × price +sell: avg_cost unchanged + cash_balance += qty × price +``` + +- **When a held ticker has no cached price, value it at `avg_cost`** (P&L reads as zero rather than as a crash). +- **Realized P&L is not tracked.** Selling at a profit simply moves value into cash and the position disappears from the table. The P&L chart on total portfolio value is the single source of performance truth. This is a deliberate simplification, not an oversight — do not add a `realized_pnl` column. +- **Round `cash_balance` to 2 decimals** after every trade, or the header eventually reads `9999.999999999998`. +- **After a sell, delete the position row if `quantity < 1e-9`.** Floating-point residue of `2.8e-16` shares renders as `0.00` but would otherwise keep the ticker in the positions table and in the tracked feed forever. ### Default Seed Data - One user profile: `id="default"`, `cash_balance=10000.0` - Ten watchlist entries: AAPL, GOOGL, MSFT, AMZN, TSLA, NVDA, META, JPM, V, NFLX +`POST /api/reset` restores exactly this state: truncate `positions`, `trades`, `portfolio_snapshots`, and `chat_messages`; set `cash_balance` to 10000.0; replace the watchlist with the ten defaults; and re-sync the market data source's tracked tickers to match. + --- -## 8. API Endpoints +## 8. API Contract + +Every endpoint's exact request and response shape is specified here. This section is the contract between the Backend and Frontend agents — neither should invent field names. Field naming is `snake_case` throughout, matching the database and the existing `PriceUpdate.to_dict()`. + +### Conventions + +- **Errors**: every non-2xx response is `{"error": "human readable message"}`. Status codes: `400` invalid input or failed business rule (insufficient cash, insufficient shares, bad ticker), `404` unknown resource, `503` LLM unavailable. FastAPI's default `422` body is replaced by an exception handler so the shape is uniform. +- **Money** is a JSON number rounded to 2 decimals. **Quantities** are numbers with up to 6 decimals. **Percentages** in REST responses are fractions (`0.0221` = 2.21%) — note this differs from the SSE `change_percent` field, which is already in percent units and is frozen for backward compatibility with the shipped market module. +- All timestamps in REST responses are ISO 8601 UTC strings, matching the database. + +### System + +**`GET /api/health`** → `200` +```json +{"status": "ok", "market_source": "simulator", "llm_mock": false} +``` + +**`POST /api/reset`** → `200` — restores the seeded state described in §7. +```json +{"cash_balance": 10000.0, "watchlist": ["AAPL", "GOOGL", "MSFT", "AMZN", "TSLA", "NVDA", "META", "JPM", "V", "NFLX"]} +``` ### Market Data -| Method | Path | Description | -|--------|------|-------------| -| GET | `/api/stream/prices` | SSE stream of live price updates | + +**`GET /api/stream/prices`** → SSE. Payload shape is specified in §6 and is frozen. + +**`GET /api/prices/{ticker}/history?limit=600`** → `200` — rolling in-memory history for the main chart. Empty `points` for an untracked ticker (not a 404 — the chart just draws nothing). +```json +{"ticker": "AAPL", "points": [{"timestamp": 1755873791.482, "price": 190.52}]} +``` ### Portfolio -| Method | Path | Description | -|--------|------|-------------| -| GET | `/api/portfolio` | Current positions, cash balance, total value, unrealized P&L | -| POST | `/api/portfolio/trade` | Execute a trade: `{ticker, quantity, side}` | -| GET | `/api/portfolio/history` | Portfolio value snapshots over time (for P&L chart) | + +**`GET /api/portfolio`** → `200` — one call gives the frontend everything the header, positions table, and heatmap need. Prices and P&L are computed server-side using the §7 formulas so the frontend never joins two sources. +```json +{ + "cash_balance": 8055.90, + "positions_value": 2003.60, + "total_value": 10059.50, + "unrealized_pnl": 59.50, + "positions": [ + { + "ticker": "AAPL", + "quantity": 10.0, + "avg_cost": 190.25, + "current_price": 194.46, + "market_value": 1944.60, + "unrealized_pnl": 42.10, + "pct_change": 0.0221, + "weight": 0.1933 + } + ] +} +``` + +**`POST /api/portfolio/trade`** — request: +```json +{"ticker": "AAPL", "side": "buy", "quantity": 10} +``` +→ `200`: +```json +{ + "trade": {"id": "uuid", "ticker": "AAPL", "side": "buy", "quantity": 10.0, "price": 194.46, "executed_at": "2026-08-22T14:03:11.482000+00:00"}, + "cash_balance": 8055.90, + "position": {"ticker": "AAPL", "quantity": 10.0, "avg_cost": 194.46} +} +``` +`position` is `null` when a sell closes the position. Validation, all `400`: + +| Condition | Message | +|---|---| +| `quantity <= 0`, NaN, or infinite | `Quantity must be a positive number` | +| `side` not `buy`/`sell` | `Side must be 'buy' or 'sell'` | +| Ticker fails the §6 pattern | `Invalid ticker symbol` | +| No cached price for the ticker | `No price available for TSLA yet` | +| Buy where `qty × price > cash_balance` | `Insufficient cash: need $1944.60, have $500.00` | +| Sell where `qty > held quantity` | `Insufficient shares: you hold 3 AAPL` | + +No shorting, no margin. Fill price is the cache price read at request time. + +**`GET /api/portfolio/history?limit=500&since=`** → `200` — for the P&L chart. Newest-last, capped at `limit` (default 500, max 2000); `since` is optional. +```json +{"snapshots": [{"total_value": 10000.0, "recorded_at": "2026-08-22T14:00:00+00:00"}]} +``` + +**`GET /api/trades?limit=50`** → `200` — for the blotter. Newest-first. +```json +{"trades": [{"id": "uuid", "ticker": "AAPL", "side": "buy", "quantity": 10.0, "price": 194.46, "executed_at": "..."}]} +``` ### Watchlist -| Method | Path | Description | -|--------|------|-------------| -| GET | `/api/watchlist` | Current watchlist tickers with latest prices | -| POST | `/api/watchlist` | Add a ticker: `{ticker}` | -| DELETE | `/api/watchlist/{ticker}` | Remove a ticker | + +**`GET /api/watchlist`** → `200` — `price` and its siblings are `null` until the first tick for that ticker. +```json +{ + "tickers": [ + {"ticker": "AAPL", "price": 194.46, "previous_price": 194.40, "change": 0.06, "direction": "up", "added_at": "..."}, + {"ticker": "PYPL", "price": null, "previous_price": null, "change": null, "direction": null, "added_at": "..."} + ] +} +``` + +**`POST /api/watchlist`** — request `{"ticker": "pypl"}` → `201 {"ticker": "PYPL", "added_at": "..."}`. Normalized and validated per §6. Adding a ticker already present is a no-op returning `200`, not an error. + +**`DELETE /api/watchlist/{ticker}`** → `204`. `404` if not on the watchlist. Per §6, the ticker stays in the price feed if a position is held. ### Chat -| Method | Path | Description | -|--------|------|-------------| -| POST | `/api/chat` | Send a message, receive complete JSON response (message + executed actions) | -### System -| Method | Path | Description | -|--------|------|-------------| -| GET | `/api/health` | Health check (for Docker/deployment) | +**`POST /api/chat`** — request `{"message": "buy 10 apple"}`, max 2000 characters (`400` beyond that). See §9 for behavior. → `200`: +```json +{ + "id": "uuid", + "message": "Bought 10 AAPL at $194.46. Your tech concentration is now 68% — consider diversifying.", + "actions": { + "trades": [ + {"ticker": "AAPL", "side": "buy", "quantity": 10, "status": "executed", "price": 194.46}, + {"ticker": "TSLA", "side": "sell", "quantity": 50, "status": "rejected", "error": "Insufficient shares: you hold 10 TSLA"} + ], + "watchlist_changes": [{"ticker": "PYPL", "action": "add", "status": "executed"}] + }, + "created_at": "..." +} +``` +`actions` always has both arrays, possibly empty. Every entry carries `status` of `"executed"` or `"rejected"`, and rejected entries carry `error`. + +**`GET /api/chat?limit=50`** → `200` — so the chat panel survives a page refresh. Oldest-first, the last `limit` messages. +```json +{"messages": [{"id": "uuid", "role": "user", "content": "buy 10 apple", "actions": null, "created_at": "..."}]} +``` + +**`DELETE /api/chat`** → `204` — clears conversation history only. Does not touch the portfolio. --- ## 9. LLM Integration -When writing code to make calls to LLMs, use cerebras-inference skill to use LiteLLM via OpenRouter to the `openrouter/openai/gpt-oss-120b` model with Cerebras as the inference provider. Structured Outputs should be used to interpret the results. +When writing code to make calls to LLMs, use the `cerebras` skill to call LiteLLM via OpenRouter against the `openrouter/openai/gpt-oss-120b` model with Cerebras as the inference provider. Structured Outputs interpret the result. + +There is an `OPENROUTER_API_KEY` in the `.env` file in the project root. -There is an OPENROUTER_API_KEY in the .env file in the project root. +> **Verify before building**: confirm that strict JSON-schema structured output actually works end-to-end through OpenRouter's Cerebras routing for this model. If it does not, fall back to prompt-enforced JSON plus the parse-failure path below — the rest of this section is unchanged either way. ### How It Works When the user sends a chat message, the backend: -1. Loads the user's current portfolio context (cash, positions with P&L, watchlist with live prices, total portfolio value) -2. Loads recent conversation history from the `chat_messages` table +1. Loads the user's current portfolio context (cash, positions with P&L, watchlist with live prices, total portfolio value) using the §7 formulas +2. Loads the **last 20 messages** from `chat_messages` (oldest-first) 3. Constructs a prompt with a system message, portfolio context, conversation history, and the user's new message -4. Calls the LLM via LiteLLM → OpenRouter, requesting structured output, using the cerebras-inference skill -5. Parses the complete structured JSON response -6. Auto-executes any trades or watchlist changes specified in the response -7. Stores the message and executed actions in `chat_messages` -8. Returns the complete JSON response to the frontend (no token-by-token streaming — Cerebras inference is fast enough that a loading indicator is sufficient) +4. Calls the LLM via LiteLLM → OpenRouter, requesting structured output +5. Parses the structured JSON response +6. Validates and executes any trades or watchlist changes specified in the response +7. Stores the user message and the assistant message (with its `actions` JSON) in `chat_messages` +8. Returns the response shape specified in §8 (no token-by-token streaming — Cerebras inference is fast enough that a loading indicator suffices) -### Structured Output Schema +Only one chat request may be in flight at a time; the frontend disables the input while waiting. Combined with the 2000-character message cap and the 20-message history window, this bounds both latency and OpenRouter spend — a retry loop on a failing endpoint would otherwise burn real credits. -The LLM is instructed to respond with JSON matching this schema: +### Structured Output Schema ```json { @@ -314,9 +517,7 @@ The LLM is instructed to respond with JSON matching this schema: } ``` -- `message` (required): The conversational text shown to the user -- `trades` (optional): Array of trades to auto-execute. Each trade goes through the same validation as manual trades (sufficient cash for buys, sufficient shares for sells) -- `watchlist_changes` (optional): Array of watchlist modifications +**All three fields are required.** `trades` and `watchlist_changes` are `[]` when there is nothing to do. Required-with-empty-array is markedly more reliable with structured outputs than optional-and-absent, and it removes a null check from every consumer. ### Auto-Execution @@ -325,24 +526,46 @@ Trades specified by the LLM execute automatically — no confirmation dialog. Th - It creates an impressive, fluid demo experience - It demonstrates agentic AI capabilities — the core theme of the course -If a trade fails validation (e.g., insufficient cash), the error is included in the chat response so the LLM can inform the user. +**Execution is best-effort and ordered.** Each action runs through exactly the same validation as a manual trade (§8). If the second of three trades fails, the first and third still execute. Every action is recorded in the response with `status` and, on failure, `error`. + +**Failures are reported without a second LLM call.** The model has already written its `message` by the time validation runs, so it cannot narrate a failure it never saw. Instead: + +- The backend appends a deterministic note to the stored assistant message, e.g. `\n\nNote: sell 50 TSLA was not executed — insufficient shares: you hold 10 TSLA.` +- The frontend additionally renders each action as an inline chip beside the message — green for executed, red with the error text for rejected. + +These compose, cost nothing, and add no latency. A second round-trip feeding failures back to the model would be more "agentic" but doubles cost and latency for a message the user can already read. + +### Malformed Response Handling + +The endpoint must never return a 500 into the chat panel. On a response that fails to parse or fails schema validation: retry once, then return +```json +{"message": "I had trouble forming a response — please try again.", "actions": {"trades": [], "watchlist_changes": []}} +``` +with `200`. If OpenRouter itself is unreachable or unauthorized, return `503` with the standard error envelope so the frontend can distinguish "the AI is down" from "the AI is confused". ### System Prompt Guidance -The LLM should be prompted as "FinAlly, an AI trading assistant" with instructions to: +Prompt the LLM as "FinAlly, an AI trading assistant" with instructions to: - Analyze portfolio composition, risk concentration, and P&L - Suggest trades with reasoning - Execute trades when the user asks or agrees - Manage the watchlist proactively - Be concise and data-driven in responses +- Use uppercase ticker symbols - Always respond with valid structured JSON ### LLM Mock Mode -When `LLM_MOCK=true`, the backend returns deterministic mock responses instead of calling OpenRouter. This enables: -- Fast, free, reproducible E2E tests -- Development without an API key -- CI/CD pipelines +When `LLM_MOCK=true`, the backend returns deterministic responses without calling OpenRouter — no API key needed, free, instant, reproducible. **The mock's behavior is part of the contract**, because E2E tests assert on it. It is keyword-driven on the lowercased user message, first match wins: + +| Trigger | `message` | `trades` | `watchlist_changes` | +|---|---|---|---| +| contains `buy` | `Mock: buying 1 share of AAPL.` | `[{"ticker":"AAPL","side":"buy","quantity":1}]` | `[]` | +| contains `sell` | `Mock: selling 1 share of AAPL.` | `[{"ticker":"AAPL","side":"sell","quantity":1}]` | `[]` | +| contains `watch` | `Mock: adding PYPL to your watchlist.` | `[]` | `[{"ticker":"PYPL","action":"add"}]` | +| anything else | `Mock: your portfolio is worth $X.` (X from live context) | `[]` | `[]` | + +Mock responses flow through the identical execution and validation path as real ones, so a mocked `sell` with no position produces a genuine rejection — which is exactly what the E2E error-path test needs. --- @@ -352,22 +575,33 @@ When `LLM_MOCK=true`, the backend returns deterministic mock responses instead o The frontend is a single-page application with a dense, terminal-inspired layout. The specific component architecture and layout system is up to the Frontend Engineer, but the UI should include these elements: -- **Watchlist panel** — grid/table of watched tickers with: ticker symbol, current price (flashing green/red on change), daily change %, and a sparkline mini-chart (accumulated from SSE since page load) -- **Main chart area** — larger chart for the currently selected ticker, with at minimum price over time. Clicking a ticker in the watchlist selects it here. -- **Portfolio heatmap** — treemap visualization where each rectangle is a position, sized by portfolio weight, colored by P&L (green = profit, red = loss) -- **P&L chart** — line chart showing total portfolio value over time, using data from `portfolio_snapshots` -- **Positions table** — tabular view of all positions: ticker, quantity, avg cost, current price, unrealized P&L, % change -- **Trade bar** — simple input area: ticker field, quantity field, buy button, sell button. Market orders, instant fill. -- **AI chat panel** — docked/collapsible sidebar. Message input, scrolling conversation history, loading indicator while waiting for LLM response. Trade executions and watchlist changes shown inline as confirmations. -- **Header** — portfolio total value (updating live), connection status indicator, cash balance +- **Watchlist panel** — grid/table of watched tickers with: ticker symbol, current price (flashing green/red on change), change, and a sparkline mini-chart accumulated from SSE since page load. Tickers with no price yet show `—`. +- **Main chart area** — larger chart for the currently selected ticker. Backfilled on selection from `GET /api/prices/{ticker}/history`, then extended live from the SSE stream. Clicking a ticker in the watchlist selects it here. +- **Portfolio heatmap** — treemap where each rectangle is a position, sized by `weight` and colored by `unrealized_pnl` (green = profit, red = loss) +- **P&L chart** — line chart of total portfolio value over time from `GET /api/portfolio/history` +- **Positions table** — ticker, quantity, avg cost, current price, unrealized P&L, % change — rendered directly from `GET /api/portfolio`, which already computes all of it +- **Trade blotter** — compact newest-first list of executed trades from `GET /api/trades`, beneath the positions table +- **Trade bar** — ticker field, quantity field (accepts fractional input), buy button, sell button. Market orders, instant fill. Server-side validation errors are shown inline verbatim — the messages in §8 are written to be user-facing. +- **AI chat panel** — docked/collapsible sidebar. Message input (2000-char cap, disabled while a request is in flight), scrolling history restored on load from `GET /api/chat`, loading indicator while waiting. Executed and rejected actions rendered as inline chips beside each assistant message. +- **Header** — portfolio total value (updating live), cash balance, connection status indicator, and a reset button (with a confirm step, since it is the one destructive action in the app) + +### Data Flow + +Two sources, and it is worth being precise about which owns what: + +- **SSE** owns live prices. Every event carries the full cache, so the client replaces its price map wholesale — no merging logic, no missed-update reconciliation. +- **REST** owns everything else. Re-fetch `GET /api/portfolio` and `GET /api/trades` after any trade or chat response; poll `GET /api/portfolio/history` every 30 seconds to match the snapshot cadence. +- Portfolio *values* shown in the header can be recomputed client-side from the SSE price map between fetches, using the §7 formulas, so the total ticks along with prices rather than jumping every 30 seconds. ### Technical Notes -- Use `EventSource` for SSE connection to `/api/stream/prices` -- Canvas-based charting library preferred (Lightweight Charts or Recharts) for performance -- Price flash effect: on receiving a new price, briefly apply a CSS class with background color transition, then remove it +- Use `EventSource` for SSE to `/api/stream/prices`. Connection indicator: green on `onopen`, yellow on `onerror` (EventSource retries automatically), red after repeated failures or a `: ping` gap exceeding ~40 seconds. +- Remember the two SSE quirks from §6: `timestamp` is Unix epoch **seconds as a float** (multiply by 1000 for `Date`), and `change_percent` is **already a percentage**. +- **Recharts for every chart** — the main line chart, the sparklines, and the `` heatmap. Lightweight Charts has no treemap, so choosing it would force a second charting library for one component; at 10 tickers and 500ms updates Recharts is comfortably fast enough. +- Price flash effect: on receiving a new price, briefly apply a CSS class with a background-color transition, then remove it - All API calls go to the same origin (`/api/*`) — no CORS configuration needed - Tailwind CSS for styling with a custom dark theme +- Next.js static export means no server components, route handlers, middleware, or image optimization — this is a client-rendered SPA that Next happens to bundle. Set `output: 'export'` and `images: {unoptimized: true}`, and keep everything under a single route. --- @@ -378,48 +612,58 @@ The frontend is a single-page application with a dense, terminal-inspired layout ``` Stage 1: Node 20 slim - Copy frontend/ - - npm install && npm run build (produces static export) + - npm ci && npm run build (produces static export in out/) Stage 2: Python 3.12 slim - Install uv - Copy backend/ - - uv sync (install Python dependencies from lockfile) - - Copy frontend build output into a static/ directory + - uv sync --frozen (install Python dependencies from lockfile) + - Copy frontend build output into static/ - Expose port 8000 - CMD: uvicorn serving FastAPI app ``` -FastAPI serves the static frontend files and all API routes on port 8000. +### Route Mounting Order -### Docker Volume +FastAPI serves both the API and the static frontend on port 8000, so mounting order matters: -The SQLite database persists via a named Docker volume: +1. Mount all `/api/*` routers **first** +2. Mount static assets +3. Register a catch-all last that returns `index.html` for any unmatched path, so client-side routing and hard refreshes work + +A `StaticFiles(html=True)` mount at `/` registered before the API routers will shadow every endpoint. This is the single most common way to break this architecture. + +### Persistence + +The SQLite database persists via a bind mount, so the file is visible and deletable on the host: ```bash -docker run -v finally-data:/app/db -p 8000:8000 --env-file .env finally +docker run -v "$(pwd)/db:/app/db" -p 8000:8000 --env-file .env finally ``` -The `db/` directory in the project root maps to `/app/db` in the container. The backend writes `finally.db` to this path. +The `db/` directory in the project root maps to `/app/db` in the container; the backend writes `finally.db` there. On Windows the PowerShell scripts use `${PWD}` and quote the path, since spaces in the path are common. ### Start/Stop Scripts **`scripts/start_mac.sh`** (macOS/Linux): -- Builds the Docker image if not already built (or if `--build` flag passed) -- Runs the container with the volume mount, port mapping, and `.env` file +- Builds the Docker image if not already built (or if `--build` is passed) +- Runs the container with the bind mount, port mapping, and `.env` file - Prints the URL to access the app - Optionally opens the browser **`scripts/stop_mac.sh`** (macOS/Linux): - Stops and removes the running container -- Does NOT remove the volume (data persists) +- Does NOT touch `db/` (data persists) -**`scripts/start_windows.ps1`** / **`scripts/stop_windows.ps1`**: PowerShell equivalents for Windows. +**`scripts/start_windows.ps1`** / **`scripts/stop_windows.ps1`**: PowerShell equivalents. -All scripts should be idempotent — safe to run multiple times. +All scripts are idempotent — safe to run multiple times. They are the only supported launch path; there is no `docker-compose.yml` to keep in sync. ### Optional Cloud Deployment -The container is designed to deploy to AWS App Runner, Render, or any container platform. A Terraform configuration for App Runner may be provided in a `deploy/` directory as a stretch goal, but is not part of the core build. +The container can deploy to AWS App Runner, Render, or any container platform, and a Terraform configuration may be provided in `deploy/` as a stretch goal. + +**If you deploy it, put it behind authentication.** The app has no login by design, and `POST /api/chat` spends the deployer's OpenRouter credits and auto-executes trades on every call. Publicly reachable, that is an open API-key proxy. Use basic auth at the platform edge, an IP allowlist, or don't deploy. Locally this is a non-issue — the warning exists only because the plan invites deployment. --- @@ -428,29 +672,90 @@ The container is designed to deploy to AWS App Runner, Render, or any container ### Unit Tests (within `frontend/` and `backend/`) **Backend (pytest)**: -- Market data: simulator generates valid prices, GBM math is correct, Massive API response parsing works, both implementations conform to the abstract interface -- Portfolio: trade execution logic, P&L calculations, edge cases (selling more than owned, buying with insufficient cash, selling at a loss) -- LLM: structured output parsing handles all valid schemas, graceful handling of malformed responses, trade validation within chat flow -- API routes: correct status codes, response shapes, error handling +- Market data: complete — 73 tests, 84% coverage (see `MARKET_DATA_SUMMARY.md`). Extend with tests for the rolling price history and the SSE keepalive. +- Portfolio: trade execution, the §7 formulas, and every row of the §8 validation table — plus fractional-residue cleanup (sell-all leaves no row) and cash rounding +- Ticker tracking: removing a watchlist ticker with an open position keeps it in the feed; selling to zero off-watchlist removes it +- Snapshots: no snapshot written while a held ticker has no price; none written on trade execution +- LLM: structured output parsing, the retry-then-fallback path on malformed responses, best-effort partial execution, and the mock-mode table in §9 +- API routes: status codes, the exact response shapes in §8, and the uniform error envelope **Frontend (React Testing Library or similar)**: - Component rendering with mock data - Price flash animation triggers correctly on price changes +- SSE payload handling — the map-shaped event, the float timestamp, and the already-percent `change_percent` - Watchlist CRUD operations -- Portfolio display calculations -- Chat message rendering and loading state +- Chat message rendering, loading state, and executed/rejected action chips ### E2E Tests (in `test/`) -**Infrastructure**: A separate `docker-compose.test.yml` in `test/` that spins up the app container plus a Playwright container. This keeps browser dependencies out of the production image. +**Infrastructure**: Playwright runs **on the host** against the container started by `scripts/start_mac.sh` (`npx playwright test`, `baseURL: http://localhost:8000`). This tests exactly the artifact users run, with no orchestration to build or debug. A containerized runner can be added later for CI, but it is not the documented path. -**Environment**: Tests run with `LLM_MOCK=true` by default for speed and determinism. +**Environment**: the container runs with `LLM_MOCK=true`, making chat responses deterministic per the §9 table. **Key Scenarios**: - Fresh start: default watchlist appears, $10k balance shown, prices are streaming - Add and remove a ticker from the watchlist -- Buy shares: cash decreases, position appears, portfolio updates -- Sell shares: cash increases, position updates or disappears +- Buy shares: cash decreases, position appears, portfolio and blotter update +- Sell shares: cash increases, position updates or disappears entirely +- Rejected trade: buying beyond available cash shows the inline error and changes nothing - Portfolio visualization: heatmap renders with correct colors, P&L chart has data points -- AI chat (mocked): send a message, receive a response, trade execution appears inline -- SSE resilience: disconnect and verify reconnection +- AI chat (mocked): send `buy`, see the response and an executed-trade chip; send `sell` with no position, see a rejected chip with the error +- Chat persistence: reload the page, history is still there +- Reset: returns to $10k with the default watchlist and an empty positions table +- SSE resilience: disconnect and verify reconnection and the connection indicator + +--- + +## 13. Decisions Log + +A documentation review on 2026-08-22 raised 30 questions and gaps. Their resolutions are now written into the sections above rather than listed here; this log records what was decided, where it landed, and what remains open. + +### Resolved and incorporated + +| Issue | Decision | Where | +|---|---|---| +| Named volume vs. bind mount contradiction | Bind mount `./db:/app/db` | §3, §11 | +| Held ticker removed from watchlist loses its price | Tracked set = `watchlist ∪ positions` | §6 | +| Arbitrary ticker input undefined | Normalize + regex at the API boundary; no allowlist | §6 | +| Chat history written but unreadable | `GET /api/chat`, `DELETE /api/chat` | §8 | +| "LLM informs the user of the failure" was circular | Deterministic backend note + frontend chips, no second call | §9 | +| Multi-trade partial failure semantics | Best-effort and ordered, every action reports status | §9 | +| No price data on page load | `GET /api/portfolio` computes prices and P&L server-side | §8 | +| Valuation with a missing price | Fall back to `avg_cost`; skip the snapshot entirely | §7 | +| P&L formulas would drift across four call sites | Written once, canonically | §7 | +| Fractional-share residue | Delete the position below `1e-9` | §7 | +| Trade validation edges | Full table with user-facing messages | §8 | +| Timestamp representation | ISO UTC in the DB, Unix float in the price layer | §7 | +| `LLM_MOCK` behavior undefined but asserted on | Keyword table, part of the contract | §9 | +| Massive polling looks like a dead connection | `: ping` keepalive every 15s (**TODO**) | §6 | +| Unbounded prompt growth | Last 20 messages, 2000-char cap, one request in flight | §9 | +| Malformed LLM output | Retry once, then a canned 200; 503 only if the provider is down | §9 | +| Main chart had no data source | 600-point rolling history + `GET /api/prices/{ticker}/history` (**TODO**) | §6, §8 | +| Two candidate chart libraries | Recharts for all of it | §3, §10 | +| No way to reset the demo | `POST /api/reset` | §7, §8 | +| `trades` table never surfaced | `GET /api/trades` + a blotter | §8, §10 | +| SQLite blocking the event loop | WAL, short-lived connections, threadpool dispatch | §7 | +| Unbounded snapshot history query | `limit` / `since` parameters | §8 | +| Snapshot written on every trade | Dropped — a trade does not change total value | §7 | +| Unauthenticated LLM endpoint if deployed | Explicit warning on the deployment path | §11 | +| `backend/db/` collides with root `db/` | Renamed to `backend/app/db/` | §4 | +| Playwright container was heavy infrastructure | Host-run Playwright is the documented path | §12 | +| Redundant `docker-compose.yml` | Removed; the scripts are the only launch path | §4, §11 | +| Realized P&L absent — oversight or intent? | Intentional, and now stated so no agent adds it | §7 | +| Ticker casing, cash rounding, absent FKs | Stated explicitly | §6, §7 | +| Two different `.env` mechanisms conflated | Both documented and distinguished | §5 | +| LLM schema fields "optional" | All three required, empty arrays for none | §9 | +| No request/response shapes anywhere | §8 rewritten as a full contract | §8 | + +### Corrected during the review + +Two findings were wrong once checked against the shipped code, and the plan now documents what actually exists: + +- **SSE already sends a full snapshot on connect.** `_generate_events` starts at `last_version = -1`, so the first iteration always pushes the entire cache — including after a reconnect. No snapshot endpoint or event type is needed. The payload is also a **map keyed by ticker in one event**, not one event per ticker, and it was worth freezing that in §6 before the frontend agent assumed otherwise. +- **Unknown tickers already work in the simulator.** `DEFAULT_PARAMS`, `CROSS_GROUP_CORR`, and a `random.uniform(50, 300)` seed price handle any symbol, and `add_ticker` writes to the cache immediately. Only API-level validation and the Massive-side behavior were missing. + +### Still open + +1. **Verify structured outputs on `gpt-oss-120b` via OpenRouter/Cerebras** before building the chat path (§9). The fallback is specified; the question is whether it is needed. +2. **Next.js is retained.** Static export disables essentially everything Next adds over a plain SPA, and Vite + React would be simpler and build faster in Docker — but this is a course capstone and the framework choice may be curricular. The concrete problems it caused are now fixed in place (§10 export config, §11 mount ordering). Worth a deliberate decision by the plan owner rather than a silent swap. +3. **Three TODOs** are new work introduced by this review, all backend: the SSE keepalive, the rolling price history plus its endpoint, and `POST /api/reset`. diff --git a/planning/REVIEW.md b/planning/REVIEW.md new file mode 100644 index 000000000..65eeeee4c --- /dev/null +++ b/planning/REVIEW.md @@ -0,0 +1,73 @@ +# Review: changes since last commit + +## Findings + +### High - First-launch setup is still not coherent + +`planning/PLAN.md:15-20` promises a single Docker command/start script and an AI chat panel that is ready immediately. The environment contract then marks `OPENROUTER_API_KEY` as required (`planning/PLAN.md:128-137`), the README tells users to add that key before running (`README.md:29-35`), and `planning/PLAN.md:487` states that the key exists in the project-root `.env`. At the same time, the plan says `LLM_MOCK=true` works without an API key (`planning/PLAN.md:557-568`) and E2E runs in mock mode (`planning/PLAN.md:691-693`). + +That leaves first launch ambiguous: either the app can boot and show a usable mocked/degraded chat without OpenRouter credentials, or the "ready to assist" experience requires a secret the user must supply. Make `OPENROUTER_API_KEY` required only when `LLM_MOCK != true`, commit/document a real `.env.example`, and define the default first-launch mode. + +### High - Persisted portfolios need startup market-source rehydration + +The plan correctly defines the tracked ticker set as `watchlist union non-zero positions` (`planning/PLAN.md:183-194`) and says the SQLite database persists across container restarts (`planning/PLAN.md:638-644`). The explicit add/remove rules only cover watchlist/trade requests, and reset performs a re-sync (`planning/PLAN.md:345`), but there is no startup/lifespan rule that loads persisted watchlist and position tickers from SQLite into the market data source. + +After a restart with existing holdings, the database can contain positions while the market source only tracks defaults. That would leave off-watchlist holdings without live prices, cause `GET /api/portfolio` to fall back to `avg_cost`, and make snapshots stall under the "skip if any held ticker has no cached price" rule. Add a startup reconciliation step: after DB initialization and before serving streams, load `watchlist union positions(quantity > 0)` and call `source.add_ticker()` for each. + +### High - Backend writes need serialization, not just frontend throttling + +The plan defines cash/position/trade mutations (`planning/PLAN.md:404-427`), LLM auto-execution (`planning/PLAN.md:522-529`), reset (`planning/PLAN.md:345`), and short-lived SQLite connections dispatched off the event loop (`planning/PLAN.md:247-253`). It only bounds chat concurrency in the frontend (`planning/PLAN.md:504`), which does not protect manual trades, multiple browser tabs, direct API calls, or reset racing with trade/chat actions. + +Require a backend transaction boundary around each trade, such as `BEGIN IMMEDIATE`, so the cash check, cash update, position update/delete, and trade insert succeed or fail together. Also route manual trades, LLM trades, watchlist changes that affect the market source, reset, and snapshot writes through one per-user write lock or equivalent service path so they cannot observe or create half-applied state. + +### Medium - The new review agent delegates to a nested Codex process + +`.claude/agents/change-reviewer.md:6-11` tells the subagent not to review changes itself and instead to run `codex exec "Please review all changes since the last commit and write your feedback to planning/REVIEW.md"`. That makes the Claude agent a wrapper around another autonomous writer of the same file, with no guard against overwriting existing feedback, no status propagation, and no fallback if `codex` is unavailable in the caller's shell. + +If this agent is meant to provide independent review, have it perform the review directly from the current worktree and write findings. If the intent is specifically "invoke Codex", make that explicit in the command name/description and add failure handling so users do not get a silent no-op or a clobbered review file. + +### Medium - Malformed LLM fallback violates the chat response contract + +`POST /api/chat` normally returns `id`, `message`, `actions`, and `created_at` (`planning/PLAN.md:457-472`). The malformed-response fallback returns only `message` and `actions` (`planning/PLAN.md:538-544`), and the plan does not say whether the user/assistant messages are stored when fallback is used. + +Make the fallback return the exact same envelope as the success path, including `id` and `created_at`, with empty action arrays. Also specify persistence behavior: either store both messages so refresh matches what the user saw, or explicitly do not store fallback responses and adjust frontend/test expectations. + +### Medium - AI watchlist removal is promised but not specified + +The UX promises watchlist add/remove manually or via AI chat (`planning/PLAN.md:31-32`). Manual add/remove endpoints are specified (`planning/PLAN.md:451-453`), but the LLM schema only shows `{"action": "add"}` and never defines allowed watchlist action values (`planning/PLAN.md:506-520`). The mock mode also only adds PYPL (`planning/PLAN.md:561-566`). + +Define `watchlist_changes[].action` as `"add" | "remove"` and specify result behavior for duplicate adds, removing a missing ticker, and removing a ticker that is still held. If AI removal is not intended for v1, remove that promise from the UX section. + +### Medium - P&L history can be empty on first launch and after reset + +The P&L chart reads from `GET /api/portfolio/history` (`planning/PLAN.md:581`), snapshots are written every 30 seconds (`planning/PLAN.md:310-315`), and reset truncates `portfolio_snapshots` (`planning/PLAN.md:345`). The E2E list expects the P&L chart to have data points (`planning/PLAN.md:701`), but after a fresh DB or reset there may be no point until the first snapshot tick. + +Either seed/write an initial baseline snapshot during DB initialization and after reset, or specify a frontend empty state and update E2E expectations to wait for the first snapshot. For a demo app, a baseline point is simpler and makes the chart feel intentionally populated. + +### Medium - Quantity precision is stated but not enforceable as written + +The schema stores quantities as SQLite `REAL` (`planning/PLAN.md:278-294`), REST says quantities have up to 6 decimals (`planning/PLAN.md:355-356`), and validation only rejects non-positive/NaN/infinite values (`planning/PLAN.md:416-425`). There is no rule for `0.0000004`, `1.1234567`, or how rounding should affect cash and average cost. + +Add an API-boundary rule for quantity scale: reject more than 6 decimal places with a specific `400` message, or round to 6 decimals and document it. If exact behavior matters, use `Decimal` in service code or store integer micro-shares/cents while still returning JSON numbers. + +### Low - The README bind-mount command is POSIX-only without saying so + +The README changed the quick-start command to `docker run -v "$(pwd)/db:/app/db" ...` (`README.md:33-35`). That matches the plan's macOS/Linux path, but the README presents it as the only quick start. The plan separately says Windows scripts need `${PWD}` quoting because spaces are common (`planning/PLAN.md:644`), and those scripts are also listed as the supported launch path (`planning/PLAN.md:646-660`). + +Either label the README command as macOS/Linux and add the Windows command/script path, or make README quick start defer to the start scripts once they exist. The previous named-volume command was platform-neutral; the bind mount needs platform-specific documentation. + +### Low - The uniform error envelope needs framework-validation coverage + +The API convention says every non-2xx response is `{"error": "human readable message"}` and FastAPI's default `422` body is replaced (`planning/PLAN.md:353-356`). The plan does not define status/message behavior for malformed JSON, missing required fields, wrong types, invalid query parameters, or path parameter validation. + +Add a short table for framework-level validation errors and require a `RequestValidationError`/JSON decode handler that maps those cases into the same envelope. This prevents frontend agents from special-casing FastAPI's default validation response. + +### Low - The new doc-review command is too loose to be reliable + +`.claude/commands/doc-review.md:1` contains typos and says to review a planning file named `$ARGUMENTS`, then append "questions, clarifications or feedback" to a new section. It does not say what to do when the file is missing, whether `$ARGUMENTS` must be a basename under `planning/`, or what heading format should be used. + +Tighten the command so it validates the target file under `planning/`, fails clearly when the argument is missing, and appends under a stable heading. That will make repeated doc reviews less likely to scatter duplicate sections. + +## Notes + +I did not flag `db/.gitkeep`; it matches the new bind-mount contract. The `.gitignore` additions for `db/*.db`, `db/*.db-wal`, and `db/*.db-shm` also match the planned SQLite/WAL runtime files.