From f830039adb880bf53429b7078ef36d0ed34434f4 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Thu, 26 Feb 2026 16:07:25 +0000
Subject: [PATCH 1/3] Initial plan
From 2146575d5f42dc8b8473d01295933a03dfe1e98f Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Thu, 26 Feb 2026 16:21:21 +0000
Subject: [PATCH 2/3] Add Trip Playback mode, remove Historic View, add
trip_data.py for CitiBike data download
Co-authored-by: Vaibhav-Hariani <62775035+Vaibhav-Hariani@users.noreply.github.com>
---
.gitignore | 3 +-
frontend/app.py | 453 +++++++++++++++++++++++++++---------------
frontend/trip_data.py | 149 ++++++++++++++
3 files changed, 439 insertions(+), 166 deletions(-)
create mode 100644 frontend/trip_data.py
diff --git a/.gitignore b/.gitignore
index 5e1b1d1..ff74613 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
__pycache__
api_keys.py
-.vscode
\ No newline at end of file
+.vscode
+processed_data/trips/
diff --git a/frontend/app.py b/frontend/app.py
index 73673bf..c3e87ad 100644
--- a/frontend/app.py
+++ b/frontend/app.py
@@ -1,4 +1,6 @@
+import json
import streamlit as st
+import streamlit.components.v1 as components
import numpy as np
import folium
from streamlit_folium import st_folium
@@ -8,6 +10,7 @@
from postgres_manager import DBManager
from globals import *
from api_keys import *
+from trip_data import load_trips
def haversine_distance(lat1, lon1, lat2, lon2):
@@ -31,17 +34,6 @@ def get_db_manager():
return manager
-@st.cache_data(show_spinner="Fetching historic timestamps from PostgreSQL...")
-def fetch_timestamps():
- db_manager = get_db_manager()
- return db_manager.get_timestamps()
-
-
-@st.cache_data(show_spinner="Fetching historic snapshot from PostgreSQL...")
-def fetch_artifact(timestamp):
- db_manager = get_db_manager()
- return db_manager.get_artifact(timestamp)
-
@st.cache_resource
def get_gmaps_client():
@@ -412,52 +404,192 @@ def general_view_render(map, station_list, gbfs_status):
return stations_added
-def add_historic_view_stations(m, station_list, historic_data):
- """Add stations to map with historic data"""
- if not station_list or not historic_data:
- return 0
+# ---------------------------------------------------------------------------
+# Trip Playback helpers
+# ---------------------------------------------------------------------------
- # Create a lookup dict for historic data by station_id
- historic_dict = {item["station_id"]: item for item in historic_data}
+@st.cache_data(show_spinner="Loading CitiBike trip data (this may take a moment)…")
+def load_playback_trips():
+ """Load and return a sample of CitiBike trip data from 2024 to present."""
+ return load_trips(sample_per_month=5000)
- stations_added = 0
- for station in station_list:
- station_id = str(station["station_id"])
- if station_id in historic_dict:
- historic = historic_dict[station_id]
- bikes = historic["bikes_available"]
- ebikes = historic["ebikes_available"]
- docks = historic["docks_available"]
- regular_bikes = get_regular_bikes_count(bikes, ebikes)
- # Get color based on availability
- color = get_color_for_availability(bikes, ebikes, docks)
+@st.cache_data(show_spinner=False)
+def get_trip_route(start_lat: float, start_lng: float, end_lat: float, end_lng: float):
+ """Return a list of {lat, lng} dicts for the Google Maps bicycling route."""
+ client = get_gmaps_client()
+ try:
+ result = client.directions(
+ f"{start_lat},{start_lng}",
+ f"{end_lat},{end_lng}",
+ mode="bicycling",
+ alternatives=False,
+ )
+ if result:
+ encoded = result[0]["overview_polyline"]["points"]
+ decoded = polyline.decode(encoded)
+ return [{"lat": lat, "lng": lng} for lat, lng in decoded]
+ except Exception as exc:
+ logger.warning("Failed to get Google Maps route (%s, %s) -> (%s, %s): %s",
+ start_lat, start_lng, end_lat, end_lng, exc)
+ return [
+ {"lat": start_lat, "lng": start_lng},
+ {"lat": end_lat, "lng": end_lng},
+ ]
+
+
+@st.cache_data(show_spinner="Computing trip routes via Google Maps…")
+def compute_trip_routes(trips_df):
+ """
+ Compute Google Maps bicycling routes for every row in trips_df.
+ Returns a list of dicts: {path: [{lat, lng}…], duration: float (seconds)}.
+ Routes are cached so repeated calls for the same (start, end) are free.
+ """
+ result = []
+ for _, row in trips_df.iterrows():
+ try:
+ path = get_trip_route(
+ # Round to 4 d.p. (~11 m) so nearby trips share cached routes
+ round(float(row["start_lat"]), 4),
+ round(float(row["start_lng"]), 4),
+ round(float(row["end_lat"]), 4),
+ round(float(row["end_lng"]), 4),
+ )
+ result.append({"path": path, "duration": float(row["duration_seconds"])})
+ except Exception:
+ continue
+ return result
- # Create popup with station info
- popup_html = f"""
-
- {station.get('name', 'Station ' + station_id)}
-
- 🚲 Regular Bikes: {regular_bikes}
- ⚡ E-Bikes: {ebikes}
- 🅿️ Docks: {docks}
-
- """
- folium.CircleMarker(
- location=[station["latitude"], station["longitude"]],
- radius=6,
- color=color,
- fill=True,
- fill_color=color,
- fill_opacity=0.7,
- opacity=0.9,
- weight=2,
- popup=folium.Popup(popup_html, max_width=250),
- ).add_to(m)
- stations_added += 1
+def render_trip_playback(trip_routes: list, playback_speed: int, gmaps_key: str, height: int = 700) -> None:
+ """
+ Render animated CitiBike trip playback using the Google Maps JavaScript API.
+
+ Always maintains 10 active trips; when a trip ends a replacement begins
+ within a random 0-10 simulated-second window. The route is visualised with
+ a spotlight effect: a bright window ±15% around the current position, with
+ the far-past fading out and the far-future shown faintly.
+ """
+ trips_json = json.dumps(trip_routes)
+
+ html = f"""
+
+Initialising…
+
+
+"""
+
+ components.html(html, height=height + 10, scrolling=False)
- return stations_added
def init_session_states():
# Initialize session state
@@ -479,13 +611,15 @@ def init_session_states():
st.session_state["app_mode"] = "Route Finder"
if "route_written" not in st.session_state:
st.session_state["route_written"] = False
+ if "playback_active" not in st.session_state:
+ st.session_state["playback_active"] = False
def main():
# Mode selection
st.sidebar.title("Mode Selection")
app_mode = st.sidebar.radio(
"Choose Mode:",
- ["Route Finder", "General View", "Historic View"],
+ ["Route Finder", "General View", "Trip Playback"],
)
st.sidebar.divider()
@@ -575,122 +709,111 @@ def main():
st.rerun()
- elif app_mode == "Historic View":
- st.sidebar.subheader("Historic Station View")
- db_manager = get_db_manager()
- if db_manager:
- # Use explicit timestamp list for precise historic selection (cached)
- timestamps = fetch_timestamps()
- if not timestamps:
- st.error("No historic data available")
- st.stop()
- min_time = timestamps[0]
- max_time = timestamps[-1]
- st.sidebar.write("📅 Data Range:")
- st.sidebar.write(f"From: {min_time.strftime('%Y-%m-%d %H:%M')}")
- st.sidebar.write(f"To: {max_time.strftime('%Y-%m-%d %H:%M')}")
- if "historic_timestamp" not in st.session_state:
- st.session_state["historic_timestamp"] = min_time
- # Allow selecting an exact available timestamp (discrete options)
- selected_datetime = st.sidebar.select_slider(
- "Select Time:",
- options=timestamps,
- value=st.session_state.get("historic_timestamp", min_time),
- key="historic_time_slider",
- )
- speed = st.sidebar.slider(
- "Playback Speed (Seconds/Step)",
- 0.1,
- 60.0,
- 10.0,
- step=0.1,
- key="historic_speed_slider",
- )
- st.session_state["historic_timestamp"] = selected_datetime
- st.sidebar.info(
- "Stations are color-coded:\n\n🟢 Green = >10% of bikes are e-bikes\n\n🔵 Blue = Regular bikes available\n\n🔴 Red = No bikes available\n\n⚫ Grey = Out of service"
- )
- # Update metadata table for historic view
- db_manager.update_metadata(
- in_type=HISTORIC, viewing_timestamp=selected_datetime, speed=speed
- )
- else:
- st.sidebar.error("Could not connect to database")
-
- # Map rendering section - common for all modes
- db_manager = get_db_manager()
-
- # Create base map centered on Manhattan
- m = folium.Map(
- location=[40.7589, -73.9851], # Manhattan center
- zoom_start=13,
- tiles="CartoDB dark_matter",
- )
-
- # Render map based on selected mode
- if app_mode == "Route Finder":
- if st.session_state["run"] and o_c and d_c:
- # Optimized call to find routes and stations (2 + num_routes DB calls)
- result = find_route_stations(o_c, d_c, bike_type_value, station_threshold, num_routes)
-
- if result[0] is not None:
- route_dict, paths, stations_per_route, start_station, end_station, features = result
-
- # Write to database only if not already written
- if route_dict and not st.session_state.get("route_written", False):
- # Convert dict to list and write atomically
- route_list = [
- {"station_id": sid, "color": color}
- for sid, color in route_dict.items()
- ]
- db_manager.clear_route()
- db_manager.set_route_stations(route_list)
- st.session_state["route_written"] = True
-
- # Render routes on map
- render_routes(
- m, paths, o_c, d_c, stations_per_route,
- st.session_state.get("selected_route"), bike_type_value,
- start_station, end_station
- )
-
- # Show route info
- st.subheader("Route Information")
- for i, feature in enumerate(features, 1):
- with st.expander(f"Route {i}", expanded=(i == 1)):
- distance_km = feature["properties"]["distance"] / 1000
- duration_min = feature["properties"]["duration"] / 60
- st.write(f"Distance: {distance_km:.2f} km")
- st.write(f"Duration: {duration_min:.0f} minutes")
+ elif app_mode == "Trip Playback":
+ st.sidebar.subheader("Trip Playback")
+ playback_speed = st.sidebar.slider(
+ "Playback Speed (× real time)",
+ min_value=1,
+ max_value=200,
+ value=30,
+ step=1,
+ key="playback_speed_slider",
+ )
+ n_trips = st.sidebar.slider(
+ "Trips to preload",
+ min_value=20,
+ max_value=200,
+ value=100,
+ step=10,
+ key="n_trips_slider",
+ )
+ st.sidebar.info(
+ "Animates real CitiBike trips from 2024 using Google Maps routing.\n\n"
+ "10 trips run simultaneously; when one ends a new one begins."
+ )
+ if st.sidebar.button(
+ "▶ Start Playback", type="primary", use_container_width=True, key="start_playback"
+ ):
+ st.session_state["playback_active"] = True
+ if st.session_state.get("playback_active"):
+ if st.sidebar.button(
+ "⏹ Stop Playback", use_container_width=True, key="stop_playback"
+ ):
+ st.session_state["playback_active"] = False
+
+ # Map rendering section
+ if app_mode == "Trip Playback":
+ # Render Google Maps JS animation (no Folium map)
+ if st.session_state.get("playback_active"):
+ trips_df = load_playback_trips()
+ if trips_df is not None and len(trips_df) > 0:
+ routes = compute_trip_routes(trips_df.head(n_trips))
+ if routes:
+ render_trip_playback(routes, playback_speed, GOOGLE_MAPS)
+ else:
+ st.error("Could not compute routes. Check your Google Maps API key.")
else:
- st.error("Could not find a valid route. Please try different locations.")
- elif app_mode == "General View":
- # General View mode - show all stations with live data
- station_list = db_manager.get_all_stations()
- gbfs_status = get_station_status()
- if station_list and gbfs_status:
- stations_added = general_view_render(m, station_list, gbfs_status)
- st.success(f"Displaying {stations_added} stations with live availability data")
+ st.error(
+ "No trip data available. Check internet connectivity — "
+ "CitiBike trip files are downloaded from s3.amazonaws.com."
+ )
else:
- st.error("Could not load station data")
-
- elif app_mode == "Historic View":
- # Historic View mode - show stations with historic data
- station_list = db_manager.get_all_stations()
- if station_list and "historic_timestamp" in st.session_state:
- selected_timestamp = st.session_state["historic_timestamp"]
- # Fetch historic snapshot for the selected timestamp (cached per-timestamp)
- historic_data = fetch_artifact(selected_timestamp)
- if historic_data:
- stations_added = add_historic_view_stations(m, station_list, historic_data)
- st.success(f"Displaying {stations_added} stations at {selected_timestamp.strftime('%Y-%m-%d %H:%M')}")
+ st.info(
+ "Click **▶ Start Playback** in the sidebar to begin the animated "
+ "CitiBike trip overlay."
+ )
+ else:
+ db_manager = get_db_manager()
+
+ # Create base map centered on Manhattan
+ m = folium.Map(
+ location=[40.7589, -73.9851],
+ zoom_start=13,
+ tiles="CartoDB dark_matter",
+ )
+
+ if app_mode == "Route Finder":
+ if st.session_state["run"] and o_c and d_c:
+ result = find_route_stations(o_c, d_c, bike_type_value, station_threshold, num_routes)
+
+ if result[0] is not None:
+ route_dict, paths, stations_per_route, start_station, end_station, features = result
+
+ if route_dict and not st.session_state.get("route_written", False):
+ route_list = [
+ {"station_id": sid, "color": color}
+ for sid, color in route_dict.items()
+ ]
+ db_manager.clear_route()
+ db_manager.set_route_stations(route_list)
+ st.session_state["route_written"] = True
+
+ render_routes(
+ m, paths, o_c, d_c, stations_per_route,
+ st.session_state.get("selected_route"), bike_type_value,
+ start_station, end_station
+ )
+
+ st.subheader("Route Information")
+ for i, feature in enumerate(features, 1):
+ with st.expander(f"Route {i}", expanded=(i == 1)):
+ distance_km = feature["properties"]["distance"] / 1000
+ duration_min = feature["properties"]["duration"] / 60
+ st.write(f"Distance: {distance_km:.2f} km")
+ st.write(f"Duration: {duration_min:.0f} minutes")
+ else:
+ st.error("Could not find a valid route. Please try different locations.")
+
+ elif app_mode == "General View":
+ station_list = db_manager.get_all_stations()
+ gbfs_status = get_station_status()
+ if station_list and gbfs_status:
+ stations_added = general_view_render(m, station_list, gbfs_status)
+ st.success(f"Displaying {stations_added} stations with live availability data")
else:
- st.warning("No historic data available for selected time")
- else:
- st.error("Could not load station data")
-
- # Display the map
- st_folium(m, use_container_width=True, height=1200, returned_objects=[])
+ st.error("Could not load station data")
+
+ st_folium(m, use_container_width=True, height=1200, returned_objects=[])
if __name__ == "__main__":
init_session_states()
diff --git a/frontend/trip_data.py b/frontend/trip_data.py
new file mode 100644
index 0000000..52c9764
--- /dev/null
+++ b/frontend/trip_data.py
@@ -0,0 +1,149 @@
+"""
+CitiBike historic trip data downloader and manager.
+Dynamically downloads trip data from 2024 to present from CitiBike's S3 bucket.
+"""
+
+import io
+import logging
+import os
+import zipfile
+from datetime import date
+
+import pandas as pd
+import requests
+
+TRIPDATA_BASE_URL = "https://s3.amazonaws.com/tripdata"
+_HERE = os.path.dirname(os.path.abspath(__file__))
+TRIP_CACHE_DIR = os.path.join(_HERE, "..", "processed_data", "trips")
+
+logger = logging.getLogger(__name__)
+
+TRIP_COLUMNS = [
+ "started_at",
+ "ended_at",
+ "start_station_id",
+ "end_station_id",
+ "start_lat",
+ "start_lng",
+ "end_lat",
+ "end_lng",
+]
+
+
+def get_available_months():
+ """Return (year, month) tuples from January 2024 through today."""
+ result, year, month = [], 2024, 1
+ today = date.today()
+ while (year, month) <= (today.year, today.month):
+ result.append((year, month))
+ month += 1
+ if month > 12:
+ month = 1
+ year += 1
+ return result
+
+
+def _zip_url(year: int, month: int) -> str:
+ return f"{TRIPDATA_BASE_URL}/{year}{month:02d}-citibike-tripdata.csv.zip"
+
+
+def _cache_path(year: int, month: int) -> str:
+ os.makedirs(TRIP_CACHE_DIR, exist_ok=True)
+ return os.path.join(TRIP_CACHE_DIR, f"{year}{month:02d}-citibike-tripdata.csv.gz")
+
+
+def download_month(year: int, month: int):
+ """
+ Download and cache trip data for the given month.
+ Returns the local cache path on success, or None if unavailable.
+ Skips download if a cache file already exists.
+ """
+ path = _cache_path(year, month)
+ if os.path.exists(path):
+ return path
+
+ url = _zip_url(year, month)
+ logger.info("Downloading %s", url)
+ try:
+ resp = requests.get(url, timeout=120)
+ if resp.status_code == 404:
+ logger.debug("No data for %d-%02d", year, month)
+ return None
+ resp.raise_for_status()
+
+ with zipfile.ZipFile(io.BytesIO(resp.content)) as zf:
+ csv_files = [n for n in zf.namelist() if n.lower().endswith(".csv")]
+ if not csv_files:
+ return None
+ with zf.open(csv_files[0]) as fh:
+ df = pd.read_csv(
+ fh,
+ usecols=lambda c: c in TRIP_COLUMNS,
+ dtype={"start_station_id": str, "end_station_id": str},
+ )
+
+ df = df.dropna(
+ subset=[
+ "start_station_id",
+ "end_station_id",
+ "started_at",
+ "ended_at",
+ "start_lat",
+ "start_lng",
+ "end_lat",
+ "end_lng",
+ ]
+ )
+ df = df[df["start_station_id"] != df["end_station_id"]]
+ df.to_csv(path, index=False, compression="gzip")
+ logger.info("Saved %d trips for %d-%02d", len(df), year, month)
+ return path
+ except Exception as exc:
+ if hasattr(exc, 'response') and exc.response is not None and exc.response.status_code != 404:
+ logger.error("HTTP %s downloading %d-%02d: %s", exc.response.status_code, year, month, exc)
+ else:
+ logger.error("Failed to download %d-%02d: %s", year, month, exc)
+ return None
+
+
+def load_trips(sample_per_month: int = 5000) -> pd.DataFrame:
+ """
+ Load (downloading if necessary) CitiBike trip data from 2024 to present.
+ Returns a DataFrame sorted by started_at with a duration_seconds column.
+ Returns an empty DataFrame if no data is available.
+ """
+ frames = []
+ for year, month in get_available_months():
+ path = _cache_path(year, month)
+ if not os.path.exists(path):
+ path = download_month(year, month)
+ if not path or not os.path.exists(path):
+ continue
+ try:
+ df = pd.read_csv(
+ path,
+ compression="gzip",
+ dtype={"start_station_id": str, "end_station_id": str},
+ )
+ if len(df) > sample_per_month:
+ df = df.sample(n=sample_per_month, random_state=42)
+ frames.append(df)
+ except Exception as exc:
+ logger.error("Failed to read %s: %s", path, exc)
+
+ if not frames:
+ return pd.DataFrame()
+
+ combined = pd.concat(frames, ignore_index=True)
+ combined["started_at"] = pd.to_datetime(combined["started_at"], errors="coerce")
+ combined["ended_at"] = pd.to_datetime(combined["ended_at"], errors="coerce")
+ combined = combined.dropna(subset=["started_at", "ended_at"])
+ combined["duration_seconds"] = (
+ (combined["ended_at"] - combined["started_at"]).dt.total_seconds()
+ )
+ # Keep sane durations: 1 minute to 3 hours
+ combined = combined[
+ (combined["duration_seconds"] >= 60) & (combined["duration_seconds"] <= 10800)
+ ]
+ combined = combined.sort_values("started_at").reset_index(drop=True)
+ return combined
From 9a5aea53247671f52683a21aca6be9d57f03d5e9 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Thu, 26 Feb 2026 16:22:38 +0000
Subject: [PATCH 3/3] Add missing logging import and logger instance to app.py
Co-authored-by: Vaibhav-Hariani <62775035+Vaibhav-Hariani@users.noreply.github.com>
---
frontend/app.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/frontend/app.py b/frontend/app.py
index c3e87ad..1933679 100644
--- a/frontend/app.py
+++ b/frontend/app.py
@@ -1,4 +1,5 @@
import json
+import logging
import streamlit as st
import streamlit.components.v1 as components
import numpy as np
@@ -12,6 +13,8 @@
from api_keys import *
from trip_data import load_trips
+logger = logging.getLogger(__name__)
+
def haversine_distance(lat1, lon1, lat2, lon2):
R = 6371000