Skip to content

Commit bca4062

Browse files
stephen-derosastephen-derosa
committed
Throughput tooling for csv generation and visualization
1 parent 1bc08ab commit bca4062

11 files changed

Lines changed: 2337 additions & 3 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,3 +34,4 @@ livekit.log
3434
web/
3535
*trace.json
3636
compile_commands.json
37+
**data_track_throughput_results/

‎README.md‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -135,14 +135,15 @@ cmake --build --preset macos-release
135135
📖 **For complete build instructions, troubleshooting, and platform-specific notes, see [README_BUILD.md](README_BUILD.md)**
136136

137137
### Building with Docker
138-
The Dockerfile COPYs folders/files required to build the CPP SDK into the image.
138+
The Docker setup is split into a reusable base image and an SDK image layered on top of it.
139139
**NOTE:** this has only been tested on Linux
140140
```bash
141-
docker build -t livekit-cpp-sdk . -f docker/Dockerfile
141+
docker build -t livekit-cpp-sdk-base . -f docker/Dockerfile.base
142+
docker build --build-arg BASE_IMAGE=livekit-cpp-sdk-base -t livekit-cpp-sdk . -f docker/Dockerfile.sdk
142143
docker run -it --network host livekit-cpp-sdk:latest bash
143144
```
144145

145-
__NOTE:__ if you are building your own Dockerfile, you will likely need to set the same `ENV` variables as in `docker/Dockerfile`, but to the relevant directories:
146+
__NOTE:__ if you are building your own Dockerfile, you will likely need to set the same `ENV` variables as in `docker/Dockerfile.base`, but to the relevant directories:
146147
```bash
147148
export CC=$HOME/gcc-14/bin/gcc
148149
export CXX=$HOME/gcc-14/bin/g++

‎benchmarks/data_track_throughput/CMakeLists.txt‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
# Copyright 2026 LiveKit, Inc.
2+
#
3+
# Standalone CMake build for the data-track throughput experiment.
4+
# All paths are relative to CMAKE_CURRENT_SOURCE_DIR so this directory
5+
# can be moved or renamed freely.
6+
7+
cmake_minimum_required(VERSION 3.20)
8+
project(DataTrackThroughput LANGUAGES CXX)
9+
10+
set(CMAKE_CXX_STANDARD 17)
11+
set(CMAKE_CXX_STANDARD_REQUIRED ON)
12+
13+
# ---- Dependencies --------------------------------------------------------
14+
15+
find_package(LiveKit CONFIG REQUIRED)
16+
17+
find_package(nlohmann_json 3.11 QUIET)
18+
if(NOT nlohmann_json_FOUND)
19+
include(FetchContent)
20+
FetchContent_Declare(
21+
nlohmann_json
22+
GIT_REPOSITORY https://github.com/nlohmann/json.git
23+
GIT_TAG v3.11.3
24+
GIT_SHALLOW TRUE
25+
)
26+
FetchContent_MakeAvailable(nlohmann_json)
27+
endif()
28+
29+
# ---- Targets -------------------------------------------------------------
30+
31+
set(_targets DataTrackThroughputProducer DataTrackThroughputConsumer)
32+
33+
add_executable(DataTrackThroughputProducer producer.cpp)
34+
add_executable(DataTrackThroughputConsumer consumer.cpp)
35+
36+
foreach(_target ${_targets})
37+
target_include_directories(${_target} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}")
38+
target_link_libraries(${_target} PRIVATE LiveKit::livekit nlohmann_json::nlohmann_json)
39+
endforeach()
40+
41+
# ---- RPATH ---------------------------------------------------------------
42+
43+
if(UNIX)
44+
if(APPLE)
45+
set_target_properties(${_targets} PROPERTIES
46+
BUILD_RPATH "@loader_path"
47+
INSTALL_RPATH "@loader_path"
48+
)
49+
else()
50+
set_target_properties(${_targets} PROPERTIES
51+
BUILD_RPATH "$ORIGIN"
52+
INSTALL_RPATH "$ORIGIN"
53+
BUILD_RPATH_USE_ORIGIN TRUE
54+
)
55+
endif()
56+
endif()
57+
58+
# ---- Copy SDK shared libraries next to executables -----------------------
59+
60+
get_target_property(_lk_location LiveKit::livekit LOCATION)
61+
if(_lk_location)
62+
get_filename_component(_lk_lib_dir "${_lk_location}" DIRECTORY)
63+
else()
64+
get_target_property(_lk_location LiveKit::livekit IMPORTED_LOCATION)
65+
if(NOT _lk_location)
66+
get_target_property(_lk_location LiveKit::livekit IMPORTED_LOCATION_RELEASE)
67+
endif()
68+
if(NOT _lk_location)
69+
get_target_property(_lk_location LiveKit::livekit IMPORTED_LOCATION_DEBUG)
70+
endif()
71+
if(_lk_location)
72+
get_filename_component(_lk_lib_dir "${_lk_location}" DIRECTORY)
73+
endif()
74+
endif()
75+
76+
if(_lk_lib_dir)
77+
if(WIN32)
78+
file(GLOB _sdk_shared_libs "${_lk_lib_dir}/../bin/*.dll" "${_lk_lib_dir}/*.dll")
79+
elseif(APPLE)
80+
file(GLOB _sdk_shared_libs "${_lk_lib_dir}/*.dylib")
81+
else()
82+
file(GLOB _sdk_shared_libs "${_lk_lib_dir}/*.so" "${_lk_lib_dir}/*.so.*")
83+
endif()
84+
85+
foreach(_target ${_targets})
86+
foreach(_lib ${_sdk_shared_libs})
87+
get_filename_component(_lib_name "${_lib}" NAME)
88+
add_custom_command(TARGET ${_target} POST_BUILD
89+
COMMAND ${CMAKE_COMMAND} -E copy_if_different
90+
"${_lib}" "$<TARGET_FILE_DIR:${_target}>/${_lib_name}"
91+
COMMENT "Copying ${_lib_name} next to ${_target}"
92+
)
93+
endforeach()
94+
endforeach()
95+
endif()

‎benchmarks/data_track_throughput/README.md‎

Lines changed: 273 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,273 @@
1+
# Data Track Throughput Experiment
2+
3+
Coordinated producer and consumer for benchmarking `LocalDataTrack` /
4+
`RemoteDataTrack` throughput across a sweep of payload sizes and publish rates.
5+
6+
## What It Does
7+
8+
- `producer.cpp`
9+
- Publishes a data track named `data-track-throughput`
10+
- Runs a default sweep of payload sizes and publish rates (see
11+
**Test Bounds** below)
12+
- Calls the consumer over RPC before and after each scenario
13+
14+
- `consumer.cpp`
15+
- Registers a room data-frame callback for the producer's data track
16+
- Receives every frame and records arrival timestamps
17+
- Logs validation warnings (size mismatches, header mismatches, etc.) to stderr
18+
- Tracks duplicates and missing messages
19+
- Appends raw data to scenario-level and per-message CSV files
20+
21+
## Design Principles
22+
23+
- **Raw data only in CSV.** The consumer writes only directly measured values
24+
(counts, byte totals, microsecond timestamps). All derived metrics (throughput,
25+
latency percentiles, delivery ratio, etc.) are computed at analysis time by
26+
`scripts/plot_throughput.py`.
27+
- **Fixed packet size per scenario.** Each scenario uses a single
28+
`packet_size_bytes`. This ensures every message in a run is the same size,
29+
making aggregate measurements unambiguous.
30+
- **Minimal measurement overhead.** The hot `onDataFrame` callback captures the
31+
arrival timestamp first, then appends to an in-memory vector under a brief
32+
mutex. File I/O happens only at finalization after all data is collected.
33+
34+
## Test Bounds
35+
36+
All bounds are defined in `common.h`. A scenario is any combination of
37+
(payload size, publish rate) that passes all three constraints below.
38+
39+
### Hard Limits
40+
41+
| Parameter | Min | Max |
42+
|-----------|-----|-----|
43+
| Packet size | 1 KiB | 256 MiB |
44+
| Publish rate | 1 Hz | 50k Hz |
45+
46+
### Data-Rate Budget
47+
48+
Every scenario must satisfy:
49+
50+
```
51+
packet_size_bytes * desired_rate_hz <= 10 Gbps (1.25 GB/s)
52+
```
53+
54+
This naturally allows small messages at very high rates and large messages at
55+
low rates while preventing any single scenario from attempting an unreasonable
56+
throughput that would destabilize the connection.
57+
58+
### Default Sweep Grid
59+
60+
The default sweep iterates over 13 payload sizes and 13 publish rates, skipping
61+
any combination that exceeds the data-rate budget:
62+
63+
**Payload sizes:** 1 KiB, 4 KiB, 16 KiB, 64 KiB, 128 KiB, 256 KiB, 512 KiB,
64+
1 MiB, 2 MiB, 4 MiB, 16 MiB, 64 MiB, 256 MiB
65+
66+
**Publish rates:** 1, 5, 10, 25, 50, 100, 200, 500, 1k, 5k, 10k, 20k, 50k Hz
67+
68+
The budget clips larger payloads to lower rates. For example:
69+
70+
| Payload | Max rate allowed |
71+
|---------|-----------------|
72+
| 1 KiB | 50k Hz (all rates) |
73+
| 16 KiB | 50k Hz (all rates) |
74+
| 64 KiB | 10k Hz |
75+
| 256 KiB | 1k Hz |
76+
| 1 MiB | 1k Hz |
77+
| 4 MiB | 200 Hz |
78+
| 64 MiB | 10 Hz |
79+
| 256 MiB | 1 Hz |
80+
81+
The budget clips larger payloads to lower rates. For example:
82+
83+
| Payload | Max rate allowed |
84+
|---------|-----------------|
85+
| 1 KiB | 50k Hz (all rates) |
86+
| 16 KiB | 50k Hz (all rates) |
87+
| 64 KiB | 10k Hz |
88+
| 256 KiB | 1k Hz |
89+
| 1 MiB | 1k Hz |
90+
| 4 MiB | 200 Hz |
91+
| 64 MiB | 10 Hz |
92+
| 256 MiB | 1 Hz |
93+
94+
Single-scenario mode (`--rate-hz`, `--packet-size`, `--num-msgs`) bypasses the
95+
default grid and only enforces the hard limits and data-rate budget, allowing
96+
any valid combination to be tested explicitly.
97+
98+
## CSV Output
99+
100+
The consumer writes raw measurement data only. All derived metrics are computed
101+
at analysis time by `scripts/plot_throughput.py`.
102+
103+
### `throughput_summary.csv`
104+
105+
One row per scenario. Contains only raw counts, byte totals, and microsecond
106+
timestamps:
107+
108+
| Column | Description |
109+
|--------|-------------|
110+
| `run_id` | Unique scenario identifier |
111+
| `scenario_name` | Human-readable scenario label |
112+
| `desired_rate_hz` | Requested publish rate |
113+
| `packet_size_bytes` | Fixed packet size for this scenario |
114+
| `messages_requested` | Number of messages the producer was told to send |
115+
| `messages_attempted` | Number of messages the producer tried to send |
116+
| `messages_enqueued` | Number of messages successfully enqueued |
117+
| `messages_enqueue_failed` | Number of enqueue failures |
118+
| `messages_received` | Unique messages received by consumer |
119+
| `messages_missed` | `messages_requested - messages_received` |
120+
| `duplicate_messages` | Number of duplicate frames received |
121+
| `attempted_bytes` | Total bytes the producer attempted to send |
122+
| `enqueued_bytes` | Total bytes successfully enqueued |
123+
| `received_bytes` | Total bytes received by consumer |
124+
| `first_send_time_us` | Timestamp of first send (microseconds since epoch) |
125+
| `last_send_time_us` | Timestamp of last send |
126+
| `first_arrival_time_us` | Timestamp of first arrival at consumer |
127+
| `last_arrival_time_us` | Timestamp of last arrival at consumer |
128+
129+
### `throughput_messages.csv`
130+
131+
One row per received frame. Raw observation data only:
132+
133+
| Column | Description |
134+
|--------|-------------|
135+
| `run_id` | Scenario identifier |
136+
| `sequence` | Message sequence number |
137+
| `payload_bytes` | Actual payload size received |
138+
| `send_time_us` | Producer send timestamp (microseconds since epoch) |
139+
| `arrival_time_us` | Consumer arrival timestamp (microseconds since epoch) |
140+
| `is_duplicate` | 1 if this sequence was already seen, 0 otherwise |
141+
142+
## Prerequisites
143+
144+
- CMake 3.20+
145+
- C++17 compiler
146+
- The LiveKit C++ SDK, built and installed (see below)
147+
148+
## Building
149+
150+
All commands below assume you are in **this directory**
151+
(`data_track_throughput/`).
152+
153+
### 1. Build and install the SDK
154+
155+
From the SDK repository root:
156+
157+
```bash
158+
./build.sh # builds the SDK (debug by default)
159+
cmake --install build-debug --prefix local-install
160+
```
161+
162+
### 2. Configure this experiment
163+
164+
```bash
165+
cmake -S . -B build \
166+
-DCMAKE_PREFIX_PATH="$(cd ../../local-install && pwd)"
167+
```
168+
169+
> Adjust the `CMAKE_PREFIX_PATH` to wherever the SDK was installed. The path
170+
> above assumes this directory lives two levels below the repository root; it
171+
> works regardless of the parent directory's name.
172+
173+
### 3. Build
174+
175+
```bash
176+
cmake --build build
177+
```
178+
179+
The executables and required shared libraries are placed in `build/`.
180+
181+
## Build Targets
182+
183+
- `DataTrackThroughputConsumer`
184+
- `DataTrackThroughputProducer`
185+
186+
## Running
187+
188+
## Generate Tokens
189+
190+
```bash
191+
# producer
192+
lk token create \
193+
--api-key devkey \
194+
--api-secret secret \
195+
-i producer \
196+
--join \
197+
--valid-for 99999h \
198+
--room robo_room \
199+
--grant '{"canPublish":true,"canSubscribe":true,"canPublishData":true}'
200+
201+
# consumer
202+
lk token create \
203+
--api-key devkey \
204+
--api-secret secret \
205+
-i consumer \
206+
--join \
207+
--valid-for 99999h \
208+
--room robo_room \
209+
--grant '{"canPublish":true,"canSubscribe":true,"canPublishData":true}'
210+
```
211+
212+
Start the local server:
213+
```bash
214+
LIVEKIT_CONFIG="enable_data_tracks: true" livekit-server --dev
215+
```
216+
217+
Start the consumer first:
218+
219+
```bash
220+
./build/DataTrackThroughputConsumer <ws-url> <consumer-token>
221+
```
222+
223+
Then start the producer:
224+
225+
```bash
226+
./build/DataTrackThroughputProducer <ws-url> <producer-token> --consumer consumer
227+
```
228+
229+
If you omit `--consumer`, the producer expects exactly one remote participant
230+
to already be in the room.
231+
232+
## Single Scenario
233+
234+
Instead of the full sweep, you can run one scenario:
235+
236+
```bash
237+
./build/DataTrackThroughputProducer \
238+
<ws-url> <producer-token> \
239+
--consumer <consumer-identity> \
240+
--rate-hz 50 \
241+
--packet-size 1mb \
242+
--num-msgs 25
243+
```
244+
245+
## Plotting
246+
247+
Generate plots from a benchmark output directory:
248+
249+
```bash
250+
python3 scripts/plot_throughput.py data_track_throughput_results
251+
```
252+
253+
By default the script writes PNGs into `data_track_throughput_results/plots/`.
254+
Pass `--output-dir <path>` to override the output location.
255+
256+
All derived metrics (throughput, latency percentiles, delivery ratio, receive
257+
rate, interarrival times) are computed from the raw CSV timestamps and counts
258+
at plot time.
259+
260+
### Generated Plots
261+
262+
From `throughput_summary.csv` + `throughput_messages.csv`:
263+
264+
| File | Description |
265+
|------|-------------|
266+
| `expected_vs_actual_throughput.png` | Scatter plot comparing expected vs actual receive throughput (Mbps). Points are colored by desired publish rate and sized by payload. An ideal y=x reference line is overlaid. |
267+
| `dropped_messages_vs_expected_throughput.png` | Scatter plot of missed/dropped message count vs expected throughput, colored by payload size (log scale). |
268+
| `actual_throughput_heatmap.png` | Heatmap of actual receive throughput (Mbps) with payload size on the y-axis and desired rate on the x-axis. |
269+
| `delivery_ratio_heatmap.png` | Heatmap of delivery ratio (received / requested) over the same payload-size x rate grid. |
270+
| `p50_latency_heatmap.png` | Heatmap of median (P50) send-to-receive latency (ms) over the same grid. |
271+
| `p95_latency_heatmap.png` | Heatmap of P95 send-to-receive latency (ms) over the same grid. |
272+
| `message_latency_histogram.png` | Histogram of per-message latency (ms) across all received frames. |
273+
| `message_interarrival_series.png` | Time-series line plot of inter-arrival gaps (ms) for every received message, ordered by run then arrival time. |

0 commit comments

Comments
 (0)