diff --git a/README.md b/README.md index eb30ec0..22cceca 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,24 @@ vs `EXISTS`). This tool runs the queries, times them, and shows the query plan b --- +## What you'll learn + +Every benchmark ends with a conclusion the tool derives from the run it just did — never a +hand-written blurb, so it cannot drift away from the numbers above it. The point of each one is to +show **where the cost is actually paid**, because that is what decides the fix: + +| Benchmark | The lesson | Who pays | +|---|---|---| +| **SELECT \* vs columns** | You are not straining the database, you are straining your serializer. Fetching 16 columns instead of 3 costs ~5x end to end while PostgreSQL's own time barely moves — the bill is decoding rows into objects and, in a real service, re-serializing them to JSON. | **The client** — your API process and the frontend waiting on it | +| **Index usage** | An index speeds up *finding* rows, never *sending* them. The same index is worth ~8x to PostgreSQL but only ~2x to the caller, because `SELECT *` ships the same wide rows either way. | **The result set** | +| **OFFSET vs keyset** | This one really *is* the database's problem. `OFFSET 100000` makes PostgreSQL walk and discard 100,000 rows to return 100. No client tuning helps; only changing the query does. | **PostgreSQL** | +| **JOIN vs IN vs EXISTS** | They do not answer the same question. `JOIN` emits one row per child, `IN`/`EXISTS` one row per parent — 2.4x more data for the same users. Choose by the shape you need, not by a stopwatch. | **The result set** | + +Read together they make one point: *"the query is slow"* is not a diagnosis. The same symptom has a +different cure depending on whether the time is going to the planner, the wire, or your driver. + +--- + ## How the measurements work The numbers are only worth something if the method is. This is what the tool does on every data point: @@ -328,10 +346,16 @@ class CityFilterBenchmark(BenchmarkBase): | `required_tables` | `list[str]` | Enforced by `check_requirements()` | `BenchmarkResult` carries `queries` (a list of `QueryResult`), a `comparison_table` DataFrame, a -`plot_buffer` PNG, and optionally `explain_plans`. When `explain_plans` is empty the runner collects +`plot_buffer` PNG, a `takeaway`, and optionally `explain_plans`. When `explain_plans` is empty the runner collects them automatically — supply them yourself when the plan must be captured at a specific moment, as `index_usage` does. +**End with a conclusion.** Set `takeaway=` on your `BenchmarkResult`: a `Takeaway` carries a +`verdict`, a `CostCentre` (`DATABASE`, `CLIENT` or `TRANSFER`), the `points` of evidence behind it +and the `advice` that follows. Build every number in it from the measurements you just took — a +takeaway that repeats a number the chart does not show is how a benchmark starts lying. It refuses +to be constructed without evidence, on purpose. + **Helpers worth using:** `measure(cursor, query, params)` returns a `Measurement` with `rows_fetched`, `wall_ms`, `server_ms` and a `client_share` property, doing the warm-up, the median and the `EXPLAIN` for you. `table_row_count(conn)` lets you size your data points to the dataset diff --git a/app/history.py b/app/history.py index c0ce243..f74648e 100644 --- a/app/history.py +++ b/app/history.py @@ -12,6 +12,20 @@ TIMESTAMP_FORMAT = "%Y-%m-%dT%H-%M-%S" +def _takeaway_to_dict(takeaway) -> dict | None: + if takeaway is None: + return None + return { + "verdict": takeaway.verdict, + "cost_centre": { + "label": takeaway.cost_centre.label, + "description": takeaway.cost_centre.description, + }, + "points": takeaway.points, + "advice": takeaway.advice, + } + + def _unique_dir(dirname: str) -> Path: """Two runs within the same second would otherwise overwrite each other.""" candidate = RESULTS_DIR / dirname @@ -50,6 +64,7 @@ def save_result(result: BenchmarkResult, params: dict) -> str: "description": result.description, "timestamp": timestamp, "params": params, + "takeaway": _takeaway_to_dict(result.takeaway), "queries": [ {"name": q.name, "query": q.query, "limits": q.limits, "times": q.times, "rows_fetched": q.rows_fetched} diff --git a/app/routes.py b/app/routes.py index eb21af9..2f68e44 100644 --- a/app/routes.py +++ b/app/routes.py @@ -22,6 +22,7 @@ def _results_page(**kwargs): "selected_benchmark": None, "selected_size": "medium", "sizes": SIZES, + "takeaway": None, } return render_template("results.html", **{**defaults, **kwargs}) @@ -73,6 +74,7 @@ def generate(): benchmark_title=result.title, benchmark_description=result.description, explain_plans=result.explain_plans or None, + takeaway=result.takeaway, result_id=result_id, ) @@ -108,6 +110,7 @@ def view_result(result_id): return render_template( "view_result.html", result=data, + takeaway=data.get("takeaway"), plot_data=data.get("plot_data"), comparison_table=data.get("comparison_table"), explain_plans=data.get("explain_plans"), diff --git a/app/templates/results.html b/app/templates/results.html index 3dc4959..ff78800 100644 --- a/app/templates/results.html +++ b/app/templates/results.html @@ -90,6 +90,36 @@
{{ benchmark_title }}
{% endif %} + {% if takeaway %} +
+
+ +
What this tells you
+
+
+

{{ takeaway.verdict }}.

+ +
+ Cost paid by + {{ takeaway.cost_centre.label }} +
{{ takeaway.cost_centre.description }}
+
+ + + + {% if takeaway.advice %} +
+ {{ takeaway.advice }} +
+ {% endif %} +
+
+ {% endif %} + {% if plot_data %}
diff --git a/app/templates/view_result.html b/app/templates/view_result.html index e2878aa..a4f55a4 100644 --- a/app/templates/view_result.html +++ b/app/templates/view_result.html @@ -41,6 +41,36 @@
{{ result.title or result.b
+ {% if takeaway %} +
+
+ +
What this tells you
+
+
+

{{ takeaway.verdict }}.

+ +
+ Cost paid by + {{ takeaway.cost_centre.label }} +
{{ takeaway.cost_centre.description }}
+
+ + + + {% if takeaway.advice %} +
+ {{ takeaway.advice }} +
+ {% endif %} +
+
+ {% endif %} + {% if plot_data %}
diff --git a/benchmarks/base.py b/benchmarks/base.py index 507b136..2103c94 100644 --- a/benchmarks/base.py +++ b/benchmarks/base.py @@ -3,6 +3,7 @@ import time from abc import ABC, abstractmethod from dataclasses import dataclass, field +from enum import Enum from io import BytesIO import pandas as pd @@ -41,6 +42,58 @@ def client_share(self) -> float | None: return max(0.0, (self.wall_ms - self.server_ms) / self.wall_ms) +class CostCentre(Enum): + """Who actually pays for a slow query. + + Keeping these apart is the single most useful thing this tool does: a cost + paid in the driver is fixed by asking for less data, while a cost paid in + PostgreSQL is fixed by changing the query or the indexes. Optimising the + wrong one wastes weeks. + """ + + DATABASE = ( + "PostgreSQL", + "PostgreSQL itself does the extra work — more rows scanned, a worse plan. " + "Fix it with indexes or a different query shape.", + ) + CLIENT = ( + "the client", + "PostgreSQL is barely involved. The time goes to transferring rows and to the " + "driver turning bytes into objects, so the bill lands in your API process, not " + "your database. Fix it by asking for less data.", + ) + TRANSFER = ( + "the result set", + "Both sides are doing reasonable work; there is simply more data crossing the " + "wire than the question required. Fix it by returning fewer rows or columns.", + ) + + def __init__(self, label, description): + self.label = label + self.description = description + + +@dataclass +class Takeaway: + """The conclusion a benchmark reached, derived from the run it just did. + + Built from measured values rather than written by hand, so it cannot drift + away from the chart above it. + """ + + verdict: str + cost_centre: CostCentre + points: list[str] + #: What to do about it. + advice: str = "" + + def __post_init__(self): + if not self.verdict.strip(): + raise ValueError("a takeaway needs a verdict") + if not self.points: + raise ValueError("a takeaway needs evidence; a conclusion without it is an opinion") + + @dataclass class QueryResult: name: str @@ -63,6 +116,7 @@ class BenchmarkResult: comparison_table: pd.DataFrame plot_buffer: BytesIO explain_plans: dict[str, str] = field(default_factory=dict) + takeaway: "Takeaway | None" = None def server_execution_ms(cursor, query, params=None) -> float | None: @@ -120,6 +174,16 @@ def speedup(slow: float | None, fast: float | None) -> str: return f"{slow / fast:.1f}x" +def ratio(slow: float | None, fast: float | None) -> float | None: + if slow is None or fast is None or fast <= 0: + return None + return slow / fast + + +def percent(value: float | None) -> str: + return "n/a" if value is None else f"{value * 100:.0f}%" + + def plot_server_series(ax, query, color): """Dashed companion line showing PostgreSQL's own time. diff --git a/benchmarks/index_usage.py b/benchmarks/index_usage.py index a9489c6..f29e8f8 100644 --- a/benchmarks/index_usage.py +++ b/benchmarks/index_usage.py @@ -11,10 +11,13 @@ from benchmarks.base import ( BenchmarkBase, BenchmarkResult, + CostCentre, QueryResult, + Takeaway, format_ms, measure, plot_server_series, + ratio, speedup, ) from benchmarks.registry import register @@ -79,6 +82,41 @@ def run(self, conn) -> BenchmarkResult: comparison_table=self._build_comparison(q1, q2), plot_buffer=self._build_plot(q1, q2), explain_plans={q1.name: no_index_plan, q2.name: with_index_plan}, + takeaway=self._takeaway(q1, q2), + ) + + def _takeaway(self, q1: QueryResult, q2: QueryResult) -> Takeaway: + i = len(q1.limits) // 2 + rows = q1.rows_fetched[i] + total = ratio(q1.times[i], q2.times[i]) + server = ratio(q1.server_times[i], q2.server_times[i]) + + points = [ + f"Both queries return the same {rows:,} rows; only the index differs.", + f"Inside PostgreSQL: {format_ms(q1.server_times[i])} -> " + f"{format_ms(q2.server_times[i])}" + (f" ({server:.1f}x faster)." if server else "."), + f"End to end: {format_ms(q1.times[i])} -> {format_ms(q2.times[i])}" + + (f" (only {total:.1f}x faster)." if total else "."), + ] + if server and total and server > total: + points.append( + f"The index is worth {server:.1f}x to the database but the request only sees " + f"{total:.1f}x, because shipping {rows:,} wide rows costs the same however " + "they were found." + ) + + return Takeaway( + verdict=( + f"The index made PostgreSQL {server:.1f}x faster, but the request only {total:.1f}x" + if server and total else "The index sped up the scan" + ), + cost_centre=CostCentre.TRANSFER, + points=points, + advice=( + "An index speeds up *finding* rows, never *sending* them. Indexing the filter is " + "the right move, but as long as you SELECT * the transfer cost caps what you can " + "win. Narrow the projection and the index gain reaches the caller." + ), ) def teardown(self, conn) -> None: diff --git a/benchmarks/join_vs_subquery.py b/benchmarks/join_vs_subquery.py index 7f1e88d..83770f0 100644 --- a/benchmarks/join_vs_subquery.py +++ b/benchmarks/join_vs_subquery.py @@ -11,9 +11,12 @@ from benchmarks.base import ( BenchmarkBase, BenchmarkResult, + CostCentre, QueryResult, + Takeaway, format_ms, measure, + ratio, ) from benchmarks.registry import register @@ -112,6 +115,49 @@ def run(self, conn) -> BenchmarkResult: q.name: run_explain(conn, q.query, (EXPLAIN_THRESHOLD,)) for q in (q1, q2, q3) }, + takeaway=self._takeaway(q1, q2, q3), + ) + + def _takeaway(self, q1: QueryResult, q2: QueryResult, q3: QueryResult) -> Takeaway: + i = 0 + join_rows, user_rows = q1.rows_fetched[i], q2.rows_fetched[i] + fanout = ratio(join_rows, user_rows) + + points = [ + f"JOIN returned {join_rows:,} rows; IN and EXISTS returned {user_rows:,}" + + (f" — {fanout:.1f}x more data for the same users." if fanout else "."), + "That is not a tie-break, it is a different answer: JOIN emits one row per " + "matching order, IN and EXISTS one row per matching user.", + f"Inside PostgreSQL, JOIN is the fastest of the three " + f"({format_ms(q1.server_times[i])} vs {format_ms(q2.server_times[i])} and " + f"{format_ms(q3.server_times[i])}).", + f"End to end it is the slowest ({format_ms(q1.times[i])} vs " + f"{format_ms(q2.times[i])} and {format_ms(q3.times[i])}), because those extra " + "rows still have to reach you.", + ] + if q2.server_times[i] and q3.server_times[i]: + close = abs(q2.server_times[i] - q3.server_times[i]) / max(q2.server_times[i], 0.001) + if close < 0.35: + points.append( + "IN and EXISTS land within noise of each other: the planner rewrites both " + "into the same semi-join. Check the EXPLAIN output below." + ) + + return Takeaway( + verdict=( + f"JOIN shipped {fanout:.1f}x more rows than IN and EXISTS to answer " + "the same question" + if fanout else "The three patterns do not return the same rows" + ), + cost_centre=CostCentre.TRANSFER, + points=points, + advice=( + "Pick the pattern by the shape of the answer you need, not by a benchmark. If you " + "need columns from the child table, JOIN is the only option that gives them. If " + "you only need to know the parent matched, IN or EXISTS say so without " + "duplicating the parent row per child — and you avoid a DISTINCT to undo the " + "damage later." + ), ) def teardown(self, conn) -> None: diff --git a/benchmarks/pagination.py b/benchmarks/pagination.py index e278334..53d7e5a 100644 --- a/benchmarks/pagination.py +++ b/benchmarks/pagination.py @@ -12,9 +12,12 @@ BenchmarkBase, BenchmarkNotApplicable, BenchmarkResult, + CostCentre, QueryResult, + Takeaway, format_ms, measure, + ratio, speedup, table_row_count, ) @@ -81,6 +84,43 @@ def run(self, conn) -> BenchmarkResult: q1.name: run_explain(conn, QUERY_OFFSET, (PAGE_SIZE, deepest)), q2.name: run_explain(conn, QUERY_KEYSET, (deepest, PAGE_SIZE)), }, + takeaway=self._takeaway(q1, q2), + ) + + def _takeaway(self, q1: QueryResult, q2: QueryResult) -> Takeaway: + last = -1 + page = q1.limits[last] // PAGE_SIZE + 1 + server = ratio(q1.server_times[last], q2.server_times[last]) + first_server = q1.server_times[0] + deep_server = q1.server_times[last] + + points = [ + f"Every point returns one page of {PAGE_SIZE} rows, so the amount of data sent " + "back is identical throughout — only the depth changes.", + f"OFFSET grows with depth: {format_ms(first_server)} on page 1, " + f"{format_ms(deep_server)} on page {page:,}.", + f"Keyset does not: {format_ms(q2.server_times[0])} on page 1, " + f"{format_ms(q2.server_times[last])} on page {page:,}.", + ] + if server: + points.append( + f"At page {page:,} that is {server:.0f}x more database time for identical output." + ) + + return Takeaway( + verdict=( + f"By page {page:,}, OFFSET makes PostgreSQL do {server:.0f}x more work " + f"to return the same {PAGE_SIZE} rows" + if server else "OFFSET degrades with page depth while keyset stays flat" + ), + cost_centre=CostCentre.DATABASE, + points=points, + advice=( + "Unlike SELECT *, this one really is the database's problem: OFFSET has to walk " + "and discard every row it skips, so page 1,000 costs a thousand pages of work. " + "Keyset pagination asks the index to jump straight to the last id you saw. " + "Trading OFFSET for a WHERE id > ? is the fix — no amount of client tuning helps." + ), ) def _build_comparison(self, q1, q2): diff --git a/benchmarks/select_star.py b/benchmarks/select_star.py index c5759b8..7b13015 100644 --- a/benchmarks/select_star.py +++ b/benchmarks/select_star.py @@ -11,10 +11,14 @@ BenchmarkBase, BenchmarkNotApplicable, BenchmarkResult, + CostCentre, QueryResult, + Takeaway, format_ms, measure, + percent, plot_server_series, + ratio, speedup, table_row_count, ) @@ -74,6 +78,50 @@ def run(self, conn) -> BenchmarkResult: queries=[q1, q2], comparison_table=self._build_comparison(q1, q2), plot_buffer=self._build_plot(q1, q2), + takeaway=self._takeaway(q1, q2), + ) + + def _takeaway(self, q1: QueryResult, q2: QueryResult) -> Takeaway: + i = len(q1.limits) // 2 + rows = q1.rows_fetched[i] + total = ratio(q1.times[i], q2.times[i]) + server = ratio(q1.server_times[i], q2.server_times[i]) + client_share = None + if q1.server_times[i] is not None and q1.times[i] > 0: + client_share = (q1.times[i] - q1.server_times[i]) / q1.times[i] + + points = [ + f"Same {rows:,} rows either way — the only difference is 16 columns vs 3.", + f"Total time: {format_ms(q1.times[i])} vs {format_ms(q2.times[i])}" + + (f" ({total:.1f}x)." if total else "."), + f"Inside PostgreSQL: {format_ms(q1.server_times[i])} vs " + f"{format_ms(q2.server_times[i])}" + + (f" ({server:.1f}x)." if server else "."), + ] + if client_share is not None: + points.append( + f"{percent(client_share)} of the SELECT * time is spent outside the database, " + "in transfer and in the driver building Python objects." + ) + if server is not None and server < 1: + points.append( + "PostgreSQL is actually *faster* for SELECT *: returning the stored row needs " + "no projection work, while picking 3 columns means building a new one." + ) + + return Takeaway( + verdict=( + f"Asking for every column cost {total:.1f}x more time for exactly the same rows" + if total else "Asking for every column cost more time for the same rows" + ), + cost_centre=CostCentre.CLIENT, + points=points, + advice=( + "The database was never the bottleneck here — your serializer is. Every column you " + "select has to be decoded into an object and, in a real service, serialized again " + "to JSON and pushed to the frontend. That bill is paid by your API process and by " + "whoever is waiting on the other end, so select the columns you need and no more." + ), ) def _build_comparison(self, q1: QueryResult, q2: QueryResult) -> pd.DataFrame: diff --git a/tests/test_history.py b/tests/test_history.py index 8e4bc44..39497c8 100644 --- a/tests/test_history.py +++ b/tests/test_history.py @@ -94,3 +94,24 @@ def test_save_result_does_not_overwrite_a_run_from_the_same_second(tmp_path, mon assert first != second _, total = list_results() assert total == 2 + + +def test_saved_result_keeps_the_conclusion_for_the_history_view(tmp_path, monkeypatch): + """A stored run that lost its takeaway is just a chart again.""" + from benchmarks.base import CostCentre, Takeaway + + monkeypatch.setattr("app.history.RESULTS_DIR", tmp_path) + result = _make_result() + result.takeaway = Takeaway( + verdict="Asking for every column cost 5.2x more", + cost_centre=CostCentre.CLIENT, + points=["98% of the time is outside PostgreSQL"], + advice="Select the columns you need.", + ) + + loaded = load_result(save_result(result, {"size": "small"})) + + assert loaded["takeaway"]["verdict"] == "Asking for every column cost 5.2x more" + assert loaded["takeaway"]["cost_centre"]["label"] == "the client" + assert loaded["takeaway"]["points"] == ["98% of the time is outside PostgreSQL"] + assert loaded["takeaway"]["advice"] == "Select the columns you need." diff --git a/tests/test_integration.py b/tests/test_integration.py index 992c8d9..f9c4138 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -219,3 +219,48 @@ def test_every_benchmark_reports_server_side_timings(conn): for query in result.queries: assert len(query.server_times) == len(query.times) assert all(t is not None for t in query.server_times), f"{name}/{query.name}" + + +@pytest.mark.parametrize("name", [bm.name for bm in all_benchmarks()]) +def test_every_benchmark_states_a_conclusion(conn, name): + """The point of the tool is the lesson, not the chart.""" + bm = get(name) + bm.setup(conn) + try: + result = bm.run(conn) + finally: + bm.teardown(conn) + + takeaway = result.takeaway + assert takeaway is not None, f"{name} produced numbers but no conclusion" + assert takeaway.verdict.strip() + assert takeaway.points, "a conclusion with no evidence is an opinion" + assert takeaway.cost_centre is not None + + +def test_select_star_blames_the_client_not_the_database(conn): + """The whole lesson: SELECT * is not a database problem.""" + from benchmarks.base import CostCentre + + bm = get("select_star") + bm.setup(conn) + try: + result = bm.run(conn) + finally: + bm.teardown(conn) + + assert result.takeaway.cost_centre is CostCentre.CLIENT + + +def test_pagination_blames_the_database(conn): + """Unlike SELECT *, a deep OFFSET really is PostgreSQL doing extra work.""" + from benchmarks.base import CostCentre + + bm = get("pagination") + bm.setup(conn) + try: + result = bm.run(conn) + finally: + bm.teardown(conn) + + assert result.takeaway.cost_centre is CostCentre.DATABASE diff --git a/tests/test_takeaway.py b/tests/test_takeaway.py new file mode 100644 index 0000000..04640a5 --- /dev/null +++ b/tests/test_takeaway.py @@ -0,0 +1,32 @@ +"""A benchmark that shows numbers without stating what they mean is a quiz, +not a lesson. Every benchmark must close with a conclusion, and that +conclusion has to be derived from the run it just did.""" +import pytest + +from benchmarks.base import CostCentre, Takeaway + + +def test_takeaway_names_where_the_cost_is_actually_paid(): + t = Takeaway( + verdict="Fetching 16 columns cost 5.2x more.", + cost_centre=CostCentre.CLIENT, + points=["98% of the gap is outside PostgreSQL"], + ) + + assert t.cost_centre is CostCentre.CLIENT + assert "driver" in t.cost_centre.description.lower() + + +def test_cost_centre_distinguishes_the_database_from_the_client(): + assert CostCentre.DATABASE is not CostCentre.CLIENT + assert "postgresql" in CostCentre.DATABASE.description.lower() + + +def test_takeaway_requires_at_least_one_piece_of_evidence(): + with pytest.raises(ValueError, match="evidence"): + Takeaway(verdict="Trust me.", cost_centre=CostCentre.CLIENT, points=[]) + + +def test_takeaway_verdict_must_not_be_empty(): + with pytest.raises(ValueError, match="verdict"): + Takeaway(verdict=" ", cost_centre=CostCentre.CLIENT, points=["x"])