From 40e786b6b02a19613e99b3f49e8e6edf3aa7939e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20S=C3=A1nchez?= <100514206+jonaas-dev@users.noreply.github.com> Date: Sun, 6 Sep 2026 15:59:24 +0200 Subject: [PATCH 1/5] fix(benchmarks): measure what the charts claim to measure Three of the four benchmarks were reporting numbers that did not support their own thesis. Verified against a real PostgreSQL 17 instance. - pagination held OFFSET at 500000 and varied the page size instead, so on the small and medium datasets both queries returned zero rows while the chart still rendered. It now varies the offset with a fixed page size. - index_usage captured both EXPLAIN plans after CREATE INDEX, making them byte-identical and the "no index" label false. The unindexed plan is now captured before the index exists. - select_star swept LIMITs from 1M down regardless of table size, so every data point returned the whole table. Limits now derive from the real row count, and rows_fetched is shown in the comparison table. - join_vs_subquery dropped and recreated a table called `orders`, which would destroy a user table of that name. Renamed to sqlperf_orders. All four now share measure(), which discards a warm-up run and reports the median of five. Previously each point was a single cold run with the slow query always going first, biasing every result toward its own conclusion. required_tables is now enforced by check_requirements() instead of being a decorative attribute, and BenchmarkNotApplicable replaces silently charting an empty result set. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_014abw4B6YUf54giaEyPQpbo --- benchmarks/base.py | 50 ++++++++- benchmarks/index_usage.py | 98 +++++++++--------- benchmarks/join_vs_subquery.py | 155 ++++++++++++++-------------- benchmarks/pagination.py | 122 +++++++++++----------- benchmarks/registry.py | 1 - benchmarks/select_star.py | 67 +++++++----- tests/test_benchmark.py | 87 +++++++++++++--- tests/test_integration.py | 182 +++++++++++++++++++++++++++++++++ tests/test_measure.py | 56 ++++++++++ 9 files changed, 588 insertions(+), 230 deletions(-) create mode 100644 tests/test_integration.py create mode 100644 tests/test_measure.py diff --git a/benchmarks/base.py b/benchmarks/base.py index 92622aa..1217a72 100644 --- a/benchmarks/base.py +++ b/benchmarks/base.py @@ -1,16 +1,22 @@ +import statistics +import time from abc import ABC, abstractmethod from dataclasses import dataclass, field from io import BytesIO import pandas as pd +REPETITIONS = 5 + @dataclass class QueryResult: name: str query: str times: list[float] - limits: list[int] + #: x-axis values for this benchmark — a LIMIT, an OFFSET or a filter value + #: depending on what the benchmark varies. Same length as `times`. + limits: list rows_fetched: list[int] @@ -25,12 +31,54 @@ class BenchmarkResult: explain_plans: dict[str, str] = field(default_factory=dict) +def measure(cursor, query, params=None, repetitions=REPETITIONS, clock=time.perf_counter): + """Time a query as the median of `repetitions` runs after one warm-up. + + The warm-up matters more than the repetitions: without it whichever query + runs first pays for the cold cache, which systematically favours whatever + the benchmark runs second. + + Returns (rows_fetched, median_milliseconds). + """ + cursor.execute(query, params) + rows = cursor.fetchall() + + timings = [] + for _ in range(repetitions): + start = clock() + cursor.execute(query, params) + rows = cursor.fetchall() + timings.append((clock() - start) * 1000) + + return len(rows), statistics.median(timings) + + +def table_row_count(conn, table: str = "users") -> int: + with conn.cursor() as cur: + cur.execute(f"SELECT count(*) FROM {table}") + return cur.fetchone()[0] + + +class BenchmarkNotApplicable(RuntimeError): + """Raised when the current dataset is too small for the benchmark to mean anything.""" + + class BenchmarkBase(ABC): name: str = "" title: str = "" description: str = "" required_tables: list[str] = [] + def check_requirements(self, conn) -> None: + """Fail loudly before measuring if a required table is missing.""" + for table in self.required_tables: + with conn.cursor() as cur: + cur.execute("SELECT to_regclass(%s)", (table,)) + if cur.fetchone()[0] is None: + raise BenchmarkNotApplicable( + f"{self.name} requires a '{table}' table that does not exist" + ) + @abstractmethod def setup(self, conn) -> None: pass diff --git a/benchmarks/index_usage.py b/benchmarks/index_usage.py index 42a88e8..9dcbafa 100644 --- a/benchmarks/index_usage.py +++ b/benchmarks/index_usage.py @@ -1,15 +1,28 @@ -import time import io + import matplotlib + matplotlib.use("Agg") -from matplotlib.figure import Figure -from matplotlib.backends.backend_agg import FigureCanvasAgg import pandas as pd +from matplotlib.backends.backend_agg import FigureCanvasAgg +from matplotlib.figure import Figure -from benchmarks.base import BenchmarkBase, BenchmarkResult, QueryResult -from benchmarks.registry import register -from app.config import TMP_DIR from app.benchmark import run_explain +from benchmarks.base import ( + BenchmarkBase, + BenchmarkResult, + QueryResult, + measure, +) +from benchmarks.registry import register + +QUERY = "SELECT * FROM users WHERE age = %s" +INDEX_NAME = "sqlperf_idx_users_age" +THRESHOLDS = [25, 30, 35, 40, 45] +EXPLAIN_AGE = 35 + +NO_INDEX_LABEL = "Seq Scan (no index)" +WITH_INDEX_LABEL = "Index Scan (with B-tree)" @register @@ -20,77 +33,66 @@ class IndexUsageBenchmark(BenchmarkBase): required_tables = ["users"] def setup(self, conn) -> None: + self.check_requirements(conn) with conn.cursor() as cur: - cur.execute("DROP INDEX IF EXISTS idx_users_age") + cur.execute(f"DROP INDEX IF EXISTS {INDEX_NAME}") conn.commit() def run(self, conn) -> BenchmarkResult: - query_no_idx = "SELECT * FROM users WHERE age = %s" - query_with_idx = "SELECT * FROM users WHERE age = %s" - - thresholds = [25, 30, 35, 40, 45] - - q1_times, q2_times = [], [] - q1_rows, q2_rows = [], [] + q1_times, q1_rows = [], [] + q2_times, q2_rows = [], [] with conn.cursor() as cur: - for t in thresholds: - rows, elapsed = self._measure(query_no_idx, (t,), cur) + for age in THRESHOLDS: + rows, elapsed = measure(cur, QUERY, (age,)) q1_times.append(elapsed) q1_rows.append(rows) - cur.execute("CREATE INDEX idx_users_age ON users(age)") + # Captured before the index exists: running both EXPLAINs after + # CREATE INDEX makes them identical and the "no index" label a lie. + no_index_plan = run_explain(conn, QUERY, (EXPLAIN_AGE,)) + + with conn.cursor() as cur: + cur.execute(f"CREATE INDEX {INDEX_NAME} ON users(age)") + cur.execute("ANALYZE users") conn.commit() - for t in thresholds: - rows, elapsed = self._measure(query_with_idx, (t,), cur) + for age in THRESHOLDS: + rows, elapsed = measure(cur, QUERY, (age,)) q2_times.append(elapsed) q2_rows.append(rows) + with_index_plan = run_explain(conn, QUERY, (EXPLAIN_AGE,)) + q1 = QueryResult( - name="Seq Scan (no index)", - query=query_no_idx.replace("%s", "35"), - times=q1_times, limits=thresholds, rows_fetched=q1_rows, + name=NO_INDEX_LABEL, query=QUERY, + times=q1_times, limits=THRESHOLDS, rows_fetched=q1_rows, ) q2 = QueryResult( - name="Index Scan (with B-tree)", - query=query_with_idx.replace("%s", "35"), - times=q2_times, limits=thresholds, rows_fetched=q2_rows, + name=WITH_INDEX_LABEL, query=QUERY, + times=q2_times, limits=THRESHOLDS, rows_fetched=q2_rows, ) - explain_plans = { - q1.name: run_explain(conn, q1.query, (35,)), - q2.name: run_explain(conn, q2.query, (35,)), - } - - comparison = self._build_comparison(q1, q2, "age =") - plot = self._build_plot(q1, q2) - return BenchmarkResult( name=self.name, title=self.title, description=self.description, - queries=[q1, q2], comparison_table=comparison, plot_buffer=plot, - explain_plans=explain_plans, + queries=[q1, q2], + comparison_table=self._build_comparison(q1, q2), + plot_buffer=self._build_plot(q1, q2), + explain_plans={q1.name: no_index_plan, q2.name: with_index_plan}, ) def teardown(self, conn) -> None: with conn.cursor() as cur: - cur.execute("DROP INDEX IF EXISTS idx_users_age") + cur.execute(f"DROP INDEX IF EXISTS {INDEX_NAME}") conn.commit() - def _measure(self, query, params, cursor): - start = time.perf_counter() - cursor.execute(query, params) - rows = cursor.fetchall() - elapsed = (time.perf_counter() - start) * 1000 - return len(rows), elapsed - - def _build_comparison(self, q1, q2, label): + def _build_comparison(self, q1, q2): rows = [] - for i, t in enumerate(q1.limits): - diff = q1.times[i] - q2.times[i] + for i, age in enumerate(q1.limits): speedup = f"{q1.times[i] / q2.times[i]:.1f}x" if q2.times[i] > 0 else "N/A" rows.append({ - f"{label}": t, + "age =": age, + "rows_matched": q1.rows_fetched[i], "without_index": f"{q1.times[i]:.2f} ms", "with_index": f"{q2.times[i]:.2f} ms", "speedup": speedup, @@ -105,7 +107,7 @@ def _build_plot(self, q1, q2): ax.plot(q2.limits, q2.times, marker="s", label=q2.name, color="#2ecc71") ax.set_title(self.title) ax.set_xlabel("age = value") - ax.set_ylabel("Execution Time (ms)") + ax.set_ylabel("Median execution time (ms)") ax.set_xticks(q1.limits) ax.grid(True, linestyle="--", alpha=0.6) ax.legend() diff --git a/benchmarks/join_vs_subquery.py b/benchmarks/join_vs_subquery.py index 3c70e66..ca9336b 100644 --- a/benchmarks/join_vs_subquery.py +++ b/benchmarks/join_vs_subquery.py @@ -1,122 +1,118 @@ -import time import io + import matplotlib + matplotlib.use("Agg") -from matplotlib.figure import Figure -from matplotlib.backends.backend_agg import FigureCanvasAgg import pandas as pd +from matplotlib.backends.backend_agg import FigureCanvasAgg +from matplotlib.figure import Figure -from benchmarks.base import BenchmarkBase, BenchmarkResult, QueryResult -from benchmarks.registry import register -from app.config import TMP_DIR from app.benchmark import run_explain +from benchmarks.base import ( + BenchmarkBase, + BenchmarkResult, + QueryResult, + measure, +) +from benchmarks.registry import register + +# Namespaced so the benchmark can never drop a table the user owns. +ORDERS = "sqlperf_orders" + +QUERY_JOIN = f""" + SELECT u.id, u.name, u.email, o.amount + FROM users u + INNER JOIN {ORDERS} o ON u.id = o.user_id + WHERE o.amount > %s +""" +QUERY_SUBQUERY = f""" + SELECT id, name, email + FROM users + WHERE id IN (SELECT user_id FROM {ORDERS} WHERE amount > %s) +""" +QUERY_EXISTS = f""" + SELECT id, name, email + FROM users u + WHERE EXISTS (SELECT 1 FROM {ORDERS} o WHERE o.user_id = u.id AND o.amount > %s) +""" + +THRESHOLDS = [50, 100, 200, 300, 400] +EXPLAIN_THRESHOLD = 100 @register class JoinVsSubqueryBenchmark(BenchmarkBase): name = "join_vs_subquery" title = "JOIN vs subquery for filtering" - description = "Compares JOIN and IN (subquery) for relational filtering" + description = "Compares JOIN, IN (subquery) and EXISTS for relational filtering" required_tables = ["users"] def setup(self, conn) -> None: + self.check_requirements(conn) with conn.cursor() as cur: - cur.execute("DROP TABLE IF EXISTS orders") - cur.execute(""" - CREATE TABLE orders ( + cur.execute(f"DROP TABLE IF EXISTS {ORDERS}") + cur.execute(f""" + CREATE TABLE {ORDERS} ( id SERIAL PRIMARY KEY, user_id INT REFERENCES users(id), amount DECIMAL(10,2), created_at TIMESTAMP DEFAULT NOW() ) """) - cur.execute(""" - INSERT INTO orders (user_id, amount) + cur.execute(f""" + INSERT INTO {ORDERS} (user_id, amount) SELECT id, (random() * 500)::DECIMAL(10,2) FROM users WHERE random() < 0.3 """) + cur.execute(f"CREATE INDEX ON {ORDERS}(user_id)") + cur.execute(f"ANALYZE {ORDERS}") conn.commit() def run(self, conn) -> BenchmarkResult: - query_join = """ - SELECT u.id, u.name, u.email, o.amount - FROM users u - INNER JOIN orders o ON u.id = o.user_id - WHERE o.amount > %s - """ - query_subquery = """ - SELECT id, name, email - FROM users - WHERE id IN (SELECT user_id FROM orders WHERE amount > %s) - """ - query_exists = """ - SELECT id, name, email - FROM users u - WHERE EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id AND o.amount > %s) - """ - - thresholds = [50, 100, 200, 300, 400] - q1_times, q2_times, q3_times = [], [], [] - q1_rows, q2_rows, q3_rows = [], [], [] + series = {QUERY_JOIN: ([], []), QUERY_SUBQUERY: ([], []), QUERY_EXISTS: ([], [])} with conn.cursor() as cur: - for t in thresholds: - r1, t1 = self._measure(query_join, (t,), cur) - r2, t2 = self._measure(query_subquery, (t,), cur) - r3, t3 = self._measure(query_exists, (t,), cur) - q1_times.append(t1) - q2_times.append(t2) - q3_times.append(t3) - q1_rows.append(r1) - q2_rows.append(r2) - q3_rows.append(r3) - - q1 = QueryResult( - name="JOIN", query=query_join.strip(), - times=q1_times, limits=thresholds, rows_fetched=q1_rows, - ) - q2 = QueryResult( - name="IN (subquery)", query=query_subquery.strip(), - times=q2_times, limits=thresholds, rows_fetched=q2_rows, - ) - q3 = QueryResult( - name="EXISTS", query=query_exists.strip(), - times=q3_times, limits=thresholds, rows_fetched=q3_rows, + for threshold in THRESHOLDS: + for query, (times, rows_fetched) in series.items(): + rows, elapsed = measure(cur, query, (threshold,)) + times.append(elapsed) + rows_fetched.append(rows) + + q1, q2, q3 = ( + QueryResult( + name=name, query=query.strip(), + times=series[query][0], limits=THRESHOLDS, rows_fetched=series[query][1], + ) + for name, query in ( + ("JOIN", QUERY_JOIN), + ("IN (subquery)", QUERY_SUBQUERY), + ("EXISTS", QUERY_EXISTS), + ) ) - comparison = self._build_comparison(q1, q2, q3) - plot = self._build_plot(q1, q2, q3) - - explain_plans = { - q1.name: run_explain(conn, q1.query, (100,)), - q2.name: run_explain(conn, q2.query, (100,)), - q3.name: run_explain(conn, q3.query, (100,)), - } - return BenchmarkResult( name=self.name, title=self.title, description=self.description, - queries=[q1, q2, q3], comparison_table=comparison, plot_buffer=plot, - explain_plans=explain_plans, + queries=[q1, q2, q3], + comparison_table=self._build_comparison(q1, q2, q3), + plot_buffer=self._build_plot(q1, q2, q3), + explain_plans={ + q.name: run_explain(conn, q.query, (EXPLAIN_THRESHOLD,)) + for q in (q1, q2, q3) + }, ) def teardown(self, conn) -> None: with conn.cursor() as cur: - cur.execute("DROP TABLE IF EXISTS orders") + cur.execute(f"DROP TABLE IF EXISTS {ORDERS}") conn.commit() - def _measure(self, query, params, cursor): - start = time.perf_counter() - cursor.execute(query, params) - rows = cursor.fetchall() - elapsed = (time.perf_counter() - start) * 1000 - return len(rows), elapsed - def _build_comparison(self, q1, q2, q3): rows = [] - for i, t in enumerate(q1.limits): + for i, threshold in enumerate(q1.limits): rows.append({ - "amount >": t, + "amount >": threshold, + "rows_matched": q1.rows_fetched[i], "join": f"{q1.times[i]:.2f} ms", "in_subquery": f"{q2.times[i]:.2f} ms", "exists": f"{q3.times[i]:.2f} ms", @@ -127,12 +123,11 @@ def _build_plot(self, q1, q2, q3): fig = Figure(figsize=(10, 6)) canvas = FigureCanvasAgg(fig) ax = fig.add_subplot(111) - ax.plot(q1.limits, q1.times, marker="o", label=q1.name) - ax.plot(q2.limits, q2.times, marker="s", label=q2.name) - ax.plot(q3.limits, q3.times, marker="^", label=q3.name) + for query, marker in ((q1, "o"), (q2, "s"), (q3, "^")): + ax.plot(query.limits, query.times, marker=marker, label=query.name) ax.set_title(self.title) - ax.set_xlabel("amount threshold") - ax.set_ylabel("Execution Time (ms)") + ax.set_xlabel("amount > threshold") + ax.set_ylabel("Median execution time (ms)") ax.set_xticks(q1.limits) ax.grid(True, linestyle="--", alpha=0.6) ax.legend() diff --git a/benchmarks/pagination.py b/benchmarks/pagination.py index e95484e..4a75433 100644 --- a/benchmarks/pagination.py +++ b/benchmarks/pagination.py @@ -1,15 +1,28 @@ -import time import io + import matplotlib + matplotlib.use("Agg") -from matplotlib.figure import Figure -from matplotlib.backends.backend_agg import FigureCanvasAgg import pandas as pd +from matplotlib.backends.backend_agg import FigureCanvasAgg +from matplotlib.figure import Figure -from benchmarks.base import BenchmarkBase, BenchmarkResult, QueryResult -from benchmarks.registry import register -from app.config import TMP_DIR from app.benchmark import run_explain +from benchmarks.base import ( + BenchmarkBase, + BenchmarkNotApplicable, + BenchmarkResult, + QueryResult, + measure, + table_row_count, +) +from benchmarks.registry import register + +PAGE_SIZE = 100 +DATA_POINTS = 6 + +QUERY_OFFSET = "SELECT * FROM users ORDER BY id LIMIT %s OFFSET %s" +QUERY_KEYSET = "SELECT * FROM users WHERE id > %s ORDER BY id LIMIT %s" @register @@ -20,75 +33,70 @@ class PaginationBenchmark(BenchmarkBase): required_tables = ["users"] def setup(self, conn) -> None: - with conn.cursor() as cur: - cur.execute("DROP INDEX IF EXISTS idx_users_id_cursor") - cur.execute("CREATE INDEX idx_users_id_cursor ON users(id)") - conn.commit() + self.check_requirements(conn) + + def _offsets(self, total: int) -> list[int]: + """The thesis is about page *number*, so the offset is what must vary. + + Every offset must leave a full page behind it, otherwise both queries + return zero rows and the chart measures nothing. + """ + usable = total - PAGE_SIZE + if usable < DATA_POINTS: + raise BenchmarkNotApplicable( + f"pagination needs more than {PAGE_SIZE + DATA_POINTS} rows, found {total}" + ) + step = usable // (DATA_POINTS - 1) + return [step * i for i in range(DATA_POINTS)] def run(self, conn) -> BenchmarkResult: - page_sizes = [100, 500, 1000, 5000, 10000] + offsets = self._offsets(table_row_count(conn)) - q1_times, q2_times = [], [] - q1_rows, q2_rows = [], [] + q1_times, q1_rows = [], [] + q2_times, q2_rows = [], [] with conn.cursor() as cur: - for page_size in page_sizes: - offset = 500000 - - query_offset = "SELECT * FROM users ORDER BY id LIMIT %s OFFSET %s" - rows, t1 = self._measure(query_offset, cur, (page_size, offset)) - q1_times.append(t1) + for offset in offsets: + rows, elapsed = measure(cur, QUERY_OFFSET, (PAGE_SIZE, offset)) + q1_times.append(elapsed) q1_rows.append(rows) - query_keyset = "SELECT * FROM users WHERE id > %s ORDER BY id LIMIT %s" - rows, t2 = self._measure(query_keyset, cur, (offset, page_size)) - q2_times.append(t2) + rows, elapsed = measure(cur, QUERY_KEYSET, (offset, PAGE_SIZE)) + q2_times.append(elapsed) q2_rows.append(rows) q1 = QueryResult( - name="OFFSET", query="SELECT * FROM users ORDER BY id LIMIT %s OFFSET %s", - times=q1_times, limits=page_sizes, rows_fetched=q1_rows, + name="OFFSET", query=QUERY_OFFSET, + times=q1_times, limits=offsets, rows_fetched=q1_rows, ) q2 = QueryResult( - name="Keyset (WHERE id >)", query="SELECT * FROM users WHERE id > %s ORDER BY id LIMIT %s", - times=q2_times, limits=page_sizes, rows_fetched=q2_rows, + name="Keyset (WHERE id >)", query=QUERY_KEYSET, + times=q2_times, limits=offsets, rows_fetched=q2_rows, ) - comparison = self._build_comparison(q1, q2) - plot = self._build_plot(q1, q2) - - explain_plans = { - q1.name: run_explain(conn, "SELECT * FROM users ORDER BY id LIMIT %s OFFSET %s", (100, 500000)), - q2.name: run_explain(conn, "SELECT * FROM users WHERE id > %s ORDER BY id LIMIT %s", (500000, 100)), - } - + deepest = offsets[-1] return BenchmarkResult( name=self.name, title=self.title, description=self.description, - queries=[q1, q2], comparison_table=comparison, plot_buffer=plot, - explain_plans=explain_plans, + queries=[q1, q2], + comparison_table=self._build_comparison(q1, q2), + plot_buffer=self._build_plot(q1, q2), + explain_plans={ + q1.name: run_explain(conn, QUERY_OFFSET, (PAGE_SIZE, deepest)), + q2.name: run_explain(conn, QUERY_KEYSET, (deepest, PAGE_SIZE)), + }, ) - def teardown(self, conn) -> None: - with conn.cursor() as cur: - cur.execute("DROP INDEX IF EXISTS idx_users_id_cursor") - conn.commit() - - def _measure(self, query, cursor, params=None): - start = time.perf_counter() - cursor.execute(query, params) - rows = cursor.fetchall() - elapsed = (time.perf_counter() - start) * 1000 - return len(rows), elapsed - def _build_comparison(self, q1, q2): rows = [] - for i, limit in enumerate(q1.limits): - diff = q1.times[i] - q2.times[i] + for i, offset in enumerate(q1.limits): + speedup = f"{q1.times[i] / q2.times[i]:.1f}x" if q2.times[i] > 0 else "N/A" rows.append({ - "page_size": limit, - "offset": f"{q1.times[i]:.2f} ms", - "keyset": f"{q2.times[i]:.2f} ms", - "diff": f"{diff:.2f} ms", + "offset": offset, + "page": offset // PAGE_SIZE + 1, + "rows_fetched": q1.rows_fetched[i], + "offset_ms": f"{q1.times[i]:.2f} ms", + "keyset_ms": f"{q2.times[i]:.2f} ms", + "speedup": speedup, }) return pd.DataFrame(rows) @@ -98,9 +106,9 @@ def _build_plot(self, q1, q2): ax = fig.add_subplot(111) ax.plot(q1.limits, q1.times, marker="o", label=q1.name, color="#e74c3c") ax.plot(q2.limits, q2.times, marker="s", label=q2.name, color="#2ecc71") - ax.set_title(self.title) - ax.set_xlabel("Page size (OFFSET at 500K)") - ax.set_ylabel("Execution Time (ms)") + ax.set_title(f"{self.title} (page size {PAGE_SIZE})") + ax.set_xlabel("OFFSET (rows skipped)") + ax.set_ylabel("Median execution time (ms)") ax.set_xticks(q1.limits) ax.grid(True, linestyle="--", alpha=0.6) ax.legend() diff --git a/benchmarks/registry.py b/benchmarks/registry.py index ea29877..96f9d04 100644 --- a/benchmarks/registry.py +++ b/benchmarks/registry.py @@ -4,7 +4,6 @@ from benchmarks.base import BenchmarkBase - _registry: dict[str, BenchmarkBase] = {} diff --git a/benchmarks/select_star.py b/benchmarks/select_star.py index f02038b..484cd30 100644 --- a/benchmarks/select_star.py +++ b/benchmarks/select_star.py @@ -1,14 +1,24 @@ -import time import io + import matplotlib + matplotlib.use("Agg") -from matplotlib.figure import Figure -from matplotlib.backends.backend_agg import FigureCanvasAgg import pandas as pd +from matplotlib.backends.backend_agg import FigureCanvasAgg +from matplotlib.figure import Figure -from benchmarks.base import BenchmarkBase, BenchmarkResult, QueryResult +from app.config import QUERIES_DIR, QUERY_1_NAME, QUERY_2_NAME +from benchmarks.base import ( + BenchmarkBase, + BenchmarkNotApplicable, + BenchmarkResult, + QueryResult, + measure, + table_row_count, +) from benchmarks.registry import register -from app.config import START, STEP, QUERY_1_NAME, QUERY_2_NAME, TMP_DIR, QUERIES_DIR + +DATA_POINTS = 10 @register @@ -22,19 +32,21 @@ def _get_query(self, filename: str) -> str: with open(QUERIES_DIR / filename) as f: return f.read() - def _measure(self, query: str, limit: int, cursor) -> tuple[list, float]: - start = time.perf_counter() - cursor.execute(query, (limit,)) - rows = cursor.fetchall() - elapsed = (time.perf_counter() - start) * 1000 - return rows, elapsed + def _limits(self, total: int) -> list[int]: + """LIMITs must stay inside the table: a LIMIT above the row count + returns the whole table every time and flattens the curve into noise.""" + step = total // DATA_POINTS + if step < 1: + raise BenchmarkNotApplicable( + f"select_star needs at least {DATA_POINTS} rows, found {total}" + ) + return [step * i for i in range(1, DATA_POINTS + 1)] def setup(self, conn) -> None: - TMP_DIR.mkdir(parents=True, exist_ok=True) + self.check_requirements(conn) def run(self, conn) -> BenchmarkResult: - matplotlib.use("Agg", force=True) - limits = list(range(START, 0, -STEP)) + limits = self._limits(table_row_count(conn)) query_1 = self._get_query("query_1.sql") query_2 = self._get_query("query_2.sql") @@ -43,12 +55,12 @@ def run(self, conn) -> BenchmarkResult: with conn.cursor() as cursor: for limit in limits: - rows_1, t1 = self._measure(query_1, limit, cursor) - rows_2, t2 = self._measure(query_2, limit, cursor) + rows_1, t1 = measure(cursor, query_1, (limit,)) + rows_2, t2 = measure(cursor, query_2, (limit,)) q1_times.append(t1) q2_times.append(t2) - q1_rows.append(len(rows_1) if rows_1 else 0) - q2_rows.append(len(rows_2) if rows_2 else 0) + q1_rows.append(rows_1) + q2_rows.append(rows_2) q1 = QueryResult( name=QUERY_1_NAME, query=query_1, @@ -59,21 +71,20 @@ def run(self, conn) -> BenchmarkResult: times=q2_times, limits=limits, rows_fetched=q2_rows, ) - comparison = self._build_comparison(q1, q2) - plot = self._build_plot(q1, q2) - return BenchmarkResult( name=self.name, title=self.title, description=self.description, - queries=[q1, q2], comparison_table=comparison, plot_buffer=plot, + queries=[q1, q2], + comparison_table=self._build_comparison(q1, q2), + plot_buffer=self._build_plot(q1, q2), ) def _build_comparison(self, q1: QueryResult, q2: QueryResult) -> pd.DataFrame: rows = [] for i, limit in enumerate(q1.limits): - diff = q1.times[i] - q2.times[i] speedup = f"{q1.times[i] / q2.times[i]:.1f}x" if q2.times[i] > 0 else "N/A" rows.append({ "limit": limit, + "rows_fetched": q1.rows_fetched[i], "select_star": f"{q1.times[i]:.2f} ms", "select_columns": f"{q2.times[i]:.2f} ms", "speedup": speedup, @@ -84,11 +95,11 @@ def _build_plot(self, q1: QueryResult, q2: QueryResult) -> io.BytesIO: fig = Figure(figsize=(10, 6)) canvas = FigureCanvasAgg(fig) ax = fig.add_subplot(111) - ax.plot(q1.limits, q1.times, marker="o", label=q1.name) - ax.plot(q2.limits, q2.times, marker="s", label=q2.name) - ax.set_title(self.title) - ax.set_xlabel("LIMIT") - ax.set_ylabel("Execution Time (ms)") + ax.plot(q1.limits, q1.times, marker="o", label=q1.name, color="#e74c3c") + ax.plot(q2.limits, q2.times, marker="s", label=q2.name, color="#2ecc71") + ax.set_title(f"{self.title} (median of {len(q1.limits)} points)") + ax.set_xlabel("LIMIT (rows fetched)") + ax.set_ylabel("Median execution time (ms)") ax.set_xticks(q1.limits) ax.grid(True, linestyle="--", alpha=0.6) ax.legend() diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index c12c340..c185cad 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -1,14 +1,15 @@ import io +from unittest.mock import MagicMock + import pandas as pd -from unittest.mock import MagicMock, patch + +from app.benchmark import list_benchmarks, result_to_plot_data, run_explain from benchmarks.base import BenchmarkResult, QueryResult -from benchmarks.registry import all, get, default, _registry -from benchmarks.select_star import SelectStarBenchmark from benchmarks.index_usage import IndexUsageBenchmark from benchmarks.join_vs_subquery import JoinVsSubqueryBenchmark from benchmarks.pagination import PaginationBenchmark -from app.benchmark import run_benchmark, result_to_plot_data, list_benchmarks, run_explain -from app.config import QUERIES_DIR +from benchmarks.registry import all, default, get +from benchmarks.select_star import SelectStarBenchmark def test_registry_discovery(): @@ -90,9 +91,23 @@ def test_list_benchmarks(): def test_result_to_plot_data(): - q1 = QueryResult(name="Q1", query="SELECT 1", times=[1.0, 2.0], limits=[100, 200], rows_fetched=[100, 200]) - q2 = QueryResult(name="Q2", query="SELECT 2", times=[0.5, 1.0], limits=[100, 200], rows_fetched=[100, 200]) - comparison = pd.DataFrame({"limit": [100, 200], "q1": ["1.00 ms", "2.00 ms"], "q2": ["0.50 ms", "1.00 ms"]}) + q1 = QueryResult( + name="Q1", + query="SELECT 1", + times=[1.0, 2.0], + limits=[100, 200], + rows_fetched=[100, 200], + ) + q2 = QueryResult( + name="Q2", + query="SELECT 2", + times=[0.5, 1.0], + limits=[100, 200], + rows_fetched=[100, 200], + ) + comparison = pd.DataFrame( + {"limit": [100, 200], "q1": ["1.00 ms", "2.00 ms"], "q2": ["0.50 ms", "1.00 ms"]}, + ) buf = io.BytesIO() buf.write(b"\x89PNG") @@ -110,8 +125,20 @@ def test_result_to_plot_data(): def test_select_star_build_comparison(): bm = SelectStarBenchmark() - q1 = QueryResult(name="Q1", query="SELECT 1", times=[10.0, 5.0], limits=[1000, 500], rows_fetched=[1000, 500]) - q2 = QueryResult(name="Q2", query="SELECT 2", times=[2.0, 1.0], limits=[1000, 500], rows_fetched=[1000, 500]) + q1 = QueryResult( + name="Q1", + query="SELECT 1", + times=[10.0, 5.0], + limits=[1000, 500], + rows_fetched=[1000, 500], + ) + q2 = QueryResult( + name="Q2", + query="SELECT 2", + times=[2.0, 1.0], + limits=[1000, 500], + rows_fetched=[1000, 500], + ) table = bm._build_comparison(q1, q2) @@ -123,8 +150,20 @@ def test_select_star_build_comparison(): def test_select_star_build_plot(): bm = SelectStarBenchmark() - q1 = QueryResult(name="Q1", query="SELECT 1", times=[1.0, 2.0], limits=[100, 200], rows_fetched=[100, 200]) - q2 = QueryResult(name="Q2", query="SELECT 2", times=[0.5, 1.0], limits=[100, 200], rows_fetched=[100, 200]) + q1 = QueryResult( + name="Q1", + query="SELECT 1", + times=[1.0, 2.0], + limits=[100, 200], + rows_fetched=[100, 200], + ) + q2 = QueryResult( + name="Q2", + query="SELECT 2", + times=[0.5, 1.0], + limits=[100, 200], + rows_fetched=[100, 200], + ) buf = bm._build_plot(q1, q2) @@ -135,8 +174,20 @@ def test_select_star_build_plot(): def test_index_usage_build_plot(): bm = IndexUsageBenchmark() - q1 = QueryResult(name="No idx", query="SELECT 1", times=[10.0, 5.0], limits=[20, 30], rows_fetched=[1000, 500]) - q2 = QueryResult(name="Idx", query="SELECT 2", times=[2.0, 1.0], limits=[20, 30], rows_fetched=[1000, 500]) + q1 = QueryResult( + name="No idx", + query="SELECT 1", + times=[10.0, 5.0], + limits=[20, 30], + rows_fetched=[1000, 500], + ) + q2 = QueryResult( + name="Idx", + query="SELECT 2", + times=[2.0, 1.0], + limits=[20, 30], + rows_fetched=[1000, 500], + ) buf = bm._build_plot(q1, q2) @@ -158,7 +209,13 @@ def test_join_vs_subquery_build_plot(): def test_pagination_build_plot(): bm = PaginationBenchmark() - q1 = QueryResult(name="OFFSET", query="SELECT 1", times=[10.0], limits=[100], rows_fetched=[100]) + q1 = QueryResult( + name="OFFSET", + query="SELECT 1", + times=[10.0], + limits=[100], + rows_fetched=[100], + ) q2 = QueryResult(name="Keyset", query="SELECT 2", times=[2.0], limits=[100], rows_fetched=[100]) buf = bm._build_plot(q1, q2) diff --git a/tests/test_integration.py b/tests/test_integration.py new file mode 100644 index 0000000..274fee6 --- /dev/null +++ b/tests/test_integration.py @@ -0,0 +1,182 @@ +"""Integration tests against a real PostgreSQL instance. + +Skipped automatically when no database is reachable, so the unit suite still +runs offline. CI always provides one, so these gates are never silently skipped +there. +""" +import os + +import pytest + +from benchmarks import all as all_benchmarks +from benchmarks.registry import get +from sql.seed import SIZES, row_count, seed + + +def _connect(): + import psycopg2 + + return psycopg2.connect( + host=os.getenv("TEST_DB_HOST", "localhost"), + port=int(os.getenv("TEST_DB_PORT", "5432")), + user=os.getenv("TEST_DB_USER", "user"), + password=os.getenv("TEST_DB_PASSWORD", "password"), + database=os.getenv("TEST_DB_NAME", "test_db"), + ) + + +@pytest.fixture(scope="module") +def conn(): + try: + connection = _connect() + except Exception as exc: + pytest.skip(f"no PostgreSQL available: {exc}") + seed(connection, "small") + yield connection + connection.close() + + +def test_seed_creates_exactly_the_requested_number_of_rows(conn): + seed(conn, "small") + assert row_count(conn) == SIZES["small"] + + +def test_seed_leaves_no_null_values_in_generated_columns(conn): + with conn.cursor() as cur: + cur.execute( + "SELECT count(*) FROM users " + "WHERE name IS NULL OR surname IS NULL " + "OR city IS NULL OR country IS NULL" + ) + assert cur.fetchone()[0] == 0 + + +@pytest.mark.parametrize("name", [bm.name for bm in all_benchmarks()]) +def test_every_benchmark_measures_a_non_empty_result_set(conn, name): + """A benchmark that fetches zero rows is measuring nothing.""" + bm = get(name) + bm.setup(conn) + try: + result = bm.run(conn) + finally: + bm.teardown(conn) + + for query in result.queries: + assert all(r > 0 for r in query.rows_fetched), ( + f"{name}/{query.name} fetched no rows: {query.rows_fetched}" + ) + + +def test_select_star_derives_its_limits_from_the_real_row_count(conn): + """A LIMIT above the row count makes every data point identical.""" + total = row_count(conn) + bm = get("select_star") + bm.setup(conn) + try: + result = bm.run(conn) + finally: + bm.teardown(conn) + + for query in result.queries: + assert max(query.limits) <= total + assert len(set(query.rows_fetched)) > 1, ( + f"every LIMIT returned the same {query.rows_fetched[0]} rows" + ) + + +def test_index_usage_reports_two_different_query_plans(conn): + """The 'no index' plan must be captured before the index exists.""" + bm = get("index_usage") + bm.setup(conn) + try: + result = bm.run(conn) + finally: + bm.teardown(conn) + + plans = result.explain_plans + no_index = next(v for k, v in plans.items() if "no index" in k.lower()) + with_index = next(v for k, v in plans.items() if "no index" not in k.lower()) + + assert "Seq Scan" in no_index + assert "Index" in with_index + assert no_index != with_index + + +def test_pagination_varies_the_offset_not_the_page_size(conn): + """The thesis is that OFFSET degrades with page *number*.""" + bm = get("pagination") + bm.setup(conn) + try: + result = bm.run(conn) + finally: + bm.teardown(conn) + + offsets = result.queries[0].limits + assert len(set(offsets)) > 1, "offset must vary across data points" + assert max(offsets) < row_count(conn) + + +def test_join_benchmark_leaves_a_pre_existing_orders_table_untouched(conn): + """The tool must never drop a table it did not create.""" + with conn.cursor() as cur: + cur.execute("DROP TABLE IF EXISTS orders") + cur.execute("CREATE TABLE orders (id SERIAL PRIMARY KEY, note TEXT)") + cur.execute("INSERT INTO orders (note) VALUES ('user data')") + conn.commit() + + bm = get("join_vs_subquery") + bm.setup(conn) + try: + bm.run(conn) + finally: + bm.teardown(conn) + + with conn.cursor() as cur: + cur.execute("SELECT note FROM orders") + assert cur.fetchall() == [("user data",)] + cur.execute("DROP TABLE orders") + conn.commit() + + +def test_generate_route_seeds_the_requested_dataset_size(conn): + """The size selector must change the data, not just the label.""" + from app import create_app + from app.config import Config + + class RouteConfig(Config): + def __init__(self): + super().__init__() + self.DB_HOST = os.getenv("TEST_DB_HOST", "localhost") + self.DB_PORT = int(os.getenv("TEST_DB_PORT", "5432")) + self.DB_USER = os.getenv("TEST_DB_USER", "user") + self.DB_PASSWORD = os.getenv("TEST_DB_PASSWORD", "password") + self.DB_NAME = os.getenv("TEST_DB_NAME", "test_db") + + seed(conn, "medium") + assert row_count(conn) == SIZES["medium"] + conn.commit() # release ACCESS SHARE so the route's TRUNCATE can proceed + + client = create_app(RouteConfig).test_client() + response = client.get("/generate?benchmark=select_star&size=small") + + assert response.status_code == 200 + assert row_count(conn) == SIZES["small"] + + +def test_seed_fails_fast_instead_of_hanging_on_a_locked_table(conn): + """A blocked TRUNCATE would otherwise hang a gunicorn worker forever.""" + import psycopg2 + + seed(conn, "small") + conn.commit() + + blocker = _connect() + with blocker.cursor() as cur: + cur.execute("SELECT count(*) FROM users") # holds ACCESS SHARE + try: + other = _connect() + with pytest.raises(psycopg2.errors.LockNotAvailable): + seed(other, "medium") + other.close() + finally: + blocker.close() diff --git a/tests/test_measure.py b/tests/test_measure.py new file mode 100644 index 0000000..6d46286 --- /dev/null +++ b/tests/test_measure.py @@ -0,0 +1,56 @@ +import pytest + +from benchmarks.base import REPETITIONS, measure + + +class FakeCursor: + """Records every execute so we can assert on warm-up and repetitions.""" + + def __init__(self, rows=3): + self.executed = [] + self._rows = [(i,) for i in range(rows)] + + def execute(self, query, params=None): + self.executed.append((query, params)) + + def fetchall(self): + return self._rows + + +def test_measure_discards_a_warmup_run_before_timing(): + cursor = FakeCursor() + + measure(cursor, "SELECT 1", repetitions=3) + + assert len(cursor.executed) == 4, "expected 1 warm-up + 3 timed runs" + + +def test_measure_returns_the_number_of_rows_fetched(): + cursor = FakeCursor(rows=7) + + rows, _ = measure(cursor, "SELECT 1") + + assert rows == 7 + + +def test_measure_passes_params_through_on_every_run(): + cursor = FakeCursor() + + measure(cursor, "SELECT %s", params=(42,), repetitions=2) + + assert all(params == (42,) for _, params in cursor.executed) + + +def test_measure_reports_the_median_not_the_first_run(): + """One slow cold run must not drag the reported time; mean would give 20.8ms.""" + cursor = FakeCursor() + # start/end pairs in seconds -> deltas of 100ms, 1ms, 1ms, 1ms, 1ms + ticks = iter([0, 0.100, 0, 0.001, 0, 0.001, 0, 0.001, 0, 0.001]) + + _, elapsed = measure(cursor, "SELECT 1", repetitions=5, clock=lambda: next(ticks)) + + assert elapsed == pytest.approx(1.0) + + +def test_default_repetitions_is_more_than_one(): + assert REPETITIONS > 1 From af57f4a7c49c80500eacc23ceeadc018314dae33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20S=C3=A1nchez?= <100514206+jonaas-dev@users.noreply.github.com> Date: Sun, 6 Sep 2026 15:59:41 +0200 Subject: [PATCH 2/5] feat(seed): make the dataset size selector actually change the data The size parameter was validated, stored in metadata and rendered in the UI, but never reached the seeder: init.sql hardcoded 1M rows in a PL/pgSQL loop, sql/seed.py was never invoked by anything, and DB_SEED_SIZE was read into Config and unused. Every stored result recorded a size that had never been applied. - init.sql is now schema only; seeding belongs to sql/seed.py. - seed() is set-based (INSERT ... SELECT generate_series) instead of a row-by-row loop, which takes 1M rows from minutes to seconds and makes reseeding viable from a web request. - seed() is idempotent on the exact row count, so switching size reseeds in both directions. - A lock_timeout guards the TRUNCATE: a concurrent reader would otherwise block it forever, and a client-side timeout cannot interrupt libpq waiting on the socket, so the worker would hang. Also fixes an off-by-one in the old init.sql array indexing: casting a float to INT in PostgreSQL rounds rather than truncates, so `(random() * 10)::INT + 1` produced index 11 on a 10-element array. Measured over 100k samples, 4.9% of rows got NULL for name, surname, city and country. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_014abw4B6YUf54giaEyPQpbo --- app/config.py | 5 -- app/templates/results.html | 9 ++- sql/__init__.py | 0 sql/init.sql | 31 +--------- sql/seed.py | 117 ++++++++++++++++++++++++------------- 5 files changed, 85 insertions(+), 77 deletions(-) create mode 100644 sql/__init__.py diff --git a/app/config.py b/app/config.py index f4bde16..b846a47 100644 --- a/app/config.py +++ b/app/config.py @@ -1,13 +1,9 @@ import os from pathlib import Path - BASE_DIR = Path(__file__).resolve().parent.parent QUERIES_DIR = BASE_DIR / 'queries' -TMP_DIR = BASE_DIR / 'executions_tmp' -START = 1000000 -STEP = 100000 QUERY_1_NAME = 'SELECT *' QUERY_2_NAME = 'SELECT id, name, email' @@ -20,5 +16,4 @@ def __init__(self): self.DB_PASSWORD = os.getenv('DB_PASSWORD', 'password') self.DB_NAME = os.getenv('DB_NAME', 'test_db') self.DB_SEED_SIZE = os.getenv('DB_SEED_SIZE', 'medium') - self.SECRET_KEY = os.getenv('SECRET_KEY', 'dev-secret-change-in-prod') self.TESTING = False diff --git a/app/templates/results.html b/app/templates/results.html index d07f4d5..f7e120d 100644 --- a/app/templates/results.html +++ b/app/templates/results.html @@ -51,10 +51,13 @@

Find out why your queries are slow

Dataset +
Changing the size reseeds the users table.