diff --git a/.dockerignore b/.dockerignore index 4c8d40a..562f5de 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,18 +1,21 @@ -__pycache__/ -*.pyc -*.pyo .git/ +.github/ .gitignore +.dockerignore .env -.env.example .venv/ -executions_tmp/ -executions/ +.pytest_cache/ +.ruff_cache/ +__pycache__/ +*.pyc +*.pyo +results/ tests/ +conftest.py requirements-dev.txt -pyproject.toml README.md +CONTRIBUTING.md +SECURITY.md LICENSE -.dockerignore Dockerfile docker-compose.yml diff --git a/.env.example b/.env.example index 59df30d..514ea03 100644 --- a/.env.example +++ b/.env.example @@ -3,3 +3,9 @@ DB_PORT=5432 DB_USER=user DB_PASSWORD=password DB_NAME=test_db + +# small (10k) | medium (100k) | large (1M) +DB_SEED_SIZE=medium + +# DEBUG | INFO | WARNING | ERROR +LOG_LEVEL=INFO diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c60cefe --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,58 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + + services: + postgres: + image: postgres:17 + env: + POSTGRES_USER: user + POSTGRES_PASSWORD: password + POSTGRES_DB: test_db + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 5s + --health-timeout 5s + --health-retries 10 + + env: + TEST_DB_HOST: localhost + TEST_DB_PORT: 5432 + TEST_DB_USER: user + TEST_DB_PASSWORD: password + TEST_DB_NAME: test_db + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version-file: .python-version + cache: pip + + - name: Install dependencies + run: pip install -r requirements.txt -r requirements-dev.txt + + - name: Create schema + run: | + # pg_isready answers before the POSTGRES_DB bootstrap finishes, so + # wait on the database itself rather than on the server. + until psql "$DSN" -c 'SELECT 1' >/dev/null 2>&1; do sleep 1; done + psql "$DSN" -f sql/init.sql + env: + DSN: postgresql://user:password@localhost:5432/test_db + + - name: Lint + run: ruff check . + + - name: Test + run: pytest --cov --cov-report=term-missing diff --git a/.gitignore b/.gitignore index ae44d7c..3080e75 100644 --- a/.gitignore +++ b/.gitignore @@ -173,3 +173,11 @@ cython_debug/ # PyPI configuration file .pypirc + +# macOS +.DS_Store + +# Tool caches created by the container at runtime +.matplotlib/ +.gunicorn/ +.ruff_cache/ diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..e4fba21 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..8137a84 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,41 @@ +# Contributing + +## Setup + +```bash +python -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt -r requirements-dev.txt +docker compose up -d db +psql "postgresql://user:password@localhost:5432/test_db" -f sql/init.sql +``` + +## Before opening a PR + +```bash +ruff check . +pytest --cov +``` + +Both run in CI against a real PostgreSQL service, so integration tests will not be skipped there. + +## Adding a benchmark + +See [Adding your own benchmark](README.md#adding-your-own-benchmark). The rules that matter: + +- **Use `measure()`** from `benchmarks.base` — it does the warm-up and the median. Hand-rolled + `time.perf_counter()` around a single run reintroduces the cold-cache bias. +- **Derive your data points from `table_row_count(conn)`**, never hardcode them. A benchmark whose + parameters exceed the dataset silently measures an empty result set. +- **Prefix any object you create with `sqlperf_`** and drop it in `teardown()`. The tool must never + destroy a table it did not create. +- **Raise `BenchmarkNotApplicable`** when the dataset is too small for your benchmark to mean + anything, instead of returning a chart built on nothing. + +Every benchmark is automatically covered by the integration suite, which asserts that it fetches a +non-empty result set. If your benchmark cannot satisfy that, it is measuring the wrong thing. + +## Commits + +[Conventional Commits](https://www.conventionalcommits.org/): `feat:`, `fix:`, `chore:`, `docs:`, +`test:`. Keep PRs small. diff --git a/Dockerfile b/Dockerfile index 4bfe0ee..03cee75 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,13 +9,19 @@ RUN pip install --no-cache-dir -r requirements.txt COPY . . -RUN chown -R appuser:appuser /app +# results/ is a mounted volume; it must exist and be writable before the +# volume inherits its ownership. +RUN mkdir -p /app/results && chown -R appuser:appuser /app USER appuser +ENV HOME=/tmp \ + MPLCONFIGDIR=/tmp/matplotlib \ + PYTHONUNBUFFERED=1 + EXPOSE 8000 HEALTHCHECK --interval=30s --timeout=10s --retries=3 \ CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8000/')" || exit 1 -CMD ["gunicorn", "--workers", "3", "--bind", "0.0.0.0:8000", "wsgi:app"] +CMD ["gunicorn", "--workers", "3", "--timeout", "300", "--bind", "0.0.0.0:8000", "wsgi:app"] diff --git a/README.md b/README.md index a804104..610c5e6 100644 --- a/README.md +++ b/README.md @@ -1,143 +1,135 @@

🔍 SQL Performance Benchmark

- Identify and understand SQL performance issues — with real numbers and EXPLAIN ANALYZE + Measure SQL performance patterns on PostgreSQL — with honest numbers and real EXPLAIN ANALYZE output

Python License Docker - Tests + CI

--- ## What is this? -A toolkit for **detecting and understanding SQL performance problems** — not just measuring them. +A toolkit for **measuring common SQL performance patterns** against a real PostgreSQL database. -Every query has a cost. Some are obvious (`SELECT *` on a wide table), some are subtle (`OFFSET` pagination that degrades under load), and some depend on context (`JOIN` vs `subquery` vs `EXISTS`). This tool runs real benchmarks against PostgreSQL, measures the actual impact, and shows you **why** one approach is faster with `EXPLAIN ANALYZE`. - -**Built for developers who want data, not opinions.** +Every query has a cost. Some are obvious (`SELECT *` on a wide table), some are subtle (`OFFSET` +pagination that degrades with page depth), and some depend entirely on context (`JOIN` vs `subquery` +vs `EXISTS`). This tool runs the queries, times them, and shows the query plan behind each result.

Landing page
- Choose a benchmark, select dataset size, and click Generate + Choose a benchmark and a dataset size, then click Generate

--- -## Benchmark examples +## How the measurements work + +The numbers are only worth something if the method is. This is what the tool does on every data point: + +| | Why | +|---|---| +| **One discarded warm-up run** | Without it, whichever query runs first pays for the cold cache. That alone was enough to make the "slow" query look slow. | +| **Median of 5 timed runs** | A single sample is dominated by scheduler noise. The median ignores the outlier that a mean would carry. | +| **Data points derived from the real row count** | A `LIMIT` above the table size returns the whole table every time, which flattens the curve into a straight line of noise. | +| **`rows_fetched` shown in every table** | So you can see the query actually returned something. A benchmark over an empty result set measures nothing. | +| **Timing includes `fetchall()`** | The cost of `SELECT *` is largely transferring and materialising the columns, so the client-side fetch is part of what is being measured. | + +### What this is not -The toolkit includes several built-in benchmarks that demonstrate common performance patterns. Each one tests a specific scenario, measures execution time across different data sizes, and explains the query plan. +These are **wall-clock timings on a synthetic dataset in a container**, not a claim about your +production database. Row counts, hardware, PostgreSQL version, `work_mem`, concurrency and data +distribution all move these numbers. Use the tool to see *why* a plan changes — read the +`EXPLAIN ANALYZE` output, not just the chart. --- -### 📊 SELECT * vs SELECT columns +## Benchmarks -> *The most common performance trap* +### 📊 SELECT * vs SELECT columns ```sql --- Fetches all 16 columns -SELECT * FROM users LIMIT 1000000; - --- Fetches only 3 columns -SELECT id, name, email FROM users LIMIT 1000000; +SELECT * FROM users LIMIT %s; +SELECT id, name, email FROM users LIMIT %s; ``` -**Why it matters**: Every extra column adds I/O, memory, and network overhead. On a table with 16 columns and 1M rows, `SELECT *` transfers **16x more data** than selecting specific columns. +The `users` table has 16 columns, one of them a `TEXT` bio that dominates the row width. Selecting 3 +narrow columns instead of all 16 cuts the bytes PostgreSQL has to read, materialise and ship to the +client. The LIMITs sweep from 10% to 100% of the table, so the curve reflects a growing result set +rather than the same query run ten times.

SELECT * vs columns benchmark -
- Chart shows execution time comparison with EXPLAIN ANALYZE output

---- - ### ⚡ Index usage -> *Why B-tree indexes are not optional* - ```sql --- Full table scan (no index) -SELECT * FROM users WHERE age > 30; - --- Index scan (after CREATE INDEX) -SELECT * FROM users WHERE age > 30; +SELECT * FROM users WHERE age = %s; -- before CREATE INDEX +SELECT * FROM users WHERE age = %s; -- after CREATE INDEX ``` -**Why it matters**: Without an index, PostgreSQL reads **every row** in the table. With a B-tree index on the filtered column, it jumps directly to matching rows. +Same query, twice: once with no index on `age`, then again after `CREATE INDEX`. **The "no index" +plan is captured before the index exists** — otherwise both `EXPLAIN` runs report the same indexed +plan and the label lies about what you are looking at. + +Note that on a small dataset PostgreSQL may legitimately still choose a sequential scan: reading +10,000 rows is cheaper than an index lookup plus heap fetches. That is the planner being right, not +the benchmark being broken — pick a larger dataset size to see the crossover.

Index usage benchmark -
- Red line: without index (Seq Scan). Green line: with B-tree index (Index Scan)

---- - ### 🔗 JOIN vs subquery vs EXISTS -> *Three ways to filter related data* - ```sql --- Pattern 1: JOIN (when you need columns from both tables) -SELECT u.id, u.name, o.amount -FROM users u INNER JOIN orders o ON u.id = o.user_id -WHERE o.amount > 100; - --- Pattern 2: IN subquery (when you only need the main table) -SELECT id, name FROM users -WHERE id IN (SELECT user_id FROM orders WHERE amount > 100); - --- Pattern 3: EXISTS (when you only need to check existence) -SELECT id, name FROM users u -WHERE EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id AND o.amount > 100); +SELECT u.id, u.name, u.email, o.amount +FROM users u INNER JOIN sqlperf_orders o ON u.id = o.user_id +WHERE o.amount > %s; + +SELECT id, name, email FROM users +WHERE id IN (SELECT user_id FROM sqlperf_orders WHERE amount > %s); + +SELECT id, name, email FROM users u +WHERE EXISTS (SELECT 1 FROM sqlperf_orders o WHERE o.user_id = u.id AND o.amount > %s); ``` -**Why it matters**: Each pattern has different performance characteristics depending on data distribution, indexes, and result set size. There is no universal "fastest" — only fastest **for your case**. +Each pattern has different characteristics depending on data distribution, indexes and result set +size. There is no universal "fastest" — often the planner rewrites `IN` and `EXISTS` into the same +plan, which the `EXPLAIN` output will show you directly.

JOIN vs subquery benchmark -
- Three patterns compared across different filtering thresholds

---- - ### 📄 OFFSET vs keyset pagination -> *Why `OFFSET 500000` is slow* - ```sql --- OFFSET: must scan and discard all skipped rows -SELECT * FROM users ORDER BY id LIMIT 10 OFFSET 500000; - --- Keyset: jumps directly to position -SELECT * FROM users WHERE id > 500000 ORDER BY id LIMIT 10; +SELECT * FROM users ORDER BY id LIMIT 100 OFFSET %s; -- scans and discards +SELECT * FROM users WHERE id > %s ORDER BY id LIMIT 100; -- jumps straight there ``` -**Why it matters**: OFFSET pagination degrades linearly with page number. At page 50,000, PostgreSQL reads 500K rows just to discard them. Keyset pagination stays **constant** regardless of position. +The page size is held at 100 and **the offset is what varies**, from the first page to the deepest +one the dataset allows. That is the whole point: `OFFSET` has to walk and throw away every skipped +row, so its cost grows with page *depth*, while keyset pagination stays flat.

Pagination benchmark -
- Red: OFFSET degrades. Green: Keyset stays constant

---- - ### 📜 Historical results -Every benchmark run is saved with timestamp. Compare results over time via the `/history` endpoint. +Every run is saved with a timestamp under `results/` and browsable at `/history`.

History page -
- All benchmark runs saved for comparison

--- @@ -148,18 +140,29 @@ Every benchmark run is saved with timestamp. Compare results over time via the ` git clone https://github.com/jonaas-dev/sql-performance.git cd sql-performance cp .env.example .env -docker-compose up --build +docker compose up --build ``` -Open **http://localhost:8000**, select a benchmark, choose dataset size, and click **Generate**. +Open **http://localhost:8000**. + +Compose brings up PostgreSQL, seeds it to `DB_SEED_SIZE`, and only then starts the app. The database +and the app are both bound to `127.0.0.1`, so nothing is exposed outside your machine. ### Dataset sizes -| Size | Rows | Use case | -|------|------|----------| -| `small` | 10,000 | Development, quick tests | -| `medium` | 100,000 | Daily benchmarking | -| `large` | 1,000,000 | Serious performance testing | +| Size | Rows | Seed time | +|------|------|-----------| +| `small` | 10,000 | ~1s | +| `medium` | 100,000 | ~2s | +| `large` | 1,000,000 | ~15s | + +Set the initial size with `DB_SEED_SIZE` in `.env`. **Changing the size in the web UI reseeds the +`users` table** — it truncates and regenerates the data so the selector reflects reality rather than +being a label on an unchanged dataset. + +> ⚠️ **The tool writes to the database it connects to.** It truncates `users` when reseeding, and +> creates and drops `sqlperf_orders` and `sqlperf_idx_users_age` around the relevant benchmarks. +> Point it at a throwaway database, never at one with data you care about. --- @@ -169,14 +172,14 @@ Open **http://localhost:8000**, select a benchmark, choose dataset size, and cli sql-performance/ ├── app/ │ ├── __init__.py # Flask app factory -│ ├── config.py # Configuration from .env +│ ├── config.py # Configuration from environment │ ├── db.py # PostgreSQL connection │ ├── benchmark.py # Benchmark runner + EXPLAIN ANALYZE │ ├── history.py # Historical results storage (filesystem) │ ├── routes.py # Flask routes │ └── templates/ # Jinja2 templates ├── benchmarks/ -│ ├── base.py # BenchmarkBase ABC — all benchmarks extend this +│ ├── base.py # BenchmarkBase ABC + the shared measure() helper │ ├── registry.py # Auto-discovery — drop a file, it's registered │ ├── select_star.py # SELECT * vs columns │ ├── index_usage.py # B-tree index impact @@ -184,73 +187,69 @@ sql-performance/ │ └── pagination.py # OFFSET vs keyset ├── queries/ # SQL files (loaded by benchmarks) ├── sql/ -│ ├── init.sql # DDL + seed data +│ ├── init.sql # Schema only │ └── seed.py # Parametrized seeder (small/medium/large) -├── tests/ # pytest tests (33 tests) +├── tests/ # pytest — unit + PostgreSQL integration ├── wsgi.py # Gunicorn entry point -├── docker-compose.yml # PostgreSQL + Flask app -└── Dockerfile # Production container +├── docker-compose.yml # PostgreSQL + seeder + app +└── Dockerfile ``` --- ## Adding your own benchmark -The plugin architecture makes it easy to add new benchmarks. Here's the full process: - -### Step 1: Create a benchmark file - -Create a new `.py` file in `benchmarks/` (e.g., `benchmarks/deadlock_demo.py`): +Drop a `.py` file in `benchmarks/`. The registry discovers it on import — no registration code. ```python -import time import io + import matplotlib matplotlib.use("Agg") -from matplotlib.figure import Figure -from matplotlib.backends.backend_agg import FigureCanvasAgg import pandas as pd +from matplotlib.backends.backend_agg import FigureCanvasAgg +from matplotlib.figure import Figure -from benchmarks.base import BenchmarkBase, BenchmarkResult, QueryResult +from benchmarks.base import ( + BenchmarkBase, BenchmarkResult, QueryResult, measure, table_row_count, +) from benchmarks.registry import register +QUERY = "SELECT * FROM users WHERE city = %s" + @register -class DeadlockDemo(BenchmarkBase): - name = "deadlock_demo" - title = "Deadlock detection patterns" - description = "Compare LOCK timeout vs row-level locking strategies" +class CityFilterBenchmark(BenchmarkBase): + name = "city_filter" + title = "Filtering by city with and without an index" + description = "Compare a seq scan against a B-tree index on a low-cardinality column" required_tables = ["users"] def setup(self, conn) -> None: - """Create indexes, temp tables, or test data before the benchmark.""" - with conn.cursor() as cur: - cur.execute( - "CREATE INDEX IF NOT EXISTS idx_users_city ON users(city)" - ) - conn.commit() + self.check_requirements(conn) # fails loudly if `users` is missing def run(self, conn) -> BenchmarkResult: - """Execute queries, measure times, build results.""" - cities = ["New York", "London", "Tokyo", "Berlin", "Sydney"] - times = [] + # Cities that actually exist in the seed data. + cities = ["New York", "Los Angeles", "Chicago", "Houston", "Phoenix"] + times, rows_fetched = [], [] with conn.cursor() as cur: for city in cities: - start = time.perf_counter() - cur.execute("SELECT * FROM users WHERE city = %s", (city,)) - cur.fetchall() - elapsed = (time.perf_counter() - start) * 1000 + rows, elapsed = measure(cur, QUERY, (city,)) # warm-up + median times.append(elapsed) + rows_fetched.append(rows) - queries = [QueryResult( + query = QueryResult( name="Filter by city", - query="SELECT * FROM users WHERE city = %s", - times=times, limits=cities, rows_fetched=1000, - )] + query=QUERY, + times=times, + limits=cities, # x-axis values + rows_fetched=rows_fetched, + ) comparison = pd.DataFrame({ "city": cities, + "rows_matched": rows_fetched, "time_ms": [round(t, 2) for t in times], }) @@ -259,9 +258,9 @@ class DeadlockDemo(BenchmarkBase): ax = fig.add_subplot(111) ax.plot(cities, times, marker="o") ax.set_xlabel("City") - ax.set_ylabel("Time (ms)") - ax.set_title("Deadlock Demo") + ax.set_ylabel("Median execution time (ms)") ax.grid(True, alpha=0.3) + fig.tight_layout() buf = io.BytesIO() canvas.print_png(buf) @@ -269,53 +268,41 @@ class DeadlockDemo(BenchmarkBase): return BenchmarkResult( name=self.name, title=self.title, description=self.description, - queries=queries, comparison_table=comparison, - plot_buffer=buf, explain_plans={}, + queries=[query], comparison_table=comparison, plot_buffer=buf, ) def teardown(self, conn) -> None: - """Clean up any created objects.""" - with conn.cursor() as cur: - cur.execute("DROP INDEX IF EXISTS idx_users_city") - conn.commit() + """Drop anything setup() created. Prefix objects with `sqlperf_`.""" ``` -### Step 2: That's it - -The registry auto-discovers new files in `benchmarks/`. No import, no registration code. Just drop the file and restart. - -### Step 3: What each method does +### The contract -| Method | Purpose | Examples | -|--------|---------|----------| -| `setup(conn)` | Prepare the database before measuring | Create indexes, temp tables, seed test data | -| `run(conn)` | Execute queries, measure times, return results | Run queries with `time.perf_counter()`, build plot | -| `teardown(conn)` | Clean up after the benchmark | Drop indexes, temp tables | - -### Step 4: Required attributes +| Method | Purpose | +|--------|---------| +| `setup(conn)` | Prepare the database. Call `self.check_requirements(conn)` first. | +| `run(conn)` | Execute queries with `measure()`, return a `BenchmarkResult`. | +| `teardown(conn)` | Drop anything `setup()` created. Optional. | | Attribute | Type | Purpose | |-----------|------|---------| -| `name` | `str` | Unique identifier (used in URL: `?benchmark=name`) | -| `title` | `str` | Human-readable name (shown in UI) | +| `name` | `str` | Unique id, used in the URL: `?benchmark=name` | +| `title` | `str` | Shown in the UI | | `description` | `str` | What this benchmark tests | -| `required_tables` | `list[str]` | Tables that must exist before setup | - -### Step 5: Return value +| `required_tables` | `list[str]` | Enforced by `check_requirements()` | -Your `run()` method must return a `BenchmarkResult` with: +`BenchmarkResult` carries `queries` (a list of `QueryResult`), a `comparison_table` DataFrame, a +`plot_buffer` PNG, and optionally `explain_plans`. When `explain_plans` is empty the runner collects +them automatically — supply them yourself when the plan must be captured at a specific moment, as +`index_usage` does. -- **`queries`**: List of `QueryResult` objects (name, query text, execution times, data points) -- **`comparison_table`**: A pandas DataFrame shown in the UI table -- **`plot_buffer`**: A `BytesIO` containing a PNG image (the chart) -- **`explain_plans`**: Dict of query name → EXPLAIN ANALYZE text (optional, auto-collected if empty) +**Helpers worth using:** `measure(cursor, query, params)` gives you the warm-up plus median for free, +and `table_row_count(conn)` lets you size your data points to the dataset instead of hardcoding them. +Raise `BenchmarkNotApplicable` when the dataset is too small for your benchmark to mean anything. --- ## Configuration -All settings via environment variables (see `.env.example`): - | Variable | Default | Description | |----------|---------|-------------| | `DB_HOST` | `localhost` | PostgreSQL host | @@ -323,7 +310,8 @@ All settings via environment variables (see `.env.example`): | `DB_USER` | `user` | Database user | | `DB_PASSWORD` | `password` | Database password | | `DB_NAME` | `test_db` | Database name | -| `DB_SEED_SIZE` | `medium` | Dataset size (`small`/`medium`/`large`) | +| `DB_SEED_SIZE` | `medium` | Initial dataset size (`small`/`medium`/`large`) | +| `LOG_LEVEL` | `INFO` | Python logging level | --- @@ -332,22 +320,32 @@ All settings via environment variables (see `.env.example`): | Route | Description | |-------|-------------| | `GET /` | Landing page with benchmark selector | -| `GET /generate?benchmark=&size=` | Run a benchmark | -| `GET /history` | List all historical benchmark runs | -| `GET /results/` | View a specific historical result | +| `GET /generate?benchmark=&size=` | Reseed if needed, then run a benchmark | +| `GET /history` | List historical runs (paginated) | +| `GET /results/` | View a stored result | --- -## Testing +## Development ```bash python -m venv .venv source .venv/bin/activate pip install -r requirements.txt -r requirements-dev.txt -pytest --cov=app + +ruff check . +pytest --cov ``` -**33 tests** covering registry discovery, benchmark metadata, plot generation, config, history, and routes. +The suite is split in two: + +- **Unit tests** run anywhere with no database. +- **Integration tests** (`tests/test_integration.py`) need PostgreSQL and are skipped when none is + reachable. They are the ones that guard the measurement contract — that every benchmark fetches + rows, that data points stay inside the dataset, and that the two index plans actually differ. + +Point them at a database with `TEST_DB_HOST`, `TEST_DB_PORT`, `TEST_DB_USER`, `TEST_DB_PASSWORD` and +`TEST_DB_NAME`. CI always provides one, so the integration gates never silently skip there. --- diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..e3973fd --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,17 @@ +# Security Policy + +## Scope + +This is a benchmarking tool meant to run locally against a throwaway database. It has no +authentication and is not designed to be exposed to a network. `docker compose` binds both +PostgreSQL and the app to `127.0.0.1` for that reason. + +**The tool writes to the database it connects to**: it truncates `users` when reseeding, and creates +and drops `sqlperf_orders` and `sqlperf_idx_users_age`. Never point it at a database holding data you +care about. + +## Reporting a vulnerability + +Open a [security advisory](../../security/advisories/new) rather than a public issue. + +Please include the version or commit, reproduction steps, and the impact you observed. diff --git a/app/__init__.py b/app/__init__.py index ae027aa..61ce609 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,6 +1,8 @@ import logging import os + import matplotlib + matplotlib.use("Agg") from flask import Flask @@ -18,22 +20,12 @@ def create_app(config_class=None): else: config = config_class - app.config['DB_HOST'] = config.DB_HOST - app.config['DB_PORT'] = config.DB_PORT - app.config['DB_USER'] = config.DB_USER - app.config['DB_PASSWORD'] = config.DB_PASSWORD - app.config['DB_NAME'] = config.DB_NAME - app.config['SECRET_KEY'] = config.SECRET_KEY - app.config['TESTING'] = getattr(config, 'TESTING', False) + app.config["DB_CONFIG"] = config + app.config["TESTING"] = getattr(config, "TESTING", False) - app._db_config = config - - log_level = getattr( - logging, os.getenv('LOG_LEVEL', 'INFO').upper(), logging.INFO - ) logging.basicConfig( - level=log_level, - format='%(asctime)s [%(levelname)s] %(name)s: %(message)s', + level=getattr(logging, os.getenv("LOG_LEVEL", "INFO").upper(), logging.INFO), + format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", ) from app.routes import bp diff --git a/app/config.py b/app/config.py index f4bde16..b846a47 100644 --- a/app/config.py +++ b/app/config.py @@ -1,13 +1,9 @@ import os from pathlib import Path - BASE_DIR = Path(__file__).resolve().parent.parent QUERIES_DIR = BASE_DIR / 'queries' -TMP_DIR = BASE_DIR / 'executions_tmp' -START = 1000000 -STEP = 100000 QUERY_1_NAME = 'SELECT *' QUERY_2_NAME = 'SELECT id, name, email' @@ -20,5 +16,4 @@ def __init__(self): self.DB_PASSWORD = os.getenv('DB_PASSWORD', 'password') self.DB_NAME = os.getenv('DB_NAME', 'test_db') self.DB_SEED_SIZE = os.getenv('DB_SEED_SIZE', 'medium') - self.SECRET_KEY = os.getenv('SECRET_KEY', 'dev-secret-change-in-prod') self.TESTING = False diff --git a/app/history.py b/app/history.py index ebaad3d..c0ce243 100644 --- a/app/history.py +++ b/app/history.py @@ -1,20 +1,48 @@ +import base64 import json from datetime import datetime from pathlib import Path -from benchmarks.base import BenchmarkResult +import pandas as pd + from app.config import BASE_DIR +from benchmarks.base import BenchmarkResult RESULTS_DIR = BASE_DIR / "results" +TIMESTAMP_FORMAT = "%Y-%m-%dT%H-%M-%S" + + +def _unique_dir(dirname: str) -> Path: + """Two runs within the same second would otherwise overwrite each other.""" + candidate = RESULTS_DIR / dirname + suffix = 2 + while candidate.exists(): + candidate = RESULTS_DIR / f"{dirname}-{suffix}" + suffix += 1 + return candidate + + +def _resolve(result_id: str) -> Path | None: + """Reject anything that escapes RESULTS_DIR. + + `result_id` comes straight from the URL and is used to build a filesystem + path, so it is validated rather than trusted. + """ + if not result_id or "/" in result_id or "\\" in result_id or result_id.startswith("."): + return None + + resolved = (RESULTS_DIR / result_id).resolve() + if resolved.parent != RESULTS_DIR.resolve(): + return None + return resolved def save_result(result: BenchmarkResult, params: dict) -> str: RESULTS_DIR.mkdir(parents=True, exist_ok=True) - timestamp = datetime.now().strftime("%Y-%m-%dT%H-%M-%S") - dirname = f"{timestamp}_{result.name}" - result_dir = RESULTS_DIR / dirname - result_dir.mkdir(parents=True, exist_ok=True) + timestamp = datetime.now().strftime(TIMESTAMP_FORMAT) + result_dir = _unique_dir(f"{timestamp}_{result.name}") + result_dir.mkdir(parents=True) metadata = { "benchmark": result.name, @@ -23,14 +51,13 @@ def save_result(result: BenchmarkResult, params: dict) -> str: "timestamp": timestamp, "params": params, "queries": [ - {"name": q.name, "query": q.query, "limits": q.limits, "times": q.times} + {"name": q.name, "query": q.query, "limits": q.limits, "times": q.times, + "rows_fetched": q.rows_fetched} for q in result.queries ], } (result_dir / "metadata.json").write_text(json.dumps(metadata, indent=2)) - (result_dir / "plot.png").write_bytes(result.plot_buffer.getvalue()) - result.comparison_table.to_csv(result_dir / "comparison.csv", index=False) if result.explain_plans: @@ -38,7 +65,7 @@ def save_result(result: BenchmarkResult, params: dict) -> str: json.dumps(result.explain_plans, indent=2) ) - return dirname + return result_dir.name def list_results(limit: int = 8, offset: int = 0) -> tuple[list[dict], int]: @@ -47,38 +74,40 @@ def list_results(limit: int = 8, offset: int = 0) -> tuple[list[dict], int]: all_results = [] for d in sorted(RESULTS_DIR.iterdir(), reverse=True): - if not d.is_dir(): - continue metadata_file = d / "metadata.json" - if metadata_file.exists(): + if not d.is_dir() or not metadata_file.exists(): + continue + try: meta = json.loads(metadata_file.read_text()) - meta["id"] = d.name - all_results.append(meta) + except json.JSONDecodeError: + continue + meta["id"] = d.name + all_results.append(meta) - total = len(all_results) - return all_results[offset : offset + limit], total + return all_results[offset : offset + limit], len(all_results) def load_result(result_id: str) -> dict | None: - result_dir = RESULTS_DIR / result_id - if not result_dir.exists(): + result_dir = _resolve(result_id) + if result_dir is None or not result_dir.is_dir(): return None metadata_file = result_dir / "metadata.json" if not metadata_file.exists(): return None - meta = json.loads(metadata_file.read_text()) + try: + meta = json.loads(metadata_file.read_text()) + except json.JSONDecodeError: + return None meta["id"] = result_id plot_file = result_dir / "plot.png" if plot_file.exists(): - import base64 meta["plot_data"] = base64.b64encode(plot_file.read_bytes()).decode("utf-8") csv_file = result_dir / "comparison.csv" if csv_file.exists(): - import pandas as pd meta["comparison_table"] = pd.read_csv(csv_file) explain_file = result_dir / "explain_plans.json" diff --git a/app/img/screenshot_index_usage.png b/app/img/screenshot_index_usage.png index a35ac48..fed6c42 100644 Binary files a/app/img/screenshot_index_usage.png and b/app/img/screenshot_index_usage.png differ diff --git a/app/img/screenshot_join.png b/app/img/screenshot_join.png index 91aed6a..9e84c36 100644 Binary files a/app/img/screenshot_join.png and b/app/img/screenshot_join.png differ diff --git a/app/img/screenshot_pagination.png b/app/img/screenshot_pagination.png index 2c93fd1..daad71c 100644 Binary files a/app/img/screenshot_pagination.png and b/app/img/screenshot_pagination.png differ diff --git a/app/img/screenshot_select_star.png b/app/img/screenshot_select_star.png index 267b2b4..03089e8 100644 Binary files a/app/img/screenshot_select_star.png and b/app/img/screenshot_select_star.png differ diff --git a/app/routes.py b/app/routes.py index d95a109..eb21af9 100644 --- a/app/routes.py +++ b/app/routes.py @@ -1,66 +1,63 @@ import logging -from flask import Blueprint, render_template, request +from flask import Blueprint, current_app, render_template, request -from app.benchmark import run_benchmark, result_to_plot_data, list_benchmarks +from app.benchmark import list_benchmarks, result_to_plot_data, run_benchmark from app.db import get_db_connection -from app.history import save_result, list_results, load_result +from app.history import list_results, load_result, save_result from benchmarks import get as get_benchmark +from benchmarks.base import BenchmarkNotApplicable +from sql.seed import SIZES, resolve_size, seed logger = logging.getLogger(__name__) bp = Blueprint("main", __name__) -VALID_SIZES = {"small", "medium", "large"} + +def _results_page(**kwargs): + defaults = { + "plot_data": None, + "comparison_table": None, + "benchmarks": list_benchmarks(), + "selected_benchmark": None, + "selected_size": "medium", + "sizes": SIZES, + } + return render_template("results.html", **{**defaults, **kwargs}) @bp.route("/") def landing(): - benchmarks = list_benchmarks() - return render_template( - "results.html", - plot_data=None, - comparison_table=None, - benchmarks=benchmarks, - selected_benchmark=None, - selected_size="medium", - ) + return _results_page() @bp.route("/generate") def generate(): benchmark_name = request.args.get("benchmark", "select_star") - size = request.args.get("size", "medium") - - if size not in VALID_SIZES: - size = "medium" + size = resolve_size(request.args.get("size")) if get_benchmark(benchmark_name) is None: - benchmarks = list_benchmarks() - return render_template( - "results.html", - plot_data=None, - comparison_table=None, - benchmarks=benchmarks, + return _results_page( selected_benchmark=benchmark_name, selected_size=size, error=f"Unknown benchmark: {benchmark_name}", ) try: - conn = get_db_connection() + conn = get_db_connection(current_app.config["DB_CONFIG"]) try: + seed(conn, size) result = run_benchmark(conn, benchmark_name) finally: conn.close() + except BenchmarkNotApplicable as e: + logger.warning("Benchmark not applicable: %s", e) + return _results_page( + selected_benchmark=benchmark_name, selected_size=size, error=str(e) + ) except Exception as e: logger.exception("Benchmark failed: %s", e) - benchmarks = list_benchmarks() - return render_template( - "results.html", - plot_data=None, - comparison_table=None, - benchmarks=benchmarks, + return _results_page( selected_benchmark=benchmark_name, selected_size=size, error="Benchmark execution failed. Please check your configuration and try again.", @@ -68,12 +65,9 @@ def generate(): result_id = save_result(result, {"benchmark": benchmark_name, "size": size}) - benchmarks = list_benchmarks() - return render_template( - "results.html", + return _results_page( plot_data=result_to_plot_data(result), comparison_table=result.comparison_table, - benchmarks=benchmarks, selected_benchmark=benchmark_name, selected_size=size, benchmark_title=result.title, @@ -87,14 +81,12 @@ def generate(): def history(): page = max(1, request.args.get("page", 1, type=int)) per_page = 8 - offset = (page - 1) * per_page - results, total = list_results(limit=per_page, offset=offset) - total_pages = max(1, -(-total // per_page)) + results, total = list_results(limit=per_page, offset=(page - 1) * per_page) return render_template( "history.html", results=results, page=page, - total_pages=total_pages, + total_pages=max(1, -(-total // per_page)), total=total, ) @@ -103,14 +95,14 @@ def history(): def view_result(result_id): data = load_result(result_id) if data is None: - results, _ = list_results() + results, total = list_results() return render_template( "history.html", results=results, error="Result not found", page=1, total_pages=1, - total=len(results), + total=total, ) return render_template( diff --git a/app/templates/results.html b/app/templates/results.html index d07f4d5..f7e120d 100644 --- a/app/templates/results.html +++ b/app/templates/results.html @@ -51,10 +51,13 @@

Find out why your queries are slow

Dataset +
Changing the size reseeds the users table.