diff --git a/.github/workflows/benchmark-pages.yml b/.github/workflows/benchmark-pages.yml new file mode 100644 index 0000000..fa93e13 --- /dev/null +++ b/.github/workflows/benchmark-pages.yml @@ -0,0 +1,98 @@ +name: Benchmark and Publish Pages + +on: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: pages + cancel-in-progress: true + +jobs: + benchmark-and-build: + runs-on: ubuntu-latest + env: + PYTHONUNBUFFERED: "1" + DATASET_XZ_URL: "https://huggingface.co/datasets/ladybugdb/livejournal-4m-35m/resolve/main/livejournal-csr.duckdb.xz" + DATASET_XZ_PATH: "livejournal-csr.duckdb.xz" + DATASET_DB_PATH: "livejournal-csr.duckdb" + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Setup Java (JDK 21) + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "21" + + - name: Setup uv + uses: astral-sh/setup-uv@v3 + + - name: Install dependencies + run: uv sync --group docs + + - name: Download and unpack benchmark data + run: | + curl -L "$DATASET_XZ_URL" -o "$DATASET_XZ_PATH" + xz -d -f "$DATASET_XZ_PATH" + test -f "$DATASET_DB_PATH" + + - name: Run icebug scenario + run: | + uv run python benchmark_pagerank_memory.py compare \ + --engine icebug \ + --duckdb-memory-limit 200MB \ + --output ci-icebug-results.csv + + - name: Run graphframes scenario + env: + SPARK_TESTING: "1" + run: | + uv run python benchmark_pagerank_memory.py compare \ + --engine graphframes \ + --duckdb-memory-limit 2048MB \ + --spark-driver-memory 4G \ + --output ci-graphframes-results.csv + + - name: Generate results markdown and plots + run: | + uv run python scripts/render_results.py \ + --icebug-csv ci-icebug-results.csv \ + --graphframes-csv ci-graphframes-results.csv \ + --template templates/results.md.j2 \ + --results-md docs/results.md \ + --assets-dir docs/assets + + - name: Configure Pages + uses: actions/configure-pages@v5 + + - name: Build MkDocs site + run: uv run mkdocs build --strict + + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@v3 + with: + path: site + + deploy: + needs: benchmark-and-build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8c6099e --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.dir-locals.el +*.pyc +*.xz \ No newline at end of file diff --git a/benchmark_pagerank_memory.py b/benchmark_pagerank_memory.py index d0dcdbf..d9a0515 100644 --- a/benchmark_pagerank_memory.py +++ b/benchmark_pagerank_memory.py @@ -290,7 +290,7 @@ def build_spark(args: argparse.Namespace): .config("spark.jars.packages", args.spark_package) ) spark = spark_builder.getOrCreate() - spark.sparkContext.setLogLevel("WARN") + spark.sparkContext.setLogLevel("ERROR") return spark diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..4a9bb42 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,25 @@ +# Icebug vs GraphFrames + +This site presents a focused benchmark comparison between **icebug** and +**GraphFrames** on the LiveJournal graph dataset. + +The benchmark data is stored in **icebug-format** (CSR arrays inside DuckDB). +icebug consumes this format directly. + +GraphFrames does not natively consume icebug-format, so each GraphFrames run +includes a long **prepare** step that converts data to Parquet before the +algorithm starts. + +The benchmark runs on a standard GitHub-hosted runner in CI with fixed memory +constraints, captures execution and memory metrics, and publishes reproducible +results. + +| Component | Version | Notes | +|---|---|---| +| Runner | GitHub Actions `ubuntu-latest` | Standard GitHub-hosted Linux runner | +| Python | `3.12` | Installed via `actions/setup-python` | +| GraphFrames | `0.11.0` | `graphframes-py` package | +| icebug | `12.6` | Installed from `uv.lock` | +| Java | JDK `21` | Installed via `actions/setup-java` (Temurin) | + +Go to the [Results](results.md) page for the latest generated table and plots. diff --git a/docs/results.md b/docs/results.md new file mode 100644 index 0000000..4126f5d --- /dev/null +++ b/docs/results.md @@ -0,0 +1,3 @@ +# Results + +Results are generated automatically by CI. diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..8dc1d5a --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,7 @@ +site_name: Icebug vs GraphFrames +site_description: Minimal benchmark site for Icebug and GraphFrames comparison +theme: + name: mkdocs +nav: + - Home: index.md + - Results: results.md diff --git a/pyproject.toml b/pyproject.toml index dedf31b..5ec49a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,13 @@ dependencies = [ "typing-extensions>=4.15.0", ] +[dependency-groups] +docs = [ + "jinja2>=3.1.6", + "matplotlib>=3.10.3", + "mkdocs>=1.6.1", +] + [tool.uv] package = false diff --git a/scripts/render_results.py b/scripts/render_results.py new file mode 100644 index 0000000..91dddf0 --- /dev/null +++ b/scripts/render_results.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import csv +from pathlib import Path + +from jinja2 import Environment, FileSystemLoader +import matplotlib.pyplot as plt +import numpy as np + + +def _as_float(value: str | None) -> float | None: + if value is None or value == "": + return None + try: + return float(value) + except ValueError: + return None + + +def load_row(path: Path, scenario_name: str) -> dict[str, str]: + with path.open("r", encoding="utf-8", newline="") as handle: + reader = csv.DictReader(handle) + row = next(reader, None) + if row is None: + raise ValueError(f"No rows in CSV: {path}") + + status = "ok" if row.get("returncode") == "0" else "failed" + total_seconds = _as_float(row.get("total_seconds")) + prepare_seconds = _as_float(row.get("prepare_seconds")) + if prepare_seconds is None: + prepare_seconds = _as_float(row.get("load_seconds")) + peak_rss_human = row.get("peak_rss_human", "n/a") + peak_rss_bytes = _as_float(row.get("peak_rss_bytes")) or 0.0 + + return { + "scenario": scenario_name, + "status": status, + "peak_rss_human": peak_rss_human, + "peak_rss_bytes": f"{peak_rss_bytes}", + "peak_rss_gib": f"{peak_rss_bytes / (1024 ** 3)}", + "prepare_seconds": f"{prepare_seconds:.2f}" if prepare_seconds is not None else "n/a", + "prepare_seconds_raw": "" if prepare_seconds is None else f"{prepare_seconds}", + "total_seconds": f"{total_seconds:.2f}" if total_seconds is not None else "n/a", + "total_seconds_raw": "" if total_seconds is None else f"{total_seconds}", + } + + +def render_markdown(rows: list[dict[str, str]], template_dir: Path, template_name: str, output_path: Path) -> None: + env = Environment(loader=FileSystemLoader(template_dir.as_posix()), autoescape=False) + template = env.get_template(template_name) + output = template.render(rows=rows) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(output, encoding="utf-8") + + +def plot_metric(rows: list[dict[str, str]], value_key: str, title: str, ylabel: str, output_path: Path) -> None: + labels = [row["scenario"] for row in rows] + values: list[float] = [] + for row in rows: + raw = row.get(value_key, "") + try: + values.append(float(raw)) + except ValueError: + values.append(0.0) + + fig, ax = plt.subplots(figsize=(8, 4.5)) + bars = ax.bar(labels, values, color=["#2f6f5f", "#b45a3c"]) + ax.set_title(title) + ax.set_ylabel(ylabel) + ax.grid(axis="y", linestyle="--", alpha=0.3) + ax.set_axisbelow(True) + + for bar, value in zip(bars, values): + ax.text(bar.get_x() + bar.get_width() / 2.0, bar.get_height(), f"{value:.2f}", ha="center", va="bottom", fontsize=9) + + fig.tight_layout() + output_path.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(output_path, dpi=150) + plt.close(fig) + + +def plot_time_grouped(rows: list[dict[str, str]], output_path: Path) -> None: + labels = [row["scenario"] for row in rows] + prepare_values: list[float] = [] + total_values: list[float] = [] + + for row in rows: + prepare_raw = row.get("prepare_seconds_raw", "") + total_raw = row.get("total_seconds_raw", "") + try: + prepare_values.append(float(prepare_raw)) + except ValueError: + prepare_values.append(0.0) + try: + total_values.append(float(total_raw)) + except ValueError: + total_values.append(0.0) + + x = np.arange(len(labels)) + width = 0.36 + + fig, ax = plt.subplots(figsize=(8, 4.5)) + prepare_bars = ax.bar(x - width / 2.0, prepare_values, width, label="Prepare time", color="#2f6f5f") + total_bars = ax.bar(x + width / 2.0, total_values, width, label="Total time", color="#b45a3c") + + ax.set_title("Runtime Comparison") + ax.set_ylabel("Time (seconds)") + ax.set_xticks(x) + ax.set_xticklabels(labels) + ax.grid(axis="y", linestyle="--", alpha=0.3) + ax.set_axisbelow(True) + ax.legend() + + for bars in (prepare_bars, total_bars): + for bar in bars: + value = bar.get_height() + ax.text(bar.get_x() + bar.get_width() / 2.0, value, f"{value:.2f}", ha="center", va="bottom", fontsize=9) + + fig.tight_layout() + output_path.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(output_path, dpi=150) + plt.close(fig) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--icebug-csv", required=True) + parser.add_argument("--graphframes-csv", required=True) + parser.add_argument("--template", default="templates/results.md.j2") + parser.add_argument("--results-md", default="docs/results.md") + parser.add_argument("--assets-dir", default="docs/assets") + args = parser.parse_args() + + icebug = load_row(Path(args.icebug_csv), "icebug") + graphframes = load_row(Path(args.graphframes_csv), "graphframes") + rows = [icebug, graphframes] + + template_path = Path(args.template) + render_markdown( + rows, + template_dir=template_path.parent, + template_name=template_path.name, + output_path=Path(args.results_md), + ) + + assets_dir = Path(args.assets_dir) + plot_metric( + rows, + value_key="peak_rss_gib", + title="Memory Usage Comparison", + ylabel="Peak RSS (GiB)", + output_path=assets_dir / "memory_usage_comparison.png", + ) + plot_time_grouped(rows, output_path=assets_dir / "runtime_comparison.png") + + +if __name__ == "__main__": + main() diff --git a/templates/results.md.j2 b/templates/results.md.j2 new file mode 100644 index 0000000..c33b166 --- /dev/null +++ b/templates/results.md.j2 @@ -0,0 +1,17 @@ +# Results + +Generated from CI benchmark runs. + +| Scenario | Status | Peak RSS | Prepare Time (s) | Total Time (s) | +|---|---|---:|---:|---:| +{% for row in rows -%} +| {{ row.scenario }} | {{ row.status }} | {{ row.peak_rss_human }} | {{ row.prepare_seconds }} | {{ row.total_seconds }} | +{% endfor %} + +## Memory Usage Comparison + +![Memory usage comparison](assets/memory_usage_comparison.png) + +## Runtime Comparison + +![Runtime comparison](assets/runtime_comparison.png)