From 99ac5cd69b2c2c3355d3b3d5d92343ef4758a0ed Mon Sep 17 00:00:00 2001 From: semyonsinchenko Date: Wed, 6 May 2026 11:09:41 +0200 Subject: [PATCH 1/8] feat: manual pagerank instead of graphx --- benchmark_pagerank_memory.py | 91 +++++++++++++++++++++++++++++++++++- graphframes-load-results.csv | 4 +- 2 files changed, 92 insertions(+), 3 deletions(-) diff --git a/benchmark_pagerank_memory.py b/benchmark_pagerank_memory.py index 4898b10..d0dcdbf 100644 --- a/benchmark_pagerank_memory.py +++ b/benchmark_pagerank_memory.py @@ -346,6 +346,95 @@ def run_graphframes_load(args: argparse.Namespace) -> None: print("RESULT_JSON=" + json.dumps(result, sort_keys=True)) +def run_graphframes_v2(args: argparse.Namespace) -> None: + from graphframes import GraphFrame + from graphframes.lib import Pregel + from pyspark.sql import functions as F + + t_start = time.time() + temp_dir: tempfile.TemporaryDirectory[str] | None = None + if args.edge_parquet: + parquet_dir = args.edge_parquet + else: + temp_dir = tempfile.TemporaryDirectory(prefix="livejournal-graphframes-") + parquet_dir = temp_dir.name + + prep = prepare_edges_parquet(args.db, parquet_dir, args.duckdb_memory_limit) + t_prepared = time.time() + + args.spark_app_name = "livejournal-graphframes-pagerank" + spark = build_spark(args) + + try: + vertices = spark.read.parquet(prep["vertex_path"]) + edges = spark.read.parquet(prep["edge_path"]) + graph = GraphFrame(vertices, edges) + t_built = time.time() + + # We need to add outDegree as an attribute to adjust ranks + out_degrees = graph.outDegrees + graph = GraphFrame(out_degrees, edges) + + pagerank_kwargs: dict[str, Any] = { + "resetProbability": args.reset_probability, + "tol": args.tol, + } + if args.max_iter is not None: + pagerank_kwargs = { + "resetProbability": args.reset_probability, + "maxIter": args.max_iter, + } + # is it known? can I use a constant here instead of count? + num_vertices = vertices.count() + damping_factor = args.reset_probability + tol = args.tol + + # Expression to compute the updated rank + pr_expression = F.coalesce(Pregel.msg(), F.lit(0.0)) * F.lit(1.0 - damping_factor) + F.lit(damping_factor / num_vertices) + + # Since it is the simplest PageRank over directed Graph it is trivial for Pregel + result = ( + graph + .pregel + .withVertexColumn( + "rank", + F.lit(1.0) / F.lit(num_vertices), # initial PR value + pr_expression, + ) + .sendMsgToDst(Pregel.src("rank") / Pregel.src("outDegree")) + .aggMsgs(F.sum(Pregel.msg())) + .setMaxIter(100_000) # just something big + .setSkipMessagesFromNonActiveVertices(False) # otherwise it won't work + .setStopIfAllNonActiveVertices(True) # pregel-like vertex voting + .setUseLocalCheckpoints(True) # due to age and adoption GraphFrames has terrible defaults and we cannot just change them + .setUpdateActiveVertexExpression(F.abs(pr_expression - F.col("rank")) > F.lit(tol)) # simple | old - new | > tol + .run() + ) + + # Why to count edges again? For spark it is equal to re-read from disk + score_count = result.count() + # edge_count = results.edges.count() + + # Pregel calls return persisted DataFrames in GF, so better to release + _ = result.unpersist() + t_done = time.time() + finally: + spark.stop() + if temp_dir is not None and not args.keep_edge_parquet: + temp_dir.cleanup() + + result = { + **prep, + "prepare_seconds": t_prepared - t_start, + "build_seconds": t_built - t_prepared, + "pagerank_seconds": t_done - t_built, + "total_seconds": t_done - t_start, + "score_count": score_count, + } + print("RESULT_JSON=" + json.dumps(result, sort_keys=True)) + + + def run_graphframes(args: argparse.Namespace) -> None: from graphframes import GraphFrame @@ -474,7 +563,7 @@ def add_common(p: argparse.ArgumentParser) -> None: graphframes.add_argument("--spark-master", default="local[*]") graphframes.add_argument("--spark-driver-memory", default="16g") graphframes.add_argument("--spark-shuffle-partitions", type=int, default=200) - graphframes.set_defaults(func=run_graphframes) + graphframes.set_defaults(func=run_graphframes_v2) graphframes_load = subparsers.add_parser("graphframes-load") add_common(graphframes_load) diff --git a/graphframes-load-results.csv b/graphframes-load-results.csv index 6102e0e..7017222 100644 --- a/graphframes-load-results.csv +++ b/graphframes-load-results.csv @@ -1,2 +1,2 @@ -engine,returncode,peak_rss_bytes,peak_rss_human,samples,build_seconds,count_seconds,degree_count,degree_seconds,degree_sum,directed,edge_count,edge_path,materialize_seconds,n_edges,n_nodes,prepare_seconds,total_seconds,vertex_count,vertex_path -graphframes-load,0,12850573312,11.97 GiB,243,5.7083985805511475,1.4602880477905273,3997962,9.520802021026611,138724756,False,69362378,/tmp/livejournal-graphframes-4w0tc3i4/edges.parquet,10.981090068817139,69362378,3997962,7.729346036911011,24.418834686279297,3997962,/tmp/livejournal-graphframes-4w0tc3i4/vertices.parquet +engine,returncode,peak_rss_bytes,peak_rss_human,samples,build_seconds,directed,edge_path,n_edges,n_nodes,pagerank_seconds,prepare_seconds,score_count,total_seconds,vertex_path +graphframes,0,12870094848,11.99 GiB,463,4.972158193588257,False,/tmp/livejournal-graphframes-zpdr2dqk/edges.parquet,69362378,3997962,38.26660180091858,12.153156995773315,3997962,55.39191699028015,/tmp/livejournal-graphframes-zpdr2dqk/vertices.parquet From f7b39f17e4995f3a929a82034fd48cad8513fec0 Mon Sep 17 00:00:00 2001 From: semyonsinchenko Date: Wed, 6 May 2026 11:17:09 +0200 Subject: [PATCH 2/8] feat: 4g --- graphframes-load-results.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graphframes-load-results.csv b/graphframes-load-results.csv index 7017222..884482d 100644 --- a/graphframes-load-results.csv +++ b/graphframes-load-results.csv @@ -1,2 +1,2 @@ engine,returncode,peak_rss_bytes,peak_rss_human,samples,build_seconds,directed,edge_path,n_edges,n_nodes,pagerank_seconds,prepare_seconds,score_count,total_seconds,vertex_path -graphframes,0,12870094848,11.99 GiB,463,4.972158193588257,False,/tmp/livejournal-graphframes-zpdr2dqk/edges.parquet,69362378,3997962,38.26660180091858,12.153156995773315,3997962,55.39191699028015,/tmp/livejournal-graphframes-zpdr2dqk/vertices.parquet +graphframes,0,12872552448,11.99 GiB,532,5.619204998016357,False,/tmp/livejournal-graphframes-iervtaul/edges.parquet,69362378,3997962,46.17072868347168,13.699648380279541,3997962,65.48958206176758,/tmp/livejournal-graphframes-iervtaul/vertices.parquet From 4e3e222aca1769d1656def5d20f6aa711cf46c31 Mon Sep 17 00:00:00 2001 From: semyonsinchenko Date: Thu, 7 May 2026 12:42:16 +0200 Subject: [PATCH 3/8] feat: pages --- .github/workflows/benchmark-pages.yml | 96 +++++++++++++++++++++ .gitignore | 3 + docs/index.md | 9 ++ docs/results.md | 3 + mkdocs.yml | 7 ++ pyproject.toml | 7 ++ scripts/render_results.py | 117 ++++++++++++++++++++++++++ templates/results.md.j2 | 17 ++++ 8 files changed, 259 insertions(+) create mode 100644 .github/workflows/benchmark-pages.yml create mode 100644 .gitignore create mode 100644 docs/index.md create mode 100644 docs/results.md create mode 100644 mkdocs.yml create mode 100644 scripts/render_results.py create mode 100644 templates/results.md.j2 diff --git a/.github/workflows/benchmark-pages.yml b/.github/workflows/benchmark-pages.yml new file mode 100644 index 0000000..ef14093 --- /dev/null +++ b/.github/workflows/benchmark-pages.yml @@ -0,0 +1,96 @@ +name: Benchmark and Publish Pages + +on: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: pages + cancel-in-progress: true + +jobs: + benchmark-and-build: + runs-on: ubuntu-latest + env: + PYTHONUNBUFFERED: "1" + DATASET_XZ_URL: "https://huggingface.co/datasets/ladybugdb/livejournal-4m-35m/resolve/main/livejournal-csr.duckdb.xz" + DATASET_XZ_PATH: "livejournal-csr.duckdb.xz" + DATASET_DB_PATH: "livejournal-csr.duckdb" + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Setup Java (JDK 21) + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "21" + + - name: Setup uv + uses: astral-sh/setup-uv@v3 + + - name: Install dependencies + run: uv sync --group docs + + - name: Download and unpack benchmark data + run: | + curl -L "$DATASET_XZ_URL" -o "$DATASET_XZ_PATH" + xz -d -f "$DATASET_XZ_PATH" + test -f "$DATASET_DB_PATH" + + - name: Run icebug scenario + run: | + uv run python benchmark_pagerank_memory.py compare \ + --engine icebug \ + --duckdb-memory-limit 200MB \ + --output ci-icebug-results.csv + + - name: Run graphframes scenario + run: | + uv run python benchmark_pagerank_memory.py compare \ + --engine graphframes \ + --duckdb-memory-limit 1024MB \ + --spark-driver-memory 4G \ + --output ci-graphframes-results.csv + + - name: Generate results markdown and plots + run: | + uv run python scripts/render_results.py \ + --icebug-csv ci-icebug-results.csv \ + --graphframes-csv ci-graphframes-results.csv \ + --template templates/results.md.j2 \ + --results-md docs/results.md \ + --assets-dir docs/assets + + - name: Configure Pages + uses: actions/configure-pages@v5 + + - name: Build MkDocs site + run: uv run mkdocs build --strict + + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@v3 + with: + path: site + + deploy: + needs: benchmark-and-build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8c6099e --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.dir-locals.el +*.pyc +*.xz \ No newline at end of file diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..a1658e7 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,9 @@ +# Icebug vs GraphFrames + +This site presents a focused benchmark comparison between **icebug** and +**GraphFrames** on the LiveJournal graph dataset. + +The benchmark runs both engines in CI with fixed memory constraints, captures +execution and memory metrics, and publishes reproducible results. + +Go to the [Results](results.md) page for the latest generated table and plots. diff --git a/docs/results.md b/docs/results.md new file mode 100644 index 0000000..4126f5d --- /dev/null +++ b/docs/results.md @@ -0,0 +1,3 @@ +# Results + +Results are generated automatically by CI. diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..8dc1d5a --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,7 @@ +site_name: Icebug vs GraphFrames +site_description: Minimal benchmark site for Icebug and GraphFrames comparison +theme: + name: mkdocs +nav: + - Home: index.md + - Results: results.md diff --git a/pyproject.toml b/pyproject.toml index dedf31b..5ec49a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,13 @@ dependencies = [ "typing-extensions>=4.15.0", ] +[dependency-groups] +docs = [ + "jinja2>=3.1.6", + "matplotlib>=3.10.3", + "mkdocs>=1.6.1", +] + [tool.uv] package = false diff --git a/scripts/render_results.py b/scripts/render_results.py new file mode 100644 index 0000000..3093417 --- /dev/null +++ b/scripts/render_results.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import csv +from pathlib import Path + +from jinja2 import Environment, FileSystemLoader +import matplotlib.pyplot as plt + + +def _as_float(value: str | None) -> float | None: + if value is None or value == "": + return None + try: + return float(value) + except ValueError: + return None + + +def load_row(path: Path, scenario_name: str) -> dict[str, str]: + with path.open("r", encoding="utf-8", newline="") as handle: + reader = csv.DictReader(handle) + row = next(reader, None) + if row is None: + raise ValueError(f"No rows in CSV: {path}") + + status = "ok" if row.get("returncode") == "0" else "failed" + total_seconds = _as_float(row.get("total_seconds")) + peak_rss_human = row.get("peak_rss_human", "n/a") + peak_rss_bytes = _as_float(row.get("peak_rss_bytes")) or 0.0 + + return { + "scenario": scenario_name, + "status": status, + "peak_rss_human": peak_rss_human, + "peak_rss_bytes": f"{peak_rss_bytes}", + "total_seconds": f"{total_seconds:.2f}" if total_seconds is not None else "n/a", + "total_seconds_raw": "" if total_seconds is None else f"{total_seconds}", + } + + +def render_markdown(rows: list[dict[str, str]], template_dir: Path, template_name: str, output_path: Path) -> None: + env = Environment(loader=FileSystemLoader(template_dir.as_posix()), autoescape=False) + template = env.get_template(template_name) + output = template.render(rows=rows) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(output, encoding="utf-8") + + +def plot_metric(rows: list[dict[str, str]], value_key: str, title: str, ylabel: str, output_path: Path) -> None: + labels = [row["scenario"] for row in rows] + values: list[float] = [] + for row in rows: + raw = row.get(value_key, "") + try: + values.append(float(raw)) + except ValueError: + values.append(0.0) + + fig, ax = plt.subplots(figsize=(8, 4.5)) + bars = ax.bar(labels, values, color=["#2f6f5f", "#b45a3c"]) + ax.set_title(title) + ax.set_ylabel(ylabel) + ax.grid(axis="y", linestyle="--", alpha=0.3) + ax.set_axisbelow(True) + + for bar, value in zip(bars, values): + ax.text(bar.get_x() + bar.get_width() / 2.0, bar.get_height(), f"{value:.2f}", ha="center", va="bottom", fontsize=9) + + fig.tight_layout() + output_path.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(output_path, dpi=150) + plt.close(fig) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--icebug-csv", required=True) + parser.add_argument("--graphframes-csv", required=True) + parser.add_argument("--template", default="templates/results.md.j2") + parser.add_argument("--results-md", default="docs/results.md") + parser.add_argument("--assets-dir", default="docs/assets") + args = parser.parse_args() + + icebug = load_row(Path(args.icebug_csv), "compare + engine icebug") + graphframes = load_row(Path(args.graphframes_csv), "compare + engine graphframes") + rows = [icebug, graphframes] + + template_path = Path(args.template) + render_markdown( + rows, + template_dir=template_path.parent, + template_name=template_path.name, + output_path=Path(args.results_md), + ) + + assets_dir = Path(args.assets_dir) + plot_metric( + rows, + value_key="peak_rss_bytes", + title="Memory Usage Comparison", + ylabel="Peak RSS (bytes)", + output_path=assets_dir / "memory_usage_comparison.png", + ) + plot_metric( + rows, + value_key="total_seconds_raw", + title="Runtime Comparison", + ylabel="Total time (seconds)", + output_path=assets_dir / "runtime_comparison.png", + ) + + +if __name__ == "__main__": + main() diff --git a/templates/results.md.j2 b/templates/results.md.j2 new file mode 100644 index 0000000..8beb23a --- /dev/null +++ b/templates/results.md.j2 @@ -0,0 +1,17 @@ +# Results + +Generated from CI benchmark runs. + +| Scenario | Status | Peak RSS | Total Time (s) | +|---|---|---:|---:| +{% for row in rows -%} +| {{ row.scenario }} | {{ row.status }} | {{ row.peak_rss_human }} | {{ row.total_seconds }} | +{% endfor %} + +## Memory Usage Comparison + +![Memory usage comparison](assets/memory_usage_comparison.png) + +## Runtime Comparison + +![Runtime comparison](assets/runtime_comparison.png) From bf92ff1c7c0aef4b51b24debabba5f62bdefd7da Mon Sep 17 00:00:00 2001 From: semyonsinchenko Date: Thu, 7 May 2026 12:54:01 +0200 Subject: [PATCH 4/8] feat: adjust mpl --- benchmark_pagerank_memory.py | 4 +++- scripts/render_results.py | 9 +++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/benchmark_pagerank_memory.py b/benchmark_pagerank_memory.py index d0dcdbf..0308f35 100644 --- a/benchmark_pagerank_memory.py +++ b/benchmark_pagerank_memory.py @@ -98,6 +98,8 @@ def run_child(mode: str, args: argparse.Namespace) -> dict[str, Any]: args.spark_driver_memory, "--spark-shuffle-partitions", str(args.spark_shuffle_partitions), + "spark.ui.enabled", # it adds some overhead that we barely want to measure + "false", ] ) if args.edge_parquet: @@ -290,7 +292,7 @@ def build_spark(args: argparse.Namespace): .config("spark.jars.packages", args.spark_package) ) spark = spark_builder.getOrCreate() - spark.sparkContext.setLogLevel("WARN") + spark.sparkContext.setLogLevel("ERROR") return spark diff --git a/scripts/render_results.py b/scripts/render_results.py index 3093417..31d63c2 100644 --- a/scripts/render_results.py +++ b/scripts/render_results.py @@ -36,6 +36,7 @@ def load_row(path: Path, scenario_name: str) -> dict[str, str]: "status": status, "peak_rss_human": peak_rss_human, "peak_rss_bytes": f"{peak_rss_bytes}", + "peak_rss_gib": f"{peak_rss_bytes / (1024 ** 3)}", "total_seconds": f"{total_seconds:.2f}" if total_seconds is not None else "n/a", "total_seconds_raw": "" if total_seconds is None else f"{total_seconds}", } @@ -84,8 +85,8 @@ def main() -> None: parser.add_argument("--assets-dir", default="docs/assets") args = parser.parse_args() - icebug = load_row(Path(args.icebug_csv), "compare + engine icebug") - graphframes = load_row(Path(args.graphframes_csv), "compare + engine graphframes") + icebug = load_row(Path(args.icebug_csv), "icebug") + graphframes = load_row(Path(args.graphframes_csv), "graphframes") rows = [icebug, graphframes] template_path = Path(args.template) @@ -99,9 +100,9 @@ def main() -> None: assets_dir = Path(args.assets_dir) plot_metric( rows, - value_key="peak_rss_bytes", + value_key="peak_rss_gib", title="Memory Usage Comparison", - ylabel="Peak RSS (bytes)", + ylabel="Peak RSS (GiB)", output_path=assets_dir / "memory_usage_comparison.png", ) plot_metric( From 3f431f3c0195f92e19caefeb5051655d5d4ce58e Mon Sep 17 00:00:00 2001 From: semyonsinchenko Date: Thu, 7 May 2026 13:01:28 +0200 Subject: [PATCH 5/8] fix: fix --- .github/workflows/benchmark-pages.yml | 2 ++ benchmark_pagerank_memory.py | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark-pages.yml b/.github/workflows/benchmark-pages.yml index ef14093..d17db93 100644 --- a/.github/workflows/benchmark-pages.yml +++ b/.github/workflows/benchmark-pages.yml @@ -57,6 +57,8 @@ jobs: --output ci-icebug-results.csv - name: Run graphframes scenario + env: + SPARK_TESTING: "1" run: | uv run python benchmark_pagerank_memory.py compare \ --engine graphframes \ diff --git a/benchmark_pagerank_memory.py b/benchmark_pagerank_memory.py index 0308f35..d9a0515 100644 --- a/benchmark_pagerank_memory.py +++ b/benchmark_pagerank_memory.py @@ -98,8 +98,6 @@ def run_child(mode: str, args: argparse.Namespace) -> dict[str, Any]: args.spark_driver_memory, "--spark-shuffle-partitions", str(args.spark_shuffle_partitions), - "spark.ui.enabled", # it adds some overhead that we barely want to measure - "false", ] ) if args.edge_parquet: From 293783b8769110c76e4ae5e2131ca12e85766e30 Mon Sep 17 00:00:00 2001 From: semyonsinchenko Date: Thu, 7 May 2026 13:10:47 +0200 Subject: [PATCH 6/8] feat: separate prepare vs run time --- .github/workflows/benchmark-pages.yml | 2 +- scripts/render_results.py | 57 +++++++++++++++++++++++---- templates/results.md.j2 | 6 +-- 3 files changed, 54 insertions(+), 11 deletions(-) diff --git a/.github/workflows/benchmark-pages.yml b/.github/workflows/benchmark-pages.yml index d17db93..fa93e13 100644 --- a/.github/workflows/benchmark-pages.yml +++ b/.github/workflows/benchmark-pages.yml @@ -62,7 +62,7 @@ jobs: run: | uv run python benchmark_pagerank_memory.py compare \ --engine graphframes \ - --duckdb-memory-limit 1024MB \ + --duckdb-memory-limit 2048MB \ --spark-driver-memory 4G \ --output ci-graphframes-results.csv diff --git a/scripts/render_results.py b/scripts/render_results.py index 31d63c2..6c161c4 100644 --- a/scripts/render_results.py +++ b/scripts/render_results.py @@ -8,6 +8,7 @@ from jinja2 import Environment, FileSystemLoader import matplotlib.pyplot as plt +import numpy as np def _as_float(value: str | None) -> float | None: @@ -28,6 +29,9 @@ def load_row(path: Path, scenario_name: str) -> dict[str, str]: status = "ok" if row.get("returncode") == "0" else "failed" total_seconds = _as_float(row.get("total_seconds")) + prepare_seconds = _as_float(row.get("prepare_seconds")) + if prepare_seconds is None: + prepare_seconds = _as_float(row.get("load_seconds")) peak_rss_human = row.get("peak_rss_human", "n/a") peak_rss_bytes = _as_float(row.get("peak_rss_bytes")) or 0.0 @@ -37,6 +41,8 @@ def load_row(path: Path, scenario_name: str) -> dict[str, str]: "peak_rss_human": peak_rss_human, "peak_rss_bytes": f"{peak_rss_bytes}", "peak_rss_gib": f"{peak_rss_bytes / (1024 ** 3)}", + "prepare_seconds": f"{prepare_seconds:.2f}" if prepare_seconds is not None else "n/a", + "prepare_seconds_raw": "" if prepare_seconds is None else f"{prepare_seconds}", "total_seconds": f"{total_seconds:.2f}" if total_seconds is not None else "n/a", "total_seconds_raw": "" if total_seconds is None else f"{total_seconds}", } @@ -76,6 +82,49 @@ def plot_metric(rows: list[dict[str, str]], value_key: str, title: str, ylabel: plt.close(fig) +def plot_time_grouped(rows: list[dict[str, str]], output_path: Path) -> None: + labels = [row["scenario"] for row in rows] + total_values: list[float] = [] + prepare_values: list[float] = [] + + for row in rows: + total_raw = row.get("total_seconds_raw", "") + prepare_raw = row.get("prepare_seconds_raw", "") + try: + total_values.append(float(total_raw)) + except ValueError: + total_values.append(0.0) + try: + prepare_values.append(float(prepare_raw)) + except ValueError: + prepare_values.append(0.0) + + x = np.arange(len(labels)) + width = 0.36 + + fig, ax = plt.subplots(figsize=(8, 4.5)) + total_bars = ax.bar(x - width / 2.0, total_values, width, label="Total time", color="#2f6f5f") + prepare_bars = ax.bar(x + width / 2.0, prepare_values, width, label="Prepare time", color="#b45a3c") + + ax.set_title("Runtime Comparison") + ax.set_ylabel("Time (seconds)") + ax.set_xticks(x) + ax.set_xticklabels(labels) + ax.grid(axis="y", linestyle="--", alpha=0.3) + ax.set_axisbelow(True) + ax.legend() + + for bars in (total_bars, prepare_bars): + for bar in bars: + value = bar.get_height() + ax.text(bar.get_x() + bar.get_width() / 2.0, value, f"{value:.2f}", ha="center", va="bottom", fontsize=9) + + fig.tight_layout() + output_path.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(output_path, dpi=150) + plt.close(fig) + + def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--icebug-csv", required=True) @@ -105,13 +154,7 @@ def main() -> None: ylabel="Peak RSS (GiB)", output_path=assets_dir / "memory_usage_comparison.png", ) - plot_metric( - rows, - value_key="total_seconds_raw", - title="Runtime Comparison", - ylabel="Total time (seconds)", - output_path=assets_dir / "runtime_comparison.png", - ) + plot_time_grouped(rows, output_path=assets_dir / "runtime_comparison.png") if __name__ == "__main__": diff --git a/templates/results.md.j2 b/templates/results.md.j2 index 8beb23a..c33b166 100644 --- a/templates/results.md.j2 +++ b/templates/results.md.j2 @@ -2,10 +2,10 @@ Generated from CI benchmark runs. -| Scenario | Status | Peak RSS | Total Time (s) | -|---|---|---:|---:| +| Scenario | Status | Peak RSS | Prepare Time (s) | Total Time (s) | +|---|---|---:|---:|---:| {% for row in rows -%} -| {{ row.scenario }} | {{ row.status }} | {{ row.peak_rss_human }} | {{ row.total_seconds }} | +| {{ row.scenario }} | {{ row.status }} | {{ row.peak_rss_human }} | {{ row.prepare_seconds }} | {{ row.total_seconds }} | {% endfor %} ## Memory Usage Comparison From 4f00f58580e1e2743bb9c9616718219baf77d409 Mon Sep 17 00:00:00 2001 From: semyonsinchenko Date: Thu, 7 May 2026 13:21:31 +0200 Subject: [PATCH 7/8] feat: some updates --- docs/index.md | 20 ++++++++++++++++++-- scripts/render_results.py | 21 ++++++++++++--------- templates/results.md.j2 | 6 +++--- 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/docs/index.md b/docs/index.md index a1658e7..4a9bb42 100644 --- a/docs/index.md +++ b/docs/index.md @@ -3,7 +3,23 @@ This site presents a focused benchmark comparison between **icebug** and **GraphFrames** on the LiveJournal graph dataset. -The benchmark runs both engines in CI with fixed memory constraints, captures -execution and memory metrics, and publishes reproducible results. +The benchmark data is stored in **icebug-format** (CSR arrays inside DuckDB). +icebug consumes this format directly. + +GraphFrames does not natively consume icebug-format, so each GraphFrames run +includes a long **prepare** step that converts data to Parquet before the +algorithm starts. + +The benchmark runs on a standard GitHub-hosted runner in CI with fixed memory +constraints, captures execution and memory metrics, and publishes reproducible +results. + +| Component | Version | Notes | +|---|---|---| +| Runner | GitHub Actions `ubuntu-latest` | Standard GitHub-hosted Linux runner | +| Python | `3.12` | Installed via `actions/setup-python` | +| GraphFrames | `0.11.0` | `graphframes-py` package | +| icebug | `12.6` | Installed from `uv.lock` | +| Java | JDK `21` | Installed via `actions/setup-java` (Temurin) | Go to the [Results](results.md) page for the latest generated table and plots. diff --git a/scripts/render_results.py b/scripts/render_results.py index 6c161c4..dd915d5 100644 --- a/scripts/render_results.py +++ b/scripts/render_results.py @@ -32,6 +32,7 @@ def load_row(path: Path, scenario_name: str) -> dict[str, str]: prepare_seconds = _as_float(row.get("prepare_seconds")) if prepare_seconds is None: prepare_seconds = _as_float(row.get("load_seconds")) + run_seconds = _as_float(row.get("pagerank_seconds")) peak_rss_human = row.get("peak_rss_human", "n/a") peak_rss_bytes = _as_float(row.get("peak_rss_bytes")) or 0.0 @@ -43,6 +44,8 @@ def load_row(path: Path, scenario_name: str) -> dict[str, str]: "peak_rss_gib": f"{peak_rss_bytes / (1024 ** 3)}", "prepare_seconds": f"{prepare_seconds:.2f}" if prepare_seconds is not None else "n/a", "prepare_seconds_raw": "" if prepare_seconds is None else f"{prepare_seconds}", + "run_seconds": f"{run_seconds:.2f}" if run_seconds is not None else "n/a", + "run_seconds_raw": "" if run_seconds is None else f"{run_seconds}", "total_seconds": f"{total_seconds:.2f}" if total_seconds is not None else "n/a", "total_seconds_raw": "" if total_seconds is None else f"{total_seconds}", } @@ -84,27 +87,27 @@ def plot_metric(rows: list[dict[str, str]], value_key: str, title: str, ylabel: def plot_time_grouped(rows: list[dict[str, str]], output_path: Path) -> None: labels = [row["scenario"] for row in rows] - total_values: list[float] = [] prepare_values: list[float] = [] + run_values: list[float] = [] for row in rows: - total_raw = row.get("total_seconds_raw", "") prepare_raw = row.get("prepare_seconds_raw", "") - try: - total_values.append(float(total_raw)) - except ValueError: - total_values.append(0.0) + run_raw = row.get("run_seconds_raw", "") try: prepare_values.append(float(prepare_raw)) except ValueError: prepare_values.append(0.0) + try: + run_values.append(float(run_raw)) + except ValueError: + run_values.append(0.0) x = np.arange(len(labels)) width = 0.36 fig, ax = plt.subplots(figsize=(8, 4.5)) - total_bars = ax.bar(x - width / 2.0, total_values, width, label="Total time", color="#2f6f5f") - prepare_bars = ax.bar(x + width / 2.0, prepare_values, width, label="Prepare time", color="#b45a3c") + prepare_bars = ax.bar(x - width / 2.0, prepare_values, width, label="Prepare time", color="#2f6f5f") + run_bars = ax.bar(x + width / 2.0, run_values, width, label="Run time", color="#b45a3c") ax.set_title("Runtime Comparison") ax.set_ylabel("Time (seconds)") @@ -114,7 +117,7 @@ def plot_time_grouped(rows: list[dict[str, str]], output_path: Path) -> None: ax.set_axisbelow(True) ax.legend() - for bars in (total_bars, prepare_bars): + for bars in (prepare_bars, run_bars): for bar in bars: value = bar.get_height() ax.text(bar.get_x() + bar.get_width() / 2.0, value, f"{value:.2f}", ha="center", va="bottom", fontsize=9) diff --git a/templates/results.md.j2 b/templates/results.md.j2 index c33b166..f0ee715 100644 --- a/templates/results.md.j2 +++ b/templates/results.md.j2 @@ -2,10 +2,10 @@ Generated from CI benchmark runs. -| Scenario | Status | Peak RSS | Prepare Time (s) | Total Time (s) | -|---|---|---:|---:|---:| +| Scenario | Status | Peak RSS | Prepare Time (s) | Run Time (s) | Total Time (s) | +|---|---|---:|---:|---:|---:| {% for row in rows -%} -| {{ row.scenario }} | {{ row.status }} | {{ row.peak_rss_human }} | {{ row.prepare_seconds }} | {{ row.total_seconds }} | +| {{ row.scenario }} | {{ row.status }} | {{ row.peak_rss_human }} | {{ row.prepare_seconds }} | {{ row.run_seconds }} | {{ row.total_seconds }} | {% endfor %} ## Memory Usage Comparison From 4ad8892de2d0bcc5eed9edeac14343e9489c224b Mon Sep 17 00:00:00 2001 From: semyonsinchenko Date: Thu, 7 May 2026 13:37:12 +0200 Subject: [PATCH 8/8] feat: adjust plots and table --- scripts/render_results.py | 15 ++++++--------- templates/results.md.j2 | 6 +++--- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/scripts/render_results.py b/scripts/render_results.py index dd915d5..91dddf0 100644 --- a/scripts/render_results.py +++ b/scripts/render_results.py @@ -32,7 +32,6 @@ def load_row(path: Path, scenario_name: str) -> dict[str, str]: prepare_seconds = _as_float(row.get("prepare_seconds")) if prepare_seconds is None: prepare_seconds = _as_float(row.get("load_seconds")) - run_seconds = _as_float(row.get("pagerank_seconds")) peak_rss_human = row.get("peak_rss_human", "n/a") peak_rss_bytes = _as_float(row.get("peak_rss_bytes")) or 0.0 @@ -44,8 +43,6 @@ def load_row(path: Path, scenario_name: str) -> dict[str, str]: "peak_rss_gib": f"{peak_rss_bytes / (1024 ** 3)}", "prepare_seconds": f"{prepare_seconds:.2f}" if prepare_seconds is not None else "n/a", "prepare_seconds_raw": "" if prepare_seconds is None else f"{prepare_seconds}", - "run_seconds": f"{run_seconds:.2f}" if run_seconds is not None else "n/a", - "run_seconds_raw": "" if run_seconds is None else f"{run_seconds}", "total_seconds": f"{total_seconds:.2f}" if total_seconds is not None else "n/a", "total_seconds_raw": "" if total_seconds is None else f"{total_seconds}", } @@ -88,26 +85,26 @@ def plot_metric(rows: list[dict[str, str]], value_key: str, title: str, ylabel: def plot_time_grouped(rows: list[dict[str, str]], output_path: Path) -> None: labels = [row["scenario"] for row in rows] prepare_values: list[float] = [] - run_values: list[float] = [] + total_values: list[float] = [] for row in rows: prepare_raw = row.get("prepare_seconds_raw", "") - run_raw = row.get("run_seconds_raw", "") + total_raw = row.get("total_seconds_raw", "") try: prepare_values.append(float(prepare_raw)) except ValueError: prepare_values.append(0.0) try: - run_values.append(float(run_raw)) + total_values.append(float(total_raw)) except ValueError: - run_values.append(0.0) + total_values.append(0.0) x = np.arange(len(labels)) width = 0.36 fig, ax = plt.subplots(figsize=(8, 4.5)) prepare_bars = ax.bar(x - width / 2.0, prepare_values, width, label="Prepare time", color="#2f6f5f") - run_bars = ax.bar(x + width / 2.0, run_values, width, label="Run time", color="#b45a3c") + total_bars = ax.bar(x + width / 2.0, total_values, width, label="Total time", color="#b45a3c") ax.set_title("Runtime Comparison") ax.set_ylabel("Time (seconds)") @@ -117,7 +114,7 @@ def plot_time_grouped(rows: list[dict[str, str]], output_path: Path) -> None: ax.set_axisbelow(True) ax.legend() - for bars in (prepare_bars, run_bars): + for bars in (prepare_bars, total_bars): for bar in bars: value = bar.get_height() ax.text(bar.get_x() + bar.get_width() / 2.0, value, f"{value:.2f}", ha="center", va="bottom", fontsize=9) diff --git a/templates/results.md.j2 b/templates/results.md.j2 index f0ee715..c33b166 100644 --- a/templates/results.md.j2 +++ b/templates/results.md.j2 @@ -2,10 +2,10 @@ Generated from CI benchmark runs. -| Scenario | Status | Peak RSS | Prepare Time (s) | Run Time (s) | Total Time (s) | -|---|---|---:|---:|---:|---:| +| Scenario | Status | Peak RSS | Prepare Time (s) | Total Time (s) | +|---|---|---:|---:|---:| {% for row in rows -%} -| {{ row.scenario }} | {{ row.status }} | {{ row.peak_rss_human }} | {{ row.prepare_seconds }} | {{ row.run_seconds }} | {{ row.total_seconds }} | +| {{ row.scenario }} | {{ row.status }} | {{ row.peak_rss_human }} | {{ row.prepare_seconds }} | {{ row.total_seconds }} | {% endfor %} ## Memory Usage Comparison