From a866b75b2a54e3ab730b9c75c3ab764ead1c51f2 Mon Sep 17 00:00:00 2001 From: Mark Saroufim Date: Wed, 17 Jun 2026 14:28:24 -0700 Subject: [PATCH 1/2] Use kernelbot leaderboard rankings API --- kernelboard/api/leaderboard.py | 158 +++++++--------------------- tests/api/test_leaderboard_api.py | 166 ++++++++++++++---------------- 2 files changed, 112 insertions(+), 212 deletions(-) diff --git a/kernelboard/api/leaderboard.py b/kernelboard/api/leaderboard.py index e1ed9ba..6c4c008 100644 --- a/kernelboard/api/leaderboard.py +++ b/kernelboard/api/leaderboard.py @@ -1,8 +1,10 @@ import logging +import os import time from http import HTTPStatus from typing import Any, List +import requests from flask import Blueprint from kernelboard.lib.db import get_db_connection @@ -20,29 +22,42 @@ def leaderboard(leaderboard_id: int): total_start = time.perf_counter() - # 1. Database connection - db_conn_start = time.perf_counter() - conn = get_db_connection() - db_conn_time = (time.perf_counter() - db_conn_start) * 1000 - - # 2. Query execution - query = _get_query() - query_start = time.perf_counter() - with conn.cursor() as cur: - cur.execute(query, {"leaderboard_id": leaderboard_id}) - result = cur.fetchone() - query_time = (time.perf_counter() - query_start) * 1000 + kernelbot_start = time.perf_counter() + try: + response = requests.get( + f"{get_cluster_manager_endpoint()}/leaderboard/{leaderboard_id}/rankings", + timeout=10, + ) + except requests.RequestException as exc: + logger.exception("kernelbot leaderboard request failed", exc_info=exc) + return http_error( + "could not fetch leaderboard rankings", + 10000 + HTTPStatus.BAD_GATEWAY, + HTTPStatus.BAD_GATEWAY, + ) + kernelbot_time = (time.perf_counter() - kernelbot_start) * 1000 - if is_result_invalid(result): + if response.status_code == HTTPStatus.NOT_FOUND: return http_error( f"canonot find leaderboard with id {leaderboard_id}", 10000 + HTTPStatus.NOT_FOUND, HTTPStatus.NOT_FOUND, ) - data = result[0] + if not response.ok: + logger.error( + "kernelbot leaderboard request failed: status=%s body=%s", + response.status_code, + response.text, + ) + return http_error( + "could not fetch leaderboard rankings", + 10000 + HTTPStatus.BAD_GATEWAY, + HTTPStatus.BAD_GATEWAY, + ) + + data = response.json() - # 3. Data transformation transform_start = time.perf_counter() res = to_api_leaderboard_item(data) transform_time = (time.perf_counter() - transform_start) * 1000 @@ -52,10 +67,9 @@ def leaderboard(leaderboard_id: int): # Log timing breakdown logger.info( "[Perf] leaderboard_id=%s | " - "db_conn=%.2fms | query=%.2fms | transform=%.2fms | total=%.2fms", + "kernelbot=%.2fms | transform=%.2fms | total=%.2fms", leaderboard_id, - db_conn_time, - query_time, + kernelbot_time, transform_time, total_time, ) @@ -118,109 +132,11 @@ def to_api_leaderboard_item(data: dict[str, Any]): } -def _get_query(): - query = """ - WITH - - -- Basic info about the leaderboard. - leaderboard_info AS ( - SELECT - name, - deadline, - task->>'lang' AS lang, - description AS description, - task->'files'->>'reference.py' AS reference, - task->'benchmarks' AS benchmarks - FROM leaderboard.leaderboard - WHERE id = %(leaderboard_id)s - ), - - -- All the different GPU types for this leaderboard. - gpu_types AS ( - SELECT DISTINCT gpu_type - FROM leaderboard.gpu_type - WHERE leaderboard_id = %(leaderboard_id)s - ), - - -- Total submission count per user per GPU type for this leaderboard. - submission_counts AS ( - SELECT s.user_id, r.runner, COUNT(DISTINCT s.id) AS submission_count - FROM leaderboard.submission s - JOIN leaderboard.runs r ON r.submission_id = s.id - WHERE s.leaderboard_id = %(leaderboard_id)s - GROUP BY s.user_id, r.runner - ), - - -- All the runs on this leaderboard. For each user and GPU type, the - -- user's runs on that GPU type are ranked by score. - ranked_runs AS ( - SELECT r.runner AS runner, - u.user_name AS user_name, - r.score AS score, - s.submission_time AS submission_time, - s.file_name AS file_name, - r.submission_id AS submission_id, - COALESCE(sc.submission_count, 0) AS submission_count, - RANK() OVER (PARTITION BY r.runner, u.id ORDER BY r.score ASC) AS rank - FROM leaderboard.runs r - JOIN leaderboard.submission s ON r.submission_id = s.id - LEFT JOIN leaderboard.user_info u ON s.user_id = u.id - LEFT JOIN submission_counts sc ON s.user_id = sc.user_id AND r.runner = sc.runner - WHERE NOT r.secret - AND r.score IS NOT NULL - AND r.passed - AND s.leaderboard_id = %(leaderboard_id)s - AND NOT EXISTS ( - SELECT 1 - FROM leaderboard.runs sr - WHERE sr.submission_id = s.id - AND sr.secret - AND sr.runner = r.runner - AND sr.passed = FALSE - ) - ), - - -- From ranked_runs, keep only the top run per user. - top_runs AS (SELECT * FROM ranked_runs WHERE rank = 1) - - SELECT jsonb_build_object( - 'rankings', (SELECT jsonb_object_agg(g.gpu_type, ( - SELECT jsonb_agg( - jsonb_build_object( - 'user_name', r.user_name, - 'score', r.score, - 'file_name', r.file_name, - 'submission_id', r.submission_id, - 'submission_count', r.submission_count, - 'submission_time', r.submission_time - ) - ORDER BY r.score ASC - ) - FROM top_runs r WHERE r.runner = g.gpu_type))), - - 'leaderboard', (SELECT jsonb_build_object( - 'name', name, - 'deadline', deadline, - 'lang', lang, - 'description', description, - 'reference', reference, - 'benchmarks', benchmarks, - 'gpu_types', (SELECT jsonb_agg(gpu_type) FROM gpu_types) - ) FROM leaderboard_info) - ) AS result FROM (SELECT gpu_type FROM gpu_types) g; - """ - return query - - -def is_result_invalid(result): - if result is None: - return True - if len(result) == 0: - return True - if not result[0] or not result[0]["leaderboard"]: - return True - - return False +def get_cluster_manager_endpoint(): + env_var = os.getenv("DISCORD_CLUSTER_MANAGER_API_BASE_URL", "") + if not env_var: + logger.warning("DISCORD_CLUSTER_MANAGER_API_BASE_URL is not set") + return env_var.rstrip("/") # ai generated code hardcoded user_id diff --git a/tests/api/test_leaderboard_api.py b/tests/api/test_leaderboard_api.py index a27e5a2..f5efb34 100644 --- a/tests/api/test_leaderboard_api.py +++ b/tests/api/test_leaderboard_api.py @@ -1,25 +1,71 @@ -from kernelboard.lib.db import get_db_connection +from unittest.mock import MagicMock, patch -def test_leaderboard(client): +def kernelbot_leaderboard_payload(rankings=None): + return { + "rankings": rankings + if rankings is not None + else { + "H100": [ + { + "user_name": "Alice", + "score": 1.25, + "file_name": "submission.py", + "submission_id": 123, + "submission_count": 2, + "submission_time": "2026-06-17T00:00:00Z", + } + ], + "A100": [], + }, + "leaderboard": { + "name": "conv2d", + "deadline": "2026-06-29T17:00:00-07:00", + "lang": "py", + "description": "description\\ntext", + "reference": "def ref():\\n pass", + "benchmarks": [{"n": 32}], + "gpu_types": ["H100", "A100"], + }, + } + + +def mock_kernelbot_response(payload=None, status_code=200): + response = MagicMock() + response.status_code = status_code + response.ok = 200 <= status_code < 400 + response.text = "" + response.json.return_value = payload if payload is not None else kernelbot_leaderboard_payload() + return response + + +@patch("kernelboard.api.leaderboard.requests.get") +def test_leaderboard(mock_get, client): + mock_get.return_value = mock_kernelbot_response() + response = client.get("/api/leaderboard/339") + assert response.status_code == 200 assert b"conv2d" in response.data + mock_get.assert_called_once_with( + "test-secret/leaderboard/339/rankings", + timeout=10, + ) + +@patch("kernelboard.api.leaderboard.requests.get") +def test_nonexistent_leaderboard(mock_get, client): + mock_get.return_value = mock_kernelbot_response(status_code=404) -def test_nonexistent_leaderboard(client): response = client.get("/api/leaderboard/1000000") assert response.status_code == 404 -def test_leaderboard_no_submissions(client, app): - with app.app_context(): - conn = get_db_connection() - with conn.cursor() as cur: - cur.execute( - "UPDATE leaderboard.submission SET leaderboard_id = 340 WHERE leaderboard_id = 339" - ) - conn.commit() # Commit update so the web reque sees it. +@patch("kernelboard.api.leaderboard.requests.get") +def test_leaderboard_no_submissions(mock_get, client): + mock_get.return_value = mock_kernelbot_response( + kernelbot_leaderboard_payload(rankings={"H100": [], "A100": []}) + ) response = client.get("/api/leaderboard/339") assert response.status_code == 200 @@ -28,86 +74,24 @@ def test_leaderboard_no_submissions(client, app): assert res["data"]["rankings"] == {} -def test_failed_secret_benchmark_hides_public_leaderboard_run(client, app): - with app.app_context(): - conn = get_db_connection() - with conn.cursor() as cur: - cur.execute( - """ - INSERT INTO leaderboard.submission - (id, leaderboard_id, file_name, user_id, code_id, submission_time, done) - VALUES - (900001, 339, 'hidden_secret_fail.py', '123456789012345', 13, NOW(), TRUE), - (900002, 339, 'visible_public_pass.py', '234567890123456', 13, NOW(), TRUE) - """ - ) - cur.execute( - """ - INSERT INTO leaderboard.runs - ( - id, - submission_id, - start_time, - end_time, - mode, - secret, - runner, - score, - passed, - compilation, - meta, - result, - system_info - ) - VALUES - ( - 900001, - 900001, - NOW(), - NOW(), - 'leaderboard', - FALSE, - 'H100', - -999, - TRUE, - '{}', - '{}', - '{}', - '{}' - ), - ( - 900002, - 900001, - NOW(), - NOW(), - 'benchmark', - TRUE, - 'H100', - NULL, - FALSE, - '{}', - '{}', - '{}', - '{}' - ), - ( - 900003, - 900002, - NOW(), - NOW(), - 'leaderboard', - FALSE, - 'H100', - -998, - TRUE, - '{}', - '{}', - '{}', - '{}' - ) - """ - ) - conn.commit() +@patch("kernelboard.api.leaderboard.requests.get") +def test_leaderboard_delegates_secret_filtering_to_kernelbot(mock_get, client): + mock_get.return_value = mock_kernelbot_response( + kernelbot_leaderboard_payload( + rankings={ + "H100": [ + { + "user_name": "Bob", + "score": -998, + "file_name": "visible_public_pass.py", + "submission_id": 900002, + "submission_count": 1, + "submission_time": "2026-06-17T00:00:00Z", + } + ] + } + ) + ) response = client.get("/api/leaderboard/339") assert response.status_code == 200 From 7a2114371f9fc34d7e7f4c4c0b461a3fb9c911da Mon Sep 17 00:00:00 2001 From: Mark Saroufim Date: Wed, 17 Jun 2026 14:52:17 -0700 Subject: [PATCH 2/2] Use kernelbot rankings in leaderboard views --- kernelboard/api/leaderboard.py | 96 ++---------------- kernelboard/leaderboard.py | 149 +++------------------------- kernelboard/lib/kernelbot_client.py | 96 ++++++++++++++++++ tests/api/test_leaderboard_api.py | 8 +- tests/conftest.py | 3 +- tests/test_leaderboard.py | 89 +++++++---------- tests/test_leaderboard_sql_guard.py | 55 ++++++++++ 7 files changed, 218 insertions(+), 278 deletions(-) create mode 100644 kernelboard/lib/kernelbot_client.py create mode 100644 tests/test_leaderboard_sql_guard.py diff --git a/kernelboard/api/leaderboard.py b/kernelboard/api/leaderboard.py index 6c4c008..47bcf95 100644 --- a/kernelboard/api/leaderboard.py +++ b/kernelboard/api/leaderboard.py @@ -1,15 +1,13 @@ import logging -import os import time from http import HTTPStatus -from typing import Any, List +from typing import List -import requests from flask import Blueprint from kernelboard.lib.db import get_db_connection +from kernelboard.lib.kernelbot_client import get_leaderboard_rankings, to_leaderboard_view from kernelboard.lib.status_code import http_error, http_success -from kernelboard.lib.time import to_time_left logger = logging.getLogger(__name__) @@ -23,43 +21,25 @@ def leaderboard(leaderboard_id: int): total_start = time.perf_counter() kernelbot_start = time.perf_counter() - try: - response = requests.get( - f"{get_cluster_manager_endpoint()}/leaderboard/{leaderboard_id}/rankings", - timeout=10, - ) - except requests.RequestException as exc: - logger.exception("kernelbot leaderboard request failed", exc_info=exc) - return http_error( - "could not fetch leaderboard rankings", - 10000 + HTTPStatus.BAD_GATEWAY, - HTTPStatus.BAD_GATEWAY, - ) + data, status_code = get_leaderboard_rankings(leaderboard_id) kernelbot_time = (time.perf_counter() - kernelbot_start) * 1000 - if response.status_code == HTTPStatus.NOT_FOUND: + if status_code == HTTPStatus.NOT_FOUND: return http_error( - f"canonot find leaderboard with id {leaderboard_id}", + f"cannot find leaderboard with id {leaderboard_id}", 10000 + HTTPStatus.NOT_FOUND, HTTPStatus.NOT_FOUND, ) - if not response.ok: - logger.error( - "kernelbot leaderboard request failed: status=%s body=%s", - response.status_code, - response.text, - ) + if status_code != HTTPStatus.OK: return http_error( "could not fetch leaderboard rankings", 10000 + HTTPStatus.BAD_GATEWAY, HTTPStatus.BAD_GATEWAY, ) - data = response.json() - transform_start = time.perf_counter() - res = to_api_leaderboard_item(data) + res = to_leaderboard_view(data) transform_time = (time.perf_counter() - transform_start) * 1000 total_time = (time.perf_counter() - total_start) * 1000 @@ -77,68 +57,6 @@ def leaderboard(leaderboard_id: int): return http_success(res) -# converts db record to api -def to_api_leaderboard_item(data: dict[str, Any]): - leaderboard_data = data["leaderboard"] - name = leaderboard_data["name"] - deadline = leaderboard_data["deadline"] - time_left = to_time_left(deadline) - - lang = leaderboard_data["lang"] - if lang == "py": - lang = "Python" - - description = leaderboard_data["description"] or "" - description = description.replace("\\n", "\n") - - reference = leaderboard_data["reference"] or "" - reference = reference.replace("\\n", "\n") - - benchmarks = leaderboard_data.get("benchmarks") or [] - - gpu_types = leaderboard_data["gpu_types"] - gpu_types.sort() - - rankings = {} - for gpu_type, ranking_ in data["rankings"].items(): - ranking = [] - prev_score = None - - if ranking_ is not None: - for i, entry in enumerate(ranking_): - entry["rank"] = i + 1 - - if prev_score is not None: - entry["prev_score"] = entry["score"] - prev_score - else: - entry["prev_score"] = None - - ranking.append(entry) - - prev_score = entry["score"] - - if len(ranking) > 0: - rankings[gpu_type] = ranking - return { - "name": name, - "deadline": deadline, - "time_left": time_left, - "lang": lang, - "gpu_types": gpu_types, - "description": description, - "reference": reference, - "benchmarks": benchmarks, - "rankings": rankings, - } - - -def get_cluster_manager_endpoint(): - env_var = os.getenv("DISCORD_CLUSTER_MANAGER_API_BASE_URL", "") - if not env_var: - logger.warning("DISCORD_CLUSTER_MANAGER_API_BASE_URL is not set") - return env_var.rstrip("/") - - # ai generated code hardcoded user_id HARDCODED_USER_ID = "261278773" diff --git a/kernelboard/leaderboard.py b/kernelboard/leaderboard.py index 353a0b5..99238e8 100644 --- a/kernelboard/leaderboard.py +++ b/kernelboard/leaderboard.py @@ -1,146 +1,29 @@ +from http import HTTPStatus + from flask import Blueprint, abort, render_template -from kernelboard.lib.db import get_db_connection -from kernelboard.lib.time import to_time_left +from kernelboard.lib.kernelbot_client import get_leaderboard_rankings, to_leaderboard_view blueprint = Blueprint("leaderboard", __name__, url_prefix="/leaderboard") @blueprint.route("/") def leaderboard(leaderboard_id: int): - query = """ - WITH - - -- Basic info about the leaderboard. - leaderboard_info AS ( - SELECT - name, - deadline, - task->>'lang' AS lang, - description AS description, - task->'files'->>'reference.py' AS reference - FROM leaderboard.leaderboard - WHERE id = %(leaderboard_id)s - ), - - -- All the different GPU types for this leaderboard. - gpu_types AS ( - SELECT DISTINCT gpu_type - FROM leaderboard.gpu_type - WHERE leaderboard_id = %(leaderboard_id)s - ), - - -- All the runs on this leaderboard. For each user and GPU type, the - -- user's runs on that GPU type are ranked by score. - ranked_runs AS ( - SELECT r.runner AS runner, - u.user_name AS user_name, - r.score AS score, - s.submission_time AS submission_time, - s.file_name AS file_name, - RANK() OVER (PARTITION BY r.runner, u.id ORDER BY r.score ASC) AS rank - FROM leaderboard.runs r - JOIN leaderboard.submission s ON r.submission_id = s.id - LEFT JOIN leaderboard.user_info u ON s.user_id = u.id - WHERE NOT r.secret - AND r.score IS NOT NULL - AND r.passed - AND s.leaderboard_id = %(leaderboard_id)s - AND NOT EXISTS ( - SELECT 1 - FROM leaderboard.runs sr - WHERE sr.submission_id = s.id - AND sr.secret - AND sr.runner = r.runner - AND sr.passed = FALSE - ) - ), - - -- From ranked_runs, keep only the top run per user. - top_runs AS (SELECT * FROM ranked_runs WHERE rank = 1) - - SELECT jsonb_build_object( - 'rankings', (SELECT jsonb_object_agg(g.gpu_type, ( - SELECT jsonb_agg( - jsonb_build_object( - 'user_name', r.user_name, - 'score', r.score, - 'file_name', r.file_name - ) - ORDER BY r.score ASC - ) - FROM top_runs r WHERE r.runner = g.gpu_type))), - - 'leaderboard', (SELECT jsonb_build_object( - 'name', name, - 'deadline', deadline, - 'lang', lang, - 'description', description, - 'reference', reference, - 'gpu_types', (SELECT jsonb_agg(gpu_type) FROM gpu_types) - ) FROM leaderboard_info) - ) AS result FROM (SELECT gpu_type FROM gpu_types) g; - """ - - conn = get_db_connection() - - with conn.cursor() as cur: - cur.execute(query, {"leaderboard_id": leaderboard_id}) - result = cur.fetchone() - - if result is None or not result[0] or not result[0]["leaderboard"]: + data, status_code = get_leaderboard_rankings(leaderboard_id) + if status_code == HTTPStatus.NOT_FOUND: abort(404) + if status_code != HTTPStatus.OK: + abort(502) - data = result[0] - - # Extract leaderboard info - leaderboard_data = data["leaderboard"] - name = leaderboard_data["name"] - deadline = leaderboard_data["deadline"] - time_left = to_time_left(deadline) - - lang = leaderboard_data["lang"] - if lang == "py": - lang = "Python" - - description = leaderboard_data["description"] or "" - description = description.replace("\\n", "\n") - - reference = leaderboard_data["reference"] or "" - reference = reference.replace("\\n", "\n") - - gpu_types = leaderboard_data["gpu_types"] - gpu_types.sort() - - rankings = {} - for gpu_type, ranking_ in data["rankings"].items(): - ranking = [] - prev_score = None - - if ranking_ is not None: - for i, entry in enumerate(ranking_): - entry["rank"] = i + 1 - - if prev_score is not None: - entry["prev_score"] = entry["score"] - prev_score - else: - entry["prev_score"] = None - - ranking.append(entry) - - prev_score = entry["score"] - - if len(ranking) > 0: - rankings[gpu_type] = ranking - + view = to_leaderboard_view(data) return render_template( "leaderboard.html", - name=name, - deadline=deadline, - time_left=time_left, - lang=lang, - gpu_types=gpu_types, - description=description, - reference=reference, - rankings=rankings, + name=view["name"], + deadline=view["deadline"], + time_left=view["time_left"], + lang=view["lang"], + gpu_types=view["gpu_types"], + description=view["description"], + reference=view["reference"], + rankings=view["rankings"], ) diff --git a/kernelboard/lib/kernelbot_client.py b/kernelboard/lib/kernelbot_client.py new file mode 100644 index 0000000..ed5a61a --- /dev/null +++ b/kernelboard/lib/kernelbot_client.py @@ -0,0 +1,96 @@ +import logging +import os +from http import HTTPStatus +from typing import Any + +import requests + +logger = logging.getLogger(__name__) + + +def get_cluster_manager_endpoint(): + env_var = os.getenv("DISCORD_CLUSTER_MANAGER_API_BASE_URL", "") + if not env_var: + logger.warning("DISCORD_CLUSTER_MANAGER_API_BASE_URL is not set") + return env_var.rstrip("/") + + +def get_leaderboard_rankings(leaderboard_id: int) -> tuple[dict[str, Any] | None, int]: + try: + response = requests.get( + f"{get_cluster_manager_endpoint()}/leaderboard/{leaderboard_id}/rankings", + timeout=10, + ) + except requests.RequestException as exc: + logger.exception("kernelbot leaderboard request failed: %s", exc) + return None, HTTPStatus.BAD_GATEWAY + + if response.status_code == HTTPStatus.NOT_FOUND: + return None, HTTPStatus.NOT_FOUND + + if not response.ok: + logger.error( + "kernelbot leaderboard request failed: status=%s body=%s", + response.status_code, + response.text, + ) + return None, HTTPStatus.BAD_GATEWAY + + return response.json(), HTTPStatus.OK + + +def to_leaderboard_view(data: dict[str, Any]): + from kernelboard.lib.time import to_time_left + + leaderboard_data = data["leaderboard"] + name = leaderboard_data["name"] + deadline = leaderboard_data["deadline"] + time_left = to_time_left(deadline) + + lang = leaderboard_data["lang"] + if lang == "py": + lang = "Python" + + description = leaderboard_data["description"] or "" + description = description.replace("\\n", "\n") + + reference = leaderboard_data["reference"] or "" + reference = reference.replace("\\n", "\n") + + benchmarks = leaderboard_data.get("benchmarks") or [] + + gpu_types = leaderboard_data["gpu_types"] + gpu_types.sort() + + rankings = {} + for gpu_type, ranking_ in data["rankings"].items(): + ranking = [] + prev_score = None + + if ranking_ is not None: + for i, entry in enumerate(ranking_): + entry["rank"] = i + 1 + + if prev_score is not None: + entry["prev_score"] = entry["score"] - prev_score + else: + entry["prev_score"] = None + + ranking.append(entry) + + prev_score = entry["score"] + + if len(ranking) > 0: + rankings[gpu_type] = ranking + + return { + "name": name, + "deadline": deadline, + "time_left": time_left, + "lang": lang, + "gpu_types": gpu_types, + "description": description, + "reference": reference, + "benchmarks": benchmarks, + "rankings": rankings, + } diff --git a/tests/api/test_leaderboard_api.py b/tests/api/test_leaderboard_api.py index f5efb34..fa55ab1 100644 --- a/tests/api/test_leaderboard_api.py +++ b/tests/api/test_leaderboard_api.py @@ -39,7 +39,7 @@ def mock_kernelbot_response(payload=None, status_code=200): return response -@patch("kernelboard.api.leaderboard.requests.get") +@patch("kernelboard.lib.kernelbot_client.requests.get") def test_leaderboard(mock_get, client): mock_get.return_value = mock_kernelbot_response() @@ -53,7 +53,7 @@ def test_leaderboard(mock_get, client): ) -@patch("kernelboard.api.leaderboard.requests.get") +@patch("kernelboard.lib.kernelbot_client.requests.get") def test_nonexistent_leaderboard(mock_get, client): mock_get.return_value = mock_kernelbot_response(status_code=404) @@ -61,7 +61,7 @@ def test_nonexistent_leaderboard(mock_get, client): assert response.status_code == 404 -@patch("kernelboard.api.leaderboard.requests.get") +@patch("kernelboard.lib.kernelbot_client.requests.get") def test_leaderboard_no_submissions(mock_get, client): mock_get.return_value = mock_kernelbot_response( kernelbot_leaderboard_payload(rankings={"H100": [], "A100": []}) @@ -74,7 +74,7 @@ def test_leaderboard_no_submissions(mock_get, client): assert res["data"]["rankings"] == {} -@patch("kernelboard.api.leaderboard.requests.get") +@patch("kernelboard.lib.kernelbot_client.requests.get") def test_leaderboard_delegates_secret_filtering_to_kernelbot(mock_get, client): mock_get.return_value = mock_kernelbot_response( kernelbot_leaderboard_payload( diff --git a/tests/conftest.py b/tests/conftest.py index 3170a8b..d7e0d20 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,4 @@ +import os import random import secrets import string @@ -131,7 +132,7 @@ def db_server(): "-f", "tests/data.sql", ], - env={"PGPASSWORD": password}, + env={**os.environ, "PGPASSWORD": password}, ) if result.returncode != 0: diff --git a/tests/test_leaderboard.py b/tests/test_leaderboard.py index dc6b3ba..e796730 100644 --- a/tests/test_leaderboard.py +++ b/tests/test_leaderboard.py @@ -1,64 +1,51 @@ -from kernelboard.lib.db import get_db_connection +def kernelbot_leaderboard_payload(rankings=None, description="description"): + return { + "rankings": rankings + if rankings is not None + else { + "H100": [ + {"user_name": "Alice", "score": 1.25, "file_name": "submission.py"} + ] + }, + "leaderboard": { + "name": "conv2d", + "deadline": "2026-06-29T17:00:00-07:00", + "lang": "py", + "description": description, + "reference": "def ref():\\n pass", + "benchmarks": [{"n": 32}], + "gpu_types": ["H100"], + }, + } -def test_leaderboard(client): - response = client.get("/leaderboard/339") - assert response.status_code == 200 - assert b"conv2d" in response.data +def view_from_payload(payload): + from kernelboard.lib.kernelbot_client import to_leaderboard_view + return to_leaderboard_view(payload) -def test_nonexistent_leaderboard(client): - response = client.get("/leaderboard/1000000") - assert response.status_code == 404 +def test_leaderboard_view(): + view = view_from_payload(kernelbot_leaderboard_payload()) -def test_leaderboard_no_submissions(client, app): - with app.app_context(): - conn = get_db_connection() - with conn.cursor() as cur: - cur.execute( - "UPDATE leaderboard.submission SET leaderboard_id = 340 WHERE leaderboard_id = 339" - ) - conn.commit() # Commit update so the web request sees it. + assert view["name"] == "conv2d" + assert view["rankings"]["H100"][0]["user_name"] == "Alice" - response = client.get("/leaderboard/339") - assert response.status_code == 200 - assert b"No submissions yet" in response.data +def test_leaderboard_view_formats_language(): + view = view_from_payload(kernelbot_leaderboard_payload()) -def test_leaderboard_mathjax(client, app): - # Test that MathJax script is not included for a standard description. - response_no_math = client.get("/leaderboard/339") - assert response_no_math.status_code == 200 - mathjax_script = b'script id="mathjax"' - assert mathjax_script not in response_no_math.data + assert view["lang"] == "Python" - # Test that MathJax script is included when description contains LaTeX. - with app.app_context(): - conn = get_db_connection() - with conn.cursor() as cur: - # Fetch original description. - cur.execute( - "SELECT description FROM leaderboard.leaderboard WHERE id = 339" - ) - result = cur.fetchone() - description = result[0] - # Update the description to include a LaTeX expression. - latex = r"$$\sum_{i=1?^n i$$" - new_description = f"{latex} {description}" +def test_leaderboard_no_submissions(): + view = view_from_payload(kernelbot_leaderboard_payload(rankings={"H100": []})) - cur.execute( - """ - UPDATE leaderboard.leaderboard - SET description = %(new_desc)s - WHERE id = 339 - """, - {"new_desc": new_description}, - ) - conn.commit() + assert view["rankings"] == {} - response_with_math = client.get("/leaderboard/339") - assert response_with_math.status_code == 200 - assert mathjax_script in response_with_math.data - assert latex.encode("utf-8") in response_with_math.data + +def test_leaderboard_preserves_latex_description(): + latex = r"$$\sum_{i=1?^n i$$" + view = view_from_payload(kernelbot_leaderboard_payload(description=f"{latex} description")) + + assert view["description"] == f"{latex} description" diff --git a/tests/test_leaderboard_sql_guard.py b/tests/test_leaderboard_sql_guard.py new file mode 100644 index 0000000..2c569b2 --- /dev/null +++ b/tests/test_leaderboard_sql_guard.py @@ -0,0 +1,55 @@ +import ast +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[1] + + +def _function_body(module_path: str, function_name: str): + tree = ast.parse((PROJECT_ROOT / module_path).read_text()) + for node in tree.body: + if isinstance(node, ast.FunctionDef) and node.name == function_name: + return node + raise AssertionError(f"{function_name} not found in {module_path}") + + +def _called_name(node: ast.Call) -> str: + func = node.func + if isinstance(func, ast.Name): + return func.id + if isinstance(func, ast.Attribute): + return func.attr + return "" + + +def _string_constants(node: ast.AST): + for child in ast.walk(node): + if isinstance(child, ast.Constant) and isinstance(child.value, str): + yield child.value.upper() + + +def assert_no_local_ranking_sql(module_path: str, function_name: str): + function = _function_body(module_path, function_name) + forbidden_calls = {"execute", "executemany", "get_db_connection"} + sql_tokens = ("SELECT ", "INSERT ", "UPDATE ", "DELETE ", "WITH ") + + calls = [ + _called_name(node) + for node in ast.walk(function) + if isinstance(node, ast.Call) + ] + assert not (set(calls) & forbidden_calls) + + sql_strings = [ + value + for value in _string_constants(function) + if any(value.lstrip().startswith(token) for token in sql_tokens) + ] + assert sql_strings == [] + + +def test_api_leaderboard_route_delegates_rankings_to_kernelbot(): + assert_no_local_ranking_sql("kernelboard/api/leaderboard.py", "leaderboard") + + +def test_legacy_leaderboard_route_delegates_rankings_to_kernelbot(): + assert_no_local_ranking_sql("kernelboard/leaderboard.py", "leaderboard")