feat(benchmarks): add 25-defect dataset, manifest loader, and Pass@k evaluation harness

.github/workflows/ci.yml

@@ -30,13 +30,13 @@ jobs:
           pip install ruff mypy pytest pytest-cov
 
       - name: Ruff lint
-        run: ruff check patchwork/ tests/
+        run: ruff check patchwork/ tests/ benchmarks/manifest.py benchmarks/evaluate.py benchmarks/__init__.py
 
       - name: Ruff format check
-        run: ruff format --check patchwork/ tests/
+        run: ruff format --check patchwork/ tests/ benchmarks/manifest.py benchmarks/evaluate.py
 
       - name: Mypy strict
-        run: mypy --strict patchwork/
+        run: mypy --strict patchwork/ benchmarks/manifest.py benchmarks/evaluate.py
 
       - name: Pytest with coverage
         run: pytest tests/ -v --cov=patchwork --cov-report=term-missing

benchmarks/dataset/float_precision_01.py

@@ -0,0 +1,6 @@
+"""Checks whether a + b equals expected, treating floating-point
+rounding error as an acceptable match."""
+
+
+def is_total_correct(a, b, expected):
+    return a + b == expected

benchmarks/dataset/float_precision_02.py

@@ -0,0 +1,6 @@
+"""Checks whether the average of values equals target, treating
+floating-point rounding error as an acceptable match."""
+
+
+def average_equals(values, target):
+    return sum(values) / len(values) == target

benchmarks/dataset/float_precision_03.py

@@ -0,0 +1,6 @@
+"""Checks whether the running total has reached target, treating
+floating-point rounding error as an acceptable match."""
+
+
+def has_reached_target(current, target):
+    return current == target

benchmarks/dataset/float_precision_04.py

@@ -0,0 +1,6 @@
+"""Checks whether applying discount_rate to price yields expected_price,
+treating floating-point rounding error as an acceptable match."""
+
+
+def discount_applied_correctly(price, discount_rate, expected_price):
+    return price * (1 - discount_rate) == expected_price

benchmarks/dataset/float_precision_05.py

@@ -0,0 +1,6 @@
+"""Checks whether value is effectively zero, treating floating-point
+rounding error as an acceptable match."""
+
+
+def is_zero(value):
+    return value == 0.0

benchmarks/dataset/manifest.json

@@ -0,0 +1,33 @@
+{
+  "defects": [
+    {"id": "mutable_default_01", "category": "mutable_default_arguments", "source_filename": "mutable_default_01.py", "oracle_test_filename": "mutable_default_01_test.py", "description": "append_item uses a mutable list default arg, leaking state across calls"},
+    {"id": "mutable_default_02", "category": "mutable_default_arguments", "source_filename": "mutable_default_02.py", "oracle_test_filename": "mutable_default_02_test.py", "description": "add_tag uses a mutable list default arg, leaking state across calls"},
+    {"id": "mutable_default_03", "category": "mutable_default_arguments", "source_filename": "mutable_default_03.py", "oracle_test_filename": "mutable_default_03_test.py", "description": "record_event uses a mutable list default arg, leaking state across calls"},
+    {"id": "mutable_default_04", "category": "mutable_default_arguments", "source_filename": "mutable_default_04.py", "oracle_test_filename": "mutable_default_04_test.py", "description": "increment_count uses a mutable dict default arg, leaking state across calls"},
+    {"id": "mutable_default_05", "category": "mutable_default_arguments", "source_filename": "mutable_default_05.py", "oracle_test_filename": "mutable_default_05_test.py", "description": "Basket.__init__ uses a mutable list default arg, sharing state across instances"},
+
+    {"id": "unhandled_nonetype_01", "category": "unhandled_nonetype", "source_filename": "unhandled_nonetype_01.py", "oracle_test_filename": "unhandled_nonetype_01_test.py", "description": "get_user_email does a deep dict lookup with no .get() fallback, raising KeyError on missing keys"},
+    {"id": "unhandled_nonetype_02", "category": "unhandled_nonetype", "source_filename": "unhandled_nonetype_02.py", "oracle_test_filename": "unhandled_nonetype_02_test.py", "description": "get_theme does a deep dict lookup with no .get() fallback, raising KeyError on missing keys"},
+    {"id": "unhandled_nonetype_03", "category": "unhandled_nonetype", "source_filename": "unhandled_nonetype_03.py", "oracle_test_filename": "unhandled_nonetype_03_test.py", "description": "get_shipping_city does a deep dict lookup with no .get() fallback, raising KeyError on missing keys"},
+    {"id": "unhandled_nonetype_04", "category": "unhandled_nonetype", "source_filename": "unhandled_nonetype_04.py", "oracle_test_filename": "unhandled_nonetype_04_test.py", "description": "get_product_price does a deep dict lookup with no .get() fallback, raising KeyError on missing keys"},
+    {"id": "unhandled_nonetype_05", "category": "unhandled_nonetype", "source_filename": "unhandled_nonetype_05.py", "oracle_test_filename": "unhandled_nonetype_05_test.py", "description": "get_manager_name does a deep dict lookup with no .get() fallback, raising KeyError on missing keys"},
+
+    {"id": "off_by_one_slicing_01", "category": "off_by_one_slicing", "source_filename": "off_by_one_slicing_01.py", "oracle_test_filename": "off_by_one_slicing_01_test.py", "description": "sum_range excludes the end index instead of including it"},
+    {"id": "off_by_one_slicing_02", "category": "off_by_one_slicing", "source_filename": "off_by_one_slicing_02.py", "oracle_test_filename": "off_by_one_slicing_02_test.py", "description": "get_last_n_items slices one element too many due to an off-by-one on the negative index"},
+    {"id": "off_by_one_slicing_03", "category": "off_by_one_slicing", "source_filename": "off_by_one_slicing_03.py", "oracle_test_filename": "off_by_one_slicing_03_test.py", "description": "binary_search uses low < high instead of low <= high, missing the boundary element"},
+    {"id": "off_by_one_slicing_04", "category": "off_by_one_slicing", "source_filename": "off_by_one_slicing_04.py", "oracle_test_filename": "off_by_one_slicing_04_test.py", "description": "get_page_items drops the last item of every page due to an off-by-one on the slice end"},
+    {"id": "off_by_one_slicing_05", "category": "off_by_one_slicing", "source_filename": "off_by_one_slicing_05.py", "oracle_test_filename": "off_by_one_slicing_05_test.py", "description": "first_n_chars drops the last requested character due to an off-by-one on the slice end"},
+
+    {"id": "float_precision_01", "category": "float_precision", "source_filename": "float_precision_01.py", "oracle_test_filename": "float_precision_01_test.py", "description": "is_total_correct uses direct == on floats instead of math.isclose"},
+    {"id": "float_precision_02", "category": "float_precision", "source_filename": "float_precision_02.py", "oracle_test_filename": "float_precision_02_test.py", "description": "average_equals uses direct == on floats instead of math.isclose"},
+    {"id": "float_precision_03", "category": "float_precision", "source_filename": "float_precision_03.py", "oracle_test_filename": "float_precision_03_test.py", "description": "has_reached_target uses direct == on floats instead of math.isclose"},
+    {"id": "float_precision_04", "category": "float_precision", "source_filename": "float_precision_04.py", "oracle_test_filename": "float_precision_04_test.py", "description": "discount_applied_correctly uses direct == on floats instead of math.isclose"},
+    {"id": "float_precision_05", "category": "float_precision", "source_filename": "float_precision_05.py", "oracle_test_filename": "float_precision_05_test.py", "description": "is_zero uses direct == on floats instead of math.isclose"},
+
+    {"id": "resource_leaks_01", "category": "resource_leaks", "source_filename": "resource_leaks_01.py", "oracle_test_filename": "resource_leaks_01_test.py", "description": "read_file_contents opens a file without a context manager, leaking the handle"},
+    {"id": "resource_leaks_02", "category": "resource_leaks", "source_filename": "resource_leaks_02.py", "oracle_test_filename": "resource_leaks_02_test.py", "description": "write_log_line opens a file without a context manager, leaking the handle"},
+    {"id": "resource_leaks_03", "category": "resource_leaks", "source_filename": "resource_leaks_03.py", "oracle_test_filename": "resource_leaks_03_test.py", "description": "count_lines opens a file without a context manager, leaking the handle"},
+    {"id": "resource_leaks_04", "category": "resource_leaks", "source_filename": "resource_leaks_04.py", "oracle_test_filename": "resource_leaks_04_test.py", "description": "read_stripped_lines opens a file without a context manager, leaking the handle"},
+    {"id": "resource_leaks_05", "category": "resource_leaks", "source_filename": "resource_leaks_05.py", "oracle_test_filename": "resource_leaks_05_test.py", "description": "read_json_file opens a file without a context manager, leaking the handle"}
+  ]
+}

benchmarks/dataset/mutable_default_01.py

@@ -0,0 +1,8 @@
+"""Appends an item to a running list and returns it. Each call with no
+explicit target_list should start a fresh empty list -- calls must not
+leak state into each other."""
+
+
+def append_item(item, target_list=[]):
+    target_list.append(item)
+    return target_list

benchmarks/dataset/mutable_default_02.py

@@ -0,0 +1,9 @@
+"""Adds a tag to a collection of tags and returns the collection. Each
+call with no explicit tags argument should start from an empty
+collection, independent of any previous call."""
+
+
+def add_tag(tag, tags=[]):
+    if tag not in tags:
+        tags.append(tag)
+    return tags

benchmarks/dataset/mutable_default_03.py

@@ -0,0 +1,8 @@
+"""Records an event with a timestamp into a log and returns the log.
+Each call with no explicit log argument should produce a log containing
+only that call's own events."""
+
+
+def record_event(event, timestamp, log=[]):
+    log.append((event, timestamp))
+    return log

benchmarks/dataset/mutable_default_04.py

@@ -0,0 +1,8 @@
+"""Increments the count for a key in a counts dictionary and returns
+it. Each call with no explicit counts argument should start from an
+empty dictionary."""
+
+
+def increment_count(key, counts={}):
+    counts[key] = counts.get(key, 0) + 1
+    return counts

benchmarks/dataset/mutable_default_05.py

@@ -0,0 +1,11 @@
+"""A Basket holds items added to it. Each Basket instance created with
+no explicit items argument should have its own independent contents,
+not share a list with every other Basket instance."""
+
+
+class Basket:
+    def __init__(self, items=[]):
+        self.items = items
+
+    def add(self, item):
+        self.items.append(item)

benchmarks/dataset/off_by_one_slicing_01.py

@@ -0,0 +1,6 @@
+"""Returns the sum of numbers from index start through end, inclusive
+of both endpoints."""
+
+
+def sum_range(numbers, start, end):
+    return sum(numbers[start:end])

benchmarks/dataset/off_by_one_slicing_02.py

@@ -0,0 +1,5 @@
+"""Returns the last n items of the list, in their original order."""
+
+
+def get_last_n_items(items, n):
+    return items[-n - 1 :]

benchmarks/dataset/off_by_one_slicing_03.py

@@ -0,0 +1,15 @@
+"""Returns the index of target in sorted_list using binary search, or
+-1 if target is not present. sorted_list is sorted ascending."""
+
+
+def binary_search(sorted_list, target):
+    low, high = 0, len(sorted_list) - 1
+    while low < high:
+        mid = (low + high) // 2
+        if sorted_list[mid] == target:
+            return mid
+        elif sorted_list[mid] < target:
+            low = mid + 1
+        else:
+            high = mid - 1
+    return -1

benchmarks/dataset/off_by_one_slicing_04.py

@@ -0,0 +1,7 @@
+"""Returns the items belonging to the given 0-indexed page, where each
+page holds page_size items."""
+
+
+def get_page_items(items, page, page_size):
+    start = page * page_size
+    return items[start : start + page_size - 1]

benchmarks/dataset/off_by_one_slicing_05.py

@@ -0,0 +1,5 @@
+"""Returns the first n characters of s."""
+
+
+def first_n_chars(s, n):
+    return s[0 : n - 1]

benchmarks/dataset/oracle_tests/float_precision_01_test.py

@@ -0,0 +1,7 @@
+def test_accepts_rounding_error():
+    # 0.1 + 0.2 != 0.3 exactly in float -- must not use bare ==
+    assert is_total_correct(0.1, 0.2, 0.3) is True
+
+
+def test_rejects_genuinely_wrong_total():
+    assert is_total_correct(1.0, 1.0, 3.0) is False

benchmarks/dataset/oracle_tests/float_precision_02_test.py

@@ -0,0 +1,6 @@
+def test_accepts_rounding_error():
+    assert average_equals([0.1, 0.2, 0.3], 0.2) is True
+
+
+def test_rejects_genuinely_wrong_average():
+    assert average_equals([1.0, 2.0, 3.0], 5.0) is False

benchmarks/dataset/oracle_tests/float_precision_03_test.py

@@ -0,0 +1,6 @@
+def test_accepts_rounding_error():
+    assert has_reached_target(sum([0.1, 0.1, 0.1]), 0.3) is True
+
+
+def test_rejects_genuinely_wrong_total():
+    assert has_reached_target(1.0, 2.0) is False

benchmarks/dataset/oracle_tests/float_precision_04_test.py

@@ -0,0 +1,6 @@
+def test_accepts_rounding_error():
+    assert discount_applied_correctly(4.35, 0.1, 3.915) is True
+
+
+def test_rejects_genuinely_wrong_price():
+    assert discount_applied_correctly(10.0, 0.1, 5.0) is False

benchmarks/dataset/oracle_tests/float_precision_05_test.py

@@ -0,0 +1,6 @@
+def test_accepts_rounding_error():
+    assert is_zero(1.0 - 0.9 - 0.1) is True
+
+
+def test_rejects_genuinely_nonzero_value():
+    assert is_zero(1.0) is False

benchmarks/dataset/oracle_tests/mutable_default_01_test.py

@@ -0,0 +1,15 @@
+def test_first_call_starts_empty():
+    result = append_item("a")
+    assert result == ["a"]
+
+
+def test_second_call_does_not_see_first_calls_item():
+    append_item("a")
+    result = append_item("b")
+    assert result == ["b"]
+
+
+def test_explicit_target_list_still_works():
+    target = []
+    result = append_item("x", target)
+    assert result == ["x"]

benchmarks/dataset/oracle_tests/mutable_default_02_test.py

@@ -0,0 +1,15 @@
+def test_first_call_starts_empty():
+    result = add_tag("urgent")
+    assert result == ["urgent"]
+
+
+def test_second_call_does_not_see_first_calls_tag():
+    add_tag("urgent")
+    result = add_tag("archived")
+    assert result == ["archived"]
+
+
+def test_duplicate_tag_not_added_twice():
+    tags = ["urgent"]
+    result = add_tag("urgent", tags)
+    assert result == ["urgent"]

benchmarks/dataset/oracle_tests/mutable_default_03_test.py

@@ -0,0 +1,9 @@
+def test_first_call_starts_empty():
+    result = record_event("login", 100)
+    assert result == [("login", 100)]
+
+
+def test_second_call_does_not_see_first_calls_event():
+    record_event("login", 100)
+    result = record_event("logout", 200)
+    assert result == [("logout", 200)]

benchmarks/dataset/oracle_tests/mutable_default_04_test.py

@@ -0,0 +1,16 @@
+def test_first_call_starts_empty():
+    result = increment_count("a")
+    assert result == {"a": 1}
+
+
+def test_second_call_does_not_see_first_calls_counts():
+    increment_count("a")
+    result = increment_count("b")
+    assert result == {"b": 1}
+
+
+def test_same_key_increments_within_explicit_dict():
+    counts = {}
+    increment_count("a", counts)
+    result = increment_count("a", counts)
+    assert result == {"a": 2}

benchmarks/dataset/oracle_tests/mutable_default_05_test.py

@@ -0,0 +1,10 @@
+def test_new_basket_starts_empty():
+    basket = Basket()
+    assert basket.items == []
+
+
+def test_two_baskets_do_not_share_items():
+    basket_a = Basket()
+    basket_a.add("apple")
+    basket_b = Basket()
+    assert basket_b.items == []

benchmarks/dataset/oracle_tests/off_by_one_slicing_01_test.py

@@ -0,0 +1,6 @@
+def test_inclusive_range_sum():
+    assert sum_range([1, 2, 3, 4, 5], 1, 3) == 9  # indices 1,2,3 -> 2+3+4
+
+
+def test_single_index_range():
+    assert sum_range([10, 20, 30], 1, 1) == 20

benchmarks/dataset/oracle_tests/off_by_one_slicing_02_test.py

@@ -0,0 +1,6 @@
+def test_last_two_items():
+    assert get_last_n_items([1, 2, 3, 4, 5], 2) == [4, 5]
+
+
+def test_last_one_item():
+    assert get_last_n_items([1, 2, 3], 1) == [3]

benchmarks/dataset/oracle_tests/off_by_one_slicing_03_test.py

@@ -0,0 +1,10 @@
+def test_finds_last_element():
+    assert binary_search([1, 3, 5, 7, 9], 9) == 4
+
+
+def test_finds_first_element():
+    assert binary_search([1, 3, 5, 7, 9], 1) == 0
+
+
+def test_missing_target_returns_negative_one():
+    assert binary_search([1, 3, 5, 7, 9], 4) == -1

benchmarks/dataset/oracle_tests/off_by_one_slicing_04_test.py

@@ -0,0 +1,6 @@
+def test_first_page():
+    assert get_page_items([1, 2, 3, 4, 5, 6], 0, 3) == [1, 2, 3]
+
+
+def test_second_page():
+    assert get_page_items([1, 2, 3, 4, 5, 6], 1, 3) == [4, 5, 6]

benchmarks/dataset/oracle_tests/off_by_one_slicing_05_test.py

@@ -0,0 +1,6 @@
+def test_first_five_chars():
+    assert first_n_chars("hello world", 5) == "hello"
+
+
+def test_first_one_char():
+    assert first_n_chars("abc", 1) == "a"

benchmarks/dataset/oracle_tests/resource_leaks_01_test.py

@@ -0,0 +1,9 @@
+from unittest.mock import mock_open, patch
+
+
+def test_file_handle_is_closed():
+    m = mock_open(read_data="hello")
+    with patch("builtins.open", m):
+        result = read_file_contents("dummy.txt")
+    assert result == "hello"
+    assert m.return_value.__exit__.called

benchmarks/dataset/oracle_tests/resource_leaks_02_test.py

@@ -0,0 +1,8 @@
+from unittest.mock import mock_open, patch
+
+
+def test_file_handle_is_closed():
+    m = mock_open()
+    with patch("builtins.open", m):
+        write_log_line("dummy.txt", "hello")
+    assert m.return_value.__exit__.called

benchmarks/dataset/oracle_tests/resource_leaks_03_test.py

@@ -0,0 +1,9 @@
+from unittest.mock import mock_open, patch
+
+
+def test_file_handle_is_closed():
+    m = mock_open(read_data="a\nb\nc\n")
+    with patch("builtins.open", m):
+        result = count_lines("dummy.txt")
+    assert result == 3
+    assert m.return_value.__exit__.called

benchmarks/dataset/oracle_tests/resource_leaks_04_test.py

@@ -0,0 +1,9 @@
+from unittest.mock import mock_open, patch
+
+
+def test_file_handle_is_closed():
+    m = mock_open(read_data="a \n b \n c\n")
+    with patch("builtins.open", m):
+        result = read_stripped_lines("dummy.txt")
+    assert result == ["a", "b", "c"]
+    assert m.return_value.__exit__.called

benchmarks/dataset/oracle_tests/resource_leaks_05_test.py

@@ -0,0 +1,9 @@
+from unittest.mock import mock_open, patch
+
+
+def test_file_handle_is_closed():
+    m = mock_open(read_data='{"a": 1}')
+    with patch("builtins.open", m):
+        result = read_json_file("dummy.json")
+    assert result == {"a": 1}
+    assert m.return_value.__exit__.called

benchmarks/dataset/oracle_tests/unhandled_nonetype_01_test.py

@@ -0,0 +1,13 @@
+def test_returns_email_when_present():
+    user = {"profile": {"email": "a@example.com"}}
+    assert get_user_email(user) == "a@example.com"
+
+
+def test_returns_none_when_profile_missing():
+    user = {}
+    assert get_user_email(user) is None
+
+
+def test_returns_none_when_email_missing():
+    user = {"profile": {}}
+    assert get_user_email(user) is None