diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index caf9ca9..08c5bf7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,13 +30,13 @@ jobs: pip install ruff mypy pytest pytest-cov - name: Ruff lint - run: ruff check patchwork/ tests/ + run: ruff check patchwork/ tests/ benchmarks/manifest.py benchmarks/evaluate.py benchmarks/__init__.py - name: Ruff format check - run: ruff format --check patchwork/ tests/ + run: ruff format --check patchwork/ tests/ benchmarks/manifest.py benchmarks/evaluate.py - name: Mypy strict - run: mypy --strict patchwork/ + run: mypy --strict patchwork/ benchmarks/manifest.py benchmarks/evaluate.py - name: Pytest with coverage run: pytest tests/ -v --cov=patchwork --cov-report=term-missing diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/dataset/float_precision_01.py b/benchmarks/dataset/float_precision_01.py new file mode 100644 index 0000000..5ea8485 --- /dev/null +++ b/benchmarks/dataset/float_precision_01.py @@ -0,0 +1,6 @@ +"""Checks whether a + b equals expected, treating floating-point +rounding error as an acceptable match.""" + + +def is_total_correct(a, b, expected): + return a + b == expected diff --git a/benchmarks/dataset/float_precision_02.py b/benchmarks/dataset/float_precision_02.py new file mode 100644 index 0000000..2ba521e --- /dev/null +++ b/benchmarks/dataset/float_precision_02.py @@ -0,0 +1,6 @@ +"""Checks whether the average of values equals target, treating +floating-point rounding error as an acceptable match.""" + + +def average_equals(values, target): + return sum(values) / len(values) == target diff --git a/benchmarks/dataset/float_precision_03.py b/benchmarks/dataset/float_precision_03.py new file mode 100644 index 0000000..ed752fc --- /dev/null +++ b/benchmarks/dataset/float_precision_03.py @@ -0,0 +1,6 @@ +"""Checks whether the running total has reached target, treating +floating-point rounding error as an acceptable match.""" + + +def has_reached_target(current, target): + return current == target diff --git a/benchmarks/dataset/float_precision_04.py b/benchmarks/dataset/float_precision_04.py new file mode 100644 index 0000000..4d27d94 --- /dev/null +++ b/benchmarks/dataset/float_precision_04.py @@ -0,0 +1,6 @@ +"""Checks whether applying discount_rate to price yields expected_price, +treating floating-point rounding error as an acceptable match.""" + + +def discount_applied_correctly(price, discount_rate, expected_price): + return price * (1 - discount_rate) == expected_price diff --git a/benchmarks/dataset/float_precision_05.py b/benchmarks/dataset/float_precision_05.py new file mode 100644 index 0000000..0873d00 --- /dev/null +++ b/benchmarks/dataset/float_precision_05.py @@ -0,0 +1,6 @@ +"""Checks whether value is effectively zero, treating floating-point +rounding error as an acceptable match.""" + + +def is_zero(value): + return value == 0.0 diff --git a/benchmarks/dataset/manifest.json b/benchmarks/dataset/manifest.json new file mode 100644 index 0000000..6209c88 --- /dev/null +++ b/benchmarks/dataset/manifest.json @@ -0,0 +1,33 @@ +{ + "defects": [ + {"id": "mutable_default_01", "category": "mutable_default_arguments", "source_filename": "mutable_default_01.py", "oracle_test_filename": "mutable_default_01_test.py", "description": "append_item uses a mutable list default arg, leaking state across calls"}, + {"id": "mutable_default_02", "category": "mutable_default_arguments", "source_filename": "mutable_default_02.py", "oracle_test_filename": "mutable_default_02_test.py", "description": "add_tag uses a mutable list default arg, leaking state across calls"}, + {"id": "mutable_default_03", "category": "mutable_default_arguments", "source_filename": "mutable_default_03.py", "oracle_test_filename": "mutable_default_03_test.py", "description": "record_event uses a mutable list default arg, leaking state across calls"}, + {"id": "mutable_default_04", "category": "mutable_default_arguments", "source_filename": "mutable_default_04.py", "oracle_test_filename": "mutable_default_04_test.py", "description": "increment_count uses a mutable dict default arg, leaking state across calls"}, + {"id": "mutable_default_05", "category": "mutable_default_arguments", "source_filename": "mutable_default_05.py", "oracle_test_filename": "mutable_default_05_test.py", "description": "Basket.__init__ uses a mutable list default arg, sharing state across instances"}, + + {"id": "unhandled_nonetype_01", "category": "unhandled_nonetype", "source_filename": "unhandled_nonetype_01.py", "oracle_test_filename": "unhandled_nonetype_01_test.py", "description": "get_user_email does a deep dict lookup with no .get() fallback, raising KeyError on missing keys"}, + {"id": "unhandled_nonetype_02", "category": "unhandled_nonetype", "source_filename": "unhandled_nonetype_02.py", "oracle_test_filename": "unhandled_nonetype_02_test.py", "description": "get_theme does a deep dict lookup with no .get() fallback, raising KeyError on missing keys"}, + {"id": "unhandled_nonetype_03", "category": "unhandled_nonetype", "source_filename": "unhandled_nonetype_03.py", "oracle_test_filename": "unhandled_nonetype_03_test.py", "description": "get_shipping_city does a deep dict lookup with no .get() fallback, raising KeyError on missing keys"}, + {"id": "unhandled_nonetype_04", "category": "unhandled_nonetype", "source_filename": "unhandled_nonetype_04.py", "oracle_test_filename": "unhandled_nonetype_04_test.py", "description": "get_product_price does a deep dict lookup with no .get() fallback, raising KeyError on missing keys"}, + {"id": "unhandled_nonetype_05", "category": "unhandled_nonetype", "source_filename": "unhandled_nonetype_05.py", "oracle_test_filename": "unhandled_nonetype_05_test.py", "description": "get_manager_name does a deep dict lookup with no .get() fallback, raising KeyError on missing keys"}, + + {"id": "off_by_one_slicing_01", "category": "off_by_one_slicing", "source_filename": "off_by_one_slicing_01.py", "oracle_test_filename": "off_by_one_slicing_01_test.py", "description": "sum_range excludes the end index instead of including it"}, + {"id": "off_by_one_slicing_02", "category": "off_by_one_slicing", "source_filename": "off_by_one_slicing_02.py", "oracle_test_filename": "off_by_one_slicing_02_test.py", "description": "get_last_n_items slices one element too many due to an off-by-one on the negative index"}, + {"id": "off_by_one_slicing_03", "category": "off_by_one_slicing", "source_filename": "off_by_one_slicing_03.py", "oracle_test_filename": "off_by_one_slicing_03_test.py", "description": "binary_search uses low < high instead of low <= high, missing the boundary element"}, + {"id": "off_by_one_slicing_04", "category": "off_by_one_slicing", "source_filename": "off_by_one_slicing_04.py", "oracle_test_filename": "off_by_one_slicing_04_test.py", "description": "get_page_items drops the last item of every page due to an off-by-one on the slice end"}, + {"id": "off_by_one_slicing_05", "category": "off_by_one_slicing", "source_filename": "off_by_one_slicing_05.py", "oracle_test_filename": "off_by_one_slicing_05_test.py", "description": "first_n_chars drops the last requested character due to an off-by-one on the slice end"}, + + {"id": "float_precision_01", "category": "float_precision", "source_filename": "float_precision_01.py", "oracle_test_filename": "float_precision_01_test.py", "description": "is_total_correct uses direct == on floats instead of math.isclose"}, + {"id": "float_precision_02", "category": "float_precision", "source_filename": "float_precision_02.py", "oracle_test_filename": "float_precision_02_test.py", "description": "average_equals uses direct == on floats instead of math.isclose"}, + {"id": "float_precision_03", "category": "float_precision", "source_filename": "float_precision_03.py", "oracle_test_filename": "float_precision_03_test.py", "description": "has_reached_target uses direct == on floats instead of math.isclose"}, + {"id": "float_precision_04", "category": "float_precision", "source_filename": "float_precision_04.py", "oracle_test_filename": "float_precision_04_test.py", "description": "discount_applied_correctly uses direct == on floats instead of math.isclose"}, + {"id": "float_precision_05", "category": "float_precision", "source_filename": "float_precision_05.py", "oracle_test_filename": "float_precision_05_test.py", "description": "is_zero uses direct == on floats instead of math.isclose"}, + + {"id": "resource_leaks_01", "category": "resource_leaks", "source_filename": "resource_leaks_01.py", "oracle_test_filename": "resource_leaks_01_test.py", "description": "read_file_contents opens a file without a context manager, leaking the handle"}, + {"id": "resource_leaks_02", "category": "resource_leaks", "source_filename": "resource_leaks_02.py", "oracle_test_filename": "resource_leaks_02_test.py", "description": "write_log_line opens a file without a context manager, leaking the handle"}, + {"id": "resource_leaks_03", "category": "resource_leaks", "source_filename": "resource_leaks_03.py", "oracle_test_filename": "resource_leaks_03_test.py", "description": "count_lines opens a file without a context manager, leaking the handle"}, + {"id": "resource_leaks_04", "category": "resource_leaks", "source_filename": "resource_leaks_04.py", "oracle_test_filename": "resource_leaks_04_test.py", "description": "read_stripped_lines opens a file without a context manager, leaking the handle"}, + {"id": "resource_leaks_05", "category": "resource_leaks", "source_filename": "resource_leaks_05.py", "oracle_test_filename": "resource_leaks_05_test.py", "description": "read_json_file opens a file without a context manager, leaking the handle"} + ] +} \ No newline at end of file diff --git a/benchmarks/dataset/mutable_default_01.py b/benchmarks/dataset/mutable_default_01.py new file mode 100644 index 0000000..2781545 --- /dev/null +++ b/benchmarks/dataset/mutable_default_01.py @@ -0,0 +1,8 @@ +"""Appends an item to a running list and returns it. Each call with no +explicit target_list should start a fresh empty list -- calls must not +leak state into each other.""" + + +def append_item(item, target_list=[]): + target_list.append(item) + return target_list \ No newline at end of file diff --git a/benchmarks/dataset/mutable_default_02.py b/benchmarks/dataset/mutable_default_02.py new file mode 100644 index 0000000..a6e27fc --- /dev/null +++ b/benchmarks/dataset/mutable_default_02.py @@ -0,0 +1,9 @@ +"""Adds a tag to a collection of tags and returns the collection. Each +call with no explicit tags argument should start from an empty +collection, independent of any previous call.""" + + +def add_tag(tag, tags=[]): + if tag not in tags: + tags.append(tag) + return tags \ No newline at end of file diff --git a/benchmarks/dataset/mutable_default_03.py b/benchmarks/dataset/mutable_default_03.py new file mode 100644 index 0000000..15a243a --- /dev/null +++ b/benchmarks/dataset/mutable_default_03.py @@ -0,0 +1,8 @@ +"""Records an event with a timestamp into a log and returns the log. +Each call with no explicit log argument should produce a log containing +only that call's own events.""" + + +def record_event(event, timestamp, log=[]): + log.append((event, timestamp)) + return log \ No newline at end of file diff --git a/benchmarks/dataset/mutable_default_04.py b/benchmarks/dataset/mutable_default_04.py new file mode 100644 index 0000000..be38056 --- /dev/null +++ b/benchmarks/dataset/mutable_default_04.py @@ -0,0 +1,8 @@ +"""Increments the count for a key in a counts dictionary and returns +it. Each call with no explicit counts argument should start from an +empty dictionary.""" + + +def increment_count(key, counts={}): + counts[key] = counts.get(key, 0) + 1 + return counts \ No newline at end of file diff --git a/benchmarks/dataset/mutable_default_05.py b/benchmarks/dataset/mutable_default_05.py new file mode 100644 index 0000000..76e6edd --- /dev/null +++ b/benchmarks/dataset/mutable_default_05.py @@ -0,0 +1,11 @@ +"""A Basket holds items added to it. Each Basket instance created with +no explicit items argument should have its own independent contents, +not share a list with every other Basket instance.""" + + +class Basket: + def __init__(self, items=[]): + self.items = items + + def add(self, item): + self.items.append(item) \ No newline at end of file diff --git a/benchmarks/dataset/off_by_one_slicing_01.py b/benchmarks/dataset/off_by_one_slicing_01.py new file mode 100644 index 0000000..1b08f6e --- /dev/null +++ b/benchmarks/dataset/off_by_one_slicing_01.py @@ -0,0 +1,6 @@ +"""Returns the sum of numbers from index start through end, inclusive +of both endpoints.""" + + +def sum_range(numbers, start, end): + return sum(numbers[start:end]) diff --git a/benchmarks/dataset/off_by_one_slicing_02.py b/benchmarks/dataset/off_by_one_slicing_02.py new file mode 100644 index 0000000..55ed447 --- /dev/null +++ b/benchmarks/dataset/off_by_one_slicing_02.py @@ -0,0 +1,5 @@ +"""Returns the last n items of the list, in their original order.""" + + +def get_last_n_items(items, n): + return items[-n - 1 :] diff --git a/benchmarks/dataset/off_by_one_slicing_03.py b/benchmarks/dataset/off_by_one_slicing_03.py new file mode 100644 index 0000000..a81367a --- /dev/null +++ b/benchmarks/dataset/off_by_one_slicing_03.py @@ -0,0 +1,15 @@ +"""Returns the index of target in sorted_list using binary search, or +-1 if target is not present. sorted_list is sorted ascending.""" + + +def binary_search(sorted_list, target): + low, high = 0, len(sorted_list) - 1 + while low < high: + mid = (low + high) // 2 + if sorted_list[mid] == target: + return mid + elif sorted_list[mid] < target: + low = mid + 1 + else: + high = mid - 1 + return -1 diff --git a/benchmarks/dataset/off_by_one_slicing_04.py b/benchmarks/dataset/off_by_one_slicing_04.py new file mode 100644 index 0000000..f5a3ea1 --- /dev/null +++ b/benchmarks/dataset/off_by_one_slicing_04.py @@ -0,0 +1,7 @@ +"""Returns the items belonging to the given 0-indexed page, where each +page holds page_size items.""" + + +def get_page_items(items, page, page_size): + start = page * page_size + return items[start : start + page_size - 1] diff --git a/benchmarks/dataset/off_by_one_slicing_05.py b/benchmarks/dataset/off_by_one_slicing_05.py new file mode 100644 index 0000000..aba6387 --- /dev/null +++ b/benchmarks/dataset/off_by_one_slicing_05.py @@ -0,0 +1,5 @@ +"""Returns the first n characters of s.""" + + +def first_n_chars(s, n): + return s[0 : n - 1] diff --git a/benchmarks/dataset/oracle_tests/float_precision_01_test.py b/benchmarks/dataset/oracle_tests/float_precision_01_test.py new file mode 100644 index 0000000..21ee016 --- /dev/null +++ b/benchmarks/dataset/oracle_tests/float_precision_01_test.py @@ -0,0 +1,7 @@ +def test_accepts_rounding_error(): + # 0.1 + 0.2 != 0.3 exactly in float -- must not use bare == + assert is_total_correct(0.1, 0.2, 0.3) is True + + +def test_rejects_genuinely_wrong_total(): + assert is_total_correct(1.0, 1.0, 3.0) is False diff --git a/benchmarks/dataset/oracle_tests/float_precision_02_test.py b/benchmarks/dataset/oracle_tests/float_precision_02_test.py new file mode 100644 index 0000000..44d5702 --- /dev/null +++ b/benchmarks/dataset/oracle_tests/float_precision_02_test.py @@ -0,0 +1,6 @@ +def test_accepts_rounding_error(): + assert average_equals([0.1, 0.2, 0.3], 0.2) is True + + +def test_rejects_genuinely_wrong_average(): + assert average_equals([1.0, 2.0, 3.0], 5.0) is False diff --git a/benchmarks/dataset/oracle_tests/float_precision_03_test.py b/benchmarks/dataset/oracle_tests/float_precision_03_test.py new file mode 100644 index 0000000..f4e96e1 --- /dev/null +++ b/benchmarks/dataset/oracle_tests/float_precision_03_test.py @@ -0,0 +1,6 @@ +def test_accepts_rounding_error(): + assert has_reached_target(sum([0.1, 0.1, 0.1]), 0.3) is True + + +def test_rejects_genuinely_wrong_total(): + assert has_reached_target(1.0, 2.0) is False diff --git a/benchmarks/dataset/oracle_tests/float_precision_04_test.py b/benchmarks/dataset/oracle_tests/float_precision_04_test.py new file mode 100644 index 0000000..0cc241a --- /dev/null +++ b/benchmarks/dataset/oracle_tests/float_precision_04_test.py @@ -0,0 +1,6 @@ +def test_accepts_rounding_error(): + assert discount_applied_correctly(4.35, 0.1, 3.915) is True + + +def test_rejects_genuinely_wrong_price(): + assert discount_applied_correctly(10.0, 0.1, 5.0) is False diff --git a/benchmarks/dataset/oracle_tests/float_precision_05_test.py b/benchmarks/dataset/oracle_tests/float_precision_05_test.py new file mode 100644 index 0000000..64c1279 --- /dev/null +++ b/benchmarks/dataset/oracle_tests/float_precision_05_test.py @@ -0,0 +1,6 @@ +def test_accepts_rounding_error(): + assert is_zero(1.0 - 0.9 - 0.1) is True + + +def test_rejects_genuinely_nonzero_value(): + assert is_zero(1.0) is False diff --git a/benchmarks/dataset/oracle_tests/mutable_default_01_test.py b/benchmarks/dataset/oracle_tests/mutable_default_01_test.py new file mode 100644 index 0000000..e61b83c --- /dev/null +++ b/benchmarks/dataset/oracle_tests/mutable_default_01_test.py @@ -0,0 +1,15 @@ +def test_first_call_starts_empty(): + result = append_item("a") + assert result == ["a"] + + +def test_second_call_does_not_see_first_calls_item(): + append_item("a") + result = append_item("b") + assert result == ["b"] + + +def test_explicit_target_list_still_works(): + target = [] + result = append_item("x", target) + assert result == ["x"] diff --git a/benchmarks/dataset/oracle_tests/mutable_default_02_test.py b/benchmarks/dataset/oracle_tests/mutable_default_02_test.py new file mode 100644 index 0000000..3a4bfd2 --- /dev/null +++ b/benchmarks/dataset/oracle_tests/mutable_default_02_test.py @@ -0,0 +1,15 @@ +def test_first_call_starts_empty(): + result = add_tag("urgent") + assert result == ["urgent"] + + +def test_second_call_does_not_see_first_calls_tag(): + add_tag("urgent") + result = add_tag("archived") + assert result == ["archived"] + + +def test_duplicate_tag_not_added_twice(): + tags = ["urgent"] + result = add_tag("urgent", tags) + assert result == ["urgent"] diff --git a/benchmarks/dataset/oracle_tests/mutable_default_03_test.py b/benchmarks/dataset/oracle_tests/mutable_default_03_test.py new file mode 100644 index 0000000..fae8ea8 --- /dev/null +++ b/benchmarks/dataset/oracle_tests/mutable_default_03_test.py @@ -0,0 +1,9 @@ +def test_first_call_starts_empty(): + result = record_event("login", 100) + assert result == [("login", 100)] + + +def test_second_call_does_not_see_first_calls_event(): + record_event("login", 100) + result = record_event("logout", 200) + assert result == [("logout", 200)] diff --git a/benchmarks/dataset/oracle_tests/mutable_default_04_test.py b/benchmarks/dataset/oracle_tests/mutable_default_04_test.py new file mode 100644 index 0000000..d6b410b --- /dev/null +++ b/benchmarks/dataset/oracle_tests/mutable_default_04_test.py @@ -0,0 +1,16 @@ +def test_first_call_starts_empty(): + result = increment_count("a") + assert result == {"a": 1} + + +def test_second_call_does_not_see_first_calls_counts(): + increment_count("a") + result = increment_count("b") + assert result == {"b": 1} + + +def test_same_key_increments_within_explicit_dict(): + counts = {} + increment_count("a", counts) + result = increment_count("a", counts) + assert result == {"a": 2} diff --git a/benchmarks/dataset/oracle_tests/mutable_default_05_test.py b/benchmarks/dataset/oracle_tests/mutable_default_05_test.py new file mode 100644 index 0000000..82e103c --- /dev/null +++ b/benchmarks/dataset/oracle_tests/mutable_default_05_test.py @@ -0,0 +1,10 @@ +def test_new_basket_starts_empty(): + basket = Basket() + assert basket.items == [] + + +def test_two_baskets_do_not_share_items(): + basket_a = Basket() + basket_a.add("apple") + basket_b = Basket() + assert basket_b.items == [] diff --git a/benchmarks/dataset/oracle_tests/off_by_one_slicing_01_test.py b/benchmarks/dataset/oracle_tests/off_by_one_slicing_01_test.py new file mode 100644 index 0000000..ce56c1c --- /dev/null +++ b/benchmarks/dataset/oracle_tests/off_by_one_slicing_01_test.py @@ -0,0 +1,6 @@ +def test_inclusive_range_sum(): + assert sum_range([1, 2, 3, 4, 5], 1, 3) == 9 # indices 1,2,3 -> 2+3+4 + + +def test_single_index_range(): + assert sum_range([10, 20, 30], 1, 1) == 20 diff --git a/benchmarks/dataset/oracle_tests/off_by_one_slicing_02_test.py b/benchmarks/dataset/oracle_tests/off_by_one_slicing_02_test.py new file mode 100644 index 0000000..153fc58 --- /dev/null +++ b/benchmarks/dataset/oracle_tests/off_by_one_slicing_02_test.py @@ -0,0 +1,6 @@ +def test_last_two_items(): + assert get_last_n_items([1, 2, 3, 4, 5], 2) == [4, 5] + + +def test_last_one_item(): + assert get_last_n_items([1, 2, 3], 1) == [3] diff --git a/benchmarks/dataset/oracle_tests/off_by_one_slicing_03_test.py b/benchmarks/dataset/oracle_tests/off_by_one_slicing_03_test.py new file mode 100644 index 0000000..112add2 --- /dev/null +++ b/benchmarks/dataset/oracle_tests/off_by_one_slicing_03_test.py @@ -0,0 +1,10 @@ +def test_finds_last_element(): + assert binary_search([1, 3, 5, 7, 9], 9) == 4 + + +def test_finds_first_element(): + assert binary_search([1, 3, 5, 7, 9], 1) == 0 + + +def test_missing_target_returns_negative_one(): + assert binary_search([1, 3, 5, 7, 9], 4) == -1 diff --git a/benchmarks/dataset/oracle_tests/off_by_one_slicing_04_test.py b/benchmarks/dataset/oracle_tests/off_by_one_slicing_04_test.py new file mode 100644 index 0000000..25cf739 --- /dev/null +++ b/benchmarks/dataset/oracle_tests/off_by_one_slicing_04_test.py @@ -0,0 +1,6 @@ +def test_first_page(): + assert get_page_items([1, 2, 3, 4, 5, 6], 0, 3) == [1, 2, 3] + + +def test_second_page(): + assert get_page_items([1, 2, 3, 4, 5, 6], 1, 3) == [4, 5, 6] diff --git a/benchmarks/dataset/oracle_tests/off_by_one_slicing_05_test.py b/benchmarks/dataset/oracle_tests/off_by_one_slicing_05_test.py new file mode 100644 index 0000000..2b91648 --- /dev/null +++ b/benchmarks/dataset/oracle_tests/off_by_one_slicing_05_test.py @@ -0,0 +1,6 @@ +def test_first_five_chars(): + assert first_n_chars("hello world", 5) == "hello" + + +def test_first_one_char(): + assert first_n_chars("abc", 1) == "a" diff --git a/benchmarks/dataset/oracle_tests/resource_leaks_01_test.py b/benchmarks/dataset/oracle_tests/resource_leaks_01_test.py new file mode 100644 index 0000000..2f9a8df --- /dev/null +++ b/benchmarks/dataset/oracle_tests/resource_leaks_01_test.py @@ -0,0 +1,9 @@ +from unittest.mock import mock_open, patch + + +def test_file_handle_is_closed(): + m = mock_open(read_data="hello") + with patch("builtins.open", m): + result = read_file_contents("dummy.txt") + assert result == "hello" + assert m.return_value.__exit__.called diff --git a/benchmarks/dataset/oracle_tests/resource_leaks_02_test.py b/benchmarks/dataset/oracle_tests/resource_leaks_02_test.py new file mode 100644 index 0000000..4d3838f --- /dev/null +++ b/benchmarks/dataset/oracle_tests/resource_leaks_02_test.py @@ -0,0 +1,8 @@ +from unittest.mock import mock_open, patch + + +def test_file_handle_is_closed(): + m = mock_open() + with patch("builtins.open", m): + write_log_line("dummy.txt", "hello") + assert m.return_value.__exit__.called diff --git a/benchmarks/dataset/oracle_tests/resource_leaks_03_test.py b/benchmarks/dataset/oracle_tests/resource_leaks_03_test.py new file mode 100644 index 0000000..7643867 --- /dev/null +++ b/benchmarks/dataset/oracle_tests/resource_leaks_03_test.py @@ -0,0 +1,9 @@ +from unittest.mock import mock_open, patch + + +def test_file_handle_is_closed(): + m = mock_open(read_data="a\nb\nc\n") + with patch("builtins.open", m): + result = count_lines("dummy.txt") + assert result == 3 + assert m.return_value.__exit__.called diff --git a/benchmarks/dataset/oracle_tests/resource_leaks_04_test.py b/benchmarks/dataset/oracle_tests/resource_leaks_04_test.py new file mode 100644 index 0000000..5512301 --- /dev/null +++ b/benchmarks/dataset/oracle_tests/resource_leaks_04_test.py @@ -0,0 +1,9 @@ +from unittest.mock import mock_open, patch + + +def test_file_handle_is_closed(): + m = mock_open(read_data="a \n b \n c\n") + with patch("builtins.open", m): + result = read_stripped_lines("dummy.txt") + assert result == ["a", "b", "c"] + assert m.return_value.__exit__.called diff --git a/benchmarks/dataset/oracle_tests/resource_leaks_05_test.py b/benchmarks/dataset/oracle_tests/resource_leaks_05_test.py new file mode 100644 index 0000000..dabf975 --- /dev/null +++ b/benchmarks/dataset/oracle_tests/resource_leaks_05_test.py @@ -0,0 +1,9 @@ +from unittest.mock import mock_open, patch + + +def test_file_handle_is_closed(): + m = mock_open(read_data='{"a": 1}') + with patch("builtins.open", m): + result = read_json_file("dummy.json") + assert result == {"a": 1} + assert m.return_value.__exit__.called diff --git a/benchmarks/dataset/oracle_tests/unhandled_nonetype_01_test.py b/benchmarks/dataset/oracle_tests/unhandled_nonetype_01_test.py new file mode 100644 index 0000000..7a0e0ef --- /dev/null +++ b/benchmarks/dataset/oracle_tests/unhandled_nonetype_01_test.py @@ -0,0 +1,13 @@ +def test_returns_email_when_present(): + user = {"profile": {"email": "a@example.com"}} + assert get_user_email(user) == "a@example.com" + + +def test_returns_none_when_profile_missing(): + user = {} + assert get_user_email(user) is None + + +def test_returns_none_when_email_missing(): + user = {"profile": {}} + assert get_user_email(user) is None diff --git a/benchmarks/dataset/oracle_tests/unhandled_nonetype_02_test.py b/benchmarks/dataset/oracle_tests/unhandled_nonetype_02_test.py new file mode 100644 index 0000000..5c24a54 --- /dev/null +++ b/benchmarks/dataset/oracle_tests/unhandled_nonetype_02_test.py @@ -0,0 +1,13 @@ +def test_returns_theme_when_present(): + config = {"app": {"settings": {"theme": "dark"}}} + assert get_theme(config) == "dark" + + +def test_returns_none_when_app_missing(): + config = {} + assert get_theme(config) is None + + +def test_returns_none_when_settings_missing(): + config = {"app": {}} + assert get_theme(config) is None diff --git a/benchmarks/dataset/oracle_tests/unhandled_nonetype_03_test.py b/benchmarks/dataset/oracle_tests/unhandled_nonetype_03_test.py new file mode 100644 index 0000000..5154aa4 --- /dev/null +++ b/benchmarks/dataset/oracle_tests/unhandled_nonetype_03_test.py @@ -0,0 +1,13 @@ +def test_returns_city_when_present(): + order = {"shipping": {"address": {"city": "Lahore"}}} + assert get_shipping_city(order) == "Lahore" + + +def test_returns_none_when_shipping_missing(): + order = {} + assert get_shipping_city(order) is None + + +def test_returns_none_when_address_missing(): + order = {"shipping": {}} + assert get_shipping_city(order) is None diff --git a/benchmarks/dataset/oracle_tests/unhandled_nonetype_04_test.py b/benchmarks/dataset/oracle_tests/unhandled_nonetype_04_test.py new file mode 100644 index 0000000..972041d --- /dev/null +++ b/benchmarks/dataset/oracle_tests/unhandled_nonetype_04_test.py @@ -0,0 +1,13 @@ +def test_returns_price_when_present(): + product = {"pricing": {"amount": 19.99}} + assert get_product_price(product) == 19.99 + + +def test_returns_none_when_pricing_missing(): + product = {} + assert get_product_price(product) is None + + +def test_returns_none_when_amount_missing(): + product = {"pricing": {}} + assert get_product_price(product) is None diff --git a/benchmarks/dataset/oracle_tests/unhandled_nonetype_05_test.py b/benchmarks/dataset/oracle_tests/unhandled_nonetype_05_test.py new file mode 100644 index 0000000..cb9fdb9 --- /dev/null +++ b/benchmarks/dataset/oracle_tests/unhandled_nonetype_05_test.py @@ -0,0 +1,13 @@ +def test_returns_name_when_present(): + employee = {"manager": {"name": "Sam"}} + assert get_manager_name(employee) == "Sam" + + +def test_returns_none_when_manager_missing(): + employee = {} + assert get_manager_name(employee) is None + + +def test_returns_none_when_name_missing(): + employee = {"manager": {}} + assert get_manager_name(employee) is None diff --git a/benchmarks/dataset/resource_leaks_01.py b/benchmarks/dataset/resource_leaks_01.py new file mode 100644 index 0000000..7f06996 --- /dev/null +++ b/benchmarks/dataset/resource_leaks_01.py @@ -0,0 +1,6 @@ +"""Reads and returns the full contents of a text file.""" + + +def read_file_contents(path): + f = open(path) + return f.read() diff --git a/benchmarks/dataset/resource_leaks_02.py b/benchmarks/dataset/resource_leaks_02.py new file mode 100644 index 0000000..e036f21 --- /dev/null +++ b/benchmarks/dataset/resource_leaks_02.py @@ -0,0 +1,6 @@ +"""Appends a line of text to a log file.""" + + +def write_log_line(path, line): + f = open(path, "a") + f.write(line + "\n") diff --git a/benchmarks/dataset/resource_leaks_03.py b/benchmarks/dataset/resource_leaks_03.py new file mode 100644 index 0000000..376c9f2 --- /dev/null +++ b/benchmarks/dataset/resource_leaks_03.py @@ -0,0 +1,7 @@ +"""Returns the number of lines in a text file.""" + + +def count_lines(path): + f = open(path) + lines = f.readlines() + return len(lines) diff --git a/benchmarks/dataset/resource_leaks_04.py b/benchmarks/dataset/resource_leaks_04.py new file mode 100644 index 0000000..23a415a --- /dev/null +++ b/benchmarks/dataset/resource_leaks_04.py @@ -0,0 +1,7 @@ +"""Reads a text file and returns its lines with surrounding whitespace +stripped.""" + + +def read_stripped_lines(path): + f = open(path) + return [line.strip() for line in f] diff --git a/benchmarks/dataset/resource_leaks_05.py b/benchmarks/dataset/resource_leaks_05.py new file mode 100644 index 0000000..6f0e570 --- /dev/null +++ b/benchmarks/dataset/resource_leaks_05.py @@ -0,0 +1,8 @@ +"""Reads and parses a JSON file, returning the resulting object.""" + +import json + + +def read_json_file(path): + f = open(path) + return json.load(f) diff --git a/benchmarks/dataset/unhandled_nonetype_01.py b/benchmarks/dataset/unhandled_nonetype_01.py new file mode 100644 index 0000000..c2708f4 --- /dev/null +++ b/benchmarks/dataset/unhandled_nonetype_01.py @@ -0,0 +1,6 @@ +"""Returns the user's email address from their profile, or None if the +profile is missing or has no email set.""" + + +def get_user_email(user): + return user["profile"]["email"] diff --git a/benchmarks/dataset/unhandled_nonetype_02.py b/benchmarks/dataset/unhandled_nonetype_02.py new file mode 100644 index 0000000..0c5f42d --- /dev/null +++ b/benchmarks/dataset/unhandled_nonetype_02.py @@ -0,0 +1,6 @@ +"""Returns the configured UI theme name, or None if not set anywhere +in the config.""" + + +def get_theme(config): + return config["app"]["settings"]["theme"] diff --git a/benchmarks/dataset/unhandled_nonetype_03.py b/benchmarks/dataset/unhandled_nonetype_03.py new file mode 100644 index 0000000..4d16735 --- /dev/null +++ b/benchmarks/dataset/unhandled_nonetype_03.py @@ -0,0 +1,6 @@ +"""Returns the shipping city for an order, or None if shipping info is +incomplete or missing.""" + + +def get_shipping_city(order): + return order["shipping"]["address"]["city"] diff --git a/benchmarks/dataset/unhandled_nonetype_04.py b/benchmarks/dataset/unhandled_nonetype_04.py new file mode 100644 index 0000000..835d900 --- /dev/null +++ b/benchmarks/dataset/unhandled_nonetype_04.py @@ -0,0 +1,6 @@ +"""Returns a product's price amount, or None if pricing info is +missing.""" + + +def get_product_price(product): + return product["pricing"]["amount"] diff --git a/benchmarks/dataset/unhandled_nonetype_05.py b/benchmarks/dataset/unhandled_nonetype_05.py new file mode 100644 index 0000000..b29ecee --- /dev/null +++ b/benchmarks/dataset/unhandled_nonetype_05.py @@ -0,0 +1,6 @@ +"""Returns the name of an employee's manager, or None if the employee +has no manager set.""" + + +def get_manager_name(employee): + return employee["manager"]["name"] diff --git a/benchmarks/evaluate.py b/benchmarks/evaluate.py new file mode 100644 index 0000000..761860a --- /dev/null +++ b/benchmarks/evaluate.py @@ -0,0 +1,208 @@ +""" +benchmarks.evaluate +===================== +Runs the full agent graph against every defect in the dataset and grades +the result against that defect's oracle test -- never against the SLM's +own self-written test suite, since a model that writes weak tests would +otherwise score as "fixed" even when it isn't. + +structured_llm is injected, same pattern as graph.py, so tests can mock +it and never touch a real Ollama server. The real run (main()) uses the +actual model. +""" + +from __future__ import annotations + +import logging +import time +from pathlib import Path +from typing import Final + +from langchain_core.runnables import Runnable +from pydantic import BaseModel, Field + +from benchmarks.manifest import ( + DefectRecord, + load_defect_source, + load_manifest, + load_oracle_test, +) +from patchwork.graph import build_patchwork_graph, build_structured_llm +from patchwork.state import CodeAuditOutput, create_initial_state +from patchwork.telemetry.profiler import profile_call +from patchwork.tools.sandbox import run_pytest_sandbox + +logger = logging.getLogger("benchmarks.evaluate") + +RESULTS_PATH: Final[Path] = Path(__file__).parent / "results.json" +DEFAULT_MAX_RETRIES: Final[int] = 3 +DEFAULT_ORACLE_TIMEOUT_SEC: Final[int] = 15 + + +class DefectResult(BaseModel): + defect_id: str + category: str + passed_oracle: bool # the real grade -- did the fix actually work + passed_own_tests: bool # informational only -- the SLM grading itself + retry_count: int + max_retries: int + duration_sec: float = Field(ge=0.0) + peak_vram_mb: float | None = None + gpu_available: bool = False + error: str | None = None # populated only if the run itself crashed + + +class EvaluationSummary(BaseModel): + total_defects: int + pass_at_1: int # passed_oracle True with retry_count == 0 + pass_at_1_rate: float + pass_overall: int # passed_oracle True at any retry_count <= max_retries + pass_overall_rate: float + avg_duration_sec: float + avg_peak_vram_mb: float | None + results: list[DefectResult] = Field(default_factory=list) + + +def _grade_against_oracle(record: DefectRecord, final_code: str) -> bool: + oracle = load_oracle_test(record) + result = run_pytest_sandbox( + final_code, oracle, timeout_sec=DEFAULT_ORACLE_TIMEOUT_SEC + ) + return result.passed + + +def evaluate_defect( + record: DefectRecord, + structured_llm: Runnable[str, CodeAuditOutput], + max_retries: int = DEFAULT_MAX_RETRIES, +) -> DefectResult: + source = load_defect_source(record) + graph = build_patchwork_graph(structured_llm) + initial = create_initial_state( + record.source_filename, source, max_retries=max_retries + ) + + try: + final_state, telemetry = profile_call(graph.invoke, initial) + except (RuntimeError, ConnectionError, TimeoutError, OSError) as exc: + # crash-isolation boundary for the batch harness: an unexpected + # transport/runtime failure on one defect must not kill the rest + # of a 25-defect run. graph.py already handles the SLM schema + # failures internally -- this only catches what escapes that. + logger.error( + "defect_evaluation_crashed", + extra={ + "event": "defect_evaluation_crashed", + "defect_id": record.id, + "error": str(exc), + }, + ) + return DefectResult( + defect_id=record.id, + category=record.category, + passed_oracle=False, + passed_own_tests=False, + retry_count=0, + max_retries=max_retries, + duration_sec=0.0, + error=str(exc), + ) + + sandbox_result = final_state["sandbox_result"] + passed_own_tests = sandbox_result.passed if sandbox_result else False + passed_oracle = _grade_against_oracle(record, final_state["current_code"]) + + return DefectResult( + defect_id=record.id, + category=record.category, + passed_oracle=passed_oracle, + passed_own_tests=passed_own_tests, + retry_count=final_state["retry_count"], + max_retries=max_retries, + duration_sec=telemetry.duration_sec, + peak_vram_mb=telemetry.peak_vram_mb, + gpu_available=telemetry.gpu_available, + ) + + +def evaluate_all( + structured_llm: Runnable[str, CodeAuditOutput], + max_retries: int = DEFAULT_MAX_RETRIES, +) -> EvaluationSummary: + manifest = load_manifest() + results: list[DefectResult] = [] + + for record in manifest.defects: + logger.info( + "evaluating_defect", + extra={"event": "evaluating_defect", "defect_id": record.id}, + ) + result = evaluate_defect(record, structured_llm, max_retries) + results.append(result) + + total = len(results) + pass_at_1 = sum(1 for r in results if r.passed_oracle and r.retry_count == 0) + pass_overall = sum(1 for r in results if r.passed_oracle) + durations = [r.duration_sec for r in results] + vram_values = [r.peak_vram_mb for r in results if r.peak_vram_mb is not None] + + return EvaluationSummary( + total_defects=total, + pass_at_1=pass_at_1, + pass_at_1_rate=pass_at_1 / total if total else 0.0, + pass_overall=pass_overall, + pass_overall_rate=pass_overall / total if total else 0.0, + avg_duration_sec=sum(durations) / total if total else 0.0, + avg_peak_vram_mb=(sum(vram_values) / len(vram_values)) if vram_values else None, + results=results, + ) + + +def save_results(summary: EvaluationSummary, path: Path = RESULTS_PATH) -> None: + path.write_text(summary.model_dump_json(indent=2), encoding="utf-8") + + +def _print_summary(summary: EvaluationSummary) -> None: + print(f"\n{'=' * 60}") + print(f"Total defects: {summary.total_defects}") + print( + f"Pass@1: {summary.pass_at_1}/{summary.total_defects} ({summary.pass_at_1_rate:.0%})" + ) + print( + f"Pass (overall): {summary.pass_overall}/{summary.total_defects} ({summary.pass_overall_rate:.0%})" + ) + print(f"Avg duration: {summary.avg_duration_sec:.2f}s") + vram_line = ( + f"{summary.avg_peak_vram_mb:.0f}MB" + if summary.avg_peak_vram_mb + else "N/A (no GPU)" + ) + print(f"Avg peak VRAM: {vram_line}") + print(f"{'=' * 60}\n") + + for result in summary.results: + status = "PASS" if result.passed_oracle else "FAIL" + note = f" -- CRASHED: {result.error}" if result.error else "" + print( + f" [{status}] {result.defect_id:28} retries={result.retry_count}/{result.max_retries}{note}" + ) + + +def main() -> None: + print("Building structured LLM client...") + structured_llm = build_structured_llm() + + print("Running 25-defect benchmark. This will take a while on a 3B model...\n") + start = time.perf_counter() + summary = evaluate_all(structured_llm) + elapsed = time.perf_counter() - start + + _print_summary(summary) + print(f"Total wall time: {elapsed:.1f}s") + + save_results(summary) + print(f"Results written to {RESULTS_PATH}") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/manifest.py b/benchmarks/manifest.py new file mode 100644 index 0000000..27be07b --- /dev/null +++ b/benchmarks/manifest.py @@ -0,0 +1,62 @@ +""" +benchmarks.manifest +===================== +Schema for the 25-defect benchmark dataset. Each defect has a buggy +source file AND a separate oracle test file the agent never sees -- +evaluate.py grades against the oracle, not the SLM's own self-written +tests, since a model that writes weak tests would otherwise score as +"fixed" even when it isn't. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Final + +from pydantic import BaseModel, Field, model_validator + +DATASET_DIR: Final[Path] = Path(__file__).parent / "dataset" +ORACLE_TESTS_DIR: Final[Path] = DATASET_DIR / "oracle_tests" +MANIFEST_PATH: Final[Path] = DATASET_DIR / "manifest.json" + + +class DefectRecord(BaseModel): + id: str = Field(min_length=1) # e.g. "mutable_default_01" + category: str = Field(min_length=1) # e.g. "mutable_default_arguments" + source_filename: str = Field(min_length=1) # relative to dataset/ + oracle_test_filename: str = Field(min_length=1) # relative to dataset/oracle_tests/ + description: str = Field(min_length=1) + + +class DefectManifest(BaseModel): + defects: list[DefectRecord] = Field(default_factory=list) + + @model_validator(mode="after") + def _check_unique_ids(self) -> DefectManifest: + # a duplicate id would let one defect silently shadow another in + # any dict/lookup keyed by id (evaluate.py's test _record() helper + # does exactly that) -- fail loudly at load time, not later at an + # unrelated call site + seen: set[str] = set() + duplicates: set[str] = set() + for defect in self.defects: + if defect.id in seen: + duplicates.add(defect.id) + seen.add(defect.id) + if duplicates: + raise ValueError( + f"Duplicate defect id(s) in manifest: {sorted(duplicates)}" + ) + return self + + +def load_manifest(path: Path = MANIFEST_PATH) -> DefectManifest: + return DefectManifest.model_validate_json(path.read_text(encoding="utf-8")) + + +def load_defect_source(record: DefectRecord) -> str: + return (DATASET_DIR / record.source_filename).read_text(encoding="utf-8") + + +def load_oracle_test(record: DefectRecord) -> str: + return (ORACLE_TESTS_DIR / record.oracle_test_filename).read_text(encoding="utf-8") diff --git a/patchwork/telemetry/profiler.py b/patchwork/telemetry/profiler.py index 79fa4d6..96a0ad0 100644 --- a/patchwork/telemetry/profiler.py +++ b/patchwork/telemetry/profiler.py @@ -19,7 +19,7 @@ from functools import wraps from typing import Final, ParamSpec, TypeVar -import pynvml # type: ignore[import-untyped] # nvidia-ml-py ships no py.typed marker +import pynvml from pydantic import BaseModel, Field logger = logging.getLogger("patchwork.telemetry.profiler") diff --git a/pyproject.toml b/pyproject.toml index e69de29..dd3df37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -0,0 +1,15 @@ +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.ruff] +target-version = "py310" +extend-exclude = ["benchmarks/dataset"] + +[tool.mypy] +python_version = "3.10" +strict = true +exclude = "benchmarks/dataset/" + +[[tool.mypy.overrides]] +module = "pynvml.*" +ignore_missing_imports = true \ No newline at end of file diff --git a/scripts/manual_profiler_test.py b/scripts/manual_profiler_test.py index ddd7b68..03938c7 100644 --- a/scripts/manual_profiler_test.py +++ b/scripts/manual_profiler_test.py @@ -63,4 +63,4 @@ def main() -> None: if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/manual_reflection_test.py b/scripts/manual_reflection_test.py index f3669c7..4ec3e05 100644 --- a/scripts/manual_reflection_test.py +++ b/scripts/manual_reflection_test.py @@ -76,4 +76,4 @@ def main() -> None: if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/manual_smoke_test.py b/scripts/manual_smoke_test.py index e2c2073..96f2801 100644 --- a/scripts/manual_smoke_test.py +++ b/scripts/manual_smoke_test.py @@ -57,4 +57,4 @@ def main() -> None: if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/tests/test_evaluate.py b/tests/test_evaluate.py new file mode 100644 index 0000000..9b971f8 --- /dev/null +++ b/tests/test_evaluate.py @@ -0,0 +1,102 @@ +""" +tests/test_evaluate.py +========================= +All tests mock structured_llm -- never touch real Ollama. But grading +runs the REAL dataset, REAL oracle tests, and REAL sandbox execution, +since this module's entire job is correctly wiring those together. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +import pytest + +from benchmarks.evaluate import EvaluationSummary, evaluate_all, evaluate_defect +from benchmarks.manifest import DefectRecord, load_manifest +from patchwork.state import CodeAuditOutput + + +def _record(defect_id: str) -> DefectRecord: + manifest = load_manifest() + return next(r for r in manifest.defects if r.id == defect_id) + + +class TestEvaluateDefect: + def test_correct_fix_passes_oracle(self) -> None: + mock_llm = MagicMock() + mock_llm.invoke.return_value = CodeAuditOutput( + identified_bugs=["fixed"], + suggested_patch=( + "def append_item(item, target_list=None):\n" + " if target_list is None:\n" + " target_list = []\n" + " target_list.append(item)\n" + " return target_list\n" + ), + pytest_suite="def test_x():\n assert True\n", + ) + result = evaluate_defect(_record("mutable_default_01"), mock_llm, max_retries=0) + assert result.passed_oracle is True + assert result.error is None + + def test_wrong_fix_fails_oracle(self) -> None: + mock_llm = MagicMock() + mock_llm.invoke.return_value = CodeAuditOutput( + identified_bugs=[], + suggested_patch="def placeholder():\n pass\n", + pytest_suite="def test_x():\n assert True\n", + ) + result = evaluate_defect(_record("mutable_default_01"), mock_llm, max_retries=0) + assert result.passed_oracle is False + + def test_grading_uses_oracle_not_slms_own_tests(self) -> None: + # SLM writes a trivial/wrong self-test that would pass against + # its own broken patch -- passed_own_tests may be True, but + # passed_oracle must still correctly be False + mock_llm = MagicMock() + mock_llm.invoke.return_value = CodeAuditOutput( + identified_bugs=[], + suggested_patch=( + "def append_item(item, target_list=[]):\n" + " target_list.append(item)\n" + " return target_list\n" + ), # unfixed + pytest_suite="def test_trivial():\n assert True\n", # passes regardless + ) + result = evaluate_defect(_record("mutable_default_01"), mock_llm, max_retries=0) + assert result.passed_own_tests is True # trivial test passes + assert result.passed_oracle is False # but the real bug is still there + + def test_transport_failure_does_not_crash_evaluation(self) -> None: + mock_llm = MagicMock() + mock_llm.invoke.side_effect = ConnectionError("ollama unreachable") + result = evaluate_defect(_record("mutable_default_01"), mock_llm, max_retries=0) + assert result.passed_oracle is False + assert result.error is not None + + +@pytest.fixture(scope="module") +def summary() -> EvaluationSummary: + mock_llm = MagicMock() + mock_llm.invoke.return_value = CodeAuditOutput( + identified_bugs=[], + suggested_patch="x = 1\n", + pytest_suite="def test_x():\n assert True\n", + ) + return evaluate_all(mock_llm, max_retries=0) + + +class TestEvaluateAll: + def test_runs_all_defects_in_manifest(self, summary: EvaluationSummary) -> None: + assert summary.total_defects == 25 + assert len(summary.results) == 25 + + def test_pass_at_1_counts_only_zero_retry_passes( + self, summary: EvaluationSummary + ) -> None: + assert summary.pass_at_1 == 0 + assert summary.pass_at_1_rate == 0.0 + + def test_rates_are_fractions_of_total(self, summary: EvaluationSummary) -> None: + assert summary.pass_overall_rate == summary.pass_overall / summary.total_defects diff --git a/tests/test_graph.py b/tests/test_graph.py index 0d4a134..d3a0df2 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -8,15 +8,19 @@ from __future__ import annotations -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch from langchain_core.exceptions import OutputParserException from pydantic import ValidationError from patchwork.graph import ( + DEFAULT_MODEL, + DEFAULT_NUM_CTX, + DEFAULT_TEMPERATURE, _build_audit_prompt, _build_reflect_prompt, build_patchwork_graph, + build_structured_llm, make_audit_and_generate_node, make_reflect_and_heal_node, node_execute_tests, @@ -32,6 +36,42 @@ def _mock_llm(result: CodeAuditOutput) -> MagicMock: return mock +class TestBuildStructuredLlm: + """Mocks ChatOllama itself -- never opens a real connection to Ollama. + + Covers patchwork.graph's only two lines that no other test in this + file exercises: build_structured_llm never gets called elsewhere, + since every other test injects a MagicMock as structured_llm directly. + """ + + @patch("patchwork.graph.ChatOllama") + def test_builds_with_default_params(self, mock_chat_ollama: MagicMock) -> None: + mock_llm_instance = MagicMock() + mock_chat_ollama.return_value = mock_llm_instance + + build_structured_llm() + + mock_chat_ollama.assert_called_once_with( + model=DEFAULT_MODEL, + temperature=DEFAULT_TEMPERATURE, + num_ctx=DEFAULT_NUM_CTX, + ) + mock_llm_instance.with_structured_output.assert_called_once_with( + CodeAuditOutput + ) + + @patch("patchwork.graph.ChatOllama") + def test_builds_with_custom_params(self, mock_chat_ollama: MagicMock) -> None: + mock_llm_instance = MagicMock() + mock_chat_ollama.return_value = mock_llm_instance + + build_structured_llm(model="custom:model", temperature=0.5, num_ctx=4096) + + mock_chat_ollama.assert_called_once_with( + model="custom:model", temperature=0.5, num_ctx=4096 + ) + + class TestBuildAuditPrompt: def test_includes_source_code(self) -> None: state = create_initial_state("target.py", "def f():\n pass\n") @@ -50,6 +90,15 @@ def test_includes_lint_issue_details(self) -> None: prompt = _build_audit_prompt(state) assert "F401" in prompt + def test_reports_invalid_syntax_note(self) -> None: + # covers the "else" branch of syntax_note in _build_audit_prompt -- + # every other test here feeds valid syntax through + # node_static_analysis, so this branch was previously unexercised + state = create_initial_state("target.py", "def broken(:\n pass\n") + state = node_static_analysis(state) + prompt = _build_audit_prompt(state) + assert "INVALID SYNTAX" in prompt + class TestNodeStaticAnalysis: def test_populates_ast_and_lint_results(self) -> None: @@ -130,8 +179,9 @@ def test_passing_generated_tests_report_passed(self) -> None: new_state = node_execute_tests(state) - assert new_state["sandbox_result"] is not None - assert new_state["sandbox_result"].passed is True + sandbox_result = new_state["sandbox_result"] + assert sandbox_result is not None + assert sandbox_result.passed is True def test_failing_generated_tests_report_not_passed(self) -> None: state = create_initial_state("target.py", "def add(a, b):\n return a - b\n") @@ -139,7 +189,9 @@ def test_failing_generated_tests_report_not_passed(self) -> None: new_state = node_execute_tests(state) - assert new_state["sandbox_result"].passed is False + sandbox_result = new_state["sandbox_result"] + assert sandbox_result is not None + assert sandbox_result.passed is False class TestBuildPatchworkGraphIntegration: @@ -158,7 +210,9 @@ def test_full_pass_with_passing_patch(self) -> None: final = graph.invoke(initial) - assert final["sandbox_result"].passed is True + sandbox_result = final["sandbox_result"] + assert sandbox_result is not None + assert sandbox_result.passed is True assert final["current_code"] == mock_result.suggested_patch assert ( len(final["audit_trail"]) == 4 @@ -299,7 +353,9 @@ def test_loop_converges_after_two_failed_attempts(self) -> None: final = graph.invoke(initial) - assert final["sandbox_result"].passed is True + sandbox_result = final["sandbox_result"] + assert sandbox_result is not None + assert sandbox_result.passed is True assert final["retry_count"] == 2 assert mock_llm.invoke.call_count == 3 # 1 initial generate + 2 reflects @@ -317,7 +373,9 @@ def test_loop_stops_at_max_retries_without_hanging(self) -> None: final = graph.invoke(initial) - assert final["sandbox_result"].passed is False + sandbox_result = final["sandbox_result"] + assert sandbox_result is not None + assert sandbox_result.passed is False assert final["retry_count"] == 2 # stopped exactly at the ceiling assert ( mock_llm.invoke.call_count == 3 diff --git a/tests/test_manifest.py b/tests/test_manifest.py new file mode 100644 index 0000000..2300e8a --- /dev/null +++ b/tests/test_manifest.py @@ -0,0 +1,69 @@ +""" +tests/test_manifest.py +========================= +Loads the real manifest.json and real dataset files -- no mocking, +since this module's whole job is reading real files off disk correctly. +""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from benchmarks.manifest import ( + DefectManifest, + load_defect_source, + load_manifest, + load_oracle_test, +) + + +class TestLoadManifest: + def test_loads_all_current_defects(self) -> None: + manifest = load_manifest() + assert len(manifest.defects) >= 5 + + def test_every_defect_has_required_fields(self) -> None: + manifest = load_manifest() + for record in manifest.defects: + assert record.id + assert record.category + assert record.source_filename + assert record.oracle_test_filename + assert record.description + + def test_ids_are_unique(self) -> None: + manifest = load_manifest() + ids = [record.id for record in manifest.defects] + assert len(ids) == len(set(ids)) + + def test_duplicate_id_raises_at_load_time(self) -> None: + # covers _check_unique_ids directly -- the real manifest.json + # never has duplicates, so this constructs one by hand to prove + # the validator itself fires, not just that the shipped file + # happens to be clean + duplicate_json = ( + '{"defects": [' + '{"id": "dup", "category": "x", "source_filename": "a.py", ' + '"oracle_test_filename": "a_test.py", "description": "d1"},' + '{"id": "dup", "category": "x", "source_filename": "b.py", ' + '"oracle_test_filename": "b_test.py", "description": "d2"}' + "]}" + ) + with pytest.raises(ValidationError): + DefectManifest.model_validate_json(duplicate_json) + + +class TestLoadDefectFiles: + def test_every_source_file_exists_and_loads(self) -> None: + manifest = load_manifest() + for record in manifest.defects: + source = load_defect_source(record) + assert len(source) > 0 + + def test_every_oracle_test_file_exists_and_loads(self) -> None: + manifest = load_manifest() + for record in manifest.defects: + oracle = load_oracle_test(record) + assert len(oracle) > 0 + assert "def test_" in oracle diff --git a/tests/test_profiler.py b/tests/test_profiler.py index 4fe9861..e218a5a 100644 --- a/tests/test_profiler.py +++ b/tests/test_profiler.py @@ -1,14 +1,20 @@ """ tests/test_profiler.py ======================== -This sandbox and CI both have no NVIDIA GPU, so every test here runs -against the real no-GPU fallback path -- which is exactly the path that -matters most to get right, since it's the one CI will always exercise. +The no-GPU fallback path is mocked (patching pynvml.nvmlInit to raise), +not relied on from the host's actual hardware -- a dev machine with a +real GPU must see identical, deterministic results here as CI (which +has no GPU). Testing against real hardware state would make these tests +flake depending on what machine runs them; that's what +scripts/manual_profiler_test.py is for instead. """ from __future__ import annotations import time +from unittest.mock import MagicMock, patch + +import pynvml from patchwork.telemetry.profiler import ( GPUProfiler, @@ -19,34 +25,55 @@ class TestGPUProfilerNoGPU: - def test_context_manager_does_not_raise_without_gpu(self) -> None: + """Every test here patches pynvml.nvmlInit to force the no-GPU path, + so results are identical regardless of the host's actual hardware.""" + + @patch("patchwork.telemetry.profiler.pynvml.nvmlInit") + def test_context_manager_does_not_raise_without_gpu( + self, mock_init: MagicMock + ) -> None: + mock_init.side_effect = pynvml.NVMLError_LibraryNotFound() with GPUProfiler() as profiler: time.sleep(0.05) result = profiler.result() assert isinstance(result, TelemetryResult) - def test_gpu_available_false_without_driver(self) -> None: + @patch("patchwork.telemetry.profiler.pynvml.nvmlInit") + def test_gpu_available_false_without_driver(self, mock_init: MagicMock) -> None: + mock_init.side_effect = pynvml.NVMLError_LibraryNotFound() with GPUProfiler() as profiler: pass assert profiler.result().gpu_available is False - def test_peak_vram_none_without_gpu(self) -> None: + @patch("patchwork.telemetry.profiler.pynvml.nvmlInit") + def test_peak_vram_none_without_gpu(self, mock_init: MagicMock) -> None: + mock_init.side_effect = pynvml.NVMLError_LibraryNotFound() with GPUProfiler() as profiler: pass assert profiler.result().peak_vram_mb is None - def test_duration_still_measured_without_gpu(self) -> None: + @patch("patchwork.telemetry.profiler.pynvml.nvmlInit") + def test_duration_still_measured_without_gpu(self, mock_init: MagicMock) -> None: + mock_init.side_effect = pynvml.NVMLError_LibraryNotFound() with GPUProfiler(interval=0.01) as profiler: time.sleep(0.1) result = profiler.result() assert result.duration_sec >= 0.1 - def test_error_message_populated_when_nvml_unavailable(self) -> None: + @patch("patchwork.telemetry.profiler.pynvml.nvmlInit") + def test_error_message_populated_when_nvml_unavailable( + self, mock_init: MagicMock + ) -> None: + mock_init.side_effect = pynvml.NVMLError_LibraryNotFound() with GPUProfiler() as profiler: pass assert profiler.result().error_message is not None - def test_result_is_pydantic_model_and_json_serializable(self) -> None: + @patch("patchwork.telemetry.profiler.pynvml.nvmlInit") + def test_result_is_pydantic_model_and_json_serializable( + self, mock_init: MagicMock + ) -> None: + mock_init.side_effect = pynvml.NVMLError_LibraryNotFound() with GPUProfiler() as profiler: pass payload = profiler.result().model_dump_json()