Fix: dupe-date-price heuristic no longer drops distinct parcels of a multi-parcel deed

openavmkit/sales_scrutiny_study.py

@@ -36,6 +36,7 @@
     combine_dfs,
 )
 from openavmkit.utilities.excel import write_to_excel
+from openavmkit.utilities.sales_scrutiny import flag_dupe_date_price
 from openavmkit.utilities.settings import get_fields_categorical, _apply_dd_to_df_cols, area_unit, get_locations, warn_if_location_collapsed
 
 
@@ -615,20 +616,7 @@ def run_heuristics(
             warnings.warn(f"You provided a `deed_id`: \"{deed_id}\", but it wasn't found in in the sales dataframe, so no deed-based sales validation heuristic can be run")
 
     # 2 -- Flag sales made on the same date for the same price
-
-    if jurisdiction != None:
-        df_sales["date_price"] = df_sales[jurisdiction].astype(str) + "---" + df_sales["sale_date"].astype(str) + "---" + df_sales["sale_price"].astype(str)
-    else:
-        df_sales["date_price"] = df_sales["sale_date"].astype(str) + "---" + df_sales["sale_price"].astype(str)
-    vcs_date_price = df_sales["date_price"].value_counts()
-    idx_dupe_date_price = vcs_date_price[vcs_date_price > 1].index.values
-    df_sales.loc[
-        df_sales["date_price"].isin(idx_dupe_date_price),
-        "flag_dupe_date_price",
-    ] = True
-
-    # drop extraneous column
-    df_sales = df_sales.drop(columns="date_price")
+    df_sales = flag_dupe_date_price(df_sales, jurisdiction)
 
     #### Misclassified vacant sales detection heuristics
 

openavmkit/utilities/sales_scrutiny.py

@@ -0,0 +1,34 @@
+import pandas as pd
+
+
+def flag_dupe_date_price(df_sales: pd.DataFrame, jurisdiction=None) -> pd.DataFrame:
+    """Flag same-parcel duplicate sales that share a date and price."""
+    if jurisdiction is not None:
+        date_price = (
+            df_sales[jurisdiction].astype(str)
+            + "---"
+            + df_sales["sale_date"].astype(str)
+            + "---"
+            + df_sales["sale_price"].astype(str)
+        )
+    else:
+        date_price = (
+            df_sales["sale_date"].astype(str)
+            + "---"
+            + df_sales["sale_price"].astype(str)
+        )
+    # Distinct parcels can legitimately share one date/price in a multi-parcel
+    # deed. Include the parcel key so only same-parcel repeats are flagged.
+    if "key" in df_sales.columns:
+        parcel_id = df_sales["key"].astype(str)
+        missing_key = df_sales["key"].isna() | df_sales["key"].astype(str).str.strip().eq("")
+        if missing_key.any() and "key_sale" in df_sales.columns:
+            parcel_id = parcel_id.mask(missing_key, df_sales["key_sale"].astype(str))
+        date_price = date_price + "---" + parcel_id
+    elif "key_sale" in df_sales.columns:
+        date_price = date_price + "---" + df_sales["key_sale"].astype(str)
+
+    dupes = date_price.value_counts()
+    dupe_keys = dupes[dupes > 1].index.values
+    df_sales.loc[date_price.isin(dupe_keys), "flag_dupe_date_price"] = True
+    return df_sales

tests/test_sales_scrutiny_heuristics.py

@@ -0,0 +1,132 @@
+"""Tests for the duplicate-detection heuristic ``flag_dupe_date_price``.
+
+Regression coverage for ``flag_dupe_date_price``: distinct parcels conveyed in a
+single multi-parcel deed share a sale date and price but are NOT duplicates and
+must be kept, while a genuine same-parcel repeat must still be flagged.
+"""
+import pandas as pd
+
+from openavmkit.utilities.sales_scrutiny import flag_dupe_date_price
+
+
+def _sale(key, key_sale, date, price):
+    return {
+        "key": key,
+        "key_sale": key_sale,
+        "sale_date": date,
+        "sale_price": price,
+    }
+
+
+def _df(sales_rows):
+    return pd.DataFrame(sales_rows)
+
+
+def _flagged_sales(df):
+    if "flag_dupe_date_price" not in df.columns:
+        return set()
+    return set(df.loc[df["flag_dupe_date_price"].eq(True), "key_sale"])
+
+
+def test_dupe_date_price_keeps_distinct_parcels_in_one_deed():
+    # Three DISTINCT parcels conveyed in a single multi-parcel deed: same date,
+    # same (deed-total) price. These are not duplicate reports and must not be
+    # flagged.
+    df = _df(
+        [
+            _sale("p1", "s1", "2020-01-01", 75000),
+            _sale("p2", "s2", "2020-01-01", 75000),
+            _sale("p3", "s3", "2020-01-01", 75000),
+        ]
+    )
+    out = flag_dupe_date_price(df)
+    assert _flagged_sales(out) == set()
+
+
+def test_dupe_date_price_still_flags_same_parcel_repeat():
+    # The SAME parcel reported twice at the same date and price is a genuine
+    # duplicate report and must still be flagged; the distinct-parcel deed lots
+    # alongside it must not be.
+    df = _df(
+        [
+            _sale("p1", "s1", "2020-01-01", 75000),
+            _sale("p2", "s2", "2020-01-01", 75000),
+            _sale("p5", "s4", "2021-01-01", 50000),
+            _sale("p5", "s5", "2021-01-01", 50000),
+        ]
+    )
+    out = flag_dupe_date_price(df)
+    assert _flagged_sales(out) == {"s4", "s5"}
+
+
+def test_dupe_date_price_keeps_one_parcel_sold_twice_on_different_dates():
+    # One parcel with two legitimate sales at DIFFERENT dates is not a duplicate
+    # and neither row must be flagged.
+    df = _df(
+        [
+            _sale("p1", "s1", "2018-05-01", 40000),
+            _sale("p1", "s2", "2022-09-01", 60000),
+        ]
+    )
+    out = flag_dupe_date_price(df)
+    assert _flagged_sales(out) == set()
+
+
+def test_dupe_date_price_with_jurisdiction_keeps_distinct_parcels():
+    # The jurisdiction-scoped branch must still keep distinct parcels that share
+    # a date and price, and still flag a true same-parcel repeat.
+    sales = [
+        _sale("p1", "s1", "2020-01-01", 75000),
+        _sale("p2", "s2", "2020-01-01", 75000),
+        _sale("p3", "s3", "2021-01-01", 50000),
+        _sale("p3", "s4", "2021-01-01", 50000),
+    ]
+    for row in sales:
+        row["county"] = "Acme"
+    df = _df(sales)
+    out = flag_dupe_date_price(df, jurisdiction="county")
+    assert _flagged_sales(out) == {"s3", "s4"}
+
+
+def test_dupe_date_price_without_parcel_key_falls_back_to_sale_key():
+    df = pd.DataFrame(
+        {
+            "key_sale": ["s1", "s2", "s3", "s3"],
+            "sale_date": ["2020-01-01", "2020-01-01", "2021-01-01", "2021-01-01"],
+            "sale_price": [75000, 75000, 50000, 50000],
+        }
+    )
+
+    out = flag_dupe_date_price(df)
+
+    assert _flagged_sales(out) == {"s3"}
+
+
+def test_dupe_date_price_null_parcel_key_falls_back_to_sale_key():
+    df = _df(
+        [
+            _sale(None, "s1", "2020-01-01", 75000),
+            _sale(None, "s2", "2020-01-01", 75000),
+            _sale(None, "s3", "2021-01-01", 50000),
+            _sale(None, "s3", "2021-01-01", 50000),
+        ]
+    )
+
+    out = flag_dupe_date_price(df)
+
+    assert _flagged_sales(out) == {"s3"}
+
+
+def test_dupe_date_price_blank_parcel_key_falls_back_to_sale_key():
+    df = _df(
+        [
+            _sale("", "s1", "2020-01-01", 75000),
+            _sale(" ", "s2", "2020-01-01", 75000),
+            _sale("", "s3", "2021-01-01", 50000),
+            _sale("", "s3", "2021-01-01", 50000),
+        ]
+    )
+
+    out = flag_dupe_date_price(df)
+
+    assert _flagged_sales(out) == {"s3"}