diff --git a/openavmkit/sales_scrutiny_study.py b/openavmkit/sales_scrutiny_study.py index f5f68a46..facc6ea1 100644 --- a/openavmkit/sales_scrutiny_study.py +++ b/openavmkit/sales_scrutiny_study.py @@ -36,6 +36,7 @@ combine_dfs, ) from openavmkit.utilities.excel import write_to_excel +from openavmkit.utilities.sales_scrutiny import flag_dupe_date_price from openavmkit.utilities.settings import get_fields_categorical, _apply_dd_to_df_cols, area_unit, get_locations, warn_if_location_collapsed @@ -615,20 +616,7 @@ def run_heuristics( warnings.warn(f"You provided a `deed_id`: \"{deed_id}\", but it wasn't found in in the sales dataframe, so no deed-based sales validation heuristic can be run") # 2 -- Flag sales made on the same date for the same price - - if jurisdiction != None: - df_sales["date_price"] = df_sales[jurisdiction].astype(str) + "---" + df_sales["sale_date"].astype(str) + "---" + df_sales["sale_price"].astype(str) - else: - df_sales["date_price"] = df_sales["sale_date"].astype(str) + "---" + df_sales["sale_price"].astype(str) - vcs_date_price = df_sales["date_price"].value_counts() - idx_dupe_date_price = vcs_date_price[vcs_date_price > 1].index.values - df_sales.loc[ - df_sales["date_price"].isin(idx_dupe_date_price), - "flag_dupe_date_price", - ] = True - - # drop extraneous column - df_sales = df_sales.drop(columns="date_price") + df_sales = flag_dupe_date_price(df_sales, jurisdiction) #### Misclassified vacant sales detection heuristics diff --git a/openavmkit/utilities/sales_scrutiny.py b/openavmkit/utilities/sales_scrutiny.py new file mode 100644 index 00000000..779ac03e --- /dev/null +++ b/openavmkit/utilities/sales_scrutiny.py @@ -0,0 +1,34 @@ +import pandas as pd + + +def flag_dupe_date_price(df_sales: pd.DataFrame, jurisdiction=None) -> pd.DataFrame: + """Flag same-parcel duplicate sales that share a date and price.""" + if jurisdiction is not None: + date_price = ( + df_sales[jurisdiction].astype(str) + + "---" + + df_sales["sale_date"].astype(str) + + "---" + + df_sales["sale_price"].astype(str) + ) + else: + date_price = ( + df_sales["sale_date"].astype(str) + + "---" + + df_sales["sale_price"].astype(str) + ) + # Distinct parcels can legitimately share one date/price in a multi-parcel + # deed. Include the parcel key so only same-parcel repeats are flagged. + if "key" in df_sales.columns: + parcel_id = df_sales["key"].astype(str) + missing_key = df_sales["key"].isna() | df_sales["key"].astype(str).str.strip().eq("") + if missing_key.any() and "key_sale" in df_sales.columns: + parcel_id = parcel_id.mask(missing_key, df_sales["key_sale"].astype(str)) + date_price = date_price + "---" + parcel_id + elif "key_sale" in df_sales.columns: + date_price = date_price + "---" + df_sales["key_sale"].astype(str) + + dupes = date_price.value_counts() + dupe_keys = dupes[dupes > 1].index.values + df_sales.loc[date_price.isin(dupe_keys), "flag_dupe_date_price"] = True + return df_sales diff --git a/tests/test_sales_scrutiny_heuristics.py b/tests/test_sales_scrutiny_heuristics.py new file mode 100644 index 00000000..9d8d6cef --- /dev/null +++ b/tests/test_sales_scrutiny_heuristics.py @@ -0,0 +1,132 @@ +"""Tests for the duplicate-detection heuristic ``flag_dupe_date_price``. + +Regression coverage for ``flag_dupe_date_price``: distinct parcels conveyed in a +single multi-parcel deed share a sale date and price but are NOT duplicates and +must be kept, while a genuine same-parcel repeat must still be flagged. +""" +import pandas as pd + +from openavmkit.utilities.sales_scrutiny import flag_dupe_date_price + + +def _sale(key, key_sale, date, price): + return { + "key": key, + "key_sale": key_sale, + "sale_date": date, + "sale_price": price, + } + + +def _df(sales_rows): + return pd.DataFrame(sales_rows) + + +def _flagged_sales(df): + if "flag_dupe_date_price" not in df.columns: + return set() + return set(df.loc[df["flag_dupe_date_price"].eq(True), "key_sale"]) + + +def test_dupe_date_price_keeps_distinct_parcels_in_one_deed(): + # Three DISTINCT parcels conveyed in a single multi-parcel deed: same date, + # same (deed-total) price. These are not duplicate reports and must not be + # flagged. + df = _df( + [ + _sale("p1", "s1", "2020-01-01", 75000), + _sale("p2", "s2", "2020-01-01", 75000), + _sale("p3", "s3", "2020-01-01", 75000), + ] + ) + out = flag_dupe_date_price(df) + assert _flagged_sales(out) == set() + + +def test_dupe_date_price_still_flags_same_parcel_repeat(): + # The SAME parcel reported twice at the same date and price is a genuine + # duplicate report and must still be flagged; the distinct-parcel deed lots + # alongside it must not be. + df = _df( + [ + _sale("p1", "s1", "2020-01-01", 75000), + _sale("p2", "s2", "2020-01-01", 75000), + _sale("p5", "s4", "2021-01-01", 50000), + _sale("p5", "s5", "2021-01-01", 50000), + ] + ) + out = flag_dupe_date_price(df) + assert _flagged_sales(out) == {"s4", "s5"} + + +def test_dupe_date_price_keeps_one_parcel_sold_twice_on_different_dates(): + # One parcel with two legitimate sales at DIFFERENT dates is not a duplicate + # and neither row must be flagged. + df = _df( + [ + _sale("p1", "s1", "2018-05-01", 40000), + _sale("p1", "s2", "2022-09-01", 60000), + ] + ) + out = flag_dupe_date_price(df) + assert _flagged_sales(out) == set() + + +def test_dupe_date_price_with_jurisdiction_keeps_distinct_parcels(): + # The jurisdiction-scoped branch must still keep distinct parcels that share + # a date and price, and still flag a true same-parcel repeat. + sales = [ + _sale("p1", "s1", "2020-01-01", 75000), + _sale("p2", "s2", "2020-01-01", 75000), + _sale("p3", "s3", "2021-01-01", 50000), + _sale("p3", "s4", "2021-01-01", 50000), + ] + for row in sales: + row["county"] = "Acme" + df = _df(sales) + out = flag_dupe_date_price(df, jurisdiction="county") + assert _flagged_sales(out) == {"s3", "s4"} + + +def test_dupe_date_price_without_parcel_key_falls_back_to_sale_key(): + df = pd.DataFrame( + { + "key_sale": ["s1", "s2", "s3", "s3"], + "sale_date": ["2020-01-01", "2020-01-01", "2021-01-01", "2021-01-01"], + "sale_price": [75000, 75000, 50000, 50000], + } + ) + + out = flag_dupe_date_price(df) + + assert _flagged_sales(out) == {"s3"} + + +def test_dupe_date_price_null_parcel_key_falls_back_to_sale_key(): + df = _df( + [ + _sale(None, "s1", "2020-01-01", 75000), + _sale(None, "s2", "2020-01-01", 75000), + _sale(None, "s3", "2021-01-01", 50000), + _sale(None, "s3", "2021-01-01", 50000), + ] + ) + + out = flag_dupe_date_price(df) + + assert _flagged_sales(out) == {"s3"} + + +def test_dupe_date_price_blank_parcel_key_falls_back_to_sale_key(): + df = _df( + [ + _sale("", "s1", "2020-01-01", 75000), + _sale(" ", "s2", "2020-01-01", 75000), + _sale("", "s3", "2021-01-01", 50000), + _sale("", "s3", "2021-01-01", 50000), + ] + ) + + out = flag_dupe_date_price(df) + + assert _flagged_sales(out) == {"s3"}