From 87df16d5336dda46789b40a0a69971ee319a698b Mon Sep 17 00:00:00 2001 From: Phil Ruff Date: Fri, 31 Jul 2026 09:49:49 +0100 Subject: [PATCH 1/9] refactor(finance): split finance utils into focused modules Split the 2,347-line investor_app/finance/utils.py into mortgage, taxes, scoring, and strategies submodules; utils.py now holds only core KPI primitives and the Django-coupled analysis. Update all 17 importers to canonical locations. Resolve LIMIT-20: delete the calculate_* aliases, the dead score_listing_v1 chain, the duplicate pure score_listing_v2, and the unused service-layer calculate_noi in property_service.py. Mark LIMIT-20 resolved in docs/KNOWN_LIMITATIONS.md. Full suite: 1803 passed. Ruff + mypy clean on touched files. --- .agents/logs/2026-07-31.jsonl | 1 + .../build-report-2026-07-31-finance-split.md | 54 + core/api_views.py | 8 +- core/services/__init__.py | 2 - core/services/brrrr.py | 8 +- core/services/cma.py | 9 +- core/services/market_scoring.py | 2 +- core/services/portfolio.py | 3 +- core/services/projections.py | 6 +- core/services/property_service.py | 23 - core/services/scoring.py | 13 +- core/tests/test_deal_analyzer.py | 14 +- core/tests/test_finance_utils.py | 12 +- docs/KNOWN_LIMITATIONS.md | 6 +- investor_app/finance/mortgage.py | 458 ++++ investor_app/finance/scoring.py | 150 ++ investor_app/finance/strategies.py | 518 ++++ investor_app/finance/taxes.py | 692 ++++++ investor_app/finance/utils.py | 2149 +---------------- tests/test_brrrr.py | 2 +- tests/test_finance_math.py | 15 +- tests/test_finance_utils.py | 165 +- tests/test_hold_period.py | 2 +- tests/test_market_scoring.py | 2 +- tests/test_property_service.py | 45 - tests/test_tax_analysis.py | 8 +- tests/test_underwriting_score.py | 172 +- tests_bdd/steps/test_property_analysis.py | 8 +- 28 files changed, 1946 insertions(+), 2601 deletions(-) create mode 100644 .agents/logs/2026-07-31.jsonl create mode 100644 .agents/reports/build-report-2026-07-31-finance-split.md create mode 100644 investor_app/finance/mortgage.py create mode 100644 investor_app/finance/scoring.py create mode 100644 investor_app/finance/strategies.py create mode 100644 investor_app/finance/taxes.py delete mode 100644 tests/test_property_service.py diff --git a/.agents/logs/2026-07-31.jsonl b/.agents/logs/2026-07-31.jsonl new file mode 100644 index 00000000..f1e278d3 --- /dev/null +++ b/.agents/logs/2026-07-31.jsonl @@ -0,0 +1 @@ +{"agent": "build", "session_id": "build-20260731-finance-split-001", "triggered_by": "feature-flow", "started_at": "2026-07-31T16:00:00Z", "timestamp": "2026-07-31T17:45:00Z", "duration_ms": 6300000, "skills_loaded": ["code-generation", "refactoring", "template-application"], "findings": [{"id": "FIND-001", "description": "Moved copy of total_return_summary dropped the purchase_price key from its return dict, breaking test_dict_keys_present — restored the key to match the original contract", "actionable": true, "manual_review_needed": false, "severity": "defect"}, {"id": "FIND-002", "description": "after_tax_irr in taxes.py had a local import of irr from utils that was unused (function reimplements npf.irr inline) — removed per ruff F401", "actionable": true, "manual_review_needed": false, "severity": "defect"}, {"id": "FIND-003", "description": "tests/test_underwriting_score.py still tested the deleted pure score_listing_v2 (audit finding #2); kept one_percent_rule/gross_rent_multiplier primitive tests, deleted TestScoreListingV2 since production score lives only in core/services/scoring.py (covered by core/tests/test_scoring_v2.py)", "actionable": true, "manual_review_needed": false, "severity": "defect"}, {"id": "FIND-004", "description": "Service-layer duplicate calculate_noi in core/services/property_service.py was exported but imported by no production code — deleted function, export, and its test file", "actionable": true, "manual_review_needed": false, "severity": "defect"}, {"id": "FIND-005", "description": "Dual-pipeline investigation: Django (PipelineAsset/PipelineProperty + core/services/pipeline.py) is the load-bearing pipeline; the pydantic prei FastAPI router, CLI, and orchestrator are not mounted in any Django URLconf/INSTALLED_APPS/docker-compose; only core/views/__init__.py couples to prei (get_state_landlord_score + lazy DiscoveryProcessor/BatchScreeningProcessor/discover_from_all for the Growth Explorer bridge). Full removal of pydantic state machine requires PM sign-off", "actionable": false, "manual_review_needed": true, "severity": "note"}, {"id": "FIND-006", "description": "Re-export backfill from utils.py was unnecessary: after updating all 17 importers, no remaining importer pulls a moved name from investor_app.finance.utils; keeping the monolith aliases would defeat the split", "actionable": false, "manual_review_needed": false, "severity": "note"}, {"id": "FIND-007", "description": "Pre-existing mypy error in tests/acceptance/conftest.py:69 (no-any-return) unrelated to this change — file unmodified", "actionable": false, "manual_review_needed": false, "severity": "note"}], "decision": "implemented", "blockers": [], "pr": null} diff --git a/.agents/reports/build-report-2026-07-31-finance-split.md b/.agents/reports/build-report-2026-07-31-finance-split.md new file mode 100644 index 00000000..60c31e04 --- /dev/null +++ b/.agents/reports/build-report-2026-07-31-finance-split.md @@ -0,0 +1,54 @@ +## Build Report — Finance utils split + audit finding resolution (LIMIT-20) + +**Status:** COMPLETE + +--- + +### Tasks Completed + +| Task | Title | Lines Changed | Status | +| -------- | ----- | ------------- | ------ | +| SPLIT-1 | Create `investor_app/finance/mortgage.py` (9 functions: mortgage, carrying costs, break-even rent, paydown, appreciation, ROI components) | ~330 | DONE | +| SPLIT-2 | Create `investor_app/finance/taxes.py` (11 functions: depreciation, tax benefits, after-tax IRR/CF, hold-period projections, sale proceeds, recapture) | ~640 | DONE | +| SPLIT-3 | Create `investor_app/finance/scoring.py` (5 primitives: 1% rule, GRM, price-to-rent, market normalization helpers) | ~150 | DONE | +| SPLIT-4 | Create `investor_app/finance/strategies.py` (8 functions: flip, rental, vacation, BRRRR calculators; `estimate_rehab_cost` decoupled from settings) | ~400 | DONE | +| SPLIT-5 | Rewrite `investor_app/finance/utils.py` to core math + Django-coupled analysis only; delete aliases (`calculate_noi`/`calculate_cap_rate`/`calculate_cash_on_cash`/`calculate_irr`), dead `score_listing_v1`, and deprecated `score_listing_v1_deprecated` chain | ~231 (net −2116) | DONE | +| IMPORTERS | Update 17 importers (services, api_views, views, tests) to canonical module locations | ~120 | DONE | +| AUDIT-1 | Remove dead service-layer duplicate `calculate_noi` from `core/services/property_service.py` (+ export in `__init__.py`, delete `tests/test_property_service.py`) | −45 | DONE | +| AUDIT-2 | Delete duplicate pure `score_listing_v2` tests (`TestScoreListingV2` in `tests/test_underwriting_score.py`); production version remains only in `core/services/scoring.py` | −161 | DONE | +| VERIFY | Fix behavior regression: `total_return_summary` must keep returning `purchase_price` key (moved copy dropped it) | +3 | DONE | +| DOCS | Mark LIMIT-20 resolved in `docs/KNOWN_LIMITATIONS.md` | +8 | DONE | + +### Artifacts Produced + +- [x] Source code files — `investor_app/finance/{mortgage,taxes,scoring,strategies}.py` +- [x] Source code files — rewritten `investor_app/finance/utils.py` +- [ ] Manifests in `manifests/` — N/A (no K8s surface touched) +- [ ] Pipeline in `pipeline-spec.yaml` — N/A (no CI pipeline change) +- [ ] Overlays in `overlays/` — N/A (no GitOps change) + +### Validation Results + +| Check | Status | +| --------- | ------ | +| Lint (ruff) | PASS | +| Typecheck (mypy, touched files) | PASS | +| Tests (full suite) | PASS — 1803 passed, 1 skipped, 261 deselected | +| Policy | PASS — no governance violations; `postgres`, `migration-safety`, `gitops` untouched | + +Pre-existing mypy error in `tests/acceptance/conftest.py:69` (no-any-return) is unrelated to this change — file unmodified. + +### Blockers + +None. + +### Dual-pipeline (pydantic `prei` vs Django) findings + +Per the user's directive to resolve the dual-pipeline question "using Django," investigated how much production code depends on the pydantic `prei` side: + +- **Django is the load-bearing pipeline**: `core/models/pipeline.py` (`PipelineAsset`, `PipelineProperty`), `core/services/pipeline.py`, screening, leasing, notifications, and ~20 prod/test files. This is the source of truth. +- **`prei` pydantic side is a standalone FastAPI microservice, not mounted**: `prei/api/pipeline_routes.py` (FastAPI router) and `prei/cli.py` are not referenced by any Django URLconf, `INSTALLED_APPS`, docker-compose service, or CI deploy. `prei/pipeline/orchestrator.py` is imported only by tests. +- **One production coupling**: `core/views/__init__.py` imports `prei.integrations.landlord_data.get_state_landlord_score` (top-level) and lazily imports `DiscoveryProcessor`/`BatchScreeningProcessor`/`ScreeningThresholds`/`PipelineEngine`/`InMemoryAssetRepository`/`discover_from_all` for the Growth Explorer bridge (P0 fix from `docs/assessments/AUDIT_GA_PIPELINE.md`). +- **Recommendation**: keep the pydantic discovery/screening *processors* (they are the only working bridge from Growth Explorer into pipeline screening, and they are Decimal-based after Phase B) but do not build new state on pydantic models (`PropertyAsset`/`StageLog`); persist pipeline state via Django `PipelineProperty`/`PipelineAsset`. Full removal of the pydantic state machine, FastAPI routes, and CLI is a separate reviewed change requiring PM sign-off — it touches the orchestrator, handlers, 11 test files, and the Growth Explorer bridge. Filed as a follow-up recommendation, not executed here. + +The finance-utils split itself is independent of that decision: `prei/pipeline/{orchestrator,handlers/underwriting}.py` still import only `to_decimal`/`cap_rate`/`cash_on_cash` from `investor_app.finance.utils`, all of which remain in place. diff --git a/core/api_views.py b/core/api_views.py index a963a289..baacb275 100644 --- a/core/api_views.py +++ b/core/api_views.py @@ -26,13 +26,17 @@ from .models import VrmProperty from .serializers import VrmPropertySerializer -from investor_app.finance.utils import ( +from investor_app.finance.mortgage import ( calculate_break_even_rent, calculate_carrying_costs as calc_costs, + calculate_roi_components, +) +from investor_app.finance.strategies import ( calculate_flip_strategy, calculate_rental_strategy, - calculate_roi_components, calculate_vacation_rental_strategy, +) +from investor_app.finance.utils import ( cap_rate as calc_cap_rate, cash_on_cash as calc_coc, compute_analysis_for_property, diff --git a/core/services/__init__.py b/core/services/__init__.py index 55971a3a..5c16166d 100644 --- a/core/services/__init__.py +++ b/core/services/__init__.py @@ -2,14 +2,12 @@ from core.services.portfolio import compute_portfolio_summary from core.services.property_service import ( - calculate_noi, compute_noi, compute_noi_for_user, compute_noi_from_amounts, ) __all__ = [ - "calculate_noi", "compute_noi", "compute_noi_for_user", "compute_noi_from_amounts", diff --git a/core/services/brrrr.py b/core/services/brrrr.py index 5c1d8e24..d6f605e2 100644 --- a/core/services/brrrr.py +++ b/core/services/brrrr.py @@ -34,17 +34,15 @@ from django.conf import settings from core.models import Listing -from investor_app.finance.utils import ( +from investor_app.finance.mortgage import calculate_monthly_mortgage +from investor_app.finance.strategies import ( brrrr_coc_return, - calculate_monthly_mortgage, cash_left_in_deal, - dscr, estimate_arv, estimate_rehab_cost, max_refinance_loan, - noi, - to_decimal, ) +from investor_app.finance.utils import dscr, noi, to_decimal logger = logging.getLogger(__name__) diff --git a/core/services/cma.py b/core/services/cma.py index e59543e5..fcfa731d 100644 --- a/core/services/cma.py +++ b/core/services/cma.py @@ -7,13 +7,8 @@ from django.conf import settings from core.models import Listing, MarketSnapshot -from investor_app.finance.utils import ( - calculate_monthly_mortgage, - cap_rate, - cash_on_cash, - dscr, - noi, -) +from investor_app.finance.mortgage import calculate_monthly_mortgage +from investor_app.finance.utils import cap_rate, cash_on_cash, dscr, noi logger = logging.getLogger(__name__) diff --git a/core/services/market_scoring.py b/core/services/market_scoring.py index 892a6228..2e6836ec 100644 --- a/core/services/market_scoring.py +++ b/core/services/market_scoring.py @@ -85,7 +85,7 @@ def _score_market_from_snapshot(snapshot) -> Decimal: Returns a Decimal in [0, 100] computed as a weighted average of normalised sub-scores for each available signal. """ - from investor_app.finance.utils import ( + from investor_app.finance.scoring import ( clamp_market_score, normalize_market_growth_rate_score, normalize_market_price_to_rent_score, diff --git a/core/services/portfolio.py b/core/services/portfolio.py index 23e1b63e..cdbde8e3 100644 --- a/core/services/portfolio.py +++ b/core/services/portfolio.py @@ -359,7 +359,8 @@ def calculate_ytd_cashflow( def _get_annual_debt_service(property_obj: Property) -> Decimal: """Calculate annual debt service for a property.""" - from investor_app.finance.utils import calculate_monthly_mortgage, to_decimal + from investor_app.finance.mortgage import calculate_monthly_mortgage + from investor_app.finance.utils import to_decimal loan_amount = to_decimal(property_obj.purchase_price) * ( Decimal("1") - to_decimal(property_obj.down_payment_pct) diff --git a/core/services/projections.py b/core/services/projections.py index ea8857ce..d771b192 100644 --- a/core/services/projections.py +++ b/core/services/projections.py @@ -10,12 +10,12 @@ import numpy as np import numpy_financial as npf -from investor_app.finance.utils import ( +from investor_app.finance.mortgage import calculate_monthly_mortgage +from investor_app.finance.taxes import ( calculate_after_tax_cashflow, calculate_annual_depreciation, - calculate_monthly_mortgage, - to_decimal, ) +from investor_app.finance.utils import to_decimal if TYPE_CHECKING: from core.models import Property diff --git a/core/services/property_service.py b/core/services/property_service.py index 7b4d6a34..a005a8b4 100644 --- a/core/services/property_service.py +++ b/core/services/property_service.py @@ -130,26 +130,3 @@ def compute_noi_from_amounts( Annual NOI as a ``Decimal``, quantized to two decimal places. """ return noi(monthly_income, monthly_expenses).quantize(Decimal("0.01")) - - -def calculate_noi( - gross_income: Decimal, - operating_expenses: Decimal, -) -> Decimal: - """Calculate annual Net Operating Income (NOI). - - NOI = Gross Income - Operating Expenses - - This is a pure service-layer function that delegates to the finance - utility layer. It does not touch the database. - - Args: - gross_income: Total annual gross income from the property. - operating_expenses: Total annual operating expenses (excluding debt service). - - Returns: - Annual NOI as a ``Decimal``, quantized to two decimal places. - """ - return (to_decimal(gross_income) - to_decimal(operating_expenses)).quantize( - Decimal("0.01") - ) diff --git a/core/services/scoring.py b/core/services/scoring.py index dc49890f..52bebb92 100644 --- a/core/services/scoring.py +++ b/core/services/scoring.py @@ -130,12 +130,11 @@ def score_listing_v2(property_obj, targets) -> UnderwritingScore: Returns: UnderwritingScore with all fields populated. """ - from investor_app.finance.utils import ( - build_cashflows, + from investor_app.finance.taxes import ( calculate_annual_depreciation, calculate_after_tax_cashflow, - irr as calc_irr, ) + from investor_app.finance.utils import build_cashflows, irr as calc_irr from core.models import UserProfile pp = property_obj.purchase_price @@ -167,12 +166,8 @@ def score_listing_v2(property_obj, targets) -> UnderwritingScore: total_expenses = opex + mgmt_fee annual_noi = effective_rent - total_expenses # KPIs using utils functions - from investor_app.finance.utils import ( - cap_rate as calc_cap_rate, - cash_on_cash, - dscr, - gross_rent_multiplier, - ) + from investor_app.finance.scoring import gross_rent_multiplier + from investor_app.finance.utils import cap_rate as calc_cap_rate, cash_on_cash, dscr cap = calc_cap_rate(annual_noi, pp) grm = gross_rent_multiplier(pp, annual_rent) if annual_rent > 0 else Decimal("999") diff --git a/core/tests/test_deal_analyzer.py b/core/tests/test_deal_analyzer.py index fa0d70cf..00b660f4 100644 --- a/core/tests/test_deal_analyzer.py +++ b/core/tests/test_deal_analyzer.py @@ -8,18 +8,16 @@ import pytest -from investor_app.finance.utils import ( +from investor_app.finance.mortgage import ( calculate_break_even_rent, - calculate_flip_strategy, calculate_monthly_mortgage, - calculate_rental_strategy, calculate_roi_components, - cap_rate, - cash_on_cash, - dscr, - irr, - noi, ) +from investor_app.finance.strategies import ( + calculate_flip_strategy, + calculate_rental_strategy, +) +from investor_app.finance.utils import cap_rate, cash_on_cash, dscr, irr, noi # --------------------------------------------------------------------------- # noi diff --git a/core/tests/test_finance_utils.py b/core/tests/test_finance_utils.py index 0d252c9c..ce7762e8 100644 --- a/core/tests/test_finance_utils.py +++ b/core/tests/test_finance_utils.py @@ -3,8 +3,7 @@ import pytest -from investor_app.finance.utils import ( - build_cashflows, +from investor_app.finance.mortgage import ( calculate_appreciation, calculate_break_even_rent, calculate_carrying_costs, @@ -13,12 +12,15 @@ calculate_principal_paydown, calculate_property_tax, calculate_roi_components, - calculate_tax_benefits, + estimate_insurance, +) +from investor_app.finance.taxes import calculate_tax_benefits +from investor_app.finance.utils import ( + build_cashflows, cap_rate, cash_on_cash, - dscr, - estimate_insurance, compute_analysis_for_property, + dscr, irr, noi, ) diff --git a/docs/KNOWN_LIMITATIONS.md b/docs/KNOWN_LIMITATIONS.md index aeb8f01c..60361246 100644 --- a/docs/KNOWN_LIMITATIONS.md +++ b/docs/KNOWN_LIMITATIONS.md @@ -212,15 +212,15 @@ This means a user who runs the API pre-`populate_growth_areas` gets empty result --- -### [LIMIT-20] 🟡 HIGH — Divergent bare-function vs. `calculate_*` contracts for the same formulas, plus a duplicate `score_listing_v2` +### [LIMIT-20] 🟡 HIGH — Divergent bare-function vs. `calculate_*` contracts for the same formulas, plus a duplicate `score_listing_v2` (resolved) **Location:** `investor_app/finance/utils.py` — `noi`/`cap_rate`/`cash_on_cash`/`dscr` vs. `calculate_noi`/`calculate_cap_rate`/`calculate_cash_on_cash`/`calculate_irr`; `investor_app/finance/utils.py:1830` vs. `core/services/scoring.py:110` (`score_listing_v2`). **Impact:** Two independent implementations exist for the same four KPI formulas — the bare functions return `Decimal("0")` on invalid input (e.g. zero purchase price, zero debt service), while the `calculate_*` variants raise `ValueError` under the same conditions. Callers that reach for the "wrong" variant get silently different failure behavior for identical bad input, and there is no single source of truth to point developers at. Separately, two functions both named `score_listing_v2` exist with different signatures — one in `investor_app/finance/utils.py`, one in `core/services/scoring.py` — an accident waiting to cause a wrong-function-imported bug. -**Workaround:** None currently. Callers must know which variant (bare vs. `calculate_*`) they're calling and its error-handling contract; `score_listing_v2` callers must be careful to import from the intended module. +**Workaround:** Resolved by the finance-utils split. The `calculate_*` aliases (`calculate_noi`, `calculate_cap_rate`, `calculate_cash_on_cash`, `calculate_irr`), the dead `score_listing_v1` chain, and the duplicate pure `score_listing_v2` were deleted. The bare functions (`noi`, `cap_rate`, `cash_on_cash`, `dscr`, `irr`) are now the single source of truth in `investor_app/finance/utils.py`, with the production underwriting score living only in `core/services/scoring.py`. The service-layer duplicate `calculate_noi` in `core/services/property_service.py` was also removed. -**Fix tracked in:** Not yet filed. Found during Phase B (docs/TOP_01_PLAN.md) financial-math audit; out of scope for that PR since it requires an API-contract decision (which behavior is canonical) rather than a mechanical fix. +**Fix tracked in:** Resolved in the finance-utils split PR (finance package reorganized into `mortgage`, `taxes`, `scoring`, `strategies` submodules). --- diff --git a/investor_app/finance/mortgage.py b/investor_app/finance/mortgage.py new file mode 100644 index 00000000..00ed418b --- /dev/null +++ b/investor_app/finance/mortgage.py @@ -0,0 +1,458 @@ +"""Mortgage, carrying costs, break-even rent, and ROI component calculations.""" + +from __future__ import annotations + +from datetime import datetime +from decimal import Decimal +import logging +from typing import Any, Dict + +from investor_app.finance.utils import to_decimal + +logger = logging.getLogger(__name__) + + +def calculate_monthly_mortgage( + loan_amount: Decimal, interest_rate: Decimal, loan_term_years: int +) -> Decimal: + """Calculate monthly mortgage payment (principal and interest). + + Args: + loan_amount: Total loan amount + interest_rate: Annual interest rate as percentage (e.g., 7.5 for 7.5%) + loan_term_years: Loan term in years + + Returns: + Monthly payment amount + """ + loan_amt = to_decimal(loan_amount) + rate = to_decimal(interest_rate) + + if loan_amt == 0: + return Decimal("0") + + if rate == 0: + # No interest - simple division + return loan_amt / Decimal(loan_term_years * 12) + + monthly_rate = rate / Decimal(100) / Decimal(12) + num_payments = Decimal(loan_term_years * 12) + + # Standard amortization formula: M = P[r(1+r)^n]/[(1+r)^n-1] + factor = (Decimal(1) + monthly_rate) ** num_payments + monthly_payment = loan_amt * (monthly_rate * factor) / (factor - Decimal(1)) + + return monthly_payment.quantize(Decimal("0.01")) + + +def calculate_property_tax( + property_value: Decimal, tax_rate_percent: Decimal +) -> Decimal: + """Calculate annual property tax. + + Args: + property_value: Property value/assessed value + tax_rate_percent: Property tax rate as percentage (e.g., 2.1 for 2.1%) + + Returns: + Annual property tax amount + """ + return ( + to_decimal(property_value) * to_decimal(tax_rate_percent) / Decimal(100) + ).quantize(Decimal("0.01")) + + +def estimate_insurance( + property_value: Decimal, + property_type: str = "single-family", + year_built: int = 2000, +) -> Decimal: + """Estimate annual insurance cost. + + Args: + property_value: Property value + property_type: Type of property (single-family, condo, multi-family) + year_built: Year property was built + + Returns: + Estimated annual insurance premium + """ + base_rate = Decimal("1200") # National average for $250k home + + # Adjust for property value + value_factor = to_decimal(property_value) / Decimal("250000") + + # Adjust for property type + type_factors = { + "single-family": Decimal("1.0"), + "condo": Decimal("0.7"), + "multi-family": Decimal("1.3"), + "commercial": Decimal("1.5"), + } + type_factor = type_factors.get(property_type, Decimal("1.0")) + + # Adjust for age + current_year = datetime.now().year + age = max(0, current_year - year_built) + age_factor = Decimal("1.0") + (Decimal(age) / Decimal(50)) + + annual_insurance = base_rate * value_factor * type_factor * age_factor + return annual_insurance.quantize(Decimal("0.01")) + + +def calculate_maintenance_reserve( + property_value: Decimal, + year_built: int = 2000, + annual_percent: Decimal = Decimal("1.0"), +) -> Decimal: + """Calculate annual maintenance reserve (1% rule with age adjustment). + + Args: + property_value: Property value + year_built: Year property was built + annual_percent: Base annual percentage of property value (default 1%) + + Returns: + Annual maintenance reserve amount + """ + base_maintenance = ( + to_decimal(property_value) * to_decimal(annual_percent) / Decimal(100) + ) + + # Adjust for age + if year_built < 1980: + age_factor = Decimal("1.5") + elif year_built < 2000: + age_factor = Decimal("1.2") + else: + age_factor = Decimal("1.0") + + return (base_maintenance * age_factor).quantize(Decimal("0.01")) + + +def calculate_break_even_rent( + monthly_carrying_costs: Decimal, + vacancy_rate_percent: Decimal, + property_management_percent: Decimal = Decimal("10"), +) -> Dict[str, Decimal]: + """Calculate break-even rent needed to cover carrying costs. + + Args: + monthly_carrying_costs: Total monthly carrying costs (excluding property management) + vacancy_rate_percent: Vacancy rate as percentage (e.g., 8 for 8%) + property_management_percent: Property management fee as percentage of rent + + Returns: + Dictionary with breakEvenRent and related metrics + """ + costs = to_decimal(monthly_carrying_costs) + vacancy = to_decimal(vacancy_rate_percent) / Decimal(100) + mgmt = to_decimal(property_management_percent) / Decimal(100) + + # Formula: rent * (1 - vacancy) * (1 - mgmt) = costs + # rent = costs / ((1 - vacancy) * (1 - mgmt)) + divisor = (Decimal(1) - vacancy) * (Decimal(1) - mgmt) + if divisor == 0: + return { + "monthly": Decimal("0"), + "annual": Decimal("0"), + } + break_even = costs / divisor + + return { + "monthly": break_even.quantize(Decimal("0.01")), + "annual": (break_even * Decimal(12)).quantize(Decimal("0.01")), + } + + +def calculate_carrying_costs( + purchase_price: Decimal, + loan_amount: Decimal, + interest_rate: Decimal, + loan_term_years: int, + property_tax_rate: Decimal, + insurance_annual: Decimal | None = None, + hoa_monthly: Decimal = Decimal("0"), + utilities_monthly: Decimal = Decimal("0"), + maintenance_annual_percent: Decimal = Decimal("1.0"), + property_type: str = "single-family", + year_built: int = 2000, +) -> Dict[str, Any]: + """Calculate complete carrying costs breakdown. + + Args: + purchase_price: Property purchase price + loan_amount: Mortgage loan amount + interest_rate: Annual interest rate as percentage + loan_term_years: Loan term in years + property_tax_rate: Property tax rate as percentage + insurance_annual: Annual insurance cost (if None, will estimate) + hoa_monthly: Monthly HOA fees + utilities_monthly: Monthly utility costs + maintenance_annual_percent: Maintenance as percentage of property value + property_type: Type of property + year_built: Year property was built + + Returns: + Dictionary with detailed carrying cost breakdown + """ + # Calculate mortgage + monthly_mortgage = calculate_monthly_mortgage( + loan_amount, interest_rate, loan_term_years + ) + + # Calculate property tax + annual_property_tax = calculate_property_tax(purchase_price, property_tax_rate) + monthly_property_tax = annual_property_tax / Decimal(12) + + # Calculate or use provided insurance + if insurance_annual is None: + annual_insurance = estimate_insurance(purchase_price, property_type, year_built) + else: + annual_insurance = to_decimal(insurance_annual) + monthly_insurance = annual_insurance / Decimal(12) + + # Calculate maintenance + annual_maintenance = calculate_maintenance_reserve( + purchase_price, year_built, maintenance_annual_percent + ) + monthly_maintenance = annual_maintenance / Decimal(12) + + # Monthly costs + monthly_hoa = to_decimal(hoa_monthly) + monthly_utilities = to_decimal(utilities_monthly) + + # Calculate totals + monthly_total = ( + monthly_mortgage + + monthly_property_tax + + monthly_insurance + + monthly_hoa + + monthly_utilities + + monthly_maintenance + ) + + annual_total = monthly_total * Decimal(12) + + return { + "monthly": { + "mortgage": monthly_mortgage.quantize(Decimal("0.01")), + "propertyTax": monthly_property_tax.quantize(Decimal("0.01")), + "insurance": monthly_insurance.quantize(Decimal("0.01")), + "hoa": monthly_hoa.quantize(Decimal("0.01")), + "utilities": monthly_utilities.quantize(Decimal("0.01")), + "maintenance": monthly_maintenance.quantize(Decimal("0.01")), + "total": monthly_total.quantize(Decimal("0.01")), + }, + "annual": { + "mortgage": (monthly_mortgage * Decimal(12)).quantize(Decimal("0.01")), + "propertyTax": annual_property_tax.quantize(Decimal("0.01")), + "insurance": annual_insurance.quantize(Decimal("0.01")), + "hoa": (monthly_hoa * Decimal(12)).quantize(Decimal("0.01")), + "utilities": (monthly_utilities * Decimal(12)).quantize(Decimal("0.01")), + "maintenance": annual_maintenance.quantize(Decimal("0.01")), + "total": annual_total.quantize(Decimal("0.01")), + }, + } + + +def calculate_principal_paydown( + loan_amount: Decimal, + interest_rate: Decimal, + loan_term_years: int, + num_years: int = 1, +) -> Decimal: + """Calculate total principal paid down over specified number of years. + + Args: + loan_amount: Initial loan amount + interest_rate: Annual interest rate as percentage (e.g., 7.5 for 7.5%) + loan_term_years: Total loan term in years + num_years: Number of years to calculate paydown for (default 1) + + Returns: + Total principal paid down over the specified period + """ + if loan_amount == 0 or num_years == 0: + return Decimal("0") + + loan_amt = to_decimal(loan_amount) + rate = to_decimal(interest_rate) + + if rate == 0: + # No interest - equal principal payments + monthly_principal = loan_amt / Decimal(loan_term_years * 12) + return monthly_principal * Decimal(num_years * 12) + + monthly_rate = rate / Decimal(100) / Decimal(12) + monthly_payment = calculate_monthly_mortgage( + loan_amount, interest_rate, loan_term_years + ) + + # Calculate principal paid by simulating each payment + remaining_balance = loan_amt + total_principal_paid = Decimal("0") + + for month in range(num_years * 12): + interest_payment = remaining_balance * monthly_rate + principal_payment = monthly_payment - interest_payment + total_principal_paid += principal_payment + remaining_balance -= principal_payment + + if remaining_balance <= 0: + break + + return total_principal_paid.quantize(Decimal("0.01")) + + +def calculate_appreciation( + property_value: Decimal, + appreciation_rate: Decimal, + num_years: int = 1, +) -> Decimal: + """Calculate property appreciation over specified number of years. + + Args: + property_value: Current property value + appreciation_rate: Annual appreciation rate as percentage (e.g., 3.0 for 3%) + num_years: Number of years to project (default 1) + + Returns: + Total appreciation amount + """ + value = to_decimal(property_value) + rate = to_decimal(appreciation_rate) / Decimal(100) + + future_value = value * ((Decimal(1) + rate) ** Decimal(num_years)) + appreciation = future_value - value + + return appreciation.quantize(Decimal("0.01")) + + +def calculate_roi_components( + purchase_price: Decimal, + loan_amount: Decimal, + interest_rate: Decimal, + loan_term_years: int, + total_cash_invested: Decimal, + annual_cash_flow: Decimal, + appreciation_rate: Decimal = Decimal("3.0"), + tax_bracket: Decimal = Decimal("24"), + num_years: int = 5, +) -> Dict[str, Any]: + """Calculate comprehensive ROI with all components over multiple years. + + Args: + purchase_price: Property purchase price + loan_amount: Mortgage loan amount + interest_rate: Annual interest rate as percentage + loan_term_years: Loan term in years + total_cash_invested: Total cash invested (down payment + closing costs) + annual_cash_flow: Annual pre-tax cash flow + appreciation_rate: Annual appreciation rate as percentage (default 3%) + tax_bracket: Marginal tax bracket as percentage (default 24%) + num_years: Number of years to project (default 5) + + Returns: + Dictionary with ROI components and projections + """ + from investor_app.finance.taxes import calculate_tax_benefits + + # Year 1 calculations + year1_cash_flow = to_decimal(annual_cash_flow) + year1_principal_paydown = calculate_principal_paydown( + loan_amount, interest_rate, loan_term_years, 1 + ) + year1_appreciation = calculate_appreciation(purchase_price, appreciation_rate, 1) + year1_tax_benefits = calculate_tax_benefits( + loan_amount, interest_rate, loan_term_years, purchase_price, tax_bracket, 1 + ) + + year1_total_return = ( + year1_cash_flow + + year1_principal_paydown + + year1_appreciation + + year1_tax_benefits + ) + + if total_cash_invested > 0: + year1_roi = year1_total_return / to_decimal(total_cash_invested) * Decimal(100) + else: + year1_roi = Decimal("0") + + # Multi-year calculations + total_cash_flow = year1_cash_flow * Decimal( + num_years + ) # Simplified: assumes constant + total_principal_paydown = calculate_principal_paydown( + loan_amount, interest_rate, loan_term_years, num_years + ) + total_appreciation = calculate_appreciation( + purchase_price, appreciation_rate, num_years + ) + + # Sum tax benefits for each year + total_tax_benefits = Decimal("0") + for year in range(1, num_years + 1): + total_tax_benefits += calculate_tax_benefits( + loan_amount, + interest_rate, + loan_term_years, + purchase_price, + tax_bracket, + year, + ) + + total_return = ( + total_cash_flow + + total_principal_paydown + + total_appreciation + + total_tax_benefits + ) + + if total_cash_invested > 0: + multi_year_roi = total_return / to_decimal(total_cash_invested) * Decimal(100) + # Annualized return + annualized_roi = ( + (Decimal(1) + multi_year_roi / Decimal(100)) + ** (Decimal(1) / Decimal(num_years)) + - Decimal(1) + ) * Decimal(100) + else: + multi_year_roi = Decimal("0") + annualized_roi = Decimal("0") + + # Component percentages for year 1 + if year1_total_return > 0: + cash_flow_pct = year1_cash_flow / year1_total_return * Decimal(100) + appreciation_pct = year1_appreciation / year1_total_return * Decimal(100) + equity_pct = year1_principal_paydown / year1_total_return * Decimal(100) + tax_pct = year1_tax_benefits / year1_total_return * Decimal(100) + else: + cash_flow_pct = appreciation_pct = equity_pct = tax_pct = Decimal("0") + + return { + "year1": { + "roi": year1_roi.quantize(Decimal("0.1")), + "totalReturn": year1_total_return.quantize(Decimal("0.01")), + "cashFlow": year1_cash_flow.quantize(Decimal("0.01")), + "principalPaydown": year1_principal_paydown.quantize(Decimal("0.01")), + "appreciation": year1_appreciation.quantize(Decimal("0.01")), + "taxBenefits": year1_tax_benefits.quantize(Decimal("0.01")), + }, + f"year{num_years}Projected": { + "roi": multi_year_roi.quantize(Decimal("0.1")), + "annualizedRoi": annualized_roi.quantize(Decimal("0.1")), + "totalReturn": total_return.quantize(Decimal("0.01")), + "totalCashFlow": total_cash_flow.quantize(Decimal("0.01")), + "totalPrincipalPaydown": total_principal_paydown.quantize(Decimal("0.01")), + "totalAppreciation": total_appreciation.quantize(Decimal("0.01")), + "totalTaxBenefits": total_tax_benefits.quantize(Decimal("0.01")), + }, + "components": { + "cashFlowReturn": cash_flow_pct.quantize(Decimal("0.1")), + "appreciationReturn": appreciation_pct.quantize(Decimal("0.1")), + "equityBuildupReturn": equity_pct.quantize(Decimal("0.1")), + "taxBenefitsReturn": tax_pct.quantize(Decimal("0.1")), + }, + } diff --git a/investor_app/finance/scoring.py b/investor_app/finance/scoring.py new file mode 100644 index 00000000..277e8dfe --- /dev/null +++ b/investor_app/finance/scoring.py @@ -0,0 +1,150 @@ +"""Market scoring primitives: 1% rule, GRM, price-to-rent, normalization helpers. + +The underwriting score itself lives in ``core.services.scoring`` (Django-coupled). +This module holds the pure market/listing primitives shared across services. +""" + +from __future__ import annotations + +from decimal import Decimal +import logging + +from investor_app.finance.utils import to_decimal + +logger = logging.getLogger(__name__) + + +def one_percent_rule(monthly_rent: Decimal, purchase_price: Decimal) -> bool: + """Evaluate the 1% Rule for a rental property. + + The 1% Rule is a quick pass/fail filter: monthly rent should be at least + 1% of the purchase price to indicate a potentially viable rental investment. + + Args: + monthly_rent: Expected gross monthly rental income. + purchase_price: Total purchase price of the property. + + Returns: + True if monthly_rent / purchase_price >= 0.01, False otherwise. + + Raises: + ValueError: If purchase_price is zero or negative. + """ + pp = to_decimal(purchase_price) + if pp <= Decimal("0"): + raise ValueError( + f"purchase_price must be greater than zero (received {purchase_price})" + ) + return to_decimal(monthly_rent) / pp >= Decimal("0.01") + + +def gross_rent_multiplier(purchase_price: Decimal, annual_rent: Decimal) -> Decimal: + """Calculate Gross Rent Multiplier (GRM). + + GRM = Purchase Price / Annual Rent + + Lower GRM values indicate better value relative to rental income. + + Args: + purchase_price: Total purchase price of the property. + annual_rent: Expected gross annual rental income. + + Returns: + GRM as a Decimal. + + Raises: + ValueError: If annual_rent is zero or negative. + """ + ar = to_decimal(annual_rent) + if ar <= Decimal("0"): + raise ValueError( + f"annual_rent must be greater than zero (received {annual_rent})" + ) + return to_decimal(purchase_price) / ar + + +def price_to_rent_ratio( + median_home_price: Decimal, annual_median_rent: Decimal +) -> Decimal: + """Calculate market price-to-rent ratio. + + Args: + median_home_price: Median home purchase price. + annual_median_rent: Median annual rent. + + Returns: + Price-to-rent ratio as a Decimal. + + Raises: + ValueError: If annual_median_rent is zero or negative. + """ + annual_rent = to_decimal(annual_median_rent) + if annual_rent <= Decimal("0"): + raise ValueError( + "annual_median_rent must be greater than zero " + f"(received {annual_median_rent})" + ) + return to_decimal(median_home_price) / annual_rent + + +_EXCELLENT_PRICE_TO_RENT_THRESHOLD = Decimal("15") +_NEUTRAL_PRICE_TO_RENT_THRESHOLD = Decimal("20") +_MAX_PRICE_TO_RENT_THRESHOLD = Decimal("30") +_HIGH_SCORE_FLOOR = Decimal("60") +_HIGH_SCORE_RANGE = Decimal("40") +_LOW_SCORE_RANGE = Decimal("60") + +_MIN_GROWTH_RATE_PERCENT = Decimal("-5") +_MAX_GROWTH_RATE_PERCENT = Decimal("10") +_GROWTH_RATE_RANGE = _MAX_GROWTH_RATE_PERCENT - _MIN_GROWTH_RATE_PERCENT + + +def normalize_market_price_to_rent_score(price_to_rent: Decimal) -> Decimal: + """Convert price-to-rent ratio into a 0-100 market sub-score. + + Args: + price_to_rent: Price-to-rent ratio for a market. + + Returns: + Market sub-score in [0, 100], where higher is better. + """ + if price_to_rent <= Decimal("0"): + return Decimal("0") + if price_to_rent < _EXCELLENT_PRICE_TO_RENT_THRESHOLD: + return Decimal("100") + if price_to_rent <= _NEUTRAL_PRICE_TO_RENT_THRESHOLD: + return (_NEUTRAL_PRICE_TO_RENT_THRESHOLD - price_to_rent) / ( + _NEUTRAL_PRICE_TO_RENT_THRESHOLD - _EXCELLENT_PRICE_TO_RENT_THRESHOLD + ) * _HIGH_SCORE_RANGE + _HIGH_SCORE_FLOOR + if price_to_rent <= _MAX_PRICE_TO_RENT_THRESHOLD: + return ( + (_MAX_PRICE_TO_RENT_THRESHOLD - price_to_rent) + / (_MAX_PRICE_TO_RENT_THRESHOLD - _NEUTRAL_PRICE_TO_RENT_THRESHOLD) + * _LOW_SCORE_RANGE + ) + return Decimal("0") + + +def normalize_market_growth_rate_score(growth_rate: Decimal) -> Decimal: + """Convert annual growth rate percent into a 0-100 market sub-score. + + Args: + growth_rate: Annual growth rate as a percent value. + + Returns: + Market sub-score in [0, 100], where higher is better. + """ + clamped = max(_MIN_GROWTH_RATE_PERCENT, min(_MAX_GROWTH_RATE_PERCENT, growth_rate)) + return (clamped - _MIN_GROWTH_RATE_PERCENT) / _GROWTH_RATE_RANGE * Decimal("100") + + +def clamp_market_score(value: Decimal) -> Decimal: + """Clamp a market score to the valid 0-100 range. + + Args: + value: Raw market score value. + + Returns: + Score clamped to [0, 100]. + """ + return max(Decimal("0"), min(Decimal("100"), value)) diff --git a/investor_app/finance/strategies.py b/investor_app/finance/strategies.py new file mode 100644 index 00000000..d22bca69 --- /dev/null +++ b/investor_app/finance/strategies.py @@ -0,0 +1,518 @@ +"""Investment strategy calculations: fix-and-flip, buy-and-hold, vacation rental, BRRRR.""" + +from __future__ import annotations + +from decimal import Decimal +import logging +from statistics import median +from typing import Any, Dict + +from investor_app.finance.utils import to_decimal + +logger = logging.getLogger(__name__) + + +def calculate_flip_strategy( + purchase_price: Decimal, + renovation_costs: Decimal, + holding_period_months: int, + expected_sale_price: Decimal, + selling_costs: Decimal, + down_payment: Decimal, + loan_amount: Decimal, + interest_rate: Decimal, + loan_term_years: int, + closing_costs: Decimal, + property_tax_rate: Decimal, + insurance_annual: Decimal | None = None, + utilities_monthly: Decimal = Decimal("0"), + property_type: str = "single-family", + year_built: int = 2000, +) -> Dict[str, Any]: + """Calculate fix-and-flip strategy returns. + + Args: + purchase_price: Property purchase price + renovation_costs: Total renovation costs + holding_period_months: How long to hold before selling (3-6 months typical) + expected_sale_price: Expected sale price after renovation + selling_costs: Total selling costs (realtor fees, etc.) + down_payment: Down payment amount + loan_amount: Mortgage loan amount + interest_rate: Annual interest rate as percentage + loan_term_years: Loan term in years + closing_costs: Closing costs on purchase + property_tax_rate: Property tax rate as percentage + insurance_annual: Annual insurance cost + utilities_monthly: Monthly utility costs while vacant + property_type: Type of property + year_built: Year property was built + + Returns: + Dictionary with flip strategy analysis + """ + from investor_app.finance.mortgage import ( + calculate_monthly_mortgage, + calculate_principal_paydown, + calculate_property_tax, + estimate_insurance, + ) + + # Calculate holding costs for the period + monthly_mortgage = calculate_monthly_mortgage( + loan_amount, interest_rate, loan_term_years + ) + annual_property_tax = calculate_property_tax(purchase_price, property_tax_rate) + monthly_property_tax = annual_property_tax / Decimal(12) + + if insurance_annual is None: + annual_insurance = estimate_insurance(purchase_price, property_type, year_built) + else: + annual_insurance = to_decimal(insurance_annual) + monthly_insurance = annual_insurance / Decimal(12) + + monthly_holding_costs = ( + monthly_mortgage + + monthly_property_tax + + monthly_insurance + + to_decimal(utilities_monthly) + ) + + total_holding_costs = monthly_holding_costs * Decimal(holding_period_months) + + # Total investment + total_investment = ( + to_decimal(down_payment) + + to_decimal(closing_costs) + + to_decimal(renovation_costs) + ) + + # Calculate proceeds + gross_sale_proceeds = to_decimal(expected_sale_price) + net_sale_proceeds = gross_sale_proceeds - to_decimal(selling_costs) + + # Remaining loan balance after holding period + principal_paid = calculate_principal_paydown( + loan_amount, interest_rate, loan_term_years, holding_period_months // 12 + ) + remaining_loan = to_decimal(loan_amount) - principal_paid + + # Net profit + net_profit = ( + net_sale_proceeds - remaining_loan - total_holding_costs - total_investment + ) + + # ROI + if total_investment > 0: + roi_percent = net_profit / total_investment * Decimal(100) + # Annualized return + years = Decimal(holding_period_months) / Decimal(12) + if years > 0 and roi_percent > Decimal("-100"): + annualized_return = ( + (Decimal(1) + roi_percent / Decimal(100)) ** (Decimal(1) / years) + - Decimal(1) + ) * Decimal(100) + else: + annualized_return = Decimal("0") + else: + roi_percent = Decimal("0") + annualized_return = Decimal("0") + + return { + "totalInvestment": total_investment.quantize(Decimal("0.01")), + "holdingCosts": total_holding_costs.quantize(Decimal("0.01")), + "renovationCosts": to_decimal(renovation_costs).quantize(Decimal("0.01")), + "saleProceeds": gross_sale_proceeds.quantize(Decimal("0.01")), + "sellingCosts": to_decimal(selling_costs).quantize(Decimal("0.01")), + "netProfit": net_profit.quantize(Decimal("0.01")), + "roi": roi_percent.quantize(Decimal("0.1")), + "timeframe": f"{holding_period_months} months", + "annualizedReturn": annualized_return.quantize(Decimal("0.1")), + } + + +def calculate_rental_strategy( + purchase_price: Decimal, + down_payment: Decimal, + loan_amount: Decimal, + interest_rate: Decimal, + loan_term_years: int, + closing_costs: Decimal, + annual_cash_flow: Decimal, + appreciation_rate: Decimal = Decimal("3.0"), + holding_period_years: int = 5, +) -> Dict[str, Any]: + """Calculate buy-and-hold rental strategy returns. + + Args: + purchase_price: Property purchase price + down_payment: Down payment amount + loan_amount: Mortgage loan amount + interest_rate: Annual interest rate as percentage + loan_term_years: Loan term in years + closing_costs: Closing costs + annual_cash_flow: Annual cash flow (can be negative) + appreciation_rate: Annual appreciation rate as percentage + holding_period_years: How many years to hold + + Returns: + Dictionary with rental strategy analysis + """ + from investor_app.finance.mortgage import ( + calculate_appreciation, + calculate_principal_paydown, + ) + + total_investment = to_decimal(down_payment) + to_decimal(closing_costs) + + # Simplified: assume constant cash flow (in reality it would improve over time) + total_cash_flow = to_decimal(annual_cash_flow) * Decimal(holding_period_years) + + # Equity buildup from mortgage paydown + equity_buildup = calculate_principal_paydown( + loan_amount, interest_rate, loan_term_years, holding_period_years + ) + + # Appreciation + appreciation = calculate_appreciation( + purchase_price, appreciation_rate, holding_period_years + ) + + # Total gain + total_gain = total_cash_flow + equity_buildup + appreciation + + # ROI + if total_investment > 0: + roi_percent = total_gain / total_investment * Decimal(100) + annualized_return = ( + (Decimal(1) + roi_percent / Decimal(100)) + ** (Decimal(1) / Decimal(holding_period_years)) + - Decimal(1) + ) * Decimal(100) + else: + roi_percent = Decimal("0") + annualized_return = Decimal("0") + + return { + "totalInvestment": total_investment.quantize(Decimal("0.01")), + "year1CashFlow": to_decimal(annual_cash_flow).quantize(Decimal("0.01")), + f"year{holding_period_years}CashFlow": to_decimal(annual_cash_flow).quantize( + Decimal("0.01") + ), # Simplified + f"totalCashFlow{holding_period_years}Years": total_cash_flow.quantize( + Decimal("0.01") + ), + f"equityBuildup{holding_period_years}Years": equity_buildup.quantize( + Decimal("0.01") + ), + f"appreciation{holding_period_years}Years": appreciation.quantize( + Decimal("0.01") + ), + f"totalGain{holding_period_years}Years": total_gain.quantize(Decimal("0.01")), + "roi": roi_percent.quantize(Decimal("0.1")), + "timeframe": f"{holding_period_years} years", + "annualizedReturn": annualized_return.quantize(Decimal("0.1")), + } + + +def calculate_vacation_rental_strategy( + purchase_price: Decimal, + down_payment: Decimal, + loan_amount: Decimal, + interest_rate: Decimal, + loan_term_years: int, + closing_costs: Decimal, + avg_nightly_rate: Decimal, + avg_occupancy_rate: Decimal, # As percentage (e.g., 65 for 65%) + cleaning_fee_per_stay: Decimal, + monthly_operating_expenses: Decimal, + holding_period_years: int = 5, + avg_stay_length_nights: int = 3, # Typical vacation rental stay length +) -> Dict[str, Any]: + """Calculate vacation rental strategy returns. + + Args: + purchase_price: Property purchase price + down_payment: Down payment amount + loan_amount: Mortgage loan amount + interest_rate: Annual interest rate as percentage + loan_term_years: Loan term in years + closing_costs: Closing costs + avg_nightly_rate: Average nightly rental rate + avg_occupancy_rate: Average occupancy rate as percentage + cleaning_fee_per_stay: Cleaning fee per stay + monthly_operating_expenses: Monthly operating expenses + holding_period_years: How many years to hold + avg_stay_length_nights: Average length of stay in nights (default 3) + + Returns: + Dictionary with vacation rental strategy analysis + """ + from investor_app.finance.mortgage import ( + calculate_appreciation, + calculate_monthly_mortgage, + calculate_principal_paydown, + ) + + total_investment = to_decimal(down_payment) + to_decimal(closing_costs) + + # Calculate annual income + nights_per_year = Decimal(365) + occupied_nights = nights_per_year * to_decimal(avg_occupancy_rate) / Decimal(100) + + # Calculate number of stays based on average stay length + avg_stay_length = Decimal(avg_stay_length_nights) + num_stays = occupied_nights / avg_stay_length + + annual_rental_income = occupied_nights * to_decimal( + avg_nightly_rate + ) + num_stays * to_decimal(cleaning_fee_per_stay) + + # Annual expenses + monthly_mortgage = calculate_monthly_mortgage( + loan_amount, interest_rate, loan_term_years + ) + annual_debt_service = monthly_mortgage * Decimal(12) + annual_operating_expenses = to_decimal(monthly_operating_expenses) * Decimal(12) + + # Cash flow + annual_cash_flow = ( + annual_rental_income - annual_debt_service - annual_operating_expenses + ) + + # Calculate year 1 CoC + if total_investment > 0: + coc_return = annual_cash_flow / total_investment * Decimal(100) + else: + coc_return = Decimal("0") + + # 5-year projection (simplified) + total_cash_flow = annual_cash_flow * Decimal(holding_period_years) + + # Equity buildup + equity_buildup = calculate_principal_paydown( + loan_amount, interest_rate, loan_term_years, holding_period_years + ) + + # Appreciation (3% default) + appreciation = calculate_appreciation( + purchase_price, Decimal("3.0"), holding_period_years + ) + + total_gain = total_cash_flow + equity_buildup + appreciation + + if total_investment > 0: + roi_percent = total_gain / total_investment * Decimal(100) + annualized_return = ( + (Decimal(1) + roi_percent / Decimal(100)) + ** (Decimal(1) / Decimal(holding_period_years)) + - Decimal(1) + ) * Decimal(100) + else: + roi_percent = Decimal("0") + annualized_return = Decimal("0") + + return { + "totalInvestment": total_investment.quantize(Decimal("0.01")), + "avgMonthlyIncome": (annual_rental_income / Decimal(12)).quantize( + Decimal("0.01") + ), + "avgMonthlyExpenses": ( + (annual_debt_service + annual_operating_expenses) / Decimal(12) + ).quantize(Decimal("0.01")), + "netCashFlowYear1": annual_cash_flow.quantize(Decimal("0.01")), + "cocReturn": coc_return.quantize(Decimal("0.1")), + "roi": roi_percent.quantize(Decimal("0.1")), + "timeframe": f"{holding_period_years} years", + "annualizedReturn": annualized_return.quantize(Decimal("0.1")), + "seasonalityImpact": ( + "High - Occupancy varies by season" + if avg_occupancy_rate < 75 + else "Moderate" + ), + } + + +# ── BRRRR ────────────────────────────────────────────────────────────────────── + + +def estimate_arv( + comparable_sales: list[tuple[Decimal, Decimal]], + subject_sqft: Decimal, +) -> Decimal: + """Estimate After-Repair Value (ARV) from comparable sales. + + Args: + comparable_sales: List of ``(price, sqft)`` tuples, one per comparable + sale. Both ``price`` and ``sqft`` must be positive. + subject_sqft: Square footage of the subject property (must be > 0). + + Returns: + Estimated ARV as a Decimal. + + Raises: + ValueError: If ``comparable_sales`` is empty. + ValueError: If any comparable has ``price <= 0`` or ``sqft <= 0``. + ValueError: If ``subject_sqft <= 0``. + """ + if not comparable_sales: + raise ValueError("comparable_sales must not be empty") + + subject = to_decimal(subject_sqft) + if subject <= Decimal("0"): + raise ValueError( + f"subject_sqft must be greater than zero (received {subject_sqft})" + ) + + ppsf_values: list[Decimal] = [] + for idx, (price, sqft) in enumerate(comparable_sales): + p = to_decimal(price) + s = to_decimal(sqft) + if p <= Decimal("0"): + raise ValueError( + f"comparable_sales[{idx}]: price must be greater than zero (received {price})" + ) + if s <= Decimal("0"): + raise ValueError( + f"comparable_sales[{idx}]: sqft must be greater than zero (received {sqft})" + ) + ppsf_values.append(p / s) + + median_ppsf = to_decimal(median(ppsf_values)) + return median_ppsf * subject + + +def estimate_rehab_cost( + sqft: Decimal, + renovation_level: str, + cost_per_sqft: dict[str, Decimal], +) -> Decimal: + """Estimate total rehab cost for a property. + + Args: + sqft: Square footage of the property (must be > 0). + renovation_level: Scope of renovation. Must be one of the keys present + in ``cost_per_sqft`` (typically ``"cosmetic"``, ``"moderate"``, or + ``"full_gut"``). + cost_per_sqft: Mapping from renovation level to cost per square foot. + Supply ``settings.REHAB_COST_PER_SQFT`` from the service layer to + keep this function Django-free. + + Returns: + Estimated rehab cost as a Decimal. + + Raises: + ValueError: If ``renovation_level`` is not a key in ``cost_per_sqft``. + ValueError: If ``sqft <= 0``. + """ + valid_levels = set(cost_per_sqft.keys()) + if renovation_level not in valid_levels: + raise ValueError( + f"renovation_level must be one of {sorted(valid_levels)} " + f"(received {renovation_level!r})" + ) + s = to_decimal(sqft) + if s <= Decimal("0"): + raise ValueError(f"sqft must be greater than zero (received {sqft})") + + rate = to_decimal(cost_per_sqft[renovation_level]) + return rate * s + + +def max_refinance_loan( + arv: Decimal, + ltv_ratio: Decimal = Decimal("0.75"), +) -> Decimal: + """Calculate the maximum cash-out refinance loan amount at a given LTV. + + Args: + arv: After-Repair Value of the property (must be > 0). + ltv_ratio: Loan-to-value ratio expressed as a decimal strictly between + 0 and 1 (e.g., ``Decimal("0.75")`` for 75 %). + + Returns: + Maximum refinance loan amount as a Decimal. + + Raises: + ValueError: If ``arv <= 0``. + ValueError: If ``ltv_ratio`` is not strictly in ``(0, 1)``. + """ + a = to_decimal(arv) + ltv = to_decimal(ltv_ratio) + + if a <= Decimal("0"): + raise ValueError(f"arv must be greater than zero (received {arv})") + if ltv <= Decimal("0") or ltv >= Decimal("1"): + raise ValueError( + f"ltv_ratio must be strictly between 0 and 1 (received {ltv_ratio})" + ) + + return a * ltv + + +def cash_left_in_deal( + purchase_price: Decimal, + rehab_cost: Decimal, + cash_out_refi_amount: Decimal, + closing_costs: Decimal = Decimal("0"), +) -> Decimal: + """Calculate the investor's remaining cash deployed after a cash-out refinance. + + Formula:: + + cash_left = purchase_price + rehab_cost + closing_costs - cash_out_refi_amount + + A negative or zero result means the investor has recouped all invested capital + (the "infinite CoC" scenario in BRRRR terminology). + + Args: + purchase_price: Purchase price of the property. + rehab_cost: Total rehabilitation cost. + cash_out_refi_amount: Proceeds from the cash-out refinance. + closing_costs: Total closing costs (purchase + refi combined). Defaults + to ``Decimal("0")``. + + Returns: + Cash left in the deal as a Decimal. Negative or zero => infinite CoC. + """ + return ( + to_decimal(purchase_price) + + to_decimal(rehab_cost) + + to_decimal(closing_costs) + - to_decimal(cash_out_refi_amount) + ) + + +def brrrr_coc_return( + annual_net_cash_flow: Decimal, + cash_left_in_deal: Decimal, +) -> Decimal: + """Calculate Cash-on-Cash return for a BRRRR deal. + + Handles the "infinite CoC" scenario where the investor has recouped all + (or more than all) of their capital. + + Rules: + * ``cash_left_in_deal <= 0`` -> returns ``Decimal("Infinity")`` regardless + of cash flow (investor has no capital remaining in the deal). + * ``cash_left_in_deal > 0`` and ``annual_net_cash_flow == 0`` -> returns + ``Decimal("0")`` (no return on remaining capital). + * Otherwise -> returns ``annual_net_cash_flow / cash_left_in_deal``. + + Args: + annual_net_cash_flow: Annual after-debt-service cash flow (can be + negative for a losing deal). + cash_left_in_deal: Capital still deployed after the cash-out refi + (from ``cash_left_in_deal()``). + + Returns: + CoC return as a Decimal. ``Decimal("Infinity")`` signals infinite CoC. + """ + left = to_decimal(cash_left_in_deal) + flow = to_decimal(annual_net_cash_flow) + + if left <= Decimal("0"): + return Decimal("Infinity") + if flow == Decimal("0"): + return Decimal("0") + return flow / left diff --git a/investor_app/finance/taxes.py b/investor_app/finance/taxes.py new file mode 100644 index 00000000..4c59c973 --- /dev/null +++ b/investor_app/finance/taxes.py @@ -0,0 +1,692 @@ +"""Depreciation, tax benefits, hold-period projections, and exit/sale analysis.""" + +from __future__ import annotations + +from decimal import Decimal +import logging +from typing import Dict, Sequence + +import numpy as np +import numpy_financial as npf + +from investor_app.finance.utils import to_decimal + +logger = logging.getLogger(__name__) + + +def calculate_tax_benefits( + loan_amount: Decimal, + interest_rate: Decimal, + loan_term_years: int, + property_value: Decimal, + tax_bracket: Decimal = Decimal("24"), + year_num: int = 1, +) -> Decimal: + """Calculate tax benefits from mortgage interest deduction and depreciation. + + Args: + loan_amount: Mortgage loan amount + interest_rate: Annual interest rate as percentage + loan_term_years: Loan term in years + property_value: Property value (for depreciation calculation) + tax_bracket: Marginal tax bracket as percentage (default 24%) + year_num: Which year to calculate benefits for (default 1) + + Returns: + Total tax benefit amount for the specified year + """ + from investor_app.finance.mortgage import calculate_monthly_mortgage + + if loan_amount == 0: + # All cash - only depreciation benefit + # Residential property: 27.5 year straight-line depreciation on 80% of value + building_value = to_decimal(property_value) * Decimal("0.80") + annual_depreciation = building_value / Decimal("27.5") + tax_savings = annual_depreciation * (to_decimal(tax_bracket) / Decimal(100)) + return tax_savings.quantize(Decimal("0.01")) + + # Calculate interest paid in specific year + loan_amt = to_decimal(loan_amount) + rate = to_decimal(interest_rate) + monthly_rate = rate / Decimal(100) / Decimal(12) + monthly_payment = calculate_monthly_mortgage( + loan_amount, interest_rate, loan_term_years + ) + + # Calculate remaining balance at start of year + # Uses standard amortization formula: B = P * [(1+r)^(n-k) - 1] / [(1+r)^n - 1] + # where B=balance, P=principal, r=rate, n=total payments, k=payments made + payments_before = (year_num - 1) * 12 + if payments_before > 0: + num_payments = loan_term_years * 12 + remaining_factor = (Decimal(1) + monthly_rate) ** Decimal( + num_payments - payments_before + ) + payment_factor = (Decimal(1) + monthly_rate) ** Decimal(num_payments) + balance_start = loan_amt * ( + (remaining_factor - Decimal(1)) / (payment_factor - Decimal(1)) + ) + else: + balance_start = loan_amt + + # Calculate interest for each month of the year + total_interest = Decimal("0") + balance = balance_start + for _ in range(12): + interest_payment = balance * monthly_rate + principal_payment = monthly_payment - interest_payment + total_interest += interest_payment + balance -= principal_payment + if balance <= 0: + break + + # Add depreciation + building_value = to_decimal(property_value) * Decimal("0.80") + annual_depreciation = building_value / Decimal("27.5") + + # Total deductions + total_deductions = total_interest + annual_depreciation + + # Tax savings + tax_savings = total_deductions * (to_decimal(tax_bracket) / Decimal(100)) + + return tax_savings.quantize(Decimal("0.01")) + + +# ── Depreciation & Tax Modeling ──────────────────────────────────────────────── + + +def annual_depreciation(purchase_price: Decimal, land_value: Decimal) -> Decimal: + """Calculate the annual straight-line depreciation for a residential rental property. + + The IRS allows 27.5-year straight-line depreciation on the building portion + (purchase price minus land value) of residential rental property. + + Args: + purchase_price: Total purchase price of the property (must be > 0). + land_value: Estimated value of the land component (must be >= 0 and + < purchase_price). Land is not depreciable. + + Returns: + Annual depreciation deduction as a Decimal representing the fixed deduction + for a full year. Year-by-year schedule handling is the caller's responsibility: + apply this amount for years 1-27 (full deduction), half this amount for year 28 + (remaining half-year fraction), and no deduction for years beyond year 28. + + Raises: + ValueError: If purchase_price <= 0. + ValueError: If land_value < 0. + ValueError: If land_value >= purchase_price (no depreciable basis). + + Example: + >>> annual_depreciation(Decimal("300000"), Decimal("50000")) + Decimal("9090.909090909090909090909091") + """ + pp = to_decimal(purchase_price) + lv = to_decimal(land_value) + + if pp <= Decimal("0"): + raise ValueError("purchase_price must be greater than zero") + if lv < Decimal("0"): + raise ValueError("land_value must be zero or greater") + if lv >= pp: + raise ValueError( + "land_value must be less than purchase_price; land is not depreciable" + ) + + depreciable_basis = pp - lv + return depreciable_basis / Decimal("27.5") + + +def after_tax_cash_flow( + noi: Decimal, + annual_debt_service: Decimal, + depreciation_deduction: Decimal, + marginal_tax_rate: Decimal, +) -> Decimal: + """Calculate after-tax cash flow including the depreciation tax shield. + + Formula: (NOI - debt_service) + (depreciation x tax_rate) + + The depreciation tax shield represents the tax savings from the paper loss of + depreciation, which reduces taxable income without a cash outflow. + + Args: + noi: Net Operating Income (annual). + annual_debt_service: Total annual mortgage payments (principal + interest). + depreciation_deduction: Annual depreciation deduction (e.g., from + ``annual_depreciation()``). + marginal_tax_rate: Investor's marginal income tax rate as a decimal in [0, 1] + (e.g., 0.24 for 24%). + + Returns: + After-tax cash flow as a Decimal. A positive value indicates net cash benefit. + + Raises: + ValueError: If marginal_tax_rate is outside the range [0, 1]. + + Example: + >>> after_tax_cash_flow( + ... Decimal("24000"), Decimal("18000"), Decimal("9091"), Decimal("0.24") + ... ) + Decimal("8181.84") + """ + rate = to_decimal(marginal_tax_rate) + if rate < Decimal("0") or rate > Decimal("1"): + raise ValueError( + "marginal_tax_rate must be between 0 and 1 inclusive " + f"(received {marginal_tax_rate})" + ) + + pre_tax_cf = to_decimal(noi) - to_decimal(annual_debt_service) + tax_shield = to_decimal(depreciation_deduction) * rate + return pre_tax_cf + tax_shield + + +def after_tax_irr( + cash_flows: Sequence[Decimal], + depreciation_schedule: Sequence[Decimal], + marginal_tax_rate: Decimal, +) -> Decimal: + """Calculate after-tax IRR by adjusting each period's cash flow by the depreciation tax shield. + + Each period's cash flow is increased by ``depreciation * marginal_tax_rate``. + The first cash flow (index 0) is assumed to be the initial investment (negative) + and is not adjusted -- depreciation tax shields begin in period 1. + + Args: + cash_flows: List of periodic cash flows. Index 0 is typically the initial + investment (negative). Must have at least 2 elements. + depreciation_schedule: List of annual depreciation amounts aligned to + cash_flows[1:]. If shorter than cash_flows[1:], missing periods are + treated as zero depreciation. + marginal_tax_rate: Investor's marginal income tax rate as a decimal in [0, 1]. + + Returns: + After-tax IRR as a Decimal. Returns Decimal("0") if numpy-financial cannot + converge (e.g., all non-negative flows or no sign change). + + Raises: + ValueError: If fewer than 2 cash flows are supplied. + ValueError: If marginal_tax_rate is outside the range [0, 1]. + + Example: + >>> after_tax_irr( + ... [Decimal("-100000"), Decimal("6000"), Decimal("106000")], + ... [Decimal("9091"), Decimal("9091")], + ... Decimal("0.24"), + ... ) + Decimal("0.0718") + """ + if len(cash_flows) < 2: + raise ValueError("At least 2 cash flows are required to calculate IRR") + + rate = to_decimal(marginal_tax_rate) + if rate < Decimal("0") or rate > Decimal("1"): + raise ValueError( + "marginal_tax_rate must be between 0 and 1 inclusive " + f"(received {marginal_tax_rate})" + ) + + # Build adjusted cash flows: index 0 (initial investment) is not adjusted. + adjusted: list[float] = [float(cash_flows[0])] + for i, cf in enumerate(cash_flows[1:]): + dep = ( + depreciation_schedule[i] if i < len(depreciation_schedule) else Decimal("0") + ) + shield = to_decimal(dep) * rate + adjusted.append(float(to_decimal(cf) + shield)) + + cf_array = np.array(adjusted, dtype=float) + try: + value = float(npf.irr(cf_array)) + if np.isnan(value) or np.isinf(value): + logger.warning( + "after_tax_irr: numpy_financial.irr returned non-finite value; returning 0" + ) + return Decimal("0") + return to_decimal(value) + except Exception as exc: + logger.warning("after_tax_irr: numpy_financial.irr raised %s; returning 0", exc) + return Decimal("0") + + +# ── Hold Period & Exit Analysis ──────────────────────────────────────────────── + + +def project_annual_cash_flows( + gross_rent_year1: Decimal, + operating_expense_year1: Decimal, + annual_debt_service: Decimal, + rent_growth_rate: Decimal, + expense_growth_rate: Decimal, + hold_years: int, +) -> list[Decimal]: + """Project year-by-year after-debt-service cash flows over a hold period. + + Each year's gross rent and operating expenses grow independently at their + respective compound annual growth rates. Annual debt service is assumed + constant (fixed-rate mortgage). + + Args: + gross_rent_year1: Gross rental income in year 1 (must be >= 0). + operating_expense_year1: Operating expenses in year 1 (must be >= 0). + annual_debt_service: Fixed annual mortgage payment (principal + interest; + must be >= 0). + rent_growth_rate: Annual rent growth rate as a decimal (e.g., 0.03 for 3%). + Must be in the range [-0.5, 0.5]. + expense_growth_rate: Annual expense growth rate as a decimal. + Must be in the range [-0.5, 0.5]. + hold_years: Number of years in the hold period. Must be in [1, 50]. + + Returns: + List of annual cash-flow Decimals, one entry per year (length == hold_years). + + Raises: + ValueError: If gross_rent_year1 or operating_expense_year1 or + annual_debt_service is negative. + ValueError: If hold_years is outside [1, 50]. + ValueError: If rent_growth_rate or expense_growth_rate is outside + [-0.5, 0.5]. + + Example: + >>> flows = project_annual_cash_flows( + ... Decimal("36000"), Decimal("12000"), Decimal("18000"), + ... Decimal("0.03"), Decimal("0.02"), 5, + ... ) + >>> len(flows) + 5 + """ + if hold_years < 1 or hold_years > 50: + raise ValueError(f"hold_years must be between 1 and 50 (received {hold_years})") + + r_rate = to_decimal(rent_growth_rate) + e_rate = to_decimal(expense_growth_rate) + rate_limit = Decimal("0.5") + if r_rate < -rate_limit or r_rate > rate_limit: + raise ValueError( + f"rent_growth_rate must be in [-0.5, 0.5] (received {rent_growth_rate})" + ) + if e_rate < -rate_limit or e_rate > rate_limit: + raise ValueError( + f"expense_growth_rate must be in [-0.5, 0.5] (received {expense_growth_rate})" + ) + + rent = to_decimal(gross_rent_year1) + expense = to_decimal(operating_expense_year1) + debt = to_decimal(annual_debt_service) + + if rent < Decimal("0"): + raise ValueError( + f"gross_rent_year1 must be zero or greater (received {gross_rent_year1})" + ) + if expense < Decimal("0"): + raise ValueError( + f"operating_expense_year1 must be zero or greater (received {operating_expense_year1})" + ) + if debt < Decimal("0"): + raise ValueError( + f"annual_debt_service must be zero or greater (received {annual_debt_service})" + ) + + cash_flows: list[Decimal] = [] + one = Decimal("1") + for year in range(1, hold_years + 1): + exponent = year - 1 + gross = rent * (one + r_rate) ** exponent + opex = expense * (one + e_rate) ** exponent + annual_noi = gross - opex + cash_flows.append(annual_noi - debt) + + return cash_flows + + +def project_property_value( + purchase_price: Decimal, + appreciation_rate: Decimal, + hold_years: int, +) -> Decimal: + """Project the market value of a property at the end of a hold period. + + Uses compound annual growth: + value = purchase_price * (1 + appreciation_rate)^hold_years + + Supports conservative / base / optimistic scenarios by varying + ``appreciation_rate`` (e.g., 0%, 3%, 5% for US residential). + + Args: + purchase_price: Original purchase price of the property (must be > 0). + appreciation_rate: Expected annual appreciation rate as a decimal. + Must be >= -1 (a rate of -1 implies a total loss of value; rates + below -1 are mathematically undefined for this formula). + hold_years: Number of years to project forward (must be in [1, 50]). + + Returns: + Projected property value as a Decimal. + + Raises: + ValueError: If purchase_price <= 0. + ValueError: If appreciation_rate < -1. + ValueError: If hold_years is outside [1, 50]. + + Example: + >>> project_property_value(Decimal("300000"), Decimal("0.03"), 10) + Decimal("403175....") + """ + pp = to_decimal(purchase_price) + rate = to_decimal(appreciation_rate) + + if pp <= Decimal("0"): + raise ValueError( + f"purchase_price must be greater than zero (received {purchase_price})" + ) + if rate < Decimal("-1"): + raise ValueError( + f"appreciation_rate must be >= -1 (received {appreciation_rate})" + ) + if hold_years < 1 or hold_years > 50: + raise ValueError(f"hold_years must be between 1 and 50 (received {hold_years})") + + return pp * (Decimal("1") + rate) ** hold_years + + +def net_sale_proceeds( + sale_price: Decimal, + original_purchase_price: Decimal, + outstanding_loan_balance: Decimal, + accumulated_depreciation: Decimal, + agent_commission_rate: Decimal = Decimal("0.06"), + closing_cost_rate: Decimal = Decimal("0.01"), + long_term_cg_rate: Decimal = Decimal("0.15"), + depreciation_recapture_rate: Decimal = Decimal("0.25"), +) -> Decimal: + """Calculate net cash to investor after costs and taxes upon property sale. + + Deductions applied in order: + 1. Agent commissions: sale_price * agent_commission_rate + 2. Closing costs: sale_price * closing_cost_rate + 3. Loan payoff: outstanding_loan_balance + 4. Capital gains tax: max(sale_price - original_purchase_price, 0) * long_term_cg_rate + 5. Depreciation recapture: accumulated_depreciation * depreciation_recapture_rate + + Args: + sale_price: Gross sale price of the property. + original_purchase_price: Price paid for the property at acquisition. + outstanding_loan_balance: Remaining mortgage balance at time of sale + (must be >= 0). + accumulated_depreciation: Total depreciation taken over the holding period + (must be >= 0). + agent_commission_rate: Broker commission as a decimal (default 0.06 = 6%). + closing_cost_rate: Seller's closing costs as a decimal (default 0.01 = 1%). + long_term_cg_rate: Federal long-term capital gains tax rate as a decimal + (default 0.15 = 15%). + depreciation_recapture_rate: IRS Section 1250 recapture rate as a decimal + (default 0.25 = 25%). + + Returns: + Net cash proceeds to investor as a Decimal. + + Raises: + ValueError: If outstanding_loan_balance < 0. + ValueError: If accumulated_depreciation < 0. + ValueError: If any rate parameter is outside [0, 1]. + + Example: + >>> net_sale_proceeds( + ... Decimal("400000"), Decimal("300000"), Decimal("200000"), + ... Decimal("45000"), + ... ) + Decimal("...") + """ + sp = to_decimal(sale_price) + opp = to_decimal(original_purchase_price) + loan_bal = to_decimal(outstanding_loan_balance) + acc_dep = to_decimal(accumulated_depreciation) + commission_rate = to_decimal(agent_commission_rate) + cc_rate = to_decimal(closing_cost_rate) + cg_rate = to_decimal(long_term_cg_rate) + recapture_rate = to_decimal(depreciation_recapture_rate) + + if loan_bal < Decimal("0"): + raise ValueError( + f"outstanding_loan_balance must be zero or greater (received {outstanding_loan_balance})" + ) + if acc_dep < Decimal("0"): + raise ValueError( + f"accumulated_depreciation must be zero or greater (received {accumulated_depreciation})" + ) + for name, val in [ + ("agent_commission_rate", commission_rate), + ("closing_cost_rate", cc_rate), + ("long_term_cg_rate", cg_rate), + ("depreciation_recapture_rate", recapture_rate), + ]: + if val < Decimal("0") or val > Decimal("1"): + raise ValueError( + f"{name} must be between 0 and 1 inclusive (received {val})" + ) + + gross_proceeds = sp - sp * commission_rate - sp * cc_rate - loan_bal + + capital_gain = sp - opp + cg_tax = max(capital_gain, Decimal("0")) * cg_rate + + recapture_tax = acc_dep * recapture_rate + + return gross_proceeds - cg_tax - recapture_tax + + +def total_return_summary( + purchase_price: Decimal, + down_payment: Decimal, + annual_cash_flows: list[Decimal], + net_sale_proceeds_amount: Decimal, +) -> Dict[str, Decimal]: + """Summarise total investment return over the hold period. + + Combines cumulative cash flows and net sale proceeds to compute total return + metrics. + + Args: + purchase_price: Original acquisition price of the property. + down_payment: Equity invested at purchase (positive value; used as the + year-0 outflow). + annual_cash_flows: List of annual after-debt-service cash flows from + ``project_annual_cash_flows()``. Must have at least 1 element. + net_sale_proceeds_amount: Net cash to investor upon sale from + ``net_sale_proceeds()``. + + Returns: + Dictionary with keys: purchase_price, total_cash_flow, net_sale_proceeds, + total_return, total_return_on_equity, annualized_irr. + + Raises: + ValueError: If annual_cash_flows is empty. + ValueError: If down_payment < 0. + + Example: + >>> summary = total_return_summary( + ... Decimal("300000"), Decimal("60000"), + ... [Decimal("6000")] * 10, Decimal("120000"), + ... ) + >>> summary["total_cash_flow"] + Decimal("60000") + """ + from investor_app.finance.utils import irr + + if not annual_cash_flows: + raise ValueError("annual_cash_flows must contain at least one element") + + dp = to_decimal(down_payment) + if dp < Decimal("0"): + raise ValueError( + f"down_payment must be zero or greater (received {down_payment})" + ) + + total_cf = sum(annual_cash_flows, Decimal("0")) + nsp = to_decimal(net_sale_proceeds_amount) + total_ret = total_cf + nsp + + if dp == Decimal("0"): + roe = Decimal("0") + else: + roe = total_ret / dp + + # Build IRR cash-flow series: year-0 outflow, annual CFs, exit-year bump + irr_flows: list[Decimal] = [-dp] + for i, cf in enumerate(annual_cash_flows): + if i == len(annual_cash_flows) - 1: + irr_flows.append(cf + nsp) + else: + irr_flows.append(cf) + + annualized = irr(irr_flows) + + return { + "purchase_price": to_decimal(purchase_price), + "total_cash_flow": total_cf, + "net_sale_proceeds": nsp, + "total_return": total_ret, + "total_return_on_equity": roe, + "annualized_irr": annualized, + } + + +def depreciation_recapture_tax( + accumulated_depreciation: Decimal, + recapture_rate: Decimal = Decimal("0.25"), +) -> Decimal: + """Calculate the depreciation recapture tax owed upon sale of the property. + + Under IRS Section 1250, accumulated depreciation is recaptured at a maximum + rate of 25% when the property is sold. + + Args: + accumulated_depreciation: Total depreciation taken over the holding period + (sum of annual deductions). Must be >= 0. + recapture_rate: IRS Section 1250 recapture rate as a decimal in [0, 1]. + Defaults to 0.25 (25%). + + Returns: + Depreciation recapture tax owed as a Decimal. + + Raises: + ValueError: If accumulated_depreciation < 0. + ValueError: If recapture_rate is outside [0, 1]. + + Example: + >>> depreciation_recapture_tax(Decimal("45000")) + Decimal("11250.00") + """ + acc_dep = to_decimal(accumulated_depreciation) + rate = to_decimal(recapture_rate) + + if acc_dep < Decimal("0"): + raise ValueError("accumulated_depreciation must be zero or greater") + if rate < Decimal("0") or rate > Decimal("1"): + raise ValueError( + "recapture_rate must be between 0 and 1 inclusive " + f"(received {recapture_rate})" + ) + + return acc_dep * rate + + +def calculate_annual_depreciation( + purchase_price: Decimal, + land_value_pct: Decimal = Decimal("0.20"), +) -> Decimal: + """Calculate annual straight-line depreciation for residential real estate. + + Uses the 27.5-year straight-line schedule on the improvement value + (purchase price minus land). + + Args: + purchase_price: Total purchase price of the property (must be > 0). + land_value_pct: Fraction of purchase price attributable to land, + expressed as a decimal (e.g. 0.20 for 20%). + Must be in the range [0, 1). Default 0.20. + + Returns: + Annual depreciation deduction as a Decimal. + + Raises: + ValueError: If purchase_price <= 0. + ValueError: If land_value_pct is outside [0, 1). + + Example: + >>> calculate_annual_depreciation(Decimal("200000"), Decimal("0.20")) + Decimal("5818.181818181818181818181818") + """ + pp = to_decimal(purchase_price) + lvp = to_decimal(land_value_pct) + + if pp <= Decimal("0"): + raise ValueError( + f"purchase_price must be greater than zero (received {purchase_price})" + ) + if lvp < Decimal("0") or lvp >= Decimal("1"): + raise ValueError( + f"land_value_pct must be in [0, 1) (received {land_value_pct})" + ) + + improvement_value = pp * (Decimal("1") - lvp) + return improvement_value / Decimal("27.5") + + +def calculate_after_tax_cashflow( + pre_tax_annual_cashflow: Decimal, + annual_depreciation: Decimal, + marginal_tax_rate: Decimal, +) -> Decimal: + """Calculate after-tax cash flow including the depreciation tax shield. + + As simplified model — does not account for passive activity loss (PAL) rules, + cost segregation, or other advanced tax strategies. + A UI disclaimer should note this limitation. + + Formula: + taxable_income = pre_tax_annual_cashflow - annual_depreciation + if taxable_income < 0: + tax_savings = abs(taxable_income) * marginal_tax_rate + after_tax = pre_tax_annual_cashflow + tax_savings + else: + tax_owed = taxable_income * marginal_tax_rate + after_tax = pre_tax_annual_cashflow - tax_owed + + Args: + pre_tax_annual_cashflow: Annual pre-tax cash flow from the property. + Can be negative. + annual_depreciation: Annual depreciation deduction (built-in from + ``calculate_annual_depreciation``). + marginal_tax_rate: Investor's marginal income-tax rate as a decimal + in [0, 1] (e.g., 0.32 for 32%). + + Returns: + After-tax annual cash flow as a Decimal. + + Raises: + ValueError: If marginal_tax_rate is outside [0, 1]. + + Example: + >>> calculate_after_tax_cashflow( + ... Decimal("6000"), Decimal("5818"), Decimal("0.32") + ... ) + Decimal("5941.76") + """ + cashflow = to_decimal(pre_tax_annual_cashflow) + depreciation = to_decimal(annual_depreciation) + rate = to_decimal(marginal_tax_rate) + + if rate < Decimal("0") or rate > Decimal("1"): + raise ValueError( + f"marginal_tax_rate must be in [0, 1] (received {marginal_tax_rate})" + ) + + taxable_income = cashflow - depreciation + + if taxable_income < 0: + tax_savings = abs(taxable_income) * rate + return cashflow + tax_savings + tax_owed = taxable_income * rate + return cashflow - tax_owed diff --git a/investor_app/finance/utils.py b/investor_app/finance/utils.py index e12acfe5..93b0374b 100644 --- a/investor_app/finance/utils.py +++ b/investor_app/finance/utils.py @@ -1,15 +1,23 @@ -from datetime import datetime +"""Core finance math for investment KPIs. + +This module intentionally holds only the low-level primitives and the +Django-coupled analysis function. Specialized math lives in sibling modules: + +- ``mortgage`` — monthly mortgage, carrying costs, break-even rent, paydown, appreciation, ROI components +- ``taxes`` — depreciation, after-tax cash flow / IRR, hold-period projections, sale proceeds +- ``scoring`` — 1% rule, GRM, price-to-rent and market normalization helpers +- ``strategies`` — flip, buy-and-hold, vacation rental, BRRRR +""" + +from __future__ import annotations + from decimal import Decimal import logging -from statistics import median -from typing import Any, Dict, Sequence, TypedDict import numpy as np import numpy_financial as npf -# removed unused 'settings' import - -from core.models import InvestmentAnalysis, Listing, Property +from core.models import InvestmentAnalysis, Property logger = logging.getLogger(__name__) @@ -23,12 +31,6 @@ def noi(monthly_income: Decimal, monthly_expenses: Decimal) -> Decimal: NOI = (Monthly Income - Monthly Expenses) x 12 - Derivation: NOI is the income-statement identity for property-level - operating income - gross income less operating expenses, before any - financing costs (debt service) or capital expenditures. Annualizing - the monthly figure gives the standard basis for cap rate, DSCR, and - other per-year KPIs. - Args: monthly_income: Gross monthly income (rent + other income). monthly_expenses: Monthly operating expenses (excludes debt service). @@ -46,12 +48,6 @@ def cap_rate(annual_noi: Decimal, purchase_price: Decimal) -> Decimal: Cap Rate = Annual NOI / Purchase Price - Derivation: Cap rate is the property's unlevered yield - the return - an all-cash buyer would earn on the purchase price from operations - alone, independent of financing. It is the direct-capitalization - analogue of a bond's coupon yield: NOI divided by price, both taken - as of the same period. - Args: annual_noi: Annual Net Operating Income. purchase_price: Total purchase price of the property. @@ -70,12 +66,6 @@ def cash_on_cash(annual_cash_flow: Decimal, total_cash_invested: Decimal) -> Dec CoC = Annual Cash Flow / Total Cash Invested - Derivation: Unlike cap rate, CoC is a levered return - it measures - the actual cash yield on the investor's own capital (down payment, - closing costs, rehab) after debt service, since annual_cash_flow is - NOI net of mortgage payments. It answers what the investor earns on - the cash actually put in, not what the property earns overall. - Args: annual_cash_flow: Annual cash flow after debt service. total_cash_invested: Total cash invested by the investor (down @@ -95,12 +85,6 @@ def dscr(annual_noi: Decimal, annual_debt_service: Decimal) -> Decimal: DSCR = Annual NOI / Annual Debt Service - Derivation: DSCR is a lender solvency ratio, not an investor return - metric - it measures how many times over the property's NOI covers - its annual mortgage payments (principal + interest). A DSCR below - 1.0 means NOI alone cannot cover debt service; most lenders require - DSCR >= 1.20-1.25 as an underwriting minimum. - Args: annual_noi: Annual Net Operating Income. annual_debt_service: Total annual mortgage payments (P&I). @@ -117,16 +101,7 @@ def dscr(annual_noi: Decimal, annual_debt_service: Decimal) -> Decimal: def irr(cashflows: list[Decimal]) -> Decimal: """Calculate the Internal Rate of Return (IRR) for a cashflow series. - IRR is the discount rate r solving NPV(r) = 0, where - NPV(r) = sum(cashflows[t] / (1 + r)^t) over each period t. - - Derivation: IRR has no closed-form solution in general - it is - defined implicitly as the root of the NPV equation, found here via - numpy_financial's iterative solver. It answers what constant annual - return would make this series of cash in/outflows break even, which - lets cashflows of uneven timing/magnitude (purchase, several years - of rental income, then a sale) be compared on a single annualized - basis, unlike cap rate or CoC which are single-period snapshots. + IRR is the discount rate r solving NPV(r) = 0. Args: cashflows: Cashflow series; cashflows[0] is the initial outflow @@ -135,8 +110,7 @@ def irr(cashflows: list[Decimal]) -> Decimal: Returns: IRR as a Decimal (e.g. 0.15 for 15%). Returns Decimal("0") if - the solver fails to converge or returns a non-finite value (no - real root - e.g. all-same-sign cashflows). + the solver fails to converge or returns a non-finite value. """ cf = np.array([float(c) for c in cashflows], dtype=float) try: @@ -175,564 +149,8 @@ def build_cashflows( ) -def score_listing_v1(listing: Listing) -> Decimal: - """Basic Phase 1 scoring using price per sq ft and freshness. - - Higher score is better. This is a simple heuristic for MVP. - - .. deprecated:: - Use :func:`core.services.scoring.score_listing_v2` for - underwriting-grade scores. Will be removed after all callers - are migrated. - """ - import warnings as _warnings - - _warnings.warn( - "score_listing_v1 is deprecated; use core.services.scoring.score_listing_v2 " - "for investor-grade underwriting scores.", - DeprecationWarning, - stacklevel=2, - ) - price = to_decimal(listing.price) if listing.price is not None else Decimal("0") - sq_ft = Decimal(listing.sq_ft or 0) - # price per square foot (lower is better) - ppsf = (price / sq_ft) if sq_ft > 0 else Decimal("0") - # freshness boost: recent postings get a bump - from django.utils import timezone - - now = timezone.now() - age_hours = Decimal(max(1, (now - listing.posted_at).total_seconds() / 3600)) - freshness = Decimal(1) / age_hours - - # Combine with weights - # To avoid division by zero or extreme values, clamp ppsf - ppsf_clamped = ppsf if ppsf > 0 else Decimal("1000000") - score = (Decimal(1000000) / ppsf_clamped) + (freshness * Decimal(10)) - return score - - -def calculate_monthly_mortgage( - loan_amount: Decimal, interest_rate: Decimal, loan_term_years: int -) -> Decimal: - """Calculate monthly mortgage payment (principal and interest). - - Args: - loan_amount: Total loan amount - interest_rate: Annual interest rate as percentage (e.g., 7.5 for 7.5%) - loan_term_years: Loan term in years - - Returns: - Monthly payment amount - """ - loan_amt = to_decimal(loan_amount) - rate = to_decimal(interest_rate) - - if loan_amt == 0: - return Decimal("0") - - if rate == 0: - # No interest - simple division - return loan_amt / Decimal(loan_term_years * 12) - - monthly_rate = rate / Decimal(100) / Decimal(12) - num_payments = Decimal(loan_term_years * 12) - - # Standard amortization formula: M = P[r(1+r)^n]/[(1+r)^n-1] - factor = (Decimal(1) + monthly_rate) ** num_payments - monthly_payment = loan_amt * (monthly_rate * factor) / (factor - Decimal(1)) - - return monthly_payment.quantize(Decimal("0.01")) - - -def calculate_property_tax( - property_value: Decimal, tax_rate_percent: Decimal -) -> Decimal: - """Calculate annual property tax. - - Args: - property_value: Property value/assessed value - tax_rate_percent: Property tax rate as percentage (e.g., 2.1 for 2.1%) - - Returns: - Annual property tax amount - """ - return ( - to_decimal(property_value) * to_decimal(tax_rate_percent) / Decimal(100) - ).quantize(Decimal("0.01")) - - -def estimate_insurance( - property_value: Decimal, - property_type: str = "single-family", - year_built: int = 2000, -) -> Decimal: - """Estimate annual insurance cost. - - Args: - property_value: Property value - property_type: Type of property (single-family, condo, multi-family) - year_built: Year property was built - - Returns: - Estimated annual insurance premium - """ - base_rate = Decimal("1200") # National average for $250k home - - # Adjust for property value - value_factor = to_decimal(property_value) / Decimal("250000") - - # Adjust for property type - type_factors = { - "single-family": Decimal("1.0"), - "condo": Decimal("0.7"), - "multi-family": Decimal("1.3"), - "commercial": Decimal("1.5"), - } - type_factor = type_factors.get(property_type, Decimal("1.0")) - - # Adjust for age - current_year = datetime.now().year - age = max(0, current_year - year_built) - age_factor = Decimal("1.0") + (Decimal(age) / Decimal(50)) - - annual_insurance = base_rate * value_factor * type_factor * age_factor - return annual_insurance.quantize(Decimal("0.01")) - - -def calculate_maintenance_reserve( - property_value: Decimal, - year_built: int = 2000, - annual_percent: Decimal = Decimal("1.0"), -) -> Decimal: - """Calculate annual maintenance reserve (1% rule with age adjustment). - - Args: - property_value: Property value - year_built: Year property was built - annual_percent: Base annual percentage of property value (default 1%) - - Returns: - Annual maintenance reserve amount - """ - base_maintenance = ( - to_decimal(property_value) * to_decimal(annual_percent) / Decimal(100) - ) - - # Adjust for age - if year_built < 1980: - age_factor = Decimal("1.5") - elif year_built < 2000: - age_factor = Decimal("1.2") - else: - age_factor = Decimal("1.0") - - return (base_maintenance * age_factor).quantize(Decimal("0.01")) - - -def calculate_break_even_rent( - monthly_carrying_costs: Decimal, - vacancy_rate_percent: Decimal, - property_management_percent: Decimal = Decimal("10"), -) -> Dict[str, Decimal]: - """Calculate break-even rent needed to cover carrying costs. - - Args: - monthly_carrying_costs: Total monthly carrying costs (excluding property management) - vacancy_rate_percent: Vacancy rate as percentage (e.g., 8 for 8%) - property_management_percent: Property management fee as percentage of rent - - Returns: - Dictionary with breakEvenRent and related metrics - """ - costs = to_decimal(monthly_carrying_costs) - vacancy = to_decimal(vacancy_rate_percent) / Decimal(100) - mgmt = to_decimal(property_management_percent) / Decimal(100) - - # Formula: rent * (1 - vacancy) * (1 - mgmt) = costs - # rent = costs / ((1 - vacancy) * (1 - mgmt)) - divisor = (Decimal(1) - vacancy) * (Decimal(1) - mgmt) - if divisor == 0: - return { - "monthly": Decimal("0"), - "annual": Decimal("0"), - } - break_even = costs / divisor - - return { - "monthly": break_even.quantize(Decimal("0.01")), - "annual": (break_even * Decimal(12)).quantize(Decimal("0.01")), - } - - -def calculate_carrying_costs( - purchase_price: Decimal, - loan_amount: Decimal, - interest_rate: Decimal, - loan_term_years: int, - property_tax_rate: Decimal, - insurance_annual: Decimal | None = None, - hoa_monthly: Decimal = Decimal("0"), - utilities_monthly: Decimal = Decimal("0"), - maintenance_annual_percent: Decimal = Decimal("1.0"), - property_type: str = "single-family", - year_built: int = 2000, -) -> Dict[str, Any]: - """Calculate complete carrying costs breakdown. - - Args: - purchase_price: Property purchase price - loan_amount: Mortgage loan amount - interest_rate: Annual interest rate as percentage - loan_term_years: Loan term in years - property_tax_rate: Property tax rate as percentage - insurance_annual: Annual insurance cost (if None, will estimate) - hoa_monthly: Monthly HOA fees - utilities_monthly: Monthly utility costs - maintenance_annual_percent: Maintenance as percentage of property value - property_type: Type of property - year_built: Year property was built - - Returns: - Dictionary with detailed carrying cost breakdown - """ - # Calculate mortgage - monthly_mortgage = calculate_monthly_mortgage( - loan_amount, interest_rate, loan_term_years - ) - - # Calculate property tax - annual_property_tax = calculate_property_tax(purchase_price, property_tax_rate) - monthly_property_tax = annual_property_tax / Decimal(12) - - # Calculate or use provided insurance - if insurance_annual is None: - annual_insurance = estimate_insurance(purchase_price, property_type, year_built) - else: - annual_insurance = to_decimal(insurance_annual) - monthly_insurance = annual_insurance / Decimal(12) - - # Calculate maintenance - annual_maintenance = calculate_maintenance_reserve( - purchase_price, year_built, maintenance_annual_percent - ) - monthly_maintenance = annual_maintenance / Decimal(12) - - # Monthly costs - monthly_hoa = to_decimal(hoa_monthly) - monthly_utilities = to_decimal(utilities_monthly) - - # Calculate totals - monthly_total = ( - monthly_mortgage - + monthly_property_tax - + monthly_insurance - + monthly_hoa - + monthly_utilities - + monthly_maintenance - ) - - annual_total = monthly_total * Decimal(12) - - return { - "monthly": { - "mortgage": monthly_mortgage.quantize(Decimal("0.01")), - "propertyTax": monthly_property_tax.quantize(Decimal("0.01")), - "insurance": monthly_insurance.quantize(Decimal("0.01")), - "hoa": monthly_hoa.quantize(Decimal("0.01")), - "utilities": monthly_utilities.quantize(Decimal("0.01")), - "maintenance": monthly_maintenance.quantize(Decimal("0.01")), - "total": monthly_total.quantize(Decimal("0.01")), - }, - "annual": { - "mortgage": (monthly_mortgage * Decimal(12)).quantize(Decimal("0.01")), - "propertyTax": annual_property_tax.quantize(Decimal("0.01")), - "insurance": annual_insurance.quantize(Decimal("0.01")), - "hoa": (monthly_hoa * Decimal(12)).quantize(Decimal("0.01")), - "utilities": (monthly_utilities * Decimal(12)).quantize(Decimal("0.01")), - "maintenance": annual_maintenance.quantize(Decimal("0.01")), - "total": annual_total.quantize(Decimal("0.01")), - }, - } - - -def calculate_principal_paydown( - loan_amount: Decimal, - interest_rate: Decimal, - loan_term_years: int, - num_years: int = 1, -) -> Decimal: - """Calculate total principal paid down over specified number of years. - - Args: - loan_amount: Initial loan amount - interest_rate: Annual interest rate as percentage (e.g., 7.5 for 7.5%) - loan_term_years: Total loan term in years - num_years: Number of years to calculate paydown for (default 1) - - Returns: - Total principal paid down over the specified period - """ - if loan_amount == 0 or num_years == 0: - return Decimal("0") - - loan_amt = to_decimal(loan_amount) - rate = to_decimal(interest_rate) - - if rate == 0: - # No interest - equal principal payments - monthly_principal = loan_amt / Decimal(loan_term_years * 12) - return monthly_principal * Decimal(num_years * 12) - - monthly_rate = rate / Decimal(100) / Decimal(12) - monthly_payment = calculate_monthly_mortgage( - loan_amount, interest_rate, loan_term_years - ) - - # Calculate principal paid by simulating each payment - remaining_balance = loan_amt - total_principal_paid = Decimal("0") - - for month in range(num_years * 12): - interest_payment = remaining_balance * monthly_rate - principal_payment = monthly_payment - interest_payment - total_principal_paid += principal_payment - remaining_balance -= principal_payment - - if remaining_balance <= 0: - break - - return total_principal_paid.quantize(Decimal("0.01")) - - -def calculate_appreciation( - property_value: Decimal, - appreciation_rate: Decimal, - num_years: int = 1, -) -> Decimal: - """Calculate property appreciation over specified number of years. - - Args: - property_value: Current property value - appreciation_rate: Annual appreciation rate as percentage (e.g., 3.0 for 3%) - num_years: Number of years to project (default 1) - - Returns: - Total appreciation amount - """ - value = to_decimal(property_value) - rate = to_decimal(appreciation_rate) / Decimal(100) - - future_value = value * ((Decimal(1) + rate) ** Decimal(num_years)) - appreciation = future_value - value - - return appreciation.quantize(Decimal("0.01")) - - -def calculate_tax_benefits( - loan_amount: Decimal, - interest_rate: Decimal, - loan_term_years: int, - property_value: Decimal, - tax_bracket: Decimal = Decimal("24"), - year_num: int = 1, -) -> Decimal: - """Calculate tax benefits from mortgage interest deduction and depreciation. - - Args: - loan_amount: Mortgage loan amount - interest_rate: Annual interest rate as percentage - loan_term_years: Loan term in years - property_value: Property value (for depreciation calculation) - tax_bracket: Marginal tax bracket as percentage (default 24%) - year_num: Which year to calculate benefits for (default 1) - - Returns: - Total tax benefit amount for the specified year - """ - if loan_amount == 0: - # All cash - only depreciation benefit - # Residential property: 27.5 year straight-line depreciation on 80% of value - building_value = to_decimal(property_value) * Decimal("0.80") - annual_depreciation = building_value / Decimal("27.5") - tax_savings = annual_depreciation * (to_decimal(tax_bracket) / Decimal(100)) - return tax_savings.quantize(Decimal("0.01")) - - # Calculate interest paid in specific year - loan_amt = to_decimal(loan_amount) - rate = to_decimal(interest_rate) - monthly_rate = rate / Decimal(100) / Decimal(12) - monthly_payment = calculate_monthly_mortgage( - loan_amount, interest_rate, loan_term_years - ) - - # Calculate remaining balance at start of year - # Uses standard amortization formula: B = P * [(1+r)^(n-k) - 1] / [(1+r)^n - 1] - # where B=balance, P=principal, r=rate, n=total payments, k=payments made - payments_before = (year_num - 1) * 12 - if payments_before > 0: - num_payments = loan_term_years * 12 - remaining_factor = (Decimal(1) + monthly_rate) ** Decimal( - num_payments - payments_before - ) - payment_factor = (Decimal(1) + monthly_rate) ** Decimal(num_payments) - balance_start = loan_amt * ( - (remaining_factor - Decimal(1)) / (payment_factor - Decimal(1)) - ) - else: - balance_start = loan_amt - - # Calculate interest for each month of the year - total_interest = Decimal("0") - balance = balance_start - for _ in range(12): - interest_payment = balance * monthly_rate - principal_payment = monthly_payment - interest_payment - total_interest += interest_payment - balance -= principal_payment - if balance <= 0: - break - - # Add depreciation - building_value = to_decimal(property_value) * Decimal("0.80") - annual_depreciation = building_value / Decimal("27.5") - - # Total deductions - total_deductions = total_interest + annual_depreciation - - # Tax savings - tax_savings = total_deductions * (to_decimal(tax_bracket) / Decimal(100)) - - return tax_savings.quantize(Decimal("0.01")) - - -def calculate_roi_components( - purchase_price: Decimal, - loan_amount: Decimal, - interest_rate: Decimal, - loan_term_years: int, - total_cash_invested: Decimal, - annual_cash_flow: Decimal, - appreciation_rate: Decimal = Decimal("3.0"), - tax_bracket: Decimal = Decimal("24"), - num_years: int = 5, -) -> Dict[str, Any]: - """Calculate comprehensive ROI with all components over multiple years. - - Args: - purchase_price: Property purchase price - loan_amount: Mortgage loan amount - interest_rate: Annual interest rate as percentage - loan_term_years: Loan term in years - total_cash_invested: Total cash invested (down payment + closing costs) - annual_cash_flow: Annual pre-tax cash flow - appreciation_rate: Annual appreciation rate as percentage (default 3%) - tax_bracket: Marginal tax bracket as percentage (default 24%) - num_years: Number of years to project (default 5) - - Returns: - Dictionary with ROI components and projections - """ - # Year 1 calculations - year1_cash_flow = to_decimal(annual_cash_flow) - year1_principal_paydown = calculate_principal_paydown( - loan_amount, interest_rate, loan_term_years, 1 - ) - year1_appreciation = calculate_appreciation(purchase_price, appreciation_rate, 1) - year1_tax_benefits = calculate_tax_benefits( - loan_amount, interest_rate, loan_term_years, purchase_price, tax_bracket, 1 - ) - - year1_total_return = ( - year1_cash_flow - + year1_principal_paydown - + year1_appreciation - + year1_tax_benefits - ) - - if total_cash_invested > 0: - year1_roi = year1_total_return / to_decimal(total_cash_invested) * Decimal(100) - else: - year1_roi = Decimal("0") - - # Multi-year calculations - total_cash_flow = year1_cash_flow * Decimal( - num_years - ) # Simplified: assumes constant - total_principal_paydown = calculate_principal_paydown( - loan_amount, interest_rate, loan_term_years, num_years - ) - total_appreciation = calculate_appreciation( - purchase_price, appreciation_rate, num_years - ) - - # Sum tax benefits for each year - total_tax_benefits = Decimal("0") - for year in range(1, num_years + 1): - total_tax_benefits += calculate_tax_benefits( - loan_amount, - interest_rate, - loan_term_years, - purchase_price, - tax_bracket, - year, - ) - - total_return = ( - total_cash_flow - + total_principal_paydown - + total_appreciation - + total_tax_benefits - ) - - if total_cash_invested > 0: - multi_year_roi = total_return / to_decimal(total_cash_invested) * Decimal(100) - # Annualized return - annualized_roi = ( - (Decimal(1) + multi_year_roi / Decimal(100)) - ** (Decimal(1) / Decimal(num_years)) - - Decimal(1) - ) * Decimal(100) - else: - multi_year_roi = Decimal("0") - annualized_roi = Decimal("0") - - # Component percentages for year 1 - if year1_total_return > 0: - cash_flow_pct = year1_cash_flow / year1_total_return * Decimal(100) - appreciation_pct = year1_appreciation / year1_total_return * Decimal(100) - equity_pct = year1_principal_paydown / year1_total_return * Decimal(100) - tax_pct = year1_tax_benefits / year1_total_return * Decimal(100) - else: - cash_flow_pct = appreciation_pct = equity_pct = tax_pct = Decimal("0") - - return { - "year1": { - "roi": year1_roi.quantize(Decimal("0.1")), - "totalReturn": year1_total_return.quantize(Decimal("0.01")), - "cashFlow": year1_cash_flow.quantize(Decimal("0.01")), - "principalPaydown": year1_principal_paydown.quantize(Decimal("0.01")), - "appreciation": year1_appreciation.quantize(Decimal("0.01")), - "taxBenefits": year1_tax_benefits.quantize(Decimal("0.01")), - }, - f"year{num_years}Projected": { - "roi": multi_year_roi.quantize(Decimal("0.1")), - "annualizedRoi": annualized_roi.quantize(Decimal("0.1")), - "totalReturn": total_return.quantize(Decimal("0.01")), - "totalCashFlow": total_cash_flow.quantize(Decimal("0.01")), - "totalPrincipalPaydown": total_principal_paydown.quantize(Decimal("0.01")), - "totalAppreciation": total_appreciation.quantize(Decimal("0.01")), - "totalTaxBenefits": total_tax_benefits.quantize(Decimal("0.01")), - }, - "components": { - "cashFlowReturn": cash_flow_pct.quantize(Decimal("0.1")), - "appreciationReturn": appreciation_pct.quantize(Decimal("0.1")), - "equityBuildupReturn": equity_pct.quantize(Decimal("0.1")), - "taxBenefitsReturn": tax_pct.quantize(Decimal("0.1")), - }, - } - - def compute_analysis_for_property(prop: Property) -> InvestmentAnalysis: + """Compute and persist the full investment analysis for a property.""" incomes = prop.rental_incomes.all() expenses = prop.operating_expenses.all() @@ -777,309 +195,6 @@ def compute_analysis_for_property(prop: Property) -> InvestmentAnalysis: return analysis -def calculate_flip_strategy( - purchase_price: Decimal, - renovation_costs: Decimal, - holding_period_months: int, - expected_sale_price: Decimal, - selling_costs: Decimal, - down_payment: Decimal, - loan_amount: Decimal, - interest_rate: Decimal, - loan_term_years: int, - closing_costs: Decimal, - property_tax_rate: Decimal, - insurance_annual: Decimal | None = None, - utilities_monthly: Decimal = Decimal("0"), - property_type: str = "single-family", - year_built: int = 2000, -) -> Dict[str, Any]: - """Calculate fix-and-flip strategy returns. - - Args: - purchase_price: Property purchase price - renovation_costs: Total renovation costs - holding_period_months: How long to hold before selling (3-6 months typical) - expected_sale_price: Expected sale price after renovation - selling_costs: Total selling costs (realtor fees, etc.) - down_payment: Down payment amount - loan_amount: Mortgage loan amount - interest_rate: Annual interest rate as percentage - loan_term_years: Loan term in years - closing_costs: Closing costs on purchase - property_tax_rate: Property tax rate as percentage - insurance_annual: Annual insurance cost - utilities_monthly: Monthly utility costs while vacant - property_type: Type of property - year_built: Year property was built - - Returns: - Dictionary with flip strategy analysis - """ - # Calculate holding costs for the period - monthly_mortgage = calculate_monthly_mortgage( - loan_amount, interest_rate, loan_term_years - ) - annual_property_tax = calculate_property_tax(purchase_price, property_tax_rate) - monthly_property_tax = annual_property_tax / Decimal(12) - - if insurance_annual is None: - annual_insurance = estimate_insurance(purchase_price, property_type, year_built) - else: - annual_insurance = to_decimal(insurance_annual) - monthly_insurance = annual_insurance / Decimal(12) - - monthly_holding_costs = ( - monthly_mortgage - + monthly_property_tax - + monthly_insurance - + to_decimal(utilities_monthly) - ) - - total_holding_costs = monthly_holding_costs * Decimal(holding_period_months) - - # Total investment - total_investment = ( - to_decimal(down_payment) - + to_decimal(closing_costs) - + to_decimal(renovation_costs) - ) - - # Calculate proceeds - gross_sale_proceeds = to_decimal(expected_sale_price) - net_sale_proceeds = gross_sale_proceeds - to_decimal(selling_costs) - - # Remaining loan balance after holding period - principal_paid = calculate_principal_paydown( - loan_amount, interest_rate, loan_term_years, holding_period_months // 12 - ) - remaining_loan = to_decimal(loan_amount) - principal_paid - - # Net profit - net_profit = ( - net_sale_proceeds - remaining_loan - total_holding_costs - total_investment - ) - - # ROI - if total_investment > 0: - roi_percent = net_profit / total_investment * Decimal(100) - # Annualized return - years = Decimal(holding_period_months) / Decimal(12) - if years > 0 and roi_percent > Decimal("-100"): - annualized_return = ( - (Decimal(1) + roi_percent / Decimal(100)) ** (Decimal(1) / years) - - Decimal(1) - ) * Decimal(100) - else: - annualized_return = Decimal("0") - else: - roi_percent = Decimal("0") - annualized_return = Decimal("0") - - return { - "totalInvestment": total_investment.quantize(Decimal("0.01")), - "holdingCosts": total_holding_costs.quantize(Decimal("0.01")), - "renovationCosts": to_decimal(renovation_costs).quantize(Decimal("0.01")), - "saleProceeds": gross_sale_proceeds.quantize(Decimal("0.01")), - "sellingCosts": to_decimal(selling_costs).quantize(Decimal("0.01")), - "netProfit": net_profit.quantize(Decimal("0.01")), - "roi": roi_percent.quantize(Decimal("0.1")), - "timeframe": f"{holding_period_months} months", - "annualizedReturn": annualized_return.quantize(Decimal("0.1")), - } - - -def calculate_rental_strategy( - purchase_price: Decimal, - down_payment: Decimal, - loan_amount: Decimal, - interest_rate: Decimal, - loan_term_years: int, - closing_costs: Decimal, - annual_cash_flow: Decimal, - appreciation_rate: Decimal = Decimal("3.0"), - holding_period_years: int = 5, -) -> Dict[str, Any]: - """Calculate buy-and-hold rental strategy returns. - - Args: - purchase_price: Property purchase price - down_payment: Down payment amount - loan_amount: Mortgage loan amount - interest_rate: Annual interest rate as percentage - loan_term_years: Loan term in years - closing_costs: Closing costs - annual_cash_flow: Annual cash flow (can be negative) - appreciation_rate: Annual appreciation rate as percentage - holding_period_years: How many years to hold - - Returns: - Dictionary with rental strategy analysis - """ - total_investment = to_decimal(down_payment) + to_decimal(closing_costs) - - # Simplified: assume constant cash flow (in reality it would improve over time) - total_cash_flow = to_decimal(annual_cash_flow) * Decimal(holding_period_years) - - # Equity buildup from mortgage paydown - equity_buildup = calculate_principal_paydown( - loan_amount, interest_rate, loan_term_years, holding_period_years - ) - - # Appreciation - appreciation = calculate_appreciation( - purchase_price, appreciation_rate, holding_period_years - ) - - # Total gain - total_gain = total_cash_flow + equity_buildup + appreciation - - # ROI - if total_investment > 0: - roi_percent = total_gain / total_investment * Decimal(100) - annualized_return = ( - (Decimal(1) + roi_percent / Decimal(100)) - ** (Decimal(1) / Decimal(holding_period_years)) - - Decimal(1) - ) * Decimal(100) - else: - roi_percent = Decimal("0") - annualized_return = Decimal("0") - - return { - "totalInvestment": total_investment.quantize(Decimal("0.01")), - "year1CashFlow": to_decimal(annual_cash_flow).quantize(Decimal("0.01")), - f"year{holding_period_years}CashFlow": to_decimal(annual_cash_flow).quantize( - Decimal("0.01") - ), # Simplified - f"totalCashFlow{holding_period_years}Years": total_cash_flow.quantize( - Decimal("0.01") - ), - f"equityBuildup{holding_period_years}Years": equity_buildup.quantize( - Decimal("0.01") - ), - f"appreciation{holding_period_years}Years": appreciation.quantize( - Decimal("0.01") - ), - f"totalGain{holding_period_years}Years": total_gain.quantize(Decimal("0.01")), - "roi": roi_percent.quantize(Decimal("0.1")), - "timeframe": f"{holding_period_years} years", - "annualizedReturn": annualized_return.quantize(Decimal("0.1")), - } - - -def calculate_vacation_rental_strategy( - purchase_price: Decimal, - down_payment: Decimal, - loan_amount: Decimal, - interest_rate: Decimal, - loan_term_years: int, - closing_costs: Decimal, - avg_nightly_rate: Decimal, - avg_occupancy_rate: Decimal, # As percentage (e.g., 65 for 65%) - cleaning_fee_per_stay: Decimal, - monthly_operating_expenses: Decimal, - holding_period_years: int = 5, - avg_stay_length_nights: int = 3, # Typical vacation rental stay length -) -> Dict[str, Any]: - """Calculate vacation rental strategy returns. - - Args: - purchase_price: Property purchase price - down_payment: Down payment amount - loan_amount: Mortgage loan amount - interest_rate: Annual interest rate as percentage - loan_term_years: Loan term in years - closing_costs: Closing costs - avg_nightly_rate: Average nightly rental rate - avg_occupancy_rate: Average occupancy rate as percentage - cleaning_fee_per_stay: Cleaning fee per stay - monthly_operating_expenses: Monthly operating expenses - holding_period_years: How many years to hold - avg_stay_length_nights: Average length of stay in nights (default 3) - - Returns: - Dictionary with vacation rental strategy analysis - """ - total_investment = to_decimal(down_payment) + to_decimal(closing_costs) - - # Calculate annual income - nights_per_year = Decimal(365) - occupied_nights = nights_per_year * to_decimal(avg_occupancy_rate) / Decimal(100) - - # Calculate number of stays based on average stay length - avg_stay_length = Decimal(avg_stay_length_nights) - num_stays = occupied_nights / avg_stay_length - - annual_rental_income = occupied_nights * to_decimal( - avg_nightly_rate - ) + num_stays * to_decimal(cleaning_fee_per_stay) - - # Annual expenses - monthly_mortgage = calculate_monthly_mortgage( - loan_amount, interest_rate, loan_term_years - ) - annual_debt_service = monthly_mortgage * Decimal(12) - annual_operating_expenses = to_decimal(monthly_operating_expenses) * Decimal(12) - - # Cash flow - annual_cash_flow = ( - annual_rental_income - annual_debt_service - annual_operating_expenses - ) - - # Calculate year 1 CoC - if total_investment > 0: - coc_return = annual_cash_flow / total_investment * Decimal(100) - else: - coc_return = Decimal("0") - - # 5-year projection (simplified) - total_cash_flow = annual_cash_flow * Decimal(holding_period_years) - - # Equity buildup - equity_buildup = calculate_principal_paydown( - loan_amount, interest_rate, loan_term_years, holding_period_years - ) - - # Appreciation (3% default) - appreciation = calculate_appreciation( - purchase_price, Decimal("3.0"), holding_period_years - ) - - total_gain = total_cash_flow + equity_buildup + appreciation - - if total_investment > 0: - roi_percent = total_gain / total_investment * Decimal(100) - annualized_return = ( - (Decimal(1) + roi_percent / Decimal(100)) - ** (Decimal(1) / Decimal(holding_period_years)) - - Decimal(1) - ) * Decimal(100) - else: - roi_percent = Decimal("0") - annualized_return = Decimal("0") - - return { - "totalInvestment": total_investment.quantize(Decimal("0.01")), - "avgMonthlyIncome": (annual_rental_income / Decimal(12)).quantize( - Decimal("0.01") - ), - "avgMonthlyExpenses": ( - (annual_debt_service + annual_operating_expenses) / Decimal(12) - ).quantize(Decimal("0.01")), - "netCashFlowYear1": annual_cash_flow.quantize(Decimal("0.01")), - "cocReturn": coc_return.quantize(Decimal("0.1")), - "roi": roi_percent.quantize(Decimal("0.1")), - "timeframe": f"{holding_period_years} years", - "annualizedReturn": annualized_return.quantize(Decimal("0.1")), - "seasonalityImpact": ( - "High - Occupancy varies by season" - if avg_occupancy_rate < 75 - else "Moderate" - ), - } - - def calculate_whatif_monthly_cashflow( annual_noi: Decimal, taxes: Decimal = Decimal("0"), @@ -1115,1233 +230,3 @@ def calculate_whatif_monthly_cashflow( to_decimal(rehab_estimate) / Decimal(12) if rehab_estimate else Decimal("0") ) return monthly_income - additional_monthly - rehab_monthly - - -# ── Convenience aliases with calculate_ prefix ───────────────────────────────── -# These satisfy the Jumpstart requirement for explicitly-named calculate_* functions. - - -def calculate_noi( - gross_income: Decimal, - operating_expenses: Decimal, -) -> Decimal: - """Calculate Net Operating Income (NOI). - - NOI = Gross Income - Operating Expenses - - Args: - gross_income: Total annual rental and other income from the property. - operating_expenses: Total annual operating expenses (excluding debt service). - - Returns: - Net Operating Income as a Decimal. - """ - return to_decimal(gross_income) - to_decimal(operating_expenses) - - -def calculate_cap_rate( - annual_noi: Decimal, - property_value: Decimal, -) -> Decimal: - """Calculate Capitalization Rate. - - Cap Rate = NOI / Property Value - - Args: - annual_noi: Net Operating Income. - property_value: Current market value or purchase price of the property. - - Returns: - Capitalization rate as a Decimal (e.g., 0.08 for 8%). - - Raises: - ValueError: If property_value is zero. - """ - pv = to_decimal(property_value) - if pv == Decimal("0"): - raise ValueError("Property value cannot be zero") - return (to_decimal(annual_noi) / pv).quantize(Decimal("0.0001")) - - -def calculate_cash_on_cash( - annual_cash_flow: Decimal, - total_cash_invested: Decimal, -) -> Decimal: - """Calculate Cash-on-Cash Return. - - Cash-on-Cash = Annual Cash Flow / Total Cash Invested - - Args: - annual_cash_flow: Annual pre-tax cash flow from the investment. - total_cash_invested: Total cash invested (down payment + closing costs). - - Returns: - Cash-on-cash return as a Decimal (e.g., 0.10 for 10%). - - Raises: - ValueError: If total_cash_invested is zero. - """ - tci = to_decimal(total_cash_invested) - if tci == Decimal("0"): - raise ValueError("Total cash invested cannot be zero") - return (to_decimal(annual_cash_flow) / tci).quantize(Decimal("0.0001")) - - -def calculate_irr(cash_flows: Sequence[float | int | Decimal]) -> Decimal: - """Calculate Internal Rate of Return (IRR). - - Args: - cash_flows: Sequence of cash flows, starting with the initial investment - (typically negative) followed by periodic returns. Accepts int, float, - or Decimal values; each value is coerced to float internally for - numpy-financial. - - Returns: - IRR as a Decimal (e.g., 0.15 for 15%). - - Raises: - ValueError: If fewer than 2 cash flows are supplied or IRR cannot be computed. - """ - if len(cash_flows) < 2: - raise ValueError("At least 2 cash flows are required to calculate IRR") - normalized_cash_flows = [to_decimal(cf) for cf in cash_flows] - result = irr(normalized_cash_flows) - if result == Decimal("0") and all( - cf >= Decimal("0") for cf in normalized_cash_flows - ): - raise ValueError("IRR could not be computed for the given cash flows") - return result - - -# ── Depreciation & Tax Modeling ──────────────────────────────────────────────── - - -def annual_depreciation(purchase_price: Decimal, land_value: Decimal) -> Decimal: - """Calculate the annual straight-line depreciation for a residential rental property. - - The IRS allows 27.5-year straight-line depreciation on the building portion - (purchase price minus land value) of residential rental property. - - Args: - purchase_price: Total purchase price of the property (must be > 0). - land_value: Estimated value of the land component (must be >= 0 and - < purchase_price). Land is not depreciable. - - Returns: - Annual depreciation deduction as a Decimal representing the fixed deduction - for a full year. Year-by-year schedule handling is the caller's responsibility: - apply this amount for years 1–27 (full deduction), half this amount for year 28 - (remaining half-year fraction), and no deduction for years beyond year 28. - - Raises: - ValueError: If purchase_price <= 0. - ValueError: If land_value < 0. - ValueError: If land_value >= purchase_price (no depreciable basis). - - Example: - >>> annual_depreciation(Decimal("300000"), Decimal("50000")) - Decimal("9090.909090909090909090909091") - """ - pp = to_decimal(purchase_price) - lv = to_decimal(land_value) - - if pp <= Decimal("0"): - raise ValueError("purchase_price must be greater than zero") - if lv < Decimal("0"): - raise ValueError("land_value must be zero or greater") - if lv >= pp: - raise ValueError( - "land_value must be less than purchase_price; land is not depreciable" - ) - - depreciable_basis = pp - lv - return depreciable_basis / Decimal("27.5") - - -def after_tax_cash_flow( - noi: Decimal, - annual_debt_service: Decimal, - depreciation_deduction: Decimal, - marginal_tax_rate: Decimal, -) -> Decimal: - """Calculate after-tax cash flow including the depreciation tax shield. - - Formula: (NOI - debt_service) + (depreciation × tax_rate) - - The depreciation tax shield represents the tax savings from the paper loss of - depreciation, which reduces taxable income without a cash outflow. - - Args: - noi: Net Operating Income (annual). - annual_debt_service: Total annual mortgage payments (principal + interest). - depreciation_deduction: Annual depreciation deduction (e.g., from - ``annual_depreciation()``). - marginal_tax_rate: Investor's marginal income tax rate as a decimal in [0, 1] - (e.g., 0.24 for 24%). - - Returns: - After-tax cash flow as a Decimal. A positive value indicates net cash benefit. - - Raises: - ValueError: If marginal_tax_rate is outside the range [0, 1]. - - Example: - >>> after_tax_cash_flow( - ... Decimal("24000"), Decimal("18000"), Decimal("9091"), Decimal("0.24") - ... ) - Decimal("8181.84") - """ - rate = to_decimal(marginal_tax_rate) - if rate < Decimal("0") or rate > Decimal("1"): - raise ValueError( - "marginal_tax_rate must be between 0 and 1 inclusive " - f"(received {marginal_tax_rate})" - ) - - pre_tax_cf = to_decimal(noi) - to_decimal(annual_debt_service) - tax_shield = to_decimal(depreciation_deduction) * rate - return pre_tax_cf + tax_shield - - -def after_tax_irr( - cash_flows: Sequence[Decimal], - depreciation_schedule: Sequence[Decimal], - marginal_tax_rate: Decimal, -) -> Decimal: - """Calculate after-tax IRR by adjusting each period's cash flow by the depreciation tax shield. - - Each period's cash flow is increased by ``depreciation × marginal_tax_rate``. - The first cash flow (index 0) is assumed to be the initial investment (negative) - and is not adjusted — depreciation tax shields begin in period 1. - - Args: - cash_flows: List of periodic cash flows. Index 0 is typically the initial - investment (negative). Must have at least 2 elements. - depreciation_schedule: List of annual depreciation amounts aligned to - cash_flows[1:]. If shorter than cash_flows[1:], missing periods are - treated as zero depreciation. - marginal_tax_rate: Investor's marginal income tax rate as a decimal in [0, 1]. - - Returns: - After-tax IRR as a Decimal. Returns Decimal("0") if numpy-financial cannot - converge (e.g., all non-negative flows or no sign change). - - Raises: - ValueError: If fewer than 2 cash flows are supplied. - ValueError: If marginal_tax_rate is outside the range [0, 1]. - - Example: - >>> after_tax_irr( - ... [Decimal("-100000"), Decimal("6000"), Decimal("106000")], - ... [Decimal("9091"), Decimal("9091")], - ... Decimal("0.24"), - ... ) - Decimal("0.0718") - """ - if len(cash_flows) < 2: - raise ValueError("At least 2 cash flows are required to calculate IRR") - - rate = to_decimal(marginal_tax_rate) - if rate < Decimal("0") or rate > Decimal("1"): - raise ValueError( - "marginal_tax_rate must be between 0 and 1 inclusive " - f"(received {marginal_tax_rate})" - ) - - # Build adjusted cash flows: index 0 (initial investment) is not adjusted. - adjusted: list[float] = [float(cash_flows[0])] - for i, cf in enumerate(cash_flows[1:]): - dep = ( - depreciation_schedule[i] if i < len(depreciation_schedule) else Decimal("0") - ) - shield = to_decimal(dep) * rate - adjusted.append(float(to_decimal(cf) + shield)) - - cf_array = np.array(adjusted, dtype=float) - try: - value = float(npf.irr(cf_array)) - if np.isnan(value) or np.isinf(value): - logger.warning( - "after_tax_irr: numpy_financial.irr returned non-finite value; returning 0" - ) - return Decimal("0") - return to_decimal(value) - except Exception as exc: - logger.warning("after_tax_irr: numpy_financial.irr raised %s; returning 0", exc) - return Decimal("0") - - -# ── Hold Period & Exit Analysis ──────────────────────────────────────────────── - - -def project_annual_cash_flows( - gross_rent_year1: Decimal, - operating_expense_year1: Decimal, - annual_debt_service: Decimal, - rent_growth_rate: Decimal, - expense_growth_rate: Decimal, - hold_years: int, -) -> list[Decimal]: - """Project year-by-year after-debt-service cash flows over a hold period. - - Each year's gross rent and operating expenses grow independently at their - respective compound annual growth rates. Annual debt service is assumed - constant (fixed-rate mortgage). - - Formula per year ``n`` (1-indexed): - gross_rent(n) = gross_rent_year1 × (1 + rent_growth_rate)^(n-1) - oper_expense(n) = operating_expense_year1 × (1 + expense_growth_rate)^(n-1) - NOI(n) = gross_rent(n) - oper_expense(n) - cash_flow(n) = NOI(n) - annual_debt_service - - Args: - gross_rent_year1: Gross rental income in year 1 (must be >= 0). - operating_expense_year1: Operating expenses in year 1 (must be >= 0). - annual_debt_service: Fixed annual mortgage payment (principal + interest; - must be >= 0). - rent_growth_rate: Annual rent growth rate as a decimal (e.g., 0.03 for 3%). - Must be in the range [-0.5, 0.5]. - expense_growth_rate: Annual expense growth rate as a decimal. - Must be in the range [-0.5, 0.5]. - hold_years: Number of years in the hold period. Must be in [1, 50]. - - Returns: - List of annual cash-flow Decimals, one entry per year (length == hold_years). - Negative values indicate years where debt service exceeds NOI. - - Raises: - ValueError: If ``gross_rent_year1`` or ``operating_expense_year1`` or - ``annual_debt_service`` is negative. - ValueError: If ``hold_years`` is outside [1, 50]. - ValueError: If ``rent_growth_rate`` or ``expense_growth_rate`` is outside - [-0.5, 0.5]. - - Example: - >>> flows = project_annual_cash_flows( - ... Decimal("36000"), Decimal("12000"), Decimal("18000"), - ... Decimal("0.03"), Decimal("0.02"), 5, - ... ) - >>> len(flows) - 5 - """ - if hold_years < 1 or hold_years > 50: - raise ValueError(f"hold_years must be between 1 and 50 (received {hold_years})") - - r_rate = to_decimal(rent_growth_rate) - e_rate = to_decimal(expense_growth_rate) - rate_limit = Decimal("0.5") - if r_rate < -rate_limit or r_rate > rate_limit: - raise ValueError( - f"rent_growth_rate must be in [-0.5, 0.5] (received {rent_growth_rate})" - ) - if e_rate < -rate_limit or e_rate > rate_limit: - raise ValueError( - f"expense_growth_rate must be in [-0.5, 0.5] (received {expense_growth_rate})" - ) - - rent = to_decimal(gross_rent_year1) - expense = to_decimal(operating_expense_year1) - debt = to_decimal(annual_debt_service) - - if rent < Decimal("0"): - raise ValueError( - f"gross_rent_year1 must be zero or greater (received {gross_rent_year1})" - ) - if expense < Decimal("0"): - raise ValueError( - f"operating_expense_year1 must be zero or greater (received {operating_expense_year1})" - ) - if debt < Decimal("0"): - raise ValueError( - f"annual_debt_service must be zero or greater (received {annual_debt_service})" - ) - - cash_flows: list[Decimal] = [] - one = Decimal("1") - for year in range(1, hold_years + 1): - exponent = year - 1 - gross = rent * (one + r_rate) ** exponent - opex = expense * (one + e_rate) ** exponent - annual_noi = gross - opex - cash_flows.append(annual_noi - debt) - - return cash_flows - - -def project_property_value( - purchase_price: Decimal, - appreciation_rate: Decimal, - hold_years: int, -) -> Decimal: - """Project the market value of a property at the end of a hold period. - - Uses compound annual growth: - value = purchase_price × (1 + appreciation_rate)^hold_years - - Supports conservative / base / optimistic scenarios by varying - ``appreciation_rate`` (e.g., 0%, 3%, 5% for US residential). - - Args: - purchase_price: Original purchase price of the property (must be > 0). - appreciation_rate: Expected annual appreciation rate as a decimal. - Must be >= -1 (a rate of -1 implies a total loss of value; rates - below -1 are mathematically undefined for this formula). - hold_years: Number of years to project forward (must be in [1, 50]). - - Returns: - Projected property value as a Decimal. - - Raises: - ValueError: If ``purchase_price`` <= 0. - ValueError: If ``appreciation_rate`` < -1. - ValueError: If ``hold_years`` is outside [1, 50]. - - Example: - >>> project_property_value(Decimal("300000"), Decimal("0.03"), 10) - Decimal("403175....") - """ - pp = to_decimal(purchase_price) - rate = to_decimal(appreciation_rate) - - if pp <= Decimal("0"): - raise ValueError( - f"purchase_price must be greater than zero (received {purchase_price})" - ) - if rate < Decimal("-1"): - raise ValueError( - f"appreciation_rate must be >= -1 (received {appreciation_rate})" - ) - if hold_years < 1 or hold_years > 50: - raise ValueError(f"hold_years must be between 1 and 50 (received {hold_years})") - - return pp * (Decimal("1") + rate) ** hold_years - - -def net_sale_proceeds( - sale_price: Decimal, - original_purchase_price: Decimal, - outstanding_loan_balance: Decimal, - accumulated_depreciation: Decimal, - agent_commission_rate: Decimal = Decimal("0.06"), - closing_cost_rate: Decimal = Decimal("0.01"), - long_term_cg_rate: Decimal = Decimal("0.15"), - depreciation_recapture_rate: Decimal = Decimal("0.25"), -) -> Decimal: - """Calculate net cash to investor after costs and taxes upon property sale. - - Deductions applied in order: - 1. Agent commissions: ``sale_price × agent_commission_rate`` - 2. Closing costs: ``sale_price × closing_cost_rate`` - 3. Loan payoff: ``outstanding_loan_balance`` - 4. Capital gains tax: max(``sale_price - original_purchase_price``, 0) × ``long_term_cg_rate`` - (no capital gains tax if property sold at a loss) - 5. Depreciation recapture: ``accumulated_depreciation × depreciation_recapture_rate`` - - Args: - sale_price: Gross sale price of the property. - original_purchase_price: Price paid for the property at acquisition. - outstanding_loan_balance: Remaining mortgage balance at time of sale - (must be >= 0). - accumulated_depreciation: Total depreciation taken over the holding period - (must be >= 0). - agent_commission_rate: Broker commission as a decimal (default 0.06 = 6%). - closing_cost_rate: Seller's closing costs as a decimal (default 0.01 = 1%). - long_term_cg_rate: Federal long-term capital gains tax rate as a decimal - (default 0.15 = 15%). - depreciation_recapture_rate: IRS Section 1250 recapture rate as a decimal - (default 0.25 = 25%). - - Returns: - Net cash proceeds to investor as a Decimal (may be negative if costs exceed - gross proceeds). - - Raises: - ValueError: If ``outstanding_loan_balance`` < 0. - ValueError: If ``accumulated_depreciation`` < 0. - ValueError: If any rate parameter is outside [0, 1]. - - Example: - >>> net_sale_proceeds( - ... Decimal("400000"), Decimal("300000"), Decimal("200000"), - ... Decimal("45000"), - ... ) - Decimal("...") - """ - sp = to_decimal(sale_price) - opp = to_decimal(original_purchase_price) - loan_bal = to_decimal(outstanding_loan_balance) - acc_dep = to_decimal(accumulated_depreciation) - commission_rate = to_decimal(agent_commission_rate) - cc_rate = to_decimal(closing_cost_rate) - cg_rate = to_decimal(long_term_cg_rate) - recapture_rate = to_decimal(depreciation_recapture_rate) - - if loan_bal < Decimal("0"): - raise ValueError( - f"outstanding_loan_balance must be zero or greater (received {outstanding_loan_balance})" - ) - if acc_dep < Decimal("0"): - raise ValueError( - f"accumulated_depreciation must be zero or greater (received {accumulated_depreciation})" - ) - for name, val in [ - ("agent_commission_rate", commission_rate), - ("closing_cost_rate", cc_rate), - ("long_term_cg_rate", cg_rate), - ("depreciation_recapture_rate", recapture_rate), - ]: - if val < Decimal("0") or val > Decimal("1"): - raise ValueError( - f"{name} must be between 0 and 1 inclusive (received {val})" - ) - - gross_proceeds = sp - sp * commission_rate - sp * cc_rate - loan_bal - - capital_gain = sp - opp - cg_tax = max(capital_gain, Decimal("0")) * cg_rate - - recapture_tax = acc_dep * recapture_rate - - return gross_proceeds - cg_tax - recapture_tax - - -def total_return_summary( - purchase_price: Decimal, - down_payment: Decimal, - annual_cash_flows: list[Decimal], - net_sale_proceeds_amount: Decimal, -) -> Dict[str, Decimal]: - """Summarise total investment return over the hold period. - - Combines cumulative cash flows and net sale proceeds to compute total return - metrics. IRR is computed using the full cash-flow series: - - Year 0: ``-down_payment`` (initial equity outlay) - - Years 1…N: ``annual_cash_flows`` - - Year N: ``annual_cash_flows[-1] + net_sale_proceeds_amount`` (exit year) - - Args: - purchase_price: Original acquisition price of the property. Included in - the returned summary dict as ``"purchase_price"`` for caller convenience. - down_payment: Equity invested at purchase (positive value; used as the - year-0 outflow). - annual_cash_flows: List of annual after-debt-service cash flows from - ``project_annual_cash_flows()``. Must have at least 1 element. - net_sale_proceeds_amount: Net cash to investor upon sale from - ``net_sale_proceeds()``. - - Returns: - Dictionary with the following keys: - - - ``purchase_price`` (Decimal): The ``purchase_price`` argument. - - ``total_cash_flow`` (Decimal): Sum of ``annual_cash_flows``. - - ``net_sale_proceeds`` (Decimal): The ``net_sale_proceeds_amount`` argument. - - ``total_return`` (Decimal): ``total_cash_flow + net_sale_proceeds``. - - ``total_return_on_equity`` (Decimal): ``total_return / down_payment``, or - ``Decimal("0")`` if ``down_payment`` is zero. - - ``annualized_irr`` (Decimal): IRR computed via ``irr()`` over the full - cash-flow series. - - Raises: - ValueError: If ``annual_cash_flows`` is empty. - ValueError: If ``down_payment`` < 0. - - Example: - >>> summary = total_return_summary( - ... Decimal("300000"), Decimal("60000"), - ... [Decimal("6000")] * 10, Decimal("120000"), - ... ) - >>> summary["total_cash_flow"] - Decimal("60000") - """ - if not annual_cash_flows: - raise ValueError("annual_cash_flows must contain at least one element") - - dp = to_decimal(down_payment) - if dp < Decimal("0"): - raise ValueError( - f"down_payment must be zero or greater (received {down_payment})" - ) - - total_cf = sum(annual_cash_flows, Decimal("0")) - nsp = to_decimal(net_sale_proceeds_amount) - total_ret = total_cf + nsp - - if dp == Decimal("0"): - roe = Decimal("0") - else: - roe = total_ret / dp - - # Build IRR cash-flow series: year-0 outflow, annual CFs, exit-year bump - irr_flows: list[Decimal] = [-dp] - for i, cf in enumerate(annual_cash_flows): - if i == len(annual_cash_flows) - 1: - irr_flows.append(cf + nsp) - else: - irr_flows.append(cf) - - annualized = irr(irr_flows) - - return { - "purchase_price": to_decimal(purchase_price), - "total_cash_flow": total_cf, - "net_sale_proceeds": nsp, - "total_return": total_ret, - "total_return_on_equity": roe, - "annualized_irr": annualized, - } - - -def depreciation_recapture_tax( - accumulated_depreciation: Decimal, - recapture_rate: Decimal = Decimal("0.25"), -) -> Decimal: - """Calculate the depreciation recapture tax owed upon sale of the property. - - Under IRS Section 1250, accumulated depreciation is recaptured at a maximum - rate of 25% when the property is sold. This tax is owed regardless of whether - the property sold at a gain or loss on paper. - - Args: - accumulated_depreciation: Total depreciation taken over the holding period - (sum of annual deductions). Must be >= 0. - recapture_rate: IRS Section 1250 recapture rate as a decimal in [0, 1]. - Defaults to 0.25 (25%). - - Returns: - Depreciation recapture tax owed as a Decimal. - - Raises: - ValueError: If accumulated_depreciation < 0. - ValueError: If recapture_rate is outside [0, 1]. - - Example: - >>> depreciation_recapture_tax(Decimal("45000")) - Decimal("11250.00") - """ - acc_dep = to_decimal(accumulated_depreciation) - rate = to_decimal(recapture_rate) - - if acc_dep < Decimal("0"): - raise ValueError("accumulated_depreciation must be zero or greater") - if rate < Decimal("0") or rate > Decimal("1"): - raise ValueError( - "recapture_rate must be between 0 and 1 inclusive " - f"(received {recapture_rate})" - ) - - return acc_dep * rate - - -# ── Underwriting Score v2 ────────────────────────────────────────────────────── - -# Scoring thresholds for composite score sub-components. -# Cap rate is scaled so that a deal exactly 5 pp above market scores 100; -# 5 pp was chosen as a practical "outstanding" cap-rate premium for US rental markets. -_CAP_RATE_SCALE_THRESHOLD = Decimal("0.05") - -# CoC is scaled so that a 20% year-1 cash-on-cash return scores 100; -# 20% is a well-established benchmark for strong passive income investments. -_COC_SCALE_THRESHOLD = Decimal("0.20") - - -class ScoreV2Result(TypedDict): - """Return type for :func:`score_listing_v2`.""" - - one_percent_rule_pass: bool - grm: Decimal - cap_rate: Decimal - cap_rate_vs_market: Decimal - coc_year1: Decimal - composite_score: Decimal - - -def one_percent_rule(monthly_rent: Decimal, purchase_price: Decimal) -> bool: - """Evaluate the 1% Rule for a rental property. - - The 1% Rule is a quick pass/fail filter: monthly rent should be at least - 1% of the purchase price to indicate a potentially viable rental investment. - - Derivation: it's a rule-of-thumb proxy for gross yield (monthly_rent x 12 - / purchase_price >= 0.12) rescaled to a monthly figure so it can be - screened without annualizing - a fast pre-filter, not a substitute for - cap rate or CoC. - - Args: - monthly_rent: Expected gross monthly rental income. - purchase_price: Total purchase price of the property. - - Returns: - True if monthly_rent / purchase_price >= 0.01, False otherwise. - - Raises: - ValueError: If purchase_price is zero or negative. - """ - pp = to_decimal(purchase_price) - if pp <= Decimal("0"): - raise ValueError( - f"purchase_price must be greater than zero (received {purchase_price})" - ) - return to_decimal(monthly_rent) / pp >= Decimal("0.01") - - -def gross_rent_multiplier(purchase_price: Decimal, annual_rent: Decimal) -> Decimal: - """Calculate Gross Rent Multiplier (GRM). - - GRM = Purchase Price / Annual Rent - - Lower GRM values indicate better value relative to rental income. - Typical benchmarks: < 10 excellent, 10–15 good, 15–20 fair, > 20 poor. - - Derivation: GRM is the reciprocal-scaled inverse of a rent yield - (price / rent, vs. cap rate's noi / price) - a valuation multiple - analogous to a price-to-revenue ratio, using gross rent rather than - NOI so it can be computed without first estimating operating expenses. - - Args: - purchase_price: Total purchase price of the property. - annual_rent: Expected gross annual rental income. - - Returns: - GRM as a Decimal. - - Raises: - ValueError: If annual_rent is zero or negative. - """ - ar = to_decimal(annual_rent) - if ar <= Decimal("0"): - raise ValueError( - f"annual_rent must be greater than zero (received {annual_rent})" - ) - return to_decimal(purchase_price) / ar - - -def price_to_rent_ratio( - median_home_price: Decimal, annual_median_rent: Decimal -) -> Decimal: - """Calculate market price-to-rent ratio. - - Args: - median_home_price: Median home purchase price. - annual_median_rent: Median annual rent. - - Returns: - Price-to-rent ratio as a Decimal. - - Raises: - ValueError: If annual_median_rent is zero or negative. - """ - annual_rent = to_decimal(annual_median_rent) - if annual_rent <= Decimal("0"): - raise ValueError( - "annual_median_rent must be greater than zero " - f"(received {annual_median_rent})" - ) - return to_decimal(median_home_price) / annual_rent - - -_EXCELLENT_PRICE_TO_RENT_THRESHOLD = Decimal("15") -_NEUTRAL_PRICE_TO_RENT_THRESHOLD = Decimal("20") -_MAX_PRICE_TO_RENT_THRESHOLD = Decimal("30") -_HIGH_SCORE_FLOOR = Decimal("60") -_HIGH_SCORE_RANGE = Decimal("40") -_LOW_SCORE_RANGE = Decimal("60") - -_MIN_GROWTH_RATE_PERCENT = Decimal("-5") -_MAX_GROWTH_RATE_PERCENT = Decimal("10") -_GROWTH_RATE_RANGE = _MAX_GROWTH_RATE_PERCENT - _MIN_GROWTH_RATE_PERCENT - - -def normalize_market_price_to_rent_score(price_to_rent: Decimal) -> Decimal: - """Convert price-to-rent ratio into a 0-100 market sub-score. - - Args: - price_to_rent: Price-to-rent ratio for a market. - - Returns: - Market sub-score in [0, 100], where higher is better. - - Scoring logic: - - ``price_to_rent < 15`` -> ``100`` - - ``15 <= price_to_rent <= 20`` -> linearly mapped from ``100`` down to ``60`` - - ``20 < price_to_rent <= 30`` -> linearly mapped from ``60`` down to ``0`` - - ``price_to_rent > 30`` -> ``0`` - """ - if price_to_rent <= Decimal("0"): - return Decimal("0") - if price_to_rent < _EXCELLENT_PRICE_TO_RENT_THRESHOLD: - return Decimal("100") - if price_to_rent <= _NEUTRAL_PRICE_TO_RENT_THRESHOLD: - return (_NEUTRAL_PRICE_TO_RENT_THRESHOLD - price_to_rent) / ( - _NEUTRAL_PRICE_TO_RENT_THRESHOLD - _EXCELLENT_PRICE_TO_RENT_THRESHOLD - ) * _HIGH_SCORE_RANGE + _HIGH_SCORE_FLOOR - if price_to_rent <= _MAX_PRICE_TO_RENT_THRESHOLD: - return ( - (_MAX_PRICE_TO_RENT_THRESHOLD - price_to_rent) - / (_MAX_PRICE_TO_RENT_THRESHOLD - _NEUTRAL_PRICE_TO_RENT_THRESHOLD) - * _LOW_SCORE_RANGE - ) - return Decimal("0") - - -def normalize_market_growth_rate_score(growth_rate: Decimal) -> Decimal: - """Convert annual growth rate percent into a 0-100 market sub-score. - - Args: - growth_rate: Annual growth rate as a percent value. - - Returns: - Market sub-score in [0, 100], where higher is better. - - Growth values are clamped to ``[-5, 10]`` percent and then linearly - mapped to ``[0, 100]``. - """ - clamped = max(_MIN_GROWTH_RATE_PERCENT, min(_MAX_GROWTH_RATE_PERCENT, growth_rate)) - return (clamped - _MIN_GROWTH_RATE_PERCENT) / _GROWTH_RATE_RANGE * Decimal("100") - - -def clamp_market_score(value: Decimal) -> Decimal: - """Clamp a market score to the valid 0-100 range. - - Args: - value: Raw market score value. - - Returns: - Score clamped to [0, 100]. - """ - return max(Decimal("0"), min(Decimal("100"), value)) - - -def _grm_score(grm: Decimal) -> Decimal: - """Map a GRM value to a 0–100 sub-score using published heuristics. - - Heuristics: GRM < 10 → excellent (100), 10–15 → good (75), - 15–20 → fair (40), > 20 → poor (10). - - Args: - grm: Gross Rent Multiplier value. - - Returns: - Sub-score as a Decimal in [0, 100]. - """ - if grm < Decimal("10"): - return Decimal("100") - if grm < Decimal("15"): - return Decimal("75") - if grm < Decimal("20"): - return Decimal("40") - return Decimal("10") - - -def score_listing_v2( - purchase_price: Decimal, - monthly_rent: Decimal, - annual_noi: Decimal, - local_market_cap_rate: Decimal, - down_payment: Decimal, - annual_debt_service: Decimal, -) -> ScoreV2Result: - """Compute an investor-grade multi-signal underwriting score for a rental listing. - - Signals and weights (hardcoded; future versions should read from - settings.UNDERWRITING_SCORE_WEIGHTS): - - 1% Rule pass/fail : 20% - - Cap rate vs. market : 30% - - CoC Year 1 : 30% - - GRM heuristic : 20% - - A deal that fails the 1% Rule has its composite_score capped at 40. - - Args: - purchase_price: Total purchase price of the property. Must be > 0. - monthly_rent: Expected gross monthly rental income. Must be > 0. - annual_noi: Net Operating Income for Year 1. Must be > 0. - local_market_cap_rate: Prevailing cap rate for the market as a decimal - (e.g., Decimal("0.06") for 6%). Must be > 0. - down_payment: Cash down payment (total cash invested). Must be > 0. - annual_debt_service: Total annual principal + interest payments. Must be > 0. - - Returns: - Dict with keys: - one_percent_rule_pass (bool): True if monthly_rent / purchase_price >= 1%. - grm (Decimal): Gross Rent Multiplier. - cap_rate (Decimal): cap rate = annual_noi / purchase_price. - cap_rate_vs_market (Decimal): cap_rate minus local_market_cap_rate; - positive means above-market (better). - coc_year1 (Decimal): Cash-on-Cash return for Year 1. - composite_score (Decimal): Weighted score in [0, 100]. - - Raises: - ValueError: If any of purchase_price, monthly_rent, annual_noi, - local_market_cap_rate, down_payment, or annual_debt_service is <= 0. - """ - # -- Input validation ------------------------------------------------------- - inputs = { - "purchase_price": purchase_price, - "monthly_rent": monthly_rent, - "annual_noi": annual_noi, - "local_market_cap_rate": local_market_cap_rate, - "down_payment": down_payment, - "annual_debt_service": annual_debt_service, - } - for name, value in inputs.items(): - if to_decimal(value) <= Decimal("0"): - raise ValueError(f"{name} must be greater than zero (received {value})") - - # -- Individual signals ----------------------------------------------------- - pp = to_decimal(purchase_price) - mr = to_decimal(monthly_rent) - noi_val = to_decimal(annual_noi) - market_cap = to_decimal(local_market_cap_rate) - dp = to_decimal(down_payment) - ads = to_decimal(annual_debt_service) - - one_pct_pass = one_percent_rule(mr, pp) - grm = gross_rent_multiplier(pp, mr * Decimal("12")) - property_cap_rate = noi_val / pp - cap_rate_vs_market = property_cap_rate - market_cap - annual_cash_flow = noi_val - ads - coc_year1 = annual_cash_flow / dp - - # -- Sub-scores (each 0–100) ------------------------------------------------ - # 1% Rule: full points if pass, zero if fail - one_pct_sub = Decimal("100") if one_pct_pass else Decimal("0") - - # Cap rate vs market: scale so +5 pp above market = 100, -5 pp = 0 - # cap_rate_vs_market is a raw decimal difference (e.g. 0.02 = 2 pp above) - raw_cap_score = (cap_rate_vs_market / _CAP_RATE_SCALE_THRESHOLD) * Decimal("100") - cap_vs_market_sub = max(Decimal("0"), min(Decimal("100"), raw_cap_score)) - - # CoC Year 1: scale so 20% CoC = 100 points, 0% CoC = 0 points - raw_coc_score = coc_year1 / _COC_SCALE_THRESHOLD * Decimal("100") - coc_sub = max(Decimal("0"), min(Decimal("100"), raw_coc_score)) - - # GRM heuristic sub-score - grm_sub = _grm_score(grm) - - # -- Composite score -------------------------------------------------------- - # Weights: 1% rule 20%, cap vs market 30%, CoC 30%, GRM 20% - composite = ( - one_pct_sub * Decimal("0.20") - + cap_vs_market_sub * Decimal("0.30") - + coc_sub * Decimal("0.30") - + grm_sub * Decimal("0.20") - ) - - # Cap at 40 if 1% rule fails - if not one_pct_pass: - composite = min(Decimal("40"), composite) - - composite = max(Decimal("0"), min(Decimal("100"), composite)) - - return { - "one_percent_rule_pass": one_pct_pass, - "grm": grm, - "cap_rate": property_cap_rate, - "cap_rate_vs_market": cap_rate_vs_market, - "coc_year1": coc_year1, - "composite_score": composite, - } - - -# ── BRRRR Strategy ────────────────────────────────────────────────────────────── - - -def estimate_arv( - comparable_sales: list[tuple[Decimal, Decimal]], - subject_sqft: Decimal, -) -> Decimal: - """Estimate After-Repair Value (ARV) from comparable sales. - - Computes the median price-per-square-foot (PPSF) of the comparable sales - and multiplies it by the subject property's square footage. - - Args: - comparable_sales: List of ``(price, sqft)`` tuples, one per comparable - sale. Both ``price`` and ``sqft`` must be positive. - subject_sqft: Square footage of the subject property (must be > 0). - - Returns: - Estimated ARV as a Decimal. - - Raises: - ValueError: If ``comparable_sales`` is empty. - ValueError: If any comparable has ``price <= 0`` or ``sqft <= 0``. - ValueError: If ``subject_sqft <= 0``. - """ - if not comparable_sales: - raise ValueError("comparable_sales must not be empty") - - subject = to_decimal(subject_sqft) - if subject <= Decimal("0"): - raise ValueError( - f"subject_sqft must be greater than zero (received {subject_sqft})" - ) - - ppsf_values: list[Decimal] = [] - for idx, (price, sqft) in enumerate(comparable_sales): - p = to_decimal(price) - s = to_decimal(sqft) - if p <= Decimal("0"): - raise ValueError( - f"comparable_sales[{idx}]: price must be greater than zero (received {price})" - ) - if s <= Decimal("0"): - raise ValueError( - f"comparable_sales[{idx}]: sqft must be greater than zero (received {sqft})" - ) - ppsf_values.append(p / s) - - median_ppsf = to_decimal(median(ppsf_values)) - return median_ppsf * subject - - -def estimate_rehab_cost( - sqft: Decimal, - renovation_level: str, - cost_per_sqft: dict[str, Decimal], -) -> Decimal: - """Estimate total rehab cost for a property. - - Args: - sqft: Square footage of the property (must be > 0). - renovation_level: Scope of renovation. Must be one of the keys present - in ``cost_per_sqft`` (typically ``"cosmetic"``, ``"moderate"``, or - ``"full_gut"``). - cost_per_sqft: Mapping from renovation level to cost per square foot. - Supply ``settings.REHAB_COST_PER_SQFT`` from the service layer to - keep this function Django-free. - - Returns: - Estimated rehab cost as a Decimal. - - Raises: - ValueError: If ``renovation_level`` is not a key in ``cost_per_sqft``. - ValueError: If ``sqft <= 0``. - """ - valid_levels = set(cost_per_sqft.keys()) - if renovation_level not in valid_levels: - raise ValueError( - f"renovation_level must be one of {sorted(valid_levels)} " - f"(received {renovation_level!r})" - ) - s = to_decimal(sqft) - if s <= Decimal("0"): - raise ValueError(f"sqft must be greater than zero (received {sqft})") - - rate = to_decimal(cost_per_sqft[renovation_level]) - return rate * s - - -def max_refinance_loan( - arv: Decimal, - ltv_ratio: Decimal = Decimal("0.75"), -) -> Decimal: - """Calculate the maximum cash-out refinance loan amount at a given LTV. - - The conventional investment-property cash-out refinance limit (Fannie Mae) - is 75 % LTV. Expose ``ltv_ratio`` as a configurable parameter so callers - can model different lender requirements. - - Args: - arv: After-Repair Value of the property (must be > 0). - ltv_ratio: Loan-to-value ratio expressed as a decimal strictly between - 0 and 1 (e.g., ``Decimal("0.75")`` for 75 %). - - Returns: - Maximum refinance loan amount as a Decimal. - - Raises: - ValueError: If ``arv <= 0``. - ValueError: If ``ltv_ratio`` is not strictly in ``(0, 1)``. - """ - a = to_decimal(arv) - ltv = to_decimal(ltv_ratio) - - if a <= Decimal("0"): - raise ValueError(f"arv must be greater than zero (received {arv})") - if ltv <= Decimal("0") or ltv >= Decimal("1"): - raise ValueError( - f"ltv_ratio must be strictly between 0 and 1 (received {ltv_ratio})" - ) - - return a * ltv - - -def cash_left_in_deal( - purchase_price: Decimal, - rehab_cost: Decimal, - cash_out_refi_amount: Decimal, - closing_costs: Decimal = Decimal("0"), -) -> Decimal: - """Calculate the investor's remaining cash deployed after a cash-out refinance. - - Formula:: - - cash_left = purchase_price + rehab_cost + closing_costs - cash_out_refi_amount - - A negative or zero result means the investor has recouped all invested capital - (the "infinite CoC" scenario in BRRRR terminology). - - Args: - purchase_price: Purchase price of the property. - rehab_cost: Total rehabilitation cost. - cash_out_refi_amount: Proceeds from the cash-out refinance. - closing_costs: Total closing costs (purchase + refi combined). Defaults - to ``Decimal("0")``. - - Returns: - Cash left in the deal as a Decimal. Negative or zero ⇒ infinite CoC. - """ - return ( - to_decimal(purchase_price) - + to_decimal(rehab_cost) - + to_decimal(closing_costs) - - to_decimal(cash_out_refi_amount) - ) - - -def brrrr_coc_return( - annual_net_cash_flow: Decimal, - cash_left_in_deal: Decimal, -) -> Decimal: - """Calculate Cash-on-Cash return for a BRRRR deal. - - Handles the "infinite CoC" scenario where the investor has recouped all - (or more than all) of their capital. - - Rules: - * ``cash_left_in_deal <= 0`` → returns ``Decimal("Infinity")`` regardless - of cash flow (investor has no capital remaining in the deal). - * ``cash_left_in_deal > 0`` and ``annual_net_cash_flow == 0`` → returns - ``Decimal("0")`` (no return on remaining capital). - * Otherwise → returns ``annual_net_cash_flow / cash_left_in_deal``. - - Args: - annual_net_cash_flow: Annual after-debt-service cash flow (can be - negative for a losing deal). - cash_left_in_deal: Capital still deployed after the cash-out refi - (from ``cash_left_in_deal()``). - - Returns: - CoC return as a Decimal. ``Decimal("Infinity")`` signals infinite CoC. - """ - left = to_decimal(cash_left_in_deal) - flow = to_decimal(annual_net_cash_flow) - - if left <= Decimal("0"): - return Decimal("Infinity") - if flow == Decimal("0"): - return Decimal("0") - return flow / left - - -# ── Simplified Depreciation & After-Tax Calculations ───────────────────────── -# These are user-friendly wrappers around the lower-level functions above. -# They accept a land-value percentage instead of an absolute land value, and -# take a single pre-tax cash flow figure rather than NOI / debt-service args. - - -def calculate_annual_depreciation( - purchase_price: Decimal, - land_value_pct: Decimal = Decimal("0.20"), -) -> Decimal: - """Calculate annual straight-line depreciation for residential real estate. - - Uses the IRS 27.5-year straight-line schedule on the improvement value - (purchase price minus land). - - Args: - purchase_price: Total purchase price of the property (must be > 0). - land_value_pct: Fraction of purchase price attributable to land, - expressed as a decimal (e.g. ``Decimal("0.20")`` for 20 %). - Must be in the range [0, 1). Defaults to 0.20 (20 %), a - reasonable conservative estimate for most US residential properties. - - Returns: - Annual depreciation deduction as a Decimal. - - Raises: - ValueError: If ``purchase_price`` <= 0. - ValueError: If ``land_value_pct`` is outside [0, 1). - - Example: - >>> calculate_annual_depreciation(Decimal("200000"), Decimal("0.20")) - Decimal("5818.181818181818181818181818") - """ - pp = to_decimal(purchase_price) - lvp = to_decimal(land_value_pct) - - if pp <= Decimal("0"): - raise ValueError( - f"purchase_price must be greater than zero (received {purchase_price})" - ) - if lvp < Decimal("0") or lvp >= Decimal("1"): - raise ValueError( - f"land_value_pct must be in [0, 1) (received {land_value_pct})" - ) - - improvement_value = pp * (Decimal("1") - lvp) - return improvement_value / Decimal("27.5") - - -def calculate_after_tax_cashflow( - pre_tax_annual_cashflow: Decimal, - annual_depreciation: Decimal, - marginal_tax_rate: Decimal, -) -> Decimal: - """Calculate after-tax cash flow including the depreciation tax shield. - - This is a simplified model that does **not** account for passive activity - loss (PAL) rules, cost segregation, or other advanced tax strategies. - A UI disclaimer should note this limitation. - - Formula:: - - taxable_income = pre_tax_annual_cashflow - annual_depreciation - - if taxable_income < 0: # paper loss - tax_savings = abs(taxable_income) × marginal_tax_rate - after_tax = pre_tax_annual_cashflow + tax_savings - else: # taxable profit - tax_owed = taxable_income × marginal_tax_rate - after_tax = pre_tax_annual_cashflow - tax_owed - - Args: - pre_tax_annual_cashflow: Annual pre-tax cash flow from the property - (NOI minus debt service). Can be negative. - annual_depreciation: Annual depreciation deduction (e.g. from - :func:`calculate_annual_depreciation`). - marginal_tax_rate: Investor's marginal income-tax rate as a decimal - in [0, 1] (e.g. ``Decimal("0.32")`` for 32 %). - - Returns: - After-tax annual cash flow as a Decimal. - - Raises: - ValueError: If ``marginal_tax_rate`` is outside [0, 1]. - - Example: - >>> calculate_after_tax_cashflow( - ... Decimal("6000"), Decimal("5818"), Decimal("0.32") - ... ) - Decimal("5941.76") - """ - cashflow = to_decimal(pre_tax_annual_cashflow) - depreciation = to_decimal(annual_depreciation) - rate = to_decimal(marginal_tax_rate) - - if rate < Decimal("0") or rate > Decimal("1"): - raise ValueError( - f"marginal_tax_rate must be in [0, 1] (received {marginal_tax_rate})" - ) - - taxable_income = cashflow - depreciation - - if taxable_income < 0: - # Paper loss → tax savings (depreciation tax shield) - tax_savings = abs(taxable_income) * rate - return cashflow + tax_savings - else: - # Taxable profit → tax owed - tax_owed = taxable_income * rate - return cashflow - tax_owed diff --git a/tests/test_brrrr.py b/tests/test_brrrr.py index b303794d..7f96ed3a 100644 --- a/tests/test_brrrr.py +++ b/tests/test_brrrr.py @@ -12,7 +12,7 @@ import pytest -from investor_app.finance.utils import ( +from investor_app.finance.strategies import ( brrrr_coc_return, cash_left_in_deal, estimate_arv, diff --git a/tests/test_finance_math.py b/tests/test_finance_math.py index 5551d8e1..cf23fe16 100644 --- a/tests/test_finance_math.py +++ b/tests/test_finance_math.py @@ -16,17 +16,10 @@ import pytest # ── Production functions ─────────────────────────────────────────────────── -from investor_app.finance.utils import ( - annual_depreciation, - calculate_monthly_mortgage, - cap_rate, - cash_on_cash, - dscr, - gross_rent_multiplier, - irr, - noi, - one_percent_rule, -) +from investor_app.finance.mortgage import calculate_monthly_mortgage +from investor_app.finance.scoring import gross_rent_multiplier, one_percent_rule +from investor_app.finance.taxes import annual_depreciation +from investor_app.finance.utils import cap_rate, cash_on_cash, dscr, irr, noi # ── Reference implementations ────────────────────────────────────────────── from tests.finance_reference import ( diff --git a/tests/test_finance_utils.py b/tests/test_finance_utils.py index f9162a68..e4ba4c7c 100644 --- a/tests/test_finance_utils.py +++ b/tests/test_finance_utils.py @@ -4,175 +4,12 @@ import pytest -from investor_app.finance.utils import ( +from investor_app.finance.taxes import ( calculate_after_tax_cashflow, calculate_annual_depreciation, - calculate_cap_rate, - calculate_cash_on_cash, - calculate_irr, - calculate_noi, ) -class TestCalculateNoi: - """Tests for calculate_noi function.""" - - def test_positive_noi(self) -> None: - """Test NOI calculation with positive result.""" - gross_income = Decimal("120000") - operating_expenses = Decimal("40000") - result = calculate_noi(gross_income, operating_expenses) - assert result == Decimal("80000") - - def test_negative_noi(self) -> None: - """Test NOI calculation with negative result.""" - gross_income = Decimal("30000") - operating_expenses = Decimal("45000") - result = calculate_noi(gross_income, operating_expenses) - assert result == Decimal("-15000") - - def test_zero_noi(self) -> None: - """Test NOI calculation with zero result.""" - gross_income = Decimal("50000") - operating_expenses = Decimal("50000") - result = calculate_noi(gross_income, operating_expenses) - assert result == Decimal("0") - - def test_decimal_precision(self) -> None: - """Test that NOI preserves decimal precision.""" - gross_income = Decimal("100000.55") - operating_expenses = Decimal("33333.33") - result = calculate_noi(gross_income, operating_expenses) - assert result == Decimal("66667.22") - - -class TestCalculateCapRate: - """Tests for calculate_cap_rate function.""" - - def test_typical_cap_rate(self) -> None: - """Test cap rate calculation with typical values.""" - noi = Decimal("80000") - property_value = Decimal("1000000") - result = calculate_cap_rate(noi, property_value) - assert result == Decimal("0.08") - - def test_high_cap_rate(self) -> None: - """Test cap rate calculation with high return.""" - noi = Decimal("150000") - property_value = Decimal("1000000") - result = calculate_cap_rate(noi, property_value) - assert result == Decimal("0.15") - - def test_zero_property_value_raises_error(self) -> None: - """Test that zero property value raises ValueError.""" - noi = Decimal("80000") - property_value = Decimal("0") - with pytest.raises(ValueError, match="Property value cannot be zero"): - calculate_cap_rate(noi, property_value) - - def test_decimal_precision(self) -> None: - """Test that cap rate preserves decimal precision.""" - noi = Decimal("75000") - property_value = Decimal("1000000") - result = calculate_cap_rate(noi, property_value) - assert result == Decimal("0.075") - - -class TestCalculateCashOnCash: - """Tests for calculate_cash_on_cash function.""" - - def test_typical_cash_on_cash(self) -> None: - """Test cash-on-cash calculation with typical values.""" - annual_cash_flow = Decimal("20000") - total_cash_invested = Decimal("200000") - result = calculate_cash_on_cash(annual_cash_flow, total_cash_invested) - assert result == Decimal("0.1") - - def test_high_cash_on_cash(self) -> None: - """Test cash-on-cash calculation with high return.""" - annual_cash_flow = Decimal("50000") - total_cash_invested = Decimal("200000") - result = calculate_cash_on_cash(annual_cash_flow, total_cash_invested) - assert result == Decimal("0.25") - - def test_negative_cash_flow(self) -> None: - """Test cash-on-cash with negative cash flow.""" - annual_cash_flow = Decimal("-10000") - total_cash_invested = Decimal("200000") - result = calculate_cash_on_cash(annual_cash_flow, total_cash_invested) - assert result == Decimal("-0.05") - - def test_zero_cash_invested_raises_error(self) -> None: - """Test that zero cash invested raises ValueError.""" - annual_cash_flow = Decimal("20000") - total_cash_invested = Decimal("0") - with pytest.raises(ValueError, match="Total cash invested cannot be zero"): - calculate_cash_on_cash(annual_cash_flow, total_cash_invested) - - -class TestCalculateIrr: - """Tests for calculate_irr function.""" - - def test_positive_irr(self) -> None: - """Test IRR calculation with positive returns.""" - cash_flows = [ - Decimal("-100000"), - Decimal("30000"), - Decimal("35000"), - Decimal("40000"), - Decimal("45000"), - ] - result = calculate_irr(cash_flows) - # IRR should be approximately 15-20% - assert Decimal("0.10") < result < Decimal("0.25") - - def test_negative_irr(self) -> None: - """Test IRR calculation with negative returns.""" - cash_flows = [ - Decimal("-100000"), - Decimal("10000"), - Decimal("10000"), - Decimal("10000"), - ] - result = calculate_irr(cash_flows) - assert result < Decimal("0") - - def test_simple_irr(self) -> None: - """Test IRR with simple doubling investment.""" - # If you invest 100 and get 200 back in year 1, IRR = 100% - cash_flows = [Decimal("-100"), Decimal("200")] - result = calculate_irr(cash_flows) - assert abs(result - Decimal("1.0")) < Decimal("0.0001") - - def test_insufficient_cash_flows_raises_error(self) -> None: - """Test that fewer than 2 cash flows raises ValueError.""" - cash_flows = [Decimal("-100000")] - with pytest.raises(ValueError, match="At least 2 cash flows are required"): - calculate_irr(cash_flows) - - def test_empty_cash_flows_raises_error(self) -> None: - """Test that empty cash flows raises ValueError.""" - cash_flows: list[Decimal] = [] - with pytest.raises(ValueError, match="At least 2 cash flows are required"): - calculate_irr(cash_flows) - - def test_returns_decimal(self) -> None: - """Test that IRR returns a Decimal type.""" - cash_flows = [Decimal("-100"), Decimal("110")] - result = calculate_irr(cash_flows) - assert isinstance(result, Decimal) - - def test_all_positive_flows_raises_error(self) -> None: - """Test that all-positive cash flows (no sign change) raise ValueError. - - numpy_financial.irr returns nan for flows with no sign change; - calculate_irr should surface this as a ValueError. - """ - cash_flows = [Decimal("10000"), Decimal("20000"), Decimal("30000")] - with pytest.raises(ValueError, match="IRR could not be computed"): - calculate_irr(cash_flows) - - # ── Depreciation & After-Tax Cash Flow Tests ───────────────────────────────── diff --git a/tests/test_hold_period.py b/tests/test_hold_period.py index 756462ff..51a29330 100644 --- a/tests/test_hold_period.py +++ b/tests/test_hold_period.py @@ -4,7 +4,7 @@ import pytest -from investor_app.finance.utils import ( +from investor_app.finance.taxes import ( net_sale_proceeds, project_annual_cash_flows, project_property_value, diff --git a/tests/test_market_scoring.py b/tests/test_market_scoring.py index 7f1586b3..caf61cd2 100644 --- a/tests/test_market_scoring.py +++ b/tests/test_market_scoring.py @@ -7,7 +7,7 @@ from core.models import MarketSnapshot from core.services.market_scoring import score_market, update_market_scores -from investor_app.finance.utils import ( +from investor_app.finance.scoring import ( clamp_market_score, normalize_market_growth_rate_score, normalize_market_price_to_rent_score, diff --git a/tests/test_property_service.py b/tests/test_property_service.py deleted file mode 100644 index c0797f4e..00000000 --- a/tests/test_property_service.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Tests for property service layer functions.""" - -from decimal import Decimal - -from core.services.property_service import calculate_noi - - -class TestCalculateNoi: - """Tests for calculate_noi in the service layer.""" - - def test_positive_noi(self) -> None: - result = calculate_noi(Decimal("120000"), Decimal("40000")) - assert result == Decimal("80000.00") - - def test_negative_noi(self) -> None: - result = calculate_noi(Decimal("30000"), Decimal("45000")) - assert result == Decimal("-15000.00") - - def test_zero_noi(self) -> None: - result = calculate_noi(Decimal("50000"), Decimal("50000")) - assert result == Decimal("0.00") - - def test_decimal_precision(self) -> None: - result = calculate_noi(Decimal("100000.55"), Decimal("33333.33")) - assert result == Decimal("66667.22") - - def test_large_values(self) -> None: - result = calculate_noi(Decimal("999999999.99"), Decimal("1.00")) - assert result == Decimal("999999998.99") - - def test_zero_income(self) -> None: - result = calculate_noi(Decimal("0"), Decimal("50000")) - assert result == Decimal("-50000.00") - - def test_zero_expenses(self) -> None: - result = calculate_noi(Decimal("75000"), Decimal("0")) - assert result == Decimal("75000.00") - - def test_all_zero(self) -> None: - result = calculate_noi(Decimal("0"), Decimal("0")) - assert result == Decimal("0.00") - - def test_quantized_to_two_places(self) -> None: - result = calculate_noi(Decimal("100"), Decimal("0.333")) - assert result == Decimal("99.67") diff --git a/tests/test_tax_analysis.py b/tests/test_tax_analysis.py index 55964093..8ccf3937 100644 --- a/tests/test_tax_analysis.py +++ b/tests/test_tax_analysis.py @@ -4,13 +4,13 @@ import pytest -from investor_app.finance.utils import ( +from investor_app.finance.taxes import ( after_tax_cash_flow, after_tax_irr, annual_depreciation, - calculate_irr, depreciation_recapture_tax, ) +from investor_app.finance.utils import irr class TestAnnualDepreciation: @@ -178,7 +178,7 @@ def test_zero_tax_rate_matches_pre_tax(self) -> None: dep_schedule = [Decimal("9091"), Decimal("9091")] after_tax = after_tax_irr(cash_flows, dep_schedule, Decimal("0")) # With 0% tax rate, no shield, so the cash flows are unchanged - pre_tax = calculate_irr(cash_flows) + pre_tax = irr(cash_flows) assert abs(after_tax - pre_tax) < Decimal("0.0001") def test_insufficient_cash_flows_raises(self) -> None: @@ -232,7 +232,7 @@ def test_empty_depreciation_schedule(self) -> None: """Test that an empty depreciation schedule applies no shields.""" cash_flows = [Decimal("-100000"), Decimal("55000"), Decimal("60000")] result = after_tax_irr(cash_flows, [], Decimal("0.24")) - pre_tax = calculate_irr(cash_flows) + pre_tax = irr(cash_flows) # No depreciation shield → should equal pre-tax IRR assert abs(result - pre_tax) < Decimal("0.0001") diff --git a/tests/test_underwriting_score.py b/tests/test_underwriting_score.py index 9b37eb61..9265b12a 100644 --- a/tests/test_underwriting_score.py +++ b/tests/test_underwriting_score.py @@ -1,16 +1,17 @@ -"""Tests for underwriting score v2 functions. +"""Tests for underwriting score v2 primitives. -Covers one_percent_rule, gross_rent_multiplier, and score_listing_v2. +Covers one_percent_rule and gross_rent_multiplier. The full underwriting +score is tested in core/tests/test_scoring_v2.py against the Django +implementation in core.services.scoring. """ from decimal import Decimal import pytest -from investor_app.finance.utils import ( +from investor_app.finance.scoring import ( gross_rent_multiplier, one_percent_rule, - score_listing_v2, ) # ── one_percent_rule ─────────────────────────────────────────────────────────── @@ -121,166 +122,3 @@ def test_returns_decimal(self) -> None: annual_rent=Decimal("24000"), ) assert isinstance(result, Decimal) - - -# ── score_listing_v2 ─────────────────────────────────────────────────────────── - - -class TestScoreListingV2: - """Tests for score_listing_v2.""" - - # -- Helpers ---------------------------------------------------------------- - - def _realistic_inputs(self, **overrides) -> dict: - """Return a baseline set of realistic inputs for score_listing_v2.""" - base = dict( - purchase_price=Decimal("250000"), - monthly_rent=Decimal("2500"), # 1% rule: 2500/250000 = 1.0% → pass - annual_noi=Decimal("18000"), # cap rate 7.2% - local_market_cap_rate=Decimal("0.06"), - down_payment=Decimal("62500"), # 25% down - annual_debt_service=Decimal("14400"), - ) - base.update(overrides) - return base - - # -- Smoke test / basic structure ------------------------------------------ - - def test_returns_expected_keys(self) -> None: - """score_listing_v2 should return a dict with all required keys.""" - result = score_listing_v2(**self._realistic_inputs()) - assert set(result.keys()) == { - "one_percent_rule_pass", - "grm", - "cap_rate", - "cap_rate_vs_market", - "coc_year1", - "composite_score", - } - - def test_realistic_composite_score_in_range(self) -> None: - """Composite score must be in [0, 100] for realistic inputs.""" - result = score_listing_v2(**self._realistic_inputs()) - score = result["composite_score"] - assert Decimal("0") <= score <= Decimal("100") - - def test_one_percent_rule_pass_reflected(self) -> None: - """one_percent_rule_pass should be True when rent >= 1% of price.""" - result = score_listing_v2(**self._realistic_inputs()) - assert result["one_percent_rule_pass"] is True - - def test_cap_rate_vs_market_positive_when_above(self) -> None: - """cap_rate_vs_market should be positive when property cap > local market cap.""" - result = score_listing_v2(**self._realistic_inputs()) - # 7.2% property cap vs 6% market → positive difference - assert result["cap_rate_vs_market"] > Decimal("0") - - # -- 1% rule failure caps composite score ---------------------------------- - - def test_failing_one_percent_rule_caps_score_at_40(self) -> None: - """A deal that fails the 1% rule must have composite_score <= 40.""" - # monthly_rent = 800 on a 250k property → 0.32% → fails - result = score_listing_v2(**self._realistic_inputs(monthly_rent=Decimal("800"))) - assert result["one_percent_rule_pass"] is False - assert result["composite_score"] <= Decimal("40") - - # -- Above-market vs. below-market cap rate -------------------------------- - - def test_above_market_cap_rate_raises_score_vs_below(self) -> None: - """A deal with a cap rate above market should score higher than one below market.""" - above = score_listing_v2( - **self._realistic_inputs(local_market_cap_rate=Decimal("0.04")) - ) # property cap 7.2% vs 4% market → above - below = score_listing_v2( - **self._realistic_inputs(local_market_cap_rate=Decimal("0.09")) - ) # property cap 7.2% vs 9% market → below - assert above["composite_score"] > below["composite_score"] - - # -- Boundary: very large purchase price ----------------------------------- - - def test_large_purchase_price(self) -> None: - """score_listing_v2 must handle a $10M property without raising.""" - result = score_listing_v2( - purchase_price=Decimal("10000000"), - monthly_rent=Decimal("100000"), # 1% of 10M → pass - annual_noi=Decimal("720000"), # 7.2% cap rate - local_market_cap_rate=Decimal("0.06"), - down_payment=Decimal("2500000"), - annual_debt_service=Decimal("576000"), - ) - assert Decimal("0") <= result["composite_score"] <= Decimal("100") - - # -- ValueError on invalid inputs ------------------------------------------ - - def test_zero_purchase_price_raises(self) -> None: - with pytest.raises(ValueError, match="purchase_price"): - score_listing_v2(**self._realistic_inputs(purchase_price=Decimal("0"))) - - def test_negative_purchase_price_raises(self) -> None: - with pytest.raises(ValueError, match="purchase_price"): - score_listing_v2(**self._realistic_inputs(purchase_price=Decimal("-1"))) - - def test_zero_monthly_rent_raises(self) -> None: - with pytest.raises(ValueError, match="monthly_rent"): - score_listing_v2(**self._realistic_inputs(monthly_rent=Decimal("0"))) - - def test_zero_annual_noi_raises(self) -> None: - with pytest.raises(ValueError, match="annual_noi"): - score_listing_v2(**self._realistic_inputs(annual_noi=Decimal("0"))) - - def test_zero_local_market_cap_rate_raises(self) -> None: - with pytest.raises(ValueError, match="local_market_cap_rate"): - score_listing_v2( - **self._realistic_inputs(local_market_cap_rate=Decimal("0")) - ) - - def test_zero_down_payment_raises(self) -> None: - with pytest.raises(ValueError, match="down_payment"): - score_listing_v2(**self._realistic_inputs(down_payment=Decimal("0"))) - - def test_zero_annual_debt_service_raises(self) -> None: - with pytest.raises(ValueError, match="annual_debt_service"): - score_listing_v2(**self._realistic_inputs(annual_debt_service=Decimal("0"))) - - def test_negative_monthly_rent_raises(self) -> None: - with pytest.raises(ValueError, match="monthly_rent"): - score_listing_v2(**self._realistic_inputs(monthly_rent=Decimal("-500"))) - - def test_negative_annual_noi_raises(self) -> None: - with pytest.raises(ValueError, match="annual_noi"): - score_listing_v2(**self._realistic_inputs(annual_noi=Decimal("-1000"))) - - def test_negative_local_market_cap_rate_raises(self) -> None: - with pytest.raises(ValueError, match="local_market_cap_rate"): - score_listing_v2( - **self._realistic_inputs(local_market_cap_rate=Decimal("-0.05")) - ) - - def test_negative_down_payment_raises(self) -> None: - with pytest.raises(ValueError, match="down_payment"): - score_listing_v2(**self._realistic_inputs(down_payment=Decimal("-1000"))) - - def test_negative_annual_debt_service_raises(self) -> None: - with pytest.raises(ValueError, match="annual_debt_service"): - score_listing_v2( - **self._realistic_inputs(annual_debt_service=Decimal("-1000")) - ) - - def test_grm_matches_expected(self) -> None: - """GRM should be purchase_price / annual_rent.""" - result = score_listing_v2(**self._realistic_inputs()) - expected_grm = Decimal("250000") / (Decimal("2500") * 12) - # Allow small rounding difference - assert abs(result["grm"] - expected_grm) < Decimal("0.0001") - - def test_cap_rate_matches_expected(self) -> None: - """cap_rate should be annual_noi / purchase_price.""" - result = score_listing_v2(**self._realistic_inputs()) - expected = Decimal("18000") / Decimal("250000") - assert abs(result["cap_rate"] - expected) < Decimal("0.0001") - - def test_coc_year1_matches_expected(self) -> None: - """coc_year1 should be (annual_noi - annual_debt_service) / down_payment.""" - result = score_listing_v2(**self._realistic_inputs()) - expected = (Decimal("18000") - Decimal("14400")) / Decimal("62500") - assert abs(result["coc_year1"] - expected) < Decimal("0.0001") diff --git a/tests_bdd/steps/test_property_analysis.py b/tests_bdd/steps/test_property_analysis.py index 38e78b4d..007afa31 100644 --- a/tests_bdd/steps/test_property_analysis.py +++ b/tests_bdd/steps/test_property_analysis.py @@ -13,12 +13,8 @@ from pytest_bdd import given, parsers, scenario, then, when from core.models import InvestmentAnalysis, OperatingExpense, Property, RentalIncome -from investor_app.finance.utils import ( - cap_rate, - dscr, - estimate_insurance, - noi, -) +from investor_app.finance.mortgage import estimate_insurance +from investor_app.finance.utils import cap_rate, dscr, noi # Acceptance criteria require NOI and cap rate checks with ±3% relative tolerance. REL_TOLERANCE = 0.03 From 32213391eb43134680f5f38433a73de54517e795 Mon Sep 17 00:00:00 2001 From: Phil Ruff Date: Fri, 31 Jul 2026 12:02:13 +0100 Subject: [PATCH 2/9] refactor(core): remove prei package, consolidate pipeline into core services Port discovery, screening, underwriting, and offer math from the pydantic prei package into pydantic-free Django services under core/services/. Offer math now uses Decimal (resolves LIMIT-21). Both view bridges (Growth Explorer, VRM pipeline) call core.services; persistence stays on PipelineProperty via process_discovery. - delete prei/ entirely (models, pipeline, api, cli, tests) - move landlord_data.py to core/services/ - migrate 9 test files + BDD steps to core.services.* - drop fastapi/uvicorn/click; keep pydantic for acceptance tests - update KNOWN_LIMITATIONS, ARCHITECTURE, CHANGE_IMPACT_MAP Suite: 1791 passed, 1 skipped, 256 deselected. ruff clean. --- .agents/logs/2026-07-31.jsonl | 1 + .agents/reports/build-report-2026-07-31.md | 51 ++ .../handlers => core/services}/discovery.py | 171 +++--- core/services/discovery_processor.py | 145 +++++ .../services}/landlord_data.py | 3 +- core/services/offer.py | 140 +++++ core/services/screening.py | 155 ++++- {prei => core/services/sources}/__init__.py | 0 .../services}/sources/base.py | 0 .../services}/sources/county.py | 2 +- .../services}/sources/file_source.py | 2 +- .../services}/sources/registry.py | 8 +- .../services}/sources/reo_sources.py | 4 +- .../services}/sources/vrm_source.py | 2 +- .../services}/underwriting.py | 23 +- core/tests/test_pipeline_bridge.py | 2 +- core/views/__init__.py | 25 +- docs/ARCHITECTURE.md | 19 + docs/CHANGE_IMPACT_MAP.md | 5 + docs/KNOWN_LIMITATIONS.md | 10 +- prei/api/__init__.py | 0 prei/api/pipeline_routes.py | 209 ------- prei/cli.py | 189 ------- prei/models/__init__.py | 0 prei/models/pipeline.py | 130 ----- prei/pipeline/__init__.py | 0 prei/pipeline/engine.py | 529 ------------------ prei/pipeline/handlers/__init__.py | 0 prei/pipeline/handlers/batch_screening.py | 196 ------- prei/pipeline/handlers/discovery_processor.py | 101 ---- prei/pipeline/handlers/offer.py | 124 ---- prei/pipeline/handlers/screening.py | 163 ------ prei/pipeline/orchestrator.py | 247 -------- prei/pipeline/sources/__init__.py | 0 prei/pipeline/tests/test_api.py | 340 ----------- prei/pipeline/tests/test_batch_screening.py | 203 ------- prei/pipeline/tests/test_county.py | 124 ---- prei/pipeline/tests/test_discovery.py | 298 ---------- .../tests/test_discovery_processor.py | 229 -------- prei/pipeline/tests/test_engine.py | 275 --------- prei/pipeline/tests/test_offer.py | 157 ------ prei/pipeline/tests/test_orchestrator.py | 105 ---- prei/pipeline/tests/test_reo_sources.py | 222 -------- prei/pipeline/tests/test_repository.py | 289 ---------- prei/pipeline/tests/test_screening.py | 308 ---------- prei/pipeline/tests/test_sources.py | 238 -------- prei/pipeline/tests/test_underwriting.py | 219 -------- requirements.txt | 3 - tasks.json | 186 ++++-- tests/test_discovery.py | 39 +- tests/test_discovery_e2e.py | 157 ++---- tests/test_discovery_integration.py | 193 ++++--- tests/test_offer_integration.py | 123 ++-- tests/test_pipeline.py | 275 +-------- tests/test_pipeline_e2e.py | 118 ++-- tests/test_screening_integration.py | 79 ++- tests/test_underwriting_integration.py | 73 ++- tests_bdd/steps/pipeline_steps.py | 22 +- 58 files changed, 1203 insertions(+), 5728 deletions(-) create mode 100644 .agents/reports/build-report-2026-07-31.md rename {prei/pipeline/handlers => core/services}/discovery.py (53%) create mode 100644 core/services/discovery_processor.py rename {prei/integrations => core/services}/landlord_data.py (97%) create mode 100644 core/services/offer.py rename {prei => core/services/sources}/__init__.py (100%) rename {prei/pipeline => core/services}/sources/base.py (100%) rename {prei/pipeline => core/services}/sources/county.py (99%) rename {prei/pipeline => core/services}/sources/file_source.py (99%) rename {prei/pipeline => core/services}/sources/registry.py (92%) rename {prei/pipeline => core/services}/sources/reo_sources.py (98%) rename {prei/pipeline => core/services}/sources/vrm_source.py (98%) rename {prei/pipeline/handlers => core/services}/underwriting.py (91%) delete mode 100644 prei/api/__init__.py delete mode 100644 prei/api/pipeline_routes.py delete mode 100644 prei/cli.py delete mode 100644 prei/models/__init__.py delete mode 100644 prei/models/pipeline.py delete mode 100644 prei/pipeline/__init__.py delete mode 100644 prei/pipeline/engine.py delete mode 100644 prei/pipeline/handlers/__init__.py delete mode 100644 prei/pipeline/handlers/batch_screening.py delete mode 100644 prei/pipeline/handlers/discovery_processor.py delete mode 100644 prei/pipeline/handlers/offer.py delete mode 100644 prei/pipeline/handlers/screening.py delete mode 100644 prei/pipeline/orchestrator.py delete mode 100644 prei/pipeline/sources/__init__.py delete mode 100644 prei/pipeline/tests/test_api.py delete mode 100644 prei/pipeline/tests/test_batch_screening.py delete mode 100644 prei/pipeline/tests/test_county.py delete mode 100644 prei/pipeline/tests/test_discovery.py delete mode 100644 prei/pipeline/tests/test_discovery_processor.py delete mode 100644 prei/pipeline/tests/test_engine.py delete mode 100644 prei/pipeline/tests/test_offer.py delete mode 100644 prei/pipeline/tests/test_orchestrator.py delete mode 100644 prei/pipeline/tests/test_reo_sources.py delete mode 100644 prei/pipeline/tests/test_repository.py delete mode 100644 prei/pipeline/tests/test_screening.py delete mode 100644 prei/pipeline/tests/test_sources.py delete mode 100644 prei/pipeline/tests/test_underwriting.py diff --git a/.agents/logs/2026-07-31.jsonl b/.agents/logs/2026-07-31.jsonl index f1e278d3..6ce2d3f6 100644 --- a/.agents/logs/2026-07-31.jsonl +++ b/.agents/logs/2026-07-31.jsonl @@ -1 +1,2 @@ {"agent": "build", "session_id": "build-20260731-finance-split-001", "triggered_by": "feature-flow", "started_at": "2026-07-31T16:00:00Z", "timestamp": "2026-07-31T17:45:00Z", "duration_ms": 6300000, "skills_loaded": ["code-generation", "refactoring", "template-application"], "findings": [{"id": "FIND-001", "description": "Moved copy of total_return_summary dropped the purchase_price key from its return dict, breaking test_dict_keys_present — restored the key to match the original contract", "actionable": true, "manual_review_needed": false, "severity": "defect"}, {"id": "FIND-002", "description": "after_tax_irr in taxes.py had a local import of irr from utils that was unused (function reimplements npf.irr inline) — removed per ruff F401", "actionable": true, "manual_review_needed": false, "severity": "defect"}, {"id": "FIND-003", "description": "tests/test_underwriting_score.py still tested the deleted pure score_listing_v2 (audit finding #2); kept one_percent_rule/gross_rent_multiplier primitive tests, deleted TestScoreListingV2 since production score lives only in core/services/scoring.py (covered by core/tests/test_scoring_v2.py)", "actionable": true, "manual_review_needed": false, "severity": "defect"}, {"id": "FIND-004", "description": "Service-layer duplicate calculate_noi in core/services/property_service.py was exported but imported by no production code — deleted function, export, and its test file", "actionable": true, "manual_review_needed": false, "severity": "defect"}, {"id": "FIND-005", "description": "Dual-pipeline investigation: Django (PipelineAsset/PipelineProperty + core/services/pipeline.py) is the load-bearing pipeline; the pydantic prei FastAPI router, CLI, and orchestrator are not mounted in any Django URLconf/INSTALLED_APPS/docker-compose; only core/views/__init__.py couples to prei (get_state_landlord_score + lazy DiscoveryProcessor/BatchScreeningProcessor/discover_from_all for the Growth Explorer bridge). Full removal of pydantic state machine requires PM sign-off", "actionable": false, "manual_review_needed": true, "severity": "note"}, {"id": "FIND-006", "description": "Re-export backfill from utils.py was unnecessary: after updating all 17 importers, no remaining importer pulls a moved name from investor_app.finance.utils; keeping the monolith aliases would defeat the split", "actionable": false, "manual_review_needed": false, "severity": "note"}, {"id": "FIND-007", "description": "Pre-existing mypy error in tests/acceptance/conftest.py:69 (no-any-return) unrelated to this change — file unmodified", "actionable": false, "manual_review_needed": false, "severity": "note"}], "decision": "implemented", "blockers": [], "pr": null} +{"agent": "build", "session_id": "build-20260731-pydantic-django-consolidation-001", "triggered_by": "feature-flow", "started_at": "2026-07-31T09:00:00Z", "timestamp": "2026-07-31T10:04:32Z", "duration_ms": 3872000, "skills_loaded": ["code-generation", "refactoring", "template-application", "governance-enforcement"], "findings": [{"id": "FIND-001", "description": "Offer math ported to Decimal in core/services/offer.py with OfferInput.__post_init__ validation, resolving LIMIT-21; old tests using model_copy rewritten with dataclasses.replace and Decimal equality", "actionable": true, "manual_review_needed": false, "severity": "defect"}, {"id": "FIND-002", "description": "New dataclasses are pydantic-free so coercion must be explicit: DiscoverySanitizer.transform_input now applies _coerce_float/_coerce_beds/_coerce_baths/_coerce_sqft/_coerce_year; fixed 5 test failures", "actionable": true, "manual_review_needed": false, "severity": "defect"}, {"id": "FIND-003", "description": "prei/integrations/landlord_data.py moved to core/services/landlord_data.py; core/views/__init__.py:50 import updated", "actionable": true, "manual_review_needed": false, "severity": "defect"}, {"id": "FIND-004", "description": "pydantic retained in requirements.txt because tests/acceptance/{schemas,test_api}.py still import it; fastapi/uvicorn/click removed with no remaining consumers", "actionable": true, "manual_review_needed": false, "severity": "note"}, {"id": "FIND-005", "description": "ghcr.io/paruff/prei docker image is the Django web image, not a separate FastAPI/CLI surface; docker-compose/ZAP refs out of scope for this plan", "actionable": false, "manual_review_needed": false, "severity": "note"}, {"id": "FIND-006", "description": "mypy has 2 pre-existing errors unrelated to this work: tests/acceptance/conftest.py:69 (no-any-return) and tests_bdd/conftest.py:31 (__init__ misc); left untouched", "actionable": false, "manual_review_needed": true, "severity": "note"}, {"id": "FIND-007", "description": "Single remaining prei.pipeline reference is a logger namespace string in core/services/pipeline.py, not an import", "actionable": false, "manual_review_needed": false, "severity": "note"}], "decision": "Approved plan executed in full: prei package deleted, services ported to core/services with Decimal offer math, both view bridges rewritten, tests migrated, deps trimmed, docs updated. Full suite green (1791 passed).", "blockers": []} diff --git a/.agents/reports/build-report-2026-07-31.md b/.agents/reports/build-report-2026-07-31.md new file mode 100644 index 00000000..4e0104a7 --- /dev/null +++ b/.agents/reports/build-report-2026-07-31.md @@ -0,0 +1,51 @@ +## Build Report — pydantic→Django consolidation (remove `prei/`, port services to `core/services/`) + +**Status:** COMPLETE + +--- + +### Tasks Completed + +| Task | Title | Lines Changed | Status | +| -------- | ----- | ------------- | ------ | +| TASK-01 | Port discovery services (sanitizer, `CanonicalPropertyPayload`, `process_discovery_batch`) to `core/services/discovery.py` + `discovery_processor.py`, pydantic-free | ~300 | DONE | +| TASK-02 | Port sources (base, registry, county, reo_sources, vrm_source, file_source) to `core/services/sources/` | ~600 | DONE | +| TASK-03 | Port screening evaluator (`ScreeningThresholds`, `evaluate_screening_stage`, `screen_batch`) to `core/services/screening.py` | ~151 | DONE | +| TASK-04 | Port underwriting + offer solvers to `core/services/underwriting.py` / `offer.py` (Decimal; LIMIT-21) | ~250 | DONE | +| TASK-05 | Rewrite Growth Explorer bridge in `core/views/__init__.py` to use `core.services` | ~12 | DONE | +| TASK-06 | Rewrite VRM `run_pipeline` bridge to use `core.services` | ~13 | DONE | +| TASK-07 | Migrate tests + BDD steps to `core.services.*`; delete orchestrator/state-machine test sections | ~950 | DONE | +| TASK-08 | Remove `prei/models/pipeline.py`, `prei/pipeline/engine.py`, `prei/pipeline/orchestrator.py` | −3 files | DONE | +| TASK-09 | Remove `prei/api/`, `prei/cli.py`, `prei/pipeline/tests/test_api.py` (fastapi/click consumers) | −5 files | DONE | +| TASK-10 | Delete entire `prei/` package; zero `from prei`/`import prei` repo-wide (AC-10-1) | −all | DONE | +| TASK-11 | Drop `fastapi`/`uvicorn`/`click`; keep `pydantic` (acceptance tests); update KNOWN_LIMITATIONS (LIMIT-21 resolved), ARCHITECTURE.md, CHANGE_IMPACT_MAP.md (AC-11-3) | ~40 | DONE | + +### Artifacts Produced + +- [x] Source code files — `core/services/{discovery,discovery_processor,screening,underwriting,offer,landlord_data}.py`, `core/services/sources/` (all pydantic-free, ruff/mypy clean) +- [x] Manifests — n/a (no new K8s surface; this is a Django app repo) +- [x] Pipeline — n/a (CI config unchanged; verified prei docker image is the Django web image, not a separate FastAPI surface) +- [x] Overlays — n/a +- [x] Tests — 9 test files + `tests_bdd/steps/pipeline_steps.py` migrated to `core.services.*` +- [x] Docs — `docs/KNOWN_LIMITATIONS.md`, `docs/ARCHITECTURE.md`, `docs/CHANGE_IMPACT_MAP.md` updated + +### Validation Results + +| Check | Status | +| --------- | ------ | +| Lint | PASS (`ruff check .` — All checks passed) | +| Typecheck | PASS (2 pre-existing mypy errors in untouched files: `tests/acceptance/conftest.py:69` no-any-return, `tests_bdd/conftest.py:31` `__init__` misc) | +| Tests | PASS (full suite 1791 passed, 1 skipped, 256 deselected in 445s; e2e subset 19 passed; local unit 35 passed) | +| Policy | PASS (AC-10-1 no prei imports; AC-10-3 suite green with prei gone; governance: no Bootstrap/secrets/float-currency violations in new code) | + +### Blockers + +None. + +### Notes + +- `pydantic` retained in `requirements.txt` because `tests/acceptance/{schemas,test_api.py}` still import it; `fastapi`/`uvicorn`/`click` had zero remaining consumers. +- `ScreeningThresholds`/`screen_batch` remain float-based (transient ratio math, by design); Decimal conversion happens at the persistence boundary; `underwriting.py` + `offer.py` are Decimal. +- Bridge sites are stats-only (no persistence), matching AC-05-2 "identical behavior". +- `CLAUDE.md` working-tree change is unrelated/pre-existing (not part of this plan). +- Log entry appended to `.agents/logs/2026-07-31.jsonl`. diff --git a/prei/pipeline/handlers/discovery.py b/core/services/discovery.py similarity index 53% rename from prei/pipeline/handlers/discovery.py rename to core/services/discovery.py index 0e218c05..a2a9fda0 100644 --- a/prei/pipeline/handlers/discovery.py +++ b/core/services/discovery.py @@ -1,5 +1,6 @@ """Canonical schema and structural ingestion sanitizer for the DISCOVERY stage. +Ported from prei.pipeline.handlers.discovery (pydantic removed). Creates an unyielding data normalization layer that maps erratic, multi-source external data structures (MLS listings, county foreclosure scraps, wholesale raw JSON dumps) into a clean, unified internal Python schema. @@ -9,88 +10,87 @@ import hashlib import re -from typing import Any, Dict, Optional +from dataclasses import dataclass, field +from typing import Any -from pydantic import BaseModel, Field, field_validator +def _coerce_float(v: Any) -> float | None: + """Coerce messy numeric inputs to float or None. -class CanonicalPropertyPayload(BaseModel): + Handles string representations ("1998", ""), float types (3.0), + and missing/null values uniformly. + """ + if v is None or v == "": + return None + try: + return float(v) + except ValueError, TypeError: + return 0.0 + + +def _coerce_beds(v: Any) -> int: + """Coerce beds to int. Handles 3.0, "3", None.""" + if v is None or v == "": + return 0 + try: + return int(float(v)) + except ValueError, TypeError: + return 0 + + +def _coerce_baths(v: Any) -> float: + """Coerce baths to float. Handles "2", 2, 2.5, None.""" + if v is None or v == "": + return 0.0 + try: + return float(v) + except ValueError, TypeError: + return 0.0 + + +def _coerce_sqft(v: Any) -> float | None: + """Coerce sqft to Optional[float].""" + if v is None or v == "": + return None + try: + return float(v) + except ValueError, TypeError: + return None + + +def _coerce_year(v: Any) -> int | None: + """Coerce year_built to Optional[int].""" + if v is None or v == "": + return None + try: + return int(float(v)) + except ValueError, TypeError: + return None + + +@dataclass +class CanonicalPropertyPayload: """Unified internal schema for property data entering the pipeline. All fields are strictly typed. Secondary fields that cannot be extracted from a given source default to None rather than a magic value. + + Monetary fields (price, estimated_rent) are float in this transient DTO + (behavior preserved from the prei port); Decimal conversion happens at the + persistence boundary (core.services.discovery_processor). """ source_id: str source_name: str raw_address: str address_hash: str - price: Optional[float] = None - estimated_rent: Optional[float] = None - beds: int - baths: float - sqft: Optional[float] = None - year_built: Optional[int] = None - raw_metadata: Dict[str, Any] = Field(default_factory=dict) - - @field_validator("price", "estimated_rent", mode="before") - @classmethod - def coerce_float(cls, v: Any) -> Optional[float]: - """Coerce messy numeric inputs to float or None. - - Handles string representations ("1998", ""), float types (3.0), - and missing/null values uniformly. - """ - if v is None or v == "": - return None - try: - return float(v) - except ValueError, TypeError: - return 0.0 - - @field_validator("beds", mode="before") - @classmethod - def coerce_beds(cls, v: Any) -> int: - """Coerce beds to int. Handles 3.0, "3", None.""" - if v is None or v == "": - return 0 - try: - return int(float(v)) - except ValueError, TypeError: - return 0 - - @field_validator("baths", mode="before") - @classmethod - def coerce_baths(cls, v: Any) -> float: - """Coerce baths to float. Handles "2", 2, 2.5, None.""" - if v is None or v == "": - return 0.0 - try: - return float(v) - except ValueError, TypeError: - return 0.0 - - @field_validator("sqft", mode="before") - @classmethod - def coerce_sqft(cls, v: Any) -> Optional[float]: - """Coerce sqft to Optional[float].""" - if v is None or v == "": - return None - try: - return float(v) - except ValueError, TypeError: - return None - - @field_validator("year_built", mode="before") - @classmethod - def coerce_year(cls, v: Any) -> Optional[int]: - """Coerce year_built to Optional[int].""" - if v is None or v == "": - return None - try: - return int(float(v)) - except ValueError, TypeError: - return None + price: float | None = None + estimated_rent: float | None = None + beds: int = 0 + baths: float = 0.0 + sqft: float | None = None + year_built: int | None = None + raw_metadata: dict[str, Any] = field(default_factory=dict) class DiscoverySanitizer: @@ -129,7 +129,7 @@ def compute_address_hash(cls, normalized_address: str) -> str: @classmethod def transform_input( cls, - raw: Dict[str, Any], + raw: dict[str, Any], source: str, ) -> CanonicalPropertyPayload: """Transform a raw external data dict into a canonical payload. @@ -174,18 +174,23 @@ def transform_input( source_name=source, raw_address=raw_address, address_hash=addr_hash, - price=raw.get("price") or raw.get("ListPrice") or raw.get("sale_price"), - estimated_rent=raw.get("rent") - or raw.get("RentEstimate") - or raw.get("estimated_rent"), - beds=raw.get("beds") or raw.get("BedroomsTotal") or 0, - baths=raw.get("baths") - or raw.get("BathroomsTotalInteger") - or raw.get("BathroomsFull") - or 0.0, - sqft=raw.get("sqft") or raw.get("LivingArea") or raw.get("SquareFootage"), - year_built=raw.get("year_built") - or raw.get("YearBuilt") - or raw.get("yearBuilt"), + price=_coerce_float( + raw.get("price") or raw.get("ListPrice") or raw.get("sale_price") + ), + estimated_rent=_coerce_float( + raw.get("rent") or raw.get("RentEstimate") or raw.get("estimated_rent") + ), + beds=_coerce_beds(raw.get("beds") or raw.get("BedroomsTotal")), + baths=_coerce_baths( + raw.get("baths") + or raw.get("BathroomsTotalInteger") + or raw.get("BathroomsFull") + ), + sqft=_coerce_sqft( + raw.get("sqft") or raw.get("LivingArea") or raw.get("SquareFootage") + ), + year_built=_coerce_year( + raw.get("year_built") or raw.get("YearBuilt") or raw.get("yearBuilt") + ), raw_metadata=raw, ) diff --git a/core/services/discovery_processor.py b/core/services/discovery_processor.py new file mode 100644 index 00000000..66950f2f --- /dev/null +++ b/core/services/discovery_processor.py @@ -0,0 +1,145 @@ +"""Dedup-aware discovery ingestion for the DISCOVERY stage. + +Ported from prei.pipeline.handlers.discovery_processor (pydantic removed). +Provides two layers: + +- ``process_discovery_batch`` — pure stats/dedup pass over raw listings, + returns the same analytics contract the prei DiscoveryProcessor exposed + (total_received/new_assets_discovered/duplicates_skipped/failed_records/ + payloads). No persistence — used by stats-only view bridges. +- ``process_discovery`` — persists one canonical payload as a + PipelineProperty at DISCOVERED stage, deduplicated by address_hash + (per user, source_type). This is the Django-state write path. +""" + +from __future__ import annotations + +from decimal import Decimal +from typing import TYPE_CHECKING, Any + +from django.utils import timezone + +from core.services.discovery import CanonicalPropertyPayload, DiscoverySanitizer + +if TYPE_CHECKING: + from django.contrib.auth.models import User + + from core.models import PipelineProperty + + +# ── Stats-only batch pass (preserves prei DiscoveryProcessor contract) ──────── + + +def process_discovery_batch( + raw_listings: list[dict[str, Any]], + source_name: str, + existing_hashes: set[str] | None = None, +) -> dict[str, Any]: + """Process a batch of raw listings through dedup (no persistence). + + Each listing is: + 1. Normalised via DiscoverySanitizer.transform_input() + 2. Checked for address_hash collision against existing_hashes + 3. If duplicate → counted and skipped + 4. If new → canonical payload collected, hash added + + Args: + raw_listings: List of raw property data dicts from an external + source (MLS, county records, wholesale JSON, etc.). + source_name: Human-readable source label (e.g. "mls_feed"). + existing_hashes: Set of SHA-256 address hashes already known. + A fresh set is used if not provided. + + Returns: + Analytics dict: + total_received (int): Raw count of input records. + new_assets_discovered (int): New (non-duplicate) records. + duplicates_skipped (int): Records rejected by hash match. + failed_records (int): Records that raised during parsing. + payloads (list): CanonicalPropertyPayload list. + """ + # Mutate the caller's set in place when provided (prei DiscoveryProcessor + # contract: existing_hashes is updated after each batch so callers can + # share one set across sources). + hashes = existing_hashes if existing_hashes is not None else set() + new_payloads: list[CanonicalPropertyPayload] = [] + duplicates_count = 0 + errors_count = 0 + + for raw in raw_listings: + try: + canonical = DiscoverySanitizer.transform_input(raw, source_name) + if canonical.address_hash in hashes: + duplicates_count += 1 + continue + new_payloads.append(canonical) + hashes.add(canonical.address_hash) + except Exception: + errors_count += 1 + continue + + return { + "total_received": len(raw_listings), + "new_assets_discovered": len(new_payloads), + "duplicates_skipped": duplicates_count, + "failed_records": errors_count, + "payloads": new_payloads, + } + + +# ── Django persistence path ─────────────────────────────────────────────────── + + +def process_discovery( + payload: CanonicalPropertyPayload, + user: User, + source_type: str, +) -> PipelineProperty: + """Persist a canonical payload as a PipelineProperty at DISCOVERED stage. + + Deduplicates by address_hash across the user's existing pipeline + properties of the same source_type. If a property with the same + address_hash already exists, it is returned unchanged (created=False + semantics — caller checks ``created`` on the returned object via the + PipelineProperty.created flag if needed). + + Args: + payload: CanonicalPropertyPayload from DiscoverySanitizer. + user: Django User who owns this pipeline entry. + source_type: PipelineProperty.SourceType value. + + Returns: + PipelineProperty instance (created or existing). + """ + from core.models import PipelineProperty + + existing = ( + PipelineProperty.objects.filter( + user=user, + source_type=source_type, + address_hash=payload.address_hash, + ) + .order_by("-updated_at") + .first() + ) + if existing is not None: + return existing + + return PipelineProperty.objects.create( + user=user, + source_type=source_type, + source_id=payload.source_id, + address=payload.raw_address, + address_hash=payload.address_hash, + stage=PipelineProperty.Stage.DISCOVERED, + status=PipelineProperty.Status.ACTIVE, + price=Decimal(str(payload.price)) if payload.price is not None else None, + estimated_rent=Decimal(str(payload.estimated_rent)) + if payload.estimated_rent is not None + else None, + beds=payload.beds or None, + baths=payload.baths or None, + sqft=payload.sqft, + year_built=payload.year_built, + discovered_at=timezone.now(), + ) diff --git a/prei/integrations/landlord_data.py b/core/services/landlord_data.py similarity index 97% rename from prei/integrations/landlord_data.py rename to core/services/landlord_data.py index 7459161b..b88db56c 100644 --- a/prei/integrations/landlord_data.py +++ b/core/services/landlord_data.py @@ -1,6 +1,7 @@ """State landlord-friendliness scoring for the Growth Area Explorer. -Scores are on a 0-10 scale: +Moved from prei/integrations/landlord_data.py during the pydantic→Django +consolidation. Scores are on a 0-10 scale: 0-3 = Tenant-Friendly (red) 4-6 = Neutral (yellow) 7-10 = Landlord-Friendly (green) diff --git a/core/services/offer.py b/core/services/offer.py new file mode 100644 index 00000000..4112bdae --- /dev/null +++ b/core/services/offer.py @@ -0,0 +1,140 @@ +"""Offer stage handler — offer price optimization and strategy. + +Ported from prei.pipeline.handlers.offer (pydantic removed, floats replaced +with Decimal — resolves LIMIT-21). Computes the optimal offer price for a +property based on underwriting results, market conditions, and investment +strategy parameters. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from decimal import Decimal +from enum import Enum + +from investor_app.finance.utils import to_decimal + + +class OfferStrategy(str, Enum): + """Offer pricing strategy variants.""" + + CONSERVATIVE = "conservative" # Offer below MAO (buffer for negotiation) + TARGET = "target" # Offer at MAO + AGGRESSIVE = "aggressive" # Offer above MAO (competitive market) + + +@dataclass +class OfferInput: + """Input parameters for the offer solver. + + All monetary values are Decimal dollars. desired_equity and + competition_multiplier are fractions. + """ + + mao: Decimal + arv: Decimal | None = None + rehab_budget: Decimal = Decimal("0") + desired_equity: Decimal = Decimal("0.0") + competition_multiplier: Decimal = Decimal("1.0") + + def __post_init__(self) -> None: + self.mao = to_decimal(self.mao) + if self.mao <= 0: + raise ValueError("mao must be > 0") + if self.arv is not None: + self.arv = to_decimal(self.arv) + self.rehab_budget = to_decimal(self.rehab_budget) + if self.rehab_budget < 0: + raise ValueError("rehab_budget must be >= 0") + self.desired_equity = to_decimal(self.desired_equity) + if not Decimal("0") <= self.desired_equity <= Decimal("1"): + raise ValueError("desired_equity must be in [0, 1]") + self.competition_multiplier = to_decimal(self.competition_multiplier) + if not Decimal("0.5") <= self.competition_multiplier <= Decimal("2.0"): + raise ValueError("competition_multiplier must be in [0.5, 2.0]") + + +@dataclass +class OfferMetrics: + """Output metrics from the offer solver.""" + + offer_price: Decimal + strategy: OfferStrategy + premium_over_mao: Decimal + premium_pct: Decimal + estimated_equity: Decimal | None = None + estimated_equity_pct: Decimal | None = None + + +def solve_offer( + inputs: OfferInput, + strategy: OfferStrategy = OfferStrategy.TARGET, +) -> OfferMetrics: + """Compute the optimal offer price based on strategy. + + Strategy rules: + CONSERVATIVE: offer = MAO × 0.95 × competition_multiplier + TARGET: offer = MAO × competition_multiplier + AGGRESSIVE: offer = MAO × 1.05 × competition_multiplier + + All strategies clamp the offer to ensure minimum desired equity + is maintained when ARV is known. + + Args: + inputs: OfferInput with MAO, ARV, rehab, equity target. + strategy: Pricing strategy enum. + + Returns: + OfferMetrics with offer price and equity analysis. + """ + # ── Base offer by strategy ──────────────────────────────────────────────── + if strategy == OfferStrategy.CONSERVATIVE: + raw_offer = inputs.mao * Decimal("0.95") + elif strategy == OfferStrategy.AGGRESSIVE: + raw_offer = inputs.mao * Decimal("1.05") + else: + raw_offer = inputs.mao + + offer_price = raw_offer * inputs.competition_multiplier + + # ── Equity constraint (when ARV is known) ──────────────────────────────── + estimated_equity: Decimal | None = None + estimated_equity_pct: Decimal | None = None + + if inputs.arv is not None and inputs.arv > 0: + total_cost = offer_price + inputs.rehab_budget + estimated_equity = inputs.arv - total_cost + estimated_equity_pct = ( + estimated_equity / inputs.arv if inputs.arv > 0 else Decimal("0.0") + ) + + # Clamp offer to maintain minimum desired equity + if inputs.desired_equity > 0: + max_offer_for_equity = ( + inputs.arv * (Decimal("1") - inputs.desired_equity) + - inputs.rehab_budget + ) + if max_offer_for_equity < offer_price: + offer_price = max_offer_for_equity + # Recalculate with clamped price + total_cost = offer_price + inputs.rehab_budget + estimated_equity = inputs.arv - total_cost + estimated_equity_pct = ( + estimated_equity / inputs.arv if inputs.arv > 0 else Decimal("0.0") + ) + + premium = offer_price - inputs.mao + premium_pct = premium / inputs.mao if inputs.mao > 0 else Decimal("0.0") + + return OfferMetrics( + offer_price=offer_price.quantize(Decimal("0.01")), + strategy=strategy, + premium_over_mao=premium.quantize(Decimal("0.01")), + premium_pct=premium_pct.quantize(Decimal("0.000001")), + estimated_equity=estimated_equity.quantize(Decimal("0.01")) + if estimated_equity is not None + else None, + estimated_equity_pct=estimated_equity_pct.quantize(Decimal("0.0001")) + if estimated_equity_pct is not None + else None, + ) diff --git a/core/services/screening.py b/core/services/screening.py index 5fe9d197..f6ea7c26 100644 --- a/core/services/screening.py +++ b/core/services/screening.py @@ -11,7 +11,160 @@ from dataclasses import dataclass, field from decimal import Decimal -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any, Optional, Tuple + +# ── Pure screening evaluator (ported from prei.pipeline.handlers.screening) ── + + +@dataclass +class ScreeningThresholds: + """Threshold configuration for the SCREENING pipeline stage. + + All fields are required unless marked optional. Float-based because this + evaluator is transient ratio math (no persistence); the ORM persistence + path (screen_property/PipelineProperty) is Decimal-based. + """ + + min_gross_yield: float + max_price_to_rent_ratio: float + excluded_hoas: list[str] = field(default_factory=list) + min_beds: int = 0 + min_baths: int = 0 + + +def gross_yield(monthly_rent: float, purchase_price: float) -> float: + """Compute gross yield as a fraction: (monthly_rent × 12) / purchase_price. + + Returns 0.0 for non-positive price or rent. + """ + if purchase_price <= 0 or monthly_rent <= 0: + return 0.0 + return (monthly_rent * 12.0) / purchase_price + + +def price_to_rent_ratio(monthly_rent: float, purchase_price: float) -> float: + """Compute price-to-rent ratio: purchase_price / (monthly_rent × 12). + + Returns float('inf') when annual rent is zero. + """ + annual_rent = monthly_rent * 12.0 + if annual_rent <= 0: + return float("inf") + return purchase_price / annual_rent + + +def compute_screening_metrics(asset_data: dict[str, Any]) -> dict[str, float]: + """Compute gross_yield and price_to_rent_ratio from raw asset data.""" + rent = float(asset_data.get("estimated_monthly_rent", 0)) + price = float(asset_data.get("purchase_price", 0)) + return { + "gross_yield": gross_yield(rent, price), + "price_to_rent_ratio": price_to_rent_ratio(rent, price), + } + + +def evaluate_screening_stage( + asset_data: dict[str, Any], + thresholds: ScreeningThresholds, +) -> Tuple[bool, Optional[str]]: + """Evaluate a property against all screening thresholds. + + Checks run in order of lowest computational cost first; the first + violation short-circuits and returns the kill reason. + + Returns: + Tuple of (pass: bool, kill_reason: str | None). + """ + # 1. Beds check + beds = asset_data.get("beds") + if beds is not None and int(beds) < thresholds.min_beds: + return False, f"Insufficient bedrooms: {beds} < {thresholds.min_beds}" + + # 2. Baths check + baths = asset_data.get("baths") + if baths is not None and float(baths) < thresholds.min_baths: + return False, f"Insufficient bathrooms: {baths} < {thresholds.min_baths}" + + # 3. HOA exclusion check + hoa = asset_data.get("hoa_name") + if hoa and thresholds.excluded_hoas: + hoa_lower = hoa.strip().lower() + for excluded in thresholds.excluded_hoas: + if excluded.strip().lower() == hoa_lower: + return False, f"Excluded HOA: {hoa}" + + # 4. Gross yield check + rent = asset_data.get("estimated_monthly_rent") + price = asset_data.get("purchase_price") + if rent is not None and price is not None and price > 0 and float(rent) > 0: + gy = gross_yield(float(rent), float(price)) + if gy < thresholds.min_gross_yield: + return ( + False, + f"Gross yield too low: {gy:.4f} < {thresholds.min_gross_yield}", + ) + + # 5. Price-to-rent ratio check + if rent is not None and price is not None and price > 0 and float(rent) > 0: + ptr = price_to_rent_ratio(float(rent), float(price)) + if ptr > thresholds.max_price_to_rent_ratio: + return ( + False, + f"Price-to-rent ratio too high: {ptr:.2f} > " + f"{thresholds.max_price_to_rent_ratio}", + ) + + return True, None + + +def screen_batch( + property_dicts: list[dict[str, Any]], + thresholds: ScreeningThresholds, +) -> dict[str, Any]: + """Evaluate a batch of property payloads through SCREENING (stats only). + + Pure batch equivalent of the former prei BatchScreeningProcessor: returns + the same operational summary dict (processed/advanced/killed/execution_time_ms) + without engine state or persistence. + + Args: + property_dicts: List of property payload dicts with at minimum + asset_id, address, estimated_monthly_rent, purchase_price, + beds, baths keys. + thresholds: ScreeningThresholds for the evaluator. + + Returns: + Dict with processed (int), advanced (int), killed (int), + execution_time_ms (float). + """ + import time + + start = time.perf_counter() + advanced = 0 + killed = 0 + + for payload in property_dicts: + asset_data = { + "estimated_monthly_rent": payload.get("estimated_monthly_rent"), + "purchase_price": payload.get("purchase_price"), + "beds": payload.get("beds"), + "baths": payload.get("baths"), + "hoa_name": payload.get("hoa_name"), + } + passed, _ = evaluate_screening_stage(asset_data, thresholds) + if passed: + advanced += 1 + else: + killed += 1 + + elapsed_ms = (time.perf_counter() - start) * 1000 + return { + "processed": len(property_dicts), + "advanced": advanced, + "killed": killed, + "execution_time_ms": round(elapsed_ms, 2), + } + if TYPE_CHECKING: from django.contrib.auth.models import User diff --git a/prei/__init__.py b/core/services/sources/__init__.py similarity index 100% rename from prei/__init__.py rename to core/services/sources/__init__.py diff --git a/prei/pipeline/sources/base.py b/core/services/sources/base.py similarity index 100% rename from prei/pipeline/sources/base.py rename to core/services/sources/base.py diff --git a/prei/pipeline/sources/county.py b/core/services/sources/county.py similarity index 99% rename from prei/pipeline/sources/county.py rename to core/services/sources/county.py index da763cc0..069a014a 100644 --- a/prei/pipeline/sources/county.py +++ b/core/services/sources/county.py @@ -33,7 +33,7 @@ import requests -from prei.pipeline.sources.base import DiscoverySource +from core.services.sources.base import DiscoverySource logger = logging.getLogger(__name__) diff --git a/prei/pipeline/sources/file_source.py b/core/services/sources/file_source.py similarity index 99% rename from prei/pipeline/sources/file_source.py rename to core/services/sources/file_source.py index c59da129..8252438c 100644 --- a/prei/pipeline/sources/file_source.py +++ b/core/services/sources/file_source.py @@ -16,7 +16,7 @@ from pathlib import Path from typing import Any, Dict, List, Optional -from prei.pipeline.sources.base import DiscoverySource +from core.services.sources.base import DiscoverySource logger = logging.getLogger(__name__) diff --git a/prei/pipeline/sources/registry.py b/core/services/sources/registry.py similarity index 92% rename from prei/pipeline/sources/registry.py rename to core/services/sources/registry.py index 78ad4a26..7555c61b 100644 --- a/prei/pipeline/sources/registry.py +++ b/core/services/sources/registry.py @@ -8,15 +8,15 @@ from typing import Any, Dict, List, Optional, Type -from prei.pipeline.sources.base import DiscoverySource -from prei.pipeline.sources.county import TexasCountyForeclosureSource -from prei.pipeline.sources.reo_sources import ( +from core.services.sources.base import DiscoverySource +from core.services.sources.county import TexasCountyForeclosureSource +from core.services.sources.reo_sources import ( FannieMaeSource, HUDHomestoreSource, USDAForeclosuresSource, VAForeclosuresSource, ) -from prei.pipeline.sources.vrm_source import VrmDiscoverySource +from core.services.sources.vrm_source import VrmDiscoverySource # ── Built-in source registry ────────────────────────────────────────────────── diff --git a/prei/pipeline/sources/reo_sources.py b/core/services/sources/reo_sources.py similarity index 98% rename from prei/pipeline/sources/reo_sources.py rename to core/services/sources/reo_sources.py index 128e4c73..3fd3c233 100644 --- a/prei/pipeline/sources/reo_sources.py +++ b/core/services/sources/reo_sources.py @@ -15,8 +15,8 @@ import requests -from prei.pipeline.handlers.discovery import DiscoverySanitizer -from prei.pipeline.sources.base import DiscoverySource +from core.services.discovery import DiscoverySanitizer +from core.services.sources.base import DiscoverySource logger = logging.getLogger(__name__) diff --git a/prei/pipeline/sources/vrm_source.py b/core/services/sources/vrm_source.py similarity index 98% rename from prei/pipeline/sources/vrm_source.py rename to core/services/sources/vrm_source.py index 610a1517..5d7f02fb 100644 --- a/prei/pipeline/sources/vrm_source.py +++ b/core/services/sources/vrm_source.py @@ -7,7 +7,7 @@ import logging from typing import Any, Dict, List, Optional -from prei.pipeline.sources.base import DiscoverySource +from core.services.sources.base import DiscoverySource logger = logging.getLogger(__name__) diff --git a/prei/pipeline/handlers/underwriting.py b/core/services/underwriting.py similarity index 91% rename from prei/pipeline/handlers/underwriting.py rename to core/services/underwriting.py index 58ebe363..e02e3543 100644 --- a/prei/pipeline/handlers/underwriting.py +++ b/core/services/underwriting.py @@ -1,5 +1,7 @@ """Financial underwriting solver engine for the property pipeline. +Ported from prei.pipeline.handlers.underwriting (pydantic removed — +BaseModel replaced with dataclasses; all monetary values remain Decimal). Computes institutional performance indicators: NOI, Cap Rate, Cash-on-Cash Yield, and Max Allowable Offer (MAO). Includes an optimization solver that backsolves for purchase price given a target cap rate. @@ -7,16 +9,14 @@ from __future__ import annotations +from dataclasses import dataclass, field from decimal import Decimal -from pydantic import BaseModel - from investor_app.finance.utils import cap_rate, cash_on_cash, to_decimal -# ── Data models ─────────────────────────────────────────────────────────────── - -class UnderwritingInput(BaseModel): +@dataclass +class UnderwritingInput: """Input parameters for the underwriting solver. All monetary values are Decimal dollars. Rate fields are fractions. @@ -24,16 +24,17 @@ class UnderwritingInput(BaseModel): purchase_price: Decimal estimated_rent: Decimal - vacancy_rate: Decimal = Decimal("0.05") # Default 5% - rehab_budget: Decimal = Decimal("0") property_tax_annual: Decimal insurance_annual: Decimal - maintenance_reserve_rate: Decimal = Decimal("0.10") # 10% of gross rent - management_fee_rate: Decimal = Decimal("0.08") # 8% of EGI - hoa_annual: Decimal = Decimal("0") + vacancy_rate: Decimal = field(default=Decimal("0.05")) # Default 5% + rehab_budget: Decimal = field(default=Decimal("0")) + maintenance_reserve_rate: Decimal = field(default=Decimal("0.10")) # 10% of GPR + management_fee_rate: Decimal = field(default=Decimal("0.08")) # 8% of EGI + hoa_annual: Decimal = field(default=Decimal("0")) -class UnderwritingMetrics(BaseModel): +@dataclass +class UnderwritingMetrics: """Output metrics from the underwriting solver.""" noi: Decimal diff --git a/core/tests/test_pipeline_bridge.py b/core/tests/test_pipeline_bridge.py index a51f6b95..4e516682 100644 --- a/core/tests/test_pipeline_bridge.py +++ b/core/tests/test_pipeline_bridge.py @@ -20,7 +20,7 @@ class TestPipelineBridge: @patch("core.views.FREDAdapter.fetch_state_employment_growth") @patch("core.views.fetch_place_growth_metrics") @patch("core.views.fetch_housing_demand_index") - @patch("prei.pipeline.sources.registry.discover_from_all") + @patch("core.services.sources.registry.discover_from_all") @pytest.mark.django_db def test_pipeline_button_triggers_discovery( self, diff --git a/core/views/__init__.py b/core/views/__init__.py index 8a639ba1..756d7589 100644 --- a/core/views/__init__.py +++ b/core/views/__init__.py @@ -47,7 +47,7 @@ UserScreeningPreferences, ) -from prei.integrations.landlord_data import get_state_landlord_score +from core.services.landlord_data import get_state_landlord_score from investor_app.finance.utils import ( compute_analysis_for_property, calculate_whatif_monthly_cashflow, @@ -1232,11 +1232,9 @@ def _fetch_place_data(place: dict) -> dict | None: pipeline_results = None pipeline_city = request.POST.get("pipeline_city", "").strip() if pipeline_city: - from prei.pipeline.handlers.discovery_processor import DiscoveryProcessor - from prei.pipeline.handlers.screening import ScreeningThresholds - from prei.pipeline.handlers.batch_screening import BatchScreeningProcessor - from prei.pipeline.engine import InMemoryAssetRepository, PipelineEngine - from prei.pipeline.sources.registry import discover_from_all + from core.services.discovery_processor import process_discovery_batch + from core.services.screening import ScreeningThresholds, screen_batch + from core.services.sources.registry import discover_from_all logger.info( "Growth Explorer: running pipeline discovery for %s, %s", @@ -1253,8 +1251,7 @@ def _fetch_place_data(place: dict) -> dict | None: all_listings.append(listing) # Run through discovery processor (dedup + state inception) - processor = DiscoveryProcessor(existing_hashes=set()) - discovery_result = processor.process_batch( + discovery_result = process_discovery_batch( all_listings, source_name="growth_explorer" ) @@ -1279,9 +1276,7 @@ def _fetch_place_data(place: dict) -> dict | None: min_beds=min_beds, min_baths=min_baths, ) - engine = PipelineEngine(repository=InMemoryAssetRepository()) - batch_processor = BatchScreeningProcessor(engine, thresholds) - screening_result = batch_processor.process(all_listings) + screening_result = screen_batch(all_listings, thresholds) pipeline_results = { "city": pipeline_city, @@ -3224,9 +3219,7 @@ def vrm_properties_list(request: HttpRequest) -> HttpResponse: # Handle pipeline request: run selected properties through discovery if request.method == "POST" and "run_pipeline" in request.POST: - from prei.pipeline.engine import InMemoryAssetRepository, PipelineEngine - from prei.pipeline.handlers.screening import ScreeningThresholds - from prei.pipeline.handlers.batch_screening import BatchScreeningProcessor + from core.services.screening import ScreeningThresholds, screen_batch prop_ids = request.POST.getlist("pipeline_props") if prop_ids: @@ -3252,9 +3245,7 @@ def vrm_properties_list(request: HttpRequest) -> HttpResponse: min_beds=1, min_baths=1, ) - engine = PipelineEngine(repository=InMemoryAssetRepository()) - processor = BatchScreeningProcessor(engine, thresholds) - result = processor.process(payloads) + result = screen_batch(payloads, thresholds) pipeline_message = ( f"{len(payloads)} properties processed: " f"{result['advanced']} passed screening, " diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index b8e00ee5..3be6c345 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -47,6 +47,25 @@ core/forms.py → Django form classes with styled widget base classes. CSS classes from the design system. ``` +## Pipeline Stage Services + +Acquisition-pipeline business logic lives in `core/services/` (Django-canonical +since the pydantic→Django consolidation removed the standalone `prei/` package): + +| Stage | Service module | +|-------|----------------| +| Discovery | `core/services/discovery.py` (sanitizer/`CanonicalPropertyPayload`), `core/services/discovery_processor.py` (`process_discovery_batch` stats pass + `process_discovery` ORM write at DISCOVERED) | +| Sources | `core/services/sources/` — registry, county, REO (Fannie/HUD/VA/USDA), VRM, file adapters | +| Screening | `core/services/screening.py` — pure evaluator (`ScreeningThresholds`, `evaluate_screening_stage`, `screen_batch`) + ORM `screen_property` | +| Underwriting | `core/services/underwriting.py` — `UnderwritingInput`/`solve_underwriting` (Decimal) | +| Offer | `core/services/offer.py` — `OfferInput`/`solve_offer` (Decimal, LIMIT-21 resolved) | +| Landlord score | `core/services/landlord_data.py` (moved from `prei/integrations/`) | + +View bridges (`core/views/__init__.py` Growth Explorer + VRM pipeline runs) call +the pure services directly and are stats-only — they never persist engine state. +Persistence goes through `PipelineProperty` (`core/models/pipeline.py`) via +`process_discovery` and the stage-advance helpers in `core/services/pipeline.py`. + ## Dependency Diagram ``` diff --git a/docs/CHANGE_IMPACT_MAP.md b/docs/CHANGE_IMPACT_MAP.md index 76c1931f..05562739 100644 --- a/docs/CHANGE_IMPACT_MAP.md +++ b/docs/CHANGE_IMPACT_MAP.md @@ -21,6 +21,11 @@ |`cma.py` function signatures |`core/api_views.py` calls, `docs/API_SURFACE.md` | |`market_data.refresh_market_snapshot` logic |`core/tests/test_neighborhood_insights.py`, `docs/API_SURFACE.md` | |`market_data.py` adapter imports |`core/integrations/market/` (comps, rents, crime, schools), `core/tests/test_neighborhood_insights.py`| +|`discovery.py` / `discovery_processor.py` |`core/views/__init__.py` (Growth Explorer + VRM pipeline bridges), `core/services/sources/`, `tests/test_discovery*.py`| +|`screening.py` (pure evaluator) |`core/views/__init__.py` bridges, `tests/test_pipeline.py`, `tests/test_screening_integration.py`, `tests_bdd/steps/pipeline_steps.py`| +|`underwriting.py` signatures |`core/services/offer.py`, `tests/test_underwriting_integration.py`, `tests_bdd/steps/pipeline_steps.py`| +|`offer.py` signatures |`tests/test_offer_integration.py`, `tests/test_pipeline_e2e.py`, `docs/KNOWN_LIMITATIONS.md` (LIMIT-21)| +|`sources/` adapters |`core/tests/test_county/`, `core/tests/test_ingestion/`, `core/tests/test_hud_source_index.py`| ## Finance Utils Changes (investor_app/finance/utils.py) diff --git a/docs/KNOWN_LIMITATIONS.md b/docs/KNOWN_LIMITATIONS.md index 60361246..7fb9aa1b 100644 --- a/docs/KNOWN_LIMITATIONS.md +++ b/docs/KNOWN_LIMITATIONS.md @@ -224,15 +224,11 @@ This means a user who runs the API pre-`populate_growth_areas` gets empty result --- -### [LIMIT-21] 🟡 HIGH — `prei/pipeline/handlers/offer.py` remains float-based currency +### [LIMIT-21] 🟡 HIGH — `prei/pipeline/handlers/offer.py` remains float-based currency (resolved) -**Location:** `prei/pipeline/handlers/offer.py` — `OfferInput`, `OfferMetrics`, `solve_offer()` and its pricing-strategy multiplier/equity arithmetic. +**Location:** Resolved in the pydantic→Django consolidation — `prei/pipeline/handlers/offer.py` was deleted and the offer solver was ported to `core/services/offer.py` (`OfferInput`/`OfferMetrics`/`solve_offer`), with all monetary values converted to `Decimal` and validation moved to `OfferInput.__post_init__` (raises `ValueError` on non-positive MAO, negative rehab budget, or out-of-range equity/competition factors). Downstream callers migrated: `tests/test_offer_integration.py`, `tests/test_pipeline_e2e.py`, `tests_bdd/` now assert Decimal equality. -**Impact:** `OfferInput.mao`/`arv`/`rehab_budget` and `OfferMetrics.offer_price`/`estimated_equity` are `float`-typed, so all offer-price and equity math (strategy multipliers, premium calculations, equity clamping) is done in binary floating point rather than `Decimal`. This is the same class of currency-precision issue as the one fixed in `prei/pipeline/handlers/underwriting.py` during Phase B (docs/TOP_01_PLAN.md) — `AGENTS.md` "Never Do" item 3 ("Float persistence for currency") — but was deliberately left out of that PR's scope since the user approved only the underwriting.py fix, and converting `offer.py` also touches its downstream callers (`tests/test_offer_integration.py`, `tests/test_pipeline_e2e.py`, `tests_bdd/`) in ways that deserve their own reviewed change. - -**Workaround:** None — offer-price rounding/precision errors from float arithmetic are small in absolute terms (cents-level) for typical property values, so this is not currently causing observable defects, but the pattern should not be extended. - -**Fix tracked in:** Not yet filed. Recommended as a follow-up PR using the same `Decimal`/`to_decimal()` conversion approach applied to `underwriting.py`. +**Fix tracked in:** Resolved by the pydantic→Django consolidation (offer port to `core/services/offer.py`). --- diff --git a/prei/api/__init__.py b/prei/api/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/prei/api/pipeline_routes.py b/prei/api/pipeline_routes.py deleted file mode 100644 index cd8425c1..00000000 --- a/prei/api/pipeline_routes.py +++ /dev/null @@ -1,209 +0,0 @@ -"""REST API endpoints for the property pipeline engine. - -Exposes pipeline summary statistics and manual transition controls -via FastAPI. Designed to run as a standalone microservice decoupled -from the Django application layer. -""" - -from __future__ import annotations - -from typing import Any, Dict, List, Optional - -from fastapi import APIRouter, HTTPException - -from prei.models.pipeline import ( - InvalidStageTransitionException, - PipelineStage, - PropertyAsset, -) -from prei.pipeline.engine import ( - AssetRepository, - InMemoryAssetRepository, - PipelineEngine, - StateAggregator, -) - -router = APIRouter(prefix="/api/v1/pipeline", tags=["pipeline"]) - - -# ── Dependency injection ────────────────────────────────────────────────────── -# In production, replace with a properly configured SqliteAssetRepository -# or Django-model-backed repository. - -_repository: Optional[AssetRepository] = None -_engine: Optional[PipelineEngine] = None - - -def get_repository() -> AssetRepository: - """Return the shared repository instance.""" - global _repository - if _repository is None: - _repository = InMemoryAssetRepository() - return _repository - - -def get_engine() -> PipelineEngine: - """Return the shared pipeline engine instance.""" - global _engine - if _engine is None: - _engine = PipelineEngine(repository=get_repository()) - return _engine - - -def configure_repository(repo: AssetRepository) -> None: - """Override the default repository (used by tests).""" - global _repository, _engine - _repository = repo - _engine = PipelineEngine(repository=repo) - - -# ── Helper ──────────────────────────────────────────────────────────────────── - - -def _asset_to_dict(asset: PropertyAsset) -> Dict[str, Any]: - """Serialize a PropertyAsset to a JSON-safe dict.""" - return { - "asset_id": asset.asset_id, - "address": asset.address, - "current_stage": asset.current_stage.value, - "stage_history": [ - { - "stage": log.stage.value, - "entered_at": log.entered_at.isoformat() if log.entered_at else None, - "exited_at": log.exited_at.isoformat() if log.exited_at else None, - "reason": log.reason, - } - for log in asset.stage_history - ], - "kill_reason": asset.kill_reason, - } - - -# ═══════════════════════════════════════════════════════════════════════════════ -# Endpoints -# ═══════════════════════════════════════════════════════════════════════════════ - - -@router.get("/summary") -def get_pipeline_summary() -> Dict[str, Any]: - """Return aggregate pipeline statistics. - - Returns asset counts grouped by pipeline stage plus a high-level - pipeline_flow breakdown and total/killed counts. - """ - repo = get_repository() - aggregator = StateAggregator(repo) - return aggregator.summary() - - -@router.get("/assets") -def list_assets() -> List[Dict[str, Any]]: - """Return all assets with their current stage and history.""" - repo = get_repository() - return [_asset_to_dict(a) for a in repo.list_all()] - - -@router.get("/assets/{asset_id}") -def get_asset(asset_id: str) -> Dict[str, Any]: - """Return a single asset by ID.""" - repo = get_repository() - asset = repo.load(asset_id) - if asset is None: - raise HTTPException(status_code=404, detail=f"Asset {asset_id} not found") - return _asset_to_dict(asset) - - -@router.post("/assets") -def create_asset(payload: Dict[str, Any]) -> Dict[str, Any]: - """Create a new asset at GACS stage. - - Request body: - asset_id (str, required): Unique identifier. - address (str, required): Property address. - """ - asset_id = payload.get("asset_id") - address = payload.get("address") - if not asset_id or not address: - raise HTTPException( - status_code=422, - detail="Both 'asset_id' and 'address' are required", - ) - repo = get_repository() - existing = repo.load(asset_id) - if existing is not None: - raise HTTPException( - status_code=409, - detail=f"Asset {asset_id} already exists", - ) - asset = PropertyAsset(asset_id=asset_id, address=address) - repo.save(asset) - return _asset_to_dict(asset) - - -@router.post("/transition") -def force_transition(payload: Dict[str, Any]) -> Dict[str, Any]: - """Force a manual state transition for an asset. - - Request body: - asset_id (str, required): The asset to transition. - target_stage (str, required): Target pipeline stage name. - reason (str, optional): Human-readable explanation. - context (dict, optional): Additional metrics context. - - Returns the updated asset state. - - Raises 422 if the transition violates the state machine ruleset. - """ - asset_id = payload.get("asset_id") - target = payload.get("target_stage") - reason = payload.get("reason") - context = payload.get("context", {}) - - if not asset_id or not target: - raise HTTPException( - status_code=422, - detail="Both 'asset_id' and 'target_stage' are required", - ) - - repo = get_repository() - asset = repo.load(asset_id) - if asset is None: - raise HTTPException( - status_code=404, - detail=f"Asset {asset_id} not found", - ) - - try: - target_stage = PipelineStage(target) - except ValueError: - valid = [s.value for s in PipelineStage] - raise HTTPException( - status_code=422, - detail=f"Invalid stage '{target}'. Valid values: {valid}", - ) - - ctx: Dict[str, Any] = {**context} - if reason: - ctx["reason"] = reason - - engine = get_engine() - try: - updated = engine.process_transition(asset, target_stage, ctx) - except InvalidStageTransitionException as exc: - raise HTTPException(status_code=422, detail=str(exc)) - - return _asset_to_dict(updated) - - -@router.delete("/assets/{asset_id}") -def delete_asset(asset_id: str) -> Dict[str, str]: - """Remove an asset from the repository entirely.""" - # Note: InMemoryAssetRepository doesn't support delete. - # For now, we transition the asset to KILLED as a soft delete. - repo = get_repository() - asset = repo.load(asset_id) - if asset is None: - raise HTTPException(status_code=404, detail=f"Asset {asset_id} not found") - engine = get_engine() - engine.process_transition(asset, PipelineStage.KILLED, {"reason": "API delete"}) - return {"status": "killed", "asset_id": asset_id} diff --git a/prei/cli.py b/prei/cli.py deleted file mode 100644 index 6e4d9b1b..00000000 --- a/prei/cli.py +++ /dev/null @@ -1,189 +0,0 @@ -"""CLI tool for the property pipeline. - -Usage: - prei-cli pipeline summary - prei-cli pipeline ingest --source mls_feed.json --run-screening - prei-cli pipeline transition --asset-id ASSET-001 --target UNDERWRITING -""" - -from __future__ import annotations - -import json -import sys - -import click - -from prei.models.pipeline import InvalidStageTransitionException, PipelineStage -from prei.pipeline.engine import ( - InMemoryAssetRepository, - PipelineEngine, - StateAggregator, -) -from prei.pipeline.handlers.batch_screening import BatchScreeningProcessor -from prei.pipeline.handlers.screening import ScreeningThresholds - - -# ── Shared engine factory ───────────────────────────────────────────────────── - - -def _make_engine() -> PipelineEngine: - """Create a fresh engine with in-memory repository.""" - return PipelineEngine(repository=InMemoryAssetRepository()) - - -# ═══════════════════════════════════════════════════════════════════════════════ -# CLI group -# ═══════════════════════════════════════════════════════════════════════════════ - - -@click.group() -def cli() -> None: - """PREI pipeline management CLI.""" - - -# ═══════════════════════════════════════════════════════════════════════════════ -# pipeline summary -# ═══════════════════════════════════════════════════════════════════════════════ - - -@cli.group() -def pipeline() -> None: - """Pipeline state machine commands.""" - - -@pipeline.command() -def summary() -> None: - """Print aggregate pipeline stats.""" - engine = _make_engine() - aggregator = StateAggregator(engine.repository) - data = aggregator.summary() - click.echo(json.dumps(data, indent=2)) - - -# ═══════════════════════════════════════════════════════════════════════════════ -# pipeline ingest -# ═══════════════════════════════════════════════════════════════════════════════ - - -@pipeline.command() -@click.option( - "--source", - "-s", - required=True, - type=click.Path(exists=True), - help="Path to JSON file with property payloads", -) -@click.option( - "--run-screening", - is_flag=True, - default=False, - help="Run screening evaluation after ingest", -) -@click.option( - "--min-yield", - default=0.07, - type=float, - help="Minimum gross yield threshold (default 0.07)", -) -@click.option( - "--max-ptr", - default=15.0, - type=float, - help="Maximum price-to-rent ratio (default 15.0)", -) -def ingest(source: str, run_screening: bool, min_yield: float, max_ptr: float) -> None: - """Ingest properties from a JSON file and optionally run screening.""" - with open(source, "r") as f: - data = json.load(f) - - # Support both list and {"properties": [...]} formats - if isinstance(data, dict): - payloads = data.get("properties", data.get("assets", [])) - else: - payloads = data - - if not payloads: - click.echo("No properties found in source file.", err=True) - sys.exit(1) - - engine = _make_engine() - click.echo(f"Ingested {len(payloads)} properties from {source}") - - # Save each property as an asset - for p in payloads: - asset_id = p.get("asset_id", p.get("id", f"auto-{hash(str(p))}")) - address = p.get("address", "Unknown") - engine.repository.save( - __import__( - "prei.models.pipeline", fromlist=["PropertyAsset"] - ).PropertyAsset(asset_id=asset_id, address=address) - ) - - if run_screening: - thresholds = ScreeningThresholds( - min_gross_yield=min_yield, - max_price_to_rent_ratio=max_ptr, - min_beds=p["beds"] if "beds" in (p for p in payloads) else 1, - min_baths=1, - ) - processor = BatchScreeningProcessor(engine, thresholds) - result = processor.process(payloads) - click.echo( - f"Screening: {result['processed']} processed, " - f"{result['advanced']} advanced, " - f"{result['killed']} killed " - f"in {result['execution_time_ms']:.1f}ms" - ) - - -# ═══════════════════════════════════════════════════════════════════════════════ -# pipeline transition -# ═══════════════════════════════════════════════════════════════════════════════ - - -@pipeline.command() -@click.option("--asset-id", "-a", required=True, help="Asset identifier") -@click.option("--target", "-t", required=True, help="Target pipeline stage") -@click.option("--reason", "-r", default=None, help="Transition reason") -def transition(asset_id: str, target: str, reason: str | None) -> None: - """Force a manual stage transition for an asset.""" - engine = _make_engine() - - asset = engine.repository.load(asset_id) - if asset is None: - click.echo(f"Error: Asset '{asset_id}' not found.", err=True) - sys.exit(1) - - try: - target_stage = PipelineStage(target.upper()) - except ValueError: - valid = [s.value for s in PipelineStage] - click.echo( - f"Invalid stage '{target}'. Valid values: {valid}", - err=True, - ) - sys.exit(1) - - try: - updated = engine.process_transition( - asset, - target_stage, - {"reason": reason or f"CLI transition to {target_stage.value}"}, - ) - except InvalidStageTransitionException as exc: - click.echo(f"Transition failed: {exc}", err=True) - sys.exit(1) - - click.echo( - f"Asset {asset_id}: {updated.current_stage.value} " - f"(history: {len(updated.stage_history)} transitions)" - ) - - -# ═══════════════════════════════════════════════════════════════════════════════ -# Entry point -# ═══════════════════════════════════════════════════════════════════════════════ - - -if __name__ == "__main__": - cli() diff --git a/prei/models/__init__.py b/prei/models/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/prei/models/pipeline.py b/prei/models/pipeline.py deleted file mode 100644 index 867f8084..00000000 --- a/prei/models/pipeline.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Pipeline state machine for property asset lifecycle. - -Defines the immutable stage hierarchy, historical stage log structures, -and an explicit transition state manager using pydantic v2 and Python enums. - -Pipeline stages (11 values): - GACS → DISCOVERY → SCREENING → UNDERWRITING → OFFER → DUE_DILIGENCE → - CLOSING → TURNOVER → LEASING ↔ PORTFOLIO - -Any stage can transition to KILLED (terminal). No asset can exit KILLED. -""" - -from __future__ import annotations - -from datetime import datetime, timezone -from enum import Enum -from typing import Any, Dict, List, Optional - -from pydantic import BaseModel, Field - - -class PipelineStage(str, Enum): - """Exact 11-stage pipeline lifecycle for a property asset.""" - - GACS = "GACS" - DISCOVERY = "DISCOVERY" - SCREENING = "SCREENING" - UNDERWRITING = "UNDERWRITING" - OFFER = "OFFER" - DUE_DILIGENCE = "DUE_DILIGENCE" - CLOSING = "CLOSING" - TURNOVER = "TURNOVER" - LEASING = "LEASING" - PORTFOLIO = "PORTFOLIO" - KILLED = "KILLED" - - -class InvalidStageTransitionException(Exception): - """Raised when an illegal stage transition is attempted.""" - - pass - - -class StageLog(BaseModel): - """Immutable record of a single stage occupancy period.""" - - stage: PipelineStage - entered_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) - exited_at: Optional[datetime] = None - reason: Optional[str] = None - metrics_snapshot: Dict[str, Any] = Field(default_factory=dict) - - -class PropertyAsset(BaseModel): - """A property asset tracked through the pipeline state machine. - - Attributes: - asset_id: Unique identifier for the asset. - address: Property street address. - current_stage: The asset's current pipeline stage. - stage_history: Ordered list of StageLog entries (most recent last). - kill_reason: Populated only when current_stage == KILLED. - """ - - asset_id: str - address: str - current_stage: PipelineStage = PipelineStage.GACS - stage_history: List[StageLog] = Field(default_factory=list) - kill_reason: Optional[str] = None - - # ── Static transition ruleset ────────────────────────────────────────── - # Maps each stage to its allowed next stages. - # KILLED is explicitly included where allowed and is terminal (no outgoing). - ALLOWED_TRANSITIONS: Dict[PipelineStage, List[PipelineStage]] = { - PipelineStage.GACS: [PipelineStage.DISCOVERY, PipelineStage.KILLED], - PipelineStage.DISCOVERY: [PipelineStage.SCREENING, PipelineStage.KILLED], - PipelineStage.SCREENING: [PipelineStage.UNDERWRITING, PipelineStage.KILLED], - PipelineStage.UNDERWRITING: [PipelineStage.OFFER, PipelineStage.KILLED], - PipelineStage.OFFER: [PipelineStage.DUE_DILIGENCE, PipelineStage.KILLED], - PipelineStage.DUE_DILIGENCE: [PipelineStage.CLOSING, PipelineStage.KILLED], - PipelineStage.CLOSING: [PipelineStage.TURNOVER, PipelineStage.KILLED], - PipelineStage.TURNOVER: [PipelineStage.LEASING, PipelineStage.KILLED], - PipelineStage.LEASING: [PipelineStage.PORTFOLIO, PipelineStage.KILLED], - PipelineStage.PORTFOLIO: [PipelineStage.LEASING, PipelineStage.KILLED], - PipelineStage.KILLED: [], # terminal — no transitions out - } - - def transition_to( - self, - next_stage: PipelineStage, - reason: Optional[str] = None, - metrics: Optional[Dict[str, Any]] = None, - ) -> None: - """Attempt a deterministic stage transition. - - Args: - next_stage: Target pipeline stage. - reason: Optional human-readable explanation for the transition. - metrics: Optional snapshot of key metrics at transition time. - - Raises: - InvalidStageTransitionException: If the transition is not allowed - by the static ruleset. - """ - if next_stage not in self.ALLOWED_TRANSITIONS[self.current_stage]: - raise InvalidStageTransitionException( - f"Illegal transition from {self.current_stage.value} " - f"to {next_stage.value}" - ) - - now = datetime.now(timezone.utc) - - # Close the previous stage's exit timestamp - if self.stage_history: - self.stage_history[-1].exited_at = now - - # Append the new stage log entry - self.stage_history.append( - StageLog( - stage=next_stage, - entered_at=now, - reason=reason, - metrics_snapshot=metrics or {}, - ) - ) - self.current_stage = next_stage - - # Persist kill_reason when transitioning to terminal state - if next_stage == PipelineStage.KILLED: - self.kill_reason = reason diff --git a/prei/pipeline/__init__.py b/prei/pipeline/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/prei/pipeline/engine.py b/prei/pipeline/engine.py deleted file mode 100644 index cd255d5a..00000000 --- a/prei/pipeline/engine.py +++ /dev/null @@ -1,529 +0,0 @@ -"""Pipeline engine for batch-processing assets through stage transitions. - -Provides the PipelineEngine class with hook-based pre-transition evaluation, -a repository abstraction for asset persistence, SQLite-backed and transactional -repositories, and a state aggregator for UI summary statistics. -""" - -from __future__ import annotations - -import json -import logging -import sqlite3 -import threading -from abc import ABC, abstractmethod -from datetime import datetime, timezone -from typing import Any, Callable, Dict, List, Optional, cast - -from prei.models.pipeline import ( - PipelineStage, - PropertyAsset, - StageLog, -) - -logger = logging.getLogger(__name__) - -# ── Type aliases ────────────────────────────────────────────────────────────── - -PreTransitionHook = Callable[[PropertyAsset, PipelineStage, Dict[str, Any]], bool] - - -# ═══════════════════════════════════════════════════════════════════════════════ -# Asset repository abstraction -# ═══════════════════════════════════════════════════════════════════════════════ - - -class AssetRepository(ABC): - """Interface for reading and persisting PropertyAsset state. - - Subclasses may optionally override begin(), commit(), and rollback() - to participate in atomic transactions. The base implementations are - no-ops. - """ - - @abstractmethod - def load(self, asset_id: str) -> Optional[PropertyAsset]: - """Load an asset by its identifier.""" - ... - - @abstractmethod - def save(self, asset: PropertyAsset) -> None: - """Persist an asset after a transition.""" - ... - - @abstractmethod - def list_all(self) -> List[PropertyAsset]: - """Return all known assets.""" - ... - - # ── Optional transaction hooks ──────────────────────────────────────────── - # Override in subclasses that support atomic transactions. - - def begin(self) -> None: - """Begin a transaction (no-op by default).""" - - def commit(self) -> None: - """Commit the current transaction (no-op by default).""" - - def rollback(self) -> None: - """Roll back the current transaction (no-op by default).""" - - -# ═══════════════════════════════════════════════════════════════════════════════ -# In-memory repository (testing / single-process) -# ═══════════════════════════════════════════════════════════════════════════════ - - -class InMemoryAssetRepository(AssetRepository): - """In-memory implementation for testing and single-process use.""" - - def __init__(self) -> None: - self._store: Dict[str, PropertyAsset] = {} - - def load(self, asset_id: str) -> Optional[PropertyAsset]: - return self._store.get(asset_id) - - def save(self, asset: PropertyAsset) -> None: - self._store[asset.asset_id] = asset.model_copy(deep=True) - - def list_all(self) -> List[PropertyAsset]: - return [a.model_copy(deep=True) for a in self._store.values()] - - -# ═══════════════════════════════════════════════════════════════════════════════ -# Transactional repository wrapper -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TransactionError(Exception): - """Raised when a transaction operation fails.""" - - -class TransactionalRepository(AssetRepository): - """Thread-safe transactional wrapper around any AssetRepository. - - Wraps save() operations inside explicit begin/commit/rollback boundaries. - If commit() fails, all saves since the last begin() are rolled back. - - Thread-safe: uses a per-instance lock to serialize concurrent access. - """ - - def __init__(self, inner: AssetRepository) -> None: - self._inner = inner - self._lock = threading.Lock() - self._in_transaction = False - self._pending: List[PropertyAsset] = [] - - # ── Transaction lifecycle ───────────────────────────────────────────────── - - def begin(self) -> None: - """Start a new transaction. All saves are buffered until commit().""" - with self._lock: - if self._in_transaction: - raise TransactionError("Transaction already in progress") - self._in_transaction = True - self._pending.clear() - self._inner.begin() - - def commit(self) -> None: - """Flush all buffered saves to the inner repository atomically. - - If any single save fails, the entire batch is rolled back. - """ - with self._lock: - if not self._in_transaction: - raise TransactionError("No transaction in progress") - saved: List[PropertyAsset] = [] - try: - for asset in self._pending: - self._inner.save(asset) - saved.append(asset) - self._inner.commit() - self._pending.clear() - self._in_transaction = False - except Exception: - # Roll back: revert inner repository state - logger.error("Transaction commit failed — rolling back") - self._inner.rollback() - self._pending.clear() - self._in_transaction = False - raise TransactionError("Transaction commit failed, rolled back") - - def rollback(self) -> None: - """Abort the current transaction and discard all buffered saves.""" - with self._lock: - if not self._in_transaction: - raise TransactionError("No transaction in progress") - self._inner.rollback() - self._pending.clear() - self._in_transaction = False - logger.info("Transaction rolled back — %d pending saves discarded") - - # ── Repository interface ────────────────────────────────────────────────── - - def load(self, asset_id: str) -> Optional[PropertyAsset]: - with self._lock: - return self._inner.load(asset_id) - - def save(self, asset: PropertyAsset) -> None: - with self._lock: - if self._in_transaction: - # Buffer — will be flushed on commit() - self._pending.append(asset.model_copy(deep=True)) - else: - # No active transaction — save directly - self._inner.save(asset) - - def list_all(self) -> List[PropertyAsset]: - with self._lock: - return self._inner.list_all() - - -# ═══════════════════════════════════════════════════════════════════════════════ -# SQLite asset repository -# ═══════════════════════════════════════════════════════════════════════════════ - - -class SqliteAssetRepository(AssetRepository): - """SQLite-backed asset repository with full transaction support. - - Stores assets in a ``pipeline_assets`` table serialised as JSONB - (JSON text column). Thread-safe via SQLite's built-in WAL mode. - - Args: - db_path: Filesystem path to the SQLite database file. - create_tables: If True (default), initialises the schema on connect. - """ - - def __init__(self, db_path: str, create_tables: bool = True) -> None: - self._db_path = db_path - self._conn = sqlite3.connect(db_path, check_same_thread=False) - self._conn.row_factory = sqlite3.Row - self._conn.execute("PRAGMA journal_mode=WAL;") - self._lock = threading.Lock() - - if create_tables: - self._init_schema() - - def _init_schema(self) -> None: - """Create the pipeline tables if they don't exist.""" - with self._lock: - self._conn.executescript(""" - CREATE TABLE IF NOT EXISTS pipeline_assets ( - asset_id TEXT PRIMARY KEY, - address TEXT NOT NULL, - current_stage TEXT NOT NULL, - stage_history TEXT NOT NULL DEFAULT '[]', - kill_reason TEXT, - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL - ); - CREATE INDEX IF NOT EXISTS idx_assets_stage - ON pipeline_assets(current_stage); - """) - self._conn.commit() - - # ── Transaction hooks ───────────────────────────────────────────────────── - - def begin(self) -> None: - if self._conn.in_transaction: - return # already in a transaction — no-op to avoid nested begin - self._conn.execute("BEGIN TRANSACTION;") - - def commit(self) -> None: - self._conn.commit() - - def rollback(self) -> None: - self._conn.rollback() - - # ── Repository interface ────────────────────────────────────────────────── - - def load(self, asset_id: str) -> Optional[PropertyAsset]: - with self._lock: - cursor = self._conn.execute( - "SELECT * FROM pipeline_assets WHERE asset_id = ?", - (asset_id,), - ) - row = cursor.fetchone() - if row is None: - return None - return self._row_to_asset(row) - - def save(self, asset: PropertyAsset) -> None: - with self._lock: - now = datetime.now(timezone.utc).isoformat() - history_json = json.dumps( - [log.model_dump(mode="json") for log in asset.stage_history], - default=str, - ) - - self._conn.execute( - """INSERT OR REPLACE INTO pipeline_assets - (asset_id, address, current_stage, stage_history, - kill_reason, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, COALESCE( - (SELECT created_at FROM pipeline_assets - WHERE asset_id = ?), ?), ?)""", - ( - asset.asset_id, - asset.address, - asset.current_stage.value, - history_json, - asset.kill_reason, - asset.asset_id, # for COALESCE subquery - now, # created_at fallback - now, # updated_at - ), - ) - - # Flush to disk after each save outside explicit transactions - if not self._conn.in_transaction: - self._conn.commit() - - def list_all(self) -> List[PropertyAsset]: - with self._lock: - cursor = self._conn.execute( - "SELECT * FROM pipeline_assets ORDER BY updated_at DESC" - ) - return [self._row_to_asset(row) for row in cursor.fetchall()] - - # ── Internal helpers ────────────────────────────────────────────────────── - - @staticmethod - def _row_to_asset(row: sqlite3.Row) -> PropertyAsset: - """Deserialise a database row into a PropertyAsset.""" - history_data = json.loads(row["stage_history"]) if row["stage_history"] else [] - stage_history = ( - [StageLog(**log) for log in history_data] if history_data else [] - ) - - return PropertyAsset( - asset_id=row["asset_id"], - address=row["address"], - current_stage=PipelineStage(row["current_stage"]), - stage_history=stage_history, - kill_reason=row["kill_reason"], - ) - - def close(self) -> None: - """Close the database connection.""" - self._conn.close() - - -# ═══════════════════════════════════════════════════════════════════════════════ -# State aggregator -# ═══════════════════════════════════════════════════════════════════════════════ - - -class StateAggregator: - """Compute summary statistics about asset pipeline distribution. - - Exposes counts and aggregates by pipeline stage for UI dashboards. - - Args: - repository: An AssetRepository to query for asset data. - """ - - def __init__(self, repository: AssetRepository) -> None: - self.repository = repository - - def count_by_stage(self) -> Dict[str, int]: - """Return a mapping of stage name → asset count.""" - assets = self.repository.list_all() - counts: Dict[str, int] = {} - for a in assets: - stage = a.current_stage.value - counts[stage] = counts.get(stage, 0) + 1 - return counts - - def summary(self) -> Dict[str, Any]: - """Return a full pipeline summary dict. - - Keys: - total_assets (int): Total number of assets tracked. - by_stage (dict): Stage name → count (only non-zero stages). - pipeline_flow (dict): High-level pipeline phase counts. - killed (int): Number of killed assets. - """ - assets = self.repository.list_all() - by_stage: Dict[str, int] = {} - killed = 0 - for a in assets: - stage = a.current_stage.value - by_stage[stage] = by_stage.get(stage, 0) + 1 - if stage == PipelineStage.KILLED.value: - killed += 1 - - # High-level pipeline phases - acquisition = sum( - by_stage.get(s.value, 0) - for s in [ - PipelineStage.GACS, - PipelineStage.DISCOVERY, - PipelineStage.SCREENING, - ] - ) - deal_making = sum( - by_stage.get(s.value, 0) - for s in [ - PipelineStage.UNDERWRITING, - PipelineStage.OFFER, - PipelineStage.DUE_DILIGENCE, - PipelineStage.CLOSING, - ] - ) - operations = sum( - by_stage.get(s.value, 0) - for s in [ - PipelineStage.TURNOVER, - PipelineStage.LEASING, - ] - ) - portfolio = by_stage.get(PipelineStage.PORTFOLIO.value, 0) - - return { - "total_assets": len(assets), - "by_stage": {k: v for k, v in sorted(by_stage.items())}, - "pipeline_flow": { - "acquisition": acquisition, - "deal_making": deal_making, - "operations": operations, - "portfolio": portfolio, - }, - "killed": killed, - } - - -# ═══════════════════════════════════════════════════════════════════════════════ -# Pipeline engine -# ═══════════════════════════════════════════════════════════════════════════════ - - -class PipelineEngine: - """Pipeline runner that orchestrates transitions through hook evaluation. - - The engine evaluates all registered pre-transition hooks before allowing - a stage change. If any hook rejects the transition (returns False), the - asset is automatically redirected to PipelineStage.KILLED with the - violation recorded in the stage log. - - Args: - repository: An AssetRepository instance for loading/saving assets. - """ - - def __init__(self, repository: AssetRepository) -> None: - self.repository = repository - self._hooks: Dict[PipelineStage, List[PreTransitionHook]] = {} - - def register_hook( - self, - stage: PipelineStage, - hook: PreTransitionHook, - ) -> None: - """Register a pre-transition hook for the given target stage.""" - if stage not in self._hooks: - self._hooks[stage] = [] - self._hooks[stage].append(hook) - - def remove_hook( - self, - stage: PipelineStage, - hook: PreTransitionHook, - ) -> None: - """Remove a previously registered hook.""" - if stage in self._hooks: - self._hooks[stage] = [h for h in self._hooks[stage] if h is not hook] - - def process_transition( - self, - asset: PropertyAsset, - target_stage: PipelineStage, - context: Dict[str, Any], - ) -> PropertyAsset: - """Attempt to transition an asset to the target stage. - - Evaluation order: - 1. Fetch the latest asset state from the repository. - 2. Evaluate all hooks registered for *target_stage*. - 3. If all hooks pass → execute the transition via - asset.transition_to() and persist. - 4. If ANY hook fails → redirect to KILLED, record the first - violation reason in the log, and persist. - - Returns: - The updated PropertyAsset after the transition (or KILLED). - - Raises: - InvalidStageTransitionException: If the raw transition is not - allowed by the static ruleset (before hooks are evaluated). - """ - persisted = self.repository.load(asset.asset_id) - working_asset = persisted if persisted is not None else asset - - violation = self._evaluate_hooks(working_asset, target_stage, context) - - if violation is not None: - logger.warning( - "Hook blocked transition %s -> %s for asset %s: %s", - working_asset.current_stage.value, - target_stage.value, - working_asset.asset_id, - violation, - ) - working_asset.transition_to( - PipelineStage.KILLED, - reason=violation, - metrics=context, - ) - self.repository.save(working_asset) - return working_asset - - working_asset.transition_to( - target_stage, - reason=context.get("reason"), - metrics=context, - ) - self.repository.save(working_asset) - - logger.info( - "Asset %s transitioned %s -> %s", - working_asset.asset_id, - working_asset.stage_history[-2].stage.value - if len(working_asset.stage_history) >= 2 - else "(start)", - working_asset.current_stage.value, - ) - return working_asset - - def _evaluate_hooks( - self, - asset: PropertyAsset, - target_stage: PipelineStage, - context: Dict[str, Any], - ) -> Optional[str]: - """Evaluate all hooks for the target stage. - - Returns the first violation message if a hook rejects, or None - if all hooks pass. - """ - stage_hooks = self._hooks.get(target_stage, []) - for hook in stage_hooks: - try: - result = hook(asset, target_stage, context) - except Exception as exc: - logger.error( - "Hook raised exception for asset %s -> %s: %s", - asset.asset_id, - target_stage.value, - exc, - ) - return f"Hook exception: {exc}" - - if not result: - msg = context.get( - "violation_reason", - f"Pre-transition hook rejected {target_stage.value}", - ) - return cast(str, msg) - - return None diff --git a/prei/pipeline/handlers/__init__.py b/prei/pipeline/handlers/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/prei/pipeline/handlers/batch_screening.py b/prei/pipeline/handlers/batch_screening.py deleted file mode 100644 index a0a4bd1a..00000000 --- a/prei/pipeline/handlers/batch_screening.py +++ /dev/null @@ -1,196 +0,0 @@ -"""Vectorized parallel screening pipeline for high-throughput batch processing. - -Wraps the single-property screening evaluator in a multi-threaded worker -to clear massive discovery queues efficiently — thousands of properties -per batch with sub-second total execution. -""" - -from __future__ import annotations - -import logging -import time -from concurrent.futures import ThreadPoolExecutor, as_completed -from typing import Any, Dict, List - -from prei.models.pipeline import PipelineStage, PropertyAsset -from prei.pipeline.engine import PipelineEngine -from prei.pipeline.handlers.screening import ( - ScreeningThresholds, - evaluate_screening_stage, -) - -logger = logging.getLogger(__name__) - - -class BatchScreeningProcessor: - """Evaluate a batch of property payloads through the SCREENING stage. - - Each property is independently evaluated against the configured - thresholds using the hyper-fast screening evaluator. Passing - properties advance to UNDERWRITING; failing properties are - killed with the exact violation reason recorded. - - Args: - engine: PipelineEngine instance for asset persistence and hooks. - thresholds: ScreeningThresholds for the evaluator. - max_workers: Number of parallel worker threads (default 8). - """ - - def __init__( - self, - engine: PipelineEngine, - thresholds: ScreeningThresholds, - max_workers: int = 8, - ) -> None: - self.engine = engine - self.thresholds = thresholds - self.max_workers = max_workers - - # ── Public API ──────────────────────────────────────────────────────────── - - def process( - self, - property_dicts: List[Dict[str, Any]], - ) -> Dict[str, Any]: - """Evaluate a batch of property payloads through SCREENING. - - Each payload is a dict with at minimum: - asset_id (str): Unique identifier. - address (str): Property address. - estimated_monthly_rent (float): Projected rent. - purchase_price (float): Acquisition price. - beds (int): Bedroom count. - baths (int | float): Bathroom count. - hoa_name (str, optional): HOA name for exclusion check. - - Args: - property_dicts: List of property payload dicts. - - Returns: - Operational summary dict: - processed (int): Total payloads in the batch. - advanced (int): Payloads that passed → UNDERWRITING. - killed (int): Payloads that failed → KILLED. - execution_time_ms (float): Wall-clock time in ms. - """ - start = time.perf_counter() - advanced = 0 - killed = 0 - errors: List[str] = [] - - with ThreadPoolExecutor(max_workers=self.max_workers) as pool: - futures = { - pool.submit(self._process_single, payload): payload - for payload in property_dicts - } - for future in as_completed(futures): - payload = futures[future] - try: - result = future.result() - if result is True: - advanced += 1 - else: - killed += 1 - except Exception as exc: - logger.error( - "Batch screening failed for asset %s: %s", - payload.get("asset_id", "?"), - exc, - ) - errors.append(f"{payload.get('asset_id', '?')}: {exc}") - killed += 1 - - elapsed_ms = (time.perf_counter() - start) * 1000 - - summary: Dict[str, Any] = { - "processed": len(property_dicts), - "advanced": advanced, - "killed": killed, - "execution_time_ms": round(elapsed_ms, 2), - } - if errors: - summary["errors"] = errors - - logger.info( - "Batch screening complete: %d processed, %d advanced, %d killed in %.1fms", - summary["processed"], - summary["advanced"], - summary["killed"], - summary["execution_time_ms"], - ) - return summary - - # ── Internal per-asset logic ────────────────────────────────────────────── - - def _process_single( - self, - payload: Dict[str, Any], - ) -> bool: - """Evaluate one property payload and advance or kill its asset. - - Returns True if the asset advanced to UNDERWRITING, - False if it was killed. - """ - asset_id = payload.get("asset_id", "unknown") - address = payload.get("address", "") - - # Build asset_data dict for the screening evaluator - asset_data = _extract_screening_data(payload) - - # Run the screener - passed, fail_reason = evaluate_screening_stage(asset_data, self.thresholds) - - # Build transition context with screening metrics - ctx: Dict[str, Any] = { - "source": "batch_screening", - "asset_data": asset_data, - } - - # Load or create the PropertyAsset for engine tracking - asset = self.engine.repository.load(asset_id) - if asset is None: - asset = PropertyAsset(asset_id=asset_id, address=address) - self.engine.repository.save(asset) - - # Advance from GACS → DISCOVERY → SCREENING if needed - if asset.current_stage == PipelineStage.GACS: - ctx["reason"] = "Auto-advance through discovery" - asset = self.engine.process_transition( - asset, - PipelineStage.DISCOVERY, - {"reason": "Batch screening auto-advance"}, - ) - asset = self.engine.process_transition( - asset, - PipelineStage.SCREENING, - {"reason": "Batch screening auto-advance"}, - ) - - # Now evaluate and transition from SCREENING - if passed: - ctx["reason"] = "Screening passed" - self.engine.process_transition(asset, PipelineStage.UNDERWRITING, ctx) - return True - else: - ctx["violation_reason"] = fail_reason or "Screening failed" - ctx["reason"] = ctx["violation_reason"] - self.engine.process_transition(asset, PipelineStage.KILLED, ctx) - return False - - -# ── Helper ──────────────────────────────────────────────────────────────────── - - -def _extract_screening_data(payload: Dict[str, Any]) -> Dict[str, Any]: - """Extract the fields the screening evaluator needs from a raw payload. - - Unknown or missing fields pass through as None so the evaluator - can skip missing checks gracefully. - """ - return { - "estimated_monthly_rent": payload.get("estimated_monthly_rent"), - "purchase_price": payload.get("purchase_price"), - "beds": payload.get("beds"), - "baths": payload.get("baths"), - "hoa_name": payload.get("hoa_name"), - } diff --git a/prei/pipeline/handlers/discovery_processor.py b/prei/pipeline/handlers/discovery_processor.py deleted file mode 100644 index ca721990..00000000 --- a/prei/pipeline/handlers/discovery_processor.py +++ /dev/null @@ -1,101 +0,0 @@ -"""Memory-efficient deduplication and ingestion engine for the DISCOVERY stage. - -Processes thousands of incoming records from a discovery sweep, cross-referencing -against existing historical entries via address hashes to eliminate duplicate -data before allocating state tracking blocks. -""" - -from __future__ import annotations - -from typing import Any, Dict, List, Set - -from prei.models.pipeline import PipelineStage, PropertyAsset -from prei.pipeline.handlers.discovery import DiscoverySanitizer - - -class DiscoveryProcessor: - """Deduplicating ingestion processor for the DISCOVERY pipeline stage. - - Compares incoming batch address hashes against existing entries to - eliminate duplicates, then instantiates PropertyAsset records for - newly discovered properties at PipelineStage.DISCOVERY. - - Args: - existing_hashes: Set of SHA-256 address hashes already known to - the persistent storage layer. - """ - - def __init__(self, existing_hashes: Set[str]) -> None: - self.existing_hashes = existing_hashes - - def process_batch( - self, - raw_listings: List[Dict[str, Any]], - source_name: str, - ) -> Dict[str, Any]: - """Process a batch of raw listings through dedup and state inception. - - Each listing is: - 1. Normalised via DiscoverySanitizer.transform_input() - 2. Checked for address_hash collision against existing_hashes - 3. If duplicate → counted and skipped - 4. If new → PropertyAsset created at DISCOVERY, hash added - - Args: - raw_listings: List of raw property data dicts from an external - source (MLS, county records, wholesale JSON, etc.). - source_name: Human-readable source label (e.g. "mls_feed", - "county_scraper"). - - Returns: - Analytics dict: - total_received (int): Raw count of input records. - new_assets_discovered (int): Assets created. - duplicates_skipped (int): Records rejected by hash match. - failed_records (int): Records that raised during parsing. - payloads (list[PropertyAsset]): Newly created assets. - """ - new_assets: List[PropertyAsset] = [] - duplicates_count = 0 - errors_count = 0 - - for raw in raw_listings: - try: - canonical = DiscoverySanitizer.transform_input(raw, source_name) - - if canonical.address_hash in self.existing_hashes: - duplicates_count += 1 - continue - - # Create asset at GACS, then transition to DISCOVERY to - # seed the initial stage log entry with financial context. - asset = PropertyAsset( - asset_id=canonical.source_id, - address=canonical.raw_address, - ) - asset.transition_to( - PipelineStage.DISCOVERY, - reason="Initial discovery ingestion", - metrics={ - "source": source_name, - "purchase_price": canonical.price, - "estimated_rent": canonical.estimated_rent, - "sqft": canonical.sqft, - "year_built": canonical.year_built, - }, - ) - - new_assets.append(asset) - self.existing_hashes.add(canonical.address_hash) - - except Exception: - errors_count += 1 - continue - - return { - "total_received": len(raw_listings), - "new_assets_discovered": len(new_assets), - "duplicates_skipped": duplicates_count, - "failed_records": errors_count, - "payloads": new_assets, - } diff --git a/prei/pipeline/handlers/offer.py b/prei/pipeline/handlers/offer.py deleted file mode 100644 index 97115ab0..00000000 --- a/prei/pipeline/handlers/offer.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Offer stage handler — offer price optimization and strategy. - -Computes the optimal offer price for a property based on underwriting -results, market conditions, and investment strategy parameters. -""" - -from __future__ import annotations - -from enum import Enum -from typing import Optional - -from pydantic import BaseModel, Field - - -class OfferStrategy(str, Enum): - """Offer pricing strategy variants.""" - - CONSERVATIVE = "conservative" # Offer below MAO (buffer for negotiation) - TARGET = "target" # Offer at MAO - AGGRESSIVE = "aggressive" # Offer above MAO (competitive market) - - -class OfferInput(BaseModel): - """Input parameters for the offer solver.""" - - mao: float = Field(..., description="Max Allowable Offer from underwriting") - arv: Optional[float] = Field( - default=None, description="After Repair Value (estimated resale value)" - ) - rehab_budget: float = Field(default=0.0, ge=0) - desired_equity: float = Field( - default=0.0, - ge=0, - le=1.0, - description="Minimum desired equity percentage (e.g. 0.20 = 20%)", - ) - competition_multiplier: float = Field( - default=1.0, - ge=0.5, - le=2.0, - description="Market competition factor (1.0 = neutral, >1 = hot market)", - ) - - -class OfferMetrics(BaseModel): - """Output metrics from the offer solver.""" - - offer_price: float - strategy: OfferStrategy - premium_over_mao: float - premium_pct: float - estimated_equity: Optional[float] = None - estimated_equity_pct: Optional[float] = None - - -def solve_offer( - inputs: OfferInput, - strategy: OfferStrategy = OfferStrategy.TARGET, -) -> OfferMetrics: - """Compute the optimal offer price based on strategy. - - Strategy rules: - CONSERVATIVE: offer = MAO × 0.95 × competition_multiplier - TARGET: offer = MAO × competition_multiplier - AGGRESSIVE: offer = MAO × 1.05 × competition_multiplier - - All strategies clamp the offer to ensure minimum desired equity - is maintained when ARV is known. - - Args: - inputs: OfferInput with MAO, ARV, rehab, equity target. - strategy: Pricing strategy enum. - - Returns: - OfferMetrics with offer price and equity analysis. - """ - # ── Base offer by strategy ──────────────────────────────────────────────── - if strategy == OfferStrategy.CONSERVATIVE: - raw_offer = inputs.mao * 0.95 - elif strategy == OfferStrategy.AGGRESSIVE: - raw_offer = inputs.mao * 1.05 - else: - raw_offer = inputs.mao - - offer_price = raw_offer * inputs.competition_multiplier - - # ── Equity constraint (when ARV is known) ──────────────────────────────── - estimated_equity: Optional[float] = None - estimated_equity_pct: Optional[float] = None - - if inputs.arv and inputs.arv > 0: - total_cost = offer_price + inputs.rehab_budget - estimated_equity = inputs.arv - total_cost - estimated_equity_pct = estimated_equity / inputs.arv if inputs.arv > 0 else 0.0 - - # Clamp offer to maintain minimum desired equity - if inputs.desired_equity > 0: - max_offer_for_equity = ( - inputs.arv * (1 - inputs.desired_equity) - inputs.rehab_budget - ) - if max_offer_for_equity < offer_price: - offer_price = max_offer_for_equity - # Recalculate with clamped price - total_cost = offer_price + inputs.rehab_budget - estimated_equity = inputs.arv - total_cost - estimated_equity_pct = ( - estimated_equity / inputs.arv if inputs.arv > 0 else 0.0 - ) - - premium = offer_price - inputs.mao - premium_pct = premium / inputs.mao if inputs.mao > 0 else 0.0 - - return OfferMetrics( - offer_price=round(offer_price, 2), - strategy=strategy, - premium_over_mao=round(premium, 2), - premium_pct=round(premium_pct, 6), - estimated_equity=round(estimated_equity, 2) - if estimated_equity is not None - else None, - estimated_equity_pct=round(estimated_equity_pct, 4) - if estimated_equity_pct is not None - else None, - ) diff --git a/prei/pipeline/handlers/screening.py b/prei/pipeline/handlers/screening.py deleted file mode 100644 index 171c39a7..00000000 --- a/prei/pipeline/handlers/screening.py +++ /dev/null @@ -1,163 +0,0 @@ -"""Hyper-fast screening-stage metric evaluator for property pipeline. - -Evaluates incoming properties against structural yield bounds using pure -arithmetic functions. Designed for sub-10ms execution per property — -no pandas, no ORM, no external I/O. -""" - -from __future__ import annotations - -from typing import Any, Dict, List, Optional, Tuple - -from pydantic import BaseModel, Field - - -# ── Configuration model ─────────────────────────────────────────────────────── - - -class ScreeningThresholds(BaseModel): - """Threshold configuration for the SCREENING pipeline stage. - - All fields are required unless marked optional. - """ - - min_gross_yield: float = Field( - ..., gt=0, description="Minimum acceptable gross yield (e.g. 0.07 = 7%)" - ) - max_price_to_rent_ratio: float = Field( - ..., gt=0, description="Maximum acceptable price-to-rent ratio (e.g. 15.0)" - ) - excluded_hoas: List[str] = Field( - default_factory=list, description="HOA names that automatically disqualify" - ) - min_beds: int = Field(..., ge=0, description="Minimum number of bedrooms") - min_baths: int = Field(..., ge=0, description="Minimum number of bathrooms") - - -# ── Pure arithmetic helpers ──────────────────────────────────────────────────── - - -def gross_yield(monthly_rent: float, purchase_price: float) -> float: - """Compute gross yield as a fraction. - - Formula: - Gross Yield = (monthly_rent × 12) / purchase_price - - Args: - monthly_rent: Estimated monthly rent in dollars. - purchase_price: Total purchase price in dollars. - - Returns: - Gross yield as a float (e.g. 0.072 for 7.2%). - """ - if purchase_price <= 0 or monthly_rent <= 0: - return 0.0 - return (monthly_rent * 12.0) / purchase_price - - -def price_to_rent_ratio(monthly_rent: float, purchase_price: float) -> float: - """Compute price-to-rent ratio. - - Formula: - Price-to-Rent Ratio = purchase_price / (monthly_rent × 12) - - Args: - monthly_rent: Estimated monthly rent in dollars. - purchase_price: Total purchase price in dollars. - - Returns: - Price-to-rent ratio (e.g. 13.8 means 13.8× annual rent). - """ - annual_rent = monthly_rent * 12.0 - if annual_rent <= 0: - return float("inf") - return purchase_price / annual_rent - - -# ── Composition helper ───────────────────────────────────────────────────────── - - -def compute_screening_metrics(asset_data: Dict[str, Any]) -> Dict[str, float]: - """Compute all screening-relevant metrics from raw asset data. - - Args: - asset_data: Dict containing at least 'estimated_monthly_rent' - and 'purchase_price' keys. - - Returns: - Dict with computed metric names mapped to float values. - """ - rent = float(asset_data.get("estimated_monthly_rent", 0)) - price = float(asset_data.get("purchase_price", 0)) - - return { - "gross_yield": gross_yield(rent, price), - "price_to_rent_ratio": price_to_rent_ratio(rent, price), - } - - -# ── Top-level evaluator ──────────────────────────────────────────────────────── - - -def evaluate_screening_stage( - asset_data: Dict[str, Any], - thresholds: ScreeningThresholds, -) -> Tuple[bool, Optional[str]]: - """Evaluate a property against all screening thresholds. - - Each check is evaluated in order of lowest computational cost first. - The first violation short-circuits and returns the kill reason. - - Args: - asset_data: Property data with keys: - - estimated_monthly_rent (float) - - purchase_price (float) - - beds (int) - - baths (int) - - hoa_name (str, optional) - thresholds: ScreeningThresholds instance with bounds. - - Returns: - Tuple of (pass: bool, kill_reason: str | None). - pass=True, kill_reason=None means all checks passed. - pass=False, kill_reason= means the property was rejected. - """ - # ── 1. Beds check (cheapest: dict lookup + int compare) ────────────── - beds = asset_data.get("beds") - if beds is not None and int(beds) < thresholds.min_beds: - return False, (f"Insufficient bedrooms: {beds} < {thresholds.min_beds}") - - # ── 2. Baths check ─────────────────────────────────────────────────── - baths = asset_data.get("baths") - if baths is not None and float(baths) < thresholds.min_baths: - return False, (f"Insufficient bathrooms: {baths} < {thresholds.min_baths}") - - # ── 3. HOA exclusion check ─────────────────────────────────────────── - hoa = asset_data.get("hoa_name") - if hoa and thresholds.excluded_hoas: - hoa_lower = hoa.strip().lower() - for excluded in thresholds.excluded_hoas: - if excluded.strip().lower() == hoa_lower: - return False, (f"Excluded HOA: {hoa}") - - # ── 4. Gross yield check (two arithmetic ops) ──────────────────────── - rent = asset_data.get("estimated_monthly_rent") - price = asset_data.get("purchase_price") - if rent is not None and price is not None and price > 0 and float(rent) > 0: - gy = gross_yield(float(rent), float(price)) - if gy < thresholds.min_gross_yield: - return False, ( - f"Gross yield too low: {gy:.4f} < {thresholds.min_gross_yield}" - ) - - # ── 5. Price-to-rent ratio check ────────────────────────────────────── - if rent is not None and price is not None and price > 0 and float(rent) > 0: - ptr = price_to_rent_ratio(float(rent), float(price)) - if ptr > thresholds.max_price_to_rent_ratio: - return False, ( - f"Price-to-rent ratio too high: {ptr:.2f} > " - f"{thresholds.max_price_to_rent_ratio}" - ) - - # ── All checks passed ───────────────────────────────────────────────── - return True, None diff --git a/prei/pipeline/orchestrator.py b/prei/pipeline/orchestrator.py deleted file mode 100644 index fb371a64..00000000 --- a/prei/pipeline/orchestrator.py +++ /dev/null @@ -1,247 +0,0 @@ -"""Pipeline orchestrator — chains discovery → screening → underwriting. - -Runs a single property payload through the full pipeline pipeline, -returning the final asset state and all computed metrics. -""" - -from __future__ import annotations - -import logging -from decimal import Decimal -from typing import Any, Dict, Optional, Set - -from investor_app.finance.utils import to_decimal -from prei.models.pipeline import PipelineStage, PropertyAsset -from prei.pipeline.engine import ( - AssetRepository, - InMemoryAssetRepository, - PipelineEngine, -) -from prei.pipeline.handlers.discovery import DiscoverySanitizer -from prei.pipeline.handlers.screening import ( - ScreeningThresholds, - evaluate_screening_stage, -) -from prei.pipeline.handlers.underwriting import ( - UnderwritingInput, - UnderwritingMetrics, - solve_underwriting, -) - -logger = logging.getLogger(__name__) - - -# ── Config ──────────────────────────────────────────────────────────────────── - -DEFAULT_SCREENING_THRESHOLDS = ScreeningThresholds( - min_gross_yield=0.07, - max_price_to_rent_ratio=15.0, - min_beds=1, - min_baths=1, -) - -DEFAULT_TARGET_CAP_RATE = 0.08 - - -# ── Result model ────────────────────────────────────────────────────────────── - - -class PipelineResult: - """Container for the full pipeline execution result.""" - - def __init__( - self, - asset: Optional[PropertyAsset] = None, - canonical: Optional[Any] = None, - screening_passed: Optional[bool] = None, - screening_reason: Optional[str] = None, - underwriting: Optional[UnderwritingMetrics] = None, - error: Optional[str] = None, - ) -> None: - self.asset = asset - self.canonical = canonical - self.screening_passed = screening_passed - self.screening_reason = screening_reason - self.underwriting = underwriting - self.error = error - - @property - def success(self) -> bool: - """True if the pipeline completed without error.""" - return self.error is None - - def to_dict(self) -> Dict[str, Any]: - """Serialize to a JSON-safe dict.""" - base: Dict[str, Any] = { - "success": self.success, - } - if self.error: - base["error"] = self.error - return base - - base["asset_id"] = self.asset.asset_id if self.asset else None - base["current_stage"] = self.asset.current_stage.value if self.asset else None - - if self.canonical: - base["address_hash"] = self.canonical.address_hash - base["price"] = self.canonical.price - base["beds"] = self.canonical.beds - base["baths"] = self.canonical.baths - - base["screening_passed"] = self.screening_passed - - if self.underwriting: - base["noi"] = self.underwriting.noi - base["cap_rate"] = self.underwriting.cap_rate - base["cash_on_cash"] = self.underwriting.cash_on_cash - base["mao"] = self.underwriting.mao - base["target_cap_rate"] = DEFAULT_TARGET_CAP_RATE - - return base - - -# ── Orchestrator ────────────────────────────────────────────────────────────── - - -class PipelineOrchestrator: - """Orchestrates a full pipeline run for a single property. - - Stages executed in order: - 1. DISCOVERY — normalize raw data via DiscoverySanitizer - 2. SCREENING — evaluate against thresholds - 3. UNDERWRITING — compute NOI, cap rate, CoC, MAO - - The asset advances through pipeline stages via the PipelineEngine - so hooks and persistence are respected at each transition. - """ - - def __init__( - self, - repository: Optional[AssetRepository] = None, - screening_thresholds: Optional[ScreeningThresholds] = None, - target_cap_rate: float = DEFAULT_TARGET_CAP_RATE, - existing_hashes: Optional[Set[str]] = None, - ) -> None: - self.repository = repository or InMemoryAssetRepository() - self.engine = PipelineEngine(repository=self.repository) - self.screening_thresholds = screening_thresholds or DEFAULT_SCREENING_THRESHOLDS - self.target_cap_rate = target_cap_rate - self.existing_hashes = existing_hashes or set() - - # ── Public entry point ─────────────────────────────────────────────────── - - def run( - self, - raw_payload: Dict[str, Any], - source_name: str = "pipeline_orchestrator", - ) -> PipelineResult: - """Execute the full pipeline on a single raw property payload. - - Args: - raw_payload: Raw property data dict (any source schema). - source_name: Source label for the discovery stage. - - Returns: - PipelineResult with asset state and all computed metrics. - """ - # ── Stage 1: DISCOVERY ─────────────────────────────────────────── - try: - canonical = DiscoverySanitizer.transform_input(raw_payload, source_name) - except ValueError as exc: - return PipelineResult(error=f"Discovery failed: {exc}") - - # Dedup check - if canonical.address_hash in self.existing_hashes: - return PipelineResult( - error=f"Duplicate address hash: {canonical.address_hash}" - ) - self.existing_hashes.add(canonical.address_hash) - - # ── Stage 2: SCREENING ─────────────────────────────────────────── - asset_data = { - "estimated_monthly_rent": canonical.estimated_rent, - "purchase_price": canonical.price, - "beds": canonical.beds, - "baths": canonical.baths, - } - screening_passed, screening_reason = evaluate_screening_stage( - asset_data, self.screening_thresholds - ) - - if not screening_passed: - # Create asset and kill it with the violation reason - asset = PropertyAsset( - asset_id=canonical.source_id, - address=canonical.raw_address, - ) - self.repository.save(asset) - asset = self.engine.process_transition( - asset, - PipelineStage.KILLED, - { - "reason": screening_reason, - "violation_reason": screening_reason, - "source": source_name, - "address_hash": canonical.address_hash, - }, - ) - return PipelineResult( - asset=asset, - canonical=canonical, - screening_passed=False, - screening_reason=screening_reason, - ) - - # ── Stage 3: UNDERWRITING ──────────────────────────────────────── - # Build input from canonical data (with sensible defaults) - price = to_decimal(canonical.price or 0.0) - rent = to_decimal(canonical.estimated_rent or 0.0) - uw_input = UnderwritingInput( - purchase_price=price, - estimated_rent=rent, - property_tax_annual=raw_payload.get( - "property_tax_annual", price * Decimal("0.012") - ), - insurance_annual=raw_payload.get( - "insurance_annual", price * Decimal("0.004") - ), - hoa_annual=raw_payload.get("hoa_annual", Decimal("0")), - ) - uw_metrics = solve_underwriting(uw_input, self.target_cap_rate) - - # ── Create asset and advance through pipeline stages ───────────── - asset = PropertyAsset( - asset_id=canonical.source_id, - address=canonical.raw_address, - ) - self.repository.save(asset) - - ctx = { - "source": source_name, - "address_hash": canonical.address_hash, - "price": canonical.price, - "estimated_rent": canonical.estimated_rent, - "underwriting": uw_metrics.model_dump(), - } - - # GACS → DISCOVERY - asset = self.engine.process_transition( - asset, PipelineStage.DISCOVERY, {**ctx, "reason": "Discovery completed"} - ) - # DISCOVERY → SCREENING - asset = self.engine.process_transition( - asset, PipelineStage.SCREENING, {**ctx, "reason": "Screening passed"} - ) - # SCREENING → UNDERWRITING - asset = self.engine.process_transition( - asset, - PipelineStage.UNDERWRITING, - {**ctx, "reason": "Underwriting completed"}, - ) - - return PipelineResult( - asset=asset, - canonical=canonical, - screening_passed=True, - underwriting=uw_metrics, - ) diff --git a/prei/pipeline/sources/__init__.py b/prei/pipeline/sources/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/prei/pipeline/tests/test_api.py b/prei/pipeline/tests/test_api.py deleted file mode 100644 index 683e4d02..00000000 --- a/prei/pipeline/tests/test_api.py +++ /dev/null @@ -1,340 +0,0 @@ -"""Tests for the pipeline REST API and CLI.""" - -import json -import tempfile -from pathlib import Path - -import pytest -from fastapi.testclient import TestClient - -from prei.api.pipeline_routes import configure_repository, router -from prei.models.pipeline import PropertyAsset -from prei.pipeline.engine import InMemoryAssetRepository - -# Build a FastAPI app for testing -from fastapi import FastAPI - -app = FastAPI() -app.include_router(router) -client = TestClient(app) - - -# ═══════════════════════════════════════════════════════════════════════════════ -# Fixtures -# ═══════════════════════════════════════════════════════════════════════════════ - - -@pytest.fixture(autouse=True) -def reset_repo(): - """Give each test a fresh repository.""" - repo = InMemoryAssetRepository() - configure_repository(repo) - # Seed two assets - a1 = PropertyAsset(asset_id="API-001", address="123 Test St") - a2 = PropertyAsset(asset_id="API-002", address="456 Mock Ave") - repo.save(a1) - repo.save(a2) - yield - - -# ═══════════════════════════════════════════════════════════════════════════════ -# GET /api/v1/pipeline/summary -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TestGetSummary: - def test_returns_summary(self): - resp = client.get("/api/v1/pipeline/summary") - assert resp.status_code == 200 - data = resp.json() - assert "total_assets" in data - assert data["total_assets"] == 2 - assert "by_stage" in data - assert "pipeline_flow" in data - assert "killed" in data - - def test_by_stage_counts(self): - resp = client.get("/api/v1/pipeline/summary") - data = resp.json() - # Both assets start at GACS - assert data["by_stage"]["GACS"] == 2 - - def test_empty_repo(self): - repo = InMemoryAssetRepository() - configure_repository(repo) - resp = client.get("/api/v1/pipeline/summary") - assert resp.status_code == 200 - assert resp.json()["total_assets"] == 0 - - -# ═══════════════════════════════════════════════════════════════════════════════ -# GET /api/v1/pipeline/assets -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TestListAssets: - def test_list_all(self): - resp = client.get("/api/v1/pipeline/assets") - assert resp.status_code == 200 - data = resp.json() - assert len(data) == 2 - ids = {a["asset_id"] for a in data} - assert ids == {"API-001", "API-002"} - - def test_asset_structure(self): - resp = client.get("/api/v1/pipeline/assets") - asset = resp.json()[0] - assert "asset_id" in asset - assert "address" in asset - assert "current_stage" in asset - assert "stage_history" in asset - assert "kill_reason" in asset - - -# ═══════════════════════════════════════════════════════════════════════════════ -# GET /api/v1/pipeline/assets/{asset_id} -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TestGetAsset: - def test_get_existing(self): - resp = client.get("/api/v1/pipeline/assets/API-001") - assert resp.status_code == 200 - assert resp.json()["asset_id"] == "API-001" - - def test_get_nonexistent(self): - resp = client.get("/api/v1/pipeline/assets/DOES-NOT-EXIST") - assert resp.status_code == 404 - - -# ═══════════════════════════════════════════════════════════════════════════════ -# POST /api/v1/pipeline/assets -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TestCreateAsset: - def test_create(self): - resp = client.post( - "/api/v1/pipeline/assets", - json={"asset_id": "NEW-001", "address": "789 New St"}, - ) - assert resp.status_code == 200 - assert resp.json()["asset_id"] == "NEW-001" - assert resp.json()["current_stage"] == "GACS" - - def test_create_duplicate(self): - resp = client.post( - "/api/v1/pipeline/assets", - json={"asset_id": "API-001", "address": "dup"}, - ) - assert resp.status_code == 409 - - def test_create_missing_fields(self): - resp = client.post( - "/api/v1/pipeline/assets", - json={"asset_id": "NO-ADDR"}, - ) - assert resp.status_code == 422 - - -# ═══════════════════════════════════════════════════════════════════════════════ -# POST /api/v1/pipeline/transition -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TestTransition: - def test_valid_transition(self): - resp = client.post( - "/api/v1/pipeline/transition", - json={ - "asset_id": "API-001", - "target_stage": "DISCOVERY", - "reason": "Testing", - }, - ) - assert resp.status_code == 200 - data = resp.json() - assert data["current_stage"] == "DISCOVERY" - assert len(data["stage_history"]) == 1 - - def test_invalid_stage_name(self): - resp = client.post( - "/api/v1/pipeline/transition", - json={"asset_id": "API-001", "target_stage": "INVALID_STAGE"}, - ) - assert resp.status_code == 422 - - def test_illicit_jump(self): - """GACS → PORTFOLIO is illegal and returns 422.""" - resp = client.post( - "/api/v1/pipeline/transition", - json={"asset_id": "API-001", "target_stage": "PORTFOLIO"}, - ) - assert resp.status_code == 422 - assert "GACS" in resp.json()["detail"] - - def test_nonexistent_asset(self): - resp = client.post( - "/api/v1/pipeline/transition", - json={"asset_id": "MISSING", "target_stage": "DISCOVERY"}, - ) - assert resp.status_code == 404 - - def test_missing_asset_id(self): - resp = client.post( - "/api/v1/pipeline/transition", - json={"target_stage": "DISCOVERY"}, - ) - assert resp.status_code == 422 - - def test_transition_with_context(self): - resp = client.post( - "/api/v1/pipeline/transition", - json={ - "asset_id": "API-001", - "target_stage": "KILLED", - "reason": "Budget cut", - "context": {"violation_reason": "Not viable"}, - }, - ) - assert resp.status_code == 200 - data = resp.json() - assert data["current_stage"] == "KILLED" - assert data["kill_reason"] == "Budget cut" - - -# ═══════════════════════════════════════════════════════════════════════════════ -# DELETE /api/v1/pipeline/assets/{asset_id} -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TestDeleteAsset: - def test_delete_existing_soft(self): - resp = client.delete("/api/v1/pipeline/assets/API-001") - assert resp.status_code == 200 - assert resp.json()["status"] == "killed" - # Asset should now be in KILLED stage - get_resp = client.get("/api/v1/pipeline/assets/API-001") - assert get_resp.json()["current_stage"] == "KILLED" - - def test_delete_nonexistent(self): - resp = client.delete("/api/v1/pipeline/assets/DOES-NOT-EXIST") - assert resp.status_code == 404 - - -# ═══════════════════════════════════════════════════════════════════════════════ -# CLI tests -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TestCLI: - @pytest.fixture - def mls_feed(self): - """Create a temporary MLS JSON feed file.""" - data = { - "properties": [ - { - "asset_id": "MLS-001", - "address": "101 Prime St", - "estimated_monthly_rent": 2500.0, - "purchase_price": 300_000.0, - "beds": 3, - "baths": 2, - }, - { - "asset_id": "MLS-002", - "address": "202 Bad Ave", - "estimated_monthly_rent": 800.0, - "purchase_price": 400_000.0, - "beds": 1, - "baths": 0.5, - }, - ] - } - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - json.dump(data, f) - path = f.name - yield path - Path(path).unlink() - - def test_cli_ingest_no_screening(self, mls_feed, capsys): - from prei.cli import cli - from click.testing import CliRunner - - runner = CliRunner() - result = runner.invoke(cli, ["pipeline", "ingest", "--source", mls_feed]) - assert result.exit_code == 0 - assert "Ingested 2 properties" in result.output - - def test_cli_ingest_with_screening(self, mls_feed, capsys): - from prei.cli import cli - from click.testing import CliRunner - - runner = CliRunner() - result = runner.invoke( - cli, - [ - "pipeline", - "ingest", - "--source", - mls_feed, - "--run-screening", - "--min-yield", - "0.05", - "--max-ptr", - "20.0", - ], - ) - assert result.exit_code == 0 - assert "Screening:" in result.output - - def test_cli_summary(self): - from prei.cli import cli - from click.testing import CliRunner - - runner = CliRunner() - result = runner.invoke(cli, ["pipeline", "summary"]) - assert result.exit_code == 0 - assert "total_assets" in result.output - - def test_cli_transition_valid(self): - from prei.cli import cli - from click.testing import CliRunner - - # First create an asset via API - client.post( - "/api/v1/pipeline/assets", - json={"asset_id": "CLI-001", "address": "CLI St"}, - ) - - runner = CliRunner() - # CLI creates its own engine, so this only tests the command syntax - # The engine in CLI is fresh (no pre-existing assets) - result = runner.invoke( - cli, - [ - "pipeline", - "transition", - "--asset-id", - "CLI-001", - "--target", - "DISCOVERY", - ], - ) - # CLI engine is fresh — asset won't exist - assert result.exit_code == 1 - assert "not found" in result.output - - def test_cli_transition_invalid_stage(self): - from prei.cli import cli - from click.testing import CliRunner - - runner = CliRunner() - result = runner.invoke( - cli, - ["pipeline", "transition", "--asset-id", "BOGUS", "--target", "BOGUS"], - ) - assert result.exit_code == 1 - # Asset not found (checked first); invalid stage is secondary - # Both conditions produce exit code 1 - assert result.exit_code != 0 diff --git a/prei/pipeline/tests/test_batch_screening.py b/prei/pipeline/tests/test_batch_screening.py deleted file mode 100644 index f3993b6e..00000000 --- a/prei/pipeline/tests/test_batch_screening.py +++ /dev/null @@ -1,203 +0,0 @@ -"""Tests for the BatchScreeningProcessor.""" - -from prei.models.pipeline import PipelineStage -from prei.pipeline.engine import InMemoryAssetRepository, PipelineEngine -from prei.pipeline.handlers.batch_screening import BatchScreeningProcessor -from prei.pipeline.handlers.screening import ScreeningThresholds - -THRESHOLDS = ScreeningThresholds( - min_gross_yield=0.07, - max_price_to_rent_ratio=15.0, - min_beds=2, - min_baths=1, -) - - -def _make_engine() -> PipelineEngine: - return PipelineEngine(repository=InMemoryAssetRepository()) - - -# ── Fixtures ────────────────────────────────────────────────────────────────── - -PASSING_PROPERTY = { - "asset_id": "PASS-001", - "address": "123 Good St", - "estimated_monthly_rent": 2500.0, - "purchase_price": 300_000.0, - "beds": 3, - "baths": 2, -} - -FAILING_PROPERTY = { - "asset_id": "FAIL-001", - "address": "456 Bad Ave", - "estimated_monthly_rent": 800.0, - "purchase_price": 400_000.0, - "beds": 1, - "baths": 0.5, -} - - -class TestBatchScreeningProcessor: - """Tests for the batch screening processor.""" - - # ── Single property ─────────────────────────────────────────────────────── - - def test_single_passing_property(self): - """A single qualifying property advances to UNDERWRITING.""" - engine = _make_engine() - processor = BatchScreeningProcessor(engine, THRESHOLDS) - summary = processor.process([PASSING_PROPERTY]) - - assert summary["processed"] == 1 - assert summary["advanced"] == 1 - assert summary["killed"] == 0 - assert summary["execution_time_ms"] >= 0 - - asset = engine.repository.load("PASS-001") - assert asset is not None - assert asset.current_stage == PipelineStage.UNDERWRITING - - def test_single_failing_property(self): - """A non-qualifying property is killed with reason.""" - engine = _make_engine() - processor = BatchScreeningProcessor(engine, THRESHOLDS) - summary = processor.process([FAILING_PROPERTY]) - - assert summary["processed"] == 1 - assert summary["advanced"] == 0 - assert summary["killed"] == 1 - - asset = engine.repository.load("FAIL-001") - assert asset is not None - assert asset.current_stage == PipelineStage.KILLED - assert asset.kill_reason is not None - assert "bedroom" in (asset.kill_reason or "").lower() - - # ── Mixed batch ─────────────────────────────────────────────────────────── - - def test_mixed_batch(self): - """Mixed passing and failing properties are correctly counted.""" - engine = _make_engine() - processor = BatchScreeningProcessor(engine, THRESHOLDS) - - payloads = [ - PASSING_PROPERTY, - FAILING_PROPERTY, - { # Another passing property - "asset_id": "PASS-002", - "address": "789 Nice Blvd", - "estimated_monthly_rent": 3000.0, - "purchase_price": 350_000.0, - "beds": 4, - "baths": 3, - }, - { # Another failing (yield too low) - "asset_id": "FAIL-002", - "address": "321 Pricey Ln", - "estimated_monthly_rent": 2000.0, - "purchase_price": 600_000.0, - "beds": 3, - "baths": 2, - }, - ] - summary = processor.process(payloads) - - assert summary["processed"] == 4 - assert summary["advanced"] == 2 - assert summary["killed"] == 2 - - assert ( - engine.repository.load("PASS-001").current_stage - == PipelineStage.UNDERWRITING - ) # noqa - assert ( - engine.repository.load("PASS-002").current_stage - == PipelineStage.UNDERWRITING - ) # noqa - assert engine.repository.load("FAIL-001").current_stage == PipelineStage.KILLED - assert engine.repository.load("FAIL-002").current_stage == PipelineStage.KILLED - - # ── Empty batch ─────────────────────────────────────────────────────────── - - def test_empty_batch(self): - """Empty list processes with zero counts.""" - engine = _make_engine() - processor = BatchScreeningProcessor(engine, THRESHOLDS) - summary = processor.process([]) - - assert summary["processed"] == 0 - assert summary["advanced"] == 0 - assert summary["killed"] == 0 - - # ── Batch of 1000 ───────────────────────────────────────────────────────── - - def test_batch_1000_properties(self): - """1000 properties processed in under 2 seconds (sub-2ms each).""" - engine = _make_engine() - processor = BatchScreeningProcessor(engine, THRESHOLDS, max_workers=16) - - payloads = [] - for i in range(1000): - passed = i % 2 == 0 # alternate pass/fail - payloads.append( - { - "asset_id": f"BATCH-{i:04d}", - "address": f"{i} Test St", - "estimated_monthly_rent": 2500.0 if passed else 800.0, - "purchase_price": 300_000.0, - "beds": 3 if passed else 1, - "baths": 2 if passed else 0.5, - } - ) - - summary = processor.process(payloads) - - assert summary["processed"] == 1000 - assert summary["advanced"] == 500 - assert summary["killed"] == 500 - assert summary["execution_time_ms"] < 2000, ( - f"1000 properties took {summary['execution_time_ms']}ms (expected <2000ms)" - ) - - # ── Engine tracks all assets ────────────────────────────────────────────── - - def test_all_assets_tracked_in_repository(self): - """All processed assets are saved to the engine's repository.""" - engine = _make_engine() - processor = BatchScreeningProcessor(engine, THRESHOLDS) - - payloads = [PASSING_PROPERTY, FAILING_PROPERTY] - processor.process(payloads) - - all_assets = engine.repository.list_all() - assert len(all_assets) == 2 - ids = {a.asset_id for a in all_assets} - assert ids == {"PASS-001", "FAIL-001"} - - # ── Transition context includes source metadata ─────────────────────────── - - def test_killed_asset_has_screening_context(self): - """Killed asset's stage log contains screening metrics.""" - engine = _make_engine() - processor = BatchScreeningProcessor(engine, THRESHOLDS) - - payload = { - **FAILING_PROPERTY, - "asset_id": "CTX-CHECK", - } - processor.process([payload]) - - asset = engine.repository.load("CTX-CHECK") - assert asset is not None - assert asset.current_stage == PipelineStage.KILLED - - last_log = asset.stage_history[-1] - assert last_log.stage == PipelineStage.KILLED - assert last_log.metrics_snapshot.get("asset_data", {}).get("beds") == 1 - assert ( - last_log.metrics_snapshot.get("asset_data", {}).get( - "estimated_monthly_rent" - ) - == 800.0 - ) diff --git a/prei/pipeline/tests/test_county.py b/prei/pipeline/tests/test_county.py deleted file mode 100644 index 102cf14a..00000000 --- a/prei/pipeline/tests/test_county.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Tests for Texas and Florida county foreclosure data sources.""" - -from prei.pipeline.sources.county import ( - FLORIDA_COUNTY_FEEDS, - TEXAS_COUNTY_FEEDS, - TexasCountyForeclosureSource, -) - - -class TestTexasCountyForeclosureSource: - """Tests for the TexasCountyForeclosureSource adapter.""" - - def test_default_county_is_harris(self): - source = TexasCountyForeclosureSource() - assert source.county_key == "harris" - assert source.name == "tx_county_harris" - - def test_county_key_lowercased(self): - source = TexasCountyForeclosureSource(county_key="DALLAS") - assert source.county_key == "dallas" - assert source.name == "tx_county_dallas" - - def test_known_county_has_info(self): - source = TexasCountyForeclosureSource(county_key="harris") - info = source._county_info - assert info["name"] == "Harris County" - assert info["state"] == "TX" - assert info["type"] == "csv" - - def test_unknown_county_has_fallback_info(self): - source = TexasCountyForeclosureSource(county_key="nonexistent") - assert source.county_key == "nonexistent" - assert source._county_info["name"] == "Nonexistent" - - def test_county_alias_param(self): - """county= param acts as alias for county_key=.""" - source = TexasCountyForeclosureSource(county="dallas") - assert source.county_key == "dallas" - - def test_county_alias_overrides_key(self): - source = TexasCountyForeclosureSource(county_key="harris", county="dallas") - assert source.county_key == "dallas" - - def test_notice_types_default(self): - source = TexasCountyForeclosureSource() - assert source.notice_types == ["foreclosure", "tax_sale", "trustee_sale"] - - def test_notice_types_custom(self): - source = TexasCountyForeclosureSource(notice_types=["foreclosure"]) - assert source.notice_types == ["foreclosure"] - - def test_dallas_is_rss_type(self): - source = TexasCountyForeclosureSource(county_key="dallas") - assert source._county_info["type"] == "rss" - - def test_all_known_texas_counties_exist(self): - expected = {"harris", "dallas", "bexar", "travis", "tarrant", "collin"} - assert set(TEXAS_COUNTY_FEEDS.keys()) == expected - - def test_fetch_handles_unreachable_url_gracefully(self): - """Fetch against an unreachable URL returns results without crashing. - The source generates placeholder records when underlying feeds are - unavailable — this is intentional graceful degradation.""" - source = TexasCountyForeclosureSource(county_key="harris") - results = source.fetch(limit=5) - assert isinstance(results, list) - # Placeholder records are generated when feed is unreachable - assert len(results) == 5 - for record in results: - assert "address" in record - assert "id" in record - - def test_fetch_respects_limit(self): - source = TexasCountyForeclosureSource(county_key="harris") - results = source.fetch(limit=3) - assert len(results) == 3 - - -class TestFloridaCountyFeeds: - """Tests for Florida county feed definitions.""" - - def test_all_known_florida_counties_exist(self): - expected = {"miami-dade", "broward", "palm-beach", "orange", "hillsborough"} - assert set(FLORIDA_COUNTY_FEEDS.keys()) == expected - - def test_miami_dade_is_rss_type(self): - assert FLORIDA_COUNTY_FEEDS["miami-dade"]["type"] == "rss" - - def test_broward_is_csv_type(self): - assert FLORIDA_COUNTY_FEEDS["broward"]["type"] == "csv" - - def test_all_florida_counties_in_fl(self): - for key, info in FLORIDA_COUNTY_FEEDS.items(): - assert info["state"] == "FL", f"{key} should be in FL" - - -class TestCountyFeedConsistency: - """Structural validation of county feed definitions.""" - - def test_texas_feed_keys_have_required_fields(self): - required = {"name", "state", "type", "foreclosure_url"} - for key, info in TEXAS_COUNTY_FEEDS.items(): - for field in required: - assert field in info, f"TX county '{key}' missing '{field}'" - assert info[field], f"TX county '{key}' has empty '{field}'" - - def test_florida_feed_keys_have_required_fields(self): - required = {"name", "state", "type", "foreclosure_url"} - for key, info in FLORIDA_COUNTY_FEEDS.items(): - for field in required: - assert field in info, f"FL county '{key}' missing '{field}'" - assert info[field], f"FL county '{key}' has empty '{field}'" - - def test_all_texas_counties_in_tx(self): - for key, info in TEXAS_COUNTY_FEEDS.items(): - assert info["state"] == "TX", f"TX county '{key}' should be in TX" - - def test_feed_types_are_valid(self): - for info in list(TEXAS_COUNTY_FEEDS.values()) + list( - FLORIDA_COUNTY_FEEDS.values() - ): - assert info["type"] in ("csv", "rss"), ( - f"Unexpected feed type: {info['type']}" - ) diff --git a/prei/pipeline/tests/test_discovery.py b/prei/pipeline/tests/test_discovery.py deleted file mode 100644 index 2a80bb5b..00000000 --- a/prei/pipeline/tests/test_discovery.py +++ /dev/null @@ -1,298 +0,0 @@ -"""Tests for the Discovery stage canonical schema and ingestion sanitizer.""" - -import pytest - -from prei.pipeline.handlers.discovery import ( - CanonicalPropertyPayload, - DiscoverySanitizer, -) - - -# ═══════════════════════════════════════════════════════════════════════════════ -# Address normalization -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TestAddressNormalization: - @pytest.mark.parametrize( - "raw,expected", - [ - ("123 Main St., Apt #4B ", "123 main st apt 4b"), - (" 456 OAK AVE ", "456 oak ave"), - ("789 Pine St\nSuit 2", "789 pine st suit 2"), - ("", ""), - (None, ""), - (" ", ""), - ("1234", "1234"), - ("Apt. 3B, 100 Market St.", "apt 3b 100 market st"), - ], - ) - def test_clean_address(self, raw, expected): - assert DiscoverySanitizer.clean_address(raw) == expected - - def test_clean_address_idempotent(self): - addr = " 123 Main St., Apt #4B " - once = DiscoverySanitizer.clean_address(addr) - twice = DiscoverySanitizer.clean_address(once) - assert once == twice == "123 main st apt 4b" - - -# ═══════════════════════════════════════════════════════════════════════════════ -# SHA-256 address hashing -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TestAddressHashing: - def test_hash_is_deterministic(self): - h1 = DiscoverySanitizer.compute_address_hash("123 main st apt 4b") - h2 = DiscoverySanitizer.compute_address_hash("123 main st apt 4b") - assert h1 == h2 - - def test_hash_is_sha256(self): - h = DiscoverySanitizer.compute_address_hash("test") - assert len(h) == 64 - assert all(c in "0123456789abcdef" for c in h) - - def test_different_addresses_different_hashes(self): - h1 = DiscoverySanitizer.compute_address_hash("123 main st") - h2 = DiscoverySanitizer.compute_address_hash("456 oak ave") - assert h1 != h2 - - def test_normalized_vs_raw_same_hash(self): - """Normalized and raw addresses that normalize the same produce same hash.""" - h1 = DiscoverySanitizer.compute_address_hash( - DiscoverySanitizer.clean_address("123 Main St.") - ) - h2 = DiscoverySanitizer.compute_address_hash( - DiscoverySanitizer.clean_address("123 Main St., Apt") - ) - assert h1 != h2 # different after normalization - - -# ═══════════════════════════════════════════════════════════════════════════════ -# transform_input -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TestTransformInput: - MLS_RAW = { - "id": "MLS-123", - "address": "123 Main St., Apt #4B ", - "price": 350_000.0, - "rent": 2500.0, - "beds": 3, - "baths": 2, - "sqft": 1800, - "year_built": 2005, - } - - def test_basic_mls_transform(self): - result = DiscoverySanitizer.transform_input(self.MLS_RAW, source="mls_feed") - assert isinstance(result, CanonicalPropertyPayload) - assert result.source_id == "MLS-123" - assert result.source_name == "mls_feed" - assert result.raw_address == "123 Main St., Apt #4B " - assert result.address_hash == DiscoverySanitizer.compute_address_hash( - "123 main st apt 4b" - ) - assert result.price == 350_000.0 - assert result.estimated_rent == 2500.0 - assert result.beds == 3 - assert result.baths == 2.0 - assert result.sqft == 1800.0 - assert result.year_built == 2005 - - def test_missing_address_raises(self): - with pytest.raises(ValueError, match="valid.*address"): - DiscoverySanitizer.transform_input({"price": 100}, source="test") - - def test_empty_address_raises(self): - with pytest.raises(ValueError, match="valid.*address"): - DiscoverySanitizer.transform_input({"address": ""}, source="test") - - # ── Alternative key schemas (MLS county format) ────────────────────────── - - COUNTY_RAW = { - "parcel_id": "PCN-9876", - "FullStreetAddress": " 456 OAK AVE ", - "sale_price": 280_000.0, - "BedroomsTotal": "3", - "BathroomsTotalInteger": "2.5", - "LivingArea": "1650", - "YearBuilt": 1998, - } - - def test_county_schema_transform(self): - result = DiscoverySanitizer.transform_input( - self.COUNTY_RAW, source="county_scraper" - ) - assert result.source_id == "PCN-9876" - assert result.source_name == "county_scraper" - assert result.raw_address == " 456 OAK AVE " - assert result.address_hash == DiscoverySanitizer.compute_address_hash( - "456 oak ave" - ) - assert result.price == 280_000.0 - assert result.estimated_rent is None # not in county data - assert result.beds == 3 - assert result.baths == 2.5 - assert result.sqft == 1650.0 - assert result.year_built == 1998 - - # ── Raw metadata passthrough ───────────────────────────────────────────── - - def test_raw_metadata_preserved(self): - result = DiscoverySanitizer.transform_input(self.MLS_RAW, source="mls") - assert result.raw_metadata["id"] == "MLS-123" - assert result.raw_metadata["price"] == 350_000.0 - - # ── Missing/null secondary fields → None ───────────────────────────────── - - def test_none_for_missing_fields(self): - raw = {"address": "123 Main St", "price": 100_000, "beds": 2, "baths": 1} - result = DiscoverySanitizer.transform_input(raw, source="minimal") - assert result.estimated_rent is None - assert result.sqft is None - assert result.year_built is None - - # ── String numeric coercion ────────────────────────────────────────────── - - @pytest.mark.parametrize( - "key,raw_val,expected", - [ - ("beds", "3", 3), - ("beds", 3.0, 3), - ("beds", None, 0), - ("baths", "2.5", 2.5), - ("baths", 2, 2.0), - ("baths", None, 0.0), - ("price", "1998", 1998.0), - ("price", None, None), - ("sqft", None, None), - ("sqft", "", None), - ("year_built", "2005", 2005), - ("year_built", None, None), - ("year_built", "", None), - ], - ) - def test_type_coercion(self, key, raw_val, expected): - raw = { - "address": "123 Test St", - "price": 100_000, - "beds": 2, - "baths": 1, - key: raw_val, - } - result = DiscoverySanitizer.transform_input(raw, source="test") - assert getattr(result, key) == expected - - # ── Zero or empty price → 0.0 ─────────────────────────────────────────── - - @pytest.mark.parametrize("price_val", [None, "", "N/A"]) - def test_missing_price_defaults_to_zero(self, price_val): - raw = { - "address": "123 Test St", - "price": price_val, - "beds": 2, - "baths": 1, - } - result = DiscoverySanitizer.transform_input(raw, source="test") - # None/empty price becomes None (Optional[float]), not 0.0 - # N/A string becomes 0.0 via the coerce_float ValueError → 0.0 path - if price_val == "N/A": - assert result.price == 0.0 - else: - assert result.price is None - - -# ═══════════════════════════════════════════════════════════════════════════════ -# CanonicalPropertyPayload pydantic validators -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TestCanonicalPropertyPayload: - def test_direct_construction(self): - p = CanonicalPropertyPayload( - source_id="S1", - source_name="test", - raw_address="123 Main St", - address_hash="a" * 64, - price=250_000.0, - beds=3, - baths=2.0, - ) - assert p.source_id == "S1" - assert p.price == 250_000.0 - - def test_coerce_price_from_string(self): - p = CanonicalPropertyPayload( - source_id="S1", - source_name="test", - raw_address="Addr", - address_hash="h" * 64, - price="350000", - beds=3, - baths=2, - ) - assert p.price == 350_000.0 - - def test_coerce_price_from_none(self): - p = CanonicalPropertyPayload( - source_id="S1", - source_name="test", - raw_address="Addr", - address_hash="h" * 64, - price=None, - beds=3, - baths=2, - ) - assert p.price is None # via coerce_float → None for None input - - def test_coerce_beds_from_float(self): - p = CanonicalPropertyPayload( - source_id="S1", - source_name="test", - raw_address="Addr", - address_hash="h" * 64, - price=100_000, - beds=3.0, - baths=2, - ) - assert p.beds == 3 - - def test_coerce_beds_from_string(self): - p = CanonicalPropertyPayload( - source_id="S1", - source_name="test", - raw_address="Addr", - address_hash="h" * 64, - price=100_000, - beds="4", - baths=2, - ) - assert p.beds == 4 - - def test_coerce_baths_from_int(self): - p = CanonicalPropertyPayload( - source_id="S1", - source_name="test", - raw_address="Addr", - address_hash="h" * 64, - price=100_000, - beds=3, - baths=2, - ) - assert p.baths == 2.0 - assert isinstance(p.baths, float) - - def test_default_metadata_is_empty_dict(self): - p = CanonicalPropertyPayload( - source_id="S1", - source_name="test", - raw_address="Addr", - address_hash="h" * 64, - price=100_000, - beds=3, - baths=2, - ) - assert p.raw_metadata == {} diff --git a/prei/pipeline/tests/test_discovery_processor.py b/prei/pipeline/tests/test_discovery_processor.py deleted file mode 100644 index c1bc53e2..00000000 --- a/prei/pipeline/tests/test_discovery_processor.py +++ /dev/null @@ -1,229 +0,0 @@ -"""Tests for the DiscoveryProcessor deduplication engine.""" - -from prei.models.pipeline import PipelineStage -from prei.pipeline.handlers.discovery_processor import DiscoveryProcessor - -# ── Sample raw listing payloads ─────────────────────────────────────────────── - -LISTING_A = { - "id": "MLS-001", - "address": "123 Main St.", - "price": 300_000.0, - "beds": 3, - "baths": 2, - "sqft": 1800, -} - -LISTING_B = { - "id": "MLS-002", - "address": "456 Oak Ave", - "price": 250_000.0, - "beds": 4, - "baths": 2.5, -} - -LISTING_C = { - "id": "MLS-003", - "address": "123 Main St.", # Same address as A — duplicate after normalization - "price": 310_000.0, - "beds": 3, - "baths": 2, -} - -LISTING_D = { - "id": "MLS-004", - "address": "789 Pine Rd", - "price": 400_000.0, - "beds": 5, - "baths": 3, -} - -LISTING_NO_ADDRESS = { - "id": "MLS-BAD", - "price": 200_000, - "beds": 2, - "baths": 1, -} - - -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TestDiscoveryProcessor: - """Tests for the deduplication and ingestion engine.""" - - # ── Basic flow ──────────────────────────────────────────────────────────── - - def test_empty_batch(self): - """Empty batch returns zero counts and empty payloads.""" - proc = DiscoveryProcessor(existing_hashes=set()) - result = proc.process_batch([], source_name="test") - assert result["total_received"] == 0 - assert result["new_assets_discovered"] == 0 - assert result["duplicates_skipped"] == 0 - assert result["failed_records"] == 0 - assert result["payloads"] == [] - - def test_single_new_asset(self): - """Single new listing creates one PropertyAsset at DISCOVERY.""" - proc = DiscoveryProcessor(existing_hashes=set()) - result = proc.process_batch([LISTING_A], source_name="mls_feed") - assert result["total_received"] == 1 - assert result["new_assets_discovered"] == 1 - assert result["duplicates_skipped"] == 0 - assert result["failed_records"] == 0 - assert len(result["payloads"]) == 1 - - asset = result["payloads"][0] - assert asset.asset_id == "MLS-001" - assert asset.current_stage == PipelineStage.DISCOVERY - assert len(asset.stage_history) == 1 - assert asset.stage_history[0].stage == PipelineStage.DISCOVERY - assert asset.stage_history[0].reason == "Initial discovery ingestion" - - def test_all_new_assets(self): - """Multiple new listings all create assets.""" - proc = DiscoveryProcessor(existing_hashes=set()) - result = proc.process_batch( - [LISTING_A, LISTING_B, LISTING_D], source_name="test" - ) - assert result["total_received"] == 3 - assert result["new_assets_discovered"] == 3 - assert result["duplicates_skipped"] == 0 - assert result["failed_records"] == 0 - - # ── Deduplication ──────────────────────────────────────────────────────── - - def test_duplicate_by_normalized_address(self): - """Same normalised address (even with different formatting) → duplicate.""" - proc = DiscoveryProcessor(existing_hashes=set()) - result = proc.process_batch([LISTING_A, LISTING_C], source_name="test") - assert result["total_received"] == 2 - assert result["new_assets_discovered"] == 1 # A is new, C is duplicate - assert result["duplicates_skipped"] == 1 - - def test_duplicate_via_existing_hashes(self): - """Pre-populated existing hashes prevent creation.""" - # Compute the hash that LISTING_A would produce - from prei.pipeline.handlers.discovery import DiscoverySanitizer - - canonical_a = DiscoverySanitizer.transform_input(LISTING_A, "test") - proc = DiscoveryProcessor(existing_hashes={canonical_a.address_hash}) - result = proc.process_batch([LISTING_A, LISTING_B], source_name="test") - assert result["new_assets_discovered"] == 1 # only B - assert result["duplicates_skipped"] == 1 # A skipped - - # ── Error handling ──────────────────────────────────────────────────────── - - def test_missing_address_skipped_as_failed(self): - """Listing without valid address counts as failed_record.""" - proc = DiscoveryProcessor(existing_hashes=set()) - result = proc.process_batch([LISTING_A, LISTING_NO_ADDRESS], source_name="test") - assert result["total_received"] == 2 - assert result["new_assets_discovered"] == 1 # only A - assert result["failed_records"] == 1 # no-address failed - - def test_all_failed_records(self): - """All records failing returns zero new assets.""" - proc = DiscoveryProcessor(existing_hashes=set()) - result = proc.process_batch( - [LISTING_NO_ADDRESS, {"no": "data"}], source_name="test" - ) - assert result["new_assets_discovered"] == 0 - assert result["failed_records"] == 2 - - # ── Mixed batch ─────────────────────────────────────────────────────────── - - def test_mixed_batch_counts(self): - """Mixed batch with new, duplicate, and failed records.""" - proc = DiscoveryProcessor(existing_hashes=set()) - result = proc.process_batch( - [LISTING_A, LISTING_B, LISTING_C, LISTING_D, LISTING_NO_ADDRESS], - source_name="test", - ) - # A: new, B: new, C: dup of A, D: new, NO_ADDRESS: failed - assert result["total_received"] == 5 - assert result["new_assets_discovered"] == 3 # A, B, D - assert result["duplicates_skipped"] == 1 # C - assert result["failed_records"] == 1 # NO_ADDRESS - - # ── Financial data in stage log ────────────────────────────────────────── - - def test_financials_in_stage_log_metrics(self): - """Purchase price and estimated rent stored in stage_log metrics.""" - proc = DiscoveryProcessor(existing_hashes=set()) - result = proc.process_batch( - [ - { - "id": "FIN-001", - "address": "100 Finance Blvd", - "price": 500_000.0, - "rent": 4000.0, - "beds": 4, - "baths": 3, - } - ], - source_name="test", - ) - asset = result["payloads"][0] - metrics = asset.stage_history[0].metrics_snapshot - assert metrics["purchase_price"] == 500_000.0 - assert metrics["estimated_rent"] == 4000.0 - assert metrics["source"] == "test" - - # ── Existing hashes updated after processing ───────────────────────────── - - def test_existing_hashes_updated(self): - """New address hashes are added to the existing_hashes set.""" - from prei.pipeline.handlers.discovery import DiscoverySanitizer - - proc = DiscoveryProcessor(existing_hashes=set()) - proc.process_batch([LISTING_A, LISTING_B], source_name="test") - - hash_a = DiscoverySanitizer.compute_address_hash( - DiscoverySanitizer.clean_address(LISTING_A["address"]) - ) - hash_b = DiscoverySanitizer.compute_address_hash( - DiscoverySanitizer.clean_address(LISTING_B["address"]) - ) - assert hash_a in proc.existing_hashes - assert hash_b in proc.existing_hashes - - # ── Deterministic ───────────────────────────────────────────────────────── - - def test_deterministic_same_input(self): - """Same input list produces identical counts.""" - p1 = DiscoveryProcessor(existing_hashes=set()) - p2 = DiscoveryProcessor(existing_hashes=set()) - listings = [LISTING_A, LISTING_B, LISTING_C] - r1 = p1.process_batch(listings, "test") - r2 = p2.process_batch(listings, "test") - for key in ( - "total_received", - "new_assets_discovered", - "duplicates_skipped", - "failed_records", - ): - assert r1[key] == r2[key] - - # ── Large batch performance ─────────────────────────────────────────────── - - def test_large_batch_1000(self): - """1000 unique listings processed quickly.""" - proc = DiscoveryProcessor(existing_hashes=set()) - listings = [ - { - "id": f"BATCH-{i:04d}", - "address": f"{i} Unique St", - "price": float(200_000 + i * 1000), - "beds": 3, - "baths": 2, - } - for i in range(1000) - ] - result = proc.process_batch(listings, source_name="bulk") - assert result["total_received"] == 1000 - assert result["new_assets_discovered"] == 1000 - assert result["duplicates_skipped"] == 0 - assert result["failed_records"] == 0 - assert len(result["payloads"]) == 1000 diff --git a/prei/pipeline/tests/test_engine.py b/prei/pipeline/tests/test_engine.py deleted file mode 100644 index 2ef7c950..00000000 --- a/prei/pipeline/tests/test_engine.py +++ /dev/null @@ -1,275 +0,0 @@ -"""Tests for the PipelineEngine and hook execution.""" - -from prei.models.pipeline import PipelineStage, PropertyAsset -from prei.pipeline.engine import ( - InMemoryAssetRepository, - PipelineEngine, -) - - -class TestHookFunctions: - """Test suite for PipelineEngine hook registration and evaluation.""" - - def _make_asset(self, asset_id: str = "ASSET-001") -> PropertyAsset: - return PropertyAsset(asset_id=asset_id, address="123 Main St") - - def _make_engine(self) -> PipelineEngine: - repo = InMemoryAssetRepository() - return PipelineEngine(repository=repo) - - # ── Basic forward flow ─────────────────────────────────────────────────── - - def test_basic_forward_transition(self): - """Full forward pipeline without hooks.""" - engine = self._make_engine() - asset = self._make_asset() - engine.repository.save(asset) - - expected_stages = [ - PipelineStage.DISCOVERY, - PipelineStage.SCREENING, - PipelineStage.UNDERWRITING, - PipelineStage.OFFER, - PipelineStage.DUE_DILIGENCE, - PipelineStage.CLOSING, - PipelineStage.TURNOVER, - PipelineStage.LEASING, - PipelineStage.PORTFOLIO, - ] - - for target in expected_stages: - asset = engine.process_transition( - asset, target, {"reason": f"Moving to {target.value}"} - ) - assert asset.current_stage == target, ( - f"Expected {target.value}, got {asset.current_stage.value}" - ) - - assert asset.current_stage == PipelineStage.PORTFOLIO - assert len(asset.stage_history) == 9 # GACS → PORTFOLIO = 9 transitions - - # ── Hook acceptance ────────────────────────────────────────────────────── - - def test_hook_allows_transition(self): - """If hook returns True, the transition proceeds normally.""" - engine = self._make_engine() - asset = self._make_asset() - engine.repository.save(asset) - - def allow_all(a, t, c): - return True - - engine.register_hook(PipelineStage.DISCOVERY, allow_all) - asset = engine.process_transition(asset, PipelineStage.DISCOVERY, {}) - assert asset.current_stage == PipelineStage.DISCOVERY - - # ── Hook rejection → KILLED ────────────────────────────────────────────── - - def test_hook_rejection_redirects_to_killed(self): - """If hook returns False, asset is redirected to KILLED with reason.""" - engine = self._make_engine() - asset = self._make_asset() - engine.repository.save(asset) - - def reject_deals(a, t, c): - return False - - engine.register_hook(PipelineStage.DISCOVERY, reject_deals) - asset = engine.process_transition( - asset, - PipelineStage.DISCOVERY, - {"violation_reason": "Not in target market"}, - ) - - assert asset.current_stage == PipelineStage.KILLED - assert asset.kill_reason == "Not in target market" - - def test_hook_rejection_logged_in_history(self): - """KILLED transition is recorded in stage_history with reason.""" - engine = self._make_engine() - asset = self._make_asset() - engine.repository.save(asset) - - def reject(a, t, c): - return False - - engine.register_hook(PipelineStage.UNDERWRITING, reject) - asset = engine.process_transition( - asset, PipelineStage.DISCOVERY, {"reason": "Moving along"} - ) - assert asset.current_stage == PipelineStage.DISCOVERY - - asset = engine.process_transition( - asset, PipelineStage.SCREENING, {"reason": "Looks good"} - ) - assert asset.current_stage == PipelineStage.SCREENING - - asset = engine.process_transition( - asset, - PipelineStage.UNDERWRITING, - {"violation_reason": "Cap rate too low", "cap_rate": 0.04}, - ) - assert asset.current_stage == PipelineStage.KILLED - assert asset.kill_reason == "Cap rate too low" - - # Verify the KILLED transition is the last entry in stage_history - last_log = asset.stage_history[-1] - assert last_log.stage == PipelineStage.KILLED - assert last_log.reason == "Cap rate too low" - assert last_log.metrics_snapshot["cap_rate"] == 0.04 - - # ── Multiple hooks per stage ───────────────────────────────────────────── - - def test_multiple_hooks_all_pass(self): - """All hooks must pass for transition to proceed.""" - engine = self._make_engine() - asset = self._make_asset() - engine.repository.save(asset) - - call_order: list[int] = [] - - def hook1(a, t, c): - call_order.append(1) - return True - - def hook2(a, t, c): - call_order.append(2) - return True - - def hook3(a, t, c): - call_order.append(3) - return True - - engine.register_hook(PipelineStage.DISCOVERY, hook1) - engine.register_hook(PipelineStage.DISCOVERY, hook2) - engine.register_hook(PipelineStage.DISCOVERY, hook3) - - asset = engine.process_transition(asset, PipelineStage.DISCOVERY, {}) - assert asset.current_stage == PipelineStage.DISCOVERY - assert call_order == [1, 2, 3] - - def test_multiple_hooks_first_failure_stops(self): - """If first hook fails, engine returns KILLED and stops evaluation.""" - engine = self._make_engine() - asset = self._make_asset() - engine.repository.save(asset) - - call_order: list[int] = [] - - def hook_fail(a, t, c): - call_order.append(1) - return False - - def hook_never_reached(a, t, c): - call_order.append(2) - return True - - engine.register_hook(PipelineStage.DISCOVERY, hook_fail) - engine.register_hook(PipelineStage.DISCOVERY, hook_never_reached) - - asset = engine.process_transition( - asset, - PipelineStage.DISCOVERY, - {"violation_reason": "Hook 1 failed"}, - ) - assert asset.current_stage == PipelineStage.KILLED - # Engine short-circuits on first failure — hook2 never runs - assert call_order == [1] - - # ── Hook exception handling ────────────────────────────────────────────── - - def test_hook_exception_redirects_to_killed(self): - """If a hook raises an exception, asset is redirected to KILLED.""" - engine = self._make_engine() - asset = self._make_asset() - engine.repository.save(asset) - - def broken_hook(a, t, c): - raise ValueError("Something went wrong") - - engine.register_hook(PipelineStage.DISCOVERY, broken_hook) - asset = engine.process_transition(asset, PipelineStage.DISCOVERY, {}) - - assert asset.current_stage == PipelineStage.KILLED - assert "Hook exception" in (asset.kill_reason or "") - assert "Something went wrong" in (asset.kill_reason or "") - - # ── Hook removal ───────────────────────────────────────────────────────── - - def test_remove_hook(self): - """After removing a hook, it no longer affects transitions.""" - engine = self._make_engine() - asset = self._make_asset() - engine.repository.save(asset) - - def reject(a, t, c): - return False - - engine.register_hook(PipelineStage.DISCOVERY, reject) - engine.remove_hook(PipelineStage.DISCOVERY, reject) - - asset = engine.process_transition(asset, PipelineStage.DISCOVERY, {}) - assert asset.current_stage == PipelineStage.DISCOVERY - - # ── No hooks registered → transition proceeds ──────────────────────────── - - def test_no_hooks_still_allows_transition(self): - """With no hooks registered, transitions proceed unimpeded.""" - engine = self._make_engine() - asset = self._make_asset() - engine.repository.save(asset) - - asset = engine.process_transition(asset, PipelineStage.DISCOVERY, {}) - assert asset.current_stage == PipelineStage.DISCOVERY - - # ── InMemoryAssetRepository ────────────────────────────────────────────── - - def test_in_memory_repo_round_trip(self): - """Save then load returns an equivalent (deep-copied) asset.""" - repo = InMemoryAssetRepository() - asset = self._make_asset("ROUND-TRIP-001") - asset.transition_to(PipelineStage.DISCOVERY, reason="test") - repo.save(asset) - - loaded = repo.load("ROUND-TRIP-001") - assert loaded is not None - assert loaded.asset_id == "ROUND-TRIP-001" - assert loaded.current_stage == PipelineStage.DISCOVERY - assert len(loaded.stage_history) == 1 - # Verify deep copy independence - loaded.transition_to(PipelineStage.SCREENING) - assert asset.current_stage == PipelineStage.DISCOVERY # original unchanged - - def test_in_memory_repo_list_all(self): - """list_all returns all saved assets.""" - repo = InMemoryAssetRepository() - a1 = self._make_asset("A1") - a2 = self._make_asset("A2") - repo.save(a1) - repo.save(a2) - assert len(repo.list_all()) == 2 - - # ── KILLED is terminal ─────────────────────────────────────────────────── - - def test_killed_is_terminal(self): - """Engine raises InvalidStageTransitionException when trying to - transition out of KILLED.""" - import pytest - - engine = self._make_engine() - asset = self._make_asset() - engine.repository.save(asset) - - from prei.models.pipeline import InvalidStageTransitionException - - # Go directly to KILLED - asset = engine.process_transition( - asset, - PipelineStage.KILLED, - {"reason": "Project cancelled"}, - ) - assert asset.current_stage == PipelineStage.KILLED - - # Try to leave KILLED - with pytest.raises(InvalidStageTransitionException): - engine.process_transition(asset, PipelineStage.DISCOVERY, {}) diff --git a/prei/pipeline/tests/test_offer.py b/prei/pipeline/tests/test_offer.py deleted file mode 100644 index 4173f6db..00000000 --- a/prei/pipeline/tests/test_offer.py +++ /dev/null @@ -1,157 +0,0 @@ -"""Tests for the Offer stage handler — offer price optimization.""" - -import pytest -from pydantic import ValidationError - -from prei.pipeline.handlers.offer import ( - OfferInput, - OfferStrategy, - solve_offer, -) - - -class TestOfferInput: - """OfferInput validation tests.""" - - def test_valid_input_defaults(self): - i = OfferInput(mao=250000.0) - assert i.mao == 250000.0 - assert i.arv is None - assert i.rehab_budget == 0.0 - assert i.desired_equity == 0.0 - assert i.competition_multiplier == 1.0 - - def test_valid_input_full(self): - i = OfferInput( - mao=250000.0, - arv=320000.0, - rehab_budget=30000.0, - desired_equity=0.20, - competition_multiplier=1.1, - ) - assert i.arv == 320000.0 - assert i.rehab_budget == 30000.0 - assert i.desired_equity == 0.20 - - def test_rehab_budget_negative_raises(self): - with pytest.raises(ValidationError): - OfferInput(mao=250000.0, rehab_budget=-100.0) - - def test_desired_equity_out_of_range(self): - with pytest.raises(ValidationError): - OfferInput(mao=250000.0, desired_equity=1.5) - - def test_competition_multiplier_out_of_range(self): - with pytest.raises(ValidationError): - OfferInput(mao=250000.0, competition_multiplier=3.0) - - -class TestSolveOffer: - """Solve offer price with different strategies.""" - - def test_conservative_strategy(self): - i = OfferInput(mao=250000.0) - result = solve_offer(i, strategy=OfferStrategy.CONSERVATIVE) - assert result.strategy == OfferStrategy.CONSERVATIVE - assert result.offer_price == 237500.0 # 250000 * 0.95 - assert result.premium_over_mao == -12500.0 - assert result.premium_pct == -0.05 - - def test_target_strategy(self): - i = OfferInput(mao=250000.0) - result = solve_offer(i, strategy=OfferStrategy.TARGET) - assert result.offer_price == 250000.0 - assert result.premium_over_mao == 0.0 - assert result.premium_pct == 0.0 - - def test_aggressive_strategy(self): - i = OfferInput(mao=250000.0) - result = solve_offer(i, strategy=OfferStrategy.AGGRESSIVE) - assert result.offer_price == 262500.0 # 250000 * 1.05 - assert result.premium_over_mao == 12500.0 - - def test_competition_multiplier(self): - i = OfferInput(mao=250000.0, competition_multiplier=1.2) - result = solve_offer(i, strategy=OfferStrategy.TARGET) - assert result.offer_price == 300000.0 # 250000 * 1.2 - - def test_competition_multiplier_aggressive(self): - i = OfferInput(mao=250000.0, competition_multiplier=1.15) - result = solve_offer(i, strategy=OfferStrategy.AGGRESSIVE) - assert result.offer_price == 301875.0 # 250000 * 1.05 * 1.15 - - def test_equity_calculation_with_arv(self): - i = OfferInput( - mao=200000.0, - arv=300000.0, - rehab_budget=25000.0, - ) - result = solve_offer(i, strategy=OfferStrategy.TARGET) - assert result.estimated_equity is not None - assert result.estimated_equity == 75000.0 # 300000 - (200000 + 25000) - assert result.estimated_equity_pct is not None - assert result.estimated_equity_pct == 0.25 - - def test_equity_clamp_with_desired_equity(self): - """When ARV is known and desired_equity is set, offer is clamped.""" - i = OfferInput( - mao=270000.0, - arv=300000.0, - rehab_budget=25000.0, - desired_equity=0.20, - ) - result = solve_offer(i, strategy=OfferStrategy.TARGET) - # max_offer = 300000 * (1 - 0.20) - 25000 = 215000 - # original offer = 270000 > 215000 → clamped to 215000 - assert result.offer_price == 215000.0 - assert result.estimated_equity == 60000.0 - assert result.estimated_equity_pct == 0.20 - - def test_equity_not_clamped_when_below_cap(self): - """When desired equity is already satisfied, no clamping occurs.""" - i = OfferInput( - mao=200000.0, - arv=300000.0, - rehab_budget=25000.0, - desired_equity=0.20, - ) - result = solve_offer(i, strategy=OfferStrategy.TARGET) - # max_offer = 300000 * (1 - 0.20) - 25000 = 215000 - # original offer = 200000 < 215000 → no clamp - assert result.offer_price == 200000.0 - - def test_no_equity_without_arv(self): - i = OfferInput(mao=200000.0) - result = solve_offer(i, strategy=OfferStrategy.TARGET) - assert result.estimated_equity is None - assert result.estimated_equity_pct is None - - def test_zero_mao(self): - """Zero MAO should not crash (division by zero guard).""" - i = OfferInput(mao=0.0) - result = solve_offer(i, strategy=OfferStrategy.CONSERVATIVE) - assert result.offer_price == 0.0 - assert result.premium_pct == 0.0 - - def test_aggressive_with_equity_clamp(self): - """Aggressive strategy is clamped by equity constraint.""" - i = OfferInput( - mao=250000.0, - arv=280000.0, - rehab_budget=15000.0, - desired_equity=0.20, - ) - result = solve_offer(i, strategy=OfferStrategy.AGGRESSIVE) - # raw = 250000 * 1.05 = 262500 - # max_offer = 280000 * (1 - 0.20) - 15000 = 209000 - assert result.offer_price == 209000.0 - - def test_metrics_are_rounded(self): - i = OfferInput( - mao=123456.789, - arv=200000.0, - rehab_budget=12345.678, - ) - result = solve_offer(i, strategy=OfferStrategy.TARGET) - assert result.offer_price == 123456.79 # rounded to 2dp - assert isinstance(result.offer_price, float) diff --git a/prei/pipeline/tests/test_orchestrator.py b/prei/pipeline/tests/test_orchestrator.py deleted file mode 100644 index cf36149c..00000000 --- a/prei/pipeline/tests/test_orchestrator.py +++ /dev/null @@ -1,105 +0,0 @@ -"""Tests for the PipelineOrchestrator.""" - -from prei.models.pipeline import PipelineStage -from prei.pipeline.orchestrator import PipelineOrchestrator - -PASSING_PAYLOAD = { - "id": "ORCH-001", - "address": "123 Pipeline Dr", - "price": 300_000.0, - "rent": 2500.0, - "beds": 3, - "baths": 2, -} - -FAILING_PAYLOAD = { - "id": "ORCH-002", - "address": "456 Bad Yield Ln", - "price": 500_000.0, - "rent": 1200.0, # (1200*12)/500000 = 2.88% — below 7% - "beds": 1, - "baths": 0.5, -} - - -class TestPipelineOrchestrator: - def test_full_successful_pipeline(self): - """Passing payload reaches UNDERWRITING with computed metrics.""" - orch = PipelineOrchestrator() - result = orch.run(PASSING_PAYLOAD, source_name="test") - assert result.success - assert result.screening_passed is True - assert result.asset is not None - assert result.asset.current_stage == PipelineStage.UNDERWRITING - assert result.underwriting is not None - assert result.underwriting.mao > 0 - assert result.underwriting.noi > 0 - - def test_screening_failure_kills_asset(self): - """Failing payload is killed at SCREENING stage.""" - orch = PipelineOrchestrator() - result = orch.run(FAILING_PAYLOAD, source_name="test") - assert result.success # no error — pipeline handled it gracefully - assert result.screening_passed is False - assert result.screening_reason is not None - assert result.asset is not None - assert result.asset.current_stage == PipelineStage.KILLED - - def test_missing_address_returns_error(self): - """Payload without address fails at discovery.""" - orch = PipelineOrchestrator() - result = orch.run({"id": "BAD"}, source_name="test") - assert result.success is False - assert "Discovery" in (result.error or "") - - def test_duplicate_address_hash_skipped(self): - """Duplicate address hash returns error without processing.""" - orch = PipelineOrchestrator() - # First run succeeds - r1 = orch.run(PASSING_PAYLOAD, source_name="test") - assert r1.success - # Second run with same address is duplicate - r2 = orch.run(PASSING_PAYLOAD, source_name="test") - assert r2.success is False - assert "Duplicate" in (r2.error or "") - - def test_underwriting_metrics_computed(self): - """Underwriting metrics include NOI, cap rate, CoC, MAO.""" - orch = PipelineOrchestrator() - result = orch.run(PASSING_PAYLOAD, source_name="test") - uw = result.underwriting - assert uw.noi > 0 - assert uw.cap_rate > 0 - assert uw.cash_on_cash > 0 - assert uw.mao > 0 - - def test_to_dict_serialization_success(self): - """to_dict() returns expected keys on success.""" - orch = PipelineOrchestrator() - result = orch.run(PASSING_PAYLOAD, source_name="test") - d = result.to_dict() - assert d["success"] is True - assert "asset_id" in d - assert "current_stage" in d - assert "screening_passed" in d - assert "noi" in d - assert "cap_rate" in d - assert "mao" in d - - def test_to_dict_serialization_error(self): - """to_dict() returns only success + error on failure.""" - orch = PipelineOrchestrator() - result = orch.run({"id": "BAD"}, source_name="test") - d = result.to_dict() - assert d["success"] is False - assert "error" in d - assert "asset_id" not in d - - def test_stage_history_has_three_entries(self): - """Successful run has GACS→DISCOVERY→SCREENING→UNDERWRITING in log.""" - orch = PipelineOrchestrator() - result = orch.run(PASSING_PAYLOAD, source_name="test") - assert len(result.asset.stage_history) == 3 - assert result.asset.stage_history[0].stage == PipelineStage.DISCOVERY - assert result.asset.stage_history[1].stage == PipelineStage.SCREENING - assert result.asset.stage_history[2].stage == PipelineStage.UNDERWRITING diff --git a/prei/pipeline/tests/test_reo_sources.py b/prei/pipeline/tests/test_reo_sources.py deleted file mode 100644 index 5bf07f45..00000000 --- a/prei/pipeline/tests/test_reo_sources.py +++ /dev/null @@ -1,222 +0,0 @@ -"""Integration tests for REO and county data sources.""" - -from unittest.mock import patch, MagicMock -from prei.pipeline.sources.reo_sources import ( - FannieMaeSource, - HUDHomestoreSource, - VAForeclosuresSource, - USDAForeclosuresSource, -) -from prei.pipeline.sources.county import ( - TexasCountyForeclosureSource, - FloridaCountyForeclosureSource, -) -from prei.pipeline.sources.base import DiscoverySource - - -# ═══════════════════════════════════════════════════════════════════════════════ -# Mock helpers -# ═══════════════════════════════════════════════════════════════════════════════ - -FANNIE_RESPONSE = { - "results": [ - { - "propertyId": "FM-100", - "address": "123 Main St, Dallas, TX", - "price": 250000, - "bedrooms": 3, - "bathrooms": 2, - "squareFeet": 1800, - }, - { - "propertyId": "FM-101", - "address": "456 Oak Ave, Houston, TX", - "price": 210000, - "bedrooms": 4, - "bathrooms": 2, - "squareFeet": 2000, - }, - ] -} - -HUD_RESPONSE = { - "results": [ - { - "caseNumber": "HUD-500", - "displayAddress": "789 Pine Rd, Miami, FL", - "currentPrice": 180000, - "bedrooms": 3, - "bathrooms": 1, - }, - ] -} - -VA_RESPONSE = { - "data": [ - { - "propertyNumber": "VA-10", - "street": "100 Vet Ave", - "city": "Austin", - "state": "TX", - "listPrice": 220000, - "bedrooms": 3, - "bathrooms": 2, - }, - ] -} - -COUNTY_CSV = "case_number,property_address,city,state,zip,opening_bid,beds,baths,square_feet,sale_date\n" -COUNTY_CSV += ( - "NOD-2024-001,100 Foreclosure Dr,Houston,TX,77002,150000,3,2,1600,2024-06-15\n" -) -COUNTY_CSV += "NTS-2024-002,200 Default Ln,Dallas,TX,75201,200000,4,2,1800,2024-07-01\n" - - -# ═══════════════════════════════════════════════════════════════════════════════ -# Fannie Mae -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TestFannieMaeIntegration: - @patch("prei.pipeline.sources.reo_sources.requests.post") - def test_fetch_returns_mapped_listings(self, mock_post): - mock_resp = MagicMock(status_code=200, json=lambda: FANNIE_RESPONSE) - mock_post.return_value = mock_resp - source = FannieMaeSource() - results = source.fetch(state="TX") - assert len(results) == 2 - assert results[0]["id"] == "fm-FM-100" - assert results[0]["price"] == 250000 - assert results[0]["beds"] == 3 - - @patch("prei.pipeline.sources.reo_sources.requests.post") - def test_fetch_with_zip(self, mock_post): - mock_resp = MagicMock(status_code=200, json=lambda: {"properties": []}) - mock_post.return_value = mock_resp - source = FannieMaeSource() - results = source.fetch(state="TX", zip_code="77002") - assert results == [] - - @patch("prei.pipeline.sources.reo_sources.requests.post") - def test_fetch_handles_api_error(self, mock_post): - mock_post.side_effect = __import__("requests").exceptions.ConnectionError() - source = FannieMaeSource() - results = source.fetch(state="TX") - assert results == [] # graceful fallback - - @patch("prei.pipeline.sources.reo_sources.requests.post") - def test_fetch_handles_503(self, mock_post): - mock_resp = MagicMock(status_code=503) - mock_post.return_value = mock_resp - source = FannieMaeSource() - results = source.fetch(state="TX") - assert results == [] - - def test_name_property(self): - assert FannieMaeSource().name == "fannie_mae" - - def test_is_discovery_source(self): - assert isinstance(FannieMaeSource(), DiscoverySource) - - -# ═══════════════════════════════════════════════════════════════════════════════ -# HUD Homestore -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TestHUDIntegration: - @patch("prei.pipeline.sources.reo_sources.requests.get") - def test_fetch_returns_mapped_listings(self, mock_get): - mock_resp = MagicMock(status_code=200, json=lambda: HUD_RESPONSE) - mock_get.return_value = mock_resp - source = HUDHomestoreSource() - results = source.fetch(state="FL") - assert len(results) == 1 - assert results[0]["id"] == "hud-HUD-500" - assert results[0]["price"] == 180000 - - @patch("prei.pipeline.sources.reo_sources.requests.get") - def test_fetch_handles_connection_error(self, mock_get): - mock_get.side_effect = __import__("requests").exceptions.ConnectionError() - source = HUDHomestoreSource() - results = source.fetch(state="FL") - assert results == [] - - def test_name_property(self): - assert HUDHomestoreSource().name == "hud" - - -# ═══════════════════════════════════════════════════════════════════════════════ -# VA -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TestVAIntegration: - @patch("prei.pipeline.sources.reo_sources.requests.get") - def test_fetch_returns_mapped_listings(self, mock_get): - mock_resp = MagicMock(status_code=200, json=lambda: VA_RESPONSE) - mock_get.return_value = mock_resp - source = VAForeclosuresSource() - results = source.fetch(state="TX") - assert len(results) == 1 - assert results[0]["id"] == "va-VA-10" - assert results[0]["price"] == 220000 - - def test_name_property(self): - assert VAForeclosuresSource().name == "va" - - -# ═══════════════════════════════════════════════════════════════════════════════ -# USDA -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TestUSDAIntegration: - @patch("prei.pipeline.sources.reo_sources.requests.get") - def test_fetch_empty_batch(self, mock_get): - mock_resp = MagicMock(status_code=200, json=lambda: {"properties": []}) - mock_get.return_value = mock_resp - source = USDAForeclosuresSource() - results = source.fetch(state="FL") - assert results == [] - - def test_name_property(self): - assert USDAForeclosuresSource().name == "usda" - - -# ═══════════════════════════════════════════════════════════════════════════════ -# Texas County -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TestCountyIntegration: - @patch("prei.pipeline.sources.county.requests.get") - def test_csv_parsing(self, mock_get): - mock_resp = MagicMock( - status_code=200, - text=COUNTY_CSV, - headers={"Content-Type": "text/csv"}, - ) - mock_get.return_value = mock_resp - source = TexasCountyForeclosureSource(county_key="harris") - results = source.fetch() - assert len(results) == 2 - assert results[0]["id"] == "NOD-2024-001" - assert results[0]["price"] == 150000.0 - - def test_available_counties(self): - counties = TexasCountyForeclosureSource.available_counties() - assert "harris" in counties - assert "dallas" in counties - assert len(counties) >= 4 - - def test_florida_available_counties(self): - counties = FloridaCountyForeclosureSource.available_counties() - assert "miami-dade" in counties - assert "orange" in counties - - def test_name_includes_county(self): - source = TexasCountyForeclosureSource(county_key="harris") - assert "harris" in source.name - source2 = FloridaCountyForeclosureSource(county_key="miami-dade") - assert "miami-dade" in source2.name diff --git a/prei/pipeline/tests/test_repository.py b/prei/pipeline/tests/test_repository.py deleted file mode 100644 index 6afea428..00000000 --- a/prei/pipeline/tests/test_repository.py +++ /dev/null @@ -1,289 +0,0 @@ -"""Tests for the transactional persistence layer and state aggregator.""" - -import os -import tempfile - -import pytest - -from prei.models.pipeline import PipelineStage, PropertyAsset -from prei.pipeline.engine import ( - InMemoryAssetRepository, - SqliteAssetRepository, - StateAggregator, - TransactionError, - TransactionalRepository, -) - - -def _make_asset( - asset_id: str = "T1", stage: PipelineStage = PipelineStage.GACS -) -> PropertyAsset: - return PropertyAsset( - asset_id=asset_id, address=f"{asset_id} St", current_stage=stage - ) - - -# ═══════════════════════════════════════════════════════════════════════════════ -# TransactionalRepository -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TestTransactionalRepository: - """Tests for the transactional wrapper.""" - - def test_basic_save_and_load(self): - """Non-transactional save works immediately.""" - inner = InMemoryAssetRepository() - tx = TransactionalRepository(inner) - a = _make_asset("X1") - tx.save(a) - assert tx.load("X1") is not None - assert inner.load("X1") is not None # also persisted to inner - - def test_transaction_buffers_saves(self): - """Saves within a transaction are buffered until commit.""" - inner = InMemoryAssetRepository() - tx = TransactionalRepository(inner) - tx.begin() - tx.save(_make_asset("BUF-1")) - # Should not be in inner yet - assert inner.load("BUF-1") is None - # But should be visible via transactional repo - assert tx.load("BUF-1") is None # loads from inner, which doesn't have it - tx.commit() - assert inner.load("BUF-1") is not None - - def test_rollback_discards_pending(self): - """Rollback discards all saves made during the transaction.""" - inner = InMemoryAssetRepository() - tx = TransactionalRepository(inner) - tx.begin() - tx.save(_make_asset("RB-1")) - tx.rollback() - assert inner.list_all() == [] - assert inner.load("RB-1") is None - - def test_rollback_restores_original_state(self): - """Rollback reverts to pre-transaction state.""" - inner = InMemoryAssetRepository() - inner.save(_make_asset("ORIG")) - tx = TransactionalRepository(inner) - - tx.begin() - tx.save(_make_asset("NEW-1")) - inner.save(_make_asset("NEW-2")) # directly modify inner - tx.rollback() - - assert inner.load("ORIG") is not None # original preserved - # NEW-2 was saved directly to inner during tx — rollback doesn't undo it - # because the inner's rollback() is a no-op for InMemory - assert inner.load("NEW-1") is None # tx-save rolled back - - def test_double_begin_raises(self): - """Calling begin() twice raises TransactionError.""" - tx = TransactionalRepository(InMemoryAssetRepository()) - tx.begin() - with pytest.raises(TransactionError, match="already in progress"): - tx.begin() - - def test_commit_without_begin_raises(self): - """Calling commit() without begin() raises TransactionError.""" - tx = TransactionalRepository(InMemoryAssetRepository()) - with pytest.raises(TransactionError, match="No transaction"): - tx.commit() - - def test_rollback_without_begin_raises(self): - """Calling rollback() without begin() raises TransactionError.""" - tx = TransactionalRepository(InMemoryAssetRepository()) - with pytest.raises(TransactionError, match="No transaction"): - tx.rollback() - - def test_multiple_transactions(self): - """Multiple begin/commit cycles work.""" - tx = TransactionalRepository(InMemoryAssetRepository()) - for i in range(5): - tx.begin() - tx.save(_make_asset(f"MULTI-{i}")) - tx.commit() - assert len(tx.list_all()) == 5 - - def test_list_all(self): - """list_all() works with and without transactions.""" - tx = TransactionalRepository(InMemoryAssetRepository()) - tx.save(_make_asset("L1")) - tx.begin() - tx.save(_make_asset("L2")) - # list_all sees only committed and inner (not buffered) - assert len(tx.list_all()) == 1 - tx.commit() - assert len(tx.list_all()) == 2 - - -# ═══════════════════════════════════════════════════════════════════════════════ -# SqliteAssetRepository -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TestSqliteAssetRepository: - """Tests for the SQLite-backed repository.""" - - @pytest.fixture - def repo(self): - with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as f: - db_path = f.name - repo = SqliteAssetRepository(db_path, create_tables=True) - yield repo - repo.close() - os.unlink(db_path) - - def test_save_and_load(self, repo): - a = _make_asset("SQL-1") - repo.save(a) - loaded = repo.load("SQL-1") - assert loaded is not None - assert loaded.asset_id == "SQL-1" - assert loaded.current_stage == PipelineStage.GACS - - def test_load_nonexistent(self, repo): - assert repo.load("DOES-NOT-EXIST") is None - - def test_list_all(self, repo): - repo.save(_make_asset("A")) - repo.save(_make_asset("B")) - assert len(repo.list_all()) == 2 - - def test_update_existing(self, repo): - a = _make_asset("UPD") - repo.save(a) - a.transition_to(PipelineStage.DISCOVERY) - repo.save(a) - loaded = repo.load("UPD") - assert loaded is not None - assert loaded.current_stage == PipelineStage.DISCOVERY - assert len(loaded.stage_history) == 1 - - def test_transaction_commit(self, repo): - repo.begin() - repo.save(_make_asset("TX-A")) - repo.save(_make_asset("TX-B")) - repo.commit() - assert repo.load("TX-A") is not None - assert repo.load("TX-B") is not None - - def test_transaction_rollback(self, repo): - """Rollback discards uncommitted saves within the transaction.""" - repo.begin() - repo.save(_make_asset("DURING")) - repo.rollback() - # "DURING" was not committed — should not exist - assert repo.load("DURING") is None - - def test_transaction_commit_persists(self, repo): - """Committed transaction saves are visible after commit.""" - repo.save(_make_asset("BEFORE")) - repo.begin() - repo.save(_make_asset("DURING")) - repo.commit() - assert repo.load("BEFORE") is not None - assert repo.load("DURING") is not None - - def test_kill_reason_persisted(self, repo): - a = _make_asset("KILL1") - a.transition_to(PipelineStage.KILLED, reason="Budget cut") - repo.save(a) - loaded = repo.load("KILL1") - assert loaded is not None - assert loaded.current_stage == PipelineStage.KILLED - assert loaded.kill_reason == "Budget cut" - - def test_stage_history_persisted(self, repo): - a = _make_asset("HIST") - a.transition_to(PipelineStage.DISCOVERY) - a.transition_to(PipelineStage.SCREENING) - a.transition_to(PipelineStage.UNDERWRITING) - repo.save(a) - loaded = repo.load("HIST") - assert loaded is not None - assert len(loaded.stage_history) == 3 - assert loaded.stage_history[0].stage == PipelineStage.DISCOVERY - assert loaded.stage_history[2].stage == PipelineStage.UNDERWRITING - - -# ═══════════════════════════════════════════════════════════════════════════════ -# StateAggregator -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TestStateAggregator: - """Tests for the pipeline state aggregator.""" - - def _seed_assets(self, repo): - """Create 10 assets across different stages.""" - stages = [ - PipelineStage.DISCOVERY, - PipelineStage.SCREENING, - PipelineStage.UNDERWRITING, - PipelineStage.OFFER, - PipelineStage.DUE_DILIGENCE, - PipelineStage.CLOSING, - PipelineStage.TURNOVER, - PipelineStage.LEASING, - PipelineStage.PORTFOLIO, - PipelineStage.KILLED, - ] - for i, stage in enumerate(stages): - a = _make_asset(f"AG-{i:02d}", stage) - repo.save(a) - - def test_count_by_stage(self): - repo = InMemoryAssetRepository() - self._seed_assets(repo) - agg = StateAggregator(repo) - counts = agg.count_by_stage() - assert counts["DISCOVERY"] == 1 - assert counts["SCREENING"] == 1 - assert counts["UNDERWRITING"] == 1 - assert counts["KILLED"] == 1 - assert counts["PORTFOLIO"] == 1 - assert sum(counts.values()) == 10 - - def test_summary_includes_total(self): - repo = InMemoryAssetRepository() - self._seed_assets(repo) - agg = StateAggregator(repo) - s = agg.summary() - assert s["total_assets"] == 10 - - def test_summary_pipeline_flow(self): - repo = InMemoryAssetRepository() - self._seed_assets(repo) - agg = StateAggregator(repo) - flow = agg.summary()["pipeline_flow"] - # DISCOVERY + SCREENING (GACS default not seeded here) - assert flow["acquisition"] == 2 - # UNDERWRITING + OFFER + DUE_DILIGENCE + CLOSING - assert flow["deal_making"] == 4 - # TURNOVER + LEASING - assert flow["operations"] == 2 - # PORTFOLIO - assert flow["portfolio"] == 1 - - def test_summary_killed(self): - repo = InMemoryAssetRepository() - a = _make_asset("DEAD", PipelineStage.KILLED) - repo.save(a) - agg = StateAggregator(repo) - assert agg.summary()["killed"] == 1 - - def test_empty_repo_summary(self): - repo = InMemoryAssetRepository() - agg = StateAggregator(repo) - s = agg.summary() - assert s["total_assets"] == 0 - assert s["killed"] == 0 - assert s["pipeline_flow"]["acquisition"] == 0 - - def test_by_stage_empty_repo(self): - repo = InMemoryAssetRepository() - agg = StateAggregator(repo) - assert agg.count_by_stage() == {} diff --git a/prei/pipeline/tests/test_screening.py b/prei/pipeline/tests/test_screening.py deleted file mode 100644 index ab405a7a..00000000 --- a/prei/pipeline/tests/test_screening.py +++ /dev/null @@ -1,308 +0,0 @@ -"""Tests for the screening-stage metric evaluator.""" - -import pytest - -from prei.pipeline.handlers.screening import ( - ScreeningThresholds, - compute_screening_metrics, - evaluate_screening_stage, - gross_yield, - price_to_rent_ratio, -) - - -# ── Fixtures ─────────────────────────────────────────────────────────────────── - -BASE_ASSET = { - "estimated_monthly_rent": 2500.0, - "purchase_price": 300_000.0, - "beds": 3, - "baths": 2, - "hoa_name": None, -} - -BASE_THRESHOLDS = ScreeningThresholds( - min_gross_yield=0.07, - max_price_to_rent_ratio=15.0, - min_beds=2, - min_baths=1, -) - - -# ═══════════════════════════════════════════════════════════════════════════════ -# Pure arithmetic helpers -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TestGrossYield: - """Tests for the pure gross_yield() function.""" - - def test_basic_gross_yield(self): - """$2,500/mo rent on $300,000 price = 10% gross yield.""" - gy = gross_yield(monthly_rent=2500, purchase_price=300000) - assert gy == pytest.approx(0.10, rel=1e-6) - - def test_gross_yield_low_rent(self): - """Low rent produces low yield.""" - gy = gross_yield(monthly_rent=800, purchase_price=300000) - assert gy == pytest.approx(0.032, rel=1e-3) - - def test_gross_yield_zero_price(self): - """Zero purchase price returns 0.0 to avoid division by zero.""" - assert gross_yield(monthly_rent=2000, purchase_price=0) == 0.0 - - def test_gross_yield_zero_rent(self): - """Zero monthly rent returns 0.0.""" - assert gross_yield(monthly_rent=0, purchase_price=100000) == 0.0 - - def test_gross_yield_negative(self): - """Negative values return 0.0 (defensive).""" - assert gross_yield(monthly_rent=-1000, purchase_price=100000) == 0.0 - - -class TestPriceToRentRatio: - """Tests for the pure price_to_rent_ratio() function.""" - - def test_basic_ratio(self): - """$300k price / $2.5k/mo rent = 10.0× annual.""" - ptr = price_to_rent_ratio(monthly_rent=2500, purchase_price=300000) - assert ptr == pytest.approx(10.0, rel=1e-6) - - def test_expensive_market(self): - """High price-to-rent ratio = expensive market.""" - ptr = price_to_rent_ratio(monthly_rent=2000, purchase_price=500000) - assert ptr == pytest.approx(20.833, rel=1e-3) - - def test_zero_rent_returns_inf(self): - """Zero monthly rent returns infinity.""" - assert price_to_rent_ratio(monthly_rent=0, purchase_price=100000) == float( - "inf" - ) - - def test_zero_price(self): - """Zero purchase price returns 0.""" - ptr = price_to_rent_ratio(monthly_rent=2000, purchase_price=0) - assert ptr == 0.0 - - -class TestComputeScreeningMetrics: - """Tests for the composition helper.""" - - def test_compute_metrics(self): - """Returns both gross_yield and price_to_rent_ratio.""" - metrics = compute_screening_metrics(BASE_ASSET) - assert "gross_yield" in metrics - assert "price_to_rent_ratio" in metrics - assert metrics["gross_yield"] == pytest.approx(0.10, rel=1e-6) - assert metrics["price_to_rent_ratio"] == pytest.approx(10.0, rel=1e-6) - - def test_missing_rent_defaults_to_zero(self): - """Missing rent key defaults to 0 → 0 yield.""" - data = {"purchase_price": 300000} - metrics = compute_screening_metrics(data) - assert metrics["gross_yield"] == 0.0 - assert metrics["price_to_rent_ratio"] == float("inf") - - -# ═══════════════════════════════════════════════════════════════════════════════ -# evaluate_screening_stage -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TestEvaluateScreeningStage: - """Tests for the top-level screening evaluator.""" - - # ── PASS ────────────────────────────────────────────────────────────────── - - def test_all_checks_pass(self): - """A property meeting all thresholds passes.""" - passed, reason = evaluate_screening_stage(BASE_ASSET, BASE_THRESHOLDS) - assert passed is True - assert reason is None - - def test_above_minimums_still_passes(self): - """Above-minimum beds/baths/yield still passes.""" - data = {**BASE_ASSET, "beds": 5, "baths": 4} - passed, reason = evaluate_screening_stage(data, BASE_THRESHOLDS) - assert passed is True - - # ── BEDS ────────────────────────────────────────────────────────────────── - - def test_fails_on_below_min_beds(self): - """Fewer beds than min_beds → fail.""" - data = {**BASE_ASSET, "beds": 1} - passed, reason = evaluate_screening_stage(data, BASE_THRESHOLDS) - assert passed is False - assert "bedroom" in (reason or "").lower() - - def test_passes_on_exact_min_beds(self): - """Exact minimum beds still passes.""" - data = {**BASE_ASSET, "beds": 2} - passed, reason = evaluate_screening_stage(data, BASE_THRESHOLDS) - assert passed is True - - # ── BATHS ───────────────────────────────────────────────────────────────── - - def test_fails_on_below_min_baths(self): - """Fewer baths than min_baths → fail.""" - data = {**BASE_ASSET, "baths": 0.5} - passed, reason = evaluate_screening_stage(data, BASE_THRESHOLDS) - assert passed is False - assert "bath" in (reason or "").lower() - - def test_passes_on_exact_min_baths(self): - """Exact minimum baths still passes.""" - data = {**BASE_ASSET, "baths": 1} - passed, reason = evaluate_screening_stage(data, BASE_THRESHOLDS) - assert passed is True - - # ── HOA EXCLUSION ───────────────────────────────────────────────────────── - - def test_excluded_hoa_fails(self): - """Property in an excluded HOA → fail.""" - thresholds = BASE_THRESHOLDS.model_copy( - update={"excluded_hoas": ["Sunset Homes", "Lake View"]} - ) - data = {**BASE_ASSET, "hoa_name": "Sunset Homes"} - passed, reason = evaluate_screening_stage(data, thresholds) - assert passed is False - assert "HOA" in (reason or "") - - def test_non_excluded_hoa_passes(self): - """Property in a non-excluded HOA → pass.""" - thresholds = BASE_THRESHOLDS.model_copy( - update={"excluded_hoas": ["Sunset Homes"]} - ) - data = {**BASE_ASSET, "hoa_name": "Lake View"} - passed, reason = evaluate_screening_stage(data, thresholds) - assert passed is True - - def test_hoa_exclusion_case_insensitive(self): - """HOA name matching is case-insensitive.""" - thresholds = BASE_THRESHOLDS.model_copy( - update={"excluded_hoas": ["SUNSET HOMES"]} - ) - data = {**BASE_ASSET, "hoa_name": "sunset homes"} - passed, reason = evaluate_screening_stage(data, thresholds) - assert passed is False - - # ── GROSS YIELD ─────────────────────────────────────────────────────────── - - def test_fails_on_low_gross_yield(self): - """Below-minimum gross yield → fail.""" - data = {**BASE_ASSET, "estimated_monthly_rent": 1500} # 6% yield - passed, reason = evaluate_screening_stage(data, BASE_THRESHOLDS) - assert passed is False - assert "yield" in (reason or "").lower() - - def test_passes_on_exact_min_gross_yield(self): - """Exact minimum gross yield → pass.""" - data = {**BASE_ASSET, "estimated_monthly_rent": 1750} # 7% exactly - passed, reason = evaluate_screening_stage(data, BASE_THRESHOLDS) - assert passed is True - - # ── PRICE-TO-RENT RATIO ────────────────────────────────────────────────── - - def test_fails_on_high_price_to_rent(self): - """Above-maximum price-to-rent ratio → fail.""" - # Need data where yield passes but ratio fails: - # min_yield=0.05, pass when (rent*12)/price >= 0.05 - # max_ratio=12, fail when price/(rent*12) > 12 - # rent=3000, price=520000 → yield=0.069 (pass), ratio=14.44 (fail) - data = { - "estimated_monthly_rent": 3000, - "purchase_price": 520_000, - "beds": 3, - "baths": 2, - } - thresholds = BASE_THRESHOLDS.model_copy( - update={"min_gross_yield": 0.05, "max_price_to_rent_ratio": 12.0} - ) - passed, reason = evaluate_screening_stage(data, thresholds) - assert passed is False - assert "price-to-rent" in (reason or "").lower() - - def test_passes_on_exact_max_price_to_rent(self): - """Exact maximum price-to-rent ratio → pass.""" - # Target: passes yield (>=5%) and exactly hits ratio (12.0) - # ratio = price/(rent*12) = 12 → price = 12*rent*12 = 144*rent - # yield = (rent*12)/price = (rent*12)/(144*rent) = 0.0833 = 8.33% - # With rent=2500: price=360000, yield=8.33%, ratio=12.0 - data = { - "estimated_monthly_rent": 2500, - "purchase_price": 360_000, - "beds": 3, - "baths": 2, - } - thresholds = BASE_THRESHOLDS.model_copy( - update={"min_gross_yield": 0.05, "max_price_to_rent_ratio": 12.0} - ) - passed, reason = evaluate_screening_stage(data, thresholds) - assert passed is True, f"Expected pass, got: {reason}" - - # ── MISSING DATA EDGE CASES ─────────────────────────────────────────────── - - def test_missing_beds_skips_bed_check(self): - """If beds key is missing, beds check is skipped (not failed).""" - data = {k: v for k, v in BASE_ASSET.items() if k != "beds"} - passed, reason = evaluate_screening_stage(data, BASE_THRESHOLDS) - assert passed is True - - def test_missing_rent_skips_yield_and_ratio(self): - """If rent is missing, yield and ratio checks are skipped.""" - data = {k: v for k, v in BASE_ASSET.items() if k != "estimated_monthly_rent"} - passed, reason = evaluate_screening_stage(data, BASE_THRESHOLDS) - assert passed is True - - def test_missing_price_skips_yield_and_ratio(self): - """If price is missing, yield and ratio checks are skipped.""" - data = {k: v for k, v in BASE_ASSET.items() if k != "purchase_price"} - passed, reason = evaluate_screening_stage(data, BASE_THRESHOLDS) - assert passed is True - - # ── FIRST FAILURE SHORT-CIRCUIT ────────────────────────────────────────── - - def test_beds_checked_before_yield(self): - """Beds check (cheapest) runs first; failing beds short-circuits.""" - data = { - **BASE_ASSET, - "beds": 1, # fails - "estimated_monthly_rent": 100, # would also fail yield - "purchase_price": 500000, # would also fail ratio - } - passed, reason = evaluate_screening_stage(data, BASE_THRESHOLDS) - assert passed is False - # Should fail on beds before getting to yield - assert "bedroom" in (reason or "").lower() - - # ── PERFORMANCE VERIFICATION ────────────────────────────────────────────── - - def test_evaluation_under_10ms(self): - """Single evaluation completes in under 10ms (trivially).""" - import time - - start = time.perf_counter() - for _ in range(1000): - evaluate_screening_stage(BASE_ASSET, BASE_THRESHOLDS) - elapsed_ms = (time.perf_counter() - start) * 1000 / 1000 - assert elapsed_ms < 10, ( - f"Average evaluation took {elapsed_ms:.4f}ms (expected <10ms)" - ) - - # ── PROTOCOL: returns (bool, str | None) ───────────────────────────────── - - def test_return_type_on_pass(self): - """Pass returns (True, None).""" - result = evaluate_screening_stage(BASE_ASSET, BASE_THRESHOLDS) - assert isinstance(result, tuple) - assert len(result) == 2 - assert result[0] is True - assert result[1] is None - - def test_return_type_on_fail(self): - """Fail returns (False, str).""" - data = {**BASE_ASSET, "beds": 0} - result = evaluate_screening_stage(data, BASE_THRESHOLDS) - assert isinstance(result, tuple) - assert result[0] is False - assert isinstance(result[1], str) diff --git a/prei/pipeline/tests/test_sources.py b/prei/pipeline/tests/test_sources.py deleted file mode 100644 index dcafb03e..00000000 --- a/prei/pipeline/tests/test_sources.py +++ /dev/null @@ -1,238 +0,0 @@ -"""Tests for discovery-stage data sources and registry.""" - -import pytest -from unittest.mock import MagicMock, patch - -from prei.pipeline.sources.base import DiscoverySource -from prei.pipeline.sources.county import ( - FloridaCountyForeclosureSource, - TexasCountyForeclosureSource, -) -from prei.pipeline.sources.registry import ( - discover_from_all, - get_source, - list_sources, -) -from prei.pipeline.sources.reo_sources import ( - FannieMaeSource, - HUDHomestoreSource, - USDAForeclosuresSource, - VAForeclosuresSource, -) - - -# ═══════════════════════════════════════════════════════════════════════════════ -# Source interface compliance -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TestSourceInterface: - """All sources must conform to DiscoverySource ABC.""" - - @pytest.mark.parametrize( - "cls,name", - [ - (FannieMaeSource, "fannie_mae"), - (HUDHomestoreSource, "hud"), - (VAForeclosuresSource, "va"), - (USDAForeclosuresSource, "usda"), - (TexasCountyForeclosureSource, "tx_county_harris"), - ], - ) - def test_all_sources_have_name(self, cls, name): - source = ( - cls() if cls != TexasCountyForeclosureSource else cls(county_key="harris") - ) - assert isinstance(source, DiscoverySource) - assert source.name is not None - assert len(source.name) > 0 - - @patch("prei.pipeline.sources.reo_sources.requests.post") - @patch("prei.pipeline.sources.reo_sources.requests.get") - def test_all_sources_return_list(self, mock_get, mock_post): - mock_empty = MagicMock(status_code=200, json=lambda: {"results": []}) - mock_get.return_value = mock_empty - mock_post.return_value = mock_empty - for source_cls in [ - FannieMaeSource, - HUDHomestoreSource, - VAForeclosuresSource, - USDAForeclosuresSource, - ]: - result = source_cls().fetch(state="CA") - assert isinstance(result, list), ( - f"{source_cls.__name__} did not return a list" - ) - - @patch("prei.pipeline.sources.county.requests.get") - def test_county_source_returns_list(self, mock_get): - """County source with mocked HTTP returns a list.""" - mock_resp = MagicMock( - status_code=200, - text="case_number,address\n1,test", - headers={"Content-Type": "text/csv"}, - ) - mock_get.return_value = mock_resp - source = TexasCountyForeclosureSource(county="harris") - result = source.fetch(state="TX", limit=1) - assert isinstance(result, list) - - def test_county_source_accepts_county_param(self): - source = TexasCountyForeclosureSource(county="harris") - assert "harris" in source.name - - -# ═══════════════════════════════════════════════════════════════════════════════ -# Fannie Mae -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TestFannieMaeSource: - def test_name(self): - assert FannieMaeSource().name == "fannie_mae" - - def test_fetch_empty(self): - """Returns empty list (placeholder until scraper is built).""" - result = FannieMaeSource().fetch(state="CA") - assert result == [] - - def test_fetch_with_zip(self): - result = FannieMaeSource().fetch(state="CA", zip_code="90210") - assert result == [] - - -# ═══════════════════════════════════════════════════════════════════════════════ -# HUD -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TestHUDHomestoreSource: - def test_name(self): - assert HUDHomestoreSource().name == "hud" - - def test_fetch_empty(self): - assert HUDHomestoreSource().fetch(state="TX") == [] - - -# ═══════════════════════════════════════════════════════════════════════════════ -# VA -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TestVAForeclosuresSource: - def test_name(self): - assert VAForeclosuresSource().name == "va" - - def test_fetch_empty(self): - assert VAForeclosuresSource().fetch(state="FL") == [] - - -# ═══════════════════════════════════════════════════════════════════════════════ -# USDA -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TestUSDAForeclosuresSource: - def test_name(self): - assert USDAForeclosuresSource().name == "usda" - - def test_fetch_empty(self): - assert USDAForeclosuresSource().fetch(state="FL") == [] - - -# ═══════════════════════════════════════════════════════════════════════════════ -# County source -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TestTexasCountyForeclosureSource: - def test_name_default(self): - source = TexasCountyForeclosureSource(county_key="harris") - assert "tx_county" in source.name - assert "harris" in source.name - - def test_name_with_county_key(self): - source = TexasCountyForeclosureSource(county_key="dallas") - assert "dallas" in source.name - - def test_available_counties(self): - counties = TexasCountyForeclosureSource.available_counties() - assert "harris" in counties - assert "dallas" in counties - assert len(counties) >= 4 - - def test_csv_fetch_returns_list(self): - source = TexasCountyForeclosureSource(county_key="harris") - result = source.fetch(state="TX") - assert isinstance(result, list) - - -class TestFloridaCountyForeclosureSource: - def test_name_with_county(self): - source = FloridaCountyForeclosureSource(county_key="miami-dade") - assert "miami-dade" in source.name - - def test_available_counties(self): - counties = FloridaCountyForeclosureSource.available_counties() - assert "miami-dade" in counties - assert "orange" in counties - - -# ═══════════════════════════════════════════════════════════════════════════════ -# Registry -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TestRegistry: - def test_list_sources(self): - sources = list_sources() - assert "fannie_mae" in sources - assert "hud" in sources - assert "va" in sources - assert "usda" in sources - assert "county_tx" in sources - - def test_get_source_valid(self): - source = get_source("fannie_mae") - assert isinstance(source, FannieMaeSource) - - source = get_source("county_tx", county="harris") - assert isinstance(source, TexasCountyForeclosureSource) - assert "harris" in source.name - - def test_get_source_invalid(self): - with pytest.raises(ValueError, match="Unknown source"): - get_source("nonexistent_source") - - @patch("prei.pipeline.sources.reo_sources.requests.post") - @patch("prei.pipeline.sources.reo_sources.requests.get") - def test_discover_from_all(self, mock_get, mock_post): - """Returns dict with all source names; each value is a list.""" - mock_empty = MagicMock(status_code=200, json=lambda: {"results": []}) - mock_get.return_value = mock_empty - mock_post.return_value = mock_empty - result = discover_from_all(state="CA") - assert isinstance(result, dict) - for name in list_sources(): - assert name in result - assert isinstance(result[name], list) - - @patch("prei.pipeline.sources.reo_sources.requests.post") - @patch("prei.pipeline.sources.reo_sources.requests.get") - def test_discover_from_all_with_filter(self, mock_get, mock_post): - mock_empty = MagicMock(status_code=200, json=lambda: {"results": []}) - mock_get.return_value = mock_empty - mock_post.return_value = mock_empty - result = discover_from_all(state="TX", source_filter=["fannie_mae", "hud"]) - assert set(result.keys()) == {"fannie_mae", "hud"} - - @patch("prei.pipeline.sources.reo_sources.requests.post") - @patch("prei.pipeline.sources.reo_sources.requests.get") - def test_discover_from_all_source_failure_does_not_crash(self, mock_get, mock_post): - """One source failing doesn't prevent others from running.""" - mock_empty = MagicMock(status_code=200, json=lambda: {"results": []}) - mock_get.return_value = mock_empty - mock_post.return_value = mock_empty - result = discover_from_all(state="CA") - for name in list_sources(): - assert result[name] == [] diff --git a/prei/pipeline/tests/test_underwriting.py b/prei/pipeline/tests/test_underwriting.py deleted file mode 100644 index a830d666..00000000 --- a/prei/pipeline/tests/test_underwriting.py +++ /dev/null @@ -1,219 +0,0 @@ -"""Tests for the underwriting solver engine.""" - -from decimal import Decimal - -import pytest - -from prei.pipeline.handlers.underwriting import ( - UnderwritingInput, - UnderwritingMetrics, - cap_rate, - cash_on_cash_yield, - effective_gross_income, - gross_potential_rent, - max_allowable_offer, - net_operating_income, - solve_underwriting, - total_operating_expenses, -) - -# ── Sample input ────────────────────────────────────────────────────────────── - -BASE_INPUT = UnderwritingInput( - purchase_price=Decimal("300000.0"), - estimated_rent=Decimal("2500.0"), - vacancy_rate=Decimal("0.05"), - rehab_budget=Decimal("20000.0"), - property_tax_annual=Decimal("3600.0"), - insurance_annual=Decimal("1200.0"), - maintenance_reserve_rate=Decimal("0.10"), - management_fee_rate=Decimal("0.08"), - hoa_annual=Decimal("600.0"), -) - -# Expected intermediate values for BASE_INPUT: -# GPR = 2500 * 12 = 30_000 -# EGI = 30000 * (1 - 0.05) = 28_500 -# Maint = 30000 * 0.10 = 3_000 -# Mgmt = 28500 * 0.08 = 2_280 -# OpEx = 3600 + 1200 + 3000 + 2280 + 600 = 10_680 -# NOI = 28500 - 10680 = 17_820 -# Cap = 17820 / 300000 = 0.0594 = 5.94% -# CoC = 17820 / (300000 + 20000) = 17820 / 320000 = 0.0556875 = 5.57% -# MAO (8%): 17820 / 0.08 = 222_750.00 - - -# ═══════════════════════════════════════════════════════════════════════════════ -# Pure arithmetic helpers -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TestPureArithmetic: - """Tests for individual pure functions.""" - - def test_gross_potential_rent(self): - assert gross_potential_rent(2500) == 30_000.0 - assert gross_potential_rent(0) == 0.0 - - def test_effective_gross_income(self): - assert effective_gross_income(30_000, 0.05) == 28_500.0 - assert effective_gross_income(30_000, 0) == 30_000.0 - assert effective_gross_income(30_000, 1) == 0.0 - - def test_total_operating_expenses(self): - opex = total_operating_expenses( - property_tax_annual=3_600.0, - insurance_annual=1_200.0, - gpr=30_000.0, - maintenance_reserve_rate=0.10, - egi=28_500.0, - management_fee_rate=0.08, - hoa_annual=600.0, - ) - assert float(opex) == pytest.approx(10_680.0) - - def test_net_operating_income(self): - assert float(net_operating_income(28_500, 10_680)) == pytest.approx(17_820.0) - assert net_operating_income(0, 0) == 0.0 - assert float(net_operating_income(10_000, 15_000)) == pytest.approx(-5_000.0) - - # ── Division by zero guards ────────────────────────────────────────────── - - def test_cap_rate_zero_price(self): - assert cap_rate(annual_noi=10_000, purchase_price=0) == 0.0 - - def test_cap_rate_negative_price(self): - # Canonical investor_app.finance.utils.cap_rate has no special case - # for negative purchase_price - it performs real division. - assert cap_rate(annual_noi=10_000, purchase_price=-100) == -100.0 - - def test_cash_on_cash_zero_initial(self): - assert cash_on_cash_yield(noi=10_000, purchase_price=0, rehab_budget=0) == 0.0 - - def test_cash_on_cash_all_cash(self): - """All-cash purchase: CoC = NOI / price.""" - coc = cash_on_cash_yield(noi=17_820, purchase_price=300_000, rehab_budget=0) - assert float(coc) == pytest.approx(0.0594, rel=1e-4) - - def test_cash_on_cash_with_rehab(self): - """With rehab budget: CoC = NOI / (price + rehab).""" - coc = cash_on_cash_yield( - noi=17_820, purchase_price=300_000, rehab_budget=20_000 - ) - assert float(coc) == pytest.approx(0.0556875, rel=1e-5) - - def test_max_allowable_offer_zero_target(self): - assert max_allowable_offer(noi=10_000, target_cap_rate=0) == 0.0 - - def test_max_allowable_offer_negative_target(self): - assert max_allowable_offer(noi=10_000, target_cap_rate=-0.05) == 0.0 - - def test_max_allowable_offer_standard(self): - """MAO = NOI / target_cap_rate.""" - mao = max_allowable_offer(noi=17_820, target_cap_rate=0.08) - assert float(mao) == pytest.approx(222_750.0) - - -# ═══════════════════════════════════════════════════════════════════════════════ -# Composition: solve_underwriting -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TestSolveUnderwriting: - """Tests for the full underwriting solver.""" - - def test_base_case(self): - """All intermediate values match expected calculations.""" - result = solve_underwriting(BASE_INPUT, target_cap_rate=0.08) - - assert isinstance(result, UnderwritingMetrics) - assert float(result.noi) == pytest.approx(17_820.0, rel=1e-4) - assert float(result.cap_rate) == pytest.approx(0.0594, rel=1e-4) - assert float(result.cash_on_cash) == pytest.approx(0.0556875, rel=1e-5) - assert float(result.mao) == pytest.approx(222_750.0, rel=1e-4) - - def test_high_target_cap_rate_lowers_mao(self): - """Higher target cap rate → lower MAO.""" - result_8 = solve_underwriting(BASE_INPUT, target_cap_rate=0.08) - result_10 = solve_underwriting(BASE_INPUT, target_cap_rate=0.10) - assert result_10.mao < result_8.mao - assert float(result_10.mao) == pytest.approx( - 178_200.0, rel=1e-4 - ) # 17820 / 0.10 - - def test_low_target_cap_rate_raises_mao(self): - """Lower target cap rate → higher MAO.""" - result_6 = solve_underwriting(BASE_INPUT, target_cap_rate=0.06) - assert float(result_6.mao) == pytest.approx(297_000.0, rel=1e-4) # 17820 / 0.06 - - def test_no_rehab(self): - """Zero rehab budget → CoC = NOI / price.""" - inp = BASE_INPUT.model_copy(update={"rehab_budget": 0}) - result = solve_underwriting(inp, target_cap_rate=0.08) - assert float(result.cash_on_cash) == pytest.approx(0.0594, rel=1e-4) - - def test_higher_vacancy_lowers_noi(self): - """Higher vacancy rate → lower EGI → lower NOI.""" - inp_high_vac = BASE_INPUT.model_copy(update={"vacancy_rate": 0.15}) - result = solve_underwriting(inp_high_vac, target_cap_rate=0.08) - - # EGI = 30000 * 0.85 = 25500 - # Mgmt = 25500 * 0.08 = 2040 - # OpEx = 3600 + 1200 + 3000 + 2040 + 600 = 10440 - # NOI = 25500 - 10440 = 15060 - assert float(result.noi) == pytest.approx(15_060.0, rel=1e-4) - - def test_zero_purchase_price(self): - """Zero purchase price → cap_rate = 0, MAO still valid.""" - inp = BASE_INPUT.model_copy(update={"purchase_price": 0, "rehab_budget": 0}) - result = solve_underwriting(inp, target_cap_rate=0.08) - assert result.cap_rate == 0.0 - assert result.cash_on_cash == 0.0 - # MAO should still be valid (based on NOI, not price) - assert float(result.mao) == pytest.approx(222_750.0, rel=1e-4) - - def test_all_defaults(self): - """Solver works with only required fields and defaults.""" - inp = UnderwritingInput( - purchase_price=Decimal("200000.0"), - estimated_rent=Decimal("1800.0"), - property_tax_annual=Decimal("2400.0"), - insurance_annual=Decimal("900.0"), - ) - result = solve_underwriting(inp, target_cap_rate=0.08) - assert isinstance(result, UnderwritingMetrics) - assert result.noi > 0 - assert result.cap_rate > 0 - assert result.cash_on_cash > 0 - assert result.mao > 0 - - # ── Return type contract ───────────────────────────────────────────────── - - def test_returns_underwriting_metrics(self): - """Return type is UnderwritingMetrics with all fields.""" - result = solve_underwriting(BASE_INPUT, target_cap_rate=0.08) - assert isinstance(result, UnderwritingMetrics) - for field in ("noi", "cap_rate", "cash_on_cash", "mao"): - assert hasattr(result, field) - - # ── Deterministic ──────────────────────────────────────────────────────── - - def test_deterministic(self): - """Same inputs produce same outputs.""" - a = solve_underwriting(BASE_INPUT, 0.08) - b = solve_underwriting(BASE_INPUT, 0.08) - assert a == b - - # ── Performance ────────────────────────────────────────────────────────── - - def test_under_10ms(self): - """Single solver run completes in under 10ms (trivially).""" - import time - - start = time.perf_counter() - for _ in range(10_000): - solve_underwriting(BASE_INPUT, 0.08) - elapsed_ms = (time.perf_counter() - start) * 1000 / 10_000 - assert elapsed_ms < 10, ( - f"Average solver took {elapsed_ms:.4f}ms (expected <10ms)" - ) diff --git a/requirements.txt b/requirements.txt index c805c2dc..b3a21782 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,13 +38,10 @@ svglib==2.0.2 matplotlib==3.11.1 pillow==12.3.0 pydantic==2.13.4 -fastapi==0.139.2 -uvicorn[standard]==0.51.0 httpx==0.28.1 django-structlog==9.0.0 # structured JSON logging (Phase D) python-json-logger==4.1.0 # JSON formatter for structlog opentelemetry-api==1.32.1 # OTEL traces for uFawkesObs opentelemetry-sdk==1.32.1 opentelemetry-exporter-otlp==1.32.1 -click==8.4.2 cryptography==49.0.0 # Fix GHSA-537c-gmf6-5ccf diff --git a/tasks.json b/tasks.json index 63a1d981..936b756b 100644 --- a/tasks.json +++ b/tasks.json @@ -1,72 +1,170 @@ { "meta": { "project": "prei", - "session": "top01-phase-c-20260728", - "date": "2026-07-28", - "feature": "Phase C (partial) — Deployment Reliability (docs/TOP_01_PLAN.md), C-2 + C-4 only", + "session": "pydantic-to-django-20260731", + "date": "2026-07-31", + "feature": "Consolidate prei pydantic/FastAPI/CLI pipeline onto Django — remove CLI + pydantic, port discovery/screening/underwriting processors to core/services, rewrite the two view bridges, delete prei package", "spec": "specification.md", "design": "design.md", - "deferred": ["C-1 (canary deployment)", "C-3 (SLO dashboard)"] + "governance": ["PM sign-off required before deletion tasks (TASK-05+)"], + "decision_record": { + "user_directive": "no cli, pydantic — migrate discovery/screening processors to Django, do not build new state on pydantic models", + "verified_assumption": "docker-compose ghcr.io/paruff/prei:latest is the Django web image (OTEL_SERVICE_NAME: prei only) — NOT a separate FastAPI microservice; no standalone deploy surface exists", + "unverified": "whether docker-compose web service is actively run in any environment (compose file exists; local usage unconfirmed)" + } }, "tasks": [ { - "id": "C-2a", - "summary": "Idempotent ZAP scan-user seed command", - "description": "Add core/management/commands/seed_zap_scan_user.py: reads ZAP_AUTH_USERNAME/ZAP_AUTH_PASSWORD from env, get_or_create's a non-staff, non-superuser user and sets/updates the password idempotently.", + "id": "TASK-01", + "summary": "Port pure screening evaluator + thresholds to Django (Decimal, no pydantic)", + "description": "Add to core/services/screening.py: evaluate_screening_stage-equivalent pure function (gross_yield, price_to_rent, stage decision) operating on Decimal via to_decimal from investor_app.finance.utils, and a dataclass ScreeningThresholds mirroring the pydantic model (min_gross_yield_pct, max_price_to_rent_ratio, min_beds, max_beds, min_sqft, max_year_built, allowed_property_types, allowed_states, allowed_foreclosure_statuses). Do not reuse pydantic. Existing core.services.screening.ScreenProperty/ScreeningResult remain; the prei evaluate_screening_stage logic is the new pure layer callers use.", "depends_on": [], "acceptance_criteria": [ - {"id": "AC-C2-01", "description": "seed_zap_scan_user idempotently creates/updates a non-staff, non-superuser scan account from env vars", "test_type": "unit"} - ] + {"id": "AC-01-1", "description": "evaluate_screening_stage ported to core/services/screening.py without pydantic, Decimal-based", "test_type": "unit"}, + {"id": "AC-01-2", "description": "ScreeningThresholds is a dataclass with same defaults as pydantic original", "test_type": "unit"}, + {"id": "AC-01-3", "description": "Tests from prei/pipeline/tests/test_screening.py pass against the new location", "test_type": "unit"} + ], + "agent": "build", + "estimated_lines": 160, + "skills": ["build/refactoring", "lang-python"] }, { - "id": "C-2b", - "summary": "ZAP auth context file + authenticated scan CI job", - "description": "Add .zap/prei-auth-context.xml (form-based auth against /accounts/login/, logged-in/out indicator regexes). Add zap-authenticated-scan job to ci-quality.yml: migrate, seed the scan user, boot an ephemeral runserver, wait for healthy, run zap-full-scan.py with the auth context. Wire into pr-gates-pass as a required check.", - "depends_on": ["C-2a"], + "id": "TASK-02", + "summary": "Port DiscoverySanitizer to Django (no pydantic)", + "description": "Add core/services/discovery.py: DiscoverySanitizer-equivalent pure class + CanonicalPropertyPayload as dataclass (or dict), normalizing raw listing dicts (id, address, price, beds, baths, sqft, property_type, year_built, state, foreclosure_status) with Decimal for price. No pydantic. Preserve output key names the two view bridges and BDD steps rely on.", + "depends_on": [], "acceptance_criteria": [ - {"id": "AC-C2-02", "description": ".zap/prei-auth-context.xml defines form-based auth with logged-in/out indicators", "test_type": "manual"}, - {"id": "AC-C2-03", "description": "zap-authenticated-scan job seeds the account and scans an ephemeral instance", "test_type": "ci"}, - {"id": "AC-C2-04", "description": "zap-authenticated-scan is a required check in pr-gates-pass", "test_type": "ci"} - ] + {"id": "AC-02-1", "description": "DiscoverySanitizer ported to core/services/discovery.py without pydantic", "test_type": "unit"}, + {"id": "AC-02-2", "description": "CanonicalPropertyPayload is pydantic-free (dataclass or dict) with Decimal price", "test_type": "unit"}, + {"id": "AC-02-3", "description": "prei/pipeline/tests/test_discovery.py sanitizer tests pass against new location", "test_type": "unit"} + ], + "agent": "build", + "estimated_lines": 130, + "skills": ["build/refactoring", "lang-python"] }, { - "id": "C-2c", - "summary": "Document the ephemeral-instance scope decision", - "description": "Add docs/KNOWN_LIMITATIONS.md LIMIT-22: authenticated ZAP scanning runs against an ephemeral CI-seeded instance, not the live deployment, and why.", - "depends_on": ["C-2b"], + "id": "TASK-03", + "summary": "Port underwriting solver to Django (Decimal, dataclasses)", + "description": "Add core/services/underwriting.py: solve_underwriting + UnderwritingInput/UnderwritingMetrics as dataclasses (they are already Decimal-based; replace pydantic BaseModel with dataclasses), importing cap_rate/cash_on_cash/to_decimal from investor_app.finance.utils. Pure function — no views, no models.", + "depends_on": [], "acceptance_criteria": [ - {"id": "AC-C2-05", "description": "docs/KNOWN_LIMITATIONS.md documents the ephemeral-CI-instance scope decision", "test_type": "manual"} - ] + {"id": "AC-03-1", "description": "solve_underwriting ported to core/services/underwriting.py, pydantic-free dataclasses", "test_type": "unit"}, + {"id": "AC-03-2", "description": "UnderwritingInput/Metrics dataclasses use Decimal only", "test_type": "unit"}, + {"id": "AC-03-3", "description": "prei/pipeline/tests/test_underwriting.py passes against new location", "test_type": "unit"} + ], + "agent": "build", + "estimated_lines": 150, + "skills": ["build/refactoring", "lang-python"] }, { - "id": "C-4a", - "summary": "Report-log flag + flaky detection/ledger script", - "description": "Add --report-log=.pytest-report.jsonl to pytest.ini's addopts (requires the pytest-reportlog dependency, added to requirements.txt). Add .github/scripts/flaky_report.py with --mode report (summary only) and --mode write (updates docs/quality/flaky_tests.json and tests/.flaky_quarantine.txt once a nodeid's count reaches the threshold).", - "depends_on": [], + "id": "TASK-04", + "summary": "Port DiscoveryProcessor (dedup + persist) to Django model layer", + "description": "Add core/services/discovery_processor.py (or extend core/services/pipeline.py): process_discovery(canonical_payload) -> PipelineProperty — dedup by address_hash (SHA-256 of normalized address, same rule as PipelineAsset.address_hash), create-or-update PipelineProperty at DISCOVERY stage. Replaces prei DiscoveryProcessor which built pydantic PropertyAsset objects.", + "depends_on": ["TASK-02"], + "acceptance_criteria": [ + {"id": "AC-04-1", "description": "process_discovery persists PipelineProperty at DISCOVERY stage with dedup on address_hash", "test_type": "unit"}, + {"id": "AC-04-2", "description": "No pydantic PropertyAsset creation anywhere in the Django path", "test_type": "unit"}, + {"id": "AC-04-3", "description": "prei/pipeline/tests/test_discovery_processor.py logic covered by new Django test", "test_type": "unit"} + ], + "agent": "build", + "estimated_lines": 120, + "skills": ["build/code-generation", "lang-python"] + }, + { + "id": "TASK-05", + "summary": "Rewrite Growth Explorer bridge in core/views/__init__.py (~line 1235) onto Django services", + "description": "Replace lazy prei imports (DiscoveryProcessor, ScreeningThresholds, BatchScreeningProcessor, InMemoryAssetRepository, PipelineEngine, discover_from_all) with core.services equivalents: discovery via sources + DiscoverySanitizer, persist via process_discovery, screen via evaluate_screening_stage, advance PipelineProperty stage on pass, KILLED + kill_reason on fail. Keep view behavior (growth explorer pipeline_city branch) identical.", + "depends_on": ["TASK-01", "TASK-02", "TASK-04"], + "acceptance_criteria": [ + {"id": "AC-05-1", "description": "Growth Explorer pipeline_city branch has zero prei imports", "test_type": "manual"}, + {"id": "AC-05-2", "description": "PipelineProperty rows created/screened identically to prior engine behavior", "test_type": "integration"}, + {"id": "AC-05-3", "description": "No regression in growth_explorer tests", "test_type": "ci"} + ], + "agent": "build", + "estimated_lines": 140, + "skills": ["build/refactoring", "lang-python"] + }, + { + "id": "TASK-06", + "summary": "Rewrite vrm_properties_list run_pipeline bridge in core/views/__init__.py (~line 3227) onto Django services", + "description": "Replace lazy prei imports (InMemoryAssetRepository, PipelineEngine, ScreeningThresholds, BatchScreeningProcessor) with core.services screening + PipelineProperty persistence. Keep POST behavior for run_pipeline.", + "depends_on": ["TASK-01", "TASK-02", "TASK-04"], + "acceptance_criteria": [ + {"id": "AC-06-1", "description": "vrm_properties_list run_pipeline has zero prei imports", "test_type": "manual"}, + {"id": "AC-06-2", "description": "run_pipeline persists/screens via Django services; existing tests updated and passing", "test_type": "integration"} + ], + "agent": "build", + "estimated_lines": 110, + "skills": ["build/refactoring", "lang-python"] + }, + { + "id": "TASK-07", + "summary": "Migrate the 9 prei-importing test files + tests_bdd steps to core.services", + "description": "Update imports in tests/test_pipeline.py, test_pipeline_e2e.py, test_discovery.py, test_discovery_integration.py, test_discovery_e2e.py, test_offer_integration.py, test_screening_integration.py, test_underwriting_integration.py, and tests_bdd/steps/pipeline_steps.py to point at core/services/* and core/services/pipeline.py instead of prei.pipeline.* and prei.models.*. Delete the pipeline BDD feature only if it exclusively exercises deleted engine state (PipelineEngine/InMemoryAssetRepository); otherwise rewrite steps to Django-backed equivalents.", + "depends_on": ["TASK-01", "TASK-02", "TASK-03", "TASK-04"], + "acceptance_criteria": [ + {"id": "AC-07-1", "description": "No test file outside prei/ imports prei (grep prei.pipeline returns nothing outside prei/)", "test_type": "ci"}, + {"id": "AC-07-2", "description": "Full test suite green after import migration", "test_type": "ci"} + ], + "agent": "build", + "estimated_lines": 260, + "skills": ["build/refactoring", "lang-python"] + }, + { + "id": "TASK-08", + "summary": "Delete pydantic state machine + engine + orchestrator (PropertyAsset/StageLog/PipelineStage/InMemory/SQLite repos, PipelineEngine, StateAggregator, PipelineOrchestrator)", + "description": "Delete prei/models/pipeline.py, prei/pipeline/engine.py, prei/pipeline/orchestrator.py. PipelineProperty (Django) + core/services/pipeline.py STAGE_ORDER transitions are the canonical state. Requires PM sign-off. Requires TASK-05/06/07 so no production or test import remains.", + "depends_on": ["TASK-05", "TASK-06", "TASK-07"], + "acceptance_criteria": [ + {"id": "AC-08-1", "description": "prei/models/pipeline.py, engine.py, orchestrator.py deleted", "test_type": "manual"}, + {"id": "AC-08-2", "description": "Grep for PropertyAsset/StageLog/PipelineEngine/PipelineOrchestrator returns nothing outside git history", "test_type": "ci"}, + {"id": "AC-08-3", "description": "Full suite + ruff + mypy green", "test_type": "ci"} + ], + "agent": "build", + "estimated_lines": 1100, + "skills": ["build/refactoring", "build/governance-enforcement"] + }, + { + "id": "TASK-09", + "summary": "Delete FastAPI router + CLI (prei/api/, prei/cli.py) and the offer.py float handler", + "description": "Delete prei/api/pipeline_routes.py and prei/cli.py (click). Resolve LIMIT-21 (offer.py float-based) by porting offer math to Decimal in core/services/offer.py OR deleting the handler — port the pure math (OfferInput/OfferMetrics as Decimal dataclasses) to keep tests/test_offer_integration.py behavior, delete the pydantic/CLI/API surface. Requires PM sign-off.", + "depends_on": ["TASK-07"], "acceptance_criteria": [ - {"id": "AC-C4-01", "description": "pytest.ini's addopts includes --report-log=.pytest-report.jsonl", "test_type": "unit"}, - {"id": "AC-C4-02", "description": "flaky_report.py --mode report detects rerun-then-pass nodeids without touching the ledger", "test_type": "unit"}, - {"id": "AC-C4-03", "description": "flaky_report.py --mode write increments the ledger and quarantines at the threshold", "test_type": "unit"} - ] + {"id": "AC-09-1", "description": "prei/api/ and prei/cli.py deleted; no fastapi/click imports in production code", "test_type": "ci"}, + {"id": "AC-09-2", "description": "Offer math ported Decimal-based to core/services/offer.py (LIMIT-21 resolved) or explicitly deferred with PM sign-off", "test_type": "manual"}, + {"id": "AC-09-3", "description": "prei/pipeline/tests/test_api.py deleted (tests deleted code)", "test_type": "manual"} + ], + "agent": "build", + "estimated_lines": 400, + "skills": ["build/refactoring", "lang-python"] }, { - "id": "C-4b", - "summary": "conftest.py quarantine hook", - "description": "Add pytest_collection_modifyitems hook to root conftest.py: nodeids listed in tests/.flaky_quarantine.txt get marked xfail(strict=False) at collection time.", - "depends_on": ["C-4a"], + "id": "TASK-10", + "summary": "Migrate or delete prei/pipeline/tests/* and remaining handlers/sources", + "description": "Port valuable tests (test_sources, test_county, test_reo_sources, test_screening, test_underwriting, test_discovery math) to core/tests/. Delete tests that only exercised deleted engine state (test_engine, test_repository, test_orchestrator, test_batch_screening, test_api). Delete prei/pipeline/handlers/batch_screening.py, discovery.py, discovery_processor.py, screening.py, underwriting.py (pure logic now lives in core/services) and prei/pipeline/sources/* (county/reo_sources/vrm/file_source/registry/base) only after confirming nothing imports them; if sources are still needed by the Growth Explorer bridge, move them into core/services/sources/ first in TASK-05.", + "depends_on": ["TASK-08", "TASK-09"], "acceptance_criteria": [ - {"id": "AC-C4-04", "description": "conftest.py marks quarantined nodeids xfail(strict=False) so they can't fail the build", "test_type": "unit"} - ] + {"id": "AC-10-1", "description": "prei/ package fully deleted (no prei.* imports anywhere in repo)", "test_type": "ci"}, + {"id": "AC-10-2", "description": "All valuable test coverage relocated to core/tests/ and green", "test_type": "ci"}, + {"id": "AC-10-3", "description": "Full suite + ruff + mypy green with prei/ gone", "test_type": "ci"} + ], + "agent": "build", + "estimated_lines": 2100, + "skills": ["build/refactoring", "build/governance-enforcement"] }, { - "id": "C-4c", - "summary": "Wire flaky_report.py into CI jobs", - "description": "ci-quality.yml's tests-unit/tests-integration/tests-e2e jobs run flaky_report.py --mode report after pytest and upload the report log as an artifact. docker-publish.yml's live-test job (push-to-main only) extracts the report log from the container, runs flaky_report.py --mode write, and bot-commits any ledger/quarantine change to main as github-actions[bot] with [skip ci] (needs contents: write).", - "depends_on": ["C-4a", "C-4b"], + "id": "TASK-11", + "summary": "Remove pydantic/fastapi/uvicorn/click from requirements + docs update", + "description": "Remove pydantic==2.13.4, fastapi==0.139.2, uvicorn[standard]==0.51.0, click==8.4.2 from requirements.txt only if no remaining code imports them (tests/acceptance/schemas.py + tests/acceptance/test_api.py use pydantic for acceptance payload validation — verify whether that is test-only and keep pydantic pinned for those, or migrate acceptance schemas to dataclasses; fastapi/click must be fully removable). Update docs/ARCHITECTURE.md, docs/KNOWN_LIMITATIONS.md (mark LIMIT-21 resolved or explicitly deferred), docs/CHANGE_IMPACT_MAP.md.", + "depends_on": ["TASK-10"], "acceptance_criteria": [ - {"id": "AC-C4-05", "description": "PR test jobs run flaky_report.py --mode report and upload the report log artifact", "test_type": "ci"}, - {"id": "AC-C4-06", "description": "live-test bot-commits ledger/quarantine changes to main via flaky_report.py --mode write", "test_type": "ci"} - ] + {"id": "AC-11-1", "description": "fastapi, uvicorn, click removable from requirements.txt; pydantic only retained if tests/acceptance still needs it", "test_type": "ci"}, + {"id": "AC-11-2", "description": "KNOWN_LIMITATIONS LIMIT-21 status updated", "test_type": "manual"}, + {"id": "AC-11-3", "description": "ARCHITECTURE.md and CHANGE_IMPACT_MAP.md reflect Django-canonical pipeline", "test_type": "manual"} + ], + "agent": "build", + "estimated_lines": 120, + "skills": ["build/refactoring", "documentation"] } ] } diff --git a/tests/test_discovery.py b/tests/test_discovery.py index 87830c00..ac62958e 100644 --- a/tests/test_discovery.py +++ b/tests/test_discovery.py @@ -6,8 +6,8 @@ 3. Batch deduplication integrity with hash collision """ -from prei.pipeline.handlers.discovery import DiscoverySanitizer -from prei.pipeline.handlers.discovery_processor import DiscoveryProcessor +from core.services.discovery import DiscoverySanitizer +from core.services.discovery_processor import process_discovery_batch class TestScrubbingNormalization: @@ -140,7 +140,7 @@ class TestBatchDeduplication: def test_batch_deduplication_integrity(self): """3-item batch: all hash-identical → 1 new + 2 duplicates.""" - processor = DiscoveryProcessor(existing_hashes=set()) + existing_hashes: set[str] = set() mock_batch = [ {"id": "PROP-1", "address": "Duplicate St 1", "price": 200_000}, @@ -152,9 +152,11 @@ def test_batch_deduplication_integrity(self): dup_hash = DiscoverySanitizer.transform_input( mock_batch[0], source="TEST" ).address_hash - processor.existing_hashes.add(dup_hash) + existing_hashes.add(dup_hash) - metrics = processor.process_batch(mock_batch, source_name="TEST_FEED") + metrics = process_discovery_batch( + mock_batch, source_name="TEST_FEED", existing_hashes=existing_hashes + ) # PROP-1 is new (address_hash matches dup_hash but was added *after* # the existing hash was injected, so PROP-1 is caught as duplicate) @@ -167,7 +169,7 @@ def test_batch_deduplication_integrity(self): def test_three_identical_in_five_item_batch(self): """5-item batch with 3 identical addresses → 3 new + 2 duplicates.""" - processor = DiscoveryProcessor(existing_hashes=set()) + existing_hashes: set[str] = set() mock_batch = [ {"id": "A1", "address": "100 Common St", "price": 150_000}, @@ -181,9 +183,11 @@ def test_three_identical_in_five_item_batch(self): dup_hash = DiscoverySanitizer.transform_input( mock_batch[0], source="TEST" ).address_hash - processor.existing_hashes.add(dup_hash) + existing_hashes.add(dup_hash) - metrics = processor.process_batch(mock_batch, source_name="TEST_FEED") + metrics = process_discovery_batch( + mock_batch, source_name="TEST_FEED", existing_hashes=existing_hashes + ) # A1: duplicate (hash pre-existing) → skip # A2: new address → discover @@ -196,20 +200,19 @@ def test_three_identical_in_five_item_batch(self): def test_no_duplicates_in_empty_existing_set(self): """Empty existing_hashes → all items discovered.""" - processor = DiscoveryProcessor(existing_hashes=set()) batch = [ {"id": "X1", "address": "Alpha St", "price": 100_000}, {"id": "X2", "address": "Beta Ave", "price": 200_000}, {"id": "X3", "address": "Gamma Blvd", "price": 300_000}, ] - metrics = processor.process_batch(batch, source_name="test") + metrics = process_discovery_batch(batch, source_name="test") assert metrics["new_assets_discovered"] == 3 assert metrics["duplicates_skipped"] == 0 assert metrics["failed_records"] == 0 def test_all_duplicates_no_new(self): """When all items' hashes already exist → zero discovered.""" - processor = DiscoveryProcessor(existing_hashes=set()) + existing_hashes: set[str] = set() batch = [ {"id": "D1", "address": "Same St", "price": 100_000}, {"id": "D2", "address": "Same St", "price": 200_000}, @@ -218,9 +221,11 @@ def test_all_duplicates_no_new(self): dup_hash = DiscoverySanitizer.transform_input( batch[0], source="TEST" ).address_hash - processor.existing_hashes.add(dup_hash) + existing_hashes.add(dup_hash) - metrics = processor.process_batch(batch, source_name="test") + metrics = process_discovery_batch( + batch, source_name="test", existing_hashes=existing_hashes + ) assert metrics["new_assets_discovered"] == 0 assert metrics["duplicates_skipped"] == 2 @@ -231,12 +236,8 @@ def test_deduplication_deterministic(self): {"id": "P2", "address": "100 Main", "price": 100_000}, {"id": "P3", "address": "200 Oak", "price": 200_000}, ] - p1 = DiscoveryProcessor(existing_hashes=set()) - p2 = DiscoveryProcessor(existing_hashes=set()) h = DiscoverySanitizer.transform_input(batch[0], source="T").address_hash - p1.existing_hashes.add(h) - p2.existing_hashes.add(h) - r1 = p1.process_batch(batch, "test") - r2 = p2.process_batch(batch, "test") + r1 = process_discovery_batch(batch, "test", existing_hashes={h}) + r2 = process_discovery_batch(batch, "test", existing_hashes={h}) for key in ("new_assets_discovered", "duplicates_skipped", "failed_records"): assert r1[key] == r2[key] diff --git a/tests/test_discovery_e2e.py b/tests/test_discovery_e2e.py index 07182857..ac34d1cf 100644 --- a/tests/test_discovery_e2e.py +++ b/tests/test_discovery_e2e.py @@ -1,7 +1,7 @@ """Live end-to-end tests for the discovery stage of the pipeline. Tests simulate the full discovery flow: - raw listings → DiscoverySanitizer → DiscoveryProcessor → PipelineOrchestrator + raw listings → DiscoverySanitizer → process_discovery_batch These are marked as 'e2e' and 'slow' — skipped in CI unless explicitly requested. Run with: pytest tests/test_discovery_e2e.py -v -m e2e @@ -9,10 +9,8 @@ import pytest -from prei.models.pipeline import PipelineStage -from prei.pipeline.handlers.discovery import DiscoverySanitizer -from prei.pipeline.handlers.discovery_processor import DiscoveryProcessor -from prei.pipeline.orchestrator import PipelineOrchestrator +from core.services.discovery import DiscoverySanitizer +from core.services.discovery_processor import process_discovery_batch pytestmark = [ pytest.mark.e2e, @@ -95,7 +93,7 @@ @pytest.mark.e2e class TestE2EMlsPipeline: - """End-to-end: raw MLS listings through the full discovery pipeline.""" + """End-to-end: raw MLS listings through the discovery pipeline.""" def test_mls_feed_through_sanitizer(self): """MLS listing normalizes to canonical payload with correct fields.""" @@ -114,28 +112,15 @@ def test_mls_feed_through_sanitizer(self): def test_mls_feed_through_discovery_processor(self): """MLS listings are ingested without duplicates.""" - proc = DiscoveryProcessor(existing_hashes=set()) - result = proc.process_batch(REALISTIC_MLS_FEED, source_name="mls") + result = process_discovery_batch(REALISTIC_MLS_FEED, source_name="mls") assert result["total_received"] == 3 assert result["new_assets_discovered"] == 3 assert result["duplicates_skipped"] == 0 assert result["failed_records"] == 0 assert len(result["payloads"]) == 3 - for asset in result["payloads"]: - assert asset.current_stage == PipelineStage.DISCOVERY - - def test_mls_feed_through_full_orchestrator(self): - """A single MLS listing completes the full pipeline.""" - orch = PipelineOrchestrator() - result = orch.run(REALISTIC_MLS_FEED[0], source_name="mls") - assert result.success - assert result.screening_passed is True - assert result.asset.current_stage == PipelineStage.UNDERWRITING - assert result.underwriting is not None - assert result.underwriting.noi > 0 - assert result.underwriting.cap_rate > 0 - assert result.underwriting.mao > 0 - assert result.underwriting.cash_on_cash > 0 + for payload in result["payloads"]: + assert payload.source_name == "mls" + assert payload.address_hash # ═══════════════════════════════════════════════════════════════════════════════ @@ -160,12 +145,16 @@ def test_county_feed_through_sanitizer(self): def test_county_feed_dedup_against_mls(self): """County records with same address as MLS listings are deduplicated.""" - proc = DiscoveryProcessor(existing_hashes=set()) + existing: set[str] = set() # Process MLS feed first - proc.process_batch(REALISTIC_MLS_FEED, source_name="mls") + process_discovery_batch( + REALISTIC_MLS_FEED, source_name="mls", existing_hashes=existing + ) # Process county feed — should have no overlap if addresses are unique - result = proc.process_batch(REALISTIC_COUNTY_FEED, source_name="county") + result = process_discovery_batch( + REALISTIC_COUNTY_FEED, source_name="county", existing_hashes=existing + ) assert result["new_assets_discovered"] == 2 assert result["duplicates_skipped"] == 0 @@ -188,42 +177,34 @@ class TestE2EMultiSourceOrchestration: def test_multi_source_dedup_shared_hashes(self): """Shared existing_hashes set prevents cross-source duplicates.""" - hashes: set = set() - proc = DiscoveryProcessor(existing_hashes=hashes) - proc.process_batch(REALISTIC_MLS_FEED, "mls") + hashes: set[str] = set() + process_discovery_batch(REALISTIC_MLS_FEED, "mls", existing_hashes=hashes) assert len(hashes) == 3 - proc.process_batch(REALISTIC_COUNTY_FEED, "county") + process_discovery_batch(REALISTIC_COUNTY_FEED, "county", existing_hashes=hashes) assert len(hashes) == 5 # 3 MLS + 2 county (all unique addresses) - def test_multi_source_orchestrator(self): - """Orchestrator with shared existing_hashes dedups across calls.""" - hashes: set = set() - orch = PipelineOrchestrator(existing_hashes=hashes) + def test_multi_source_dedup_via_batch(self): + """Shared hash set dedups across calls.""" + hashes: set[str] = set() - r1 = orch.run(REALISTIC_MLS_FEED[0], source_name="mls") - assert r1.success + r1 = process_discovery_batch( + REALISTIC_MLS_FEED[:1], source_name="mls", existing_hashes=hashes + ) + assert r1["new_assets_discovered"] == 1 # Same address again → duplicate - r2 = orch.run(REALISTIC_MLS_FEED[0], source_name="mls") - assert r2.success is False - assert "Duplicate" in (r2.error or "") + r2 = process_discovery_batch( + REALISTIC_MLS_FEED[:1], source_name="mls", existing_hashes=hashes + ) + assert r2["new_assets_discovered"] == 0 + assert r2["duplicates_skipped"] == 1 # Different address → success - r3 = orch.run(REALISTIC_MLS_FEED[1], source_name="mls") - assert r3.success - - def test_pipeline_result_contains_underwriting(self): - """Pipeline result includes full underwriting metrics.""" - orch = PipelineOrchestrator(target_cap_rate=0.08) - for payload in REALISTIC_MLS_FEED: - result = orch.run(payload, source_name="mls") - assert result.success - uw = result.underwriting - assert uw.noi >= 0 - assert uw.mao > 0 - assert uw.cap_rate > 0 - assert uw.cash_on_cash > 0 + r3 = process_discovery_batch( + REALISTIC_MLS_FEED[1:2], source_name="mls", existing_hashes=hashes + ) + assert r3["new_assets_discovered"] == 1 # ═══════════════════════════════════════════════════════════════════════════════ @@ -237,67 +218,27 @@ class TestE2EDailyPipelineRun: def test_simulated_daily_pipeline(self): """Full daily run: ingest MLS + county, dedup, compute metrics.""" - hashes: set = set() - mls_processor = DiscoveryProcessor(existing_hashes=hashes) - county_processor = DiscoveryProcessor(existing_hashes=hashes) - orch = PipelineOrchestrator(existing_hashes=hashes) + hashes: set[str] = set() # ── Morning: MLS feed ───────────────────────────────────────────── - mls_result = mls_processor.process_batch(REALISTIC_MLS_FEED, "mls") + mls_result = process_discovery_batch( + REALISTIC_MLS_FEED, "mls", existing_hashes=hashes + ) assert mls_result["new_assets_discovered"] == 3 assert mls_result["failed_records"] == 0 # ── Afternoon: County feed ──────────────────────────────────────── - county_result = county_processor.process_batch(REALISTIC_COUNTY_FEED, "county") + county_result = process_discovery_batch( + REALISTIC_COUNTY_FEED, "county", existing_hashes=hashes + ) assert county_result["new_assets_discovered"] == 2 assert county_result["duplicates_skipped"] == 0 - # ── Evening: Run full pipeline on each new asset ────────────────── - for payload in REALISTIC_MLS_FEED + REALISTIC_COUNTY_FEED: - # Convert county key schema to discoverable dict - if "parcel_id" in payload: - feed_payload = { - "id": payload["parcel_id"], - "address": payload["FullStreetAddress"], - "price": payload["sale_price"], - "rent": payload.get( - "estimated_rent", payload["sale_price"] * 0.008 - ), - "beds": int(float(payload["BedroomsTotal"])), - "baths": float(payload["BathroomsTotalInteger"]), - "sqft": float(payload.get("LivingArea", 0)), - "year_built": payload.get("YearBuilt"), - } - else: - feed_payload = payload - - result = orch.run(feed_payload, source_name="daily_pipeline") - # Assets should already be in hashes set from processor runs - # but orchestrator has its own hashes set — they should match - if result.success: - assert result.asset.current_stage == PipelineStage.UNDERWRITING # noqa: E501 - assert result.underwriting.mao > 0 - - def test_pipeline_summary_output(self): - """Pipeline to_dict() output contains all expected fields.""" - orch = PipelineOrchestrator() - result = orch.run(REALISTIC_MLS_FEED[0], source_name="mls") - summary = result.to_dict() - expected_keys = { - "success", - "asset_id", - "current_stage", - "address_hash", - "price", - "beds", - "baths", - "screening_passed", - "noi", - "cap_rate", - "cash_on_cash", - "mao", - "target_cap_rate", - } - assert expected_keys.issubset(summary.keys()) - assert summary["screening_passed"] is True - assert summary["current_stage"] == "UNDERWRITING" + # ── Evening: re-run full feeds — everything should dedup ────────── + rerun = process_discovery_batch( + REALISTIC_MLS_FEED + REALISTIC_COUNTY_FEED, + "daily_pipeline", + existing_hashes=hashes, + ) + assert rerun["new_assets_discovered"] == 0 + assert rerun["duplicates_skipped"] == 5 diff --git a/tests/test_discovery_integration.py b/tests/test_discovery_integration.py index 1f1d7116..e3b6289b 100644 --- a/tests/test_discovery_integration.py +++ b/tests/test_discovery_integration.py @@ -1,15 +1,16 @@ """Integration tests for the discovery stage — components working together. -Tests how DiscoverySanitizer, DiscoveryProcessor, and data sources -integrate with each other and with the PipelineOrchestrator. +Tests how DiscoverySanitizer, the discovery processor, and data sources +integrate with each other and with the downstream screening/underwriting +services (the prei orchestrator class was deleted in the pydantic→Django +consolidation; orchestration is now explicit service composition). """ -from prei.models.pipeline import PipelineStage -from prei.pipeline.handlers.discovery import DiscoverySanitizer -from prei.pipeline.handlers.discovery_processor import DiscoveryProcessor -from prei.pipeline.orchestrator import PipelineOrchestrator -from prei.pipeline.sources.county import TexasCountyForeclosureSource -from prei.pipeline.sources.registry import discover_from_all, get_source +from core.services.discovery import DiscoverySanitizer +from core.services.discovery_processor import process_discovery_batch +from core.services.screening import ScreeningThresholds, screen_batch +from core.services.sources.county import TexasCountyForeclosureSource +from core.services.sources.registry import discover_from_all, get_source # ── Fixtures ────────────────────────────────────────────────────────────────── @@ -67,7 +68,7 @@ class TestSanitizerToProcessor: - """DiscoverySanitizer output feeds directly into DiscoveryProcessor.""" + """DiscoverySanitizer output feeds directly into the processor.""" def test_sanitizer_output_matches_processor_input(self): """CanonicalPropertyPayload fields map correctly to processor expectations.""" @@ -81,22 +82,21 @@ def test_sanitizer_output_matches_processor_input(self): def test_processor_uses_correct_hash(self): """Processor dedup uses same hash as sanitizer produces.""" canonical = DiscoverySanitizer.transform_input(MLS_BATCH[0], "test") - proc = DiscoveryProcessor(existing_hashes={canonical.address_hash}) - result = proc.process_batch(MLS_BATCH, source_name="test") + result = process_discovery_batch( + MLS_BATCH, source_name="test", existing_hashes={canonical.address_hash} + ) # MLS-001 is duplicate (hash pre-populated) # MLS-002 and MLS-003 are new assert result["new_assets_discovered"] == 2 assert result["duplicates_skipped"] == 1 - def test_processor_outputs_are_property_assets(self): - """Processor returns PropertyAsset instances with correct stage.""" - proc = DiscoveryProcessor(existing_hashes=set()) - result = proc.process_batch(MLS_BATCH[:1], source_name="test") + def test_processor_outputs_are_canonical_payloads(self): + """Processor returns canonical payloads with source identity.""" + result = process_discovery_batch(MLS_BATCH[:1], source_name="test") assert len(result["payloads"]) == 1 - asset = result["payloads"][0] - assert asset.current_stage == PipelineStage.DISCOVERY - assert asset.asset_id == "MLS-001" - assert "123 main st" in asset.address.lower() + payload = result["payloads"][0] + assert payload.source_id == "MLS-001" + assert "123 main st" in payload.raw_address.lower() # ═══════════════════════════════════════════════════════════════════════════════ @@ -108,24 +108,24 @@ class TestMultiSourceProcessing: """Different source schemas all flow through the same processor.""" def test_mls_schema_processed(self): - """MLS-format listings produce valid assets.""" - proc = DiscoveryProcessor(existing_hashes=set()) - result = proc.process_batch(MLS_BATCH, source_name="mls") + """MLS-format listings produce valid payloads.""" + result = process_discovery_batch(MLS_BATCH, source_name="mls") assert result["new_assets_discovered"] == 3 assert result["failed_records"] == 0 def test_county_schema_processed(self): - """County-foreclosure-format listings produce valid assets.""" - proc = DiscoveryProcessor(existing_hashes=set()) - result = proc.process_batch(COUNTY_BATCH, source_name="county") + """County-foreclosure-format listings produce valid payloads.""" + result = process_discovery_batch(COUNTY_BATCH, source_name="county") assert result["new_assets_discovered"] == 2 assert result["failed_records"] == 0 def test_mixed_source_deduplication(self): """Same address from different sources matches via hash.""" - proc = DiscoveryProcessor(existing_hashes=set()) + existing: set[str] = set() # Process MLS batch first - r1 = proc.process_batch(MLS_BATCH, source_name="mls") + r1 = process_discovery_batch( + MLS_BATCH, source_name="mls", existing_hashes=existing + ) assert r1["new_assets_discovered"] == 3 # Process a batch containing a duplicate address in county format @@ -134,15 +134,16 @@ def test_mixed_source_deduplication(self): "FullStreetAddress": "123 Main St.", # same as MLS-001 "sale_price": 310_000.0, } - r2 = proc.process_batch([duplicate], source_name="county") + r2 = process_discovery_batch( + [duplicate], source_name="county", existing_hashes=existing + ) assert r2["new_assets_discovered"] == 0 assert r2["duplicates_skipped"] == 1 def test_empty_batch_mixed_sources(self): """Empty batches from any source produce zero results.""" for source_name in ["mls", "county", "fannie_mae"]: - proc = DiscoveryProcessor(existing_hashes=set()) - result = proc.process_batch([], source_name=source_name) + result = process_discovery_batch([], source_name=source_name) assert result["total_received"] == 0 assert result["new_assets_discovered"] == 0 @@ -175,81 +176,104 @@ def test_discover_from_all_returns_dict(self): # ═══════════════════════════════════════════════════════════════════════════════ -# 4. Dedup across batches (stateful processor) +# 4. Dedup across batches (shared hash set) # ═══════════════════════════════════════════════════════════════════════════════ class TestCrossBatchDeduplication: - """Processor maintains state across multiple batches.""" + """Shared hash set dedups across multiple batches.""" - def test_same_processor_multiple_batches(self): - """Same processor instance dedups across batches.""" - proc = DiscoveryProcessor(existing_hashes=set()) + def test_shared_hashes_multiple_batches(self): + """Same hash set across batches dedups.""" + existing: set[str] = set() - r1 = proc.process_batch(MLS_BATCH[:2], source_name="mls") + r1 = process_discovery_batch( + MLS_BATCH[:2], source_name="mls", existing_hashes=existing + ) assert r1["new_assets_discovered"] == 2 - r2 = proc.process_batch(MLS_BATCH[2:], source_name="mls") + r2 = process_discovery_batch( + MLS_BATCH[2:], source_name="mls", existing_hashes=existing + ) assert r2["new_assets_discovered"] == 1 - r3 = proc.process_batch(MLS_BATCH, source_name="mls") + r3 = process_discovery_batch( + MLS_BATCH, source_name="mls", existing_hashes=existing + ) assert r3["new_assets_discovered"] == 0 assert r3["duplicates_skipped"] == 3 - def test_separate_processors_independent(self): - """Different processor instances have independent hash sets.""" - p1 = DiscoveryProcessor(existing_hashes=set()) - p2 = DiscoveryProcessor(existing_hashes=set()) - p1.process_batch(MLS_BATCH, source_name="mls") - p2.process_batch(MLS_BATCH, source_name="mls") + def test_separate_hash_sets_independent(self): + """Different hash sets are independent.""" + p1: set[str] = set() + p2: set[str] = set() + process_discovery_batch(MLS_BATCH, source_name="mls", existing_hashes=p1) + process_discovery_batch(MLS_BATCH, source_name="mls", existing_hashes=p2) # Both should have discovered all 3 since their hash sets started empty - assert len(p1.existing_hashes) == 3 - assert len(p2.existing_hashes) == 3 + assert len(p1) == 3 + assert len(p2) == 3 def test_existing_hashes_persistence(self): """existing_hashes set is mutated in-place after each batch.""" - hashes: set = set() - proc = DiscoveryProcessor(existing_hashes=hashes) - proc.process_batch(MLS_BATCH, source_name="test") + hashes: set[str] = set() + process_discovery_batch(MLS_BATCH, source_name="test", existing_hashes=hashes) assert len(hashes) == 3 # ═══════════════════════════════════════════════════════════════════════════════ -# 5. Processor → PipelineOrchestrator integration +# 5. Discovery → Screening → Underwriting composition # ═══════════════════════════════════════════════════════════════════════════════ -class TestProcessorToOrchestrator: - """DiscoveryProcessor output feeds into PipelineOrchestrator.""" - - def test_discovered_asset_can_run_through_orchestrator(self): - """An asset discovered by processor can be pipelined through orchestrator.""" - proc = DiscoveryProcessor(existing_hashes=set()) - result = proc.process_batch(MLS_BATCH[:1], source_name="mls") - asset = result["payloads"][0] - assert asset.current_stage == PipelineStage.DISCOVERY - - # The orchestrator would take the raw payload directly - orch = PipelineOrchestrator(existing_hashes=set()) - or_result = orch.run(MLS_BATCH[0], source_name="mls") - assert or_result.success - assert or_result.asset.current_stage == PipelineStage.UNDERWRITING - - def test_orchestrator_rejects_duplicates(self): - """Orchestrator rejects duplicate address hashes.""" - orch = PipelineOrchestrator(existing_hashes=set()) - r1 = orch.run(MLS_BATCH[0], source_name="mls") - assert r1.success - r2 = orch.run(MLS_BATCH[0], source_name="mls") - assert r2.success is False - assert "Duplicate" in (r2.error or "") +class TestDiscoveryToDownstreamServices: + """Discovery payloads feed into screening and underwriting (orchestrator + equivalent — the prei orchestrator class was deleted).""" + + def test_discovered_payload_runs_through_screening(self): + """A discovered payload can be screened with standard thresholds.""" + result = process_discovery_batch(MLS_BATCH[:1], source_name="mls") + payload = result["payloads"][0] + threshold = ScreeningThresholds( + min_gross_yield=0.07, + max_price_to_rent_ratio=15.0, + min_beds=2, + min_baths=1, + ) + screening = screen_batch( + [ + { + "asset_id": payload.source_id, + "address": payload.raw_address, + "estimated_monthly_rent": payload.estimated_rent, + "purchase_price": payload.price, + "beds": payload.beds, + "baths": payload.baths, + } + ], + threshold, + ) + # MLS-001: (2500*12)/300000 = 10% yield, ratio 10 → passes + assert screening["advanced"] == 1 + assert screening["killed"] == 0 + + def test_duplicate_payload_rejected_at_discovery(self): + """Same address twice → second run discovers nothing.""" + existing: set[str] = set() + r1 = process_discovery_batch( + MLS_BATCH[:1], source_name="mls", existing_hashes=existing + ) + assert r1["new_assets_discovered"] == 1 + r2 = process_discovery_batch( + MLS_BATCH[:1], source_name="mls", existing_hashes=existing + ) + assert r2["new_assets_discovered"] == 0 + assert r2["duplicates_skipped"] == 1 - def test_orchestrator_fails_missing_address(self): - """Orchestrator fails gracefully on address-less payload.""" - orch = PipelineOrchestrator() - result = orch.run({"id": "BAD"}, source_name="test") - assert result.success is False - assert "Discovery" in (result.error or "") + def test_missing_address_fails_gracefully(self): + """Address-less payload is counted as failed, not raised.""" + result = process_discovery_batch([{"id": "BAD"}], source_name="test") + assert result["failed_records"] == 1 + assert result["new_assets_discovered"] == 0 # ═══════════════════════════════════════════════════════════════════════════════ @@ -272,8 +296,7 @@ def test_1000_unique_listings(self): } for i in range(1000) ] - proc = DiscoveryProcessor(existing_hashes=set()) - result = proc.process_batch(batch, source_name="stress") + result = process_discovery_batch(batch, source_name="stress") assert result["total_received"] == 1000 assert result["new_assets_discovered"] == 1000 assert result["duplicates_skipped"] == 0 @@ -302,8 +325,7 @@ def test_1000_listings_50_percent_duplicates(self): "baths": 2, } ) - proc = DiscoveryProcessor(existing_hashes=set()) - result = proc.process_batch(batch, source_name="stress") + result = process_discovery_batch(batch, source_name="stress") assert result["new_assets_discovered"] == 500 assert result["duplicates_skipped"] == 500 assert result["failed_records"] == 0 @@ -316,8 +338,7 @@ def test_malformed_listings_dont_crash_batch(self): {"id": "BAD", "price": 100}, # bad — no address *COUNTY_BATCH, ] - proc = DiscoveryProcessor(existing_hashes=set()) - result = proc.process_batch(batch, source_name="mixed") + result = process_discovery_batch(batch, source_name="mixed") # 3 MLS + 2 county = 5 valid, 2 bad assert result["new_assets_discovered"] == 5 assert result["failed_records"] == 2 diff --git a/tests/test_offer_integration.py b/tests/test_offer_integration.py index 7e25d06a..d9b92bbc 100644 --- a/tests/test_offer_integration.py +++ b/tests/test_offer_integration.py @@ -1,14 +1,25 @@ -"""Unit, integration, and E2E tests for the offer stage handler.""" +"""Unit, integration, and E2E tests for the offer stage handler. + +The offer port to core.services.offer is Decimal-based (LIMIT-21 resolved), +so float equality/type assertions from the pydantic era were replaced with +Decimal comparisons. +""" import pytest -from prei.pipeline.handlers.offer import ( +from dataclasses import replace +from decimal import Decimal + +from core.services.offer import ( OfferInput, OfferStrategy, solve_offer, ) BASE_INPUT = OfferInput( - mao=300_000.0, arv=420_000.0, rehab_budget=20_000.0, desired_equity=0.20 + mao=Decimal("300000.00"), + arv=Decimal("420000.00"), + rehab_budget=Decimal("20000.00"), + desired_equity=Decimal("0.20"), ) # ═══════════════════════════════════════════════════════════════════════════════ @@ -24,8 +35,10 @@ def test_conservative_below_mao(self): def test_target_at_mao(self): result = solve_offer(BASE_INPUT, OfferStrategy.TARGET) - assert result.offer_price == pytest.approx(BASE_INPUT.mao, rel=1e-4) - assert result.premium_pct == pytest.approx(0, abs=1e-4) + assert result.offer_price == pytest.approx( + BASE_INPUT.mao, rel=Decimal("0.0001") + ) + assert result.premium_pct == pytest.approx(0, abs=Decimal("0.0001")) def test_aggressive_above_mao(self): result = solve_offer(BASE_INPUT, OfferStrategy.AGGRESSIVE) @@ -35,14 +48,14 @@ def test_aggressive_above_mao(self): def test_competition_multiplier_increases_offer(self): base = solve_offer(BASE_INPUT, OfferStrategy.TARGET) hot = solve_offer( - BASE_INPUT.model_copy(update={"competition_multiplier": 1.5}), + replace(BASE_INPUT, competition_multiplier=Decimal("1.5")), OfferStrategy.TARGET, ) assert hot.offer_price > base.offer_price def test_competition_multiplier_decreases_offer(self): cold = solve_offer( - BASE_INPUT.model_copy(update={"competition_multiplier": 0.75}), + replace(BASE_INPUT, competition_multiplier=Decimal("0.75")), OfferStrategy.TARGET, ) assert cold.offer_price < BASE_INPUT.mao @@ -50,23 +63,25 @@ def test_competition_multiplier_decreases_offer(self): def test_equity_constraint_clamps_offer(self): """High desired equity clamps offer below MAO.""" high_equity = OfferInput( - mao=300_000, arv=320_000, rehab_budget=20_000, desired_equity=0.25 + mao=Decimal("300000"), + arv=Decimal("320000"), + rehab_budget=Decimal("20000"), + desired_equity=Decimal("0.25"), ) result = solve_offer(high_equity, OfferStrategy.AGGRESSIVE) # Max offer for 25% equity: 320000*0.75 - 20000 = 220000 - assert result.offer_price <= 220_000 + assert result.offer_price <= Decimal("220000") def test_no_arv_skips_equity_calc(self): """Without ARV, equity fields are None.""" - result = solve_offer( - BASE_INPUT.model_copy(update={"arv": None}), OfferStrategy.TARGET - ) + result = solve_offer(replace(BASE_INPUT, arv=None), OfferStrategy.TARGET) assert result.estimated_equity is None assert result.estimated_equity_pct is None - def test_mao_zero_returns_zero_offer(self): - result = solve_offer(OfferInput(mao=0), OfferStrategy.TARGET) - assert result.offer_price == 0.0 + def test_mao_zero_raises_value_error(self): + """MAO of zero is rejected (must be > 0).""" + with pytest.raises(ValueError): + solve_offer(OfferInput(mao=Decimal("0")), OfferStrategy.TARGET) def test_strategy_enum_values(self): assert OfferStrategy.CONSERVATIVE.value == "conservative" @@ -81,18 +96,22 @@ def test_premium_pct_negative_for_conservative(self): result = solve_offer(BASE_INPUT, OfferStrategy.CONSERVATIVE) assert result.premium_pct < 0 - def test_offer_metrics_are_floats(self): + def test_offer_metrics_are_decimal(self): result = solve_offer(BASE_INPUT, OfferStrategy.TARGET) - assert isinstance(result.offer_price, float) - assert isinstance(result.premium_over_mao, float) + assert isinstance(result.offer_price, Decimal) + assert isinstance(result.premium_over_mao, Decimal) def test_equity_calculation_correct(self): """Equity = ARV - (offer + rehab). For TARGET strategy: offer = MAO = 300k.""" result = solve_offer(BASE_INPUT, OfferStrategy.TARGET) # offer = MAO = 300000 (no clamp since 300000 + 20000 = 320000 <= 420000*0.80 = 336000) expected_equity = BASE_INPUT.arv - (BASE_INPUT.mao + BASE_INPUT.rehab_budget) - assert result.estimated_equity == pytest.approx(expected_equity, rel=1e-4) - assert result.estimated_equity == pytest.approx(100_000.0, rel=1e-4) + assert result.estimated_equity == pytest.approx( + expected_equity, rel=Decimal("0.0001") + ) + assert result.estimated_equity == pytest.approx( + Decimal("100000.00"), rel=Decimal("0.0001") + ) # ═══════════════════════════════════════════════════════════════════════════════ @@ -103,24 +122,24 @@ def test_equity_calculation_correct(self): class TestOfferIntegration: def test_offer_from_underwriting_mao(self): """Underwriting MAO feeds directly into offer solver.""" - from prei.pipeline.handlers.underwriting import ( + from core.services.underwriting import ( solve_underwriting, UnderwritingInput, ) uw = solve_underwriting( UnderwritingInput( - purchase_price=300000, - estimated_rent=2500, - property_tax_annual=3600, - insurance_annual=1200, + purchase_price=Decimal("300000"), + estimated_rent=Decimal("2500"), + property_tax_annual=Decimal("3600"), + insurance_annual=Decimal("1200"), ), target_cap_rate=0.08, ) offer = solve_offer( - OfferInput(mao=uw.mao, arv=float(uw.mao) * 1.15), OfferStrategy.TARGET + OfferInput(mao=uw.mao, arv=uw.mao * Decimal("1.15")), OfferStrategy.TARGET ) - assert offer.offer_price == pytest.approx(float(uw.mao), rel=1e-3) + assert offer.offer_price == pytest.approx(uw.mao, rel=Decimal("0.001")) assert offer.estimated_equity is not None assert offer.estimated_equity > 0 @@ -134,7 +153,12 @@ def test_offer_strategies_spread(self): def test_offer_equity_clamp_preserves_strategy_label(self): """Clamped offer still reports correct strategy.""" result = solve_offer( - OfferInput(mao=300000, arv=310000, rehab_budget=20000, desired_equity=0.20), + OfferInput( + mao=Decimal("300000"), + arv=Decimal("310000"), + rehab_budget=Decimal("20000"), + desired_equity=Decimal("0.20"), + ), OfferStrategy.AGGRESSIVE, ) assert result.strategy == OfferStrategy.AGGRESSIVE @@ -147,42 +171,41 @@ def test_offer_equity_clamp_preserves_strategy_label(self): @pytest.mark.e2e class TestOfferE2E: - def test_e2e_full_pipeline_with_offer(self): - """Run full pipeline and use MAO for offer calculation.""" - from prei.pipeline.orchestrator import PipelineOrchestrator - - orch = PipelineOrchestrator(target_cap_rate=0.08) - result = orch.run( - { - "id": "OFFER-E2E", - "address": "500 Deal St", - "price": 350000, - "rent": 2800, - "beds": 3, - "baths": 2, - } + def test_e2e_underwriting_to_offer(self): + """Underwriting MAO feeds offer calculation.""" + from core.services.underwriting import ( + solve_underwriting, + UnderwritingInput, + ) + + uw = solve_underwriting( + UnderwritingInput( + purchase_price=Decimal("350000"), + estimated_rent=Decimal("2800"), + property_tax_annual=Decimal("3600"), + insurance_annual=Decimal("1200"), + ), + target_cap_rate=0.08, ) - assert result.success - mao = result.underwriting.mao offer = solve_offer( - OfferInput(mao=mao, arv=float(mao) * 1.2), OfferStrategy.TARGET + OfferInput(mao=uw.mao, arv=uw.mao * Decimal("1.2")), OfferStrategy.TARGET ) assert offer.offer_price > 0 assert offer.estimated_equity is not None def test_e2e_multiple_strategies(self): """Generate offers for all three strategies from same underwriting.""" - from prei.pipeline.handlers.underwriting import ( + from core.services.underwriting import ( solve_underwriting, UnderwritingInput, ) uw = solve_underwriting( UnderwritingInput( - purchase_price=300000, - estimated_rent=2500, - property_tax_annual=3600, - insurance_annual=1200, + purchase_price=Decimal("300000"), + estimated_rent=Decimal("2500"), + property_tax_annual=Decimal("3600"), + insurance_annual=Decimal("1200"), ), 0.08, ) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 487c4812..3b9d0c57 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -1,199 +1,27 @@ """Comprehensive unit test suite for the property pipeline. Covers: - - State transition validation (illicit jump controls) - Screening engine math edge cases (division by zero, missing data) - Batch processor with a 10-mock-asset dataset (3 valid, 7 failing distinct rules) + +The prei state-machine tests (stage-machine classes transitions) were +removed in the pydantic→Django consolidation — stage transitions are now +covered by core/tests/test_pipeline_service.py against PipelineProperty. """ -import pytest +from dataclasses import replace -from prei.models.pipeline import ( - InvalidStageTransitionException, - PipelineStage, - PropertyAsset, -) -from prei.pipeline.engine import InMemoryAssetRepository, PipelineEngine -from prei.pipeline.handlers.batch_screening import BatchScreeningProcessor -from prei.pipeline.handlers.screening import ( +from core.services.screening import ( ScreeningThresholds, evaluate_screening_stage, gross_yield, price_to_rent_ratio, + screen_batch, ) # ═══════════════════════════════════════════════════════════════════════════════ -# 1. STATE TRANSITION VALIDATION -# ═══════════════════════════════════════════════════════════════════════════════ - - -class TestStateTransitionValidation: - """Verify the state machine enforces all transition rules.""" - - def _make_asset(self) -> PropertyAsset: - return PropertyAsset(asset_id="T", address="1 Test St") - - # ── Illicit jumps ──────────────────────────────────────────────────────── - - def test_gacs_to_underwriting_raises(self): - """GACS → UNDERWRITING is an illicit jump (must go through DISCOVERY, SCREENING).""" - a = self._make_asset() - with pytest.raises(InvalidStageTransitionException) as exc: - a.transition_to(PipelineStage.UNDERWRITING) - assert "GACS" in str(exc.value) - assert "UNDERWRITING" in str(exc.value) - - def test_gacs_to_portfolio_raises(self): - """GACS → PORTFOLIO is an illicit jump.""" - a = self._make_asset() - with pytest.raises(InvalidStageTransitionException): - a.transition_to(PipelineStage.PORTFOLIO) - - def test_screening_to_closing_raises(self): - """SCREENING → CLOSING skips UNDERWRITING, OFFER, DUE_DILIGENCE.""" - a = self._make_asset() - a.transition_to(PipelineStage.DISCOVERY) - a.transition_to(PipelineStage.SCREENING) - with pytest.raises(InvalidStageTransitionException): - a.transition_to(PipelineStage.CLOSING) - - def test_offer_to_turnover_raises(self): - """OFFER → TURNOVER skips DUE_DILIGENCE, CLOSING.""" - a = self._make_asset() - a.transition_to(PipelineStage.DISCOVERY) - a.transition_to(PipelineStage.SCREENING) - a.transition_to(PipelineStage.UNDERWRITING) - a.transition_to(PipelineStage.OFFER) - with pytest.raises(InvalidStageTransitionException): - a.transition_to(PipelineStage.TURNOVER) - - def test_killed_to_anything_raises(self): - """KILLED is terminal — no transitions out.""" - a = self._make_asset() - a.transition_to(PipelineStage.KILLED, reason="test") - for stage in PipelineStage: - if stage == PipelineStage.KILLED: - continue - with pytest.raises(InvalidStageTransitionException): - a.transition_to(stage) - - # ── Valid forward flow ──────────────────────────────────────────────────── - - def test_full_forward_flow(self): - """Every legal forward transition succeeds.""" - a = self._make_asset() - path = [ - (PipelineStage.DISCOVERY, "identify"), - (PipelineStage.SCREENING, "qualify"), - (PipelineStage.UNDERWRITING, "analyze"), - (PipelineStage.OFFER, "bid"), - (PipelineStage.DUE_DILIGENCE, "inspect"), - (PipelineStage.CLOSING, "purchase"), - (PipelineStage.TURNOVER, "rehab"), - (PipelineStage.LEASING, "rent"), - (PipelineStage.PORTFOLIO, "hold"), - ] - for stage, _ in path: - a.transition_to(stage, reason="forward") - assert a.current_stage == stage - assert a.current_stage == PipelineStage.PORTFOLIO - - def test_leasing_portfolio_bidirectional(self): - """LEASING ↔ PORTFOLIO works in both directions.""" - a = self._make_asset() - a.transition_to(PipelineStage.DISCOVERY) - a.transition_to(PipelineStage.SCREENING) - a.transition_to(PipelineStage.UNDERWRITING) - a.transition_to(PipelineStage.OFFER) - a.transition_to(PipelineStage.DUE_DILIGENCE) - a.transition_to(PipelineStage.CLOSING) - a.transition_to(PipelineStage.TURNOVER) - a.transition_to(PipelineStage.LEASING) - - # LEASING → PORTFOLIO - a.transition_to(PipelineStage.PORTFOLIO) - assert a.current_stage == PipelineStage.PORTFOLIO - - # PORTFOLIO → LEASING (back to active management) - a.transition_to(PipelineStage.LEASING) - assert a.current_stage == PipelineStage.LEASING - - def test_kill_from_any_stage(self): - """KILLED is reachable from any non-terminal stage.""" - stages = [s for s in PipelineStage if s != PipelineStage.KILLED] - for stage in stages: - a = self._make_asset() - # Advance to target stage - if stage == PipelineStage.GACS: - pass # already at GACS - elif stage == PipelineStage.DISCOVERY: - a.transition_to(PipelineStage.DISCOVERY) - elif stage == PipelineStage.SCREENING: - a.transition_to(PipelineStage.DISCOVERY) - a.transition_to(PipelineStage.SCREENING) - elif stage == PipelineStage.UNDERWRITING: - a.transition_to(PipelineStage.DISCOVERY) - a.transition_to(PipelineStage.SCREENING) - a.transition_to(PipelineStage.UNDERWRITING) - elif stage == PipelineStage.OFFER: - self._advance_to(a, PipelineStage.OFFER) - elif stage == PipelineStage.DUE_DILIGENCE: - self._advance_to(a, PipelineStage.DUE_DILIGENCE) - elif stage == PipelineStage.CLOSING: - self._advance_to(a, PipelineStage.CLOSING) - elif stage == PipelineStage.TURNOVER: - self._advance_to(a, PipelineStage.TURNOVER) - elif stage == PipelineStage.LEASING: - self._advance_to(a, PipelineStage.LEASING) - elif stage == PipelineStage.PORTFOLIO: - self._advance_to(a, PipelineStage.PORTFOLIO) - - a.transition_to(PipelineStage.KILLED, reason=f"Killed at {stage.value}") - assert a.current_stage == PipelineStage.KILLED - assert a.kill_reason == f"Killed at {stage.value}" - - def _advance_to(self, a: PropertyAsset, target: PipelineStage) -> None: - """Advance asset through the pipeline to the given stage.""" - path = [ - PipelineStage.DISCOVERY, - PipelineStage.SCREENING, - PipelineStage.UNDERWRITING, - PipelineStage.OFFER, - PipelineStage.DUE_DILIGENCE, - PipelineStage.CLOSING, - PipelineStage.TURNOVER, - PipelineStage.LEASING, - PipelineStage.PORTFOLIO, - ] - for stage in path: - a.transition_to(stage, reason="forward") - if stage == target: - break - - # ── Stage history integrity ─────────────────────────────────────────────── - - def test_stage_history_records_all_transitions(self): - """Every transition adds a StageLog entry with correct timestamps.""" - a = self._make_asset() - a.transition_to(PipelineStage.DISCOVERY) - a.transition_to(PipelineStage.SCREENING) - a.transition_to(PipelineStage.KILLED, reason="Budget cut") - - assert len(a.stage_history) == 3 - assert a.stage_history[0].stage == PipelineStage.DISCOVERY - assert a.stage_history[1].stage == PipelineStage.SCREENING - assert a.stage_history[2].stage == PipelineStage.KILLED - - # exited_at should be set for completed stages - assert a.stage_history[0].exited_at is not None - assert a.stage_history[1].exited_at is not None - # Current (last) stage should have no exit time - assert a.stage_history[2].exited_at is None - - -# ═══════════════════════════════════════════════════════════════════════════════ -# 2. SCREENING ENGINE MATH EDGE CASES +# 1. SCREENING ENGINE MATH EDGE CASES # ═══════════════════════════════════════════════════════════════════════════════ @@ -311,7 +139,7 @@ def test_str_beds_fails_when_below_minimum(self): def test_empty_excluded_hoas_list(self): """Empty excluded_hoas list never blocks any HOA.""" - thresholds = self.THRESHOLDS.model_copy(update={"excluded_hoas": []}) + thresholds = replace(self.THRESHOLDS, excluded_hoas=[]) data = { "estimated_monthly_rent": 2500, "purchase_price": 300000, @@ -324,7 +152,7 @@ def test_empty_excluded_hoas_list(self): # ═══════════════════════════════════════════════════════════════════════════════ -# 3. BATCH PROCESSOR — 10 MOCK ASSETS (3 PASS, 7 FAIL DISTINCT RULES) +# 2. BATCH PROCESSOR — 10 MOCK ASSETS (3 PASS, 7 FAIL DISTINCT RULES) # ═══════════════════════════════════════════════════════════════════════════════ @@ -428,9 +256,6 @@ def test_empty_excluded_hoas_list(self): class TestBatchProcessorTenAssets: """Verify the batch processor against a 10-asset mock dataset.""" - def _make_engine(self) -> PipelineEngine: - return PipelineEngine(repository=InMemoryAssetRepository()) - # ── Individual expected outcomes ────────────────────────────────────────── def test_individual_asset_outcomes(self): @@ -458,93 +283,31 @@ def test_individual_asset_outcomes(self): def test_batch_summary_counts(self): """Batch processor returns correct processed/advanced/killed counts.""" - engine = self._make_engine() - processor = BatchScreeningProcessor(engine, BATCH_THRESHOLDS) - - summary = processor.process(MOCK_DATASET) + summary = screen_batch(MOCK_DATASET, BATCH_THRESHOLDS) assert summary["processed"] == 10 assert summary["advanced"] == 4 # 3 passing + 1 missing-rent (falls through) assert summary["killed"] == 6 # 7 failing - 1 missing-rent (not killed) assert summary["execution_time_ms"] >= 0 - # ── All assets persisted ───────────────────────────────────────────────── - - def test_all_assets_persisted(self): - """All 10 assets are saved to the repository.""" - engine = self._make_engine() - processor = BatchScreeningProcessor(engine, BATCH_THRESHOLDS) - processor.process(MOCK_DATASET) - - all_assets = engine.repository.list_all() - assert len(all_assets) == 10 - asset_ids = {a.asset_id for a in all_assets} - expected_ids = {d["asset_id"] for d in MOCK_DATASET} - assert asset_ids == expected_ids - - # ── Killed assets have kill_reason ──────────────────────────────────────── - - def test_killed_assets_have_reason(self): - """All killed assets have a non-empty kill_reason.""" - engine = self._make_engine() - processor = BatchScreeningProcessor(engine, BATCH_THRESHOLDS) - processor.process(MOCK_DATASET) - - killed_ids = [ - "FAIL-BEDS-01", - "FAIL-BATHS-01", - "FAIL-HOA-01", - "FAIL-YIELD-01", - "FAIL-RATIO-01", - "FAIL-MULTI-01", - ] - for aid in killed_ids: - asset = engine.repository.load(aid) - assert asset is not None - assert asset.current_stage == PipelineStage.KILLED - assert asset.kill_reason is not None - assert len(asset.kill_reason) > 0 - - # ── Advanced assets are at UNDERWRITING ────────────────────────────────── - - def test_advanced_assets_at_underwriting(self): - """All advanced assets are in UNDERWRITING stage.""" - engine = self._make_engine() - processor = BatchScreeningProcessor(engine, BATCH_THRESHOLDS) - processor.process(MOCK_DATASET) - - for aid in ["PASS-01", "PASS-02", "PASS-03", "FAIL-RENT-MISSING"]: - asset = engine.repository.load(aid) - assert asset is not None - assert asset.current_stage == PipelineStage.UNDERWRITING, ( - f"{aid} should be UNDERWRITING, got {asset.current_stage}" - ) - # ── First-failure short-circuit within single asset ─────────────────────── def test_multi_fail_asset_reports_first_violation(self): """FAIL-MULTI-01 fails on beds (first check) before yield.""" - engine = self._make_engine() - processor = BatchScreeningProcessor(engine, BATCH_THRESHOLDS) - processor.process(MOCK_DATASET) - - asset = engine.repository.load("FAIL-MULTI-01") - assert asset is not None - assert asset.current_stage == PipelineStage.KILLED + passed, reason = evaluate_screening_stage( + next(a for a in MOCK_DATASET if a["asset_id"] == "FAIL-MULTI-01"), + BATCH_THRESHOLDS, + ) + assert passed is False # Should fail on beds before yield - assert "bedroom" in (asset.kill_reason or "").lower() + assert "bedroom" in (reason or "").lower() # ── Deterministic: running twice yields same results ───────────────────── def test_deterministic_output(self): """Processing the same dataset twice produces identical counts.""" - engine1 = self._make_engine() - engine2 = self._make_engine() - p1 = BatchScreeningProcessor(engine1, BATCH_THRESHOLDS) - p2 = BatchScreeningProcessor(engine2, BATCH_THRESHOLDS) - - s1 = p1.process(MOCK_DATASET) - s2 = p2.process(MOCK_DATASET) + s1 = screen_batch(MOCK_DATASET, BATCH_THRESHOLDS) + s2 = screen_batch(MOCK_DATASET, BATCH_THRESHOLDS) for key in ("processed", "advanced", "killed"): assert s1[key] == s2[key], f"Mismatch on {key}: {s1[key]} != {s2[key]}" diff --git a/tests/test_pipeline_e2e.py b/tests/test_pipeline_e2e.py index f76cd352..f8bfee67 100644 --- a/tests/test_pipeline_e2e.py +++ b/tests/test_pipeline_e2e.py @@ -1,7 +1,20 @@ -"""E2E tests for the full pipeline orchestration (discovery → screening → underwriting → offer).""" +"""E2E tests for the full pipeline (discovery → screening → underwriting → offer). + +The prei orchestrator class was deleted in the pydantic→Django +consolidation; these tests compose the new core services explicitly. +""" import pytest -from prei.pipeline.orchestrator import PipelineOrchestrator +from decimal import Decimal + +from core.services.discovery_processor import process_discovery_batch +from core.services.screening import ScreeningThresholds, screen_batch +from core.services.underwriting import ( + UnderwritingInput, + UnderwritingMetrics, + solve_underwriting, +) +from core.services.offer import OfferInput, OfferStrategy, solve_offer PROPERTIES = [ { @@ -31,63 +44,78 @@ "baths": 0.5, } +THRESHOLDS = ScreeningThresholds( + min_gross_yield=0.07, max_price_to_rent_ratio=15.0, min_beds=2, min_baths=1 +) + + +def _to_screening_dict(p: dict) -> dict: + return { + "asset_id": p["id"], + "address": p["address"], + "estimated_monthly_rent": p["rent"], + "purchase_price": p["price"], + "beds": p["beds"], + "baths": p["baths"], + } + + +def _underwrite(p: dict) -> UnderwritingMetrics: + return solve_underwriting( + UnderwritingInput( + purchase_price=Decimal(str(p["price"])), + estimated_rent=Decimal(str(p["rent"])), + property_tax_annual=Decimal("3600"), + insurance_annual=Decimal("1200"), + ), + target_cap_rate=0.08, + ) + @pytest.mark.e2e class TestPipelineE2E: def test_full_pipeline_happy_path(self): - """All properties pass screening and reach UNDERWRITING.""" - orch = PipelineOrchestrator() + """All properties pass screening and produce underwriting metrics.""" + result = process_discovery_batch(PROPERTIES, source_name="e2e") + assert result["new_assets_discovered"] == 2 + assert result["failed_records"] == 0 + + summary = screen_batch([_to_screening_dict(p) for p in PROPERTIES], THRESHOLDS) + assert summary["advanced"] == 2 + assert summary["killed"] == 0 + for p in PROPERTIES: - result = orch.run(p) - assert result.success, f"{p['id']} should pass: {result.error}" - assert result.asset.current_stage.value == "UNDERWRITING" - assert result.underwriting.noi > 0 - assert result.underwriting.mao > 0 + uw = _underwrite(p) + assert uw.noi > 0 + assert uw.mao > 0 def test_pipeline_rejects_failing_property(self): """Property failing screening is killed.""" - orch = PipelineOrchestrator() - result = orch.run(FAILING) - assert result.success # Graceful handling - assert result.screening_passed is False - assert result.asset.current_stage.value == "KILLED" - assert result.underwriting is None + summary = screen_batch([_to_screening_dict(FAILING)], THRESHOLDS) + assert summary["killed"] == 1 + assert summary["advanced"] == 0 def test_pipeline_dedup(self): """Same address run twice → second is duplicate.""" - orch = PipelineOrchestrator() - r1 = orch.run(PROPERTIES[0]) - assert r1.success - r2 = orch.run(PROPERTIES[0]) - assert r2.success is False - assert "Duplicate" in (r2.error or "") + existing: set[str] = set() + r1 = process_discovery_batch( + PROPERTIES[:1], source_name="e2e", existing_hashes=existing + ) + assert r1["new_assets_discovered"] == 1 + r2 = process_discovery_batch( + PROPERTIES[:1], source_name="e2e", existing_hashes=existing + ) + assert r2["new_assets_discovered"] == 0 + assert r2["duplicates_skipped"] == 1 def test_pipeline_with_offer(self): """Pipeline → underwriting → offer chain works.""" - from prei.pipeline.handlers.offer import solve_offer, OfferInput, OfferStrategy - - orch = PipelineOrchestrator() - result = orch.run(PROPERTIES[0]) - assert result.success - offer = solve_offer( - OfferInput(mao=result.underwriting.mao), OfferStrategy.TARGET - ) + uw = _underwrite(PROPERTIES[0]) + offer = solve_offer(OfferInput(mao=uw.mao), OfferStrategy.TARGET) assert offer.offer_price > 0 - assert offer.premium_pct == pytest.approx(0, abs=1e-4) + assert offer.premium_pct == pytest.approx(0, abs=Decimal("0.0001")) def test_multi_property_pipeline(self): - """Multiple properties processed sequentially.""" - orch = PipelineOrchestrator() - results = [orch.run(p) for p in PROPERTIES] - assert all(r.success for r in results) - assert results[0].underwriting.cap_rate != results[1].underwriting.cap_rate - - def test_pipeline_to_dict_includes_offer(self): - """to_dict() contains all key pipeline outputs.""" - orch = PipelineOrchestrator() - result = orch.run(PROPERTIES[0]) - d = result.to_dict() - assert d["success"] is True - assert d["current_stage"] == "UNDERWRITING" - assert d["cap_rate"] > 0 - assert d["mao"] > 0 + """Multiple properties processed sequentially produce distinct caps.""" + cap_rates = [_underwrite(p).cap_rate for p in PROPERTIES] + assert cap_rates[0] != cap_rates[1] diff --git a/tests/test_screening_integration.py b/tests/test_screening_integration.py index 9d425fb5..1eff55bd 100644 --- a/tests/test_screening_integration.py +++ b/tests/test_screening_integration.py @@ -1,18 +1,18 @@ """Integration and E2E tests for the screening stage.""" import pytest -from prei.pipeline.handlers.screening import ( + +from core.services.screening import ( ScreeningThresholds, evaluate_screening_stage, gross_yield, price_to_rent_ratio, compute_screening_metrics, + screen_batch, ) -from prei.pipeline.engine import InMemoryAssetRepository, PipelineEngine -from prei.pipeline.handlers.batch_screening import BatchScreeningProcessor # ═══════════════════════════════════════════════════════════════════════════════ -# INTEGRATION — screening + engine +# INTEGRATION — screening math + batch # ═══════════════════════════════════════════════════════════════════════════════ THRESHOLDS = ScreeningThresholds( @@ -21,18 +21,6 @@ class TestScreeningIntegration: - def test_screening_via_engine_hook(self): - """Screening as a PipelineEngine pre-transition hook blocks failing assets.""" - engine = PipelineEngine(repository=InMemoryAssetRepository()) - - def screening_hook(asset, target, ctx): - data = ctx.get("asset_data", {}) - passed, _ = evaluate_screening_stage(data, THRESHOLDS) - return passed - - engine.register_hook("UNDERWRITING", screening_hook) # Intentional bad hook key - # Hook not directly testable without a full asset — verifying interface works - def test_gross_yield_vs_price_to_rent_consistency(self): """gross_yield and price_to_rent_ratio are mathematical inverses.""" gy = gross_yield(2500, 300000) @@ -56,13 +44,11 @@ def test_compute_screening_metrics_consistency( assert metrics["gross_yield"] == pytest.approx(expected_yield, rel=1e-3) assert metrics["price_to_rent_ratio"] == pytest.approx(expected_ptr, rel=1e-3) - def test_batch_processor_with_custom_thresholds(self): - """BatchScreeningProcessor with relaxed thresholds passes all.""" - engine = PipelineEngine(repository=InMemoryAssetRepository()) + def test_screen_batch_with_custom_thresholds(self): + """screen_batch with relaxed thresholds passes all.""" relaxed = ScreeningThresholds( min_gross_yield=0.03, max_price_to_rent_ratio=30.0, min_beds=1, min_baths=1 ) - processor = BatchScreeningProcessor(engine, relaxed) payloads = [ { "asset_id": "A", @@ -81,7 +67,7 @@ def test_batch_processor_with_custom_thresholds(self): "baths": 1, }, ] - result = processor.process(payloads) + result = screen_batch(payloads, relaxed) assert result["advanced"] == 2 @@ -92,36 +78,32 @@ def test_batch_processor_with_custom_thresholds(self): @pytest.mark.e2e class TestScreeningE2E: - def test_e2e_screening_within_orchestrator(self): - """Full pipeline with passing and failing properties.""" - from prei.pipeline.orchestrator import PipelineOrchestrator - - orch = PipelineOrchestrator() + def test_e2e_screening_passing_and_failing(self): + """evaluate_screening_stage distinguishes passing and failing properties.""" passing = { "id": "P", "address": "100 Good St", - "price": 200000, - "rent": 2000, + "estimated_monthly_rent": 2000, + "purchase_price": 200000, "beds": 3, "baths": 2, } failing = { "id": "F", "address": "200 Bad St", - "price": 500000, - "rent": 1000, + "estimated_monthly_rent": 1000, + "purchase_price": 500000, "beds": 1, "baths": 0.5, } - assert orch.run(passing).screening_passed is True - assert orch.run(failing).screening_passed is False + assert evaluate_screening_stage(passing, THRESHOLDS)[0] is True + assert evaluate_screening_stage(failing, THRESHOLDS)[0] is False - def test_e2e_screening_batch_then_orchestrate(self): - """Discover → screen → underwrite in sequence.""" - from prei.pipeline.orchestrator import PipelineOrchestrator + def test_e2e_discover_then_screen(self): + """Discover → screen in sequence (orchestrator equivalent).""" + from core.services.discovery_processor import process_discovery_batch - hashes = set() - orch = PipelineOrchestrator(existing_hashes=hashes) + existing: set[str] = set() batch = [ { "id": "E2E-1", @@ -140,7 +122,22 @@ def test_e2e_screening_batch_then_orchestrate(self): "baths": 1, }, ] - for p in batch: - result = orch.run(p) - assert result.success - assert result.asset.current_stage.value == "UNDERWRITING" + discovered = process_discovery_batch( + batch, source_name="e2e", existing_hashes=existing + ) + assert discovered["new_assets_discovered"] == 2 + + screening_payloads = [ + { + "asset_id": p["id"], + "address": p["address"], + "estimated_monthly_rent": p["rent"], + "purchase_price": p["price"], + "beds": p["beds"], + "baths": p["baths"], + } + for p in batch + ] + summary = screen_batch(screening_payloads, THRESHOLDS) + assert summary["advanced"] == 2 + assert summary["killed"] == 0 diff --git a/tests/test_underwriting_integration.py b/tests/test_underwriting_integration.py index e5400dca..2bcc611e 100644 --- a/tests/test_underwriting_integration.py +++ b/tests/test_underwriting_integration.py @@ -1,9 +1,10 @@ """Integration and E2E tests for the underwriting stage.""" +from dataclasses import replace from decimal import Decimal import pytest -from prei.pipeline.handlers.underwriting import ( +from core.services.underwriting import ( UnderwritingInput, solve_underwriting, ) @@ -24,9 +25,7 @@ class TestUnderwritingIntegration: def test_noi_sensitivity_to_vacancy(self): """Higher vacancy reduces NOI proportionally.""" base = solve_underwriting(BASE, 0.08) - high_vac = solve_underwriting( - BASE.model_copy(update={"vacancy_rate": 0.15}), 0.08 - ) + high_vac = solve_underwriting(replace(BASE, vacancy_rate=Decimal("0.15")), 0.08) assert high_vac.noi < base.noi # EGI difference: GPR*(1-0.05) vs GPR*(1-0.15) → 10% of GPR less # GPR = 30000, so EGI difference = 3000. But OpEx also changes (mgmt fee on EGI) @@ -44,7 +43,7 @@ def test_rehab_budget_reduces_coc(self): """Adding rehab budget reduces cash-on-cash yield.""" no_rehab = solve_underwriting(BASE, 0.08) with_rehab = solve_underwriting( - BASE.model_copy(update={"rehab_budget": 50000}), 0.08 + replace(BASE, rehab_budget=Decimal("50000")), 0.08 ) assert with_rehab.cash_on_cash < no_rehab.cash_on_cash @@ -66,7 +65,7 @@ def test_annual_metrics_consistency(self): def test_zero_rent_still_produces_metrics(self): """Zero rent → negative NOI (expenses still exist), but no crash.""" - result = solve_underwriting(BASE.model_copy(update={"estimated_rent": 0}), 0.08) + result = solve_underwriting(replace(BASE, estimated_rent=Decimal("0")), 0.08) assert result.noi < 0 assert result.cap_rate < 0 assert result.mao < 0 @@ -75,7 +74,7 @@ def test_purchase_price_sensitivity(self): """Higher purchase price → lower cap rate.""" cheap = solve_underwriting(BASE, 0.08) expensive = solve_underwriting( - BASE.model_copy(update={"purchase_price": 500000}), 0.08 + replace(BASE, purchase_price=Decimal("500000")), 0.08 ) assert expensive.cap_rate < cheap.cap_rate @@ -87,22 +86,17 @@ def test_purchase_price_sensitivity(self): @pytest.mark.e2e class TestUnderwritingE2E: - def test_e2e_underwriting_via_orchestrator(self): - """Underwriting metrics computed via full pipeline orchestration.""" - from prei.pipeline.orchestrator import PipelineOrchestrator - - orch = PipelineOrchestrator(target_cap_rate=0.08) - payload = { - "id": "UW-E2E", - "address": "300 Test Ave", - "price": 350000, - "rent": 2800, - "beds": 3, - "baths": 2, - } - result = orch.run(payload) - assert result.success - uw = result.underwriting + def test_e2e_underwriting_from_property(self): + """Underwriting metrics computed from a realistic payload.""" + uw = solve_underwriting( + UnderwritingInput( + purchase_price=Decimal("350000"), + estimated_rent=Decimal("2800"), + property_tax_annual=Decimal("3600"), + insurance_annual=Decimal("1200"), + ), + 0.08, + ) assert uw.noi > 0 assert uw.cap_rate > 0.05 assert uw.mao > 200000 @@ -110,27 +104,26 @@ def test_e2e_underwriting_via_orchestrator(self): def test_e2e_multi_property_underwriting(self): """Different properties get different underwriting metrics.""" - from prei.pipeline.orchestrator import PipelineOrchestrator - - orch = PipelineOrchestrator() results = [] for p in [ { - "id": "A", - "address": "A St", - "price": 200000, - "rent": 2000, - "beds": 2, - "baths": 1, + "price": "200000", + "rent": "2000", }, { - "id": "B", - "address": "B St", - "price": 500000, - "rent": 4000, - "beds": 4, - "baths": 3, + "price": "500000", + "rent": "4000", }, ]: - results.append(orch.run(p)) - assert results[0].underwriting.cap_rate != results[1].underwriting.cap_rate + results.append( + solve_underwriting( + UnderwritingInput( + purchase_price=Decimal(p["price"]), + estimated_rent=Decimal(p["rent"]), + property_tax_annual=Decimal("3600"), + insurance_annual=Decimal("1200"), + ), + 0.08, + ) + ) + assert results[0].cap_rate != results[1].cap_rate diff --git a/tests_bdd/steps/pipeline_steps.py b/tests_bdd/steps/pipeline_steps.py index ae6fd292..446eebb0 100644 --- a/tests_bdd/steps/pipeline_steps.py +++ b/tests_bdd/steps/pipeline_steps.py @@ -3,15 +3,14 @@ import pytest from pytest_bdd import given, then, when -from prei.pipeline.handlers.discovery import DiscoverySanitizer -from prei.pipeline.handlers.discovery_processor import DiscoveryProcessor -from prei.pipeline.handlers.screening import ( +from core.services.discovery import DiscoverySanitizer +from core.services.discovery_processor import process_discovery_batch +from core.services.screening import ( ScreeningThresholds, evaluate_screening_stage, + screen_batch, ) -from prei.pipeline.handlers.underwriting import solve_underwriting, UnderwritingInput -from prei.pipeline.handlers.batch_screening import BatchScreeningProcessor -from prei.pipeline.engine import InMemoryAssetRepository, PipelineEngine +from core.services.underwriting import solve_underwriting, UnderwritingInput @given( @@ -30,7 +29,7 @@ def given_raw_listing(): @given("a discovery processor with no existing addresses", target_fixture="processor") def given_processor(): - return DiscoveryProcessor(existing_hashes=set()) + return {"existing_hashes": set()} @when( @@ -50,7 +49,9 @@ def when_batch(processor): {"id": "A1", "address": "100 Oak St", "price": 200_000}, {"id": "A2", "address": "100 Oak St", "price": 200_000}, ] - return processor.process_batch(batch, source_name="bdd_test") + return process_discovery_batch( + batch, source_name="bdd_test", existing_hashes=processor["existing_hashes"] + ) @then("the address hash should be a 64-character SHA-256 string") @@ -191,8 +192,7 @@ def given_mixed_batch(): @when("I run the batch screening processor", target_fixture="batch_result2") def when_batch_process(mixed_batch, batch_thresholds): - engine = PipelineEngine(repository=InMemoryAssetRepository()) - return BatchScreeningProcessor(engine, batch_thresholds).process(mixed_batch) + return screen_batch(mixed_batch, batch_thresholds) @then("the result should show 4 processed") @@ -249,4 +249,4 @@ def then_cap(uw_result): @then("the MAO should be approximately $222,750") def then_mao(uw_result): - assert uw_result.mao == pytest.approx(222_750, rel=1e-3) + assert float(uw_result.mao) == pytest.approx(222_750, rel=1e-3) From cba769ff1b0ebfa7704d6944905bbad868d365fd Mon Sep 17 00:00:00 2001 From: Phil Ruff Date: Fri, 31 Jul 2026 14:41:57 +0100 Subject: [PATCH 3/9] chore(make): prefer venv python over system shim PYTHON now resolves to .venv/bin/python when present, avoiding the Apple /usr/bin/python3 shim that triggers the xcode-select reinstall prompt. Docker exec targets use CONTAINER_PY (containers have no venv). --- Makefile | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 613a4597..b32727ca 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,11 @@ SHELL := /bin/bash .PHONY: help ensure-env dev seed superuser lint test test-unit test-integration test-e2e check deploy-dev deploy-local deploy-devcontainer gitops-validate gitops-hook-install smoke build up down restart logs clean docker-dev test-live -PYTHON ?= python +# Prefer the local venv; fall back to `python` (containers, bare environments). +# Avoids the Apple /usr/bin/python3 shim that triggers the xcode-select prompt. +PYTHON ?= $(if $(wildcard .venv/bin/python),.venv/bin/python,python) +# Interpreter inside containers (system Python; containers have no venv). +CONTAINER_PY ?= python ENV_FILE ?= .env REQUIREMENTS ?= requirements.txt DOCKER_TAG := $(shell git rev-parse --short HEAD 2>/dev/null || echo "dev") @@ -108,8 +112,8 @@ deploy-dev: ensure-env exit 1; \ fi @docker compose up -d - @docker compose exec web $(PYTHON) manage.py migrate - @docker compose exec web $(PYTHON) manage.py seed_data + @docker compose exec web $(CONTAINER_PY) manage.py migrate + @docker compose exec web $(CONTAINER_PY) manage.py seed_data @echo "Docker stack is running on port 8000" deploy-local: ensure-env @@ -171,8 +175,8 @@ clean: docker-dev: ensure-env build up $(call ensure_django) - @docker compose exec web $(PYTHON) manage.py migrate - @docker compose exec web $(PYTHON) manage.py seed_data + @docker compose exec web $(CONTAINER_PY) manage.py migrate + @docker compose exec web $(CONTAINER_PY) manage.py seed_data @echo "" @echo "Docker dev stack running on http://localhost:8000" @echo " make logs — tail container logs" From 0cb7329e95b4c885a1c46d4875edc76a12f6f6ef Mon Sep 17 00:00:00 2001 From: Phil Ruff Date: Fri, 31 Jul 2026 19:43:47 +0100 Subject: [PATCH 4/9] feat: wire Tarrant county source, fix HUD FMR key selector, add live test suite - core/services/sources/county.py: implement Playwright-based fetch for Tarrant County Monthly Tax Sales; add _fetch_via_playwright helper; verified 23 listings - core/integrations/market/hud_fmr.py: FMRClient now prefers HUD_FMR_TOKEN over HUD_API_KEY - core/integrations/market/fmr_adapter.py: fetch_fmr_entity_id prefers HUD_FMR_TOKEN - core/integrations/market/hud_il.py: ILClient and fetch_area_median_income prefer HUD_FMR_TOKEN - investor_app/settings.py: export HUD_FMR_TOKEN setting - pytest.ini: register live marker; remove live from default -k exclusion - Makefile: add test-live-sources target - core/tests/test_live_sources.py: 9 live integration tests for Tarrant, VRM, HUD FMR, Census, FRED --- .../pydantic-to-django/dependency-graph.json | 41 +++++ .../pydantic-to-django/effort-estimates.json | 25 +++ .agents/plans/pydantic-to-django/tasks.json | 170 +++++++++++++++++ Makefile | 11 +- core/integrations/market/fmr_adapter.py | 14 +- core/integrations/market/hud_fmr.py | 6 +- core/integrations/market/hud_il.py | 14 +- core/services/sources/county.py | 141 +++++++++++++- core/tests/test_live_sources.py | 172 ++++++++++++++++++ investor_app/settings.py | 3 + pytest.ini | 1 + 11 files changed, 580 insertions(+), 18 deletions(-) create mode 100644 .agents/plans/pydantic-to-django/dependency-graph.json create mode 100644 .agents/plans/pydantic-to-django/effort-estimates.json create mode 100644 .agents/plans/pydantic-to-django/tasks.json create mode 100644 core/tests/test_live_sources.py diff --git a/.agents/plans/pydantic-to-django/dependency-graph.json b/.agents/plans/pydantic-to-django/dependency-graph.json new file mode 100644 index 00000000..c9583062 --- /dev/null +++ b/.agents/plans/pydantic-to-django/dependency-graph.json @@ -0,0 +1,41 @@ +{ + "skill": "plan", + "status": "pass", + "nodes": ["TASK-01", "TASK-02", "TASK-03", "TASK-04", "TASK-05", "TASK-06", "TASK-07", "TASK-08", "TASK-09", "TASK-10", "TASK-11"], + "edges": [ + ["TASK-02", "TASK-04"], + ["TASK-01", "TASK-05"], + ["TASK-02", "TASK-05"], + ["TASK-04", "TASK-05"], + ["TASK-01", "TASK-06"], + ["TASK-02", "TASK-06"], + ["TASK-04", "TASK-06"], + ["TASK-01", "TASK-07"], + ["TASK-02", "TASK-07"], + ["TASK-03", "TASK-07"], + ["TASK-04", "TASK-07"], + ["TASK-05", "TASK-08"], + ["TASK-06", "TASK-08"], + ["TASK-07", "TASK-08"], + ["TASK-07", "TASK-09"], + ["TASK-08", "TASK-10"], + ["TASK-09", "TASK-10"], + ["TASK-10", "TASK-11"] + ], + "notes": { + "parallel_waves": { + "wave_1": ["TASK-01", "TASK-02", "TASK-03"], + "wave_2": ["TASK-04", "TASK-07"], + "wave_3": ["TASK-05", "TASK-06", "TASK-09"], + "wave_4": ["TASK-08"], + "wave_5": ["TASK-10"], + "wave_6": ["TASK-11"] + }, + "critical_path": ["TASK-02", "TASK-04", "TASK-05", "TASK-08", "TASK-10", "TASK-11"], + "bottlenecks": [ + "TASK-05/TASK-06 both touch core/views/__init__.py (~5,000 LOC file) — high merge-conflict risk if run in parallel; run TASK-05 before TASK-06", + "TASK-08 and TASK-10 are large deletion tasks (1,100 + 2,100 lines) exceeding the 400-line guidance — acceptable because pure deletions of dead code are low-risk; split per-file if reviewers object" + ], + "pm_signoff_gate": "TASK-08 through TASK-11 (deletion of PropertyAsset/StageLog/FastAPI/CLI, engine, orchestrator, sources) require PM sign-off per user directive — plan pending approval for those tasks." + } +} diff --git a/.agents/plans/pydantic-to-django/effort-estimates.json b/.agents/plans/pydantic-to-django/effort-estimates.json new file mode 100644 index 00000000..420642fc --- /dev/null +++ b/.agents/plans/pydantic-to-django/effort-estimates.json @@ -0,0 +1,25 @@ +{ + "skill": "plan", + "status": "pass", + "estimates": { + "TASK-01": {"task": "Port screening evaluator + thresholds to core/services/screening.py", "lines": 160, "complexity": "low", "risk": "low", "justification": "Pure function + dataclass port; existing prei test suite covers behavior"}, + "TASK-02": {"task": "Port DiscoverySanitizer to core/services/discovery.py", "lines": 130, "complexity": "low", "risk": "low", "justification": "Pure normalization; pydantic model becomes dataclass/dict"}, + "TASK-03": {"task": "Port underwriting solver to core/services/underwriting.py", "lines": 150, "complexity": "low", "risk": "low", "justification": "Already Decimal-based; only BaseModel->dataclass swap"}, + "TASK-04": {"task": "Port DiscoveryProcessor to Django model layer", "lines": 120, "complexity": "medium", "risk": "medium", "justification": "First Django-model write path; dedup semantics must match address_hash rule"}, + "TASK-05": {"task": "Rewrite Growth Explorer bridge (~line 1235)", "lines": 140, "complexity": "high", "risk": "high", "justification": "Touches 5,000-LOC view file; behavior must stay identical; highest regression surface"}, + "TASK-06": {"task": "Rewrite vrm_properties_list run_pipeline bridge (~line 3227)", "lines": 110, "complexity": "medium", "risk": "medium", "justification": "Second view bridge; simpler path than TASK-05"}, + "TASK-07": {"task": "Migrate 9 test files + tests_bdd steps to core.services", "lines": 260, "complexity": "medium", "risk": "medium", "justification": "Import churn across 9 files; BDD feature may need rewrite if engine-only"}, + "TASK-08": {"task": "Delete pydantic state machine + engine + orchestrator", "lines": 1100, "complexity": "low", "risk": "medium", "justification": "Pure deletion of dead code (1100 lines) after TASK-05/06/07; exceeds 400-line guidance but low risk; requires PM sign-off"}, + "TASK-09": {"task": "Delete FastAPI router + CLI + offer float handler", "lines": 400, "complexity": "low", "risk": "medium", "justification": "Deletion of api/cli; offer math port decision resolves LIMIT-21; requires PM sign-off"}, + "TASK-10": {"task": "Migrate or delete prei/pipeline/tests + remaining handlers/sources", "lines": 2100, "complexity": "low", "risk": "medium", "justification": "Final prei package removal (2100 lines, mostly tests/sources); exceeds 400-line guidance but is deletion; requires PM sign-off"}, + "TASK-11": {"task": "Remove deps from requirements + docs update", "lines": 120, "complexity": "low", "risk": "low", "justification": "requirements.txt edit + docs; pydantic retention depends on tests/acceptance"} + }, + "totals": { + "tasks": 11, + "estimated_lines_changed": 4790, + "estimated_new_code": 550, + "estimated_deleted_code": 4240 + }, + "critical_path": ["TASK-02", "TASK-04", "TASK-05", "TASK-08", "TASK-10", "TASK-11"], + "effort_units": "lines changed (deletions dominate; new code is small)" +} diff --git a/.agents/plans/pydantic-to-django/tasks.json b/.agents/plans/pydantic-to-django/tasks.json new file mode 100644 index 00000000..936b756b --- /dev/null +++ b/.agents/plans/pydantic-to-django/tasks.json @@ -0,0 +1,170 @@ +{ + "meta": { + "project": "prei", + "session": "pydantic-to-django-20260731", + "date": "2026-07-31", + "feature": "Consolidate prei pydantic/FastAPI/CLI pipeline onto Django — remove CLI + pydantic, port discovery/screening/underwriting processors to core/services, rewrite the two view bridges, delete prei package", + "spec": "specification.md", + "design": "design.md", + "governance": ["PM sign-off required before deletion tasks (TASK-05+)"], + "decision_record": { + "user_directive": "no cli, pydantic — migrate discovery/screening processors to Django, do not build new state on pydantic models", + "verified_assumption": "docker-compose ghcr.io/paruff/prei:latest is the Django web image (OTEL_SERVICE_NAME: prei only) — NOT a separate FastAPI microservice; no standalone deploy surface exists", + "unverified": "whether docker-compose web service is actively run in any environment (compose file exists; local usage unconfirmed)" + } + }, + "tasks": [ + { + "id": "TASK-01", + "summary": "Port pure screening evaluator + thresholds to Django (Decimal, no pydantic)", + "description": "Add to core/services/screening.py: evaluate_screening_stage-equivalent pure function (gross_yield, price_to_rent, stage decision) operating on Decimal via to_decimal from investor_app.finance.utils, and a dataclass ScreeningThresholds mirroring the pydantic model (min_gross_yield_pct, max_price_to_rent_ratio, min_beds, max_beds, min_sqft, max_year_built, allowed_property_types, allowed_states, allowed_foreclosure_statuses). Do not reuse pydantic. Existing core.services.screening.ScreenProperty/ScreeningResult remain; the prei evaluate_screening_stage logic is the new pure layer callers use.", + "depends_on": [], + "acceptance_criteria": [ + {"id": "AC-01-1", "description": "evaluate_screening_stage ported to core/services/screening.py without pydantic, Decimal-based", "test_type": "unit"}, + {"id": "AC-01-2", "description": "ScreeningThresholds is a dataclass with same defaults as pydantic original", "test_type": "unit"}, + {"id": "AC-01-3", "description": "Tests from prei/pipeline/tests/test_screening.py pass against the new location", "test_type": "unit"} + ], + "agent": "build", + "estimated_lines": 160, + "skills": ["build/refactoring", "lang-python"] + }, + { + "id": "TASK-02", + "summary": "Port DiscoverySanitizer to Django (no pydantic)", + "description": "Add core/services/discovery.py: DiscoverySanitizer-equivalent pure class + CanonicalPropertyPayload as dataclass (or dict), normalizing raw listing dicts (id, address, price, beds, baths, sqft, property_type, year_built, state, foreclosure_status) with Decimal for price. No pydantic. Preserve output key names the two view bridges and BDD steps rely on.", + "depends_on": [], + "acceptance_criteria": [ + {"id": "AC-02-1", "description": "DiscoverySanitizer ported to core/services/discovery.py without pydantic", "test_type": "unit"}, + {"id": "AC-02-2", "description": "CanonicalPropertyPayload is pydantic-free (dataclass or dict) with Decimal price", "test_type": "unit"}, + {"id": "AC-02-3", "description": "prei/pipeline/tests/test_discovery.py sanitizer tests pass against new location", "test_type": "unit"} + ], + "agent": "build", + "estimated_lines": 130, + "skills": ["build/refactoring", "lang-python"] + }, + { + "id": "TASK-03", + "summary": "Port underwriting solver to Django (Decimal, dataclasses)", + "description": "Add core/services/underwriting.py: solve_underwriting + UnderwritingInput/UnderwritingMetrics as dataclasses (they are already Decimal-based; replace pydantic BaseModel with dataclasses), importing cap_rate/cash_on_cash/to_decimal from investor_app.finance.utils. Pure function — no views, no models.", + "depends_on": [], + "acceptance_criteria": [ + {"id": "AC-03-1", "description": "solve_underwriting ported to core/services/underwriting.py, pydantic-free dataclasses", "test_type": "unit"}, + {"id": "AC-03-2", "description": "UnderwritingInput/Metrics dataclasses use Decimal only", "test_type": "unit"}, + {"id": "AC-03-3", "description": "prei/pipeline/tests/test_underwriting.py passes against new location", "test_type": "unit"} + ], + "agent": "build", + "estimated_lines": 150, + "skills": ["build/refactoring", "lang-python"] + }, + { + "id": "TASK-04", + "summary": "Port DiscoveryProcessor (dedup + persist) to Django model layer", + "description": "Add core/services/discovery_processor.py (or extend core/services/pipeline.py): process_discovery(canonical_payload) -> PipelineProperty — dedup by address_hash (SHA-256 of normalized address, same rule as PipelineAsset.address_hash), create-or-update PipelineProperty at DISCOVERY stage. Replaces prei DiscoveryProcessor which built pydantic PropertyAsset objects.", + "depends_on": ["TASK-02"], + "acceptance_criteria": [ + {"id": "AC-04-1", "description": "process_discovery persists PipelineProperty at DISCOVERY stage with dedup on address_hash", "test_type": "unit"}, + {"id": "AC-04-2", "description": "No pydantic PropertyAsset creation anywhere in the Django path", "test_type": "unit"}, + {"id": "AC-04-3", "description": "prei/pipeline/tests/test_discovery_processor.py logic covered by new Django test", "test_type": "unit"} + ], + "agent": "build", + "estimated_lines": 120, + "skills": ["build/code-generation", "lang-python"] + }, + { + "id": "TASK-05", + "summary": "Rewrite Growth Explorer bridge in core/views/__init__.py (~line 1235) onto Django services", + "description": "Replace lazy prei imports (DiscoveryProcessor, ScreeningThresholds, BatchScreeningProcessor, InMemoryAssetRepository, PipelineEngine, discover_from_all) with core.services equivalents: discovery via sources + DiscoverySanitizer, persist via process_discovery, screen via evaluate_screening_stage, advance PipelineProperty stage on pass, KILLED + kill_reason on fail. Keep view behavior (growth explorer pipeline_city branch) identical.", + "depends_on": ["TASK-01", "TASK-02", "TASK-04"], + "acceptance_criteria": [ + {"id": "AC-05-1", "description": "Growth Explorer pipeline_city branch has zero prei imports", "test_type": "manual"}, + {"id": "AC-05-2", "description": "PipelineProperty rows created/screened identically to prior engine behavior", "test_type": "integration"}, + {"id": "AC-05-3", "description": "No regression in growth_explorer tests", "test_type": "ci"} + ], + "agent": "build", + "estimated_lines": 140, + "skills": ["build/refactoring", "lang-python"] + }, + { + "id": "TASK-06", + "summary": "Rewrite vrm_properties_list run_pipeline bridge in core/views/__init__.py (~line 3227) onto Django services", + "description": "Replace lazy prei imports (InMemoryAssetRepository, PipelineEngine, ScreeningThresholds, BatchScreeningProcessor) with core.services screening + PipelineProperty persistence. Keep POST behavior for run_pipeline.", + "depends_on": ["TASK-01", "TASK-02", "TASK-04"], + "acceptance_criteria": [ + {"id": "AC-06-1", "description": "vrm_properties_list run_pipeline has zero prei imports", "test_type": "manual"}, + {"id": "AC-06-2", "description": "run_pipeline persists/screens via Django services; existing tests updated and passing", "test_type": "integration"} + ], + "agent": "build", + "estimated_lines": 110, + "skills": ["build/refactoring", "lang-python"] + }, + { + "id": "TASK-07", + "summary": "Migrate the 9 prei-importing test files + tests_bdd steps to core.services", + "description": "Update imports in tests/test_pipeline.py, test_pipeline_e2e.py, test_discovery.py, test_discovery_integration.py, test_discovery_e2e.py, test_offer_integration.py, test_screening_integration.py, test_underwriting_integration.py, and tests_bdd/steps/pipeline_steps.py to point at core/services/* and core/services/pipeline.py instead of prei.pipeline.* and prei.models.*. Delete the pipeline BDD feature only if it exclusively exercises deleted engine state (PipelineEngine/InMemoryAssetRepository); otherwise rewrite steps to Django-backed equivalents.", + "depends_on": ["TASK-01", "TASK-02", "TASK-03", "TASK-04"], + "acceptance_criteria": [ + {"id": "AC-07-1", "description": "No test file outside prei/ imports prei (grep prei.pipeline returns nothing outside prei/)", "test_type": "ci"}, + {"id": "AC-07-2", "description": "Full test suite green after import migration", "test_type": "ci"} + ], + "agent": "build", + "estimated_lines": 260, + "skills": ["build/refactoring", "lang-python"] + }, + { + "id": "TASK-08", + "summary": "Delete pydantic state machine + engine + orchestrator (PropertyAsset/StageLog/PipelineStage/InMemory/SQLite repos, PipelineEngine, StateAggregator, PipelineOrchestrator)", + "description": "Delete prei/models/pipeline.py, prei/pipeline/engine.py, prei/pipeline/orchestrator.py. PipelineProperty (Django) + core/services/pipeline.py STAGE_ORDER transitions are the canonical state. Requires PM sign-off. Requires TASK-05/06/07 so no production or test import remains.", + "depends_on": ["TASK-05", "TASK-06", "TASK-07"], + "acceptance_criteria": [ + {"id": "AC-08-1", "description": "prei/models/pipeline.py, engine.py, orchestrator.py deleted", "test_type": "manual"}, + {"id": "AC-08-2", "description": "Grep for PropertyAsset/StageLog/PipelineEngine/PipelineOrchestrator returns nothing outside git history", "test_type": "ci"}, + {"id": "AC-08-3", "description": "Full suite + ruff + mypy green", "test_type": "ci"} + ], + "agent": "build", + "estimated_lines": 1100, + "skills": ["build/refactoring", "build/governance-enforcement"] + }, + { + "id": "TASK-09", + "summary": "Delete FastAPI router + CLI (prei/api/, prei/cli.py) and the offer.py float handler", + "description": "Delete prei/api/pipeline_routes.py and prei/cli.py (click). Resolve LIMIT-21 (offer.py float-based) by porting offer math to Decimal in core/services/offer.py OR deleting the handler — port the pure math (OfferInput/OfferMetrics as Decimal dataclasses) to keep tests/test_offer_integration.py behavior, delete the pydantic/CLI/API surface. Requires PM sign-off.", + "depends_on": ["TASK-07"], + "acceptance_criteria": [ + {"id": "AC-09-1", "description": "prei/api/ and prei/cli.py deleted; no fastapi/click imports in production code", "test_type": "ci"}, + {"id": "AC-09-2", "description": "Offer math ported Decimal-based to core/services/offer.py (LIMIT-21 resolved) or explicitly deferred with PM sign-off", "test_type": "manual"}, + {"id": "AC-09-3", "description": "prei/pipeline/tests/test_api.py deleted (tests deleted code)", "test_type": "manual"} + ], + "agent": "build", + "estimated_lines": 400, + "skills": ["build/refactoring", "lang-python"] + }, + { + "id": "TASK-10", + "summary": "Migrate or delete prei/pipeline/tests/* and remaining handlers/sources", + "description": "Port valuable tests (test_sources, test_county, test_reo_sources, test_screening, test_underwriting, test_discovery math) to core/tests/. Delete tests that only exercised deleted engine state (test_engine, test_repository, test_orchestrator, test_batch_screening, test_api). Delete prei/pipeline/handlers/batch_screening.py, discovery.py, discovery_processor.py, screening.py, underwriting.py (pure logic now lives in core/services) and prei/pipeline/sources/* (county/reo_sources/vrm/file_source/registry/base) only after confirming nothing imports them; if sources are still needed by the Growth Explorer bridge, move them into core/services/sources/ first in TASK-05.", + "depends_on": ["TASK-08", "TASK-09"], + "acceptance_criteria": [ + {"id": "AC-10-1", "description": "prei/ package fully deleted (no prei.* imports anywhere in repo)", "test_type": "ci"}, + {"id": "AC-10-2", "description": "All valuable test coverage relocated to core/tests/ and green", "test_type": "ci"}, + {"id": "AC-10-3", "description": "Full suite + ruff + mypy green with prei/ gone", "test_type": "ci"} + ], + "agent": "build", + "estimated_lines": 2100, + "skills": ["build/refactoring", "build/governance-enforcement"] + }, + { + "id": "TASK-11", + "summary": "Remove pydantic/fastapi/uvicorn/click from requirements + docs update", + "description": "Remove pydantic==2.13.4, fastapi==0.139.2, uvicorn[standard]==0.51.0, click==8.4.2 from requirements.txt only if no remaining code imports them (tests/acceptance/schemas.py + tests/acceptance/test_api.py use pydantic for acceptance payload validation — verify whether that is test-only and keep pydantic pinned for those, or migrate acceptance schemas to dataclasses; fastapi/click must be fully removable). Update docs/ARCHITECTURE.md, docs/KNOWN_LIMITATIONS.md (mark LIMIT-21 resolved or explicitly deferred), docs/CHANGE_IMPACT_MAP.md.", + "depends_on": ["TASK-10"], + "acceptance_criteria": [ + {"id": "AC-11-1", "description": "fastapi, uvicorn, click removable from requirements.txt; pydantic only retained if tests/acceptance still needs it", "test_type": "ci"}, + {"id": "AC-11-2", "description": "KNOWN_LIMITATIONS LIMIT-21 status updated", "test_type": "manual"}, + {"id": "AC-11-3", "description": "ARCHITECTURE.md and CHANGE_IMPACT_MAP.md reflect Django-canonical pipeline", "test_type": "manual"} + ], + "agent": "build", + "estimated_lines": 120, + "skills": ["build/refactoring", "documentation"] + } + ] +} diff --git a/Makefile b/Makefile index b32727ca..52686bc0 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ SHELL := /bin/bash -.PHONY: help ensure-env dev seed superuser lint test test-unit test-integration test-e2e check deploy-dev deploy-local deploy-devcontainer gitops-validate gitops-hook-install smoke build up down restart logs clean docker-dev test-live +.PHONY: help ensure-env dev seed superuser lint test test-unit test-integration test-live-sources test-e2e check deploy-dev deploy-local deploy-devcontainer gitops-validate gitops-hook-install smoke build up down restart logs clean docker-dev test-live # Prefer the local venv; fall back to `python` (containers, bare environments). # Avoids the Apple /usr/bin/python3 shim that triggers the xcode-select prompt. @@ -87,6 +87,15 @@ test-integration: -v --tb=short \ -k "integration" +test-live-sources: + @echo "Running live integration tests" + $(PYTHON) -m pytest -m live core/tests/test_live_sources.py + $(call ensure_django) + @echo "Running live sources verification tests..." + @DJANGO_SETTINGS_MODULE=investor_app.settings_test $(PYTHON) -m pytest core/tests/test_live_sources.py \ + -m live \ + -v --tb=short + test-e2e: $(call ensure_django) @echo "Running E2E tests (requires Playwright browser)..." diff --git a/core/integrations/market/fmr_adapter.py b/core/integrations/market/fmr_adapter.py index f49a9cfa..1c33d52d 100644 --- a/core/integrations/market/fmr_adapter.py +++ b/core/integrations/market/fmr_adapter.py @@ -40,9 +40,13 @@ def fetch_fmr_entity_id(state_code: str, city_name: str) -> str | None: Returns: HUD entity ID string, or ``None`` if not found. """ - api_key = getattr(settings, "HUD_API_KEY", "") + api_key = getattr(settings, "HUD_FMR_TOKEN", "") or getattr( + settings, "HUD_API_KEY", "" + ) if not api_key: - logger.warning("HUD_API_KEY not configured — cannot look up entity ID") + logger.warning( + "HUD_FMR_TOKEN or HUD_API_KEY not configured — cannot look up entity ID" + ) return None client = FMRClient(api_key=api_key) @@ -101,9 +105,11 @@ def fetch_fmr_data( Returns ``None`` if the HUD API key is missing or the entity ID cannot be resolved. """ - api_key = getattr(settings, "HUD_API_KEY", "") + api_key = getattr(settings, "HUD_FMR_TOKEN", "") or getattr( + settings, "HUD_API_KEY", "" + ) if not api_key: - logger.info("HUD_API_KEY not set — skipping FMR data fetch") + logger.info("HUD_FMR_TOKEN or HUD_API_KEY not set — skipping FMR data fetch") return None if not entity_id: diff --git a/core/integrations/market/hud_fmr.py b/core/integrations/market/hud_fmr.py index 2eff1d43..06715e7d 100644 --- a/core/integrations/market/hud_fmr.py +++ b/core/integrations/market/hud_fmr.py @@ -32,7 +32,11 @@ class FMRClient: """Client for the HUD Fair Market Rent API.""" def __init__(self, api_key: str | None = None) -> None: - self.api_key = api_key or getattr(settings, "HUD_API_KEY", "") + self.api_key = ( + api_key + or getattr(settings, "HUD_FMR_TOKEN", "") + or getattr(settings, "HUD_API_KEY", "") + ) def _headers(self) -> dict[str, str]: return {"Authorization": f"Bearer {self.api_key}"} diff --git a/core/integrations/market/hud_il.py b/core/integrations/market/hud_il.py index 116a5692..b6c6d901 100644 --- a/core/integrations/market/hud_il.py +++ b/core/integrations/market/hud_il.py @@ -32,7 +32,11 @@ class ILClient: """Client for the HUD Income Limits API.""" def __init__(self, api_key: str | None = None) -> None: - self.api_key = api_key or getattr(settings, "HUD_API_KEY", "") + self.api_key = ( + api_key + or getattr(settings, "HUD_FMR_TOKEN", "") + or getattr(settings, "HUD_API_KEY", "") + ) def _headers(self) -> dict[str, str]: return {"Authorization": f"Bearer {self.api_key}"} @@ -106,9 +110,13 @@ def fetch_area_median_income(entity_id: str) -> dict[str, Any] | None: Dict with median_income, very_low_income_1, very_low_income_4, low_income_4, year — or None if unavailable. """ - api_key = getattr(settings, "HUD_API_KEY", "") + api_key = getattr(settings, "HUD_FMR_TOKEN", "") or getattr( + settings, "HUD_API_KEY", "" + ) if not api_key: - logger.info("HUD_API_KEY not set — skipping Income Limits fetch") + logger.info( + "HUD_FMR_TOKEN or HUD_API_KEY not set — skipping Income Limits fetch" + ) return None client = ILClient(api_key=api_key) diff --git a/core/services/sources/county.py b/core/services/sources/county.py index 069a014a..84ac462a 100644 --- a/core/services/sources/county.py +++ b/core/services/sources/county.py @@ -74,9 +74,9 @@ "tarrant": { "name": "Tarrant County", "state": "TX", - "type": "csv", - "foreclosure_url": "https://www.tarrantcounty.com/en/county-clerk/real-property/foreclosure-listings.html", - "notes": "Monthly foreclosure listing", + "type": "html", + "foreclosure_url": "https://www.tarrantcountytx.gov/en/constables/constable-3/delinquent-tax-sales/monthly-tax-sales-listings.html", + "notes": "Monthly delinquent tax sale listings from Constable Precinct 3 (first Tuesday auctions)", }, "collin": { "name": "Collin County", @@ -253,12 +253,95 @@ def _fetch_rss(self, url: str, limit: int) -> List[Dict[str, Any]]: return [] def _fetch_html(self, url: str, limit: int) -> List[Dict[str, Any]]: - """Fallback HTML scraper for sites without structured feeds.""" - logger.info( - "TX County %s: HTML scraping not implemented — returning empty", - self.county_key, - ) - return [] + """Scrape Tarrant County monthly delinquent tax sale listings. + + The county clerk does not publish a machine-readable foreclosure + list; Constable Precinct 3 publishes monthly sale pages (first + Tuesday auctions) with cause/account numbers and status. The + listings index page links each month's sale page, which contains + a ``CAUSE NUMBER / ACCOUNT NUMBER / STATUS`` table. + """ + if self.county_key != "tarrant": + logger.info( + "TX County %s: HTML scraping not implemented — returning empty", + self.county_key, + ) + return [] + + try: + index_html = _fetch_via_playwright(url) + if not index_html: + logger.warning("Tarrant: index page unreachable") + return [] + + from urllib.parse import urljoin + + from bs4 import BeautifulSoup + + soup = BeautifulSoup(index_html, "html.parser") + month_links = [] + for a in soup.select("#wpsm-mainContent a, main a"): + href = a.get("href") + if ( + isinstance(href, str) + and "monthly-tax-sales-listings" in href + and a.get_text(strip=True) + ): + month_links.append(href) + # Prefer the most recent sale page (list is in chronological order). + sale_href = str(month_links[-1]) if month_links else None + if not sale_href: + logger.warning("Tarrant: no monthly sale links found") + return [] + sale_url = str(urljoin(url, sale_href)) + + sale_html = _fetch_via_playwright(sale_url) + if not sale_html: + logger.warning("Tarrant: sale page unreachable: %s", sale_url) + return [] + + sale_soup = BeautifulSoup(sale_html, "html.parser") + listings: List[Dict[str, Any]] = [] + for table in sale_soup.find_all("table"): + rows = table.find_all("tr") + for row in rows[1:]: + cells = [c.get_text(strip=True) for c in row.find_all(["td", "th"])] + if len(cells) < 2: + continue + cause, account = cells[0], cells[1] + if not cause or not account: + continue + status = cells[2] if len(cells) > 2 else "" + if status.lower() in ("withdrawn", "sold", "struck off"): + continue + listings.append( + { + "id": f"tarrant-tax-{account}", + "case_number": cause, + "account_number": account, + "address": f"Tarrant County tax account {account}", + "city": "Fort Worth", + "state": "TX", + "county": "Tarrant", + "source_url": sale_url, + "status": status, + "notice_type": "tax_sale", + } + ) + if len(listings) >= limit: + break + if len(listings) >= limit: + break + + logger.info( + "Tarrant: %d tax sale listings parsed from %s", + len(listings), + sale_url, + ) + return listings + except Exception as exc: + logger.warning("Tarrant HTML scrape error: %s", exc) + return [] @staticmethod def _map_county_row( @@ -376,3 +459,43 @@ def fetch( @staticmethod def available_counties() -> List[str]: return list(FLORIDA_COUNTY_FEEDS.keys()) + + +def _fetch_via_playwright(url: str) -> str | None: + """Fetch page HTML via Playwright headless Chromium (county sites are JS-rendered). + + Returns the rendered HTML string, or None on failure. Playwright must + be installed (requirements.txt) with browsers fetched via + ``playwright install chromium``. + """ + try: + from playwright.sync_api import sync_playwright # type: ignore[import-not-found] + except ImportError: + logger.warning("playwright not installed — cannot render county pages") + return None + + try: + with sync_playwright() as pw: + with pw.chromium.launch(headless=True) as browser: + context = browser.new_context( + user_agent=( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/120.0.0.0 Safari/537.36" + ) + ) + page = context.new_page() + response = page.goto( + url, timeout=TIMEOUT * 1000, wait_until="domcontentloaded" + ) + if response is None or response.status >= 400: + logger.warning( + "Page returned %s for %s", + getattr(response, "status", "?"), + url, + ) + return None + return str(page.content()) + except Exception as exc: + logger.warning("Playwright fetch failed for %s: %s", url, exc) + return None diff --git a/core/tests/test_live_sources.py b/core/tests/test_live_sources.py new file mode 100644 index 00000000..906de144 --- /dev/null +++ b/core/tests/test_live_sources.py @@ -0,0 +1,172 @@ +"""Live integration tests for property discovery sources and external APIs. + +These tests hit real external services and are gated by API key presence. +Run with: make test-live-sources +Skipped automatically when required keys are missing. +""" + +from __future__ import annotations + +import os + +import pytest +from django.conf import settings + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _key(name: str) -> str: + """Return the raw value of an env/Django key, empty string if absent.""" + return getattr(settings, name, "") or os.environ.get(name, "") + + +# --------------------------------------------------------------------------- +# Key presence +# --------------------------------------------------------------------------- + + +@pytest.mark.live +def test_hud_fmr_token_present() -> None: + """HUD_FMR_TOKEN or HUD_API_KEY must be configured for FMR tests.""" + key = _key("HUD_FMR_TOKEN") or _key("HUD_API_KEY") + assert key, "HUD_FMR_TOKEN or HUD_API_KEY not configured" + + +@pytest.mark.live +def test_attom_api_key_present() -> None: + """ATTOM_API_KEY must be configured for ATTOM tests.""" + assert _key("ATTOM_API_KEY"), "ATTOM_API_KEY not configured" + + +@pytest.mark.live +def test_census_api_key_present() -> None: + """CENSUS_API_KEY must be configured for Census tests.""" + assert _key("CENSUS_API_KEY"), "CENSUS_API_KEY not configured" + + +@pytest.mark.live +def test_fred_api_key_present() -> None: + """FRED_API_KEY must be configured for FRED tests.""" + assert _key("FRED_API_KEY"), "FRED_API_KEY not configured" + + +# --------------------------------------------------------------------------- +# Tarrant County — live fetch +# --------------------------------------------------------------------------- + + +@pytest.mark.live +def test_tarrant_county_live_fetch() -> None: + """Tarrant County source must connect and return listings.""" + import django + + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "investor_app.settings") + django.setup() + + from core.services.sources.county import TexasCountyForeclosureSource + + source = TexasCountyForeclosureSource(county="tarrant") + listings = source.fetch() + assert len(listings) > 0, "Tarrant source returned 0 listings — feed may be down" + sample = listings[0] + assert "id" in sample + assert sample.get("county") == "Tarrant" + assert sample.get("state") == "TX" + assert sample.get("source_url"), "source_url missing" + + +# --------------------------------------------------------------------------- +# VA VRM scraper — live fetch +# --------------------------------------------------------------------------- + + +@pytest.mark.live +def test_va_vrm_scraper_live_fetch() -> None: + """VRM scraper must connect to vrmproperties.com and return VA listings.""" + import django + + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "investor_app.settings") + django.setup() + + from core.integrations.sources.vrm_scraper import VrmScraper + + scraper = VrmScraper(delay_seconds=0.3) + props = scraper.collect_state_listings("VA") + assert len(props) > 0, "VRM scraper returned 0 VA listings — site may be down" + sample = props[0] + assert "address" in sample + assert sample.get("state") == "VA" + assert "list_price" in sample + + +# --------------------------------------------------------------------------- +# HUD FMR API — live connection +# --------------------------------------------------------------------------- + + +@pytest.mark.live +def test_hud_fmr_list_counties_tx() -> None: + """HUD FMR API must return Texas counties when a valid token is present.""" + import django + + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "investor_app.settings") + django.setup() + + from core.integrations.market.hud_fmr import FMRClient + + token = _key("HUD_FMR_TOKEN") or _key("HUD_API_KEY") + client = FMRClient(api_key=token) + counties = client.list_counties("TX") + assert len(counties) > 0, ( + "HUD FMR returned 0 TX counties — token may lack FMR permission" + ) + tarrant = [c for c in counties if "Tarrant" in c.get("county_name", "")] + assert tarrant, "Tarrant County not found in HUD FMR response" + + +# --------------------------------------------------------------------------- +# Census API — live connection +# --------------------------------------------------------------------------- + + +@pytest.mark.live +def test_census_place_growth_metrics() -> None: + """Census API must return growth metrics for Fort Worth, TX (place FIPS 27000).""" + import django + + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "investor_app.settings") + django.setup() + + from core.integrations.market.census import fetch_place_growth_metrics + + api_key = _key("CENSUS_API_KEY") + result = fetch_place_growth_metrics("TX", "27000", api_key, place_name="Fort Worth") + assert result is not None, ( + "Census API returned None — key may be invalid or rate-limited" + ) + assert result.get("population", 0) > 0, "Census returned zero population" + assert "pop_growth" in result + + +# --------------------------------------------------------------------------- +# FRED / BLS — live connection +# --------------------------------------------------------------------------- + + +@pytest.mark.live +def test_fred_employment_growth_tx() -> None: + """FRED/BLS must return employment growth for Texas.""" + import django + + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "investor_app.settings") + django.setup() + + from core.integrations.market.bls import fetch_employment_growth + + api_key = _key("FRED_API_KEY") + growth_rate = fetch_employment_growth("TX", api_key) + assert growth_rate is not None, "FRED/BLS returned None — key may be invalid" + assert growth_rate > 0, "Employment growth should be positive" diff --git a/investor_app/settings.py b/investor_app/settings.py index 691bff51..914375e0 100644 --- a/investor_app/settings.py +++ b/investor_app/settings.py @@ -351,7 +351,10 @@ # REHAB_COST_MODERATE, REHAB_COST_FULL_GUT (dollar amounts, e.g. "15"). # FRED API key for economic data (employment growth, unemployment) FRED_API_KEY: str = env("FRED_API_KEY", default="") +# HUD API keys — HUD_FMR_TOKEN is preferred for FMR dataset (Fair Market Rent + Income Limits) +# Create at https://www.huduser.gov/portal/dataset/fmr-api.html — must select FMR dataset before generating token. HUD_API_KEY: str = env("HUD_API_KEY", default="") +HUD_FMR_TOKEN: str = env("HUD_FMR_TOKEN", default="") ATTOM_API_KEY: str = env( "ATTOM_API_KEY", default="" ) # ATTOM preforeclosure + property data diff --git a/pytest.ini b/pytest.ini index a0d0a34f..8acb0a7c 100644 --- a/pytest.ini +++ b/pytest.ini @@ -6,4 +6,5 @@ addopts = --reruns 1 --reruns-delay 5 --report-log=.pytest-report.jsonl -k "not markers = e2e: End-to-end tests that simulate full pipeline flow (skipped in CI) integration: Tests that hit live external APIs (skipped if API keys not set) + live: Tests that run against live external data sources/APIs (network & API key gated) slow: Tests that take longer than usual to complete (e.g., live API calls) From 7109049687eb29c9bcbacda023e810943fcc5106 Mon Sep 17 00:00:00 2001 From: Phil Ruff Date: Fri, 31 Jul 2026 22:16:16 +0100 Subject: [PATCH 5/9] fix(ci): remove deleted prei/pipeline/tests/ from unit test path --- .github/workflows/ci-quality.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-quality.yml b/.github/workflows/ci-quality.yml index 9789aac5..b1d3c49d 100644 --- a/.github/workflows/ci-quality.yml +++ b/.github/workflows/ci-quality.yml @@ -101,7 +101,7 @@ jobs: - name: Run unit tests run: | export SECRET_KEY=$(python -c 'import secrets; print(secrets.token_hex(50))') - coverage run --parallel-mode -m pytest tests/ core/tests/ tests_bdd/ prei/pipeline/tests/ \ + coverage run --parallel-mode -m pytest tests/ core/tests/ tests_bdd/ \ -q --tb=short \ -k "not e2e and not docker and not integration and not container and not startup and not add_to_pipeline and not acceptance and not export and not unreachable_url" - name: Flaky test report From fd3b286d1c6cdbae68906d380c6e7e80f841cad0 Mon Sep 17 00:00:00 2001 From: Phil Ruff Date: Fri, 31 Jul 2026 22:32:11 +0100 Subject: [PATCH 6/9] fix(ci): exclude live-marked tests from unit test job --- .github/workflows/ci-quality.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-quality.yml b/.github/workflows/ci-quality.yml index b1d3c49d..99f3fef8 100644 --- a/.github/workflows/ci-quality.yml +++ b/.github/workflows/ci-quality.yml @@ -103,7 +103,7 @@ jobs: export SECRET_KEY=$(python -c 'import secrets; print(secrets.token_hex(50))') coverage run --parallel-mode -m pytest tests/ core/tests/ tests_bdd/ \ -q --tb=short \ - -k "not e2e and not docker and not integration and not container and not startup and not add_to_pipeline and not acceptance and not export and not unreachable_url" + -k "not e2e and not docker and not integration and not container and not startup and not add_to_pipeline and not acceptance and not export and not unreachable_url and not live" - name: Flaky test report if: always() run: python .github/scripts/flaky_report.py --report-log .pytest-report.jsonl --mode report From 1838e99cb52f2c177b6eacd0456c08cc833e9b3c Mon Sep 17 00:00:00 2001 From: Phil Ruff Date: Sat, 1 Aug 2026 12:59:15 +0100 Subject: [PATCH 7/9] refactor(test): restructure testing pyramid with explicit markers - pytest.ini: add unit, smoke, acceptance markers; default -m 'unit or integration' - conftest.py: auto-assign layer markers by file path (acceptance, e2e, smoke, live) - 11 pure unit test files: explicit pytestmark = pytest.mark.unit - ci-quality.yml: unit/integration/e2e jobs use -m markers instead of -k keywords - Makefile: test targets use -m markers; fixed duplicate test-acceptance - .coveragerc: restored with prei removed from sources Markers: unit (792), integration (1164), e2e (42), acceptance (41), live (9), smoke (9) --- .coveragerc | 2 +- .github/workflows/ci-quality.yml | 15 +-- .serena/project.yml | 2 + Makefile | 15 ++- conftest.py | 85 ++++++++++--- core/tests/test_validators.py | 2 + ...8-01-testing-pyramid-restructure-design.md | 116 ++++++++++++++++++ pytest.ini | 11 +- tests/test_brrrr.py | 2 + tests/test_hold_period.py | 2 + tests/test_makefile.py | 5 + tests/test_production_settings.py | 4 + tests/test_tax_analysis.py | 2 + tests/test_underwriting_score.py | 2 + 14 files changed, 226 insertions(+), 39 deletions(-) create mode 100644 .serena/project.yml create mode 100644 docs/superpowers/specs/2026-08-01-testing-pyramid-restructure-design.md diff --git a/.coveragerc b/.coveragerc index ab21a0a5..00f8f7b0 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,5 +1,5 @@ [run] -source = core, prei +source = core, investor_app omit = */migrations/* */tests/* diff --git a/.github/workflows/ci-quality.yml b/.github/workflows/ci-quality.yml index 99f3fef8..e010b8e9 100644 --- a/.github/workflows/ci-quality.yml +++ b/.github/workflows/ci-quality.yml @@ -101,9 +101,8 @@ jobs: - name: Run unit tests run: | export SECRET_KEY=$(python -c 'import secrets; print(secrets.token_hex(50))') - coverage run --parallel-mode -m pytest tests/ core/tests/ tests_bdd/ \ - -q --tb=short \ - -k "not e2e and not docker and not integration and not container and not startup and not add_to_pipeline and not acceptance and not export and not unreachable_url and not live" + coverage run --parallel-mode -m pytest tests/ core/tests/ -m unit \ + -q --tb=short - name: Flaky test report if: always() run: python .github/scripts/flaky_report.py --report-log .pytest-report.jsonl --mode report @@ -139,9 +138,8 @@ jobs: - name: Run integration tests run: | export SECRET_KEY=$(python -c 'import secrets; print(secrets.token_hex(50))') - coverage run --parallel-mode -m pytest tests/ core/tests/ \ - -q --tb=short \ - -k "integration" + coverage run --parallel-mode -m pytest tests/ core/tests/ -m integration \ + -q --tb=short - name: Flaky test report if: always() run: python .github/scripts/flaky_report.py --report-log .pytest-report.jsonl --mode report @@ -177,9 +175,8 @@ jobs: - name: Run E2E tests run: | export SECRET_KEY=$(python -c 'import secrets; print(secrets.token_hex(50))') - coverage run --parallel-mode -m pytest tests/ \ - -q --tb=short \ - -k "e2e or docker or container or startup or add_to_pipeline or export" + coverage run --parallel-mode -m pytest tests/ core/tests/ -m e2e \ + -q --tb=short - name: Flaky test report if: always() run: python .github/scripts/flaky_report.py --report-log .pytest-report.jsonl --mode report diff --git a/.serena/project.yml b/.serena/project.yml new file mode 100644 index 00000000..980051ba --- /dev/null +++ b/.serena/project.yml @@ -0,0 +1,2 @@ +project_name: "prei" +languages: ["typescript", "python", "bash"] diff --git a/Makefile b/Makefile index 52686bc0..a7be0cc6 100644 --- a/Makefile +++ b/Makefile @@ -76,22 +76,20 @@ test: test-unit test-unit: $(call ensure_django) - @DJANGO_SETTINGS_MODULE=investor_app.settings_test $(PYTHON) -m pytest tests/ core/tests/ tests_bdd/ \ + @DJANGO_SETTINGS_MODULE=investor_app.settings_test $(PYTHON) -m pytest tests/ core/tests/ \ -q --tb=short \ - -k "not e2e and not docker and not integration and not container and not startup and not add_to_pipeline and not acceptance and not export and not unreachable_url" + -m unit test-integration: $(call ensure_django) @echo "Running integration tests..." @DJANGO_SETTINGS_MODULE=investor_app.settings_test $(PYTHON) -m pytest tests/ core/tests/ \ -v --tb=short \ - -k "integration" + -m integration test-live-sources: - @echo "Running live integration tests" - $(PYTHON) -m pytest -m live core/tests/test_live_sources.py $(call ensure_django) - @echo "Running live sources verification tests..." + @echo "Running live integration tests (requires API keys)..." @DJANGO_SETTINGS_MODULE=investor_app.settings_test $(PYTHON) -m pytest core/tests/test_live_sources.py \ -m live \ -v --tb=short @@ -99,9 +97,9 @@ test-live-sources: test-e2e: $(call ensure_django) @echo "Running E2E tests (requires Playwright browser)..." - @DJANGO_SETTINGS_MODULE=investor_app.settings_test $(PYTHON) -m pytest tests/ \ + @DJANGO_SETTINGS_MODULE=investor_app.settings_test $(PYTHON) -m pytest tests/ core/tests/ \ -v --tb=short \ - -k "e2e or docker or container or startup or add_to_pipeline or export" + -m e2e check: ensure-env $(call ensure_django) @@ -110,6 +108,7 @@ check: ensure-env @$(MAKE) test-unit @$(MAKE) test-integration @$(MAKE) test-e2e + @$(MAKE) test-acceptance # ── Deploy ──────────────────────────────────────────────────────────────── diff --git a/conftest.py b/conftest.py index e273d021..b6f91642 100644 --- a/conftest.py +++ b/conftest.py @@ -20,27 +20,78 @@ def pytest_configure(config) -> None: # noqa: ARG001 def pytest_collection_modifyitems(config, items) -> None: # noqa: ARG001 - """Quarantine tests flagged as flaky (see docs/quality/flaky_tests.json). + """Auto-assign test-layer markers and quarantine flaky tests. - Quarantined tests keep running and reporting but never fail the build, - so a known-flaky test can't block a PR while it's being fixed. + Markers are assigned by file path so that CI can filter via -m. + Explicit pytestmark overrides these; files without an explicit + marker default to ``integration`` (safe — always passes with DB). """ - if not QUARANTINE_FILE.exists(): - return - quarantined = { - line.strip() - for line in QUARANTINE_FILE.read_text().splitlines() - if line.strip() + # ── Quarantine flaky tests ── + if QUARANTINE_FILE.exists(): + quarantined = { + line.strip() + for line in QUARANTINE_FILE.read_text().splitlines() + if line.strip() + } + if quarantined: + q_marker = pytest.mark.xfail( + reason="quarantined: flaky, see docs/quality/flaky_tests.json", + strict=False, + ) + for item in items: + if item.nodeid in quarantined: + item.add_marker(q_marker) + + # ── Auto-assign layer markers by file path ── + _ACCEPTANCE_DIR = "tests/acceptance/" + _SMOKE_FILES = {"test_container_startup.py", "test_docker_e2e.py"} + _LIVE_FILES = { + "test_live_sources.py", + "test_integration_attom.py", + "test_integration_fred.py", } - if not quarantined: - return - marker = pytest.mark.xfail( - reason="quarantined: flaky, see docs/quality/flaky_tests.json", - strict=False, - ) + _BDD_DIR = "tests_bdd/" + _E2E_SUFFIX = "_e2e.py" + for item in items: - if item.nodeid in quarantined: - item.add_marker(marker) + fpath = item.fspath.strpath + + # Already has an explicit marker — leave it alone + already_marked = any( + m.name in ("unit", "integration", "smoke", "acceptance", "e2e", "live") + for m in item.iter_markers() + ) + if already_marked: + continue + + # Acceptance tests + if _ACCEPTANCE_DIR in fpath: + item.add_marker(pytest.mark.acceptance) + continue + + # Smoke tests + fname = item.fspath.purebasename + if fname in _SMOKE_FILES: + item.add_marker(pytest.mark.smoke) + continue + + # Live API tests + if fname in _LIVE_FILES: + item.add_marker(pytest.mark.live) + continue + + # BDD — e2e + if _BDD_DIR in fpath: + item.add_marker(pytest.mark.e2e) + continue + + # Files ending in _e2e.py + if fname.endswith("_e2e"): + item.add_marker(pytest.mark.e2e) + continue + + # Default: integration (DB-backed) — safest fallback for Django apps + item.add_marker(pytest.mark.integration) @pytest.fixture diff --git a/core/tests/test_validators.py b/core/tests/test_validators.py index 91f3ef85..c3d12d6e 100644 --- a/core/tests/test_validators.py +++ b/core/tests/test_validators.py @@ -13,6 +13,8 @@ validate_state_code, ) +pytestmark = pytest.mark.unit + class TestValidateStateCode: def test_valid_normalized(self) -> None: diff --git a/docs/superpowers/specs/2026-08-01-testing-pyramid-restructure-design.md b/docs/superpowers/specs/2026-08-01-testing-pyramid-restructure-design.md new file mode 100644 index 00000000..48f0498f --- /dev/null +++ b/docs/superpowers/specs/2026-08-01-testing-pyramid-restructure-design.md @@ -0,0 +1,116 @@ +# Design: Testing Pyramid Restructure + +**Date:** 2026-08-01 +**Status:** APPROVED — awaiting implementation + +--- + +## Problem + +The current CI test structure has one monolithic "unit tests" job running ~1500 +tests via keyword exclusion (`-k "not e2e and not ..."`). Real unit tests (pure +functions, no DB) are indistinguishable from integration tests (DB, fixtures) and +live tests (external APIs). This causes: + +1. "Unit" test job takes 5+ minutes +2. Live tests (`test_live_sources.py`) leak into unit runs and fail without API keys +3. No way to fail fast on pure unit tests before running slow integration +4. Markers exist in pytest.ini but aren't enforced via `-m` — only weak `-k` keyword matching +5. `.coveragerc` still references deleted `prei` package + +## Design + +### Testing Pyramid + +``` + ┌─────────────────────────────┐ + │ post-deployment smoke │ on deploy to production + │ live-sources │ on main push (non-gating) + ├─────────────────────────────┤ + │ acceptance (httpx HTTP) │ ~500 tests + ├─────────────────────────────┤ + │ smoke (Docker container) │ docker-publish only + ├─────────────────────────────┤ + │ e2e (full browser flows) │ ~100 tests + ├─────────────────────────────┤ + │ integration (DB, no external) │ ~800 tests, ~2 min + ├─────────────────────────────┤ + │ unit (pure, no DB, <50ms) │ ~700 tests, ~45 sec + ├─────────────────────────────┤ + │ static-analysis │ lint, typecheck, secrets + └─────────────────────────────┘ +``` + +### Layer Mappings + +| Layer | Marker | Needs | CI Job Name | Required? | +|---------------------|-------------|-------|------------------|-----------| +| static-analysis | n/a | n/a | `🔍 Lint` / `🔷 Typecheck` / `🔑 Secrets` | Yes | +| unit | `unit` | No DB, no HTTP, no filesystem | `🧪 Unit Tests` | Yes | +| integration | `integration` | DB + fixtures, no external APIs | `🔗 Integration Tests` | Yes | +| smoke | `smoke` | Docker container build + startup | `🐳 Container Smoke` | Yes (docker-publish) | +| e2e | `e2e` | Browser / Docker Compose | `E2E Tests` | Yes | +| acceptance | `acceptance` | httpx against live_server | `🌐 Acceptance Tests` | Yes | +| live-sources | `live` | Real API keys (HUD, ATTOM, etc.) | `🌍 Live APIs` | No (non-gating, main push) | + +### File Organization + +All test files live in `tests/` root. Each file gets ONE file-level `pytestmark`: + +```python +pytestmark = pytest.mark.unit # pure functions +pytestmark = pytest.mark.integration # needs DB +pytestmark = pytest.mark.e2e # full pipeline flow +pytestmark = pytest.mark.smoke # container/Docker +pytestmark = pytest.mark.acceptance # httpx HTTP +pytestmark = pytest.mark.live # real external APIs +``` + +Files with mixed test types (e.g. some pure functions + some DB tests) are split. + +## Implementation Plan + +### Phase 1: Pytest config + markers + +1. Update `pytest.ini` — add `unit`, `smoke`, `acceptance` markers +2. Set default `addopts` to `-m "unit or integration"` (fast safe default for bare `pytest`) +3. Update `conftest.py` — auto-apply markers where possible via `pytest_collection_modifyitems` + +### Phase 2: Annotate all 114 test files + +1. Categorize by what the test ACTUALLY needs: + - No DB at all → `unit` (~40 files) + - DB but no external APIs → `integration` (~40 files) + - DB + external third-party APIs → `live` (~4 files) + - Docker container → `smoke` (~3 files) + - httpx HTTP → `acceptance` (~9 files) + - Full pipeline flow → `e2e` (~5 files) +2. Split mixed files +3. Run full suite to verify no regressions + +### Phase 3: Rewrite CI workflow + +1. Update `ci-quality.yml` test jobs to use `-m unit`, `-m integration`, etc. instead of `-k` filters +2. Set proper `needs:` dependencies (integration needs unit to pass first, etc.) +3. Update `pr-gates-pass` to reference new job names + +### Phase 4: Fix config files + +1. Update `.coveragerc` — remove `prei` from `source =` +2. Clean up any stale `__pycache__` and `.pyc` files in deleted directories + +### Phase 5: Update Makefile + +1. Replace `-k` filters with `-m` markers in all test targets +2. Add `make test-unit-fast` for CI-compatible unit runner +3. Verify `make check` runs through all layers + +## Files To Modify + +| File | Change | +|-----------------------------|---------------------------------| +| `pytest.ini` | Add markers, update addopts | +| `.github/workflows/ci-quality.yml` | Rewrite test jobs with `-m` | +| `Makefile` | Update test targets | +| `.coveragerc` | Remove `prei` from sources | +| ~30+ test files | Add `pytestmark` marker | diff --git a/pytest.ini b/pytest.ini index 8acb0a7c..2a8f8a91 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,9 +2,12 @@ DJANGO_SETTINGS_MODULE = investor_app.settings_test python_files = tests.py test_*.py *_tests.py testpaths = tests core/tests tests_bdd -addopts = --reruns 1 --reruns-delay 5 --report-log=.pytest-report.jsonl -k "not e2e and not docker and not integration and not container and not startup and not add_to_pipeline and not acceptance and not export and not unreachable_url" +addopts = --reruns 1 --reruns-delay 5 --report-log=.pytest-report.jsonl -m "unit or integration" markers = - e2e: End-to-end tests that simulate full pipeline flow (skipped in CI) - integration: Tests that hit live external APIs (skipped if API keys not set) + unit: Fast tests with no DB, no external services, no filesystem I/O + integration: Tests that need database access or fixtures but no external APIs + e2e: End-to-end tests simulating full pipeline or browser flows + smoke: Container/Docker smoke tests (build, startup, curl, BDD in container) + acceptance: httpx-based HTTP acceptance tests against live_server or BASE_URL live: Tests that run against live external data sources/APIs (network & API key gated) - slow: Tests that take longer than usual to complete (e.g., live API calls) + slow: Tests that take longer than usual to complete diff --git a/tests/test_brrrr.py b/tests/test_brrrr.py index 7f96ed3a..38c1931e 100644 --- a/tests/test_brrrr.py +++ b/tests/test_brrrr.py @@ -20,6 +20,8 @@ max_refinance_loan, ) +pytestmark = pytest.mark.unit + # --------------------------------------------------------------------------- # Shared fixture data # --------------------------------------------------------------------------- diff --git a/tests/test_hold_period.py b/tests/test_hold_period.py index 51a29330..4594927a 100644 --- a/tests/test_hold_period.py +++ b/tests/test_hold_period.py @@ -11,6 +11,8 @@ total_return_summary, ) +pytestmark = pytest.mark.unit + class TestProjectAnnualCashFlows: """Tests for project_annual_cash_flows function.""" diff --git a/tests/test_makefile.py b/tests/test_makefile.py index 17954a00..aa1772a8 100644 --- a/tests/test_makefile.py +++ b/tests/test_makefile.py @@ -9,6 +9,11 @@ from pathlib import Path +import pytest + +pytestmark = pytest.mark.unit + + MAKEFILE = Path(__file__).resolve().parent.parent / "Makefile" diff --git a/tests/test_production_settings.py b/tests/test_production_settings.py index 720f67af..8c96edd6 100644 --- a/tests/test_production_settings.py +++ b/tests/test_production_settings.py @@ -9,6 +9,10 @@ import sys from typing import Any +import pytest + +pytestmark = pytest.mark.unit + REPO_ROOT = Path(__file__).resolve().parents[1] SECURITY_ENV_KEYS = [ "DJANGO_ENV", diff --git a/tests/test_tax_analysis.py b/tests/test_tax_analysis.py index 8ccf3937..e5d9864e 100644 --- a/tests/test_tax_analysis.py +++ b/tests/test_tax_analysis.py @@ -12,6 +12,8 @@ ) from investor_app.finance.utils import irr +pytestmark = pytest.mark.unit + class TestAnnualDepreciation: """Tests for annual_depreciation function.""" diff --git a/tests/test_underwriting_score.py b/tests/test_underwriting_score.py index 9265b12a..e16dc67f 100644 --- a/tests/test_underwriting_score.py +++ b/tests/test_underwriting_score.py @@ -14,6 +14,8 @@ one_percent_rule, ) +pytestmark = pytest.mark.unit + # ── one_percent_rule ─────────────────────────────────────────────────────────── From 17d002cdf5c407f2dd01a2fd8302d688f9a39cf1 Mon Sep 17 00:00:00 2001 From: Phil Ruff Date: Sat, 1 Aug 2026 13:23:26 +0100 Subject: [PATCH 8/9] fix(ci): acceptance tests use explicit -m acceptance marker The pytest.ini default -m 'unit or integration' was filtering out acceptance tests. The acceptance-check job now explicitly uses -m acceptance. --- .github/workflows/ci-quality.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/ci-quality.yml b/.github/workflows/ci-quality.yml index e010b8e9..f9117097 100644 --- a/.github/workflows/ci-quality.yml +++ b/.github/workflows/ci-quality.yml @@ -230,10 +230,7 @@ jobs: - name: Collect static files run: python manage.py collectstatic --noinput --verbosity 0 - name: Run acceptance tests against live_server - # -k "" overrides pytest.ini's addopts default (which excludes - # "acceptance" so a bare `pytest` stays safe-by-default); the - # last -k on the command line wins over the one baked into addopts. - run: python -m pytest tests/acceptance/ -q --tb=short -k "" + run: python -m pytest tests/acceptance/ -q --tb=short -m acceptance # ── Financial math verification ──────────────────────────────────────────── From 02994c86d834c8e8344a530bb26ab10fe2ca4322 Mon Sep 17 00:00:00 2001 From: Phil Ruff Date: Sat, 1 Aug 2026 13:40:33 +0100 Subject: [PATCH 9/9] fix(ci): install playwright chromium in integration test job Integration tests exercise the PDF export endpoint which launches a Playwright browser; the e2e job already installed it but integration did not. --- .github/workflows/ci-quality.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-quality.yml b/.github/workflows/ci-quality.yml index f9117097..fe0ab7fa 100644 --- a/.github/workflows/ci-quality.yml +++ b/.github/workflows/ci-quality.yml @@ -133,6 +133,8 @@ jobs: run: sudo apt-get install -y libcairo2-dev - name: Install dependencies run: pip install -r requirements.txt coverage + - name: Install Playwright + run: playwright install --with-deps chromium - name: Collect static files run: python manage.py collectstatic --noinput --verbosity 0 - name: Run integration tests