diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..9b6eb8a --- /dev/null +++ b/.flake8 @@ -0,0 +1,14 @@ +[flake8] +# Lint for real defects — pyflakes (unused imports/vars, undefined names, f-string +# bugs) and syntax errors (E9) — rather than fighting a formatter over whitespace +# and line length. Run: make lint +select = F,E9 +exclude = + .git, + __pycache__, + build, + dist, + site, + .hypothesis, + .pytest_cache, + *.egg-info diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..d2cd94b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,24 @@ +--- +name: Bug report +about: Something isn't working as documented +title: "[Bug] " +labels: bug +--- + +**What happened** +A clear description of the bug. + +**Minimal reproduction** +The smallest code that triggers it, plus the shape/dtypes of the data it ran on: + +```python +# ... +``` + +**Expected vs. actual behavior** + +**Environment** +- philanthropy version: +- Python version: +- scikit-learn version: +- OS: diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..a35437e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: true +contact_links: + - name: 📖 Documentation + url: https://PhilanthroPy-Project.github.io/PhilanthroPy/ + about: Check the docs first — many questions are answered there. + - name: 🔗 UniSchema (event source) + url: https://github.com/PhilanthroPy-Project/UniSchema + about: Issues about the ConstituentEvent stream itself belong in UniSchema. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..5e8da38 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,17 @@ +--- +name: Feature request +about: Suggest an estimator, transformer, metric, or improvement +title: "[Feature] " +labels: enhancement +--- + +**Problem / use case** +Which advancement or fundraising workflow does this serve? + +**Proposed API** +Sketch the class or function. For estimators/transformers, note the intended +sklearn-compatible signature (`fit`/`transform`/`predict_*`). + +**Alternatives considered** + +**Would you be willing to open a PR?** diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..d791017 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,9 @@ +## What & why + + + +## Checklist +- [ ] `make ci` passes locally (lint → collection → tests → coverage ≥ 85%) +- [ ] New/changed public API has docstrings and is exported in the subpackage `__init__.py` +- [ ] Tests added or updated +- [ ] `CHANGELOG.md` updated under `[Unreleased]` diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a82469a..c5824fa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} @@ -20,6 +20,8 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install package with dev deps run: pip install -e ".[dev]" + - name: Lint (flake8 — real defects) + run: python -m flake8 philanthropy tests examples - name: Check for import/collection errors (fast gate) run: | python -m pytest tests/ --collect-only -q || { diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9a5cfbd..2e8bf7e 100755 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -51,6 +51,6 @@ jobs: - name: Install dependencies run: | pip install pytest pytest-cov hypothesis scikit-learn pandas numpy - pip install -e . + pip install -e ".[viz]" - name: Run all tests with coverage run: pytest tests/ --cov=philanthropy --cov-fail-under=85 diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f316fb..80ea15c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ All notable changes to PhilanthroPy are documented here. Format: [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) ## [Unreleased] +### Added +- `constituent_events_to_features` carries `first_name` / `last_name` through to + the donor feature table when the UniSchema feed supplies them (guarded; null + when absent). +- Community health files: `.github` issue/PR templates, + `CODE_OF_CONDUCT.md` (Contributor Covenant 2.1), and `SECURITY.md`. +- flake8 lint gate — `.flake8` (enforces pyflakes `F` + syntax `E9` defects), + a `make lint` target folded into `make ci`, and a CI step. + +### Fixed +- Cleared 31 real-defect lint violations (unused imports/variables) across the + package and tests, including two dead code blocks. ## [0.4.0] - 2026-07-18 ### Added diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..134ba9a --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,131 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall + community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or advances of + any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email address, + without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official email address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +**shivamlalakiya151299@gmail.com**. All complaints will be reviewed and +investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of +actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or permanent +ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the +community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. + +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][mozilla]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[mozilla]: https://github.com/mozilla/diversity + +For answers to common questions about this code of conduct, see the FAQ at +[https://www.contributor-covenant.org/faq][faq]. + +[faq]: https://www.contributor-covenant.org/faq diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d8e4112..605cb59 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,7 +1,8 @@ # Contributing to PhilanthroPy Thanks for helping improve PhilanthroPy! This guide covers the local checks -every change must pass before it reaches CI. +every change must pass before it reaches CI. By participating you agree to abide +by our [Code of Conduct](CODE_OF_CONDUCT.md). ## Setup diff --git a/Makefile b/Makefile index cc750ab..7c4efcc 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,8 @@ -.PHONY: check test coverage ci +.PHONY: lint check test coverage ci + +lint: + @echo "==> Linting (flake8 — real defects)..." + python -m flake8 philanthropy tests examples check: @echo "==> Checking for collection errors..." @@ -14,5 +18,5 @@ coverage: test @echo "==> Checking coverage..." python -m pytest tests/ --cov=philanthropy --cov-fail-under=85 --cov-report=term-missing -ci: coverage +ci: lint coverage @echo "==> All CI checks passed locally." diff --git a/README.md b/README.md index a7628e0..ea02f99 100755 --- a/README.md +++ b/README.md @@ -101,6 +101,7 @@ features["affinity_score"] = model.predict_affinity_score(X) | Column | Built from | |---|---| +| `constituent_email`, `first_name`, `last_name` | identity fields, carried through when present | | `total_gift_amount`, `gift_count`, `first_gift_date`, `last_gift_date` | `DONATION` events | | `event_attendance_count` | `EVENT_REGISTRATION` events | | `email_click_count` | `EMAIL_CLICK` events | diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..fc3dfb0 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,18 @@ +# Security Policy + +## Supported versions + +Security fixes land on the latest `0.4.x` release. + +| Version | Supported | +| ------- | ------------------ | +| 0.4.x | :white_check_mark: | +| < 0.4 | :x: | + +## Reporting a vulnerability + +Please **do not open a public issue** for security problems. + +Email **shivamlalakiya151299@gmail.com** with a description and, if possible, a +minimal reproduction. You can expect an acknowledgement within a few days and a +coordinated fix and disclosure. diff --git a/philanthropy/experimental/_lapse.py b/philanthropy/experimental/_lapse.py index eb6c264..ccac7af 100755 --- a/philanthropy/experimental/_lapse.py +++ b/philanthropy/experimental/_lapse.py @@ -1,4 +1,3 @@ -import numpy as np from sklearn.base import BaseEstimator, ClassifierMixin from sklearn.ensemble import RandomForestClassifier from sklearn.utils.validation import check_X_y, check_is_fitted, check_array diff --git a/philanthropy/ingest/_constituent_events.py b/philanthropy/ingest/_constituent_events.py index 28eddc8..5ad0f4c 100644 --- a/philanthropy/ingest/_constituent_events.py +++ b/philanthropy/ingest/_constituent_events.py @@ -43,6 +43,8 @@ # so an empty batch still yields a correctly-typed, downstream-safe frame. _FEATURE_DTYPES: "dict[str, object]" = { "constituent_email": "object", + "first_name": "object", + "last_name": "object", "total_gift_amount": "float64", "gift_count": "int64", "event_attendance_count": "int64", @@ -70,8 +72,9 @@ def constituent_events_to_features( events : iterable of mapping, or DataFrame Records following UniSchema's ``ConstituentEvent`` schema. Each event carries at least ``constituentEmail``, ``eventType``, ``sourceSystem``, - and ``createdAt`` (ISO-8601); ``amount``, ``externalConstituentId``, and - ``eventId`` are optional. Accepts the output of + and ``createdAt`` (ISO-8601); ``amount``, ``externalConstituentId``, + ``eventId``, ``firstName``, and ``lastName`` are optional. Accepts the + output of :func:`read_constituent_events`, a list of dicts, or a DataFrame of the same fields. reference_date : str or datetime-like, optional @@ -89,8 +92,9 @@ def constituent_events_to_features( features : pandas.DataFrame One row per constituent, indexed by ``constituent_id`` (the ``externalConstituentId`` when present, else ``constituentEmail``), with - the columns declared in ``_FEATURE_DTYPES``. Rows are sorted by - ``constituent_id`` for determinism. + the columns declared in ``_FEATURE_DTYPES`` (``first_name`` / + ``last_name`` are populated from the feed when available, else null). + Rows are sorted by ``constituent_id`` for determinism. Warns ----- @@ -173,6 +177,12 @@ def constituent_events_to_features( grouped = df.groupby("_constituent_id", sort=True) out = pd.DataFrame(index=grouped.size().index) out["constituent_email"] = grouped["constituentEmail"].first() + # Optional identity fields — carried through when the feed supplies them + # (the output is a donor-level table, not a de-identified feature store, so + # it already holds constituent_email). ``.first()`` skips nulls, so a donor + # whose name rode in on only some events still resolves. Absent column -> None. + for out_col, src in (("first_name", "firstName"), ("last_name", "lastName")): + out[out_col] = grouped[src].first() if src in df.columns else None out["total_gift_amount"] = grouped["_gift_amount"].sum() out["gift_count"] = grouped["_is_donation"].sum() out["event_attendance_count"] = grouped["_is_event"].sum() @@ -203,8 +213,11 @@ def read_constituent_events( * a single ``.json`` file holding one event (object) or many (array); * a ``.ndjson`` / ``.jsonl`` batch, one event per line; - * a directory, in which case every ``*.json``, ``*.ndjson``, and ``*.jsonl`` - file directly inside it is read and concatenated (sorted by name). + * a directory, which is walked **recursively** — every ``*.json``, + ``*.ndjson``, and ``*.jsonl`` file at any depth is read and concatenated, + sorted by relative path. This handles UniSchema's date-partitioned egress + (``{prefix}/{vendor}/{yyyy}/{mm}/{dd}/{eventId}.json``); a flat directory + still works too. ``*.manifest.json`` batch sidecars are skipped. Parameters ---------- @@ -218,10 +231,22 @@ def read_constituent_events( """ p = Path(path) if p.is_dir(): + # UniSchema's local egress is date-partitioned — it writes each event to + # {prefix}/{vendor}/{yyyy}/{mm}/{dd}/{eventId}.json (see UniSchema + # src/egress/objectKey.ts), so the files sit several levels down and a + # non-recursive scan of the top dir finds nothing. Walk the whole tree. events: "list[dict]" = [] - for child in sorted(p.iterdir()): - if child.suffix.lower() in {".json", ".ndjson", ".jsonl"}: - events.extend(_read_events_file(child)) + files = [ + f + for f in p.rglob("*") + if f.is_file() + and f.suffix.lower() in {".json", ".ndjson", ".jsonl"} + # Skip S3 batch sidecars — batch metadata, not ConstituentEvents. + and not f.name.lower().endswith(".manifest.json") + ] + # Sort by relative path so ordering is deterministic across platforms. + for child in sorted(files, key=lambda f: f.relative_to(p).as_posix()): + events.extend(_read_events_file(child)) return events return _read_events_file(p) diff --git a/philanthropy/model_selection/__init__.py b/philanthropy/model_selection/__init__.py index 8ecbc5f..1b47229 100755 --- a/philanthropy/model_selection/__init__.py +++ b/philanthropy/model_selection/__init__.py @@ -35,8 +35,6 @@ from __future__ import annotations -from typing import Optional - import numpy as np from sklearn.model_selection import BaseCrossValidator from sklearn.utils.validation import column_or_1d diff --git a/philanthropy/models/_moves.py b/philanthropy/models/_moves.py index 68f6dc9..892f873 100755 --- a/philanthropy/models/_moves.py +++ b/philanthropy/models/_moves.py @@ -1,7 +1,6 @@ from __future__ import annotations import numpy as np -import pandas as pd from sklearn.base import ClassifierMixin, BaseEstimator from sklearn.utils.validation import check_is_fitted, validate_data from sklearn.preprocessing import LabelEncoder diff --git a/philanthropy/models/_wallet.py b/philanthropy/models/_wallet.py index 9dc8eb0..904cbdd 100755 --- a/philanthropy/models/_wallet.py +++ b/philanthropy/models/_wallet.py @@ -48,7 +48,7 @@ import numpy as np from sklearn.base import BaseEstimator, RegressorMixin from sklearn.ensemble import HistGradientBoostingRegressor -from sklearn.utils.validation import check_array, check_is_fitted, check_X_y, validate_data +from sklearn.utils.validation import check_is_fitted, validate_data class ShareOfWalletRegressor(RegressorMixin, BaseEstimator): diff --git a/philanthropy/preprocessing/_encounter_recency.py b/philanthropy/preprocessing/_encounter_recency.py index 9eed255..c6dea21 100755 --- a/philanthropy/preprocessing/_encounter_recency.py +++ b/philanthropy/preprocessing/_encounter_recency.py @@ -359,7 +359,6 @@ def transform(self, X, y=None) -> np.ndarray: UserWarning, ) n = len(df) - p = f"{prefix}__" if prefix else "" parsed = pd.Series(pd.NaT, index=df.index) parsed = self._parse_dates(pd.Series([None] * n)) diff --git a/philanthropy/preprocessing/_encounters.py b/philanthropy/preprocessing/_encounters.py index 2b57f39..4f999a3 100755 --- a/philanthropy/preprocessing/_encounters.py +++ b/philanthropy/preprocessing/_encounters.py @@ -43,7 +43,7 @@ from __future__ import annotations import warnings -from typing import List, Optional +from typing import List import numpy as np import pandas as pd @@ -270,13 +270,6 @@ def fit(self, X: pd.DataFrame, y=None) -> "EncounterTransformer": self._validate_encounter_df(raw_enc) - if hasattr(X, "columns"): - input_cols = list(X.columns) - else: - # Although X must be a DataFrame based on _validate_X, we handle ndarray gracefully - n_cols = np.shape(X)[1] if len(np.shape(X)) > 1 else 1 - input_cols = [f"x{i}" for i in range(n_cols)] - self._validate_X(X) X = validate_data(self, X, dtype=None, ensure_all_finite="allow-nan", reset=True) self.n_features_in_ = X.shape[1] diff --git a/philanthropy/preprocessing/_rfm.py b/philanthropy/preprocessing/_rfm.py index eb310c4..1fa3e18 100755 --- a/philanthropy/preprocessing/_rfm.py +++ b/philanthropy/preprocessing/_rfm.py @@ -1,7 +1,7 @@ import numpy as np import pandas as pd from sklearn.base import TransformerMixin, BaseEstimator -from sklearn.utils.validation import check_is_fitted, validate_data +from sklearn.utils.validation import check_is_fitted class RFMTransformer(TransformerMixin, BaseEstimator): """ diff --git a/philanthropy/preprocessing/_wealth.py b/philanthropy/preprocessing/_wealth.py index 45afee6..d9a1a40 100755 --- a/philanthropy/preprocessing/_wealth.py +++ b/philanthropy/preprocessing/_wealth.py @@ -36,10 +36,9 @@ from __future__ import annotations -from typing import Dict, List, Literal, Optional +from typing import List, Literal, Optional import numpy as np -import pandas as pd from sklearn.base import TransformerMixin, BaseEstimator from sklearn.utils.validation import check_is_fitted, validate_data diff --git a/philanthropy/preprocessing/_wealth_percentile.py b/philanthropy/preprocessing/_wealth_percentile.py index 7a7c736..7dede32 100755 --- a/philanthropy/preprocessing/_wealth_percentile.py +++ b/philanthropy/preprocessing/_wealth_percentile.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import numpy as np import pandas as pd from sklearn.base import TransformerMixin, BaseEstimator diff --git a/philanthropy/preprocessing/transformers.py b/philanthropy/preprocessing/transformers.py index 69124be..19d092d 100755 --- a/philanthropy/preprocessing/transformers.py +++ b/philanthropy/preprocessing/transformers.py @@ -15,8 +15,7 @@ from __future__ import annotations -import warnings -from typing import Optional, Any +from typing import Any import numpy as np import pandas as pd diff --git a/philanthropy/visualisation/_plots.py b/philanthropy/visualisation/_plots.py index 9fdb3d7..4ebdc8b 100755 --- a/philanthropy/visualisation/_plots.py +++ b/philanthropy/visualisation/_plots.py @@ -1,8 +1,19 @@ -import matplotlib.pyplot as plt -import seaborn as sns +from __future__ import annotations + +from typing import TYPE_CHECKING + import pandas as pd -def plot_affinity_distribution(scores, labels=None) -> plt.Axes: +if TYPE_CHECKING: # only for the return-type hints; never imported at runtime + import matplotlib.axes + +# matplotlib and seaborn are an optional extra (`pip install philanthropy[viz]`), +# imported lazily inside each plotting function so importing the package — or +# building a Pipeline — never requires the plotting stack. `from __future__ import +# annotations` keeps the return hints from evaluating at import time. + + +def plot_affinity_distribution(scores, labels=None) -> matplotlib.axes.Axes: """ Plots a Seaborn KDE or histogram of the 0-100 affinity scores. If actual major-gift labels are provided, plots overlaid distributions @@ -20,6 +31,9 @@ def plot_affinity_distribution(scores, labels=None) -> plt.Axes: matplotlib.axes.Axes The underlying axes object for further customization. """ + import matplotlib.pyplot as plt + import seaborn as sns + fig, ax = plt.subplots(figsize=(8, 5)) if labels is not None: @@ -51,7 +65,7 @@ def plot_affinity_distribution(scores, labels=None) -> plt.Axes: return ax -def plot_retention_waterfall(starting_donors, acquired, lapsed, recovered) -> plt.Axes: +def plot_retention_waterfall(starting_donors, acquired, lapsed, recovered) -> matplotlib.axes.Axes: """ Generates a step-by-step waterfall chart showing the net change in the donor file year-over-year. @@ -72,6 +86,8 @@ def plot_retention_waterfall(starting_donors, acquired, lapsed, recovered) -> pl matplotlib.axes.Axes The underlying axes object for further customization. """ + import matplotlib.pyplot as plt + categories = ['Starting', 'Acquired', 'Lapsed', 'Recovered', 'Ending'] # Lapsed acts as a negative flow @@ -90,7 +106,7 @@ def plot_retention_waterfall(starting_donors, acquired, lapsed, recovered) -> pl ] fig, ax = plt.subplots(figsize=(10, 6)) - bars = ax.bar(categories, values, bottom=bottoms, color=colors, edgecolor='black') + ax.bar(categories, values, bottom=bottoms, color=colors, edgecolor='black') ax.set_title("Donor Retention Waterfall") ax.set_ylabel("Number of Donors") diff --git a/pyproject.toml b/pyproject.toml index e7241ba..6d69019 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,17 +42,21 @@ classifiers = [ dependencies = [ "scikit-learn>=1.6", "pandas", - "numpy", - "matplotlib", - "seaborn" + "numpy" ] [project.optional-dependencies] +viz = [ + "matplotlib", + "seaborn" +] dev = [ "pytest", "pytest-cov", "hypothesis", - "flake8" + "flake8", + "matplotlib", + "seaborn" ] docs = [ "mkdocs>=1.4.0", diff --git a/tests/test_datasets.py b/tests/test_datasets.py index 7710038..d7e1780 100755 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -4,9 +4,7 @@ Unit tests for philanthropy.datasets.generate_synthetic_donor_data. """ -import numpy as np import pandas as pd -import pytest from philanthropy.datasets import generate_synthetic_donor_data diff --git a/tests/test_donor_propensity_model.py b/tests/test_donor_propensity_model.py index fbd2c81..2f795f2 100755 --- a/tests/test_donor_propensity_model.py +++ b/tests/test_donor_propensity_model.py @@ -248,7 +248,6 @@ def test_set_params_round_trip(): # --------------------------------------------------------------------------- def test_accepts_pandas_dataframe(): - import pandas as pd df = generate_synthetic_donor_data(n_samples=100, random_state=20) feature_cols = ["total_gift_amount", "years_active", "event_attendance_count"] X_df = df[feature_cols] diff --git a/tests/test_estimators.py b/tests/test_estimators.py index 0e00379..11d8e92 100755 --- a/tests/test_estimators.py +++ b/tests/test_estimators.py @@ -1,4 +1,3 @@ -import pytest from sklearn.utils.estimator_checks import check_estimator from philanthropy.models import MajorGiftClassifier, LapsePredictor import numpy as np diff --git a/tests/test_ingest.py b/tests/test_ingest.py index 259105f..54fb608 100644 --- a/tests/test_ingest.py +++ b/tests/test_ingest.py @@ -18,7 +18,7 @@ def _event(email, etype, created, *, amount=None, source="GIVECAMPUS", - external=None, event_id=None): + external=None, event_id=None, first=None, last=None): ev = { "constituentEmail": email, "eventType": etype, @@ -31,6 +31,10 @@ def _event(email, etype, created, *, amount=None, source="GIVECAMPUS", ev["externalConstituentId"] = external if event_id is not None: ev["eventId"] = event_id + if first is not None: + ev["firstName"] = first + if last is not None: + ev["lastName"] = last return ev @@ -227,6 +231,26 @@ def test_parsing_emits_no_warning(): constituent_events_to_features(events) +def test_carries_first_last_name_when_present(): + events = [ + _event("ada@uni.edu", "DONATION", "2025-01-01T00:00:00Z", amount=100, + first="Ada", last="Lovelace"), + # A later event without names must not overwrite the resolved name. + _event("ada@uni.edu", "EVENT_REGISTRATION", "2025-02-01T00:00:00Z"), + ] + feats = constituent_events_to_features(events) + assert feats.loc["ada@uni.edu", "first_name"] == "Ada" + assert feats.loc["ada@uni.edu", "last_name"] == "Lovelace" + + +def test_names_absent_yields_null_column_not_crash(): + events = [_event("a@x.edu", "DONATION", "2025-01-01T00:00:00Z", amount=10)] + feats = constituent_events_to_features(events) + assert "first_name" in feats.columns and "last_name" in feats.columns + assert pd.isna(feats.loc["a@x.edu", "first_name"]) + assert pd.isna(feats.loc["a@x.edu", "last_name"]) + + def test_mixed_currency_warns(): events = [ _event("a@x.edu", "DONATION", "2025-01-01T00:00:00Z", amount=100), @@ -303,6 +327,63 @@ def test_read_empty_file(tmp_path): assert read_constituent_events(f) == [] +def _write_partitioned(root, vendor, day, event_id, ev): + # Mirror UniSchema's {prefix}/{vendor}/{yyyy}/{mm}/{dd}/{eventId}.json layout. + d = root / vendor / "2026" / "07" / day + d.mkdir(parents=True, exist_ok=True) + (d / f"{event_id}.json").write_text(json.dumps(ev)) + + +def test_read_recurses_into_date_partitioned_egress(tmp_path): + # UniSchema egress: events live several levels down, none in the top dir. + _write_partitioned(tmp_path, "givecampus", "19", "e1", + _event("a@x.edu", "DONATION", "2026-07-19T00:00:00Z", amount=10)) + _write_partitioned(tmp_path, "npsp", "20", "e2", + _event("b@x.edu", "DONATION", "2026-07-20T00:00:00Z", amount=20)) + events = read_constituent_events(tmp_path) + assert len(events) == 2 + feats = constituent_events_to_features(events) + assert set(feats.index) == {"a@x.edu", "b@x.edu"} + + +def test_read_skips_manifest_sidecars(tmp_path): + _write_partitioned(tmp_path, "npsp", "19", "e1", + _event("a@x.edu", "DONATION", "2026-07-19T00:00:00Z", amount=10)) + # A batch .manifest.json sidecar must be ignored (metadata, not an event). + (tmp_path / "npsp" / "2026" / "07" / "batch.manifest.json").write_text( + json.dumps({"batchId": "b1", "count": 1})) + events = read_constituent_events(tmp_path) + assert events == [_event("a@x.edu", "DONATION", "2026-07-19T00:00:00Z", amount=10)] + + +def test_read_mixes_nested_json_and_ndjson(tmp_path): + _write_partitioned(tmp_path, "givecampus", "19", "e1", + _event("a@x.edu", "DONATION", "2026-07-19T00:00:00Z", amount=10)) + nd = tmp_path / "slate" / "2026" / "07" / "20" + nd.mkdir(parents=True, exist_ok=True) + (nd / "batch.ndjson").write_text("\n".join(json.dumps(e) for e in [ + _event("b@x.edu", "DONATION", "2026-07-20T00:00:00Z", amount=20), + _event("c@x.edu", "EMAIL_CLICK", "2026-07-20T01:00:00Z"), + ]) + "\n") + events = read_constituent_events(tmp_path) + assert len(events) == 3 + assert {e["constituentEmail"] for e in events} == {"a@x.edu", "b@x.edu", "c@x.edu"} + + +def test_read_directory_ordering_is_deterministic(tmp_path): + for vendor, day, eid, email in [ + ("slate", "21", "e3", "c@x.edu"), + ("givecampus", "19", "e1", "a@x.edu"), + ("npsp", "20", "e2", "b@x.edu"), + ]: + _write_partitioned(tmp_path, vendor, day, eid, + _event(email, "DONATION", "2026-07-19T00:00:00Z", amount=1)) + order = [e["constituentEmail"] for e in read_constituent_events(tmp_path)] + # Sorted by relative path: givecampus < npsp < slate. + assert order == ["a@x.edu", "b@x.edu", "c@x.edu"] + assert order == [e["constituentEmail"] for e in read_constituent_events(tmp_path)] + + # --------------------------------------------------------------------------- # # Integration: bridge output flows into an estimator # --------------------------------------------------------------------------- # diff --git a/tests/test_leakage.py b/tests/test_leakage.py index e552432..a50c717 100755 --- a/tests/test_leakage.py +++ b/tests/test_leakage.py @@ -31,8 +31,6 @@ from __future__ import annotations -import warnings - import numpy as np import pandas as pd import pytest diff --git a/tests/test_preprocessing.py b/tests/test_preprocessing.py index 16f19a9..24035b7 100755 --- a/tests/test_preprocessing.py +++ b/tests/test_preprocessing.py @@ -40,8 +40,6 @@ try: from hypothesis import HealthCheck, given, settings from hypothesis import strategies as st - from hypothesis.extra.pandas import column, data_frames - import hypothesis.extra.pandas as hpd _HYPOTHESIS_AVAILABLE = True except ImportError: diff --git a/tests/test_preprocessing_properties.py b/tests/test_preprocessing_properties.py index b2e0e4a..76e74de 100755 --- a/tests/test_preprocessing_properties.py +++ b/tests/test_preprocessing_properties.py @@ -6,7 +6,6 @@ import numpy as np import pandas as pd -import pytest from hypothesis import given, strategies as st, settings, HealthCheck from philanthropy.preprocessing import ( FiscalYearTransformer, diff --git a/tests/test_propensity.py b/tests/test_propensity.py index cad6a7e..f75562c 100755 --- a/tests/test_propensity.py +++ b/tests/test_propensity.py @@ -3,7 +3,6 @@ """ import numpy as np -import pandas as pd import pytest from sklearn.base import clone from sklearn.exceptions import NotFittedError diff --git a/tests/test_properties.py b/tests/test_properties.py index 9941e76..2a68b09 100755 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -1,6 +1,6 @@ import numpy as np import pandas as pd -from hypothesis import given, settings, assume, strategies as st +from hypothesis import given, settings, strategies as st from hypothesis.extra.pandas import data_frames, column, range_indexes from philanthropy.preprocessing import ( FiscalYearTransformer, diff --git a/tests/test_sklearn_compat.py b/tests/test_sklearn_compat.py index 0df3975..78825e4 100755 --- a/tests/test_sklearn_compat.py +++ b/tests/test_sklearn_compat.py @@ -1,4 +1,3 @@ -import pytest from sklearn.utils.estimator_checks import parametrize_with_checks from philanthropy.preprocessing import ( EncounterRecencyTransformer, @@ -7,7 +6,6 @@ CRMCleaner, FiscalYearTransformer ) -from philanthropy.model_selection import FiscalYearGroupedSplitter ALL_ESTIMATORS = [ EncounterRecencyTransformer(), diff --git a/tests/test_sklearn_compliance.py b/tests/test_sklearn_compliance.py index a0b39e5..d06dfb1 100755 --- a/tests/test_sklearn_compliance.py +++ b/tests/test_sklearn_compliance.py @@ -22,7 +22,6 @@ from philanthropy.datasets import generate_synthetic_donor_data from philanthropy.models import ( DonorPropensityModel, - LapsePredictor, MajorGiftClassifier, ShareOfWalletRegressor, PlannedGivingIntentScorer, @@ -34,7 +33,6 @@ PlannedGivingSignalTransformer, RFMTransformer, DischargeToSolicitationWindowTransformer, - WealthPercentileTransformer, WealthScreeningImputer, ) @@ -222,7 +220,6 @@ def test_donor_propensity_model_in_pipeline_with_5fold_cv(): def test_wealth_imputer_in_pipeline_does_not_contaminate_folds(): """In 5-fold CV, fill stats must NOT be identical across all folds.""" - from sklearn.linear_model import LogisticRegression from sklearn.model_selection import KFold rng = np.random.default_rng(42) @@ -234,15 +231,6 @@ def test_wealth_imputer_in_pipeline_does_not_contaminate_folds(): }) y = rng.integers(0, 2, n) - pipe = Pipeline([ - ("imp", WealthScreeningImputer( - wealth_cols=["estimated_net_worth"], - strategy="median", - add_indicator=False, - )), - ("clf", LogisticRegression(max_iter=200)), - ]) - fill_values = [] kf = KFold(n_splits=5, shuffle=True, random_state=42) for train_idx, _ in kf.split(X, y): diff --git a/tests/test_transformers_property.py b/tests/test_transformers_property.py index 5214e01..f0e1971 100755 --- a/tests/test_transformers_property.py +++ b/tests/test_transformers_property.py @@ -1,5 +1,4 @@ import numpy as np -import pytest import pandas as pd from hypothesis import given, settings, HealthCheck from hypothesis import strategies as st diff --git a/tests/test_visualisation.py b/tests/test_visualisation.py index b8c3458..90e04d0 100755 --- a/tests/test_visualisation.py +++ b/tests/test_visualisation.py @@ -3,12 +3,18 @@ Headless test suite for philanthropy.visualisation. """ +import pytest + +# Visualisation is an optional extra (`pip install philanthropy[viz]`); skip this +# whole module rather than error-collect when matplotlib/seaborn aren't installed. +pytest.importorskip("matplotlib") +pytest.importorskip("seaborn") + import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt import numpy as np -import pytest from philanthropy.visualisation import plot_affinity_distribution, plot_retention_waterfall