diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 2e09ee7..5708c76 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -40,7 +40,7 @@ jobs: # the assertion below fails loudly instead of leaving a confusing pytest crash. uv venv .venv --python cpython-${{ matrix.python }} uv run --python .venv --no-project python -c "import sysconfig, sys; sys.exit('picked the free-threaded interpreter for .venv; expected the GIL build' if sysconfig.get_config_var('Py_GIL_DISABLED') else 0)" - uv pip install --python .venv dist/*.whl pytest + uv pip install --python .venv dist/*.whl pytest pandas - name: pytest shell: bash working-directory: crates/fec-py @@ -72,3 +72,51 @@ jobs: with: name: wheel-${{ matrix.os }} path: crates/fec-py/dist/*.whl + + test-python-freethreaded: + # D8 canary: build against the free-threaded interpreter and run the suite. + # Non-blocking on purpose — no free-threaded wheel ships until abi3t exists + # (see build-python-bindings.yml, which stays 3.11/3.14 GIL-only). + runs-on: ubuntu-latest + continue-on-error: true + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - run: rustup toolchain install stable --profile minimal + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + workspaces: ". -> target" + - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0 + - name: Build against 3.14t + shell: bash + working-directory: crates/fec-py + run: | + uv venv .venv --python cpython-3.14t + uv run --python .venv --no-project python -c "import sysconfig, sys; sys.exit(0 if sysconfig.get_config_var('Py_GIL_DISABLED') else 'expected the free-threaded interpreter')" + # The crate is abi3-py311 (Cargo.toml), but the limited API doesn't + # exist on free-threaded builds, so pyo3-build-config treats abi3 as + # a no-op there and maturin warns "abi3 does not yet support CPython + # 3.14t ... build artifacts will be version-specific" (verified + # locally 2026-09-18). The result is a version-specific cp314-cp314t + # wheel, not an abi3 one — expected, and fine since this job never + # ships a wheel. + uvx maturin@1.15.0 build --release --out dist -i .venv/bin/python + - name: Install + import with warnings as errors + shell: bash + working-directory: crates/fec-py + run: | + # A module that hasn't declared itself free-threading-safe makes 3.14t + # re-enable the GIL and emit a RuntimeWarning; treat that as a failure + # of this canary. Verified locally 2026-09-18: the module imports + # cleanly under `-W error::RuntimeWarning` on 3.14t as-is (pyo3 >= + # 0.28 defaults modules to GIL-not-used), so `#[pymodule(gil_used = + # false)]` is NOT added — it would be a no-op here. + uv pip install --python .venv dist/*.whl pytest + uv run --python .venv --no-project python -W error::RuntimeWarning -c "import libfec_parser; print(libfec_parser.__version__)" + # pandas does ship a 3.14t wheel (verified locally 2026-09-18: `uv pip + # install pandas` resolves prebuilt numpy/pandas wheels, no compile), + # so no fallback/deselect is needed here. + uv pip install --python .venv pandas + - name: pytest + shell: bash + working-directory: crates/fec-py + run: uv run --python .venv --no-project pytest tests -v -rs diff --git a/Cargo.lock b/Cargo.lock index 8c7c406..431ff91 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1533,6 +1533,7 @@ version = "0.0.32" dependencies = [ "csv", "fec-parser", + "jiff", "pyo3", ] @@ -2133,6 +2134,7 @@ version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b" dependencies = [ + "jiff", "libc", "once_cell", "portable-atomic", diff --git a/benchmarks/README.md b/benchmarks/README.md index 80ca705..b3ec030 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -39,4 +39,8 @@ libfec fastfec 1805248.fec output/ Now, +## Python bindings + +`benchmarks/python/bench.py` benchmarks the `libfec_parser` Python bindings against this +same 91 MB filing; run it with `make bench` from `crates/fec-py`. diff --git a/benchmarks/python/bench.py b/benchmarks/python/bench.py new file mode 100644 index 0000000..eeeb558 --- /dev/null +++ b/benchmarks/python/bench.py @@ -0,0 +1,242 @@ +"""Benchmarks for the ``libfec_parser`` Python bindings. + +Moved and extended from the ``plans/python/probes/bench.py`` probe (Phase 2, +ticket 19): each scenario below parses the 91 MB ``1805248.fec`` benchmark +filing (gitignored; not present in a fresh checkout) and reports rows parsed, +wall time, and peak RSS. + +Every scenario runs in its **own subprocess** — ``resource.getrusage(...).ru_maxrss`` +is a process-wide high-water mark, so measuring several scenarios in one +interpreter would have each one see the high-water mark of everything before it. + +Usage (from the repo root, with the ``libfec_parser`` dev environment active):: + + uv run python benchmarks/python/bench.py + uv run python benchmarks/python/bench.py --filing /path/to/other.fec + uv run python benchmarks/python/bench.py --only open,read + +or, from ``crates/fec-py``: ``make bench``. +""" +from __future__ import annotations + +import argparse +import subprocess +import sys +import textwrap +from pathlib import Path + +DEFAULT_FILING = Path(__file__).resolve().parents[1] / "1805248.fec" + +# Every scenario body runs after this preamble, with `sys.argv[1]` set to the +# filing path. It defines `_peak_mb()` (None on Windows, where `resource` does +# not exist) and prints one line other code never needs to parse around: +# `RESULT [extra]` or `MISSING `. +_PREAMBLE = """ +import sys, time + +def _peak_mb(): + if sys.platform == "win32": + return None + import resource + r = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + # ru_maxrss is bytes on macOS, kilobytes on Linux. + return r / 1e6 if sys.platform == "darwin" else r / 1e3 + +path = sys.argv[1] +""" + +# Each scenario is (name, description, subprocess body). The body must set +# `rows` and use `t0 = time.perf_counter()` right before the timed work, then +# print the RESULT/MISSING line itself (so it controls exactly what is timed). +_SCENARIOS: list[tuple[str, str, str]] = [ + ( + "open", + "sum(1 for _ in libfec_parser.open(p))", + """ +import libfec_parser +t0 = time.perf_counter() +rows = sum(1 for _ in libfec_parser.open(path)) +dt = time.perf_counter() - t0 +print(f"RESULT {rows} {dt} {_peak_mb()}") +""", + ), + ( + "open_sb", + 'sum(1 for _ in libfec_parser.open(p).rows("SB"))', + """ +import libfec_parser +t0 = time.perf_counter() +rows = sum(1 for _ in libfec_parser.open(path).rows("SB")) +dt = time.perf_counter() - t0 +print(f"RESULT {rows} {dt} {_peak_mb()}") +""", + ), + ( + "open_bytes", + "same as open() over p.read_bytes()", + """ +import libfec_parser +data = open(path, "rb").read() +t0 = time.perf_counter() +rows = sum(1 for _ in libfec_parser.open(data)) +dt = time.perf_counter() - t0 +peak = _peak_mb() +extra = "NA" if peak is None else (peak - len(data) / 1e6) +print(f"RESULT {rows} {dt} {peak} {extra}") +""", + ), + ( + "read", + "len(libfec_parser.read(p).rows)", + """ +import libfec_parser +t0 = time.perf_counter() +rows = len(libfec_parser.read(path).rows) +dt = time.perf_counter() - t0 +print(f"RESULT {rows} {dt} {_peak_mb()}") +""", + ), + ( + "read_20x", + "read() once, then 20x len(f.rows) -- the N2 regression guard", + """ +import libfec_parser +t0 = time.perf_counter() +f = libfec_parser.read(path) +for _ in range(20): + rows = len(f.rows) +dt = time.perf_counter() - t0 +print(f"RESULT {rows} {dt} {_peak_mb()}") +""", + ), + ( + "dataframe", + "pd.DataFrame(read(p).rows).shape", + """ +try: + import pandas as pd +except ImportError: + print("MISSING pandas not installed") +else: + import libfec_parser + t0 = time.perf_counter() + df = pd.DataFrame(libfec_parser.read(path).rows) + dt = time.perf_counter() - t0 + print(f"RESULT {df.shape[0]} {dt} {_peak_mb()}") +""", + ), + ( + "compat", + "libfec_parser.fecfile.from_file(p) (unchanged until Phase 3)", + """ +from libfec_parser import fecfile +t0 = time.perf_counter() +d = fecfile.from_file(path) +rows = sum(len(v) for v in d["itemizations"].values()) +dt = time.perf_counter() - t0 +print(f"RESULT {rows} {dt} {_peak_mb()}") +""", + ), + ( + "real", + "PyPI fecfile.from_file(p)", + """ +try: + import fecfile +except ImportError: + print("MISSING fecfile not installed") +else: + t0 = time.perf_counter() + d = fecfile.from_file(path) + rows = sum(len(v) for v in d["itemizations"].values()) + dt = time.perf_counter() - t0 + print(f"RESULT {rows} {dt} {_peak_mb()}") +""", + ), + ( + "real_iter", + "PyPI fecfile.iter_file(p)", + """ +try: + import fecfile +except ImportError: + print("MISSING fecfile not installed") +else: + t0 = time.perf_counter() + rows = sum(1 for _ in fecfile.iter_file(path)) + dt = time.perf_counter() - t0 + print(f"RESULT {rows} {dt} {_peak_mb()}") +""", + ), +] + + +def _format_peak(value: str) -> str: + if value == "NA": + return "n/a" + return f"{float(value):.1f}" + + +def _run_scenario(name: str, body: str, filing: Path) -> str: + """Run one scenario's code in a fresh subprocess; format its Markdown row.""" + code = _PREAMBLE + body + result = subprocess.run( + [sys.executable, "-c", textwrap.dedent(code), str(filing)], + capture_output=True, + text=True, + ) + if result.returncode != 0: + return f"| {name} | ERROR | ERROR | {result.stderr.strip().splitlines()[-1:] or 'see stderr'} |" + + line = "" + for candidate in result.stdout.splitlines(): + if candidate.startswith(("RESULT", "MISSING")): + line = candidate + if not line: + return f"| {name} | ERROR | ERROR | no RESULT/MISSING line in output |" + + if line.startswith("MISSING"): + reason = line[len("MISSING "):].strip() + return f"| {name} (n/a: {reason}) | n/a | n/a | n/a |" + + parts = line.split() + rows, seconds, peak = parts[1], parts[2], parts[3] + peak_str = _format_peak(peak) + if name == "open_bytes" and len(parts) > 4: + delta = parts[4] + peak_str = f"{peak_str} ({'+' if float(delta) >= 0 else ''}{float(delta):.1f} over bytes)" + return f"| {name} | {rows} | {float(seconds):.2f} s | {peak_str} MB |" + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--filing", type=Path, default=DEFAULT_FILING) + parser.add_argument( + "--only", + type=str, + default=None, + help="comma-separated scenario names to run (default: all)", + ) + args = parser.parse_args() + + if not args.filing.exists(): + print(f"error: filing not found: {args.filing}", file=sys.stderr) + raise SystemExit(1) + + only = set(args.only.split(",")) if args.only else None + scenarios = [s for s in _SCENARIOS if only is None or s[0] in only] + if only is not None: + missing = only - {s[0] for s in _SCENARIOS} + if missing: + print(f"error: unknown scenario(s): {', '.join(sorted(missing))}", file=sys.stderr) + raise SystemExit(1) + + print(f"Filing: {args.filing} ({args.filing.stat().st_size / 1e6:.1f} MB)\n") + print("| name | rows | seconds | peak MB |") + print("|---|---|---|---|") + for name, _description, body in scenarios: + print(_run_scenario(name, body, args.filing)) + + +if __name__ == "__main__": + main() diff --git a/crates/fec-parser/src/lib.rs b/crates/fec-parser/src/lib.rs index f0fd506..a885dac 100644 --- a/crates/fec-parser/src/lib.rs +++ b/crates/fec-parser/src/lib.rs @@ -57,7 +57,7 @@ pub struct FilingHeader { } impl FilingHeader { - fn from_record(hdr: csv::StringRecord) -> Result { + pub fn from_record(hdr: csv::StringRecord) -> Result { let record_type = header_get_field!(hdr, 0, "record_type"); let ef_type = header_get_field!(hdr, 1, "ef_type"); let fec_version = header_get_field!(hdr, 2, "fec_version").trim().to_owned(); @@ -323,10 +323,13 @@ impl Filing { /// Return the next itemization row in the filing, or None if at end of file. pub fn next_row(&mut self) -> Option> { - let (record, original_size) = match self.records_iter.next() { + let (record, original_size, line) = match self.records_iter.next() { Some(Ok(record)) => { let n = record.as_slice().len(); - (StringRecord::from_byte_record_lossy(record), n) + // `from_byte_record_lossy` drops the position when the record is not + // valid UTF-8, so read the line number off the `ByteRecord` first. + let line = record.position().map(|p| p.line()).unwrap_or(0); + (StringRecord::from_byte_record_lossy(record), n, line) } Some(Err(err)) => return Some(Err(FilingRowReadError::CsvError(err))), None => return None, @@ -354,6 +357,7 @@ impl Filing { Err(e) => return Some(Err(FilingRowReadError::CsvError(e))), }; let original_size = record.as_slice().len(); + let line = record.position().map(|p| p.line()).unwrap_or(0); let record = StringRecord::from_byte_record_lossy(record); let row_type = record .get(0) @@ -363,6 +367,7 @@ impl Filing { row_type, record, original_size, + line, })); } None => return None, @@ -385,6 +390,7 @@ impl Filing { row_type, record, original_size, + line, })) } } @@ -403,6 +409,8 @@ pub struct FilingRow { pub row_type: String, pub record: StringRecord, pub original_size: usize, + /// 1-based physical line of the row in the source file, or 0 if unknown. + pub line: u64, } #[cfg(test)] diff --git a/crates/fec-py/Cargo.toml b/crates/fec-py/Cargo.toml index b2a988e..b53d823 100644 --- a/crates/fec-py/Cargo.toml +++ b/crates/fec-py/Cargo.toml @@ -15,6 +15,9 @@ crate-type = ["cdylib"] # "abi3-py311" tells pyo3 (and maturin) to build using the stable ABI with minimum Python version 3.11. # "extension-module" is intentionally absent: maturin >= 1.9.4 sets PYO3_BUILD_EXTENSION_MODULE itself, # and leaving it off keeps `cargo test -p libfec_parser` linkable. -pyo3 = { version = "0.29", features = ["abi3-py311"] } +pyo3 = { version = "0.29", features = ["abi3-py311", "jiff-02"] } fec-parser = { path="../fec-parser" } csv = "1.3" +# The workspace already resolves jiff 0.2.x via fec-parser; `jiff-02` converts +# `jiff::civil::Date` to `datetime.date` (limited-API safe under abi3-py311). +jiff = "0.2" diff --git a/crates/fec-py/Makefile b/crates/fec-py/Makefile index 49717b9..0097902 100644 --- a/crates/fec-py/Makefile +++ b/crates/fec-py/Makefile @@ -1,4 +1,4 @@ -.PHONY: develop develop-release build build-release test test-network test-slow notebook notebook-check stubs clean +.PHONY: develop develop-release build build-release test test-network test-slow bench notebook notebook-check stubs clean # One-time: uv venv && uv sync --group dev @@ -22,6 +22,9 @@ test-network: develop test-slow: develop-release uv run pytest tests -m slow +bench: develop-release ## benchmarks/python/bench.py against the 91 MB filing + uv run python ../../benchmarks/python/bench.py + notebook: develop-release uv run --with jupyterlab jupyter lab examples/quickstart.ipynb diff --git a/crates/fec-py/README.md b/crates/fec-py/README.md index 6818100..8ef1cbd 100644 --- a/crates/fec-py/README.md +++ b/crates/fec-py/README.md @@ -1,28 +1,27 @@ # libfec_parser > **Alpha.** Wheels are published on [GitHub Releases](https://github.com/asg017/libfec/releases), -> not PyPI. The native `Filing` API will change in upcoming releases (see -> [`plans/python/`](../../plans/python/)); the `fecfile` module is not yet a drop-in -> replacement. +> not PyPI. The `fecfile` module is not yet a drop-in replacement for the +> [`fecfile`](https://pypi.org/project/fecfile/) package (Phase 3). Python bindings for [libfec](https://github.com/asg017/libfec)'s `.fec` parser. Parse FEC electronic filings from a path, from bytes, or straight from the FEC's website, with the parsing done in Rust. ```python -from libfec_parser import fecfile - -filing = fecfile.from_file("1721696.fec") +import libfec_parser -filing["filing"]["committee_name"] # 'PFIZER INC. PAC' -filing["filing"]["col_a_total_receipts"] # '83741.93' - -for row in filing["itemizations"]["Schedule A"]: - print(row["contributor_last_name"], row["contribution_amount"]) +with libfec_parser.open("1721696.fec") as filing: + filing.cover.filer_name # 'PFIZER INC. PAC' + filing.cover_row["col_a_total_receipts"] # 83741.93 + for row in filing.rows("SA"): + print(row["contributor_last_name"], row["contribution_amount"], row["contribution_date"]) ``` Two APIs are included: -- [`libfec_parser.fecfile`](#fecfile-api): rows as dicts keyed by column name, modeled on the [`fecfile`](https://pypi.org/project/fecfile/) package. -- [`libfec_parser.parser`](#native-api): a lower-level `Filing` class with positional fields. +- [`libfec_parser.parser`](#native-api) (also exported at the top level as `open`/`read`): the + primary API — a streaming `FilingReader` or an eager `Filing`, both with typed values. +- [`libfec_parser.fecfile`](#fecfile-api): rows as dicts keyed by column name, modeled on the + [`fecfile`](https://pypi.org/project/fecfile/) package; strings only until Phase 3. For a guided tour, including loading a filing into pandas, see [`examples/quickstart.ipynb`](examples/quickstart.ipynb). @@ -124,39 +123,95 @@ row = fecfile.parse_line(line, version) ### Differences from `fecfile` -- **Every value is a string.** Amounts are not converted to `float`, and dates stay as `YYYYMMDD`. The `as_strings` option is accepted and ignored. +- **Every value is a string.** Amounts are not converted to `float`, and dates stay as `YYYYMMDD`. The `as_strings` option is accepted and ignored — the native `Row` API returns typed values. - Only the ASCII 28-delimited format (FEC version 6 and later) is supported by `parse_header()` and `parse_line()`. - `from_http()` reads the whole response into memory before parsing, and there is no `iter_file()` / `iter_http()` yet. ## Native API ```python -from libfec_parser.parser import Filing +from libfec_parser import open, read +``` + +(`open`/`read` are also reachable as `libfec_parser.parser.open`/`.read`; `open` shadows the +builtin, so reach the real one through `builtins.open` if you need both in the same scope.) + +### `open()` vs `read()` + +`open(source)` returns a `FilingReader`: `header`, `cover` and `cover_row` are parsed eagerly, so +they're available immediately, and the itemization rows are pulled lazily as you iterate. Use it +for large filings, or when `rows(*prefixes)` lets you skip most of the file. + +`read(source)` (a thin function wrapper over the `Filing` class) parses the whole filing up front +into a `Filing`, whose `rows` is a plain `list`. `Filing(source)` is exactly `read(source)` — pick +whichever reads better. Use `read`/`Filing` when you want everything in memory at once, such as to +build a `pandas.DataFrame`. + +```python +from libfec_parser import open, read + +reader = open("1721696.fec") +next(reader).row_type # 'SA11AI' + +filing = read("1721696.fec") # same as Filing("1721696.fec") +len(filing.rows) # 1387 +``` + +### Sources + +Both `open()` and `read()` (and `Filing()` and `fec_header()`) accept the same kinds of source: + +- a filesystem path — `str` or `os.PathLike`. **A `str` is always a path**, never the filing's + contents — pass `bytes` if you already have the data in memory. +- a bytes-like object — `bytes`, `bytearray`, `memoryview`, `mmap.mmap` — read without copying. +- a binary file object: anything with a `read(n) -> bytes` method (an open file, `io.BytesIO`, an + `urlopen()` response, …), pulled a chunk at a time rather than read whole. + +A text-mode file (or anything else whose `read()` returns `str`) raises `TypeError`: + +```python +import io +import libfec_parser + +with open("1721696.fec", "rb") as f: # builtin `open`, binary mode + libfec_parser.open(f) -filing = Filing("1721696.fec") -# Filing(form_type='F3XN', filer_id='C00016683', 1387 itemizations) +libfec_parser.open(io.StringIO("not bytes")) +# TypeError: file must be opened in binary mode, e.g. open(path, 'rb') ``` -`Filing(source)` accepts a path (`str`), `bytes`, or any object with a `.read()` method that returns bytes: +### `FilingReader` ```python -import urllib.request +with libfec_parser.open("1721696.fec") as filing: + filing.id # '1721696' — the file stem, or a file object's `name` + filing.fec_version # '8.4' — shortcut for filing.header.fec_version -with urllib.request.urlopen("https://docquery.fec.gov/dcdev/posted/1721696.fec") as response: - filing = Filing(response) + for row in filing.rows("SA11AI"): # filters before a Row is even built + ... ``` -The whole filing is parsed up front. A bad path raises `IOError`, an unparseable filing raises `ValueError`. +- Iterating a `FilingReader` is single-pass: once exhausted, a second `for` yields nothing. +- `rows(*prefixes)` keeps only rows whose type starts with one of `prefixes` + (case-insensitive) and returns the reader itself, so it chains into a `for`. A second call + replaces the filter; `rows()` with no arguments clears it. +- `close()` drops the source; it's idempotent, and iterating afterwards raises `ValueError`. + Use it as a context manager (as above) rather than calling it directly. +- There is no `len()` — the row count isn't known without a full pass. ### `Filing` -| Attribute | Type | | -| --- | --- | --- | -| `header` | `Header` | The `HDR` record | -| `cover` | `Cover` | The cover page | -| `itemizations` | `list[Itemization]` | Every remaining row, in file order | +```python +from libfec_parser import read + +filing = read("1721696.fec") +filing # Filing(id='1721696', form_type='F3XN', filer_id='C00016683', 1387 rows) +len(filing) # 1387, same as len(filing.rows) +list(filing)[0] is filing.rows[0] # True — iterating a Filing iterates its rows +``` -Each access to `itemizations` copies the list, so bind it to a variable rather than indexing `filing.itemizations` in a loop. +`filing.itemizations` still works as an alias for `filing.rows`, but warns with +`DeprecationWarning` — use `rows`. ### `Header` @@ -179,28 +234,127 @@ Each access to `itemizations` copies the list, so bind it to a variable rather t | `filer_id` | `str` | Committee ID | | `filer_name` | `str` | | | `report_code` | `str \| None` | Such as `"Q1"`, `"M8"`, `"YE"` | -| `coverage_from_date` | `str \| None` | ISO formatted, `"2023-07-01"` | -| `coverage_through_date` | `str \| None` | ISO formatted | +| `coverage_from_date` | `date \| None` | | +| `coverage_through_date` | `date \| None` | | + +`cover.fields()` returns those same six values as a `dict`, dates included as `datetime.date`. +For every other column on the cover page, use `cover_row` (below). + +```python +cover = filing.cover +cover.coverage_from_date # datetime.date(2023, 7, 1) +cover.fields() +# {'form_type': 'F3XN', 'filer_id': 'C00016683', 'filer_name': 'PFIZER INC. PAC', +# 'report_code': 'M8', 'coverage_from_date': datetime.date(2023, 7, 1), +# 'coverage_through_date': datetime.date(2023, 7, 31)} +``` + +### `cover_row` + +`filing.cover_row` is the cover line itself, as a full `Row` — every column the form defines, not +just the six normalized ones above: + +```python +filing.cover_row["col_a_total_receipts"] # 83741.93 (float) +len(filing.cover_row) # 123 mapped columns, for an F3XN 8.4 cover +``` + +It follows the same rules as any other `Row` (below). Note that an unparseable cover date comes +back as `None` from `Cover.coverage_from_date`, but as its raw `str` from +`cover_row["coverage_from_date"]` — the two are not reconciled. + +### `Row` + +A `Row` is a mapping from column name to a typed value, plus positional access to the raw fields. +`len(row)` is the number of *mapped columns* — `len(row.fields())` is the raw field count, which +can differ for a short or an over-long line. + +| Access | Result | +| --- | --- | +| `row["name"]`, amount/date column, parses | typed: `float` or `datetime.date` | +| `row["name"]`, amount/date column, empty | `None` | +| `row["name"]`, amount/date column, garbage (doesn't parse) | the raw `str` | +| `row["name"]`, text column | the raw `str` (`""` if empty — text never becomes `None`) | +| `row["name"]`, any column, past the end of a short row | `None` | +| `row[i]` / `row[a:b]` (by position) | always the raw `str` / `list[str]`, whatever the column | + +```python +row = filing.rows[0] +row["contribution_amount"] # 104.17 (float) +row[20] # '104.17' (str) — same field, by position +dict(row)["contributor_last_name"] # 'Aaronson' +``` + +Because a garbage value comes back as `str` in place of the expected type, guard with +`isinstance` rather than assuming every amount parsed: + +```python +value = row["contribution_amount"] +if isinstance(value, str): + ... # the parser couldn't convert it; handle the raw text explicitly +``` + +`row.extra_fields` holds any fields past the last mapped column (`[]` unless one is non-empty), +and `row.line` is the row's 1-based physical line in the file. Two `Row`s compare and hash equal +when their `(row_type, fec_version, raw fields)` match, and a `Row` pickles and unpickles cleanly. + +### Errors -`cover.fields()` returns the same six values as a dict. For the rest of the cover page, use the [`fecfile` API](#fecfile-api). +| Exception | Raised when | +| --- | --- | +| `FileNotFoundError` | the source is a path that doesn't exist (`errno`/`filename` set, like `open()`) | +| `FecParseError` (a `FecError`, a `ValueError`) | the input isn't a parseable `.fec` filing | +| `MissingMappingError` (a `FecError`) | a row's `(row_type, fec_version)` has no column mapping — has `.row_type`, `.version`, `.line` | + +`read()`/`Filing()` are strict: a missing mapping anywhere raises out of the call. `open()` raises +it from the specific `next()` that reached that row, and the reader stays usable — but only a +`while`/`next()` loop can catch it and keep going; a `for` loop can't, because the exception comes +out of the `for` statement's own call to `__next__` before your loop body ever runs: + +```python +from libfec_parser import MissingMappingError, open + +reader = open("1721696.fec") +rows = [] +while True: + try: + rows.append(next(reader)) + except StopIteration: + break + except MissingMappingError: + continue # skip this row; the reader is still usable +``` + +### `fec_header(source, /)` + +Returns just the `fec_version` of a filing, from anything `open()` accepts. It reads only the +`HDR` record — the first line — so it works even when the rest of the file (say, an unmapped +cover form) would make `open()` raise: + +```python +libfec_parser.fec_header("1721696.fec") # '8.4' +``` -### `Itemization` +### pandas -A single row: its `row_type` (such as `"SA11AI"`) plus the raw fields, in file order and without column names. +`Row` registers as a `collections.abc.Mapping`, so a list of them is exactly what `pd.DataFrame` +wants — and because values are already typed, no per-column conversion is needed: ```python -item = filing.itemizations[0] +import pandas as pd +from libfec_parser import read -item.row_type # 'SA11AI' -len(item) # 45 -item[0] # 'SA11AI' -item[-1] # negative indexes work -item.fields() # all fields as a list[str] +df = pd.DataFrame(read("1721696.fec").rows) +df["contribution_amount"].dtype # dtype('float64') ``` -### `fec_header(contents)` +Dates land as `datetime.date` objects in an `object`-dtype column — pandas does not auto-convert +`date` to `datetime64`. That's fine for most uses; if you need `datetime64` semantics (`.dt` +accessors, resampling), convert explicitly: -Takes the `bytes` of a filing and returns just its FEC format version. +```python +pd.to_datetime(df["contribution_date"]).dt.year.min() # 2023 +``` ## Known issues @@ -210,14 +364,29 @@ differs from the `fecfile` package on exactly these points: - **`[BEGINTEXT]…[ENDTEXT]` bodies are dropped.** F99 filings parse, but their free-form text never reaches `filing["text"]`. -- **`_TODO_DUP` cover column names.** A few Form 3P cover-page columns come back with - placeholder names such as `_TODO_DUP1` instead of a real column name. +- **`_TODO_DUP` cover column names.** A few Form 3X and Form 3P cover-page columns come back + with placeholder names such as `col_a_total_receipts_TODO_DUP` — this shows up in + `cover_row.keys()` (and `dict(cover_row)`) too, not just the `fecfile` API's `filing["filing"]`. - **Non-UTF-8 bytes become U+FFFD.** Filings written in cp1252 (curly quotes, en dashes) decode lossily: the offending bytes are replaced with `�` rather than transcoded. Background and the intended fixes are in [`plans/python/`](../../plans/python/) — see [`00-decisions.md`](../../plans/python/00-decisions.md), "Deferred to `fec-parser`". +## Performance + +Measured 2026-09-18 on an Apple M4 Pro, CPython 3.13, release build, parsing the 91 MB, 408,160-row +`1805248.fec` filing (gitignored; not one of the committed fixtures). Reproduce with `make bench` +([`benchmarks/python/bench.py`](../../benchmarks/python/bench.py)). + +| Operation | Time | Peak RSS | +| --- | --- | --- | +| `open()`, streamed | 0.18 s | 31.6 MB | +| `read()` (eager `Filing`) | 0.30 s | 1,007 MB | +| `pd.DataFrame(read(p).rows)` | 5.32 s | 2,389 MB | +| PyPI `fecfile.from_file` | 7.14 s | 1,403 MB | +| PyPI `fecfile.iter_file` | 6.87 s | 36.4 MB | + ## Development Build with `maturin`, not `cargo build`, which can't link a Python extension module on its own. diff --git a/crates/fec-py/examples/quickstart.ipynb b/crates/fec-py/examples/quickstart.ipynb index 7a03d88..55511d2 100644 --- a/crates/fec-py/examples/quickstart.ipynb +++ b/crates/fec-py/examples/quickstart.ipynb @@ -2,22 +2,22 @@ "cells": [ { "cell_type": "markdown", - "id": "ccc14585", + "id": "8d3013fa", "metadata": {}, "source": [ "# `libfec_parser` quickstart\n", "\n", "`libfec_parser` is the Python binding for [libfec](https://github.com/asg017/libfec)'s Rust `.fec` parser. This notebook walks through both APIs it ships with:\n", "\n", - "1. **`libfec_parser.fecfile`** — a dict-based API modeled on the [`fecfile`](https://pypi.org/project/fecfile/) package. Rows come back as dicts keyed by column name. Start here.\n", - "2. **`libfec_parser.parser`** — a lower-level `Filing` class that gives you the raw positional fields of every row.\n", + "1. **`libfec_parser.parser`** (also exported at the top level as `open`/`read`) — the primary API: a streaming `FilingReader` or an eager `Filing`, with typed values (`float`, `datetime.date`) and full pandas interop. Start here.\n", + "2. **`libfec_parser.fecfile`** — a dict-based API modeled on the [`fecfile`](https://pypi.org/project/fecfile/) package. Rows come back as dicts keyed by column name, every value a string.\n", "\n", "The package isn't on PyPI yet, so build it from source first. From `crates/fec-py/`, `make notebook` builds the wheel and opens this notebook with everything installed. See the [README](../README.md) for other options." ] }, { "cell_type": "markdown", - "id": "b4bc15e8", + "id": "7e95d85a", "metadata": {}, "source": [ "## Get a filing\n", @@ -28,13 +28,13 @@ { "cell_type": "code", "execution_count": 1, - "id": "97559d2b", + "id": "82486d41", "metadata": { "execution": { - "iopub.execute_input": "2026-09-18T21:41:00.207314Z", - "iopub.status.busy": "2026-09-18T21:41:00.207222Z", - "iopub.status.idle": "2026-09-18T21:41:00.214682Z", - "shell.execute_reply": "2026-09-18T21:41:00.214155Z" + "iopub.execute_input": "2026-09-19T02:33:04.772427Z", + "iopub.status.busy": "2026-09-19T02:33:04.772192Z", + "iopub.status.idle": "2026-09-19T02:33:04.785945Z", + "shell.execute_reply": "2026-09-19T02:33:04.785257Z" } }, "outputs": [ @@ -61,72 +61,99 @@ }, { "cell_type": "markdown", - "id": "00edaa54", + "id": "1478f756", "metadata": {}, "source": [ - "## The `fecfile` API\n", + "## The native API\n", "\n", - "`from_file()` parses a filing into a plain dict with four keys:\n", + "`libfec_parser.open()` returns a `FilingReader`: the `HDR` record and cover page are parsed eagerly, so `header`/`cover`/`cover_row` are available right away, and itemization rows are pulled lazily as you iterate — the right choice for a large filing, or when `rows(*prefixes)` lets you skip most of it. `read()` (or `Filing(source)`, the same thing) parses everything up front into a `Filing` whose `rows` is a plain `list`, which is what you want to build a `pandas.DataFrame`.\n", "\n", - "- `header` — the `HDR` record: FEC format version and the software that produced the filing\n", - "- `filing` — the cover page (form type, committee, coverage dates, summary totals)\n", - "- `itemizations` — rows grouped by schedule: `\"Schedule A\"`, `\"Schedule B\"`, …\n", - "- `text` — free-form `TEXT` records" + "Use it as a context manager, and filter to the rows you want with `rows(*prefixes)` (matched case-insensitively, before a `Row` is even built):" ] }, { "cell_type": "code", "execution_count": 2, - "id": "c4dc682d", + "id": "f9a8ddc6", "metadata": { "execution": { - "iopub.execute_input": "2026-09-18T21:41:00.215796Z", - "iopub.status.busy": "2026-09-18T21:41:00.215719Z", - "iopub.status.idle": "2026-09-18T21:41:00.362406Z", - "shell.execute_reply": "2026-09-18T21:41:00.361857Z" + "iopub.execute_input": "2026-09-19T02:33:04.787678Z", + "iopub.status.busy": "2026-09-19T02:33:04.787528Z", + "iopub.status.idle": "2026-09-19T02:33:04.802415Z", + "shell.execute_reply": "2026-09-19T02:33:04.801947Z" } }, "outputs": [ { - "data": { - "text/plain": [ - "dict_keys(['header', 'filing', 'itemizations', 'text'])" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "Header(fec_version='8.4', software_name='FECFile', software_version='8.4')\n", + "1721696 | 8.4\n", + "\n", + "Aaronson 104.17 2023-07-14\n", + "Aaronson 104.17 2023-07-31\n", + "Aarts 20.84 2023-07-14\n", + "Aarts 20.84 2023-07-31\n", + "Adams 20.0 2023-07-14\n" + ] } ], "source": [ - "from libfec_parser import fecfile\n", + "import itertools\n", + "import libfec_parser\n", "\n", - "parsed = fecfile.from_file(str(path))\n", - "parsed.keys()" + "with libfec_parser.open(str(path)) as filing:\n", + " print(filing.header)\n", + " print(filing.id, \"|\", filing.fec_version)\n", + " print()\n", + "\n", + " for row in itertools.islice(filing.rows(\"SA\"), 5):\n", + " print(row[\"contributor_last_name\"], row[\"contribution_amount\"], row[\"contribution_date\"])\n", + "\n", + " cover_row = filing.cover_row" + ] + }, + { + "cell_type": "markdown", + "id": "3306d870", + "metadata": {}, + "source": [ + "A `Row` is a mapping from column name to a *typed* value — `contribution_amount` above is a `float`, `contribution_date` a `datetime.date`, not strings to parse yourself. `dict(row)` gives every column at once (only the populated ones shown here):" ] }, { "cell_type": "code", "execution_count": 3, - "id": "2fda64ba", + "id": "6b0815bb", "metadata": { "execution": { - "iopub.execute_input": "2026-09-18T21:41:00.363554Z", - "iopub.status.busy": "2026-09-18T21:41:00.363472Z", - "iopub.status.idle": "2026-09-18T21:41:00.365475Z", - "shell.execute_reply": "2026-09-18T21:41:00.365102Z" + "iopub.execute_input": "2026-09-19T02:33:04.803950Z", + "iopub.status.busy": "2026-09-19T02:33:04.803829Z", + "iopub.status.idle": "2026-09-19T02:33:04.807119Z", + "shell.execute_reply": "2026-09-19T02:33:04.806666Z" } }, "outputs": [ { "data": { "text/plain": [ - "{'record_type': 'HDR',\n", - " 'ef_type': 'FEC',\n", - " 'fec_version': '8.4',\n", - " 'software_name': 'FECFile',\n", - " 'software_version': '8.4',\n", - " 'report_number': '0'}" + "{'form_type': 'SA11AI',\n", + " 'filer_committee_id_number': 'C00016683',\n", + " 'transaction_id': '2023071716378-3174',\n", + " 'entity_type': 'IND',\n", + " 'contributor_last_name': 'Adams',\n", + " 'contributor_first_name': 'Dorinda',\n", + " 'contributor_middle_name': 'C',\n", + " 'contributor_street_1': '66 Hudson Blvd East',\n", + " 'contributor_city': 'New York',\n", + " 'contributor_state': 'NY',\n", + " 'contributor_zip_code': '10001',\n", + " 'contribution_date': datetime.date(2023, 7, 14),\n", + " 'contribution_amount': 20.0,\n", + " 'contribution_aggregate': 280.0,\n", + " 'contributor_employer': 'Pfizer, Inc.',\n", + " 'contributor_occupation': 'National Pharmacy Business Manager'}" ] }, "execution_count": 3, @@ -135,27 +162,27 @@ } ], "source": [ - "parsed[\"header\"]" + "{k: v for k, v in dict(row).items() if v != \"\"}" ] }, { "cell_type": "markdown", - "id": "4cc54271", + "id": "2740c3da", "metadata": {}, "source": [ - "The cover page has one key per column on the form. An F3X has more than a hundred, so here are just a few:" + "`cover_row` is the full cover line as a `Row` too — every column the form defines, typed the same way. This is where the cover-page totals live (`Cover.fields()` only keeps the six normalized attributes: form type, filer, coverage dates):" ] }, { "cell_type": "code", "execution_count": 4, - "id": "0b8b8ce5", + "id": "8195d6cb", "metadata": { "execution": { - "iopub.execute_input": "2026-09-18T21:41:00.366622Z", - "iopub.status.busy": "2026-09-18T21:41:00.366563Z", - "iopub.status.idle": "2026-09-18T21:41:00.368631Z", - "shell.execute_reply": "2026-09-18T21:41:00.368203Z" + "iopub.execute_input": "2026-09-19T02:33:04.808309Z", + "iopub.status.busy": "2026-09-19T02:33:04.808234Z", + "iopub.status.idle": "2026-09-19T02:33:04.810411Z", + "shell.execute_reply": "2026-09-19T02:33:04.809954Z" } }, "outputs": [ @@ -163,25 +190,20 @@ "name": "stdout", "output_type": "stream", "text": [ - "123 cover fields\n", - "\n", - "form_type F3XN\n", - "filer_committee_id_number C00016683\n", - "committee_name PFIZER INC. PAC\n", - "report_code M8\n", - "coverage_from_date 20230701\n", - "coverage_through_date 20230731\n", + "form_type 'F3XN'\n", + "filer_committee_id_number 'C00016683'\n", + "committee_name 'PFIZER INC. PAC'\n", + "report_code 'M8'\n", + "coverage_from_date datetime.date(2023, 7, 1)\n", + "coverage_through_date datetime.date(2023, 7, 31)\n", "col_a_cash_on_hand_beginning_period 394272.48\n", "col_a_total_receipts 83741.93\n", - "col_a_total_disbursements 57650.00\n", + "col_a_total_disbursements 57650.0\n", "col_a_cash_on_hand_close_of_period 420364.41\n" ] } ], "source": [ - "cover = parsed[\"filing\"]\n", - "print(len(cover), \"cover fields\\n\")\n", - "\n", "for key in [\n", " \"form_type\",\n", " \"filer_committee_id_number\",\n", @@ -194,123 +216,40 @@ " \"col_a_total_disbursements\",\n", " \"col_a_cash_on_hand_close_of_period\",\n", "]:\n", - " print(f\"{key:40} {cover[key]}\")" + " print(f\"{key:40} {cover_row[key]!r}\")" ] }, { "cell_type": "markdown", - "id": "f362973c", + "id": "189627d2", "metadata": {}, "source": [ - "> **Every value is a string.** Unlike the original `fecfile` package, amounts are not converted to floats and dates stay in the FEC's `YYYYMMDD` format. Convert them yourself, as shown in the pandas section below." - ] - }, - { - "cell_type": "markdown", - "id": "fb6c362e", - "metadata": {}, - "source": [ - "### Itemizations\n", + "### Into pandas\n", "\n", - "Rows are grouped by schedule. The exact line number each row was reported on is in its `form_type` (`SA11AI`, `SB23`, …)." + "`Row` registers as a `collections.abc.Mapping`, so `read(path).rows` — a plain list of them — is exactly what `pd.DataFrame` wants. Because the values are already typed, there's no `astype(float)` or `pd.to_datetime(..., format=...)` step to write: amounts are `float64` and dates are `datetime.date` objects from the moment the `DataFrame` exists." ] }, { "cell_type": "code", "execution_count": 5, - "id": "27976117", + "id": "79ff3370", "metadata": { "execution": { - "iopub.execute_input": "2026-09-18T21:41:00.369721Z", - "iopub.status.busy": "2026-09-18T21:41:00.369665Z", - "iopub.status.idle": "2026-09-18T21:41:00.371589Z", - "shell.execute_reply": "2026-09-18T21:41:00.371214Z" + "iopub.execute_input": "2026-09-19T02:33:04.811495Z", + "iopub.status.busy": "2026-09-19T02:33:04.811426Z", + "iopub.status.idle": "2026-09-19T02:33:04.920601Z", + "shell.execute_reply": "2026-09-19T02:33:04.920242Z" } }, "outputs": [ { - "data": { - "text/plain": [ - "{'Schedule A': 1354, 'Schedule B': 33}" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "{schedule: len(rows) for schedule, rows in parsed[\"itemizations\"].items()}" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "35da23b5", - "metadata": { - "execution": { - "iopub.execute_input": "2026-09-18T21:41:00.372473Z", - "iopub.status.busy": "2026-09-18T21:41:00.372411Z", - "iopub.status.idle": "2026-09-18T21:41:00.374386Z", - "shell.execute_reply": "2026-09-18T21:41:00.374096Z" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'form_type': 'SA11AI',\n", - " 'filer_committee_id_number': 'C00016683',\n", - " 'transaction_id': '2023071716378-1066',\n", - " 'entity_type': 'IND',\n", - " 'contributor_last_name': 'Aaronson',\n", - " 'contributor_first_name': 'Eric',\n", - " 'contributor_street_1': '66 Hudson Blvd East',\n", - " 'contributor_city': 'New York',\n", - " 'contributor_state': 'NY',\n", - " 'contributor_zip_code': '10001',\n", - " 'contribution_date': '20230714',\n", - " 'contribution_amount': '104.17',\n", - " 'contribution_aggregate': '1458.38',\n", - " 'contributor_employer': 'Pfizer Inc',\n", - " 'contributor_occupation': 'SVP, Chief Counsel IP & IPE'}" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "first = parsed[\"itemizations\"][\"Schedule A\"][0]\n", - "\n", - "# Only show the populated columns\n", - "{k: v for k, v in first.items() if v}" - ] - }, - { - "cell_type": "markdown", - "id": "a3bdb8d9", - "metadata": {}, - "source": [ - "### Into pandas\n", - "\n", - "Each schedule is a list of dicts, which is exactly what `pd.DataFrame` wants." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "40abb0c3", - "metadata": { - "execution": { - "iopub.execute_input": "2026-09-18T21:41:00.375369Z", - "iopub.status.busy": "2026-09-18T21:41:00.375298Z", - "iopub.status.idle": "2026-09-18T21:41:00.588101Z", - "shell.execute_reply": "2026-09-18T21:41:00.587726Z" - } - }, - "outputs": [ + "name": "stdout", + "output_type": "stream", + "text": [ + "float64\n", + "\n" + ] + }, { "data": { "text/html": [ @@ -413,18 +352,22 @@ "4 20.00 " ] }, - "execution_count": 7, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import pandas as pd\n", + "from libfec_parser import read\n", + "\n", + "filing = read(str(path))\n", + "df = pd.DataFrame(filing.rows)\n", "\n", - "receipts = pd.DataFrame(parsed[\"itemizations\"][\"Schedule A\"])\n", - "receipts[\"contribution_amount\"] = receipts[\"contribution_amount\"].astype(float)\n", - "receipts[\"contribution_date\"] = pd.to_datetime(receipts[\"contribution_date\"], format=\"%Y%m%d\")\n", + "print(df[\"contribution_amount\"].dtype)\n", + "print(type(df[\"contribution_date\"].dropna().iloc[0]))\n", "\n", + "receipts = df[df[\"form_type\"].str.startswith(\"SA\")].dropna(axis=1, how=\"all\")\n", "receipts[\n", " [\n", " \"contributor_last_name\",\n", @@ -437,16 +380,26 @@ "].head()" ] }, + { + "cell_type": "markdown", + "id": "57ae4d86", + "metadata": {}, + "source": [ + "(Dates land as `object`-dtype `datetime.date`, not `datetime64` — pandas doesn't auto-convert one to the other. If you need `datetime64` semantics, such as `.dt` accessors or resampling, convert explicitly with `pd.to_datetime(df[\"contribution_date\"])`; this notebook doesn't need to.)\n", + "\n", + "This PAC is funded by payroll deductions, so most people appear more than once in a month. Group by contributor to see who gave the most — the same analysis as before, now over typed columns:" + ] + }, { "cell_type": "code", - "execution_count": 8, - "id": "a7c19fea", + "execution_count": 6, + "id": "8ccf3842", "metadata": { "execution": { - "iopub.execute_input": "2026-09-18T21:41:00.589163Z", - "iopub.status.busy": "2026-09-18T21:41:00.589091Z", - "iopub.status.idle": "2026-09-18T21:41:00.590971Z", - "shell.execute_reply": "2026-09-18T21:41:00.590658Z" + "iopub.execute_input": "2026-09-19T02:33:04.921749Z", + "iopub.status.busy": "2026-09-19T02:33:04.921683Z", + "iopub.status.idle": "2026-09-19T02:33:04.926659Z", + "shell.execute_reply": "2026-09-19T02:33:04.926275Z" } }, "outputs": [ @@ -457,35 +410,7 @@ "1,354 itemized receipts totaling $57,161.47\n", "reported on the cover page: $57,161.47\n" ] - } - ], - "source": [ - "total = receipts[\"contribution_amount\"].sum()\n", - "print(f\"{len(receipts):,} itemized receipts totaling ${total:,.2f}\")\n", - "print(f\"reported on the cover page: ${float(cover['col_a_individuals_itemized']):,.2f}\")" - ] - }, - { - "cell_type": "markdown", - "id": "67c20a58", - "metadata": {}, - "source": [ - "This PAC is funded by payroll deductions, so most people appear more than once in a month. Group by contributor to see who gave the most:" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "42d2d765", - "metadata": { - "execution": { - "iopub.execute_input": "2026-09-18T21:41:00.591896Z", - "iopub.status.busy": "2026-09-18T21:41:00.591837Z", - "iopub.status.idle": "2026-09-18T21:41:00.597362Z", - "shell.execute_reply": "2026-09-18T21:41:00.597097Z" - } - }, - "outputs": [ + }, { "data": { "text/html": [ @@ -623,12 +548,16 @@ "Hogan Timothy SVP, Global Policy & Public Affairs 416.66 " ] }, - "execution_count": 9, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ + "total = receipts[\"contribution_amount\"].sum()\n", + "print(f\"{len(receipts):,} itemized receipts totaling ${total:,.2f}\")\n", + "print(f\"reported on the cover page: ${filing.cover_row['col_a_individuals_itemized']:,.2f}\")\n", + "\n", "(\n", " receipts.groupby([\"contributor_last_name\", \"contributor_first_name\", \"contributor_occupation\"])[\"contribution_amount\"]\n", " .agg([\"count\", \"sum\"])\n", @@ -639,271 +568,131 @@ }, { "cell_type": "markdown", - "id": "4cda5eb9", + "id": "0b1c2fe4", "metadata": {}, "source": [ - "Schedule B is where the money goes. For a corporate PAC, that's mostly contributions to candidates and other committees." + "## The `fecfile` API\n", + "\n", + "`libfec_parser.fecfile` is a dict-based API modeled on the [`fecfile`](https://pypi.org/project/fecfile/) package: `from_file()` parses a filing into a plain dict with four keys — `header`, `filing` (the cover page), `itemizations` (rows grouped by schedule: `\"Schedule A\"`, `\"Schedule B\"`, …), and `text`." ] }, { "cell_type": "code", - "execution_count": 10, - "id": "9ff31696", + "execution_count": 7, + "id": "dacb2a48", "metadata": { "execution": { - "iopub.execute_input": "2026-09-18T21:41:00.598489Z", - "iopub.status.busy": "2026-09-18T21:41:00.598435Z", - "iopub.status.idle": "2026-09-18T21:41:00.602235Z", - "shell.execute_reply": "2026-09-18T21:41:00.601914Z" + "iopub.execute_input": "2026-09-19T02:33:04.927629Z", + "iopub.status.busy": "2026-09-19T02:33:04.927574Z", + "iopub.status.idle": "2026-09-19T02:33:04.933791Z", + "shell.execute_reply": "2026-09-19T02:33:04.933428Z" } }, "outputs": [ { "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
payee_organization_namepayee_stateexpenditure_purpose_descripexpenditure_amount
19NRSC (Building Fund)DC2023 Contribution5000.0
7DSCC (Building Fund)DC2023 Contribution5000.0
32Republican Assembly Campaign CommitteeWINonfederal Contribution3750.0
24Committee to Elect a Republican SenateWINonfederal Contribution3750.0
17Mike Kelly For CongressPA2024 Primary3000.0
11Guy For CongressPA2024 Primary2500.0
20Oorah! Political Action CommitteeIN2023 Contribution2500.0
1Ann Wagner For CongressMO2024 Primary2500.0
15Lou Correa For CongressCA2024 Primary2500.0
0Alamo PACTX2023 Contribution2500.0
\n", - "
" - ], "text/plain": [ - " payee_organization_name payee_state \\\n", - "19 NRSC (Building Fund) DC \n", - "7 DSCC (Building Fund) DC \n", - "32 Republican Assembly Campaign Committee WI \n", - "24 Committee to Elect a Republican Senate WI \n", - "17 Mike Kelly For Congress PA \n", - "11 Guy For Congress PA \n", - "20 Oorah! Political Action Committee IN \n", - "1 Ann Wagner For Congress MO \n", - "15 Lou Correa For Congress CA \n", - "0 Alamo PAC TX \n", - "\n", - " expenditure_purpose_descrip expenditure_amount \n", - "19 2023 Contribution 5000.0 \n", - "7 2023 Contribution 5000.0 \n", - "32 Nonfederal Contribution 3750.0 \n", - "24 Nonfederal Contribution 3750.0 \n", - "17 2024 Primary 3000.0 \n", - "11 2024 Primary 2500.0 \n", - "20 2023 Contribution 2500.0 \n", - "1 2024 Primary 2500.0 \n", - "15 2024 Primary 2500.0 \n", - "0 2023 Contribution 2500.0 " + "dict_keys(['header', 'filing', 'itemizations', 'text'])" ] }, - "execution_count": 10, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "disbursements = pd.DataFrame(parsed[\"itemizations\"][\"Schedule B\"])\n", - "disbursements[\"expenditure_amount\"] = disbursements[\"expenditure_amount\"].astype(float)\n", + "from libfec_parser import fecfile\n", "\n", - "(\n", - " disbursements[[\"payee_organization_name\", \"payee_state\", \"expenditure_purpose_descrip\", \"expenditure_amount\"]]\n", - " .sort_values(\"expenditure_amount\", ascending=False)\n", - " .head(10)\n", - ")" + "parsed = fecfile.from_file(str(path))\n", + "parsed.keys()" ] }, { "cell_type": "markdown", - "id": "72039168", + "id": "b7a6cdd6", "metadata": {}, "source": [ - "### Only parse what you need\n", - "\n", - "`filter_itemizations` takes a list of row-type prefixes and drops everything else. On a large filing (ActBlue's reports run to several gigabytes) this saves most of the memory. An empty list skips itemizations entirely, leaving just the header and cover page." + "> **Every value is a string.** Unlike the native API above, amounts are not converted to floats and dates stay in the FEC's `YYYYMMDD` format — convert them yourself, or use `libfec_parser.open()`/`read()` instead." ] }, { "cell_type": "code", - "execution_count": 11, - "id": "ac9c1b61", + "execution_count": 8, + "id": "d3595410", "metadata": { "execution": { - "iopub.execute_input": "2026-09-18T21:41:00.603331Z", - "iopub.status.busy": "2026-09-18T21:41:00.603276Z", - "iopub.status.idle": "2026-09-18T21:41:00.605683Z", - "shell.execute_reply": "2026-09-18T21:41:00.605411Z" + "iopub.execute_input": "2026-09-19T02:33:04.934788Z", + "iopub.status.busy": "2026-09-19T02:33:04.934737Z", + "iopub.status.idle": "2026-09-19T02:33:04.936579Z", + "shell.execute_reply": "2026-09-19T02:33:04.936288Z" } }, "outputs": [ { "data": { "text/plain": [ - "{'Schedule B': 33}" + "(str, '83741.93')" ] }, - "execution_count": 11, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "only_sb = fecfile.from_file(str(path), options={\"filter_itemizations\": [\"SB\"]})\n", - "{schedule: len(rows) for schedule, rows in only_sb[\"itemizations\"].items()}" + "cover = parsed[\"filing\"]\n", + "type(cover[\"col_a_total_receipts\"]), cover[\"col_a_total_receipts\"]" + ] + }, + { + "cell_type": "markdown", + "id": "1b228047", + "metadata": {}, + "source": [ + "Itemizations are grouped by schedule, and each row is a dict keyed by column name:" ] }, { "cell_type": "code", - "execution_count": 12, - "id": "f3bb01e6", + "execution_count": 9, + "id": "0d658682", "metadata": { "execution": { - "iopub.execute_input": "2026-09-18T21:41:00.606547Z", - "iopub.status.busy": "2026-09-18T21:41:00.606496Z", - "iopub.status.idle": "2026-09-18T21:41:00.608637Z", - "shell.execute_reply": "2026-09-18T21:41:00.608373Z" + "iopub.execute_input": "2026-09-19T02:33:04.937495Z", + "iopub.status.busy": "2026-09-19T02:33:04.937447Z", + "iopub.status.idle": "2026-09-19T02:33:04.939164Z", + "shell.execute_reply": "2026-09-19T02:33:04.938875Z" } }, "outputs": [ { "data": { "text/plain": [ - "{}" + "{'Schedule B': 33, 'Schedule A': 1354}" ] }, - "execution_count": 12, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "cover_only = fecfile.loads(path.read_bytes(), options={\"filter_itemizations\": []})\n", - "cover_only[\"itemizations\"]" - ] - }, - { - "cell_type": "markdown", - "id": "26736a10", - "metadata": {}, - "source": [ - "### Other entry points\n", - "\n", - "- `loads(content)` parses `bytes`, a `str`, or a list of lines you already have in memory.\n", - "- `from_http(filing_id)` downloads from the FEC and parses in one step. It returns `None` if the filing doesn't exist.\n", - "- `parse_header(line)` and `parse_line(line, version)` parse a single record, if you're streaming a file yourself.\n", - "\n", - "One gotcha when working with lines: `.fec` fields are separated by the ASCII 28 \"file separator\" character, which Python's `str.splitlines()` treats as a line break. Split on `\"\\n\"` instead." + "{schedule: len(rows) for schedule, rows in parsed[\"itemizations\"].items()}" ] }, { "cell_type": "code", - "execution_count": 13, - "id": "f359ccc6", + "execution_count": 10, + "id": "fca5d0ad", "metadata": { "execution": { - "iopub.execute_input": "2026-09-18T21:41:00.609466Z", - "iopub.status.busy": "2026-09-18T21:41:00.609422Z", - "iopub.status.idle": "2026-09-18T21:41:00.611786Z", - "shell.execute_reply": "2026-09-18T21:41:00.611498Z" + "iopub.execute_input": "2026-09-19T02:33:04.940030Z", + "iopub.status.busy": "2026-09-19T02:33:04.939960Z", + "iopub.status.idle": "2026-09-19T02:33:04.941886Z", + "shell.execute_reply": "2026-09-19T02:33:04.941542Z" } }, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'record_type': 'HDR', 'ef_type': 'FEC', 'fec_version': '8.4', 'software_name': 'FECFile', 'software_version': '8.4', 'report_number': '0'}\n" - ] - }, { "data": { "text/plain": [ @@ -924,157 +713,81 @@ " 'contributor_occupation': 'SVP, Chief Counsel IP & IPE'}" ] }, - "execution_count": 13, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "lines = path.read_text(encoding=\"latin-1\").split(\"\\n\")\n", - "\n", - "header, version, _ = fecfile.parse_header(lines[0])\n", - "print(header)\n", + "first = parsed[\"itemizations\"][\"Schedule A\"][0]\n", "\n", - "row = fecfile.parse_line(lines[2], version)\n", - "{k: v for k, v in row.items() if v}" + "# Only show the populated columns\n", + "{k: v for k, v in first.items() if v}" ] }, { "cell_type": "markdown", - "id": "02504485", + "id": "fc2c3905", "metadata": {}, "source": [ - "## The native `Filing` API\n", - "\n", - "`libfec_parser.parser.Filing` is a thinner wrapper over the Rust parser. It accepts a path, `bytes`, or any file-like object with a `.read()` method (including an open `urlopen()` response)." - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "f49c7ad7", - "metadata": { - "execution": { - "iopub.execute_input": "2026-09-18T21:41:00.612724Z", - "iopub.status.busy": "2026-09-18T21:41:00.612674Z", - "iopub.status.idle": "2026-09-18T21:41:00.615322Z", - "shell.execute_reply": "2026-09-18T21:41:00.615065Z" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "Filing(form_type='F3XN', filer_id='C00016683', 1387 itemizations)" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from libfec_parser.parser import Filing\n", + "### Only parse what you need\n", "\n", - "filing = Filing(str(path))\n", - "filing" + "`filter_itemizations` takes a list of row-type prefixes and drops everything else. On a large filing (ActBlue's reports run to several gigabytes) this saves most of the memory. An empty list skips itemizations entirely, leaving just the header and cover page." ] }, { "cell_type": "code", - "execution_count": 15, - "id": "6d534255", + "execution_count": 11, + "id": "00d8e2de", "metadata": { "execution": { - "iopub.execute_input": "2026-09-18T21:41:00.616289Z", - "iopub.status.busy": "2026-09-18T21:41:00.616234Z", - "iopub.status.idle": "2026-09-18T21:41:00.618055Z", - "shell.execute_reply": "2026-09-18T21:41:00.617774Z" + "iopub.execute_input": "2026-09-19T02:33:04.942738Z", + "iopub.status.busy": "2026-09-19T02:33:04.942688Z", + "iopub.status.idle": "2026-09-19T02:33:04.945098Z", + "shell.execute_reply": "2026-09-19T02:33:04.944812Z" } }, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Header(fec_version='8.4', software_name='FECFile', software_version='8.4')\n", - "8.4 | FECFile 8.4\n", - "\n", - "Cover(form_type='F3XN', filer_id='C00016683', filer_name='PFIZER INC. PAC')\n" - ] - }, { "data": { "text/plain": [ - "{'form_type': 'F3XN',\n", - " 'filer_id': 'C00016683',\n", - " 'filer_name': 'PFIZER INC. PAC',\n", - " 'report_code': 'M8',\n", - " 'coverage_from_date': '2023-07-01',\n", - " 'coverage_through_date': '2023-07-31'}" + "{'Schedule B': 33}" ] }, - "execution_count": 15, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "print(filing.header)\n", - "print(filing.header.fec_version, \"|\", filing.header.software_name, filing.header.software_version)\n", - "print()\n", - "print(filing.cover)\n", - "filing.cover.fields()" + "only_sb = fecfile.from_file(str(path), options={\"filter_itemizations\": [\"SB\"]})\n", + "{schedule: len(rows) for schedule, rows in only_sb[\"itemizations\"].items()}" ] }, { "cell_type": "markdown", - "id": "6d181880", + "id": "0b6c7e00", "metadata": {}, "source": [ - "Here the cover's coverage dates are ISO formatted (`2023-07-01`), and itemizations are positional: `row_type` plus a list of string fields, with no column names attached. It's the cheaper representation when you only need to count or filter rows." - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "3dc69314", - "metadata": { - "execution": { - "iopub.execute_input": "2026-09-18T21:41:00.618940Z", - "iopub.status.busy": "2026-09-18T21:41:00.618887Z", - "iopub.status.idle": "2026-09-18T21:41:00.621560Z", - "shell.execute_reply": "2026-09-18T21:41:00.621221Z" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "Counter({'SA11AI': 1354, 'SB23': 24, 'SB29': 9})" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from collections import Counter\n", + "### Other entry points\n", "\n", - "Counter(item.row_type for item in filing.itemizations)" + "- `loads(content)` parses `bytes`, a `str`, or a list of lines you already have in memory.\n", + "- `from_http(filing_id)` downloads from the FEC and parses in one step. It returns `None` if the filing doesn't exist.\n", + "- `parse_header(line)` and `parse_line(line, version)` parse a single record, if you're streaming a file yourself.\n", + "\n", + "One gotcha when working with lines: `.fec` fields are separated by the ASCII 28 \"file separator\" character, which Python's `str.splitlines()` treats as a line break. Split on `\"\\n\"` instead." ] }, { "cell_type": "code", - "execution_count": 17, - "id": "4130a7fd", + "execution_count": 12, + "id": "679694c4", "metadata": { "execution": { - "iopub.execute_input": "2026-09-18T21:41:00.622446Z", - "iopub.status.busy": "2026-09-18T21:41:00.622393Z", - "iopub.status.idle": "2026-09-18T21:41:00.624940Z", - "shell.execute_reply": "2026-09-18T21:41:00.624705Z" + "iopub.execute_input": "2026-09-19T02:33:04.946014Z", + "iopub.status.busy": "2026-09-19T02:33:04.945965Z", + "iopub.status.idle": "2026-09-19T02:33:04.948493Z", + "shell.execute_reply": "2026-09-19T02:33:04.948124Z" } }, "outputs": [ @@ -1082,42 +795,47 @@ "name": "stdout", "output_type": "stream", "text": [ - "Itemization(row_type='SA11AI', 45 fields)\n", - "45 fields\n", - "SA11AI ... (empty)\n" + "{'record_type': 'HDR', 'ef_type': 'FEC', 'fec_version': '8.4', 'software_name': 'FECFile', 'software_version': '8.4', 'report_number': '0'}\n" ] }, { "data": { "text/plain": [ - "['SA11AI',\n", - " 'C00016683',\n", - " '2023071716378-1066',\n", - " '',\n", - " '',\n", - " 'IND',\n", - " '',\n", - " 'Aaronson',\n", - " 'Eric',\n", - " '']" + "{'form_type': 'SA11AI',\n", + " 'filer_committee_id_number': 'C00016683',\n", + " 'transaction_id': '2023071716378-1066',\n", + " 'entity_type': 'IND',\n", + " 'contributor_last_name': 'Aaronson',\n", + " 'contributor_first_name': 'Eric',\n", + " 'contributor_street_1': '66 Hudson Blvd East',\n", + " 'contributor_city': 'New York',\n", + " 'contributor_state': 'NY',\n", + " 'contributor_zip_code': '10001',\n", + " 'contribution_date': '20230714',\n", + " 'contribution_amount': '104.17',\n", + " 'contribution_aggregate': '1458.38',\n", + " 'contributor_employer': 'Pfizer Inc',\n", + " 'contributor_occupation': 'SVP, Chief Counsel IP & IPE'}" ] }, - "execution_count": 17, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "item = filing.itemizations[0]\n", - "print(item)\n", - "print(len(item), \"fields\")\n", - "print(item[0], \"...\", item[-1] or \"(empty)\")\n", - "item.fields()[:10]" + "lines = path.read_text(encoding=\"latin-1\").split(\"\\n\")\n", + "\n", + "header, version, _ = fecfile.parse_header(lines[0])\n", + "print(header)\n", + "\n", + "row = fecfile.parse_line(lines[2], version)\n", + "{k: v for k, v in row.items() if v}" ] }, { "cell_type": "markdown", - "id": "27a5ee7d", + "id": "74e29457", "metadata": {}, "source": [ "## Next steps\n", diff --git a/crates/fec-py/python/libfec_parser/__init__.py b/crates/fec-py/python/libfec_parser/__init__.py index dc45716..579f334 100644 --- a/crates/fec-py/python/libfec_parser/__init__.py +++ b/crates/fec-py/python/libfec_parser/__init__.py @@ -3,7 +3,34 @@ from importlib.metadata import version as _version from . import fecfile, parser +from .parser import ( + Cover, + FecError, + FecParseError, + Filing, + FilingReader, + Header, + MissingMappingError, + Row, + fec_header, + open, + read, +) __version__ = _version("libfec-parser") -__all__ = ["fecfile", "parser"] +__all__ = [ + "fecfile", + "parser", + "Cover", + "FecError", + "FecParseError", + "Filing", + "FilingReader", + "Header", + "MissingMappingError", + "Row", + "fec_header", + "open", + "read", +] diff --git a/crates/fec-py/python/libfec_parser/parser.py b/crates/fec-py/python/libfec_parser/parser.py index a0ba613..48de52a 100644 --- a/crates/fec-py/python/libfec_parser/parser.py +++ b/crates/fec-py/python/libfec_parser/parser.py @@ -1,14 +1,123 @@ """Parsing primitives for FEC electronic filings.""" +# `open` below shadows the builtin for the rest of this module; reach the real one +# through `builtins.open`. +import builtins # noqa: F401 (kept for modules that need the real `open`) +import mmap +import os +import warnings +from collections.abc import Iterator, Mapping +from typing import Protocol, TypeAlias + # `_native` is a single extension module; `_native.parser` is an attribute of it, # not an importable submodule, so it is bound by attribute access rather than # `from ._native.parser import ...`. from ._native import parser as _parser Cover = _parser.Cover -Filing = _parser.Filing +FilingReader = _parser.FilingReader Header = _parser.Header -Itemization = _parser.Itemization +Row = _parser.Row fec_header = _parser.fec_header +open = _parser.open + +FecError = _parser.FecError +FecParseError = _parser.FecParseError + + +class _Readable(Protocol): + """A binary file object: anything whose ``read(n)`` hands back ``bytes``.""" + + def read(self, n: int, /) -> bytes: ... + + +# The union `open()`/`Filing()` accept. Defined at runtime (not stub-only like +# `Value`) so it doubles as the annotation on `Filing.__init__` below, and so +# `parser.pyi` can spell it identically. `collections.abc.Buffer` would say +# "bytes-like" in one word, but it is 3.12+ and the floor here is 3.11. +Source: TypeAlias = ( + str | os.PathLike[str] | bytes | bytearray | memoryview | mmap.mmap | _Readable +) + + +class MissingMappingError(FecError): + """Raised by iteration for a row whose ``(row_type, fec_version)`` has no column mapping.""" + + def __init__(self, row_type: str, version: str, line: int) -> None: + super().__init__(row_type, version, line) + self.row_type, self.version, self.line = row_type, version, line + + def __str__(self) -> str: + return ( + f"no column mapping for row type {self.row_type!r} " + f"in FEC version {self.version} (line {self.line})" + ) + + +# `Row.__reduce__` names this function, and pickle resolves a callable through its +# `__module__`. `_native.parser` is an attribute, not an importable module, so point +# it at this module — the one place `_row_from_parts` can actually be imported from. +_row_from_parts = _parser._row_from_parts +_row_from_parts.__module__ = __name__ + +# pandas only treats list items as records if `isinstance(x, Mapping)`. +Mapping.register(Row) + + +class Filing: + """A whole filing in memory: header, cover and every row, parsed once. + + ``read(source)`` is the same thing as a function. For filings too large to hold, + use :func:`open`, which streams. + """ + + __slots__ = ("id", "header", "cover", "cover_row", "rows") + + def __init__(self, source: Source) -> None: + with open(source) as reader: # this module's open(), not builtins.open + self.id = reader.id + self.header = reader.header + self.cover = reader.cover + self.cover_row = reader.cover_row + self.rows: list[Row] = list(reader) # MissingMappingError propagates (Q15: eager = strict) + + @property + def fec_version(self) -> str: + return self.header.fec_version + + @property + def itemizations(self) -> list[Row]: + warnings.warn("Filing.itemizations is deprecated; use Filing.rows", DeprecationWarning, stacklevel=2) + return self.rows + + def __iter__(self) -> Iterator[Row]: + return iter(self.rows) + + def __len__(self) -> int: + return len(self.rows) + + def __repr__(self) -> str: + return ( + f"Filing(id={self.id!r}, form_type={self.cover.form_type!r}, " + f"filer_id={self.cover.filer_id!r}, {len(self.rows)} rows)" + ) + + +def read(source: Source) -> Filing: + """Parse ``source`` eagerly; see :class:`Filing`.""" + return Filing(source) + -__all__ = ["Cover", "Filing", "Header", "Itemization", "fec_header"] +__all__ = [ + "Cover", + "Filing", + "FilingReader", + "Header", + "Row", + "fec_header", + "open", + "read", + "FecError", + "FecParseError", + "MissingMappingError", +] diff --git a/crates/fec-py/python/libfec_parser/parser.pyi b/crates/fec-py/python/libfec_parser/parser.pyi index ed5c5ff..be7576d 100644 --- a/crates/fec-py/python/libfec_parser/parser.pyi +++ b/crates/fec-py/python/libfec_parser/parser.pyi @@ -5,14 +5,43 @@ checked against the built extension by `python -m mypy.stubtest` — see `crates/fec-py/Makefile`'s `stubs` target. The implementation is `src/parser.rs`. """ -from typing import Protocol, final - -__all__ = ["Cover", "Filing", "Header", "Itemization", "fec_header"] +import mmap +import os +from collections.abc import Iterator +from datetime import date +from types import TracebackType +from typing import Any, Protocol, Self, TypeAlias, final, overload + +__all__ = [ + "Cover", + "Filing", + "FilingReader", + "Header", + "Row", + "fec_header", + "open", + "read", + "FecError", + "FecParseError", + "MissingMappingError", +] + +Value: TypeAlias = str | float | date | None +"""A column's value: typed if it parses, the raw `str` if it is garbage, `None` if empty.""" class _Readable(Protocol): - """A binary file-like object: `read()` must return the filing's bytes.""" + """A binary file object: anything whose `read(n)` hands back `bytes`.""" + + def read(self, n: int, /) -> bytes: ... - def read(self) -> bytes: ... +Source: TypeAlias = ( + str | os.PathLike[str] | bytes | bytearray | memoryview | mmap.mmap | _Readable +) +"""What `open()`/`read()`/`Filing()`/`fec_header()` accept as a filing source. + +A path, any bytes-like object (read without copying), or a binary file object +(read in chunks). A text-mode file raises `TypeError`. +""" @final class Header: @@ -49,43 +78,167 @@ class Cover: @property def report_code(self) -> str | None: ... @property - def coverage_from_date(self) -> str | None: ... + def coverage_from_date(self) -> date | None: ... @property - def coverage_through_date(self) -> str | None: ... - def fields(self) -> dict[str, str | None]: + def coverage_through_date(self) -> date | None: ... + def fields(self) -> dict[str, str | date | None]: """The six cover attributes above as a dict.""" @final -class Itemization: - """One itemization row; a sequence of raw string fields.""" +class Row: + """One itemization row: a mapping from column name to typed value. + + By name the value follows the value rule (`Value` above); by position it is + the raw `str` exactly as it appears in the file. `len(row)` is the number of + mapped columns — `len(row.fields())` is the raw field count. + """ @property def row_type(self) -> str: ... - def fields(self) -> list[str]: - """Every field of the row, in file order.""" + @property + def line(self) -> int: + """The row's 1-based physical line in the file.""" - def __len__(self) -> int: ... + @property + def extra_fields(self) -> list[str]: + """Fields past the last mapped column, `[]` unless one of them is non-empty.""" + + @overload + def __getitem__(self, key: str, /) -> Value: ... + @overload def __getitem__(self, key: int, /) -> str: ... + @overload + def __getitem__(self, key: slice, /) -> list[str]: ... + def __len__(self) -> int: ... + def __iter__(self) -> Iterator[str]: ... + def __contains__(self, key: object, /) -> bool: ... + def keys(self) -> list[str]: ... + def values(self) -> list[Value]: ... + def items(self) -> list[tuple[str, Value]]: ... + def get(self, key: str, default: Value = None, /) -> Value: ... + def fields(self) -> list[str]: + """Every raw field of the row, in file order.""" + + def __eq__(self, other: object, /) -> bool: ... + def __hash__(self) -> int: ... + def __reduce__(self) -> tuple[Any, ...]: ... def __repr__(self) -> str: ... +def _row_from_parts( + row_type: str, version: str, fields: list[str], line: int, / +) -> Row: + """Rebuild a `Row` from its pickled parts; named by `Row.__reduce__`.""" + @final -class Filing: - """A fully parsed filing: header, cover and every itemization row. +class FilingReader: + """A streaming, single-pass reader over one filing. - `source` is a file path (`str`), the filing's bytes, or a binary file-like - object. `os.PathLike` is *not* accepted — pass `str(path)`. + Built by `open()`. The `HDR` and cover records are parsed eagerly, so + `header`, `cover` and `cover_row` are available before iteration; the + itemization rows are pulled lazily, a batch at a time, with the GIL released. + There is no `len()` — the row count is unknown without a full pass. """ - def __new__( - cls, source: str | bytes | bytearray | memoryview | _Readable - ) -> Filing: ... @property - def header(self) -> Header: ... + def header(self) -> Header: + """The filing's `HDR` record; the same object every time.""" + + @property + def cover(self) -> Cover: + """The six normalized cover attributes; the same object every time.""" + + @property + def cover_row(self) -> Row: + """The full cover line as a `Row`; the same object every time.""" + + @property + def id(self) -> str | None: + """The file stem of a path source, or of a file object's `name` + (`FEC-` stripped); `None` when the source does not name itself.""" + @property - def cover(self) -> Cover: ... + def fec_version(self) -> str: + """Shortcut for `header.fec_version`.""" + @property - def itemizations(self) -> list[Itemization]: ... + def source_length(self) -> int: + """The source's size in bytes, `0` if unknown.""" + + @property + def closed(self) -> bool: ... + def rows(self, *prefixes: str) -> Self: + """Only yield rows whose type starts with one of `prefixes` (case-insensitive). + + Returns the reader itself. A second call replaces the filter; `rows()` + with no arguments clears it. + """ + + def close(self) -> None: + """Drop the source. Idempotent; iterating afterwards raises `ValueError`.""" + + def __enter__(self) -> Self: ... + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + /, + ) -> None: ... + def __iter__(self) -> Self: ... + def __next__(self) -> Row: ... + def __repr__(self) -> str: ... + +def open(source: Source, /) -> FilingReader: + """Open a filing for streaming. + + `source` is a filesystem path (`str` or `os.PathLike`), a bytes-like object + (`bytes`, `bytearray`, `memoryview`, `mmap` — read without copying), or a + binary file object, which is read a chunk at a time. A missing path raises + `FileNotFoundError`, a text-mode file raises `TypeError`, and unparseable + input raises `FecParseError`. An exception raised by the source's own + `read()` propagates unchanged. + """ + +class Filing: + """A whole filing in memory: header, cover and every row, parsed once. + + `read(source)` is the same thing as a function. For filings too large to + hold, use `open()`, which streams. + """ + + id: str | None + header: Header + cover: Cover + cover_row: Row + rows: list[Row] + + def __init__(self, source: Source) -> None: ... + @property + def fec_version(self) -> str: ... + @property + def itemizations(self) -> list[Row]: + """Deprecated alias of `rows`; warns with `DeprecationWarning`.""" + + def __iter__(self) -> Iterator[Row]: ... + def __len__(self) -> int: ... def __repr__(self) -> str: ... -def fec_header(contents: bytes) -> str: - """The `fec_version` of a filing held entirely in memory.""" +def read(source: Source) -> Filing: + """Parse `source` eagerly; see `Filing`.""" + +def fec_header(source: Source, /) -> str: + """The `fec_version` of a filing, from anything `open()` accepts.""" + +class FecError(ValueError): + """Base class for libfec_parser errors.""" + +class FecParseError(FecError): + """The input is not a parseable .fec filing.""" + +class MissingMappingError(FecError): + """Raised by iteration for a row whose ``(row_type, fec_version)`` has no column mapping.""" + + row_type: str + version: str + line: int + def __init__(self, row_type: str, version: str, line: int) -> None: ... diff --git a/crates/fec-py/src/errors.rs b/crates/fec-py/src/errors.rs new file mode 100644 index 0000000..4081499 --- /dev/null +++ b/crates/fec-py/src/errors.rs @@ -0,0 +1,70 @@ +use pyo3::create_exception; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use pyo3::sync::PyOnceLock; +use pyo3::types::PyType; + +create_exception!( + libfec_parser.parser, + FecError, + PyValueError, + "Base class for libfec_parser errors." +); +create_exception!( + libfec_parser.parser, + FecParseError, + FecError, + "The input is not a parseable .fec filing." +); + +/// Wrap any displayable parser error (`anyhow::Error`, `csv::Error`, …) as a `FecParseError`. +pub fn parse_error(e: impl std::fmt::Display) -> PyErr { + FecParseError::new_err(e.to_string()) +} + +// `MissingMappingError` is defined in Python (`python/libfec_parser/parser.py`), not via +// `create_exception!`, so it can carry real `.row_type`/`.version`/`.line` attributes (see +// todos/python/12's "Recommended alternative"). The class object is cached after the first +// lookup; importing `libfec_parser.parser` here is safe because `_native` is only ever loaded +// from `libfec_parser/__init__.py`, so the package is already importable by the time a row is +// read. +static MISSING_MAPPING_ERROR: PyOnceLock> = PyOnceLock::new(); + +fn missing_mapping_class(py: Python<'_>) -> PyResult<&Py> { + MISSING_MAPPING_ERROR.get_or_try_init(py, || -> PyResult> { + Ok(py + .import("libfec_parser.parser")? + .getattr("MissingMappingError")? + .cast_into::()? + .unbind()) + }) +} + +/// Raise `libfec_parser.parser.MissingMappingError(row_type, version, line)`. +pub fn missing_mapping(py: Python<'_>, row_type: &str, version: &str, line: u64) -> PyErr { + let class = match missing_mapping_class(py) { + Ok(class) => class, + Err(e) => return e, + }; + match class.bind(py).call1((row_type, version, line)) { + Ok(instance) => PyErr::from_value(instance), + Err(e) => e, + } +} + +/// Wrap an `io::Error` from opening/stat-ing a filing path so `FileNotFoundError` (and friends) +/// carry `errno`/`filename`, matching stdlib `open()`. +pub fn io_error(e: std::io::Error, path: &std::path::Path) -> PyErr { + let errno = e.raw_os_error().unwrap_or(0); + let strerror = e.to_string(); + let filename = path.display().to_string(); + match e.kind() { + std::io::ErrorKind::NotFound => { + pyo3::exceptions::PyFileNotFoundError::new_err((errno, strerror, filename)) + } + std::io::ErrorKind::PermissionDenied => { + pyo3::exceptions::PyPermissionError::new_err((errno, strerror, filename)) + } + _ => pyo3::exceptions::PyOSError::new_err((errno, strerror, filename)), + } +} diff --git a/crates/fec-py/src/lib.rs b/crates/fec-py/src/lib.rs index c4691b1..f658adf 100644 --- a/crates/fec-py/src/lib.rs +++ b/crates/fec-py/src/lib.rs @@ -1,5 +1,8 @@ +mod errors; mod fecfile; mod parser; +mod row; +mod source; use pyo3::prelude::*; @@ -14,7 +17,11 @@ mod _native { #[pymodule] mod parser { #[pymodule_export] - use crate::parser::{fec_header, Cover, Filing, Header, Itemization}; + use crate::errors::{FecError, FecParseError}; + #[pymodule_export] + use crate::parser::{fec_header, open_filing, Cover, FilingReader, Header}; + #[pymodule_export] + use crate::row::{row_from_parts, Row}; } #[pymodule] diff --git a/crates/fec-py/src/parser.rs b/crates/fec-py/src/parser.rs index 396e839..63d8f2d 100644 --- a/crates/fec-py/src/parser.rs +++ b/crates/fec-py/src/parser.rs @@ -1,27 +1,32 @@ +//! `open(src)` → [`FilingReader`]: a streaming, single-pass reader over a filing. +//! +//! The HDR and cover records are parsed eagerly so `.header`/`.cover` are +//! available before iteration; every itemization row after them is pulled lazily, +//! [`BATCH`] rows at a time, with the GIL released for the pull. + +use std::collections::VecDeque; +use std::io::Read; +use std::sync::Mutex; +use std::thread::ThreadId; + +use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::*; use pyo3::types::PyDict; -use std::io::Cursor; -use std::path::PathBuf; -/// Python wrapper for FilingHeader -#[pyclass(module = "libfec_parser.parser", skip_from_py_object)] -#[derive(Clone)] +use crate::errors::{missing_mapping, parse_error}; +use crate::row::{schema_for, Row}; +use crate::source::{lock, lock_attached, raised_or, resolve, ErrorSlot, SourceReader}; + +/// The filing's `HDR` record. +#[pyclass(module = "libfec_parser.parser", frozen, get_all)] pub struct Header { - #[pyo3(get)] pub record_type: String, - #[pyo3(get)] pub ef_type: String, - #[pyo3(get)] pub fec_version: String, - #[pyo3(get)] pub software_name: String, - #[pyo3(get)] pub software_version: String, - #[pyo3(get)] pub report_id: Option, - #[pyo3(get)] pub report_number: Option, - #[pyo3(get)] pub comment: Option, } @@ -35,22 +40,17 @@ impl Header { } } -/// Python wrapper for FilingCover -#[pyclass(module = "libfec_parser.parser", skip_from_py_object)] -#[derive(Clone)] +/// The six normalized cover attributes, shared by every form type (Q19). +/// +/// The full cover line is `FilingReader.cover_row`. +#[pyclass(module = "libfec_parser.parser", frozen, get_all)] pub struct Cover { - #[pyo3(get)] pub form_type: String, - #[pyo3(get)] pub filer_id: String, - #[pyo3(get)] pub filer_name: String, - #[pyo3(get)] pub report_code: Option, - #[pyo3(get)] - pub coverage_from_date: Option, - #[pyo3(get)] - pub coverage_through_date: Option, + pub coverage_from_date: Option, + pub coverage_through_date: Option, } #[pymethods] @@ -62,190 +62,417 @@ impl Cover { ) } - /// Get all cover record fields as a dictionary + /// The six normalized attributes as a dict; for every column on the cover + /// page use `cover_row`. fn fields<'py>(&self, py: Python<'py>) -> PyResult> { let dict = PyDict::new(py); dict.set_item("form_type", &self.form_type)?; dict.set_item("filer_id", &self.filer_id)?; dict.set_item("filer_name", &self.filer_name)?; dict.set_item("report_code", &self.report_code)?; - dict.set_item("coverage_from_date", &self.coverage_from_date)?; - dict.set_item("coverage_through_date", &self.coverage_through_date)?; + dict.set_item("coverage_from_date", self.coverage_from_date)?; + dict.set_item("coverage_through_date", self.coverage_through_date)?; Ok(dict) } } -/// Python wrapper for FilingRow (itemization) -#[pyclass(module = "libfec_parser.parser", skip_from_py_object)] -#[derive(Clone)] -pub struct Itemization { - #[pyo3(get)] - pub row_type: String, - fields: Vec, +/// The parser's filing, over a type-erased reader. +/// +/// `Box` (not just `Read`) so the whole `Filing` is `Send` and can +/// be pulled inside `Python::detach`. +type Source = fec_parser::Filing>; + +type PendingRow = Result; + +/// How many rows one `detach` pull queues up before handing the GIL back. +const BATCH: usize = 256; + +/// A streaming reader over one filing. +/// +/// `frozen` — the ticket's sketch says "not frozen", but `Bound::get()` (which the +/// same sketch uses) is `T: PyClass + Sync`, and every field here is +/// already behind a `Mutex` or a `Py`, so `frozen` is both legal and cheaper than +/// a runtime borrow flag. +/// +/// # Locking +/// +/// Pulling a row can run arbitrary Python code (a file object's `read()`), so a +/// thread can hold `inner` while waiting to attach to the interpreter. Two rules +/// keep that from deadlocking, and both are load-bearing: +/// +/// 1. **Never block on one of these mutexes while attached.** Every lock taken +/// with the GIL held goes through [`lock_attached`]; only code running inside +/// `py.detach` (i.e. [`FilingReader::refill`]) uses the plain [`lock`]. +/// 2. **Lock order is `inner` → `prefixes` → `pending`**, and `inner` is the only +/// one ever held across another. `puller` is a leaf, held for a single +/// assignment or comparison and never across a call into Python. +#[pyclass(module = "libfec_parser.parser", frozen)] +pub struct FilingReader { + /// `None` once closed. + inner: Mutex>, + /// Rows pulled but not yet handed to Python, still as raw `FilingRow`s. + pending: Mutex>, + /// The `rows(*prefixes)` filter; `None` means "every row". + prefixes: Mutex>>, + header: Py
, + cover: Py, + cover_row: Py, + id: Option, + /// `header.fec_version`, needed for every `schema_for` lookup. + version: String, + source_length: usize, + /// An exception raised by a Python `read()` mid-pull, to re-raise as itself. + raised: ErrorSlot, + /// The thread currently inside a pull, if any. + /// + /// `inner` is a plain, non-reentrant `Mutex`, so a pathological source whose + /// `read()` calls `next()` or `close()` on the very reader reading it would + /// deadlock against itself. Recording the puller turns that into a clean + /// `RuntimeError`. + puller: Mutex>, } -#[pymethods] -impl Itemization { - fn __repr__(&self) -> String { - format!( - "Itemization(row_type='{}', {} fields)", - self.row_type, - self.fields.len() - ) - } +/// Marks `puller` for the duration of a pull, and clears it however the pull ends. +struct PullMark<'a>(&'a Mutex>); - fn __len__(&self) -> usize { - self.fields.len() +impl<'a> PullMark<'a> { + fn set(slot: &'a Mutex>) -> Self { + *lock(slot) = Some(std::thread::current().id()); + Self(slot) } +} - fn __getitem__(&self, idx: isize) -> PyResult { - let len = self.fields.len() as isize; - let actual_idx = if idx < 0 { - (len + idx) as usize - } else { - idx as usize - }; - - self.fields - .get(actual_idx) - .cloned() - .ok_or_else(|| pyo3::exceptions::PyIndexError::new_err("Index out of range")) +impl Drop for PullMark<'_> { + fn drop(&mut self) { + *lock(self.0) = None; } +} - /// Get all fields as a list - fn fields(&self) -> Vec { - self.fields.clone() +/// `row_type` matches if it starts with any of `prefixes`, case-insensitively. +/// +/// Compared over bytes: slicing a `str` at `p.len()` would panic on a multi-byte +/// boundary, and FEC row types are ASCII anyway. +fn matches_prefix(prefixes: Option<&Vec>, row_type: &str) -> bool { + match prefixes { + None => true, + Some(prefixes) => { + let row_type = row_type.as_bytes(); + prefixes.iter().any(|p| { + let p = p.as_bytes(); + row_type.len() >= p.len() && row_type[..p.len()].eq_ignore_ascii_case(p) + }) + } } } -/// Main Filing class -#[pyclass(module = "libfec_parser.parser")] -pub struct Filing { - header: Header, - cover: Cover, - itemizations: Vec, +fn closed_error() -> PyErr { + PyValueError::new_err("I/O operation on closed filing") } -#[pymethods] -impl Filing { - #[new] - #[pyo3(signature = (source))] - pub fn new(source: &Bound<'_, PyAny>) -> PyResult { - // Handle different input types: path (str), bytes, or file-like object - let (reader, source_length): (Box, usize) = - if let Ok(path_str) = source.extract::() { - // It's a file path - let path = PathBuf::from(path_str); - let file = std::fs::File::open(&path).map_err(|e| { - pyo3::exceptions::PyIOError::new_err(format!("Failed to open file: {}", e)) - })?; - let len = file - .metadata() - .map_err(|e| { - pyo3::exceptions::PyIOError::new_err(format!( - "Failed to get file metadata: {}", - e - )) - })? - .len() as usize; - (Box::new(file), len) - } else if let Ok(bytes) = source.extract::>() { - // It's bytes - let len = bytes.len(); - (Box::new(Cursor::new(bytes)), len) - } else if let Ok(bytes_like) = source.call_method0("read") { - // It's a file-like object with read() method - let bytes: Vec = bytes_like.extract()?; - let len = bytes.len(); - (Box::new(Cursor::new(bytes)), len) - } else { - return Err(pyo3::exceptions::PyTypeError::new_err( - "Source must be a file path (str), bytes, or file-like object with read() method" - )); - }; +impl FilingReader { + /// Whether this thread is already inside a pull on this reader. + fn pulling_here(&self) -> bool { + *lock(&self.puller) == Some(std::thread::current().id()) + } - // Parse the filing - let mut filing = fec_parser::Filing::from_reader( - reader, - "filing".to_string(), - source_length, - ) - .map_err(|e| { - pyo3::exceptions::PyValueError::new_err(format!("Failed to parse filing: {}", e)) - })?; - - // Convert header - let header = Header { - record_type: filing.header.record_type.clone(), - ef_type: filing.header.ef_type.clone(), - fec_version: filing.header.fec_version.clone(), - software_name: filing.header.software_name.clone(), - software_version: filing.header.software_version.clone(), - report_id: filing.header.report_id.clone(), - report_number: filing.header.report_number.clone(), - comment: filing.header.comment.clone(), - }; + /// Reject a re-entrant call before it blocks on a lock this thread holds. + fn check_not_reentrant(&self) -> PyResult<()> { + if self.pulling_here() { + return Err(PyRuntimeError::new_err( + "reader is already being read on this thread: a source's read() \ + must not call back into the FilingReader reading it", + )); + } + Ok(()) + } - // Convert cover - let cover = Cover { - form_type: filing.cover.form_type.clone(), - filer_id: filing.cover.filer_id.clone(), - filer_name: filing.cover.filer_name.clone(), - report_code: filing.cover.report_code.clone(), - coverage_from_date: filing.cover.coverage_from_date.map(|d| d.to_string()), - coverage_through_date: filing.cover.coverage_through_date.map(|d| d.to_string()), + /// Pull up to [`BATCH`] filter-passing rows into `pending`, **without the GIL**. + /// + /// Returns `true` when the source is exhausted. Rejected rows never leave Rust, + /// which is the entire point of `rows(*prefixes)`. + fn refill(&self) -> PyResult { + self.check_not_reentrant()?; + let mut guard = lock(&self.inner); + let Some(src) = guard.as_mut() else { + return Err(closed_error()); }; + let _mark = PullMark::set(&self.puller); + let prefixes = lock(&self.prefixes).clone(); - // Collect all itemizations - let mut itemizations = Vec::new(); - while let Some(row_result) = filing.next_row() { - let row = row_result.map_err(|e| { - pyo3::exceptions::PyValueError::new_err(format!("Failed to read row: {}", e)) - })?; - - let fields: Vec = row.record.iter().map(|s| s.to_string()).collect(); - itemizations.push(Itemization { - row_type: row.row_type, - fields, - }); + // Pull into a local queue rather than into `pending` directly: a batch can + // mean hundreds of Python `read()` calls, and `pending` is what every other + // thread's `next()` pops from. Holding it throughout would make them all + // wait for the whole batch (detached, so not a deadlock — just a stall). + let mut batch = VecDeque::with_capacity(BATCH); + let mut exhausted = false; + while batch.len() < BATCH { + match src.next_row() { + None => { + exhausted = true; + break; + } + Some(Ok(row)) if !matches_prefix(prefixes.as_ref(), &row.row_type) => continue, + Some(item) => batch.push_back(item), + } } + // `inner` stays held across this append — it is what serializes two + // concurrent refills, so their batches cannot interleave out of file order. + lock(&self.pending).extend(batch); + Ok(exhausted) + } - Ok(Filing { - header, - cover, - itemizations, - }) + /// Drop the source and any rows already pulled. Idempotent. + /// + /// The caller must not be inside a pull on this thread; the public entry + /// points check that first. + fn shut(&self, py: Python<'_>) { + // `take()`, then drop *after* the guard is gone: dropping the source + // releases a `PyBuffer` or a `Py`, and the latter's final decref can + // run a `__del__`, i.e. arbitrary Python, which must not happen while we + // hold `inner`. (`PyBuffer`'s own `Drop` is safe either way: it re-attaches + // via `Python::try_attach`, which is a no-op on an already-attached thread — + // `pyo3-0.29.2/src/internal/state.rs:86-91`.) + let source = lock_attached(py, &self.inner).take(); + lock_attached(py, &self.pending).clear(); + drop(source); + } +} + +#[pymethods] +impl FilingReader { + /// The filing's `HDR` record — the same object every time. + #[getter] + fn header(&self, py: Python<'_>) -> Py
{ + self.header.clone_ref(py) + } + + /// The six normalized cover attributes — the same object every time. + #[getter] + fn cover(&self, py: Python<'_>) -> Py { + self.cover.clone_ref(py) + } + + /// The full cover line as a `Row` — the same object every time. + #[getter] + fn cover_row(&self, py: Python<'_>) -> Py { + self.cover_row.clone_ref(py) } + /// The filing id: the file stem of a path, or of a file object's `name` + /// (`FEC-` stripped); `None` when the source does not name itself. #[getter] - fn header(&self) -> Header { - self.header.clone() + fn id(&self) -> Option<&str> { + self.id.as_deref() } + /// Shortcut for `reader.header.fec_version`. #[getter] - fn cover(&self) -> Cover { - self.cover.clone() + fn fec_version(&self) -> &str { + &self.version } + /// The source's size in bytes, or `0` if it is not known. #[getter] - fn itemizations(&self) -> Vec { - self.itemizations.clone() + fn source_length(&self) -> usize { + self.source_length + } + + /// Whether `close()` has been called. + #[getter] + fn closed(&self, py: Python<'_>) -> bool { + // Inside a pull on this thread we are the ones holding `inner`, and the + // source is open by definition; answer without touching the lock. + !self.pulling_here() && lock_attached(py, &self.inner).is_none() + } + + /// Only yield rows whose type starts with one of `prefixes` (case-insensitive). + /// + /// Returns the reader itself, so `for row in reader.rows("SA", "SB")` reads well. + /// The filter applies to the single remaining pass: a second call replaces it, and + /// `rows()` with no arguments clears it. + #[pyo3(signature = (*prefixes))] + fn rows<'py>(slf: Bound<'py, Self>, prefixes: Vec) -> Bound<'py, Self> { + let filter = if prefixes.is_empty() { + None + } else { + Some(prefixes) + }; + *lock_attached(slf.py(), &slf.get().prefixes) = filter; + slf + } + + /// Drop the source. Idempotent; iterating afterwards raises `ValueError`. + fn close(&self, py: Python<'_>) -> PyResult<()> { + self.check_not_reentrant()?; + self.shut(py); + Ok(()) + } + + fn __enter__(slf: Bound<'_, Self>) -> Bound<'_, Self> { + slf + } + + #[pyo3(signature = (exc_type, exc_value, traceback, /))] + fn __exit__( + &self, + py: Python<'_>, + exc_type: Option<&Bound<'_, PyAny>>, + exc_value: Option<&Bound<'_, PyAny>>, + traceback: Option<&Bound<'_, PyAny>>, + ) -> PyResult<()> { + let (_, _, _) = (exc_type, exc_value, traceback); + self.check_not_reentrant()?; + self.shut(py); + Ok(()) + } + + fn __iter__(slf: Bound<'_, Self>) -> Bound<'_, Self> { + slf + } + + /// The next row, or `StopIteration` at end of file. + /// + /// An unmapped row type raises `MissingMappingError` but leaves the reader usable — + /// the rows queued behind it are still in `pending`. A CSV error is terminal: the + /// underlying reader is poisoned, so the reader is closed before raising. + fn __next__(slf: &Bound<'_, Self>) -> PyResult>> { + let py = slf.py(); + let this = slf.get(); + loop { + let queued = lock_attached(py, &this.pending).pop_front(); + if let Some(item) = queued { + return match item { + Ok(row) => match schema_for(py, &row.row_type, &this.version) { + Some(schema) => { + Ok(Some(Py::new(py, Row::new(schema, row.record, row.line))?)) + } + None => Err(missing_mapping(py, &row.row_type, &this.version, row.line)), + }, + Err(e) => { + // Not inside a pull here: `refill` has already returned. + this.shut(py); + // A `read()` that raised comes back here wrapped in a CSV + // error; hand back the exception the source actually raised. + Err(raised_or(&this.raised, parse_error(e))) + } + }; + } + // `&FilingReader` is `Send` (every field is behind a `Mutex`/`Py`), which + // is what `Ungil` asks for; no `PyRef`/`Bound` crosses into the closure. + let exhausted = py.detach(|| this.refill())?; + if exhausted && lock_attached(py, &this.pending).is_empty() { + return Ok(None); + } + } } fn __repr__(&self) -> String { + let cover = self.cover.get(); + let id = match &self.id { + Some(id) => format!("'{id}'"), + None => "None".to_owned(), + }; format!( - "Filing(form_type='{}', filer_id='{}', {} itemizations)", - self.cover.form_type, - self.cover.filer_id, - self.itemizations.len() + "FilingReader(id={}, form_type='{}', filer_id='{}')", + id, cover.form_type, cover.filer_id ) } } +/// Open a filing for streaming. +/// +/// Parses the `HDR` and cover records eagerly; everything after them is pulled on +/// demand by iteration. +#[pyfunction] +#[pyo3(name = "open", signature = (source, /))] +pub fn open_filing(py: Python<'_>, source: &Bound<'_, PyAny>) -> PyResult { + let SourceReader { + reader, + length: source_length, + id, + raised, + } = resolve(source)?; + + let filing_id = id.clone().unwrap_or_default(); + let filing = py + .detach(move || fec_parser::Filing::from_reader(reader, filing_id, source_length)) + .map_err(|e| raised_or(&raised, parse_error(e)))?; + + let header = Header { + record_type: filing.header.record_type.clone(), + ef_type: filing.header.ef_type.clone(), + fec_version: filing.header.fec_version.clone(), + software_name: filing.header.software_name.clone(), + software_version: filing.header.software_version.clone(), + report_id: filing.header.report_id.clone(), + report_number: filing.header.report_number.clone(), + comment: filing.header.comment.clone(), + }; + let version = header.fec_version.clone(); + + let cover = Cover { + form_type: filing.cover.form_type.clone(), + filer_id: filing.cover.filer_id.clone(), + filer_name: filing.cover.filer_name.clone(), + report_code: filing.cover.report_code.clone(), + coverage_from_date: filing.cover.coverage_from_date, + coverage_through_date: filing.cover.coverage_through_date, + }; + + // `from_reader` already failed if the cover's form type had no mapping + // (`FilingCover::from_record` looks up its columns), so this cannot miss. + let cover_schema = schema_for(py, &cover.form_type, &version).ok_or_else(|| { + parse_error(format!( + "no column mapping for cover form type '{}' in FEC version '{version}'", + cover.form_type + )) + })?; + // The cover record is always the filing's second line. + let cover_row = Row::new(cover_schema, filing.cover.record.clone(), 2); + + Ok(FilingReader { + inner: Mutex::new(Some(filing)), + pending: Mutex::new(VecDeque::new()), + prefixes: Mutex::new(None), + header: Py::new(py, header)?, + cover: Py::new(py, cover)?, + cover_row: Py::new(py, cover_row)?, + id, + version, + source_length, + raised, + puller: Mutex::new(None), + }) +} + +/// The `fec_version` of a filing, from any source `open()` accepts. +/// +/// Reads only the `HDR` record — the first line — so this works even on a +/// filing whose cover has no column mapping, and never touches the rest of +/// the file. #[pyfunction] -pub fn fec_header(contents: &[u8]) -> PyResult { - let f = fec_parser::Filing::from_reader(contents, "123".to_string(), contents.len()).map_err( - |e| pyo3::exceptions::PyValueError::new_err(format!("Failed to parse filing: {}", e)), - )?; - Ok(f.header.fec_version) +#[pyo3(signature = (source, /))] +pub fn fec_header(py: Python<'_>, source: &Bound<'_, PyAny>) -> PyResult { + let SourceReader { reader, raised, .. } = resolve(source)?; + let fec_version = py + .detach(move || -> Result { + // Same `csv::ReaderBuilder` settings as `fec_parser::Filing::from_reader` + // (`crates/fec-parser/src/lib.rs`): delimiter `0x1c`, flexible, no headers. + let csv_reader = csv::ReaderBuilder::new() + .delimiter(b"\x1c"[0]) + .flexible(true) + .has_headers(false) + .from_reader(reader); + let hdr = csv_reader + .into_byte_records() + .next() + .ok_or_else(|| "no header record found".to_owned())? + .map_err(|e| e.to_string())?; + let hdr_record = csv::StringRecord::from_byte_record_lossy(hdr); + fec_parser::FilingHeader::from_record(hdr_record) + .map(|header| header.fec_version) + .map_err(|e| e.to_string()) + }) + .map_err(|e| raised_or(&raised, parse_error(e)))?; + Ok(fec_version) } diff --git a/crates/fec-py/src/row.rs b/crates/fec-py/src/row.rs new file mode 100644 index 0000000..bef3451 --- /dev/null +++ b/crates/fec-py/src/row.rs @@ -0,0 +1,357 @@ +//! `Row`: one itemization line, mapping-first. +//! +//! By name the value is typed (`float` / `datetime.date` / `str` / `None`), by +//! position it is the raw `str` exactly as it appears in the file. Column names +//! and per-column kinds are shared by every row of a `(row_type, fec_version)` +//! pair through an interned [`Schema`]. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use pyo3::exceptions::{PyIndexError, PyKeyError, PyTypeError, PyValueError}; +use pyo3::prelude::*; +use pyo3::types::{PyList, PySlice, PyString, PyTuple}; +use pyo3::IntoPyObjectExt; + +use fec_parser::mappings::{column_names_for_field, DATE_COLUMNS, FLOAT_COLUMNS}; + +/// How a column's raw string is turned into a Python value. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub enum Kind { + Str, + Float, + Date, +} + +/// The column names and kinds of one `(row_type, fec_version)` pair. +pub struct Schema { + /// The row type as it appears in the file (e.g. `"SA11AI"`). + pub row_type: String, + /// `header.fec_version`. + pub version: String, + /// Interned column names, in column order. + pub names: Vec>, + pub kinds: Vec, + pub index: HashMap, +} + +type SchemaKey = (String, String); + +static SCHEMAS: Mutex>>> = Mutex::new(None); + +/// The shared schema for `(row_type, version)`, or `None` if the mapping is unknown. +/// +/// Misses are not cached: `column_names_for_field` is cheap to retry and the +/// caller turns `None` into `MissingMappingError`. +pub fn schema_for(py: Python<'_>, row_type: &str, version: &str) -> Option> { + // `column_names_for_field` is case-insensitive on the row type, but the cache is + // keyed on the spelling in the file so `Row.row_type` round-trips it. + let key: SchemaKey = (row_type.to_owned(), version.to_owned()); + + { + let cache = SCHEMAS.lock().unwrap_or_else(|e| e.into_inner()); + if let Some(schema) = cache.as_ref().and_then(|c| c.get(&key)) { + return Some(Arc::clone(schema)); + } + } + + let columns = column_names_for_field(row_type, version).ok()?; + let schema = Arc::new(Schema { + row_type: key.0.clone(), + version: key.1.clone(), + names: columns + .iter() + .map(|name| PyString::intern(py, name).unbind()) + .collect(), + kinds: columns + .iter() + .map(|name| { + // Same precedence as the CLI's Excel export. + if DATE_COLUMNS.contains(name) { + Kind::Date + } else if FLOAT_COLUMNS.contains(name) { + Kind::Float + } else { + Kind::Str + } + }) + .collect(), + index: columns + .iter() + .enumerate() + .map(|(i, name)| (name.clone(), i)) + .collect(), + }); + + let mut cache = SCHEMAS.lock().unwrap_or_else(|e| e.into_inner()); + Some(Arc::clone( + cache + .get_or_insert_with(HashMap::new) + .entry(key) + .or_insert(schema), + )) +} + +/// One itemization row. +#[pyclass(module = "libfec_parser.parser", frozen)] +pub struct Row { + schema: Arc, + /// Raw fields, including field 0 (the row type). + record: csv::StringRecord, + /// 1-based physical line of the row in the file. + line: u64, +} + +impl Row { + pub fn new(schema: Arc, record: csv::StringRecord, line: u64) -> Self { + Row { + schema, + record, + line, + } + } + + /// The value rule (Q6): typed if it parses, the raw `str` if it is garbage, + /// `None` if it is empty. Text columns keep `""`; a short row reads its + /// missing columns as `None`. + fn value<'py>(&self, py: Python<'py>, i: usize) -> PyResult> { + let Some(raw) = self.record.get(i) else { + return Ok(py.None().into_bound(py)); + }; + let kind = self.schema.kinds[i]; + if kind == Kind::Str { + return PyString::new(py, raw).into_bound_py_any(py); + } + let trimmed = raw.trim(); + if trimmed.is_empty() { + return Ok(py.None().into_bound(py)); + } + match kind { + Kind::Float => match trimmed.parse::() { + Ok(v) => v.into_bound_py_any(py), + Err(_) => PyString::new(py, raw).into_bound_py_any(py), + }, + Kind::Date => match jiff::civil::Date::strptime("%Y%m%d", trimmed) { + Ok(d) => d.into_bound_py_any(py), + Err(_) => PyString::new(py, raw).into_bound_py_any(py), + }, + Kind::Str => unreachable!("handled above"), + } + } + + /// The column index for a name, or `None` if the name is not a column. + fn column(&self, name: &str) -> Option { + self.schema.index.get(name).copied() + } + + /// The raw field at a (possibly negative) position, over *all* fields. + fn field_at(&self, index: isize) -> PyResult<&str> { + let len = self.record.len() as isize; + let i = if index < 0 { len + index } else { index }; + if i < 0 || i >= len { + return Err(PyIndexError::new_err("row index out of range")); + } + Ok(&self.record[i as usize]) + } + + /// `(row_type, version, fields)` as a Python tuple — the identity used by + /// `__eq__` and `__hash__`. + fn identity<'py>(&self, py: Python<'py>) -> PyResult> { + PyTuple::new( + py, + [ + PyString::new(py, &self.schema.row_type).into_any(), + PyString::new(py, &self.schema.version).into_any(), + PyTuple::new(py, self.record.iter().collect::>())?.into_any(), + ], + ) + } +} + +#[pymethods] +impl Row { + /// The row type, as written in the file. + #[getter] + fn row_type(&self) -> &str { + self.record.get(0).unwrap_or("") + } + + /// The 1-based physical line of this row in the file. + #[getter] + fn line(&self) -> u64 { + self.line + } + + /// Fields past the last mapped column, but only if at least one is non-empty. + /// + /// A single trailing empty field is a stray delimiter, not data (Q16). + #[getter] + fn extra_fields(&self) -> Vec { + let mapped = self.schema.names.len(); + if self.record.len() <= mapped { + return Vec::new(); + } + let extras: Vec = self + .record + .iter() + .skip(mapped) + .map(|s| s.to_owned()) + .collect(); + if extras.iter().all(|s| s.is_empty()) { + Vec::new() + } else { + extras + } + } + + /// The number of *columns* in the mapping (`len(row.fields())` is the raw count). + fn __len__(&self) -> usize { + self.schema.names.len() + } + + /// `row[name]` → typed value, `row[i]` → raw `str`, `row[a:b]` → `list[str]`. + fn __getitem__<'py>(&self, key: &Bound<'py, PyAny>) -> PyResult> { + let py = key.py(); + if let Ok(name) = key.cast::() { + let name = name.to_cow()?; + return match self.column(&name) { + Some(i) => self.value(py, i), + None => Err(PyKeyError::new_err(name.into_owned())), + }; + } + if let Ok(slice) = key.cast::() { + let indices = slice.indices(self.record.len() as isize)?; + let mut out: Vec<&str> = Vec::new(); + let mut i = indices.start; + let mut n = indices.slicelength; + while n > 0 { + out.push(&self.record[i as usize]); + i += indices.step; + n -= 1; + } + return out.into_bound_py_any(py); + } + match key.extract::() { + Ok(index) => self.field_at(index)?.into_bound_py_any(py), + Err(_) => Err(PyTypeError::new_err(format!( + "row indices must be str, int or slice, not {}", + key.get_type().name()? + ))), + } + } + + /// Iterating a row yields its column names (mapping semantics). + fn __iter__<'py>(&self, py: Python<'py>) -> PyResult> { + Ok(PyList::new(py, self.keys(py))?.try_iter()?.into_any()) + } + + /// Column-name membership; a non-`str` key is simply not a column. + fn __contains__(&self, key: &Bound<'_, PyAny>) -> PyResult { + match key.cast::() { + Ok(name) => Ok(self.column(&name.to_cow()?).is_some()), + Err(_) => Ok(false), + } + } + + /// The column names, in column order. + fn keys(&self, py: Python<'_>) -> Vec> { + self.schema + .names + .iter() + .map(|name| name.clone_ref(py)) + .collect() + } + + /// The typed values, in column order. + fn values<'py>(&self, py: Python<'py>) -> PyResult>> { + (0..self.schema.names.len()) + .map(|i| self.value(py, i)) + .collect() + } + + /// `(name, value)` pairs, in column order. + fn items<'py>(&self, py: Python<'py>) -> PyResult, Bound<'py, PyAny>)>> { + self.schema + .names + .iter() + .enumerate() + .map(|(i, name)| Ok((name.clone_ref(py), self.value(py, i)?))) + .collect() + } + + /// Like `dict.get`: the typed value, or `default` if `key` is not a column. + #[pyo3(signature = (key, default=None, /))] + fn get<'py>( + &self, + py: Python<'py>, + key: &str, + default: Option>, + ) -> PyResult> { + match self.column(key) { + Some(i) => self.value(py, i), + None => Ok(default.unwrap_or_else(|| py.None().into_bound(py))), + } + } + + /// Every raw field, in file order, including field 0 and any extras. + fn fields(&self) -> Vec { + self.record.iter().map(|s| s.to_owned()).collect() + } + + fn __eq__(&self, other: &Bound<'_, PyAny>) -> PyResult> { + let py = other.py(); + let Ok(other) = other.cast::() else { + return Ok(py.NotImplemented()); + }; + let other = other.get(); + let equal = self.schema.row_type == other.schema.row_type + && self.schema.version == other.schema.version + && self.record == other.record; + equal.into_py_any(py) + } + + fn __hash__(&self, py: Python<'_>) -> PyResult { + self.identity(py)?.hash() + } + + fn __reduce__<'py>(&self, py: Python<'py>) -> PyResult> { + // `libfec_parser.parser`, not `_native.parser`: the latter is an attribute of + // the extension module, not importable, so pickle cannot resolve it. + let module = py.import("libfec_parser.parser")?; + let rebuild = module.getattr("_row_from_parts")?; + let args = ( + &self.schema.row_type, + &self.schema.version, + self.fields(), + self.line, + ); + PyTuple::new(py, [rebuild, args.into_bound_py_any(py)?]) + } + + fn __repr__(&self) -> String { + format!( + "Row(row_type='{}', line={}, {} fields)", + self.row_type(), + self.line, + self.record.len() + ) + } +} + +/// Rebuild a `Row` from its pickled parts. Private; exists for `Row.__reduce__`. +#[pyfunction] +#[pyo3(name = "_row_from_parts", signature = (row_type, version, fields, line, /))] +pub fn row_from_parts( + py: Python<'_>, + row_type: &str, + version: &str, + fields: Vec, + line: u64, +) -> PyResult { + let schema = schema_for(py, row_type, version).ok_or_else(|| { + PyValueError::new_err(format!( + "no column mapping for row type '{row_type}' in FEC version '{version}'" + )) + })?; + Ok(Row::new(schema, csv::StringRecord::from(fields), line)) +} diff --git a/crates/fec-py/src/source.rs b/crates/fec-py/src/source.rs new file mode 100644 index 0000000..6a3791b --- /dev/null +++ b/crates/fec-py/src/source.rs @@ -0,0 +1,315 @@ +//! Turning a Python object into something the parser can `Read`. +//! +//! [`resolve`] is the one place that decides what `open()`, `read()`, `Filing()` +//! and `fec_header()` accept. Resolution order, first match wins: +//! +//! 1. the **buffer protocol** (`bytes`, `bytearray`, `memoryview`, `mmap`, …) — +//! read straight out of the exporter's memory, no copy; +//! 2. **`str` / `os.PathLike`** — a filesystem path, opened as a `File`; +//! 3. a **text-mode file** — rejected with a `TypeError` naming `'rb'`; +//! 4. anything with a **`read`** method — pulled [`CHUNK`] bytes at a time; +//! 5. anything else — `TypeError`. +//! +//! The buffer branch has to come first: `PathBuf` extraction goes through +//! `os.fspath`, which accepts `bytes` as a path, so the path branch would swallow +//! a `bytes` source. `str` is not a buffer exporter and `pathlib.Path` has no +//! `read`, so nothing that means *path* is caught by 1 or 4. + +use std::ffi::CStr; +use std::io::{self, Cursor, Read}; +use std::path::{Path, PathBuf}; +use std::ptr; +use std::sync::{Arc, Mutex, MutexGuard}; + +use pyo3::buffer::{PyBuffer, PyUntypedBuffer}; +use pyo3::exceptions::{PyTypeError, PyValueError}; +use pyo3::prelude::*; +use pyo3::sync::MutexExt; +use pyo3::types::{PyBytes, PyString}; + +use crate::errors::io_error; + +/// How many bytes one `read()` call on a Python file object asks for. +const CHUNK: usize = 64 * 1024; + +/// A `Mutex` here is only ever held by this crate's own short critical sections, +/// so a poisoned lock means a panic mid-pull; take the data anyway rather than +/// turning every later call into a panic. +/// +/// Only for use **without** the GIL, or on a lock no GIL-holder can be waiting +/// for; see [`lock_attached`] for the other case. +pub fn lock(m: &Mutex) -> MutexGuard<'_, T> { + m.lock().unwrap_or_else(|e| e.into_inner()) +} + +/// Lock a mutex whose holder may be waiting for the GIL. +/// +/// `std::sync::Mutex::lock` keeps the GIL while it blocks, which deadlocks +/// against a thread that holds the lock and is trying to attach — exactly what a +/// pull from a Python file object does. `lock_py_attached` tries once, and on +/// contention detaches, blocks, then re-attaches +/// (`pyo3-0.29.2/src/sync.rs:407-436`), so no thread ever waits on one of this +/// crate's locks while holding the GIL. Same poison recovery as [`lock`]. +pub fn lock_attached<'a, T>(py: Python<'_>, m: &'a Mutex) -> MutexGuard<'a, T> { + m.lock_py_attached(py).unwrap_or_else(|e| e.into_inner()) +} + +/// Where an exception raised by a Python `read()` waits to be re-raised. +/// +/// The parser only ever hands back its own error types (`anyhow::Error`, +/// `FilingRowReadError`), and this crate turns those into `FecParseError` by +/// `Display`, which would lose the original Python exception. pyo3 does recover a +/// `PyErr` wrapped in `io::Error` (`From for PyErr`, +/// `pyo3-0.29.2/src/err/impls.rs:46-51`), but nothing on our path performs that +/// conversion, so the adapter also stashes the `PyErr` here and the caller prefers +/// it over the wrapped error. +pub type ErrorSlot = Arc>>; + +/// A Python source resolved into a reader, plus what could be learned about it. +pub struct SourceReader { + /// `Box` (not just `Read`) so the whole `Filing` is `Send` + /// and can be pulled inside `Python::detach`. + pub reader: Box, + /// The source's size in bytes, `0` when it cannot be known. + pub length: usize, + /// The filing id, where the source names itself (a path, or a file object's + /// `.name`); `None` for anonymous sources like `bytes` or `BytesIO`. + pub id: Option, + /// See [`ErrorSlot`]; always empty for sources that cannot run Python code. + pub raised: ErrorSlot, +} + +/// The exception a Python source raised, if any, else `fallback`. +pub fn raised_or(slot: &ErrorSlot, fallback: PyErr) -> PyErr { + lock(slot).take().unwrap_or(fallback) +} + +fn text_mode_error() -> PyErr { + PyTypeError::new_err("file must be opened in binary mode, e.g. open(path, 'rb')") +} + +/// A `Read` straight out of a Python buffer's memory — the reason to take a +/// `PyBuffer` at all. +struct BufferReader { + /// C-contiguous, one byte per item; checked before this struct is built. + buf: PyBuffer, + /// Bytes already handed out. Invariant: `pos <= buf.len_bytes()`. + pos: usize, +} + +impl Read for BufferReader { + fn read(&mut self, out: &mut [u8]) -> io::Result { + let len = self.buf.len_bytes(); + debug_assert!(self.pos <= len, "BufferReader read past its buffer"); + // `saturating_sub`, not `-`: `pos <= len` is an invariant, but if it ever + // stopped holding, wrapping here would hand `copy_nonoverlapping` a huge + // length. This way the worst case is a short read. + let n = out.len().min(len.saturating_sub(self.pos)); + if n == 0 { + return Ok(0); + } + // SAFETY: `buf` owns a `Py_buffer` view, so the exporter must keep the + // memory alive, at a fixed address and at a fixed length, until the view is + // released — which happens in `PyBuffer`'s `Drop`, i.e. no earlier than this + // struct's own drop (CPython enforces the length half: `bytearray` raises + // `BufferError` on a resize while a view is exported). `len_bytes()` is + // therefore constant, `pos <= len` is an invariant (`pos` starts at 0 and + // grows by `n = min(out.len(), len - pos)`), so `[pos, pos + n)` is inside + // the buffer, and `out` is at least `n` long and cannot overlap it. + // NOT guaranteed by the buffer protocol: that the *contents* hold still. + // Another thread can mutate a writable exporter (a `bytearray`, a writable + // `mmap`) while we copy with the GIL released. That is a data race, but a + // bounded one: the address and length are pinned, so the worst case is torn + // bytes in `out` and a `FecParseError` — never a read outside the buffer. + unsafe { + ptr::copy_nonoverlapping( + self.buf.buf_ptr().cast::().add(self.pos), + out.as_mut_ptr(), + n, + ); + } + self.pos += n; + Ok(n) + } +} + +/// A `Read` over any Python object with a `read(n)` method. +/// +/// Runs inside `Python::detach`, so every call re-attaches; it asks for at most +/// [`CHUNK`] bytes, so a huge filing behind a file object is never pulled into +/// Python whole. +struct PyReadAdapter { + obj: Py, + raised: ErrorSlot, +} + +impl PyReadAdapter { + /// Remember `err` for the caller to re-raise, and wrap a copy of it for the + /// parser's own error path. + fn stash(&self, py: Python<'_>, err: PyErr) -> io::Error { + let wrapped = err.clone_ref(py); + *lock(&self.raised) = Some(err); + io::Error::other(wrapped) + } +} + +impl Read for PyReadAdapter { + fn read(&mut self, out: &mut [u8]) -> io::Result { + if out.is_empty() { + return Ok(0); + } + let want = out.len().min(CHUNK); + Python::attach(|py| { + let data = match self.obj.bind(py).call_method1("read", (want,)) { + Ok(data) => data, + Err(e) => return Err(self.stash(py, e)), + }; + let Ok(bytes) = data.cast::() else { + // A text-mode file hands back `str`; say the same thing the + // up-front `io.TextIOBase` check says. + let e = if data.is_instance_of::() { + text_mode_error() + } else { + match data.get_type().name() { + Ok(name) => { + PyTypeError::new_err(format!("read() must return bytes, not {name}")) + } + Err(e) => e, + } + }; + return Err(self.stash(py, e)); + }; + let b = bytes.as_bytes(); + if b.len() > want { + let e = PyValueError::new_err(format!( + "read({want}) returned {} bytes, more than asked for", + b.len() + )); + return Err(self.stash(py, e)); + } + out[..b.len()].copy_from_slice(b); + Ok(b.len()) + }) + } +} + +/// The filing id for a path: its file stem, with a `FEC-` prefix stripped. +/// +/// Mirrors `fec_parser::Filing::from_path` + `from_reader`'s own stripping. +fn id_from_path(path: &Path) -> Option { + path.file_stem().map(|stem| { + let stem = stem.to_string_lossy(); + stem.strip_prefix("FEC-").unwrap_or(&stem).to_owned() + }) +} + +/// `os.fstat(src.fileno()).st_size`, or `0` if the object has no usable `fileno`. +/// +/// `BytesIO` and `urlopen()` responses raise from `fileno()`; that is not an error, +/// it just means the length is unknown. +fn length_from_fileno(source: &Bound<'_, PyAny>) -> usize { + let py = source.py(); + let stat = source + .call_method0("fileno") + .and_then(|fd| py.import("os")?.call_method1("fstat", (fd,))); + match stat { + Ok(stat) => stat + .getattr("st_size") + .and_then(|size| size.extract::()) + .unwrap_or(0), + Err(_) => 0, + } +} + +/// A file object's `.name`, if it is a `str`, as a filing id. +/// +/// So `open(builtins.open("1921705.fec", "rb")).id == "1921705"`. A file opened +/// from a raw descriptor has an `int` name and gets `None`. +fn id_from_name(source: &Bound<'_, PyAny>) -> Option { + let name: String = source.getattr("name").ok()?.extract().ok()?; + id_from_path(Path::new(&name)) +} + +/// Whether `source` is a text-mode file (`open(p)`, `io.StringIO`, …). +fn is_text_io(source: &Bound<'_, PyAny>) -> PyResult { + let text_io_base = source.py().import("io")?.getattr("TextIOBase")?; + source.is_instance(&text_io_base) +} + +/// Read a buffer exporter without copying it, or — for the rare non-contiguous +/// buffer, such as `memoryview(b)[::2]` — gather it into a `Vec` first. +fn buffer_source(py: Python<'_>, buffer: PyUntypedBuffer) -> PyResult { + let length = buffer.len_bytes(); + // Captured before `into_typed` consumes the buffer, for the error message. + let format = buffer.format().to_owned(); + let item_size = buffer.item_size(); + + let buf = buffer + .into_typed::() + .map_err(|_| not_bytes_like(&format, item_size))?; + let reader: Box = if buf.is_c_contiguous() { + Box::new(BufferReader { buf, pos: 0 }) + } else { + Box::new(Cursor::new(buf.to_vec(py)?)) + }; + Ok(SourceReader { + reader, + length, + id: None, + raised: ErrorSlot::default(), + }) +} + +/// A buffer of something other than bytes — a NumPy float array, `array('i')`, +/// `memoryview(b).cast('I')`. `PyBuffer::::get` rejects those (the format and +/// item size have to match `u8`), and so do we, with a type error rather than the +/// `BufferError` pyo3 raises. +fn not_bytes_like(format: &CStr, item_size: usize) -> PyErr { + PyTypeError::new_err(format!( + "source buffer must be bytes-like (one byte per item), \ + got format '{}' with {item_size}-byte items", + format.to_string_lossy() + )) +} + +/// Pull from a Python file object `CHUNK` bytes at a time. +fn read_source(source: &Bound<'_, PyAny>) -> SourceReader { + let raised = ErrorSlot::default(); + let adapter = PyReadAdapter { + obj: source.clone().unbind(), + raised: Arc::clone(&raised), + }; + SourceReader { + reader: Box::new(adapter), + length: length_from_fileno(source), + id: id_from_name(source), + raised, + } +} + +/// Turn a Python source into a reader; see the module docs for the order. +pub fn resolve(source: &Bound<'_, PyAny>) -> PyResult { + if let Ok(buffer) = PyUntypedBuffer::get(source) { + return buffer_source(source.py(), buffer); + } + if let Ok(path) = source.extract::() { + let file = std::fs::File::open(&path).map_err(|e| io_error(e, &path))?; + let length = file.metadata().map_err(|e| io_error(e, &path))?.len() as usize; + return Ok(SourceReader { + reader: Box::new(file), + length, + id: id_from_path(&path), + raised: ErrorSlot::default(), + }); + } + if is_text_io(source)? { + return Err(text_mode_error()); + } + if source.hasattr("read")? { + return Ok(read_source(source)); + } + Err(PyTypeError::new_err(format!( + "source must be a path, a bytes-like object, or a binary file object; got {}", + source.get_type().name()? + ))) +} diff --git a/crates/fec-py/stubtest-allowlist.txt b/crates/fec-py/stubtest-allowlist.txt index ff5a19c..111481c 100644 --- a/crates/fec-py/stubtest-allowlist.txt +++ b/crates/fec-py/stubtest-allowlist.txt @@ -4,3 +4,6 @@ # return; both are plain `dict`s at runtime, by design (see src/fecfile.rs). libfec_parser.fecfile.Options libfec_parser.fecfile.Parsed +# `Value` is the same kind of stub-only alias: the union a `Row` column can hold +# (`str | float | date | None`), used in annotations only (see src/row.rs). +libfec_parser.parser.Value diff --git a/crates/fec-py/tests/README.md b/crates/fec-py/tests/README.md index c072ca0..65afee9 100644 --- a/crates/fec-py/tests/README.md +++ b/crates/fec-py/tests/README.md @@ -6,8 +6,18 @@ This directory contains pytest-based tests for the `libfec_parser` Python packag - `test_parser.py` - Tests for the `libfec_parser.parser` module - Tests for `fec_header()` function - - Tests for `Filing` class - - Tests for `Header`, `Cover`, and `Itemization` classes + - Tests for `Header`, `Cover`, `Row` and the eager `Filing` class + - Tests for the `FecError`/`FecParseError`/`MissingMappingError` hierarchy + +- `test_reader.py` - Tests for `open()` and the `FilingReader` it returns + - Accepted sources (paths, bytes-like objects, binary file objects) and rejected ones + - `rows(*prefixes)` filtering, `close()`/context manager, `id`, `fec_version` + - Threading: the GIL released during a pull, one reader shared across threads + +- `test_pandas.py` - `pd.DataFrame(read(p).rows)` gets typed (`float64`/`date`) columns + +- `test_perf.py` - `@pytest.mark.slow` tests for the Phase 2 done-when numbers (peak RSS, + GIL-released thread progress) against the gitignored 91 MB filing - `test_fecfile.py` - Tests for the `libfec_parser.fecfile` module - Tests for `loads()` function @@ -136,11 +146,14 @@ def test_large_file_parsing(benchmark_fec_file): ## Test Coverage Current coverage includes: -- ✅ Parser module (Filing, Header, Cover, Itemization classes) +- ✅ Native `open()`/`FilingReader` streaming API: sources, filtering, threading (`test_reader.py`) +- ✅ Native `Header`, `Cover`, `Row` and eager `Filing`/`read()` (`test_parser.py`) +- ✅ pandas interop: typed `float64`/`date` columns from `Filing.rows` (`test_pandas.py`) - ✅ Fecfile module (fecfile compatibility layer) -- ✅ Error handling and edge cases +- ✅ `FecError`/`FecParseError`/`MissingMappingError` hierarchy and edge cases - ✅ Integration tests - ✅ Every committed fixture, through both APIs (`test_fixtures.py`) +- ✅ Phase 2 done-when perf numbers, opt-in via `-m slow` (`test_perf.py`) ## CI/CD diff --git a/crates/fec-py/tests/test_fixtures.py b/crates/fec-py/tests/test_fixtures.py index bc56847..8094f61 100644 --- a/crates/fec-py/tests/test_fixtures.py +++ b/crates/fec-py/tests/test_fixtures.py @@ -7,7 +7,7 @@ from pathlib import Path from libfec_parser import fecfile -from libfec_parser.parser import Filing +from libfec_parser.parser import open # name -> (fec_version, form_type, filer_id, itemization row count) EXPECTED = { @@ -25,14 +25,15 @@ def test_all_fixtures_present(all_fixture_files): def test_parser_api_parses_fixture(fec_fixture: Path): - """Filing() parses every fixture with the documented shape""" + """open() parses every fixture with the documented shape""" version, form_type, filer_id, n_rows = EXPECTED[fec_fixture.name] - filing = Filing(str(fec_fixture)) + reader = open(fec_fixture) - assert filing.header.fec_version == version - assert filing.cover.form_type == form_type - assert filing.cover.filer_id == filer_id - assert len(filing.itemizations) == n_rows + assert reader.header.fec_version == version + assert reader.cover.form_type == form_type + assert reader.cover.filer_id == filer_id + assert reader.id == fec_fixture.stem + assert sum(1 for _ in reader) == n_rows def test_fecfile_api_parses_fixture(fec_fixture: Path): @@ -49,17 +50,17 @@ def test_fecfile_api_parses_fixture(fec_fixture: Path): def test_both_apis_agree_on_row_count(fec_fixture: Path): """The parser and fecfile layers see the same number of itemizations""" - filing = Filing(str(fec_fixture)) + n_rows = sum(1 for _ in open(fec_fixture)) result = fecfile.from_file(str(fec_fixture)) - assert len(filing.itemizations) == sum( - len(v) for v in result["itemizations"].values() - ) + assert n_rows == sum(len(v) for v in result["itemizations"].values()) def test_bytes_and_path_agree(fec_fixture: Path): - """Filing(bytes) and Filing(path) produce the same filing""" - from_path = Filing(str(fec_fixture)) - from_bytes = Filing(fec_fixture.read_bytes()) + """open(bytes) and open(path) read the same filing (`id` aside)""" + from_path = open(fec_fixture) + from_bytes = open(fec_fixture.read_bytes()) - assert repr(from_path) == repr(from_bytes) + assert from_path.cover.fields() == from_bytes.cover.fields() + assert from_path.cover_row.fields() == from_bytes.cover_row.fields() + assert [r.fields() for r in from_path] == [r.fields() for r in from_bytes] diff --git a/crates/fec-py/tests/test_pandas.py b/crates/fec-py/tests/test_pandas.py new file mode 100644 index 0000000..f9b9e31 --- /dev/null +++ b/crates/fec-py/tests/test_pandas.py @@ -0,0 +1,20 @@ +""" +pandas interop for Filing.rows (Mapping.register(Row), ticket 13). + +No `importorskip`: a skipped test is a test that never runs (tests/conftest.py). +CI installs pandas alongside the wheel for the pytest step (test-python.yml). +""" +import pandas as pd # type: ignore[import-untyped] # no pandas-stubs dev dependency +from datetime import date + +from libfec_parser import read + + +def test_dataframe_from_rows_has_typed_columns(pac_fec_file): + """pd.DataFrame(filing.rows) gets typed amount/date columns via Mapping.register(Row)""" + df = pd.DataFrame(read(pac_fec_file).rows) + + assert list(df.columns)[:3] == ["form_type", "filer_committee_id_number", "transaction_id"] + assert df["contribution_amount"].dtype == "float64" # all SA rows have a float or None + assert isinstance(df["contribution_date"].dropna().iloc[0], date) # object dtype of datetime.date; pandas does not auto-convert date -> datetime64 + assert pd.to_datetime(df["contribution_date"]).dt.year.min() == 2023 diff --git a/crates/fec-py/tests/test_parser.py b/crates/fec-py/tests/test_parser.py index 5750aa3..0aeccbd 100644 --- a/crates/fec-py/tests/test_parser.py +++ b/crates/fec-py/tests/test_parser.py @@ -1,8 +1,28 @@ """ Tests for libfec_parser.parser module """ +import builtins +import errno +import pickle +from collections.abc import Mapping +from datetime import date + import pytest -from libfec_parser.parser import fec_header, Filing, Header, Cover, Itemization + +# `open` here is `libfec_parser.parser.open`, not the builtin — reach the real +# one through `builtins.open`, exactly as `parser.py` itself has to. +from libfec_parser.parser import ( + fec_header, + Filing, + Header, + Cover, + Row, + open, + read, + FecError, + FecParseError, + MissingMappingError, +) # The fixtures (`sample_fec_file`, `sample_fec_bytes`, `all_fixture_files`, …) # live in conftest.py. The primary one is tests/fixtures/1921705.fec: @@ -29,13 +49,33 @@ def test_fec_header_with_empty_bytes(self): with pytest.raises(ValueError): fec_header(b"") + def test_fec_header_accepts_path_and_file(self, sample_fec_file, sample_fec_bytes): + """`fec_header()` takes every source `open()` does""" + assert fec_header(sample_fec_bytes) == "8.5" + assert fec_header(str(sample_fec_file)) == "8.5" + assert fec_header(sample_fec_file) == "8.5" + with builtins.open(sample_fec_file, "rb") as f: + assert fec_header(f) == "8.5" + + def test_fec_header_ignores_cover(self): + """Reads only the HDR record: a filing whose cover has no mapping still + fails `open()` but `fec_header()` reads past it.""" + hdr = b"HDR\x1cFEC\x1c8.5\x1cFECfile\x1c8.5.0.0(f33)\x1c\x1c\n" + garbage_cover = b"ZZZZ\x1cwhatever\n" + filing = hdr + garbage_cover + + with pytest.raises(FecParseError): + open(filing) + + assert fec_header(filing) == "8.5" + class TestHeader: """Tests for Header class""" def test_header_attributes(self, sample_fec_file): """Test that Header has expected attributes""" - filing = Filing(str(sample_fec_file)) + filing = open(sample_fec_file) header = filing.header assert isinstance(header, Header) @@ -50,7 +90,7 @@ def test_header_attributes(self, sample_fec_file): def test_header_values(self, sample_fec_file): """Test Header values for the known fixture 1921705.fec""" - header = Filing(str(sample_fec_file)).header + header = open(sample_fec_file).header assert header.fec_version == "8.5" assert header.record_type == "HDR" @@ -59,7 +99,7 @@ def test_header_values(self, sample_fec_file): def test_header_repr(self, sample_fec_file): """Test Header __repr__""" - filing = Filing(str(sample_fec_file)) + filing = open(sample_fec_file) header = filing.header repr_str = repr(header) @@ -73,7 +113,7 @@ class TestCover: def test_cover_attributes(self, sample_fec_file): """Test that Cover has expected attributes""" - filing = Filing(str(sample_fec_file)) + filing = open(sample_fec_file) cover = filing.cover assert isinstance(cover, Cover) @@ -86,7 +126,7 @@ def test_cover_attributes(self, sample_fec_file): def test_cover_values(self, sample_fec_file): """Test Cover values for the known fixture 1921705.fec""" - cover = Filing(str(sample_fec_file)).cover + cover = open(sample_fec_file).cover assert cover.form_type == "F3N" assert cover.filer_id == "C00900860" @@ -94,7 +134,7 @@ def test_cover_values(self, sample_fec_file): def test_cover_repr(self, sample_fec_file): """Test Cover __repr__""" - filing = Filing(str(sample_fec_file)) + filing = open(sample_fec_file) cover = filing.cover repr_str = repr(cover) @@ -104,7 +144,7 @@ def test_cover_repr(self, sample_fec_file): def test_cover_fields_method(self, sample_fec_file): """Test Cover.fields() returns a dictionary""" - filing = Filing(str(sample_fec_file)) + filing = open(sample_fec_file) cover = filing.cover fields = cover.fields() @@ -114,131 +154,314 @@ def test_cover_fields_method(self, sample_fec_file): assert 'filer_name' in fields -class TestItemization: - """Tests for Itemization class, against the known fixture 1921705.fec""" +def _edit_first_row(raw: bytes, edit) -> bytes: + """Return ``raw`` with ``edit`` applied to its first itemization line (line 3).""" + lines = raw.split(b"\n") + assert lines[2].startswith(b"SA11AI"), lines[2][:20] + lines[2] = edit(lines[2]) + return b"\n".join(lines) - def test_itemization_attributes(self, sample_fec_file): - """Test that Itemization has expected attributes""" - itemization = Filing(str(sample_fec_file)).itemizations[0] - assert isinstance(itemization, Itemization) - assert itemization.row_type == "SA11AI" +class TestRow: + """Tests for Row, against the known fixture 1921705.fec. - def test_itemization_row_type_order(self, sample_fec_file): - """Test the exact itemization row types, in file order""" - filing = Filing(str(sample_fec_file)) + Its first itemization is line 3, an ``SA11AI`` with 45 fields: + ``SA11AI|C00900860|SA11AI.4264|||IND||Weed|Richard||||14 Stacey St||…`` + """ + + def test_attributes(self, sample_fec_file): + """A Row exposes row_type and is a Mapping""" + row = list(open(sample_fec_file))[0] - assert len(filing.itemizations) == 20 - assert [i.row_type for i in filing.itemizations] == ( + assert isinstance(row, Row) + assert isinstance(row, Mapping) + assert row.row_type == "SA11AI" + + def test_row_type_order(self, sample_fec_file): + """Test the exact row types, in file order""" + rows = list(open(sample_fec_file)) + + assert len(rows) == 20 + assert [r.row_type for r in rows] == ( ["SA11AI"] + ["SA11C"] * 13 + ["SA11D"] + ["SB17"] * 5 ) - def test_itemization_repr(self, sample_fec_file): - """Test Itemization __repr__""" - itemization = Filing(str(sample_fec_file)).itemizations[0] + def test_getitem_by_name_is_typed(self, sample_fec_file): + """By name: amounts are float, dates are date, text stays str (`""` if empty)""" + row = list(open(sample_fec_file))[0] - assert repr(itemization) == "Itemization(row_type='SA11AI', 45 fields)" + assert row["contribution_amount"] == 500.0 + assert isinstance(row["contribution_amount"], float) + assert row["contribution_date"] == date(2025, 7, 7) + assert row["contributor_last_name"] == "Weed" + assert row["contributor_middle_name"] == "" + assert row["contribution_purpose_descrip"] == "" - def test_itemization_len(self, sample_fec_file): - """Test Itemization __len__""" - itemization = Filing(str(sample_fec_file)).itemizations[0] + def test_getitem_unknown_name_is_key_error(self, sample_fec_file): + """An unmapped column name raises KeyError""" + row = list(open(sample_fec_file))[0] - assert len(itemization) == 45 + with pytest.raises(KeyError): + _ = row["not_a_column"] - def test_itemization_getitem_positive_index(self, sample_fec_file): - """Test Itemization __getitem__ with positive index""" - itemization = Filing(str(sample_fec_file)).itemizations[0] + def test_getitem_by_position_is_raw(self, sample_fec_file): + """By position: the raw field, negative indexes allowed""" + row = list(open(sample_fec_file))[0] - assert itemization[0] == "SA11AI" - assert itemization[1] == "C00900860" + assert row[0] == "SA11AI" + assert row[1] == "C00900860" + assert row[20] == "500.00" + assert row[-1] == row[len(row.fields()) - 1] - def test_itemization_getitem_negative_index(self, sample_fec_file): - """Test Itemization __getitem__ with negative index""" - itemization = Filing(str(sample_fec_file)).itemizations[0] + def test_getitem_out_of_bounds(self, sample_fec_file): + """An out-of-range position raises IndexError""" + row = list(open(sample_fec_file))[0] - assert itemization[-1] == itemization[len(itemization) - 1] - assert isinstance(itemization[-1], str) + with pytest.raises(IndexError): + _ = row[9999] - def test_itemization_getitem_out_of_bounds(self, sample_fec_file): - """Test Itemization __getitem__ with out of bounds index""" - itemization = Filing(str(sample_fec_file)).itemizations[0] + def test_getitem_bad_key_type(self, sample_fec_file): + """A key that is neither str, int nor slice raises TypeError""" + row = list(open(sample_fec_file))[0] - with pytest.raises(IndexError): - _ = itemization[9999] + with pytest.raises(TypeError): + _ = row[object()] # type: ignore[call-overload] # invalid key, on purpose + + def test_empty_amount_is_none(self, sample_fec_file): + """An empty amount column reads as None, not `""` or 0.0""" + row = next(iter(open(sample_fec_file).rows("SB17"))) + + assert row.fields()[21] == "" + assert row["semi_annual_refunded_bundled_amt"] is None + + def test_garbage_amount_is_raw_str(self, sample_fec_bytes): + """An amount that does not parse comes back as the raw string""" + raw = _edit_first_row( + sample_fec_bytes, lambda line: line.replace(b"\x1c500.00\x1c", b"\x1cN/A\x1c", 1) + ) + row = list(open(raw))[0] + + assert row["contribution_amount"] == "N/A" + assert row["contribution_aggregate"] == 500.0 + + def test_garbage_date_is_raw_str(self, sample_fec_bytes): + """A date that does not parse comes back as the raw string""" + raw = _edit_first_row( + sample_fec_bytes, lambda line: line.replace(b"\x1c20250707\x1c", b"\x1cnotadate\x1c", 1) + ) + row = list(open(raw))[0] + + assert row["contribution_date"] == "notadate" + + def test_slice(self, sample_fec_file): + """A slice yields raw fields as a list""" + row = list(open(sample_fec_file))[0] + + assert row[0:3] == ["SA11AI", "C00900860", "SA11AI.4264"] + assert row[:2] == ["SA11AI", "C00900860"] + assert row[-2:] == row.fields()[-2:] + assert row[0:6:2] == ["SA11AI", "SA11AI.4264", ""] - def test_itemization_fields_method(self, sample_fec_file): - """Test Itemization.fields() returns a list""" - itemization = Filing(str(sample_fec_file)).itemizations[0] - fields = itemization.fields() + def test_len_is_column_count(self, sample_fec_file): + """len(row) counts columns, not raw fields""" + row = list(open(sample_fec_file))[0] + + assert len(row) == 45 + assert len(row) == len(row.keys()) + + def test_iter_yields_names(self, sample_fec_file): + """Iterating a row yields column names, in column order""" + row = list(open(sample_fec_file))[0] + + assert list(row) == row.keys() + assert list(row)[:3] == [ + "form_type", + "filer_committee_id_number", + "transaction_id", + ] + + def test_values_and_items(self, sample_fec_file): + """values() and items() are lists of typed values""" + row = list(open(sample_fec_file))[0] + + assert isinstance(row.values(), list) + assert isinstance(row.items(), list) + assert row.items() == list(zip(row.keys(), row.values())) + assert row.values()[20] == 500.0 + + def test_dict_roundtrip(self, sample_fec_file): + """dict(row) maps every column name to its typed value""" + row = list(open(sample_fec_file))[0] + as_dict = dict(row) + + assert len(as_dict) == len(row) + assert as_dict["contributor_state"] == "MA" + assert as_dict["contribution_amount"] == 500.0 + + def test_contains(self, sample_fec_file): + """Membership is over column names; an int key is never a column""" + row = list(open(sample_fec_file))[0] + + assert "contribution_amount" in row + assert "not_a_column" not in row + assert 0 not in row + + def test_get_default(self, sample_fec_file): + """get() behaves like dict.get""" + row = list(open(sample_fec_file))[0] + + assert row.get("contribution_amount") == 500.0 + assert row.get("not_a_column") is None + assert row.get("not_a_column", "fallback") == "fallback" + + def test_fields_method(self, sample_fec_file): + """fields() returns every raw field, in file order""" + row = list(open(sample_fec_file))[0] + fields = row.fields() assert isinstance(fields, list) assert len(fields) == 45 assert all(isinstance(f, str) for f in fields) assert fields[0] == "SA11AI" + def test_extra_fields_trailing_empty_ignored(self, sample_fec_bytes): + """A stray trailing delimiter is not data""" + raw = _edit_first_row(sample_fec_bytes, lambda line: line + b"\x1c") + row = list(open(raw))[0] + + assert len(row.fields()) == 46 + assert row.extra_fields == [] + assert len(row) == 45 + + def test_extra_fields_non_empty_kept(self, sample_fec_bytes): + """A non-empty extra field is kept, positionally, and never in keys()""" + raw = _edit_first_row(sample_fec_bytes, lambda line: line + b"\x1cEXTRA") + row = list(open(raw))[0] + + assert row.extra_fields == ["EXTRA"] + assert row[45] == "EXTRA" + assert "EXTRA" not in row.keys() + assert len(row) == 45 + + def test_short_row_missing_is_none(self, sample_fec_bytes): + """Columns past the end of a short row read as None""" + raw = _edit_first_row( + sample_fec_bytes, lambda line: b"\x1c".join(line.split(b"\x1c")[:11]) + ) + row = list(open(raw))[0] + + assert len(row.fields()) == 11 + assert len(row) == 45 + assert row["contributor_prefix"] == "" # column 10, the last one present + assert row["contributor_suffix"] is None # column 11, past the end + assert row["contribution_amount"] is None + assert row["contributor_first_name"] == "Richard" + + def test_eq_hash(self, sample_fec_file): + """Rows compare and hash by (row_type, version, raw fields)""" + rows = list(open(sample_fec_file)) + + assert rows[0] == rows[0] + assert rows[0] != rows[1] + assert rows[0] != "not a row" + assert len({rows[0], rows[0]}) == 1 + assert hash(rows[0]) == hash(list(open(sample_fec_file))[0]) + + def test_pickle_roundtrip(self, sample_fec_file): + """A Row survives pickling, line number included""" + row = list(open(sample_fec_file))[0] + restored = pickle.loads(pickle.dumps(row)) + + assert restored == row + assert restored.line == row.line + assert restored["contribution_amount"] == 500.0 + + def test_repr(self, sample_fec_file): + """Test Row __repr__""" + row = list(open(sample_fec_file))[0] + + assert repr(row) == "Row(row_type='SA11AI', line=3, 45 fields)" + + def test_line(self, sample_fec_file): + """The first itemization is on line 3; the rest follow one per line""" + rows = list(open(sample_fec_file)) + + assert rows[0].line == 3 + assert [r.line for r in rows] == list(range(3, 23)) + + def test_keys_are_interned(self, sample_fec_file): + """Column names are one object per (row_type, version), not per row""" + rows = list(open(sample_fec_file)) + + assert rows[0].keys()[0] is rows[1].keys()[0] + assert rows[0].keys()[7] is rows[13].keys()[7] + + def test_unicode_replacement_row_has_line(self, fec_fixture): + """Every row of every fixture knows its line, even after lossy decoding""" + for row in open(fec_fixture): + assert row.line >= 3 + class TestFiling: - """Tests for Filing class""" - + """Tests for the eager `Filing` class: header, cover and every row, parsed once.""" + def test_filing_from_path_string(self, sample_fec_file): """Test Filing initialization with file path string""" filing = Filing(str(sample_fec_file)) - + assert isinstance(filing, Filing) assert isinstance(filing.header, Header) assert isinstance(filing.cover, Cover) - assert isinstance(filing.itemizations, list) - + assert isinstance(filing.rows, list) + + def test_filing_from_pathlib(self, sample_fec_file): + """Test Filing initialization with a pathlib.Path""" + filing = Filing(sample_fec_file) + + assert isinstance(filing, Filing) + assert len(filing.rows) == 20 + def test_filing_from_bytes(self, sample_fec_bytes): """Test Filing initialization with bytes""" filing = Filing(sample_fec_bytes) - + assert isinstance(filing, Filing) assert isinstance(filing.header, Header) assert isinstance(filing.cover, Cover) - - def test_filing_from_file_object(self, sample_fec_file): - """Test Filing initialization with file-like object""" - with open(sample_fec_file, 'rb') as f: - filing = Filing(f) - - assert isinstance(filing, Filing) - assert isinstance(filing.header, Header) - assert isinstance(filing.cover, Cover) - + def test_filing_repr(self, sample_fec_file): """Test Filing __repr__""" filing = Filing(str(sample_fec_file)) assert repr(filing) == ( - "Filing(form_type='F3N', filer_id='C00900860', 20 itemizations)" + "Filing(id='1921705', form_type='F3N', filer_id='C00900860', 20 rows)" ) def test_filing_header_property(self, sample_fec_file): """Test Filing.header property""" - filing = Filing(str(sample_fec_file)) - header = filing.header - + header = Filing(str(sample_fec_file)).header + assert isinstance(header, Header) assert header.fec_version - + def test_filing_cover_property(self, sample_fec_file): """Test Filing.cover property""" - filing = Filing(str(sample_fec_file)) - cover = filing.cover - + cover = Filing(str(sample_fec_file)).cover + assert isinstance(cover, Cover) assert cover.form_type assert cover.filer_id - + def test_filing_itemizations_property(self, sample_fec_file): - """Test Filing.itemizations property""" + """Filing.itemizations is a deprecated alias of Filing.rows""" filing = Filing(str(sample_fec_file)) - itemizations = filing.itemizations - assert isinstance(itemizations, list) + with pytest.warns(DeprecationWarning): + itemizations = filing.itemizations + + assert itemizations is filing.rows assert len(itemizations) == 20 - assert all(isinstance(item, Itemization) for item in itemizations) + assert all(isinstance(item, Row) for item in itemizations) def test_filing_many_itemizations(self, pac_fec_file): """Test a filing with many rows: 1721696.fec, v8.4 F3XN, 1,387 rows""" @@ -247,26 +470,95 @@ def test_filing_many_itemizations(self, pac_fec_file): assert filing.header.fec_version == "8.4" assert filing.cover.form_type == "F3XN" assert filing.cover.filer_id == "C00016683" - assert len(filing.itemizations) == 1387 + assert len(filing.rows) == 1387 def test_filing_with_no_itemizations(self, f99_fec_file): """Test the F99 fixture: a [BEGINTEXT] filing with zero rows""" filing = Filing(str(f99_fec_file)) assert filing.cover.form_type == "F99" - assert filing.itemizations == [] + assert filing.rows == [] + + def test_filing_from_file_object(self, sample_fec_file): + """Test Filing initialization with a binary file object""" + with builtins.open(sample_fec_file, "rb") as f: + filing = Filing(f) + + assert isinstance(filing, Filing) + assert isinstance(filing.header, Header) + assert isinstance(filing.cover, Cover) + assert len(filing.rows) == 20 def test_filing_with_invalid_path(self): """Test Filing with non-existent file path""" - with pytest.raises(IOError): + with pytest.raises(FileNotFoundError): Filing("/path/that/does/not/exist.fec") - + def test_filing_with_invalid_type(self): """Test Filing with invalid input type""" with pytest.raises(TypeError): Filing(12345) # type: ignore[arg-type] # invalid type, on purpose - + def test_filing_with_invalid_data(self): """Test Filing with invalid FEC data""" with pytest.raises(ValueError): Filing(b"invalid fec data") + + def test_rows_identity(self, sample_fec_file): + """`rows` is a plain cached attribute; so is `header`""" + filing = Filing(sample_fec_file) + + assert filing.rows is filing.rows + assert filing.header is filing.header + + def test_filing_is_iterable_and_sized(self, sample_fec_file): + """iter(filing) and len(filing) delegate to rows""" + filing = Filing(sample_fec_file) + + assert len(filing) == 20 + assert list(filing) == filing.rows + + def test_read_is_filing(self, sample_fec_file): + """read() is a thin function wrapper over Filing""" + filing = read(sample_fec_file) + + assert isinstance(filing, Filing) + assert len(filing.rows) == 20 + + def test_eager_raises_missing_mapping(self, sample_fec_bytes): + """An unmapped row type raises out of Filing(...) itself (eager = strict)""" + raw = sample_fec_bytes.replace(b"\nSA11C\x1c", b"\nZZZZ\x1c", 1) + + with pytest.raises(MissingMappingError) as ei: + Filing(raw) + + assert (ei.value.row_type, ei.value.version, ei.value.line) == ("ZZZZ", "8.5", 4) + + +class TestErrors: + """Tests for the FecError/FecParseError/MissingMappingError hierarchy""" + + def test_missing_file_is_file_not_found(self, tmp_path): + missing = tmp_path / "nope.fec" + with pytest.raises(FileNotFoundError) as ei: + open(str(missing)) + assert ei.value.errno == errno.ENOENT + assert ei.value.filename == str(missing) + + def test_invalid_source_type_is_type_error(self): + with pytest.raises(TypeError): + open(12345) # type: ignore[arg-type] # invalid type, on purpose + + def test_garbage_is_parse_error(self): + with pytest.raises(FecParseError): + open(b"invalid fec data") + + def test_hierarchy(self): + assert issubclass(FecParseError, FecError) and issubclass(FecError, ValueError) + assert issubclass(MissingMappingError, FecError) + assert FecError.__module__ == "libfec_parser.parser" + + def test_missing_mapping_error_attributes(self): + e = MissingMappingError("ZZZ", "8.4", 7) + assert (e.row_type, e.version, e.line) == ("ZZZ", "8.4", 7) + assert "ZZZ" in str(e) and "8.4" in str(e) diff --git a/crates/fec-py/tests/test_perf.py b/crates/fec-py/tests/test_perf.py new file mode 100644 index 0000000..b2af9cb --- /dev/null +++ b/crates/fec-py/tests/test_perf.py @@ -0,0 +1,158 @@ +"""Slow tests for the Phase 2 done-when numbers (roadmap: `plans/python/05-roadmap.md:64-66`). + +All ``@pytest.mark.slow`` (opt-in, see ``tests/conftest.py``), and all use +``benchmark_fec_file`` (``conftest.py``), which ``pytest.skip``s if the 91 MB +filing is absent -- the one sanctioned skip, since that file is gitignored. + +RSS is measured in a fresh subprocess: ``resource.getrusage(...).ru_maxrss`` is +a process-wide high-water mark, so measuring it in the already-running pytest +process would include whatever pytest/mypy/etc. had already allocated. +""" +import subprocess +import sys +import threading +import time + +import pytest + +from libfec_parser import read + + +def _run(code: str) -> str: + """Run `code` in a fresh interpreter; return its stdout.""" + result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True) + assert result.returncode == 0, result.stderr + return result.stdout + + +RSS_SNIPPET = """ +import resource, sys +{body} +r = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss +print(r / 1e6 if sys.platform == 'darwin' else r / 1e3) +""" + + +def _peak_mb(body: str) -> float: + """Run `body` in a fresh interpreter, returning its peak RSS in MB. + + ``ru_maxrss`` is bytes on macOS and KB on Linux; normalised above. + """ + return float(_run(RSS_SNIPPET.format(body=body)).strip()) + + +@pytest.mark.slow +@pytest.mark.skipif(sys.platform == "win32", reason="no getrusage on Windows") +def test_open_streams_under_100mb(benchmark_fec_file): + """Phase 2 done-when: iterating the 91 MB filing via open() peaks under 100 MB. + + Measured 2026-09-18 (this ticket, M-series Mac, release build, CPython 3.13): + ~32-39 MB peak RSS across a few runs, against a ~15-20 MB base interpreter -- + well under the roadmap's 100 MB done-when. The assert below is tightened to + 64 MB per ticket 19 ("if the real number lands near 60 MB, tighten to 64"): + a regression guard with headroom, not the done-when itself. + """ + body = ( + "import libfec_parser\n" + f"n = sum(1 for _ in libfec_parser.open({str(benchmark_fec_file)!r}))\n" + "assert n == 408160" + ) + peak = _peak_mb(body) + assert peak < 64, f"peak RSS {peak:.1f} MB" + + +@pytest.mark.slow +def test_read_rows_access_is_free(benchmark_fec_file): + """N2 regression guard: 20 accesses to `.rows` cost ~nothing (it's a cached list).""" + f = read(benchmark_fec_file) + t = time.perf_counter() + for _ in range(20): + len(f.rows) + assert time.perf_counter() - t < 0.01 + + +@pytest.mark.slow +def test_background_thread_progresses_during_parse(benchmark_fec_file): + """Phase 2 done-when: a second thread makes progress during a parse. + + The honest version of ``test_reader.py::test_gil_released`` (which uses the + 263 KB fixture so it stays fast on a debug build): here we parse the full + 91 MB filing once and check that a ticking background thread got at least + 50% of the ticks its measured duration implies -- duration-based, not an + absolute tick count, because ``make test-slow``/``make bench``/CI install + the --release extension, where this parse is much faster than under + ``make test``'s debug build. Same tick-counting pattern as the old + ``plans/python/probes/edge.py`` probe. + """ + import libfec_parser + + tick_interval = 0.01 + ticks: list[float] = [] + stop = threading.Event() + + def ticker() -> None: + while not stop.is_set(): + ticks.append(time.monotonic()) + time.sleep(tick_interval) + + thread = threading.Thread(target=ticker, daemon=True) + thread.start() + try: + time.sleep(0.05) # let the ticker warm up before starting the clock + n0 = len(ticks) + started = time.monotonic() + n = sum(1 for _ in libfec_parser.open(benchmark_fec_file)) + elapsed = time.monotonic() - started + n1 = len(ticks) + finally: + stop.set() + thread.join(timeout=5) + + assert n == 408160 + during = n1 - n0 + expected = elapsed / tick_interval + assert during >= 0.5 * expected, ( + f"only {during} ticks in {elapsed:.3f}s (expected ~{expected:.0f})" + ) + + +# The subprocess the zero-copy test measures: read the filing into a `bytes`, +# iterate it through `open()`, report peak RSS alongside the size of the bytes. +# Moved here from test_reader.py (ticket 15) to share this module's RSS harness. +_ZERO_COPY_SCRIPT = """ +import resource, sys +import libfec_parser + +with open(sys.argv[1], "rb") as f: + data = f.read() +rows = sum(1 for _ in libfec_parser.open(data)) +peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss +# ru_maxrss is bytes on macOS, kilobytes on Linux. +if sys.platform != "darwin": + peak *= 1024 +print(peak, len(data), rows) +""" + + +@pytest.mark.slow +def test_zero_copy_bytes(benchmark_fec_file): + """Iterating a 91 MB `bytes` costs well under 100 MB on top of the bytes + + The buffer is read in place, so the only per-row cost is the row being + handed to Python, which iteration drops again immediately. + """ + result = subprocess.run( + [sys.executable, "-c", _ZERO_COPY_SCRIPT, str(benchmark_fec_file)], + capture_output=True, + text=True, + check=True, + ) + peak, size, rows = (int(v) for v in result.stdout.split()) + + assert rows > 0 + overhead = peak - size + assert overhead < 100 * 1024 * 1024, ( + f"peak RSS {peak / 1e6:.0f} MB over a {size / 1e6:.0f} MB bytes object " + f"= {overhead / 1e6:.0f} MB of overhead" + ) + diff --git a/crates/fec-py/tests/test_reader.py b/crates/fec-py/tests/test_reader.py new file mode 100644 index 0000000..525143a --- /dev/null +++ b/crates/fec-py/tests/test_reader.py @@ -0,0 +1,734 @@ +""" +Tests for ``libfec_parser.open()`` and the ``FilingReader`` it returns. + +The fixtures (``sample_fec_file``, ``pac_fec_file``, …) live in conftest.py. +The primary one is tests/fixtures/1921705.fec: v8.5, F3N, filer C00900860 +"Jason Byors for Congress", 20 itemizations (1 SA11AI, 13 SA11C, 1 SA11D, +5 SB17) in that file order, on lines 3-22. +""" +import builtins +import io +import mmap +import shutil +import subprocess +import sys +import threading +import time +from array import array +from datetime import date + +import pytest +from libfec_parser.parser import ( + Cover, + FecParseError, + FilingReader, + Header, + MissingMappingError, + Row, + open, +) + +ROW_TYPES = ["SA11AI"] + ["SA11C"] * 13 + ["SA11D"] + ["SB17"] * 5 + + +class TestOpen: + """Accepted sources, and what `open()` hands back""" + + def test_open_path_str(self, sample_fec_file): + reader = open(str(sample_fec_file)) + + assert isinstance(reader, FilingReader) + assert len(list(reader)) == 20 + + def test_open_pathlib_path(self, sample_fec_file): + """os.PathLike goes through os.fspath""" + assert len(list(open(sample_fec_file))) == 20 + + def test_open_bytes(self, sample_fec_bytes): + assert len(list(open(sample_fec_bytes))) == 20 + + def test_open_rejects_other_types(self): + with pytest.raises(TypeError): + open(12345) # type: ignore[arg-type] # invalid type, on purpose + + def test_open_missing_path_is_file_not_found(self, tmp_path): + with pytest.raises(FileNotFoundError): + open(str(tmp_path / "nope.fec")) + + def test_open_garbage_is_parse_error(self): + with pytest.raises(ValueError): + open(b"invalid fec data") + + def test_repr(self, sample_fec_file): + assert repr(open(sample_fec_file)) == ( + "FilingReader(id='1921705', form_type='F3N', filer_id='C00900860')" + ) + + def test_no_len(self, sample_fec_file): + """The row count is unknown without a full pass, so there is no __len__""" + with pytest.raises(TypeError): + len(open(sample_fec_file)) # type: ignore[arg-type] # no __len__, on purpose + + def test_source_length(self, sample_fec_file): + assert open(sample_fec_file).source_length == sample_fec_file.stat().st_size + + def test_fec_version(self, sample_fec_file): + reader = open(sample_fec_file) + + assert reader.fec_version == "8.5" + assert reader.fec_version == reader.header.fec_version + + +class TestIteration: + """Iteration order, laziness and the rows it yields""" + + def test_iter_returns_self(self, sample_fec_file): + reader = open(sample_fec_file) + + assert iter(reader) is reader + + def test_iteration_order_and_types(self, sample_fec_file): + rows = list(open(sample_fec_file)) + + assert [r.row_type for r in rows] == ROW_TYPES + assert all(isinstance(r, Row) for r in rows) + + def test_lines(self, sample_fec_file): + assert [r.line for r in open(sample_fec_file)] == list(range(3, 23)) + + def test_single_pass(self, sample_fec_file): + """A reader is exhausted after one pass; the second yields nothing""" + reader = open(sample_fec_file) + + assert len(list(reader)) == 20 + assert list(reader) == [] + + def test_typed_values_through_reader(self, sample_fec_file): + """The version is plumbed through, so columns come back typed""" + row = next(iter(open(sample_fec_file))) + + assert row["contribution_amount"] == 500.0 + assert row["contribution_date"] == date(2025, 7, 7) + + def test_pac_row_count(self, pac_fec_file): + assert sum(1 for _ in open(pac_fec_file)) == 1387 + + def test_f99_zero_rows(self, f99_fec_file): + """The F99 fixture is a [BEGINTEXT] filing with no itemizations""" + reader = open(f99_fec_file) + + assert reader.cover.form_type == "F99" + assert list(reader) == [] + + +class TestHeaderAndCover: + """`header`/`cover`/`cover_row`, parsed eagerly and identity-stable""" + + def test_available_before_iteration(self, sample_fec_file): + reader = open(sample_fec_file) + + assert isinstance(reader.header, Header) + assert isinstance(reader.cover, Cover) + assert isinstance(reader.cover_row, Row) + + def test_header_cover_identity(self, sample_fec_file): + reader = open(sample_fec_file) + + assert reader.header is reader.header + assert reader.cover is reader.cover + assert reader.cover_row is reader.cover_row + + def test_cover_dates_are_dates(self, sample_fec_file): + cover = open(sample_fec_file).cover + + assert cover.coverage_from_date == date(2025, 7, 1) + assert cover.coverage_through_date == date(2025, 9, 30) + assert isinstance(cover.coverage_from_date, date) + + def test_cover_fields_keys_and_typed_dates(self, sample_fec_file): + fields = open(sample_fec_file).cover.fields() + + assert set(fields) == { + "form_type", + "filer_id", + "filer_name", + "report_code", + "coverage_from_date", + "coverage_through_date", + } + assert fields["coverage_from_date"] == date(2025, 7, 1) + + def test_cover_row_is_the_cover_line(self, sample_fec_file): + cover_row = open(sample_fec_file).cover_row + + assert cover_row.row_type == "F3N" + assert cover_row.line == 2 + assert cover_row["filer_committee_id_number"] == "C00900860" + + def test_cover_row_values(self, pac_fec_file): + """`cover_row` follows the same rules as any Row (Q14-Q16, Q6): + typed by name, raw by position, `row_type`/`line` from the record.""" + cover_row = open(pac_fec_file).cover_row + + assert cover_row["col_a_total_receipts"] == 83741.93 + assert cover_row["coverage_from_date"] == date(2023, 7, 1) + assert cover_row[0] == "F3XN" + assert cover_row.row_type == "F3XN" + assert cover_row.line == 2 + + def test_cover_row_is_row_and_identity(self, pac_fec_file): + reader = open(pac_fec_file) + + assert isinstance(reader.cover_row, Row) + assert reader.cover_row is reader.cover_row + + def test_cover_fields_six_keys_typed(self, sample_fec_file): + """`Cover.fields()` keeps its six keys (Q19), typed dates""" + fields = open(sample_fec_file).cover.fields() + + assert set(fields) == { + "form_type", + "filer_id", + "filer_name", + "report_code", + "coverage_from_date", + "coverage_through_date", + } + assert fields["coverage_from_date"] == date(2025, 7, 1) + + def test_cover_row_keys_count(self, pac_fec_file): + """pac_fec_file is F3XN/8.4; column_names_for_field("F3XN", "8.4") + (checked directly against fec_parser::mappings on this tip) returns + 123 columns, including the deferred `_TODO_DUP` names.""" + cover_row = open(pac_fec_file).cover_row + + assert len(cover_row) > 100 + assert len(cover_row) == 123 + + def test_header_and_cover_survive_close(self, sample_fec_file): + """They are parsed up front, so closing the source does not lose them""" + reader = open(sample_fec_file) + reader.close() + + assert reader.header.fec_version == "8.5" + assert reader.cover.form_type == "F3N" + + +class TestId: + """`reader.id`""" + + def test_id_from_path(self, sample_fec_file): + assert open(sample_fec_file).id == "1921705" + + def test_id_none_for_bytes(self, sample_fec_bytes): + assert open(sample_fec_bytes).id is None + + def test_id_strips_fec_prefix(self, sample_fec_file, tmp_path): + prefixed = tmp_path / "FEC-1921705.fec" + shutil.copyfile(sample_fec_file, prefixed) + + assert open(prefixed).id == "1921705" + + +class TestRowsFilter: + """`rows(*prefixes)` — filtered in Rust, before any Row is built""" + + def test_returns_self(self, sample_fec_file): + reader = open(sample_fec_file) + + assert reader.rows("SB") is reader + + def test_prefix_filter(self, sample_fec_file): + assert sum(1 for _ in open(sample_fec_file).rows("SB")) == 5 + + def test_prefix_filter_is_case_insensitive(self, sample_fec_file): + assert sum(1 for _ in open(sample_fec_file).rows("sa11c")) == 13 + + def test_several_prefixes(self, sample_fec_file): + assert sum(1 for _ in open(sample_fec_file).rows("SA", "SB17")) == 20 + + def test_no_match(self, sample_fec_file): + assert list(open(sample_fec_file).rows("ZZ")) == [] + + def test_no_args_clears_the_filter(self, sample_fec_file): + reader = open(sample_fec_file).rows("SB") + + assert sum(1 for _ in reader.rows()) == 20 + + def test_replaces_previous_filter(self, sample_fec_file): + reader = open(sample_fec_file).rows("SB").rows("SA11C") + + assert sum(1 for _ in reader) == 13 + + def test_rejects_non_str_prefix(self, sample_fec_file): + with pytest.raises(TypeError): + open(sample_fec_file).rows(17) # type: ignore[arg-type] # on purpose + + +class TestCloseAndContextManager: + def test_context_manager_yields_self_and_closes(self, sample_fec_file): + with open(sample_fec_file) as reader: + assert isinstance(reader, FilingReader) + assert not reader.closed + + assert reader.closed + + def test_iterate_after_close_raises(self, sample_fec_file): + reader = open(sample_fec_file) + reader.close() + + with pytest.raises(ValueError, match="closed filing"): + next(iter(reader)) + + def test_close_idempotent(self, sample_fec_file): + reader = open(sample_fec_file) + reader.close() + reader.close() + + assert reader.closed + + def test_closed_is_false_while_open(self, sample_fec_file): + assert open(sample_fec_file).closed is False + + +class TestMissingMapping: + def test_missing_mapping_is_raised_and_iteration_continues(self, sample_fec_bytes): + """An unmapped row type raises, but the reader stays usable""" + raw = sample_fec_bytes.replace(b"\nSA11C\x1c", b"\nZZZZ\x1c", 1) + reader = open(raw) + rows, errors = [], [] + + while True: + try: + rows.append(next(reader)) + except StopIteration: + break + except MissingMappingError as e: + errors.append(e) + + assert len(errors) == 1 + assert (errors[0].row_type, errors[0].version, errors[0].line) == ( + "ZZZZ", + "8.5", + 4, + ) + # every row but the one that was renamed, still in file order + expected = list(ROW_TYPES) + expected.remove("SA11C") + assert [r.row_type for r in rows] == expected + + +class TestThreading: + """The GIL is released while rows are pulled""" + + def test_gil_released(self, pac_fec_file): + """A background thread makes progress while the main thread parses. + + The workload is duration- not count-based: a release build parses the + 263 KB fixture an order of magnitude faster than a debug one, so a fixed + iteration count would finish before the ticker could say anything. + """ + parse_for = 0.25 # seconds of wall time spent parsing + ticks: list[float] = [] + stop = threading.Event() + + def ticker() -> None: + while not stop.is_set(): + ticks.append(time.monotonic()) + time.sleep(0.001) + + thread = threading.Thread(target=ticker, daemon=True) + thread.start() + try: + started = time.monotonic() + while time.monotonic() - started < parse_for: + assert sum(1 for _ in open(pac_fec_file)) == 1387 + elapsed = time.monotonic() - started + finally: + stop.set() + thread.join(timeout=5) + + # Loose on purpose: a smoke test, not a benchmark. The old eager + # binding, which held the GIL for the whole parse, got ~0 ticks here. + during = [t for t in ticks if t >= started] + assert len(during) >= 10, f"only {len(during)} ticks in {elapsed:.3f}s" + + def test_many_readers_in_threads(self, pac_fec_file): + """Four threads, four independent readers, no shared state""" + counts: list[int] = [] + lock = threading.Lock() + + def run() -> None: + n = sum(1 for _ in open(pac_fec_file)) + with lock: + counts.append(n) + + threads = [threading.Thread(target=run) for _ in range(4)] + for t in threads: + t.start() + for t in threads: + t.join(timeout=60) + + assert counts == [1387] * 4 + + +class TestBufferSources: + """Every bytes-like source goes through one zero-copy buffer path""" + + def test_open_bytearray(self, sample_fec_bytes): + assert len(list(open(bytearray(sample_fec_bytes)))) == 20 + + def test_open_memoryview(self, sample_fec_bytes): + assert len(list(open(memoryview(sample_fec_bytes)))) == 20 + + def test_open_memoryview_slice(self, sample_fec_bytes): + """A strided memoryview is not contiguous: gathered into a Vec, then parsed + + Every other byte is not a filing, so the only sane outcomes are a parse + error or a filing-shaped nothing — never a crash. + """ + try: + rows = list(open(memoryview(sample_fec_bytes)[::2])) + except FecParseError: + return + assert isinstance(rows, list) + + def test_open_mmap(self, sample_fec_file): + with builtins.open(sample_fec_file, "rb") as f: + with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mapped: + assert len(list(open(mapped))) == 20 + + def test_buffer_source_length(self, sample_fec_bytes): + assert open(bytearray(sample_fec_bytes)).source_length == len(sample_fec_bytes) + + def test_buffer_source_has_no_id(self, sample_fec_bytes): + """A buffer does not name itself""" + assert open(memoryview(sample_fec_bytes)).id is None + + def test_non_byte_buffer_raises_typeerror(self): + """`array('i')`, a NumPy float array, `memoryview(...).cast('I')`: not bytes-like""" + with pytest.raises(TypeError, match="bytes-like"): + open(array("i", [1, 2, 3])) # type: ignore[arg-type] # on purpose + + +class TestFileObjectSources: + """Binary file objects are pulled a chunk at a time, never `.read()` whole""" + + def test_open_binary_file_object(self, sample_fec_file): + with builtins.open(sample_fec_file, "rb") as f: + assert len(list(open(f))) == 20 + + def test_open_bytesio(self, sample_fec_bytes): + assert len(list(open(io.BytesIO(sample_fec_bytes)))) == 20 + + def test_open_urlopen_like(self, sample_fec_bytes): + """A response object: `read(n)` and nothing else useful""" + + class Response: + def __init__(self, data): + self._buf = io.BytesIO(data) + + def read(self, n): + return self._buf.read(n) + + assert len(list(open(Response(sample_fec_bytes)))) == 20 + + def test_read_is_chunked_not_whole(self, pac_fec_bytes_source): + """The 263 KB fixture takes several reads, each bounded by the chunk size""" + source, sizes = pac_fec_bytes_source + + assert sum(1 for _ in open(source)) == 1387 + assert len(sizes) > 1 + assert max(sizes) <= 64 * 1024 + + def test_id_from_file_object_name(self, sample_fec_file): + with builtins.open(sample_fec_file, "rb") as f: + assert open(f).id == "1921705" + + def test_id_none_without_a_name(self, sample_fec_bytes): + assert open(io.BytesIO(sample_fec_bytes)).id is None + + def test_source_length_from_fileno(self, sample_fec_file): + with builtins.open(sample_fec_file, "rb") as f: + assert open(f).source_length == sample_fec_file.stat().st_size + + def test_source_length_unknown_without_fileno(self, sample_fec_bytes): + """`BytesIO.fileno()` raises; an unknown length is 0, not an error""" + assert open(io.BytesIO(sample_fec_bytes)).source_length == 0 + + +class TestSourceErrors: + """Text-mode files, and exceptions raised by the source's own `read()`""" + + def test_text_mode_file_raises_typeerror(self, sample_fec_file): + with builtins.open(sample_fec_file) as f: + with pytest.raises(TypeError, match="'rb'"): + open(f) # type: ignore[arg-type] # text mode, on purpose + + def test_stringio_raises_typeerror(self, sample_fec_content): + with pytest.raises(TypeError, match="'rb'"): + open(io.StringIO(sample_fec_content)) # type: ignore[arg-type] # on purpose + + def test_read_returning_str_raises_typeerror(self, sample_fec_content): + """Not an `io.TextIOBase`, but still hands back `str`: same message""" + + class TextLike: + def read(self, n): + return "HDR\x1cFEC\x1c8.5" + + with pytest.raises(TypeError, match="'rb'"): + open(TextLike()) # type: ignore[arg-type] # on purpose + + def test_read_returning_junk_raises_typeerror(self): + class Junk: + def read(self, n): + return [1, 2, 3] + + with pytest.raises(TypeError, match="must return bytes"): + open(Junk()) # type: ignore[arg-type] # on purpose + + def test_read_error_propagates(self): + """The source's exception comes back as itself, not as FecParseError""" + + class Boom: + def read(self, n): + raise ZeroDivisionError("boom") + + with pytest.raises(ZeroDivisionError, match="boom"): + open(Boom()) # type: ignore[arg-type] # on purpose + + def test_read_error_mid_iteration_propagates(self, sample_fec_bytes): + """Same when the source fails after the header and cover are already parsed""" + + # The header and cover records end at byte 599 of the fixture, so 1 KB is + # enough for `open()` to succeed and not enough to finish iterating. + class BoomLater: + def __init__(self, data): + self._buf = io.BytesIO(data) + + def read(self, n): + if self._buf.tell() >= 1024: + raise ZeroDivisionError("late boom") + return self._buf.read(min(n, 512)) + + reader = open(BoomLater(sample_fec_bytes)) # type: ignore[arg-type] # on purpose + + with pytest.raises(ZeroDivisionError, match="late boom"): + list(reader) + + def test_read_returning_too_much_raises(self, sample_fec_bytes): + """A source that ignores its size argument is a bug, not a buffer overrun""" + + class TooMuch: + def read(self, n): + return sample_fec_bytes * 100 + + with pytest.raises(ValueError, match="more than asked for"): + open(TooMuch()) # type: ignore[arg-type] # on purpose + + +class TestSharedReaderThreading: + """One reader, many threads — the GIL and the reader's mutexes must not deadlock + + A binary file object is the interesting case: pulling a row runs Python code + (`read()`) from inside the reader's own locks, so a thread holding the GIL + that blocks on one of those locks closes a cycle. + """ + + def test_one_file_object_reader_shared_by_threads(self, pac_fec_file): + class SlowFile: + """A real file whose `read` yields the GIL, to force interleaving.""" + + def __init__(self, path): + self._f = builtins.open(path, "rb") + + def read(self, n): + data = self._f.read(n) + time.sleep(0.0005) + return data + + def close(self): + self._f.close() + + source = SlowFile(pac_fec_file) + reader = open(source) # type: ignore[arg-type] # a binary file object + lines: list[int] = [] + guard = threading.Lock() + failures: list[BaseException] = [] + + def drain() -> None: + try: + while True: + try: + row = next(reader) + except StopIteration: + return + with guard: + lines.append(row.line) + except BaseException as e: # noqa: BLE001 - reported, not swallowed + with guard: + failures.append(e) + + threads = [threading.Thread(target=drain, daemon=True) for _ in range(4)] + for t in threads: + t.start() + # Daemon threads plus an explicit timeout: a deadlock must fail the test, + # not hang the suite. + stuck = [] + for t in threads: + t.join(timeout=30) + if t.is_alive(): + stuck.append(t.name) + source.close() + + assert not stuck, f"threads still alive after 30s (deadlock): {stuck}" + assert not failures, f"worker raised: {failures!r}" + assert len(lines) == 1387 + assert len(set(lines)) == 1387, "a row was delivered to more than one thread" + + def test_close_from_another_thread_while_reading(self, pac_fec_file): + """`close()` holds the GIL and wants `inner`, which a pull may be holding""" + + class SlowFile: + def __init__(self, path): + self._f = builtins.open(path, "rb") + + def read(self, n): + time.sleep(0.001) + return self._f.read(n) + + reader = open(SlowFile(pac_fec_file)) # type: ignore[arg-type] # file object + started = threading.Event() + + def drain() -> None: + try: + for _ in reader: + started.set() + except ValueError: + pass # the expected "closed filing" once close() lands + + thread = threading.Thread(target=drain, daemon=True) + thread.start() + started.wait(timeout=30) + reader.close() # must not deadlock against the in-flight pull + thread.join(timeout=30) + + assert not thread.is_alive(), "close() deadlocked against an in-flight pull" + + +def test_shared_reader_across_threads(pac_fec_file): + """Four threads pulling from ONE reader: each row is delivered exactly once. + + Unlike ``TestSharedReaderThreading`` above, the source is a path — no + Python code runs from inside the reader's locks — so on the GIL build + this is a plain concurrency test. On a free-threaded build there is no + GIL to serialize `__next__`, so this is the one that actually exercises + concurrent Rust-side access to `pending`/`prefixes`. + """ + reader = open(pac_fec_file) + seen: list[Row] = [] + lock = threading.Lock() + + def pull() -> None: + for row in reader: + with lock: + seen.append(row) + + threads = [threading.Thread(target=pull, daemon=True) for _ in range(4)] + for t in threads: + t.start() + stuck = [] + for t in threads: + t.join(timeout=30) + if t.is_alive(): + stuck.append(t.name) + + assert not stuck, f"threads still alive after 30s (deadlock): {stuck}" + assert len(seen) == 1387 + assert len({r.line for r in seen}) == 1387, "a row was delivered to more than one thread" + + +# A source whose `read()` calls back into the reader that is reading it. `inner` +# is not a reentrant lock, so this has to be refused, not waited on. Run in a +# subprocess: if the guard ever regresses this must time out, not hang the suite. +_REENTRANT_SCRIPT = """ +import sys +import libfec_parser + + +class Reentrant: + def __init__(self, path, how): + self._f = open(path, "rb") + self._how = how + self.reader = None + + def read(self, n): + if self.reader is not None: + try: + next(self.reader) if self._how == "next" else self.reader.close() + except RuntimeError as e: + print("REFUSED", type(e).__name__, e) + except Exception as e: # noqa: BLE001 + print("WRONG", type(e).__name__, e) + return self._f.read(n) + + +source = Reentrant(sys.argv[1], sys.argv[2]) +source.reader = libfec_parser.open(source) +print("ROWS", sum(1 for _ in source.reader)) +""" + + +class TestReentrantSource: + @pytest.mark.parametrize("how", ["next", "close"]) + def test_source_reentering_its_own_reader_is_refused(self, sample_fec_file, how): + try: + result = subprocess.run( + [sys.executable, "-c", _REENTRANT_SCRIPT, str(sample_fec_file), how], + capture_output=True, + text=True, + timeout=60, + ) + except subprocess.TimeoutExpired: + pytest.fail(f"re-entrant {how}() deadlocked instead of raising") + + assert result.returncode == 0, result.stderr + assert "REFUSED RuntimeError" in result.stdout, result.stdout + assert "WRONG" not in result.stdout, result.stdout + # The reader survives the refusal and still delivers the filing. + assert "ROWS 20" in result.stdout, result.stdout + + def test_closed_is_answerable_during_a_pull(self, sample_fec_file): + """`closed` must not block on `inner` when this thread is the puller""" + + class Peeking: + def __init__(self, path): + self._f = builtins.open(path, "rb") + self.reader = None + self.seen = [] + + def read(self, n): + if self.reader is not None: + self.seen.append(self.reader.closed) + return self._f.read(n) + + source = Peeking(sample_fec_file) + source.reader = open(source) # type: ignore[arg-type] # a binary file object + + assert sum(1 for _ in source.reader) == 20 + assert source.seen and not any(source.seen) + + +@pytest.fixture +def pac_fec_bytes_source(pac_fec_file): + """A file object over the 263 KB fixture that records every read size.""" + sizes: list[int] = [] + buf = io.BytesIO(pac_fec_file.read_bytes()) + + class Recording: + def read(self, n): + sizes.append(n) + return buf.read(n) + + return Recording(), sizes +